diff --git a/README.md b/README.md index 31e1786..282366e 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ - `Dumas H3 Long Videos` - Inputs/outputs: the current upstream `MiniMax-H3-Longvideos` sampler surface, exposed under the existing `DumasH3LongVideos` key for saved Dumas workflows. - - The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine directly so it can track the source project again. + - The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine/runtime/audio/conditioning/shot-plan modules directly so it can track the source project again. - Upstream compatibility keys `H3LongVideos`, `H3LongVideosFL2VA`, `H3LongVideosV1`, and `H3LongVideosREF2VA` are also registered to the same class. - The old Dumas browser widget grouping script is disabled for this node because it targeted controls that no longer exist on the upstream sampler. - `handoff_frames` extends the upstream last-frame handoff: `1` keeps the current single keyframe behavior; higher values keep that final-frame keyframe and add earlier tail frames from the previous shot as claimed reference context for the next beat. @@ -44,7 +44,7 @@ - `Dumas H3 Latent Upscale Params` - Inputs: `mode`, `model_name`, `method`, `width`, `height`, `device`, `precision`, `sampler_name`, `scheduler`, `steps`, `denoise`, `megapixels`, `tile_width`, `tile_height`, `overlap`, `fade_width`, `fade_height`, `overlap_mode`, `overlap_blend`, `tile_size_mode`, `grid_rows`, `grid_cols`, `spatial_w_overlap`, `spatial_h_overlap`, `min_tile_size`, `masked_area_noise`, `brightness_match`, `dynamic_fade`, `dynamic_fade_min`, `chunk_length`, `temporal_overlap`, `resize_conditioning`, `anchor_strength` - Output: `latent_upscale_param` - - Bundles the optional latent-space upscaler settings used by `Dumas H3 Long Videos` before decode, so the main node can rebuild conditioning at the target size and run a short refinement pass with your chosen sampler, scheduler, step count, denoise, and the full upstream spatial split controls. + - Legacy helper from the abandoned Dumas Long Videos fork. The current upstream-backed `Dumas H3 Long Videos` node does not consume this socket; it uses the upstream latent-upscale controls on the Long Videos node itself. - `Dumas H3 Beat Prompt` - Inputs: authored through the custom front-end beat editor diff --git a/dumas_h3_longvideos_audio.py b/dumas_h3_longvideos_audio.py new file mode 100644 index 0000000..66a18c3 --- /dev/null +++ b/dumas_h3_longvideos_audio.py @@ -0,0 +1,319 @@ +# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos +# Copyright (c) 2026 Smite79. All rights reserved. +# Redistribution, in whole or in part, requires written permission. +# This notice may not be removed or altered. See LICENSE. +"""Audio policy shared by conditioning and soundtrack assembly.""" + +from dataclasses import dataclass + +import torch +import comfy.nested_tensor +from h3_runtime import temporal_shape + + +@dataclass(frozen=True) +class ShotAudio: + speech: bool + sounded: bool + voiced_only: bool + silence_enabled: bool + lead_seconds: float + latent_fps: int + # The tail. Everything after the line's expected end is pinned the way the lead + # pins everything before its start. All three default off, so a ShotAudio built + # the old way -- six positional arguments -- behaves exactly the old way. + line_seconds: float = 0.0 # planner's estimate of the spoken line + tail_seconds: float = 0.0 # free audio kept after that estimate; 0 = no tail pin + frame_count: int = 0 # the shot in pixel frames; the audio T comes from it + + @property + def pinned(self): + return self.silence_enabled and not self.speech and not self.sounded + + @property + def lead_frames(self): + if not self.speech or self.lead_seconds <= 0: + return 0 + return round(self.lead_seconds * self.latent_fps) + + @property + def tail_frames(self): + """Audio latent frames pinned at the END of a dialogue shot. + + The lead pins the opening so the line cannot start early; nothing pinned the + close, and a 2s line in a 9s shot left 7s of open branch in a shot the model + knows has a voice in it -- which is where speech carries on past the line, or + doubles it. The free span is lead + the line's estimate + tail_seconds; the + rest is held at encoded silence. The model chooses WHEN to speak, so the + margin is the author's dial: a clipped word costs more than a second of babble. + Off unless the shot speaks, the margin is set, and at least half a second would + be pinned -- a sliver is not worth the risk of clipping.""" + if (not self.speech or self.tail_seconds <= 0 or self.line_seconds <= 0 + or self.frame_count <= 0): + return 0 + total = temporal_shape(self.frame_count)[2] + free = self.lead_frames + round((self.line_seconds + self.tail_seconds) * self.latent_fps) + tail = total - free + return tail if tail >= round(0.5 * self.latent_fps) else 0 + + +_SILENT_UNIT = {"lat": None, "key": None} + +# --------------------------------------------------------------------------- +# NOTHING HERE IS SYNTHESISED ANY MORE. Removed on the report: "Just get rid of +# the ambient sounds all together. They sound horrid. Go back to the model's +# natural audio." +# +# What was here built the soundtrack's non-vocal half out of shaped noise: a room +# tone from the scene's own wording (synth_ambient, over a table of recipes, with +# plain_bed under it as a floor) and 21 foley recipes laid into the shots whose +# audio branch is pinned to silence (foley_for, over _hits/_band/_room and later +# _contact/_flow/_creak, timed off the picture's own movement for footsteps). +# +# It went in because a shot pinned to silence cannot get audio from the model at +# all -- prompt text never opens a branch -- so auto_sound was writing sounds into +# prompts that could not make them. That reasoning was sound and the thing it built +# still did not pass: reported first as footsteps sounding like heartbeats and a +# bathroom that tapped, and then, once both of those measured clean, as horrid +# anyway. Synthesis that measures right and sounds wrong is the end of that road. +# +# So the audio is the model's, whole. H3 is a joint model and the audio branch is +# where its sound comes from; the prompt still describes what a shot sounds like, +# which is the half that was always doing the real work. +# +# The consequence, which is real and is reported in info rather than left to be +# discovered: a shot with no line and no sound you wrote is pinned to silence and +# is now SILENT. The pin is not a bug and is deliberately untouched -- it is what +# stops a free branch filling itself with babble and a face lip-syncing to it. +# Write the sound into the beat to open the branch on purpose, or wire a recording +# to ambient_audio, which is played under the finished track and conditions +# nothing. mix_ambient below is that path, and it is all that is left here. +# --------------------------------------------------------------------------- + + +def _seamless_loop(x, n, sr): + """[C, M] -> [C, n], looped with a crossfade so the join does not click. + + Plain tiling puts a discontinuity at every repeat, once per loop length. In a + bed that is meant to sit under everything unnoticed, a regular click is the one + thing that gets noticed -- the same objection that made the silence latent + ping-pong its interior rather than tile it. Here the material is real audio + being PLAYED rather than a latent being conditioned on, so it cannot be + reversed: a room tone read backwards is fine, but footsteps are not. Crossfade + instead, which works on both.""" + m = int(x.shape[-1]) + if m <= 0: + return None + if m >= n: + return x[..., :n] + fade = min(int(0.25 * sr), m // 4) + if fade < 1: + reps = -(-n // m) + return x.repeat(1, reps)[..., :n] + # OVERLAP-ADD the tail onto the head, and shorten the unit by the overlap. The + # unit then runs x[m-fade] .. x[m-fade-1], so tiling it steps between samples + # that were adjacent in the source and there is no discontinuity anywhere. + # + # Measured, because the obvious construction is wrong: appending the crossfade + # to the END of a full-length unit leaves it finishing on x[fade-1] while the + # next repeat starts on x[0], which are not adjacent -- a 2s tone that does not + # divide evenly gave a 64x jump at the join, worse than plain tiling's 41x. + t = torch.linspace(0.0, 1.0, fade, dtype=x.dtype, device=x.device) + head = x[..., :fade] * t + x[..., m - fade:] * (1.0 - t) + unit = torch.cat([head, x[..., fade:m - fade]], dim=-1) + if int(unit.shape[-1]) < 1: + reps = -(-n // m) + return x.repeat(1, reps)[..., :n] + reps = -(-n // int(unit.shape[-1])) + return unit.repeat(1, reps)[..., :n] + + +def mix_ambient(audio, sr, bed, level): + """Lay an ambient bed UNDER a finished soundtrack. -> (waveform, note). + + The bed is PLAYED, not conditioned on: it is the file, at the level asked for, + under whatever the model generated. That is the whole reason to do it here + rather than in the sampler -- ambience needs no cooperation from a joint model, + has nothing to lip-sync to, and so cannot put a voice in a wordless shot. The + conditioning path can only steer the branch toward something bed-LIKE, and on a + shot with a line it competes with the line. + + Defensive throughout, like the silence latent: any failure returns the audio + untouched with a note saying so, because a bed is a nicety and a render is not. + """ + try: + if audio is None or bed is None or float(level or 0.0) <= 0.0: + return audio, "" + w = bed.get("waveform") if isinstance(bed, dict) else None + if w is None or not int(getattr(w, "ndim", 0)): + return audio, ("ambient_audio is wired but carries no waveform, so nothing " + "was laid under the soundtrack") + w = w[0] if w.dim() == 3 else w # [B, C, M] -> [C, M] + if w.dim() != 2 or w.shape[-1] < 2: + return audio, ("ambient_audio is too short to loop, so nothing was laid " + "under the soundtrack") + w = w.detach().to(dtype=audio.dtype, device=audio.device) + b_sr = int((bed.get("sample_rate") if isinstance(bed, dict) else 0) or 0) + # RESAMPLE, or the bed plays at the wrong speed and pitch. Linear is coarse + # for music and inaudible on a room tone, which is what this input is for. + resampled = "" + if b_sr > 0 and b_sr != int(sr): + want = max(2, int(round(w.shape[-1] * float(sr) / float(b_sr)))) + w = torch.nn.functional.interpolate( + w.unsqueeze(0), size=want, mode="linear", align_corners=False)[0] + resampled = f", resampled from {b_sr} Hz" + ch = int(audio.shape[1]) + if int(w.shape[0]) != ch: + w = (w.mean(dim=0, keepdim=True).repeat(ch, 1) if int(w.shape[0]) > ch + else w[:1].repeat(ch, 1)) + n = int(audio.shape[-1]) + loop = _seamless_loop(w, n, int(sr)) + if loop is None: + return audio, "" + out = audio + loop.unsqueeze(0) * float(level) + # NORMALISE rather than clip. Clipping a bed that pushed a loud line over + # the top distorts the LINE, which is the thing worth keeping. + peak = float(out.abs().max()) + gain = "" + if peak > 1.0: + out = out / peak + gain = f", and the mix was scaled by {1.0 / peak:.2f} to stop it clipping" + secs = w.shape[-1] / float(sr) + return out, (f"an ambient bed was laid under the whole soundtrack at level " + f"{float(level):.2f} -- {secs:.1f}s of audio{resampled}, looped " + f"with a crossfade so the join does not click{gain}. It is your " + f"file, played under what the model generated: it conditions " + f"nothing, so it cannot put a voice in a wordless shot the way " + f"an inferred bed did. Shots pinned to silence keep their silent " + f"conditioning and get the bed on top, which is what makes a " + f"wordless shot sound like a room instead of a mute") + except Exception as exc: + return audio, (f"the ambient bed could not be mixed ({type(exc).__name__}), so " + f"the soundtrack is unchanged") + + +_SILENCE_STATUS = {"asked": 0, "applied": 0, "why": ""} + + +_SILENT_SECONDS = 2 + + +_SILENT_EDGE = 4 + + +def _silent_audio_latent(audio_vae, frame_count, fps): + """A keyframe audio latent of actual SILENCE, or None if it cannot be made. + + H3 is a JOINT model: the mouth follows the audio branch. On a shot with no + scripted line the branch is otherwise unconditioned, and an unconditioned audio + branch invents a voice -- which the picture then lip-syncs to. The lips-closed + sentence is arguing with a stream that has already decided someone is talking. + + REBUILT 2026-09-05, from measurements against the real VAE rather than from + reasoning. The previous version encoded one second, kept a SINGLE interior + frame and repeated it, on the argument that silence is homogeneous. It is not, + in latent space: encoded silence has genuine frame-to-frame variation (delta + mean 0.002-0.004, max 0.021), and a repeated frame has a delta of exactly + 0.000000. That is a flat signal no encoder produces, and a model handed + conditioning outside its own distribution has every reason to disregard it -- + which is an audio branch back to inventing a voice, with the report saying + silence went on. + + The fix that version was avoiding is real too: tiling the whole encoded second + end to end leaves a 25x spike at each join (0.554 against 0.022), once per + second, which is a metronome in the conditioning of a joint model. + + So: encode two seconds, drop the padded ends, and PING-PONG the interior -- + forward, reversed, forward. Every join repeats a frame, so there is no seam, + and the interior statistics are the encoder's own. Measured over a 9s shot: + + one frame repeated peak 0.000686 delta mean 0.000000 max 0.000000 + whole 2s tiled peak 0.000314 delta mean 0.017451 max 0.554715 + interior ping-pong peak 0.000566 delta mean 0.002039 max 0.021159 + + where the encoder's own interior is mean 0.0021, max 0.0212. Decoded peak + 0.000566 on a +/-1.0 scale is about -65 dBFS: silence. + + Everything here stays defensive. Shapes are CHECKED against what the layout + expects rather than assumed, and any failure returns None so the shot falls + back to an unconditioned branch instead of breaking the render -- the caller + reports when that happens, so it is no longer a silent failure. + """ + try: + sr = int(getattr(audio_vae, "audio_sample_rate", 0) or 0) + if sr <= 0: + return None + _, _, want_t = temporal_shape(frame_count, fps) + if want_t <= 0: + return None + key = (id(audio_vae), sr) + block = _SILENT_UNIT.get("lat") if _SILENT_UNIT.get("key") == key else None + if block is None: + # CHANNELS LAST. comfy.sd.VAE.encode() does `pixel_samples.movedim(-1, 1)` + # before handing off, so the audio VAE -- which wants [B, 2, L] -- must be + # given [B, L, 2]. Passing [B, 2, L] raises inside the encoder, and an + # early version did exactly that: swallowed by the guard below, so the + # whole layer silently did nothing. + # + # Two seconds, encoded ONCE and cached. Encoding a full 15s shot instead + # cost a VAE pass big enough to OOM mid-render on a 16GB card, where the + # failure again degraded silently to no conditioning at all. + enc = audio_vae.encode(torch.zeros((1, sr * _SILENT_SECONDS, 2))) + if enc is None or enc.dim() != 4 or enc.shape[1] != 32: + return None + if enc.shape[-1] <= 2 * _SILENT_EDGE + 1: + return None + block = enc[..., _SILENT_EDGE:-_SILENT_EDGE].detach().to("cpu").clone() + _SILENT_UNIT["lat"] = block + _SILENT_UNIT["key"] = key + n = block.shape[-1] + if n < 1: + return None + # Forward, reversed, forward... Each join repeats a frame, so the seam that + # plain tiling leaves is gone while the interior variation is the encoder's. + pieces, have, i = [], 0, 0 + while have < want_t: + piece = block if i % 2 == 0 else torch.flip(block, dims=[-1]) + pieces.append(piece) + have += n + i += 1 + out = torch.cat(pieces, dim=-1)[..., :want_t].clone() + if out.shape[-1] != want_t: + return None + return out + except Exception: + return None # never fail a render for a nicety + + +def _pin_audio_silence(latent, silence, lead_frames=None, tail_frames=0): + """Start target audio at encoded silence and preserve the requested span(s). + + lead_frames None pins the whole shot. Otherwise the first lead_frames and the + last tail_frames are held at silence and the span between is left to the model + -- that is where the line goes. The tail is clipped to what the lead leaves, so + the two can never overlap. Nothing pinned at all is a no-op, reported as False + so the caller does not count it as applied.""" + try: + video, audio = latent["samples"].unbind() + silence = silence.to(device=audio.device, dtype=audio.dtype) + if silence.shape != audio.shape: + return False + audio_mask = torch.ones_like(audio[:, :1]) + if lead_frames is None: + audio_mask.zero_() + else: + t = audio.shape[-1] + n = min(t, max(0, int(lead_frames))) + m = min(t - n, max(0, int(tail_frames or 0))) + if n <= 0 and m <= 0: + return False + if n > 0: + audio_mask[..., :n] = 0 + if m > 0: + audio_mask[..., t - m:] = 0 + latent["samples"] = comfy.nested_tensor.NestedTensor((video, silence)) + latent["noise_mask"] = comfy.nested_tensor.NestedTensor( + (torch.ones_like(video[:, :1]), audio_mask)) + return True + except Exception: + return False diff --git a/dumas_h3_longvideos_conditioning.py b/dumas_h3_longvideos_conditioning.py new file mode 100644 index 0000000..af477f9 --- /dev/null +++ b/dumas_h3_longvideos_conditioning.py @@ -0,0 +1,310 @@ +# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos +# Copyright (c) 2026 Smite79. All rights reserved. +# Redistribution, in whole or in part, requires written permission. +# This notice may not be removed or altered. See LICENSE. +"""Decisions about which pictures may condition a shot.""" + +import torch +import node_helpers +from h3_runtime import (H3_FPS, AUDIO_LATENT_FPS, _empty_av_latent, _resize, ref_image_canvas, + frame_levels) +from h3_audio import _SILENCE_STATUS, _silent_audio_latent, _pin_audio_silence + + +def may_carry_room(previous_cast, current_cast, tagged_names): + """A previous frame is safe as a reference only when it adds no subject.""" + previous = [name for name in (previous_cast or ()) if name] + current = set(current_cast or ()) + tagged = set(tagged_names or ()) + return bool(previous) and all(name in current for name in previous) \ + and not any(name in tagged for name in previous) + + +def may_carry_frame(previous_cast, current_cast, tagged_names): + """A previous frame of the SAME room is safe as a reference claimed with everyone in it. + + Unlike may_carry_room, somebody this shot does not describe may be in it: the claim + names them, and they are still in that room. Refused only for an empty frame, or + one holding somebody whose own portrait also rides this shot -- two pictures of one + person is how a second one gets drawn.""" + previous = [name for name in (previous_cast or ()) if name] + current = set(current_cast or ()) + tagged = set(tagged_names or ()) + return bool(previous) and not any(name in tagged and name in current + for name in previous) + + +def recoverable_subject(cast, tagged_names, returning_names, captured): + """Return the sole safe recovered subject, or an empty string.""" + people = [name for name in (cast or ()) if name] + if len(people) != 1: + return "" + name = people[0] + return name if (name not in set(tagged_names or ()) + and name in set(returning_names or ()) + and captured.get(name) is not None) else "" + + +KEYFRAME_SAFE_AUG = 0.99 # below this, a ref aug would soften the keyframe too + +# What ONE boundary is allowed to claim it measured. Wider than any real per-pass drift, +# narrow enough that a bad frame -- a flash, a cut to black, a frame the model lost -- +# cannot swing the estimate. The median across boundaries does the real rejecting. +LEVEL_GAIN_CAP = 0.12 # in log-gain, so +-12.7% of contrast +LEVEL_OFFSET_CAP = 0.05 +# The within-shot term is believed only when boundaries AGREE on its sign, and even then +# only this far: within-shot change is often the author's (a light switched off), so it is +# the half of the signal that cannot be trusted on its own. +LEVEL_SHOT_GAIN_CAP = 0.015 +LEVEL_SHOT_OFFSET_CAP = 0.010 +LEVEL_AGREE = 2.0 / 3.0 +LEVEL_MIN_OBS = 3 +# What the correction may do to one handoff, whatever it measured. A cut should not carry +# a visible grade step: shot N's last frame reaches the video uncorrected while N+1 is +# sampled from a corrected keyframe, so an uncapped correction trades burn-in for a pop at +# every join -- the same class of complaint, differently shaped. +LEVEL_GAIN_LO, LEVEL_GAIN_HI = 0.80, 1.25 +LEVEL_OFFSET_BOUND = 0.02 +# Below this a frame is too flat for a contrast RATIO to mean anything. +LEVEL_MIN_SIGMA = 0.01 + + +class HandoffLevels: + """Takes the grade the chain adds to itself back out of the handoff. + + THE MEASUREMENT, which is the whole reason this needs no scene list. At every + boundary the render holds two pictures that are SUPPOSED to be the same frame: K, + the handoff it gave the shot, and R, frame one of what came back -- the model's own + reproduction of K, from a keyframe labelled sigma 0.001. Nothing was asked to change + between them, so everything separating them is the chain's own doing and none of it + is the author's intent. That is the one difference in the loop that can be corrected + without guessing at anybody's lighting, and R costs nothing to look at: it is the + frame trim_seam throws away. + + A beat that walks into a darker room moves K, and R follows it there. So the level is + never anchored, never compared to shot 1, and never compared to a target -- only K + against its own reproduction, boundary by boundary. + + WHAT IT WILL NOT FIX. Clipping already baked into earlier shots, because the VAE + clamps every decode and headroom spent is gone. Softening, which is a different + measurement and a different cause. Anything spatial -- ghosting, local burn, identity + drift. A tone curve with a knee in it, since this is affine per channel; the residual + in the report is how that would show itself. The first boundary, which has nothing to + measure yet. And a deliberate monotone move -- a film that dims every single beat -- + loses a bounded, reported fraction of itself.""" + + def __init__(self): + self._bg, self._bo = [], [] # per boundary: K -> R, the chain's own drift + self._sg, self._so = [], [] # per shot: R -> last frame, believed only on agreement + self.applied = [] # (gain, offset) actually used, for the report + + def observe(self, given, repro, last=None, pre_up_last=None): + """Record one boundary. given is the keyframe this shot got, repro is frame one + of what it produced, last is its final frame, pre_up_last the handoff it hands on. + + last/pre_up_last are how the pre-upscale handoff and the post-upscale output are + put in the same frame of reference: their difference IS the pipeline's own offset, + measured on one frame that went through both, so it can be subtracted from the + K->R reading instead of being mistaken for drift. With latent_upscale off they are + the same frame and the term is zero.""" + gm, gs = frame_levels(given) + rm, rs = frame_levels(repro) + if gm is None or rm is None: + return False + if float(gs.min()) < LEVEL_MIN_SIGMA or float(rs.min()) < LEVEL_MIN_SIGMA: + return False + ug = torch.zeros(3) + uo = torch.zeros(3) + lm, ls = frame_levels(last) if last is not None else (None, None) + if pre_up_last is not None and lm is not None: + pm, ps = frame_levels(pre_up_last) + if pm is not None and float(ps.min()) >= LEVEL_MIN_SIGMA: + ug = torch.log(ls / ps) + uo = lm - pm + self._bg.append((torch.log(rs / gs) - ug).clamp(-LEVEL_GAIN_CAP, LEVEL_GAIN_CAP)) + self._bo.append((rm - gm - uo).clamp(-LEVEL_OFFSET_CAP, LEVEL_OFFSET_CAP)) + if lm is not None and float(ls.min()) >= LEVEL_MIN_SIGMA: + self._sg.append(torch.log(ls / rs)) + self._so.append(lm - rm) + return True + + def _agreed(self, rows, cap): + """The median of rows, but only per channel where at least LEVEL_AGREE of them + share its sign. A within-shot change the boundaries disagree about is content, not + drift, and content must not be corrected.""" + out = torch.zeros(3) + if len(rows) < LEVEL_MIN_OBS: + return out + st = torch.stack(rows) + med = st.median(dim=0).values + agree = ((st * med.sign().unsqueeze(0)) > 0).float().mean(dim=0) + keep = agree >= LEVEL_AGREE + return torch.where(keep, med.clamp(-cap, cap), out) + + def estimate(self): + """(gain_log, offset) the chain is drifting by per boundary, per channel.""" + if not self._bg: + return None, None + g = torch.stack(self._bg).median(dim=0).values + self._agreed(self._sg, LEVEL_SHOT_GAIN_CAP) + o = torch.stack(self._bo).median(dim=0).values + self._agreed(self._so, LEVEL_SHOT_OFFSET_CAP) + return g, o + + def gains(self, strength): + """(gain, offset) as 3-vectors, or (None, None) when there is nothing worth doing. + + Separate from note() because more than one frame leaves a shot -- the handoff, + and any face captured for a return several shots later -- and they have to carry + the SAME grade. A recovered face arriving at a different exposure from the shot + around it would be a new bug of exactly the kind this is fixing.""" + g, o = self.estimate() + if g is None or strength <= 0: + return None, None + gain = torch.exp(-float(strength) * g).clamp(LEVEL_GAIN_LO, LEVEL_GAIN_HI) + off = (-float(strength) * o).clamp(-LEVEL_OFFSET_BOUND, LEVEL_OFFSET_BOUND) + # The next thing this frame meets is an 8-bit quantisation, so a correction under + # 1/255 would be erased on the way there. Claiming it would be worse than silence. + if float((gain - 1.0).abs().max()) < 1e-3 and float(off.abs().max()) < 1.0 / 255.0: + return None, None + return gain, off + + def note(self, gain, off): + """Record what was applied, and say it in one clause.""" + self.applied.append((gain.clone(), off.clone())) + return (f"gain {'/'.join(f'{float(v):.3f}' for v in gain)} " + f"level {'/'.join(f'{float(v):+.4f}' for v in off)}") + +def _keyframe_latent(vae, hand_img): + """The keyframe latent for this shot: an ENCODE of the previous shot's last frame. + + This was briefly an optimisation -- pass the previous shot's own latent straight + through and skip a VAE round trip per boundary. It was wrong, and it degraded + every shot after the first. + + A keyframe is ONE pixel frame, and H3's grid puts that at 5f -> TWO latent + frames. Slicing [:, :, -1:] off a finished shot hands over one. Worse, the video + VAE is causal: the last latent of a 72-frame sequence encodes its temporal + context, not a standalone opening frame, so even at the right count it does not + mean what a keyframe means. The spatial-size guard could not see either problem. + + The round trip is real but it is one lossy step on a correctly formed anchor, + which beats a cheap malformed one.""" + return vae.encode(hand_img) + + +def _build_ref_images(vae, images, gen_w, gen_h, mode="match"): + """(tokenizer items, DiT blocks) for a list of reference IMAGE tensors. + + The tokenizer labels each one `:` itself, in the order given here -- + so the roster the prompt refers to is decided by input order, not by anything + written in the prompt.""" + items, blocks = [], [] + for img in images: + if img is None: + continue + h, w = int(img.shape[1]), int(img.shape[2]) + tw, th = ref_image_canvas(w, h, gen_w, gen_h, mode) + resized = _resize(img[:1], tw, th, "disabled") + items.append({"type": "image", "data": resized}) + blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, + "latent": vae.encode(resized)}) + return items, blocks + + +def build_conditioning(clip, vae, audio_vae, prompt, width, height, length, + handoff=None, refs=None, + ref_noise_aug=0.999, silent=False, ref_image_size="match", + handoff_as_ref=False, speech_lead_seconds=0.0, speech_tail_frames=0): + """Encode prompt, identity references, keyframe, and audio constraints for a shot.""" + latent, fc = _empty_av_latent(width, height, length, H3_FPS) + refs = [r for r in (refs or []) if r is not None] + + hand_img = None + if handoff is not None: + hand_img = _resize(handoff[:1], width, height, "disabled") + + # REFERENCES AND THE KEYFRAME RIDE TOGETHER. This is the arrangement the node + # had before I broke it, and the reason is in ComfyUI's own layout: + # + # model_base.py:2183-2191 cond_video_latents = keyframe latents THEN ref latents + # model.py PackedLayout emits keyframe "cond" segments THEN ref "ref_img" ones + # + # The two orders agree, so both channels coexist. A shot takes its references AND + # a real keyframe: the keyframe ANCHORS the first frame, which is what continuity + # needs, while a reference only supplies identity. They are not alternatives. + # + # I had read "" as MEANING the first frame on fl2va, and rearranged the + # roster around that. It does not. Which image is the first frame is decided by + # resolved_frame_index in minimax_keyframes, not by a label's number -- the labels + # are only how the images are shown to the VLM, and what they have to line up with + # is the tags in the prompt. + # + # So references come FIRST and keep slots 1..N, which is what a sheet line's + # `Name: , ...` points at, and the handoff is appended AFTER them where + # it disturbs no numbering. It has to be in the list at all because + # tokenize_with_weights is either/or: passing minimax_ref_items makes it ignore + # `images` outright, so leaving the handoff out means the VLM is never shown where + # the shot left off and re-imagines the scenery -- same place, new room. + keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG + # One aug covers every visual condition row, references AND the keyframe. Below + # KEYFRAME_SAFE_AUG the keyframe latent would be noised and labelled at the wrong + # timestep, so the handoff stops being an anchor and rides as an extra reference + # instead: weaker continuity, but nothing pretending to anchor while carrying noise. + # ...or because the caller asked for it. A shot that introduces somebody already + # in position wants the room this picture carries and NOT the first frame it + # would force, and that is a demotion the aug knows nothing about. + carry_as_ref = bool(hand_img is not None + and (handoff_as_ref or (refs and not keyframe_ok))) + + enc_refs = refs + ([hand_img] if carry_as_ref else []) + items, blocks = ([], []) + if enc_refs: + items, blocks = _build_ref_images(vae, enc_refs, width, height, ref_image_size) + if hand_img is not None and not carry_as_ref: + items = items + [{"type": "image", "data": hand_img}] + + if items: + tokens = clip.tokenize(prompt, minimax_ref_items=items) + else: + tokens = clip.tokenize(prompt) + cond = clip.encode_from_tokens_scheduled(tokens) + + vals = {} + if blocks: + vals["minimax_refs"] = blocks + # How CLEAN the references are shown. One aug covers every conditioning + # latent, keyframe included -- which is why softening references below + # KEYFRAME_SAFE_AUG would soften the anchor too. + if ref_noise_aug is not None: + vals["minimax_visual_cond_noise_aug"] = float(ref_noise_aug) + + kfs = [] + if hand_img is not None and not carry_as_ref: + kfs.append({"resolved_frame_index": 0, + "latent": _keyframe_latent(vae, hand_img)}) + # Audio keyframes are extra conditioning rows in H3's PackedLayout. Pin the + # generated target stream instead, so the joint model also sees a quiet mouth. + # A dialogue shot pins its opening (the lead) and, past the line's estimated end, + # its close (the tail); the span between is the model's. + if silent or float(speech_lead_seconds or 0.0) > 0.0 or int(speech_tail_frames or 0) > 0: + _SILENCE_STATUS["asked"] += 1 + if audio_vae is None: + _SILENCE_STATUS["why"] = "no audio VAE is wired to the node" + else: + sil = _silent_audio_latent(audio_vae, fc, H3_FPS) + if sil is None: + _SILENCE_STATUS["why"] = ("the audio VAE would not encode a silent " + "second -- the wrong VAE is on the " + "audio_vae input") + else: + lead = None if silent else round(float(speech_lead_seconds) * + AUDIO_LATENT_FPS) + tail = 0 if silent else int(speech_tail_frames or 0) + if _pin_audio_silence(latent, sil, lead, tail): + _SILENCE_STATUS["applied"] += 1 + else: + _SILENCE_STATUS["why"] = "the silent latent did not match the shot" + if kfs: + vals["minimax_keyframes"] = kfs + if vals: + cond = node_helpers.conditioning_set_values(cond, vals) + return cond, latent, fc, carry_as_ref diff --git a/dumas_h3_longvideos_engine.py b/dumas_h3_longvideos_engine.py index f00aa31..7e52c54 100644 --- a/dumas_h3_longvideos_engine.py +++ b/dumas_h3_longvideos_engine.py @@ -66,21 +66,8 @@ HARDWARE = ( (r"cuffs?|cuffed", "cuffs", "wrists"), (r"tape", "tape", "wrists"), ) -# Material and colour survive because they decide what the thing looks like: -# "steel collar" must not come back as "collar" two shots later. -# WHAT THE AUTHOR CALLED IT. This decides how much of the wording survives into -# the guard clauses, and the guard is what every shot after the first repeats -- -# so a word missing here is a word the model stops hearing. -# -# It was twenty-odd words, and "a mirrored steel collar" came back as "steel -# collar" while "a brushed nickel collar" came back as "collar". A bare "collar" -# repeated once a shot is a bare collar, and the prior for that is a black -# leather one -- which is exactly what was reported. -# -# Hyphenated compounds pass whole ("mirror-finish", "chrome-plated"), so an -# unusual finish survives without being listed. Bare participles are NOT -# accepted: "-ed" is a verb far more often than a modifier, and capturing one -# would put an action into the name of the thing. +# Preserve visual modifiers in continuity text. Hyphenated compounds pass whole; +# arbitrary participles do not, because they are more often verbs than modifiers. _ADJ = (r"(?:[A-Za-z]+-[A-Za-z]+|" # materials r"steel|stainless|iron|metal|metallic|nickel|chrome|chromed|brass|" @@ -131,8 +118,26 @@ PART_VARIES = frozenset({"chain", "rope", "straps", "tape"}) # outlives the beat that caused it, and the clause that says so has to be # writable from any later shot. REGION_OF = ( + # UNDERWEAR IS IN HERE TOO, on both halves of the body. The torso row has + # listed a bra since the day it was written -- that is the report it exists + # for, "a bra coming back on somebody topless" -- and the leg row never got + # its counterpart, so region_of("thong") answered "". A garment that cannot + # be placed latches no bare region, so underwear coming off said nothing + # about the hips in that shot or in any shot after it, and an unspecified + # region is filled by the model's own prior -- which for a hip is underwear. + # + # Worse, a beat saying somebody is NAKED looks each worn garment's region up + # to take it off the body, so the one garment it could not place stayed + # "worn" in the state while the text said she was nude. Reported as a thong + # restored a beat after she undressed to get in the shower. + # + # No hardware. A chastity belt is in the layering vocabulary, but it is a + # restraint: it is latched and held by its own mechanism, and a bare region + # read off it would argue with that. (r"shorts|trousers|jeans|slacks|chinos|skirt|kilt|leggings|joggers|tights|" - r"pantyhose|jeggings|culottes|tracksuit\s+bottoms", "legs", + r"pantyhose|jeggings|culottes|tracksuit\s+bottoms|" + r"panties|knickers|thong|g-?string|briefs|boxers|underwear|undies|" + r"jockstrap|loincloth", "legs", "The legs are bare from the hip down"), (r"socks|stockings|hold-?ups|boots|shoes|trainers|sneakers|sandals|heels", "feet", "The feet and ankles are bare"), @@ -214,16 +219,34 @@ RELEASE_VERB = ( # the engine knew a cell and a warehouse, the sampler did not, so a scene set in # either was a room to one reader and nowhere to the other. Same fault the # garment lists had, waiting to be reported. +# MULTI-WORD ROOMS COME BEFORE BARE "room". _MOD is non-greedy, so it tries no +# modifier first and the longest place wins -- but only if the long form is here to +# win with. Without "locker\s+room", "heads to the locker room" read its destination +# as "room": _MOD swallowed "locker" and the capture took what was left, so the film +# was reported as entering "room" and the shot was told to arrive in one. +# +# A gym, a locker room and a court were in no list at all, which is worse than vague: +# with the ORIGIN unknown, travel_anchor emitted nothing, so a beat walking out of a +# gym was never told to walk and the set simply changed under the characters. PLACES = (r"hallway|hall|corridor|passage|landing|stairwell|staircase|stairs|" r"steps|bedroom|bathroom|washroom|kitchen|living\s+room|lounge|" - r"dining\s+room|study|office|garage|basement|cellar|attic|loft|porch|" + r"dining\s+room|locker\s+rooms?|changing\s+rooms?|dressing\s+rooms?|" + r"waiting\s+rooms?|utility\s+rooms?|gymnasium|gym|classroom|library|" + r"cafeteria|canteen|reception|laundry|pantry|sauna|balcony|terrace|" + r"rooftop|elevator|court|pool|showers|shower|store|shop|studio|" + r"study|office|garage|basement|cellar|attic|loft|porch|" r"veranda|garden|yard|driveway|street|alley|car\s?park|lobby|foyer|" r"doorway|cell|warehouse|barn|shed|van|truck|room") # Place words that are also ordinary verbs or everyday nouns. A reader with a # preposition in front of it ("in the study") can tell which sense is meant; the # free-text one cannot, and "she steps out", "they study the map" and "he lands # badly" are all commoner than the rooms they collide with. -PLACE_ALSO_A_VERB = {"steps", "landing", "study", "lounge", "garage", "porch"} +# "bar" and "lift" are deliberately NOT places in this file at all: bars are +# restraint hardware here ("chained to the bars") and lifting is what happens to a +# garment or a body, so behind a preposition they would both read as journeys. +PLACE_ALSO_A_VERB = {"steps", "landing", "study", "lounge", "garage", "porch", + "court", "pool", "shower", "showers", "store", "shop", + "studio", "reception"} # A room is usually described, not just named -- "the tiled bathroom", "the long # hallway". Up to three adjectives, non-greedy so the NEAREST room still wins. _ROOM_MOD = (r"(?:(?!(?:of|the|an?|and|or|to|in|into|from|with|on|at|by|for|her|" @@ -272,15 +295,21 @@ _GAP = r"(?:\s+\S+){0,4}?\s+" TAKES_OFF = (r"(?:takes?|took|taking|pulls?|pulled|peels?|peeled|strips?|" r"stripped|shrugs?|slips?|slipped|steps?|gets?|got|kicks?|" r"kicked)" + _GAP + r"(?:off|out\s+of)\b" - r"|\b(?:removes?|removed|removing|discards?|discarded|" - r"undresses|undressed|unbuttons?|unzips?|unzipped)") + r"|\b(?:removes?|removed|removing|discards?|discarded|sheds?|shedding|" + r"undresses|undressed)") PUTS_ON = (r"(?:puts?|putting|pulls?|pulled|slips?|slipped|tugs?|tugged|" r"steps?|stepped|climbs?|climbed|gets?|got|wriggles?)" + _GAP + r"(?:on|into|back\s+on)\b" r"|\b(?:dresses?\s+in|dressed\s+in|buttons?|zips?\s+up|fastens?)") +# UNZIPPING A JACKET LEAVES IT ON. These were removals, so "Owen unzips his jacket" +# took the jacket out of every later shot and called his chest bare -- and "rolls up +# his sleeves" a beat later rolled the sleeves of a shirt that had gone with it. They +# open a garment; a beat that also takes it off says so ("and takes it off"). DISPLACES = (r"(?:pulls?|pulled|pushes?|pushed|tugs?|tugged|hikes?|hiked|" r"rolls?|rolled|lifts?|lifted|yanks?|yanked|shoves?|shoved)" - + _GAP + r"(?:aside|up|down|open)\b") + + _GAP + r"(?:aside|up|down|open)\b" + r"|\b(?:unzips?|unzipped|unbuttons?|unbuttoned|unfastens?|unfastened|" + r"undoes|undid)\b") # POSTURES and the _POSTURE list built from it used to live here. Nothing read # _POSTURE -- it was a second, dead copy of the posture vocabulary, and it had @@ -440,6 +469,17 @@ def _outside_speech(text): return _SPOKEN_SPAN.sub(" ", text or "") +def staged_text(text): + """What a beat STAGES: not what anybody says, and not what the narration asks. + + A narrated question is the same kind of thing as a line of speech. "Maya waits by + the door. Will he come?" asks whether he will, which is to say he is not there -- + and the "he" read as him being present, so Will was described into the shot of + her waiting for him. Removed for deciding who is in the shot, as speech is.""" + staged = _outside_speech(text) + return " ".join(s for s in re.split(r"(?<=[.!?])\s+", staged) if not s.rstrip().endswith("?")) + + # THREE modifiers, not two: "mirrored stainless steel collar" is three words and # a noun, and the third was the first to be dropped. _HW_ONE = _rx(r"\b(" + _ADJ + r"(?:\s+" + _ADJ + r"){0,2}\s+)?(" @@ -478,6 +518,14 @@ _GARMENT_ONE = _rx(r"\b(" + _ADJ + r"(?:\s+" + _ADJ + r"){0,2}\s+)?(" _TAKES_OFF = _rx(r"\b" + TAKES_OFF + r"\b") _PUTS_ON = _rx(r"\b" + PUTS_ON + r"\b") _DISPLACES = _rx(r"\b" + DISPLACES + r"\b") +# ...and the sentence can still finish the job after the garment is named. "Kate +# unzips the denim skirt and steps out of it": the unzip opens it, the rest of the +# sentence takes it off, and the removal verb comes after the item where the reader +# above does not look. +_OPENS_GARMENT = _rx(r"\b(?:unzips?|unzipped|unbuttons?|unbuttoned|unfastens?|unfastened|" + r"undoes|undid|unhooks?|unhooked|unclasps?|unclasped)\b") +_COMPLETES_OFF = _rx(r"\b(?:off|out\s+of|away|lets?\s+(?:it|them)\s+(?:fall|drop|slide)|" + r"drops?\s+(?:it|them)|falls?\s+(?:to|down|away|off))\b") _MOVES = _rx(r"\b(?:walks?|walked|walking|goes|go|went|going|runs?|ran|running|" r"steps?|stepped|stepping|moves?|moved|moving|enters?|entered|" r"leaves?|left|leaving|crosses|crossed|crossing|climbs?|climbed|" @@ -595,11 +643,6 @@ def nudity_in(text): return out -def bare_sentence(region): - """How to say a region is bare, or "" for one with no wording.""" - return next((s for _rx, r, s in _REGION_RX if r == region), "") - - def _bare_on(p, regions): for r in ([regions] if isinstance(regions, str) else regions): if r and r not in p.bare: @@ -653,11 +696,6 @@ def _nearest_part(parts, at, ats): return "" -def hardware_in(text): - """Every piece of hardware named, as (canonical, part, as-written).""" - return [(c, p, w) for c, p, w, _at in hardware_spans(text)] - - def position_spans(text): """Every limb position named, as (name, at).""" out = [] @@ -705,9 +743,15 @@ def place_in(text): Behind a preposition, so a room has to be somewhere somebody IS. "Ana looks at the door" names no room -- and a door is not on the list in any case.""" - m = _PLACE_IN.search(text or "") + text = text or "" + m = _PLACE_IN.search(text) if not m: return "" + clause = re.split(r"[.;!?]", text[:m.end()])[-1] + present = re.search(r"\b(?:is|are|was|were|stands?|sits?|waits?|lies?|remains?)\b", + clause, re.I) + if not _MOVES.search(clause) and not present: + return "" got = re.sub(r"\s+", " ", m.group(1).lower()).strip() # A BARE "room" NAMES NOWHERE. "Ana walks into the room" says she goes # inside, not which room -- and taking it as a place produced "The shot is @@ -730,6 +774,18 @@ def garments_in(text): return out +_CLAUSE_BOUNDARY = re.compile(r"[,;]|\b(?:and|while)\b", re.I) + + +def _clause_at(text, at, boundaries=None): + """Return the clause containing character offset `at` and its start offset.""" + if boundaries is None: + boundaries = list(_CLAUSE_BOUNDARY.finditer(text or "")) + lo = max((m.end() for m in boundaries if m.end() <= at), default=0) + hi = min((m.start() for m in boundaries if m.start() > at), default=len(text)) + return text[lo:hi], lo + + # Hardware, not clothing. Taking clothes off does not unlock anything, so these # are kept out of the garment answer -- the standing rule is that hardware is @@ -740,6 +796,32 @@ _NOT_CLOTHING = re.compile( r"straitjacket|spreader|hogtie|clamps?|clips?)$", re.I) _PHRASE_ONE = _rx(r"\b(?:" + GARMENT_PHRASES + r")\b") _WORD_ONE = _rx(r"^(?:" + GARMENT_WORDS + r")s?$") +# The same list with NO optional plural, which is what says whether a trailing "s" +# belongs to the word or was added to it. The vocabulary is clean on this: garments that +# are inherently plural are listed only in the plural (boots, jeans, shorts, panties, +# tights, leggings, socks, knickers, trousers, gloves) and the rest only in the singular +# (skirt, vest, top), so "stem is itself a garment" is an exact test and not a guess. +_WORD_EXACT = _rx(r"^(?:" + GARMENT_WORDS + r")$") + + +def singular_garment(word): + """A garment word as the VOCABULARY spells it, so two readers cannot disagree. + + Reported: several women take their skirts off and the skirts are back in the next + beat. "their skirts" yields the token "skirts" while the sheet says "a denim + skirt", and every reader downstream looks the token up in the sheet -- the scrub + by pattern, infer_removals by entry head -- so a plural garment matched nothing + and the removal silently did nothing at all. One woman undressing wrote "her + skirt" and worked; the moment the subject went plural so did the garment. + + A trailing "s" comes off only when the stem is ITSELF a garment word, which is an + exact test here and not a guess: the vocabulary lists inherently plural garments + only in the plural (boots, jeans, shorts, panties, tights, leggings, socks, + knickers, trousers, gloves) and the rest only in the singular.""" + low = str(word or "").lower().strip("-") + if low.endswith("s") and _WORD_EXACT.match(low[:-1]): + return low[:-1] + return low def garment_words(text): @@ -761,12 +843,60 @@ def garment_words(text): text = _PHRASE_ONE.sub(" ", text) for word in re.findall(r"\b[\w-]{3,}\b", text): low = word.lower().strip("-") - if low in out or _NOT_CLOTHING.match(low): + if _NOT_CLOTHING.match(low): continue - if _WORD_ONE.match(low): + if not _WORD_ONE.match(low): + continue + # The token the sheet wrote, not the one the beat happened to inflect. See + # singular_garment -- shared with infer_removals so the two cannot disagree. + low = singular_garment(low) + if low not in out: out.append(low) return out +# A POSTURE DENIED IS NOT A POSTURE TAKEN. +# +# Reported: a woman chained by the ankles and forced into a squat stood up anyway. +# The chain clauses were all correct -- the metal "already drawn to its full length, +# so the position it fixes is the position that keeps" was on every shot after the +# squat. What sat beside it was the posture latch saying she was STANDING, because +# "She cannot stand." matched `stand` and nothing looked at the `cannot`. The latch +# then carried that forward, so every later shot asserted, flatly and positively, +# the one thing the chains were there to prevent. At cfg 1 a positive statement wins. +# +# The same shape as _in_a_request, which already suppresses a posture that is ASKED +# for rather than taken. Attempts are here too: "tries to stand", "struggles to get +# up", "strains to rise" are all bodies that have NOT got there, and reading them as +# arrival is the same error in a friendlier disguise. +# +# A SHORT WINDOW on purpose -- the five words before the verb. The cue always sits +# immediately in front of it ("cannot stand", "no longer able to stand"), and a wider +# reach would let a `cannot` from a different clause silence a real posture. +_POSTURE_DENIED = _rx( + r"\b(?:cannot|can\s*not|can['\u2019]?t|could\s*not|could\s*n['\u2019]?t|" + r"unable|never|not\s+able|no\s+longer\s+able|" + r"does\s*n['\u2019]?t|does\s+not|do\s*n['\u2019]?t|did\s*n['\u2019]?t|did\s+not|" + r"will\s+not|wo\s*n['\u2019]?t|fail(?:s|ed|ing)?|" + r"tr(?:y|ies|ied|ying)|attempt(?:s|ed|ing)?|struggl(?:e|es|ed|ing)|" + r"strain(?:s|ed|ing)?|fight(?:s|ing)?|want(?:s|ed)?|need(?:s|ed)?|" + r"told|ordered|asked|begs?|begged)\b") + + +def denied_posture(text, at): + """Is the posture verb at `at` negated, or only attempted, by what precedes it?""" + before = str(text or "")[:max(0, int(at))] + # A cue belongs to its OWN clause. Without stopping at the boundary, "McKenna + # cannot kneel, so she sits" reached back past the comma and silenced the sitting + # -- suppressing a posture the beat plainly states, which is the same class of + # error in the other direction. + cut = 0 + for _m in re.finditer(r"[,;:.!?]|\b(?:so|and|but|then|yet|while|as|before|after)\b", + before, re.I): + cut = _m.end() + window = " ".join(re.findall(r"[\w'\u2019]+", before[cut:])[-5:]) + return bool(_POSTURE_DENIED.search(window)) + + def posture_in(text): """The posture this beat puts a body in. '' when it does not. @@ -775,7 +905,7 @@ def posture_in(text): for that, but off the same vocabulary.""" t = text or "" hits = sorted((m.start(), name) for name, rx in _POSTURE_OF - for m in [rx.search(t)] if m) + for m in rx.finditer(t) if not denied_posture(t, m.start())) return hits[0][1] if hits else "" @@ -820,7 +950,7 @@ _TRAILING_VERB = (r"take[sn]?|took|taking|pull(?:s|ed|ing)?|peel(?:s|ed|ing)?|" r"toss(?:es|ed)?|throw[s]?|threw|kick(?:s|ed|ing)?|" r"slide[s]?|slid|wriggle[sd]?|wiggle[sd]?") # ...and verbs that are a removal on their own, needing no particle. -_UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|unzip(?:s|ped)?|" +_UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|shed(?:s|ding)?|unzip(?:s|ped)?|" r"unbutton(?:s|ed)?|unhook(?:s|ed)?|unclasp(?:s|ed)?|unfasten(?:s|ed)?|" # Hardware comes off by being UNDONE, and these were missing: a beat # saying "unlocks the belt" left it described as worn for the rest of @@ -830,11 +960,26 @@ _UNDO_VERB = (r"remove[sd]?|removing|undress(?:es|ed)?|unzip(?:s|ped)?|" r"undo(?:es)?|undid") +# WHERE A GARMENT ENDS UP ONCE IT IS OFF. A thing on the floor is not on a body, +# and this is how a beat says so -- by DESTINATION, not by verb. Two readers need +# the same list, for opposite reasons: the removal reader to call it a removal, +# the restore reader to stop calling it one. "Lets the skirt fall" drops a lifted +# skirt back over her legs; "lets the thong fall to the floor" is the thong coming +# off, and the only difference between those two sentences is this list. +FLOOR = (r"floor|ground|tiles?|tiling|lino|mat|bath\s*mat|rug|carpet|deck|boards|" + r"concrete|grass|sand|bed|sofa|couch|chair|seat|stool|bench|basket|" + r"hamper|laundry|pile|heap") +TO_THE_FLOOR = (r"(?:to|on|onto|into|in)\s+(?:the|a|an|her|his|their)?\s*" + r"(?:" + FLOOR + r")\b") +_LANDS_OFF = _rx(r"\s*(?:fall(?:s|ing)?|drop(?:s|ping)?|land(?:s|ing)?)?\s*" + + TO_THE_FLOOR) + _DISPLACE_WAY = (r"back\s+up|back\s+down|down|up|aside|open|back|" r"off\s+(?:one|her|his|their)\s+shoulders?") _DISPLACE = re.compile( r"\b(?:" + _STRIP_VERB + r"|push(?:es|ed|ing)?|shove[sd]?|roll(?:s|ed|ing)?|" - r"hitch(?:es|ed)?|hike[sd]?|open(?:s|ed)?|undo(?:es)?|unzip(?:s|ped)?|" + r"hitch(?:es|ed)?|hike[sd]?|open(?:s|ed)?|undo(?:es)?|undid|unzip(?:s|ped)?|" + r"unbutton(?:s|ed)?|unfasten(?:s|ed)?|unhook(?:s|ed)?|unclasp(?:s|ed)?|" # LIFTING A SKIRT IS DISPLACING IT, and none of these were here. Asked # for directly: "when the skirt has been lifted up to show the chastity # belt, that's when it should be shown". Lifting was not read as moving @@ -874,14 +1019,23 @@ def scene_name_for(head, scene): # a reference is pinning, so it is the worst one to describe loosely. item = re.sub(r"<\s*picture\s+\d+\s*>", " ", item, flags=re.I) item = re.sub(r"\s+", " ", item).strip() - if not item or item.split()[-1].lower() != head: - continue - # Drop a leading article or possessive; they are not description. - item = re.sub(r"^(?:a|an|the|her|his|their|its)\s+", "", item, flags=re.I) - # The longest entry wins: a sheet that names it twice described it most - # fully once, and the fuller name is the one worth carrying. - if len(item) > len(best): - best = item + # ONE ENTRY CAN HOLD SEVERAL GARMENTS, and the name is the garment's own + # part of it. "navy jacket over a white shirt" ends in "shirt", so the + # whole entry came back as the shirt's name and "the navy jacket over a + # white shirt open" was said about a shirt being unbuttoned. Split on the + # words that join garments, and cut a "with ..." tail, which describes a + # garment rather than naming it. + for part in re.split(r"\s+(?:over|under|beneath|underneath|on\s+top\s+of|and)\s+", + item, flags=re.I): + part = re.split(r"\s+with\s+", part, flags=re.I)[0].strip() + if not part or part.split()[-1].lower() != head: + continue + # Drop a leading article or possessive; they are not description. + part = re.sub(r"^(?:a|an|the|her|his|their|its)\s+", "", part, flags=re.I) + # The longest entry wins: a sheet that names it twice described it + # most fully once, and the fuller name is the one worth carrying. + if len(part) > len(best): + best = part # The author's OWN capitalisation. Lowercasing turned "PVC" into "pvc" and # "Shiny white crop top" into all-lowercase -- a different token sequence than # was written, for a brand or material name that is capitalised for a reason. @@ -910,6 +1064,13 @@ def displaced_garments(beat, scene): if not way and re.match(r"\s*(?:lift|rais|hoist|gather|bunch)", m.group(0), re.I): way = "up" + # ...and UNDOING a garment opens it. "Owen unzips his jacket" was a removal, + # then (once it was not) nothing at all -- the jacket went back to being + # described closed on the next shot, which is a jacket zipping itself up + # across a cut. + if not way and re.match(r"\s*(?:unzip|unbutton|unfasten|unhook|unclasp|undo|undid)", + m.group(0), re.I): + way = "open" if not way or not thing or thing in seen: continue # The garment has to be one the scene already dresses them in, and the head @@ -987,9 +1148,26 @@ def restored_garments(beat, scene): return [] out, low = [], scene.lower() for m in _PUT_BACK_NAMED.finditer(beat): + # ...UNLESS IT LANDS ON THE FLOOR. These verbs are the restore vocabulary + # because that is what people write for a lifted skirt -- let fall, drop, + # lower, let go of -- and the identical words take a garment OFF when the + # sentence says where it lands. "Lets the thong fall to the floor" was read + # as putting the thong back on: the author's removal, enacted backwards, and + # from there the sheet described it as worn for the rest of the film. + # Reported as a thong restored after she undressed. See FLOOR. + if _LANDS_OFF.match(beat[m.end():]): + continue thing = re.sub(r"\s+", " ", (m.group(1) or "")).strip().lower() if not thing: continue + # ...AND THE CAPTURE CANNOT RUN THROUGH A PREPOSITION. The group takes + # spaces so a sheet's "long grey skirt" comes back whole, and on "drops the + # thong on the floor" it swallowed "thong on the floor" instead -- head + # "floor" -- so the restore was keyed to the ROOM. The scene named a wet + # floor, scene_name_for handed back "tiled bathroom with a wet floor", and + # the beat was recorded as putting the bathroom back on. + if re.search(r"\b(?:on|onto|to|into|in|at|over|under|from|with|and)\b", thing): + continue head = thing.split()[-1] if len(head) < 3 or head not in low: continue @@ -1197,6 +1375,10 @@ def _alias_at(word, staged): return fallback +_AUX_FOLLOWER = re.compile(r"\s+(?:I|you|he|she|we|they|it|there|this|that|anyone|someone|" + r"everyone|anybody|somebody)\b") + + def names_in(beat, cast): """Names this beat STAGES, in the order the sentence puts them. @@ -1208,14 +1390,23 @@ def names_in(beat, cast): Speech-stripped, for the same reason it is everywhere else -- "McKenna, where are you?" is how absence gets written, and reading it as presence put a whole sheet entry into a shot the person is not in.""" - staged = _outside_speech(beat or "") + staged = staged_text(beat or "") names = [str(n) for n in (cast or []) if n] hits, found = [], set() for n in names: - m = re.search(r"\b" + re.escape(n) + r"\b", staged) - if m: + # A NAME THAT IS ALSO A WORD. "Will he come?" and "May I come in?" open a + # sentence with the name followed by who the question is about, and read as + # the person they staged Will and May -- a second character in a shot about + # somebody waiting for them. A name followed straight away by a subject + # pronoun at the start of a sentence is the verb; the next use can still be + # the person ("Will opens the gate"). + for m in re.finditer(r"\b" + re.escape(n) + r"\b", staged): + _opens = re.search(r"(?:^|[.!?]\s*[\"'\u201c]?)\s*$", staged[:m.start()]) + if _opens and _AUX_FOLLOWER.match(staged[m.end():]): + continue hits.append((m.start(), n)) found.add(n) + break # A SHEET NAME IS OFTEN LONGER THAN WHAT THE BEATS CALL HER. "Mistress Vale" # on the sheet and "the Mistress" in every beat matched nothing, so her line # was in no shot at all and the model invented her from scratch each time. @@ -1415,20 +1606,35 @@ class SceneState: subject = who[0] if who else next(iter(list(self.people) or list(cast) or [""])) - hw = hardware_in(beat) - applying = bool(hw) and bool(_APPLY.search(beat)) and not _RELEASE.search(beat) + spans = hardware_spans(beat) + garments = list(_GARMENT_ONE.finditer(beat)) + boundaries = list(_CLAUSE_BOUNDARY.finditer(beat)) if spans or garments else [] + applying = bool(spans) and bool(_APPLY.search(beat)) releasing = bool(_RELEASE.search(beat)) - if applying: - wearer = _wearer(beat, who, subject) - p = self.person(wearer) + if applying or releasing: # MODIFIERS BIND TO THE NEAREST ITEM. "handcuffs her wrists behind # her back and locks a steel collar around her neck, chained to the # wall" carries two modifiers and two items; giving both modifiers # to both items produced handcuffs chained to a wall they were never # near, and a collar held behind a back. - spans = hardware_spans(beat) for canon, part, written, at in spans: + clause, lo = _clause_at(beat, at, boundaries) + item_at = at - lo + apply_at = max((m.start() for m in _APPLY.finditer(clause) + if m.start() <= item_at), default=-1) + release_at = max((m.start() for m in _RELEASE.finditer(clause) + if m.start() <= item_at), default=-1) + if apply_at < 0 and release_at < 0: + continue + local_who = names_in(clause, cast) + wearer = _wearer(clause, local_who or who, subject) + p = self.person(wearer) + if release_at >= 0: + keys = [k for k in list(p.hardware) if k[0] == canon] + for key in keys: + changed["released"].append((wearer, p.hardware.pop(key))) + continue # KEYED BY THE PAIR. A chain on the ankles and a chain on the # wrists are two restraints; keyed by name alone the second # overwrote the first and one of them was never drawn again. @@ -1446,7 +1652,7 @@ class SceneState: # Its anchor needs no transferring either: with the tether gone from # the spans, the anchor binds to the nearest remaining item, which # is the one it was always describing. - elif releasing: + if releasing and not spans: # Whoever is actually wearing it. "The guard unlocks the handcuffs" # names only the agent, and taking the subject there tried to # release hardware from the man holding the key. @@ -1457,13 +1663,8 @@ class SceneState: # Released by NAME, whatever part it is on: an unlocking beat says # "unlocks the chain", not which of two chains, and matching the # pair left one fastened forever. - _kinds = {c for c, _pt, _w in hw} - named = [k for k in list(p.hardware) if k[0] in _kinds] - if named: - for key in named: - changed["released"].append((wearer, p.hardware.pop(key))) - elif re.search(r"\b(?:them|it|her|him|everything|all\s+of\s+it)\b", - beat, re.I): + if re.search(r"\b(?:them|it|her|him|everything|all\s+of\s+it)\b", + beat, re.I): # "the guard releases her" names no item, so all of it comes off. while p.hardware: changed["released"].append((wearer, p.hardware.popitem()[1])) @@ -1472,12 +1673,26 @@ class SceneState: # to be named -- a bare "she undresses" says nothing about which garment, # and guessing is how a garment came off a beat before the beat that # took it off. - wearer_g = _wearer(beat, who, subject) if len(who) > 1 else subject - if wearer_g: - p = self.person(wearer_g) - for g in garments_in(beat): + if subject: + for m in garments: + g = f"{(m.group(1) or '').strip()} {m.group(2)}".strip().lower() key = _garment_key(g) - if _TAKES_OFF.search(beat): + clause, lo = _clause_at(beat, m.start(), boundaries) + item_at = m.start() - lo + actions = [(x.start(), "off") for x in _TAKES_OFF.finditer(clause) + if x.start() <= item_at] + actions += [(x.start(), "on") for x in _PUTS_ON.finditer(clause) + if x.start() <= item_at] + actions += [(x.start(), "aside") for x in _DISPLACES.finditer(clause) + if x.start() <= item_at] + action = max(actions, default=(-1, ""))[1] + if action == "aside" and _OPENS_GARMENT.search(clause[:item_at]): + if _COMPLETES_OFF.search(re.split(r"[.;!?]", beat[m.end():])[0]): + action = "off" + local_who = names_in(clause, cast) + wearer_g = _wearer(clause, local_who, subject) + p = self.person(wearer_g) + if action == "off": if key not in [_garment_key(x) for x in p.removed]: p.removed.append(g) changed["removed"].append((wearer_g, g)) @@ -1485,7 +1700,7 @@ class SceneState: p.displaced = [x for x in p.displaced if _garment_key(x) != key] _bare_on(p, region_of(g)) - elif _PUTS_ON.search(beat): + elif action == "on": if key not in [_garment_key(x) for x in p.worn]: p.worn.append(g) changed["worn"].append((wearer_g, g)) @@ -1496,7 +1711,7 @@ class SceneState: # dresses is told for the rest of the film that the region is # bare, over the garment she just put on. _bare_off(p, region_of(g)) - elif _DISPLACES.search(beat): + elif action == "aside": if key not in [_garment_key(x) for x in p.displaced]: p.displaced.append(g) changed["displaced"].append((wearer_g, g)) @@ -1506,7 +1721,11 @@ class SceneState: # ever said what was on the chest. _nude = nudity_in(beat) if _nude: - for n in (who or ([subject] if subject else [])): + nude_at = min((m.start() for rx, _regions in _NUDITY_RX + for m in rx.finditer(beat)), default=len(beat)) + located = [(abs(beat.find(n) - nude_at), n) for n in who if beat.find(n) >= 0] + owners = [min(located)[1]] if located else ([subject] if subject else []) + for n in owners: q = self.person(n) _bare_on(q, _nude) # ...and it takes the garments OFF. Saying somebody is topless diff --git a/dumas_h3_longvideos_runtime.py b/dumas_h3_longvideos_runtime.py new file mode 100644 index 0000000..5861d4a --- /dev/null +++ b/dumas_h3_longvideos_runtime.py @@ -0,0 +1,739 @@ +# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos +# Copyright (c) 2026 Smite79. All rights reserved. +# Redistribution, in whole or in part, requires written permission. +# This notice may not be removed or altered. See LICENSE. +"""Sampling, decoding, resizing, memory handling, and frame assembly.""" + +import math + +import torch +import nodes +import comfy.utils +import comfy.sample +import comfy.samplers +import comfy.nested_tensor +import comfy.model_management as mm +import latent_preview + + +class FrameAccumulator: + """Build the final frame tensor once, retaining overflow only when necessary.""" + + def __init__(self, capacity, dtype, store_on_cpu): + self.capacity = int(capacity) + self.dtype = dtype + self.store_on_cpu = bool(store_on_cpu) + self.tensor = None + self.used = 0 + self.overflow = [] + + def add(self, frames): + count = int(frames.shape[0]) + if self.tensor is None and count: + device = torch.device("cpu") if self.store_on_cpu else frames.device + self.tensor = torch.empty( + (max(count, self.capacity),) + tuple(frames.shape[1:]), + dtype=self.dtype, device=device) + if (not self.overflow and self.tensor is not None + and self.used + count <= self.tensor.shape[0]): + self.tensor[self.used:self.used + count].copy_(frames) + self.used += count + return + self.overflow.append(frames.to("cpu", self.dtype, copy=True) + if self.store_on_cpu else frames) + + def release(self): + """Drop every tensor held, now, rather than whenever the collector gets to it. + + On an interrupt the render unwinds through frames the collector tears down in + its own order, and a large video buffer freed after the models it was sized + against have already gone is a free the allocator cannot explain. Deliberately + does NOT empty the cache: that is another CUDA call, and if the context is + already in a sticky error state it is one more thing to abort inside.""" + self.tensor = None + self.overflow = [] + self.used = 0 + + def finish(self): + if not self.overflow: + if self.tensor is None: + return torch.cat(self.overflow, dim=0) + if self.used == self.tensor.shape[0]: + out = self.tensor + else: + # COMPACT, never a slice. A slice of a larger buffer keeps the WHOLE + # buffer's storage alive, which is the retention this class exists to + # prevent -- and test_the_chain_is_never_held_twice measures exactly + # that, demanding no unused bytes behind the returned tensor. + # + # There is slack because the capacity is now an upper bound: it can no + # longer assume trim_seam drops a frame at every seam, since a shot that + # opens on no keyframe keeps its first frame. Over-allocating by at most + # one frame per seam and compacting once is the bounded cost. The + # alternative -- an exact guess that can be too small -- drops into the + # overflow list, which with cleanup_between_shots off holds every shot's + # decoded frames live on the GPU until the end of the run. + out = torch.empty((self.used,) + tuple(self.tensor.shape[1:]), + dtype=self.dtype, device=self.tensor.device) + out.copy_(self.tensor[:self.used]) + self.tensor = None + return out + + extra = sum(int(piece.shape[0]) for piece in self.overflow) + reference = self.tensor if self.tensor is not None else self.overflow[0] + out = torch.empty((self.used + extra,) + tuple(reference.shape[1:]), + dtype=self.dtype, device=reference.device) + if self.tensor is not None and self.used: + out[:self.used].copy_(self.tensor[:self.used]) + at = self.used + while self.overflow: + piece = self.overflow.pop(0) + count = int(piece.shape[0]) + out[at:at + count].copy_(piece) + at += count + self.tensor = None + return out + + +H3_FPS = 24 # H3 renders 24 fps, always + + +AUDIO_LATENT_FPS = 40 # audio latent frames per second + + +AUTO_TILE_T = 8 # temporal chunk for a tiled decode + + +MAX_FRAMES = 362 # H3's own ceiling (~15s) + + +CANVAS_MULTIPLE = 32 + + +REF_IMAGE_SHORT_EDGE = 2048 + + +def align_frame_count(n): + """Up to the next valid H3 frame count. The grid is 17k+5.""" + n = max(5, int(n)) + while n % 17 != 5: + n += 1 + return min(n, MAX_FRAMES) + + +def video_latent_t(fc): + return 2 if fc <= 5 else ((fc - 5) // 17) * 5 + 2 + + +def temporal_shape(length, fps=H3_FPS): + """(frame count, video latent frames, audio latent frames) for a shot. + + `fps` is accepted but deliberately IGNORED: the audio latent has to line up + with 24 fps video or the shot's sound is stretched against its picture.""" + fc = align_frame_count(length) + return fc, video_latent_t(fc), round(fc / H3_FPS * AUDIO_LATENT_FPS) + + +def ref_image_canvas(w, h, gen_w, gen_h, mode="match"): + """Pure: the (width, height) a reference image is encoded at. + + 'match' scales it (DOWN only, aspect kept) to the generation's pixel area, so a + reference costs about as much as one frame of the shot. 'max' goes to the + reference pipeline's 2048 short edge for the best identity fidelity, which on a + long chain is several times slower because the rows are re-attended every step + of every shot. Never upscales: a small reference stays small.""" + w, h = max(1, int(w)), max(1, int(h)) + if mode == "max": + scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h)) + else: + scale = min(1.0, math.sqrt((int(gen_w) * int(gen_h)) / float(w * h))) + snap = lambda v: max(CANVAS_MULTIPLE, round(v * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) + return snap(w), snap(h) + + +def _resize(image, width, height, crop): + s = image[..., :3].movedim(-1, 1) + s = comfy.utils.common_upscale(s, width, height, "lanczos", crop) + return s.movedim(1, -1) + + +def _empty_av_latent(width, height, length, fps, batch_size=1): + fc, lt, at = temporal_shape(length, fps) + video = torch.zeros([batch_size, 24, lt, height // 16, width // 16], device=mm.intermediate_device()) + audio = torch.zeros([batch_size, 32, 2, at], device=mm.intermediate_device()) + return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, fc + + +def _auto_tile_t(n_latent_frames, requested=None): + """Temporal tile for a tiled decode. An explicit value wins. + + The decode_tile_frames widget is gone, so this is where the value comes from + now. It has to come from somewhere: ComfyUI's decode_tiled_3d defaults tile_t + to 999, i.e. SPATIAL tiles only, and expanding the whole clip's time axis at + once is the single largest allocation in a run. A "tiled" decode that keeps the + full temporal extent barely lowers the peak, so the OOM retry that switches + tiling on was, without this, retrying with almost the same footprint.""" + if requested: + return int(requested) + n = int(n_latent_frames or 0) + return AUTO_TILE_T if n > AUTO_TILE_T else None + + +def _decode_video(vae, out_latent, tiled, free_first=None, tile_t=None, tile_xy=None, + keep=()): + """Decode the video latent. + + `free_first` is the diffusion model: sampling is finished, and the video VAE + needs the room for THIS decode -- the free runs immediately before it, not to + make room for the next shot. On a card where the DiT is most of the VRAM, the + decode does not fit until it goes. + + `keep` is what must NOT be evicted on the way. It was `keep_loaded=[]`, which + unloaded every resident model -- including the video VAE, which ComfyUI then + reloaded three lines later to run the decode. An evict-and-reload of the thing + about to be used, once per shot, on every card. Peak VRAM is identical either + way, since the VAE has to be resident to decode; the round trip was pure cost. + + memory_required is ASKED FOR HONESTLY, which it was not. It was 1e30, and + free_memory computes `memory_to_free = memory_required - get_free_memory(device)` + (model_management.py:887), so 1e30 means "unload everything not in keep_loaded", + every shot, in full -- skipping partially_unload entirely. + + What that evicts is the DiT, three lines before the next shot needs it again. On + a machine whose RAM is already full of finished frames there is nowhere for it to + go but disk, so the reload is a read from the drive, once per shot. Reported as + thrashing that slows the preload, and it is exactly that: the same weights being + read back at every boundary. + + The VAE knows what its own decode costs -- ComfyUI sizes it with + memory_used_decode and uses that number everywhere else. Asked for that instead, + a card with headroom frees NOTHING and the DiT simply stays. A card without + headroom frees what it needs and no more, which is what partially_unload is for. + 1e30 remains the fallback for a VAE that cannot estimate itself.""" + latent = out_latent["samples"] + if latent.is_nested: + latent = latent.unbind()[0] + if free_first is not None: + try: + mm.free_memory(_decode_headroom(vae, latent), mm.get_torch_device(), + keep_loaded=_resident(keep or (vae,))) + except Exception: + pass + # A VAE THAT ALREADY TILES DOES NOT NEED TO BE ASKED TO, AND ASKING COSTS 3x. + # + # MiniMaxH3VideoVAE.decode_tiled is, in full: + # + # def decode_tiled(self, z, **kwargs): + # return self.decode(z) + # + # Every tile_t/overlap_t/tile_x/tile_y this function computes is discarded, so + # the tiling the widget promises is not happening here -- the model tiles + # internally either way (256px spatial, 17-frame temporal), which is why + # comfy/sd.py sets handles_tiling on it. + # + # What the detour costs is the OUTPUT BUFFER. comfy's VAE.decode preallocates + # ONE result at vae_output_dtype and hands it to the model as output_buffer=, + # and MiniMaxH3VideoVAE.decode_temporal writes finalized chunks straight into + # it. Going through decode_tiled instead reaches _decode_tiled_owned, which + # calls the model with output_buffer=None -- so decode_temporal allocates its + # own at torch.float32 -- and then makes an fp16 `copy=True` of that. Two + # buffers, the larger of them at double width: + # + # tiled : fp32 2.60GB + fp16 copy 1.30GB = 3.90GB per shot + # decode: one preallocated fp16 = 1.30GB per shot + # + # at 362 frames of 1056x608. Every shot, on the node's own default. + # + # So: when the VAE owns its tiling AND can be written into, the un-tiled call IS + # the tiled one, minus the copies. Anything else keeps the old path -- this is a + # detour around a detour, not a claim that tiling is useless. + _owns_tiling = bool(getattr(vae, "handles_tiling", False) and getattr( + getattr(vae, "first_stage_model", None), "comfy_has_chunked_io", False)) + if tiled and _owns_tiling: + imgs = vae.decode(latent) + elif tiled: + # Temporal + spatial tiling. Without tile_t the VAE expands the WHOLE latent + # clip at once, which on a 243-frame 1344x768 shot is the single largest + # allocation in the run -- and on an unpruned checkpoint that is already + # streaming, it is what tips the card over. Decoding in temporal chunks + # trades a little speed for a much lower peak; None keeps ComfyUI's defaults. + args = {} + tile_t = _auto_tile_t(latent.shape[2] if latent.ndim >= 5 else 0, tile_t) + if tile_t: + args["tile_t"] = int(tile_t) + args["overlap_t"] = max(1, int(tile_t) // 8) + if tile_xy: + args["tile_x"] = int(tile_xy) + args["tile_y"] = int(tile_xy) + try: + imgs = vae.decode_tiled(latent, **args) if args else vae.decode_tiled(latent) + except TypeError: + imgs = vae.decode_tiled(latent) # older signature without tile_t + else: + imgs = vae.decode(latent) + if len(imgs.shape) == 5: + imgs = imgs.reshape(-1, imgs.shape[-3], imgs.shape[-2], imgs.shape[-1]) + return imgs + + +def _decode_audio(audio_vae, out_latent): + latent = out_latent["samples"] + if latent.is_nested: + latent = latent.unbind()[-1] + audio = audio_vae.decode(latent).movedim(-1, 1) + std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 + std[std < 1.0] = 1.0 + audio = audio / std + sr = getattr(audio_vae, "audio_sample_rate_output", getattr(audio_vae, "audio_sample_rate", 44100)) + return {"waveform": audio, "sample_rate": sr} + + +def _is_oom(e): + return isinstance(e, torch.cuda.OutOfMemoryError) or "out of memory" in str(e).lower() + + +def _deep_cleanup(): + """Release cached VRAM between shots so a long chain does not accumulate and OOM. + + It unloads NOTHING. soft_empty_cache(force) ignores `force` in current ComfyUI + (model_management.py:2050) -- the body only reaches empty_cache() and + ipc_collect() -- so this drops cached blocks, not models. The `True` is kept + only for older builds that read it; the older comment here claimed this took an + unload_all_models path, and it does not.""" + try: + mm.soft_empty_cache(True) + except TypeError: + mm.soft_empty_cache() + try: + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception: + pass + + +DECODE_HEADROOM = 1.25 # over ComfyUI's own estimate, for working allocations + + +SAMPLE_HEADROOM = 1.35 # likewise for sampling, which is the longer stretch + + +def _decode_headroom(vae, latent): + """VRAM this decode actually needs, by the VAE's own estimate. 1e30 if unknown. + + ComfyUI sizes every VAE with memory_used_decode and uses that number itself, so + it is the honest figure to hand free_memory. The alternative -- and what was here + -- is 1e30, which means "unload everything" and evicts the DiT before every + decode, three lines before the next shot reloads it. + + 1e30 on failure rather than 0: a bad estimate that frees too little turns a slow + render into an OOM, and a wrong guess should fall back to the behaviour that has + been running, not to no freeing at all.""" + try: + dtype = getattr(vae, "vae_dtype", None) or latent.dtype + need = float(vae.memory_used_decode(tuple(latent.shape), dtype)) + if need > 0: + return need * DECODE_HEADROOM + except Exception: + pass + return 1e30 + + +def _resident(models): + """The LoadedModel entries ComfyUI currently holds for `models`. + + That is the form free_memory's keep_loaded wants: it compares against the + entries in current_loaded_models, not against the ModelPatcher objects a node + is holding. Anything not matched is simply not kept, so a model that is not + resident costs nothing here.""" + out = [] + for lm in list(getattr(mm, "current_loaded_models", [])): + for m in models or (): + if m is None: + continue + try: + if lm.model is m or getattr(lm, "model", None) is getattr(m, "model", None): + if lm not in out: + out.append(lm) + except Exception: + pass + return out + + +def _image_out_dtype(): + """The dtype ComfyUI itself hands between nodes on THIS install. + + The join used to end in a hard-coded .float(), commented "back to what every + downstream node expects". That was true when it was written and is not a + constant: ComfyUI has --fp16-intermediates, and on an install running it the + VAE's own decode already returns fp16 -- VAE.vae_output_dtype() IS + model_management.intermediate_dtype() (comfy/sd.py) -- as do EmptyLatentImage + and the rest of nodes.py. So on that install the node was taking frames the + VAE handed it in fp16, widening them to fp32 nothing had asked for, and + handing them to nodes whose own convention is fp16. + + It is the largest thing this node holds, so the widening is not free: the + 2580-frame chain costed at the join is 9.3GB as fp16 and 18.5GB as fp32, + against 44.6GB of staged weights on a 62GB machine -- which is the difference + between the render finishing and the OOM killer taking the server. Reported as + exactly that, twice. + + Asked, not assumed, and never widened: whatever ComfyUI says it wants between + nodes is what the chain is built in. An install with the flag off is told + float32 and gets float32, byte for byte what it got before. Older builds have + no intermediate_dtype at all, so the fallback is the old constant.""" + try: + return mm.intermediate_dtype() + except Exception: + return torch.float32 + + +def _evict_all_but(keep_model, latent=None): + """Unload every model EXCEPT the diffusion model from the GPU. + + This is the fix for VRAM ratcheting across a long chain. soft_empty_cache() + only drops the CUDA allocator's cached blocks -- it does NOT unload models, so + ComfyUI keeps the Qwen3-VL text encoder (~14.6GB) and both VAEs resident in + current_loaded_models alongside the DiT. Each shot re-encodes the prompt + (text encoder), encodes the handoff keyframe (video VAE), then samples (DiT), + so all three compete for the card. + + ComfyUI does free ahead of each load -- load_models_gpu() calls free_memory() + for what it is about to need (model_management.py:975), so the weight path is + not purely reactive. What it cannot size for is a long chain's ACTIVATIONS on + a card where the DiT is most of the VRAM. Freeing explicitly, right after + conditioning is built and before sampling, keeps only what the sampler needs. + + ASKED FOR HONESTLY, and this is the expensive one. free_memory computes + `memory_to_free = memory_required - get_free_memory(device)`, so 1e30 meant + "unload everything but the DiT" on every shot, unconditionally -- on a 48GB card + with room for all of it as readily as on a 16GB one. What it unloads is the + ~14.6GB text encoder and both VAEs, and the next shot re-encodes the prompt and + the handoff keyframe, so all three come straight back. On a machine whose RAM is + already full of finished frames they come back from DISK, once per shot, which is + the thrashing this was reported as. + + The DiT can size its own activations -- memory_required(shape) is what ComfyUI + itself calls before a load -- so ask for that. A card with room frees nothing and + keeps the encoder resident; a card without frees exactly as much as it must. + 1e30 stays the fallback, because a bad estimate that frees too little turns a + slow render into an OOM.""" + need = 1e30 + try: + if latent is not None: + shape = latent["samples"].shape if isinstance(latent, dict) else latent.shape + need = float(keep_model.model.memory_required(tuple(shape))) * SAMPLE_HEADROOM + if not (need > 0): + need = 1e30 + except Exception: + need = 1e30 + try: + mm.free_memory(need, mm.get_torch_device(), + keep_loaded=_resident([keep_model])) + except Exception: + try: + mm.soft_empty_cache(True) + except Exception: + pass + + +def _sample_on_sigmas(model, seed, cfg, sampler_name, positive, negative, latent, sigmas): + """common_ksampler, driven by an EXTERNAL sigma schedule. + + common_ksampler derives its sigmas from (sampler_name, scheduler, steps, denoise) + and takes no schedule argument, so a schedule computed anywhere else cannot + reach it. Under PDD that is fatal rather than merely inconvenient: the heads + accept only their nine trained boundaries, and re-deriving the grid from + widgets means hitting it by coincidence and losing it again the moment a step + count changes. + + Mirrors nodes.common_ksampler's noise / mask / callback handling exactly -- the + only substitution is comfy.sample.sample_custom for comfy.sample.sample.""" + latent_image = latent["samples"] + latent_image = comfy.sample.fix_empty_latent_channels( + model, latent_image, + latent.get("downscale_ratio_spacial", None), + latent.get("downscale_ratio_temporal", None)) + noise = comfy.sample.prepare_noise(latent_image, seed, latent.get("batch_index")) + # `steps` here only sizes the progress bar -- the schedule is `sigmas`, whose + # step count is one less than its length (the trailing 0.0 is an endpoint). + callback = latent_preview.prepare_callback(model, max(len(sigmas) - 1, 1)) + samples = comfy.sample.sample_custom( + model, noise, cfg, comfy.samplers.sampler_object(sampler_name), sigmas, + positive, negative, latent_image, + noise_mask=latent.get("noise_mask"), callback=callback, + disable_pbar=not comfy.utils.PROGRESS_BAR_ENABLED, seed=seed) + out = latent.copy() + out.pop("downscale_ratio_spacial", None) + out.pop("downscale_ratio_temporal", None) + out["samples"] = samples + return out + + +RESIZE_CHUNK = 32 + + +def _stream_chunks(total): + """A collector that writes upscaled chunks into ONE destination as they land. + + Both chunk loops in _upscale_frames used `out.append(...)` then + `frames = torch.cat(out, dim=0)`. That is the shape the finished-chain join was + rebuilt to stop, at a LARGER size: the list holds the whole upscaled chain and + the cat allocates a second one, both live at the cat, and `out` is a local that + is never cleared -- so it survives the cat, survives the trailing resize, and is + still bound at the return. Meanwhile the CALLER's pre-upscale chain cannot be + dropped either, because `part = frames[s:s+batch]` is a view into it. + + At 2580 frames of 1056x608 that is 9.26GB per copy per doubling: 37GB x2 at 2x, + and 148GB x2 with the RealESRGAN_x4plus that is sitting in models/upscale_models. + Preallocating from the first chunk and copying into it removes exactly one of + those two, and drops the list at the same time. + + The destination is sized from the FIRST chunk, so the model's scale factor does + not have to be known in advance, and the frame count is the caller's own -- an + upscaler changes width and height, never the number of frames.""" + state = {"dst": None, "at": 0} + + def put(piece): + if state["dst"] is None: + state["dst"] = torch.empty((int(total),) + tuple(piece.shape[1:]), + dtype=piece.dtype, device=piece.device) + k = int(piece.shape[0]) + end = min(state["at"] + k, state["dst"].shape[0]) + if end > state["at"]: + state["dst"][state["at"]:end].copy_(piece[:end - state["at"]]) + state["at"] = end + + def done(): + d, at = state["dst"], state["at"] + if d is None: + return None + return d if at == d.shape[0] else d[:at] + + return put, done + + +def _resize_short_edge(frames, target, method="lanczos", chunk=0): + """Resize a [B,H,W,C] frame batch so its short edge == target (keeping aspect, + snapped to /32). Plain high-quality resize -- enlarges, doesn't add detail. + + IN CHUNKS, BECAUSE LANCZOS IS FOUR FULL-LENGTH COPIES. The whole chain went + into one common_upscale call, and comfy.utils.lanczos is three successive list + comprehensions over every frame at once: + + images = [Image.fromarray(...) for image in samples] # N at source size + images = [image.resize(...) for image in images] # N at target size + images = [torch.from_numpy(np.array(im).astype(np.float32)/255.) ...] + result = torch.stack(images) + return result.to(samples.device, samples.dtype) + + A comprehension builds the new list completely before rebinding the name, so at + each rebind BOTH are live; then torch.stack allocates a full copy while its list + still exists, and .to() allocates the result while the stack still exists. Note + the astype(np.float32): the input is fp16 but the two largest transients are at + DOUBLE its width. At 2580 frames to a 1080 short edge that peaked around 147GB + to produce a 29GB result, and it fires on a DOWNSCALE too. + + Chunked, the peak is the result plus one chunk's worth of that machinery. It is + bit-identical: PIL resizes each frame independently, so per-chunk and per-chain + give the same pixels. The early return for an already-correct size is kept, so + the common no-op case still allocates nothing.""" + b, h, w, c = frames.shape + if min(h, w) == target: + return frames + if h <= w: + nh = target; nw = max(32, int(round(target * w / h / 32) * 32)) + else: + nw = target; nh = max(32, int(round(target * h / w / 32) * 32)) + step = max(1, int(chunk) or RESIZE_CHUNK) + out = torch.empty((b, nh, nw, c), dtype=frames.dtype, device=frames.device) + for i in range(0, b, step): + part = comfy.utils.common_upscale( + frames[i:i + step].movedim(-1, 1), nw, nh, method, "disabled") + out[i:i + step].copy_(part.movedim(1, -1)) + del part + return out + + +def _upscale_frames(frames, mode, model_name, target_short_edge, batch=4): + """Optional post-pass upscale of the finished frames (on CPU). + mode 'model' : run a ComfyUI upscale model (Real-ESRGAN/UltraSharp class) + via the registered loader+apply nodes, chunked with cleanup + so 2000+ frames don't OOM; then fit to target short edge. + mode 'rtx' : NVIDIA RTX Video Super Resolution (Tensor Cores; fastest, + best quality for video -- needs Nvidia_RTX_Nodes_ComfyUI). + mode 'lanczos' : plain high-quality resize to the target short edge. + Any failure falls back to lanczos (or the raw frames), so it never breaks a + render. Returns (frames, note). NOTE: this SHARPENS/ENLARGES; it does not + reconstruct video detail the way a second-model (LTX 2.3) pass does.""" + if mode == "off" or frames is None or getattr(frames, "shape", [0])[0] == 0: + return frames, "" + note = "" + if mode == "rtx": + # NVIDIA RTX Video Super Resolution (Comfy-Org/Nvidia_RTX_Nodes_ComfyUI). + # Runs on RTX Tensor Cores -- far faster than ESRGAN-class models and + # generally cleaner on video, though like them it enhances/enlarges rather + # than reconstructing detail (an LTX 2.3 re-generation does that). + try: + rtx = (_find_node(["rtx", "video", "super"]) or _find_node(["rtxvideosuperresolution"]) + or _find_node(["rtx", "upscale"])) + if rtx is None: + raise RuntimeError("RTX node not installed (Nvidia_RTX_Nodes_ComfyUI)") + scale = 2 + if target_short_edge and int(target_short_edge) > 0: + cur = min(frames.shape[1], frames.shape[2]) + if cur > 0: + scale = max(1, min(4, int(round(int(target_short_edge) / cur)))) + _put, _done = _stream_chunks(frames.shape[0]) + n = frames.shape[0] + step = max(1, int(batch)) + for st in range(0, n, step): + part = frames[st:st + step] + res = None + for kw in ({"image": part, "scale": scale}, {"images": part, "scale": scale}, + {"image": part, "scale_factor": scale}, {"image": part}): + try: + res = _invoke_node(rtx, **kw); break + except TypeError: + continue + if res is None: + raise RuntimeError("RTX node signature not recognized") + _put(res.detach().to("cpu")) + del res, part + _deep_cleanup() + frames = _done() + note = f"RTX Video Super Resolution x{scale}" + if target_short_edge and int(target_short_edge) > 0: + frames = _resize_short_edge(frames, int(target_short_edge)) + note += f"; fit to {int(target_short_edge)}px short edge" + return frames, note + except Exception as e: + mode = "model" + note = f"RTX upscale unavailable ({e}); fell back to model/lanczos" + if mode == "model" and model_name and model_name != "none": + try: + loader = _find_node(["upscale", "model", "load"]) or _find_node(["loadupscalemodel"]) + applier = _find_node(["imageupscale", "model"]) or _find_node(["upscaleimageusingmodel"]) + if loader is None or applier is None: + raise RuntimeError("upscale-model nodes not found") + up_model = _invoke_node(loader, model_name=model_name) + _put, _done = _stream_chunks(frames.shape[0]) + n = frames.shape[0] + for s in range(0, n, max(1, int(batch))): + part = frames[s:s + max(1, int(batch))] + res = _invoke_node(applier, upscale_model=up_model, image=part) + _put(res.detach().to("cpu")) + del res, part + _deep_cleanup() + frames = _done() + note = f"upscaled with {model_name}" + except Exception as e: + mode = "lanczos" + note = f"model upscale unavailable ({e}); used lanczos" + if target_short_edge and int(target_short_edge) > 0: + try: + frames = _resize_short_edge(frames, int(target_short_edge)) + note = (note + "; " if note else "") + f"fit to {int(target_short_edge)}px short edge" + except Exception as e: + note = (note + "; " if note else "") + f"resize failed ({e})" + elif mode == "lanczos" and not note: + note = "lanczos selected but no target set -> unchanged" + return frames, note + + +def _find_node(substrings): + """Find a registered node whose key contains all of `substrings` (lowercased).""" + maps = getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {} + for k, v in maps.items(): + kl = k.lower() + if all(s in kl for s in substrings): + return v + return None + + +def _invoke_node(cls, **kwargs): + """Call a registered ComfyUI node (V1 FUNCTION or V3 execute) with kwargs and + return its first output. Used to reuse ComfyUI's own upscale-model loader/apply + so we don't reimplement spandrel loading or tiled scaling.""" + inst = cls() + fn = None + if getattr(cls, "FUNCTION", None) and hasattr(inst, cls.FUNCTION): + fn = getattr(inst, cls.FUNCTION) + else: + for cand in ("execute", "upscale", "load_model", "load"): + if hasattr(inst, cand): + fn = getattr(inst, cand); break + if fn is None: + raise RuntimeError("no callable entrypoint") + out = fn(**kwargs) + out = getattr(out, "result", out) + return out[0] if isinstance(out, (tuple, list)) else out + + +# --- THE GRADE THE CHAIN ADDS TO ITSELF ------------------------------------- +# Every shot boundary decodes a shot, hands its LAST frame over, and re-encodes that as +# the next shot's keyframe. The distill reproduces the keyframe faithfully enough to +# inherit whatever is already in it and SYNTHESISES frame 0 rather than copying it, so +# its own bias lands on top: S_next = a*S + b, a near 1, b above 0. Linear at best, +# geometric at worst, invisible shot to shot. And the VAE hard-clips every decode to +# 0..1, which makes the expansion a RATCHET -- headroom spent is not recoverable, so it +# shows as crushed blacks and blown highlights rather than merely as more contrast. +# +# These two are the measurement and the correction. Both work per colour channel, +# because the clip is per channel: the VAE un-whitens with ImageNet stds before it +# clamps, so the 0..1 rails sit at different distances in each channel and the blue +# floor and red ceiling bite first. A single luma number would miss the colour half. +LEVEL_POOL = 64 # cells per axis the level statistics are measured on + + +def frame_levels(img): + """(mean, std) per colour channel for one frame, as 3-vectors, or (None, None). + + Area-pooled to LEVEL_POOL first, so a pre-upscale frame and an upscaled one can be + compared: pooling measures the PICTURE's levels rather than its resolution. Measured + across a 2x resize, std agrees to 0.28% on picture-like content -- and to only 15% + on pure noise, because pooling cannot preserve variance that lives entirely at the + pixel scale. Real frames are the former, and whatever residual there is cancels + anyway: the caller measures the same pipeline difference separately and subtracts it. + + float32 throughout, deliberately: these frames are fp16 under + --fp16-intermediates, and an fp16 mean accumulated over a 1344x768 frame biases + badly enough to matter at the sizes being corrected here.""" + x = img + if x.dim() == 4: + x = x[0] + if x.dim() != 3 or int(x.shape[-1]) < 3: + return None, None + if int(x.shape[0]) < 2 or int(x.shape[1]) < 2: + return None, None + x = x[..., :3].float().permute(2, 0, 1).unsqueeze(0) + p = torch.nn.functional.adaptive_avg_pool2d(x, LEVEL_POOL)[0].reshape(3, -1) + return p.mean(dim=1), p.std(dim=1) + + +# The per-shot motion envelope lived here, measured so the built footsteps could be +# timed off the picture. Nothing is built any more -- see the note at the top of +# audio.py -- so there is nothing left to time, and a measurement with no reader is a +# measurement that rots. Removed with the synthesiser it served. + + +def apply_levels(img, gain, offset): + """Rescale a frame's contrast and level about its OWN per-channel mean. + + The pivot is the frame's own mean and never a target. That is the whole reason this + can run on any scene: a beat that walks into a darker room keeps its darkness, + because nothing here knows or cares what the level is -- only how much the last + boundary expanded it. Anchoring to shot 1 instead would cancel every deliberate + lighting change in the film, which is the opposite failure. + + Clamped into 0..1 because the next thing that happens to this frame is an 8-bit + quantisation (comfy.utils.common_upscale goes through a uint8 PIL round trip even + at the same size), so there is no headroom outside the range to borrow from.""" + x = img.float() + c = min(3, int(x.shape[-1])) + m = x[..., :c].reshape(-1, c).mean(dim=0) + g = gain[:c].to(device=x.device, dtype=x.dtype) + o = offset[:c].to(device=x.device, dtype=x.dtype) + y = x.clone() + y[..., :c] = ((x[..., :c] - m) * g + m + o).clamp(0.0, 1.0) + return y.to(img.dtype) diff --git a/dumas_h3_longvideos_shot_plan.py b/dumas_h3_longvideos_shot_plan.py new file mode 100644 index 0000000..b75b953 --- /dev/null +++ b/dumas_h3_longvideos_shot_plan.py @@ -0,0 +1,109 @@ +# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos +# Copyright (c) 2026 Smite79. All rights reserved. +# Redistribution, in whole or in part, requires written permission. +# This notice may not be removed or altered. See LICENSE. +"""Per-shot records passed from prompt planning to rendering.""" + +from dataclasses import dataclass, field + + +@dataclass +class Shot: + prompt: str + cast: list[str] + speech: bool + sounded: bool + voiced_only: bool + events: list[str] + frame_count: int = 0 + refs: list[object] = field(default_factory=list) + line_seconds: float = 0.0 # the planner's estimate of the spoken line, words / WORDS_PER_SEC + + +@dataclass +class ShotPlan: + shots: list[Shot] = field(default_factory=list) + + @property + def prompts(self): + return [shot.prompt for shot in self.shots] + + def add(self, prompt, cast, speech, sounded, voiced_only, events): + shot = Shot(prompt, list(cast or ()), bool(speech), bool(sounded), + bool(voiced_only), list(events or ())) + self.shots.append(shot) + + def set_frame_counts(self, counts): + counts = [int(n) for n in counts] + if len(counts) != len(self.shots) or any(n <= 0 for n in counts): + raise ValueError("frame counts must be positive and match the planned shots") + for shot, count in zip(self.shots, counts): + shot.frame_count = count + + def validate(self): + if any(shot.frame_count <= 0 for shot in self.shots): + raise ValueError("each shot needs a positive frame count before rendering") + return self + + def __len__(self): + return len(self.shots) + + +@dataclass +class PreparedVideo: + """Resolved inputs consumed by the render stage; model/tensor handles are shared.""" + _placed_shots: object + _first_is_plate: object + _returns: object + _soft_landing: object + _tagged_names: object + ambient_audio: object + ambient_level: float + apply_model_sampling: bool + audio_vae: object + auto_sound: bool + bared_shots: object + cfg: float + cleanup_between_shots: bool + clip: object + first_frame: object + foley_level: float + h: int + latent_upscale: str + latent_upscale_scale: float + megapixels: float + model: object + moved_shots: object + negative: object + notes: list[str] + plan: ShotPlan + ref_noise_aug: float | None + restart_after_removal: bool + revealed_shots: object + sampler_name: str + scheduler: str + seed: int + shift_audio: float + shift_video: float + sigmas: object + silence_nonspeech: bool + speech_lead_seconds: float + speech_tail_seconds: float + hold_levels: float + handoff_frames: int + staging_shots: object + steps: int + stripped_shots: object + cut_shots: object + tiled_decode: bool + trim_seam: bool + upscale: str + upscale_batch: int + upscale_model: str + upscale_target_short_edge: int + vae: object + w: int + shot_rooms: object = None # {0-based shot: (room it opens in, room it ends in)} + hardware_changed: object = None # 1-based shots that put hardware on or take it off + shot_frames: object = None # {0-based shot: (who its frames show, who is still there at its end)} + reentry_shots: object = None # {0-based shot: who walks in while the keyframe still has them} diff --git a/dumas_h3_longvideos_upstream.py b/dumas_h3_longvideos_upstream.py index d9331c3..c7570fe 100644 --- a/dumas_h3_longvideos_upstream.py +++ b/dumas_h3_longvideos_upstream.py @@ -2,38 +2,18 @@ # Copyright (c) 2026 Smite79. All rights reserved. # Redistribution, in whole or in part, requires written permission. # This notice may not be removed or altered. See LICENSE. -""" -H3 Long Videos -- chain MiniMax-H3 shots into one continuous video with audio. +"""Plan MiniMax-H3 shots, render their audio/video, and preserve continuity. -Rebuilt from scratch. The previous version grew a large prompt-engineering layer -that wrote continuity guards into every shot; measured, the user's own beat was -under 4% of the conditioning and the rest was boilerplate arguing with it. None of -that is here. What a shot is told is: your scene text, then your beat, verbatim. - -What this node does is the part a prompt cannot do -- the mechanics of chaining: - - * splits the prompt into beats on blank lines, one beat per shot; - * gives every shot the SAME length, so one seed is one noise field across the - chain (noise is drawn to the latent's shape, so unequal lengths mean unrelated - noise from the same seed, and detail resets at every cut); - * hands each shot the previous shot's last frame as its keyframe, encoded the - way H3 expects a keyframe to be encoded (one frame -> the 5f grid point); - * keeps identity references on every shot, which is the only fixed anchor a long - chain has against drift; - * anchors the audio branch to real silence on shots with no quoted line, because - H3 is a joint model and an unconditioned audio stream invents a voice that the - picture then lip-syncs to. - -Everything about what the video should CONTAIN is yours to write. +The node interface and prompt planning live here. Audio policy and synthesis, +conditioning assembly, and tensor/runtime operations have separate owner modules. """ -import gc -import json import math import os import re import sys import time +import uuid import torch @@ -43,26 +23,67 @@ import comfy.sample import comfy.samplers import comfy.nested_tensor import comfy.model_management as mm -import latent_preview -import node_helpers # The prompt engine: scene state, read beat by beat, rendered once per shot. # Imported by file path rather than by name so it resolves the same whether # ComfyUI loads this package as `custom_nodes.H3-LongVideos-V1` or bare. import importlib.util as _ilu -_eng_spec = _ilu.spec_from_file_location( - "h3_engine", os.path.join(os.path.dirname(os.path.abspath(__file__)), - "dumas_h3_longvideos_engine.py")) -engine = _ilu.module_from_spec(_eng_spec) -_eng_spec.loader.exec_module(engine) -H3_FPS = 24 # H3 renders 24 fps, always -AUDIO_LATENT_FPS = 40 # audio latent frames per second +def _load_local(name, filename): + spec = _ilu.spec_from_file_location( + name, os.path.join(os.path.dirname(os.path.abspath(__file__)), filename)) + module = _ilu.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +engine = _load_local("h3_engine", "dumas_h3_longvideos_engine.py") +_plan_module = _load_local("h3_shot_plan", "dumas_h3_longvideos_shot_plan.py") +_runtime_module = _load_local("h3_runtime", "dumas_h3_longvideos_runtime.py") +_audio_module = _load_local("h3_audio", "dumas_h3_longvideos_audio.py") +_cond_module = _load_local("h3_conditioning", "dumas_h3_longvideos_conditioning.py") +ShotPlan = _plan_module.ShotPlan +PreparedVideo = _plan_module.PreparedVideo + +# Internal helper exports retained for existing callers. +ShotAudio = _audio_module.ShotAudio +FrameAccumulator = _runtime_module.FrameAccumulator +apply_levels = _runtime_module.apply_levels +H3_FPS = _runtime_module.H3_FPS +AUDIO_LATENT_FPS = _runtime_module.AUDIO_LATENT_FPS +KEYFRAME_SAFE_AUG = _cond_module.KEYFRAME_SAFE_AUG +MAX_FRAMES = _runtime_module.MAX_FRAMES +_SILENT_UNIT = _audio_module._SILENT_UNIT +align_frame_count = _runtime_module.align_frame_count +video_latent_t = _runtime_module.video_latent_t +temporal_shape = _runtime_module.temporal_shape +_decode_video = _runtime_module._decode_video +_decode_audio = _runtime_module._decode_audio +_seamless_loop = _audio_module._seamless_loop +mix_ambient = _audio_module.mix_ambient +_is_oom = _runtime_module._is_oom +_deep_cleanup = _runtime_module._deep_cleanup +_decode_headroom = _runtime_module._decode_headroom +_resident = _runtime_module._resident +_image_out_dtype = _runtime_module._image_out_dtype +_evict_all_but = _runtime_module._evict_all_but +_SILENCE_STATUS = _audio_module._SILENCE_STATUS +_silent_audio_latent = _audio_module._silent_audio_latent +_pin_audio_silence = _audio_module._pin_audio_silence +HandoffLevels = _cond_module.HandoffLevels +_keyframe_latent = _cond_module._keyframe_latent +_sample_on_sigmas = _runtime_module._sample_on_sigmas +RESIZE_CHUNK = _runtime_module.RESIZE_CHUNK +_stream_chunks = _runtime_module._stream_chunks +_resize_short_edge = _runtime_module._resize_short_edge +_upscale_frames = _runtime_module._upscale_frames +_find_node = _runtime_module._find_node +_invoke_node = _runtime_module._invoke_node +build_conditioning = _cond_module.build_conditioning + RES_MULTIPLE = 32 -KEYFRAME_SAFE_AUG = 0.99 # below this, a ref aug would soften the keyframe too -AUTO_TILE_T = 8 # temporal chunk for a tiled decode -MAX_FRAMES = 362 # H3's own ceiling (~15s) # Latent frames decoded from the PRE-upscale latent to source the handoff. Enough # for the VAE's temporal context to produce a clean last frame, and cheap. HANDOFF_LATENT_TAIL = 8 @@ -80,18 +101,9 @@ NATIVE_RES = { } -CANVAS_MULTIPLE = 32 - - -REF_IMAGE_SHORT_EDGE = 2048 - - _LAST_MODEL_FP = {"fp": None} -_SILENT_UNIT = {"lat": None} - - def _call_node(cls, model, shift_video, shift_audio): """Call the H3 sampling node whether it uses the V1 (INPUT_TYPES/FUNCTION) or V3 (define_schema/execute) API, mapping the shift args by name.""" @@ -143,16 +155,6 @@ def _is_audio_vae(v): return None -# --- sizing ----------------------------------------------------------------- - -def align_frame_count(n): - """Up to the next valid H3 frame count. The grid is 17k+5.""" - n = max(5, int(n)) - while n % 17 != 5: - n += 1 - return min(n, MAX_FRAMES) - - def align_frame_count_nearest(n): """The NEAREST 17k+5 grid point, not the next one up. @@ -166,19 +168,6 @@ def align_frame_count_nearest(n): return min(MAX_FRAMES, lo if (n - lo) <= (hi - n) else hi) -def video_latent_t(fc): - return 2 if fc <= 5 else ((fc - 5) // 17) * 5 + 2 - - -def temporal_shape(length, fps=H3_FPS): - """(frame count, video latent frames, audio latent frames) for a shot. - - `fps` is accepted but deliberately IGNORED: the audio latent has to line up - with 24 fps video or the shot's sound is stretched against its picture.""" - fc = align_frame_count(length) - return fc, video_latent_t(fc), round(fc / H3_FPS * AUDIO_LATENT_FPS) - - def parse_resolution(choice): text = (choice or "").strip() if text in NATIVE_RES: @@ -216,9 +205,28 @@ def split_beats(prompt): paras = paragraphs(prompt) if not paras: return "", [] + # A CHARACTER SHEET WRITTEN FIRST IS NOT THE SCENE. Opening a script with who is + # in it is the natural order, and taking that paragraph as the scene stamped the + # WHOLE sheet onto every shot as prose: every person described in every shot, + # "Owen feeds the ducks." sent with Maya's full description beside it, and none + # of the per-shot scoping, the count or the mouth guard run, because the node + # believed there was no sheet. Leading sheet paragraphs stay in the beat list, + # where pull_character_sheets takes them, and the scene is the first paragraph + # that is not one. + # + # Stricter than is_character_sheet on purpose: every line has to declare a + # pronoun or an age, which a person's entry does and "Interior: a kitchen at + # night." does not -- that one is a scene heading and stays the scene. + lead = [] + while paras and is_character_sheet(paras[0]) and all( + sheet_pronoun(ln) or age_in(ln) + for ln in paras[0].splitlines() if ln.strip()): + lead.append(paras.pop(0)) + if not paras: + return "", lead if len(paras) == 1: - return "", paras - return paras[0], paras[1:] + return "", lead + paras + return paras[0], lead + paras[1:] def paragraphs(text): @@ -228,8 +236,15 @@ def paragraphs(text): # A line of a character sheet: `Name: attributes`. The directive lines are excluded # by name -- they are instructions to this node, not people. -_SHEET_LINE = re.compile(r"^\s*(?!(?:remove|off|add|wear|wardrobe)\s*:)" - r"[A-Z][\w'’-]{0,24}\s*:\s*\S", re.I) +# +# UP TO THREE WORDS, AS sheet_lines READS THEM. This allowed one, so "Mistress Vale: +# she, 45, black dress." was not a sheet line: the whole sheet paragraph rendered as a +# shot of its own and every later shot described nobody -- the people the model +# invented there were not the ones in the first shot. The first word keeps its old +# case-insensitivity; each extra word has to be capitalised, which keeps +# "Both women: tired" out, as sheet_lines does. +_SHEET_LINE = re.compile(r"^\s*(?!(?i:remove|off|add|wear|wardrobe)\s*:)" + r"[A-Za-z][\w'’-]{0,24}(?:\s+[A-Z][\w'’-]{0,24}){0,2}\s*:\s*\S") def is_character_sheet(par): @@ -368,6 +383,34 @@ _SPOKEN_SPAN = engine._SPOKEN_SPAN _outside_speech = engine._outside_speech +# AN OBJECT PRONOUN AFTER A PREPOSITION OF PROXIMITY IS SOMEBODY ELSE. +# +# "Tess kneels beside her" cannot mean Tess kneels beside herself, but with two women +# on the sheet the pronoun resolver credited "her" to Tess -- she declares "she", so it +# read as already accounted for -- and McKenna, who is in the shot and is what "her" +# refers to, lost her sheet line. A person in frame with no description is a person the +# model dresses out of nothing: reported as hair changing between shots, restraints not +# matching, and a body that is not the same size twice. +# +# DELIBERATELY NARROW, because the rule it refines is right in the ordinary case and +# there is a recorded regression on either side of it. +# * `behind` is NOT in this list. "Jon walks out and shuts the door behind him" is +# behind HIMSELF, and reading it as another person is the exact bug the resolver's +# own comment records fixing. +# * A pronoun followed by a noun is possessive, not an object: "look at her hands" is +# Nora's hands, and the two-name case is excluded anyway. +# The pronoun must end its phrase -- punctuation, a conjunction, or the end of the beat. +_PRONOUN_AT = re.compile( + r"\b(?:beside|alongside|next\s+to|opposite|toward|towards|at|to|over|onto|into|" + r"against|with|near|by)\s+(her|him|them)\b" + r"(?=\s*[.,;:!?]|\s+(?:and|but|then|while|as|so|who|before|after)\b|\s*$)", re.I) + + +def pronoun_points_away(beat): + """Does this beat aim a pronoun at somebody OTHER than the person it names?""" + return bool(_PRONOUN_AT.search(str(beat or ""))) + + def sheet_for_beat(sheet, beat, previous=None): """(the sheet lines for the people this beat involves, the names kept). @@ -438,7 +481,9 @@ def sheet_for_beat(sheet, beat, previous=None): named = ([n for n in everyone if n in named] if len(named) >= 2 else everyone) return "\n".join(ln for n, ln in rows if n in named), named - used = {m.group(0).lower() for m in _PRONOUN.finditer(beat or "")} + # Pronouns in what the beat STAGES only -- not in speech, not in a question the + # narration asks. See engine.staged_text. + used = {m.group(0).lower() for m in _PRONOUN.finditer(engine.staged_text(beat or ""))} if used: # Resolve a pronoun to the person whose sheet DECLARES it. Adding the whole # previous cast on any pronoun put someone in a shot they were not in -- @@ -454,6 +499,29 @@ def sheet_for_beat(sheet, beat, previous=None): continue # Already accounted for by somebody the beat names outright: "Nora and Dan # look at her hands" needs nobody else for "her". + # + # UNLESS THE PRONOUN POINTS AWAY FROM THEM. "Tess kneels beside her" names + # one person and aims the pronoun at another, and crediting it to Tess left + # McKenna in the shot with no description at all. Only with exactly one name + # in the beat -- with two there is somebody for the pronoun to belong to -- + # and only where exactly one other person on the sheet declares that + # pronoun, so nothing is guessed between two candidates. See _PRONOUN_AT. + # ...AND ONLY AT SOMEBODY WHO IS ACTUALLY IN THE SCENE. Read from the + # previous shot's cast, not from the sheet: off the sheet, "Tess looks at + # her" dragged whichever other woman was WRITTEN DOWN into the shot, even + # one who had left two beats earlier or never appeared at all. That is a + # random in the scene, which is the thing character_guard exists to + # prevent, reintroduced by the fix for the opposite problem. + _present = {n for n in (previous or []) if n} + _away = (len(named) == 1 and pronoun_points_away(beat)) + if _away and _present: + _others = [n for n, ln in rows + if n and n not in named and n in _present + and sheet_pronoun(ln) == group] + if len(_others) == 1: + named.append(_others[0]) + matched = True + continue if any(sheet_pronoun(ln) == group for n, ln in rows if n and n in named): matched = True continue @@ -538,7 +606,7 @@ def unresolved_pronouns(sheet, beat, previous=None): rows = sheet_lines(sheet) named = [n for n, _ in rows if n and re.search(r"\b" + re.escape(n) + r"\b", beat or "")] - used = {m.group(0).lower() for m in _PRONOUN.finditer(beat or "")} + used = {m.group(0).lower() for m in _PRONOUN.finditer(engine.staged_text(beat or ""))} out = [] for group, words in _PRONOUN_SET.items(): if not used & words: @@ -552,6 +620,147 @@ def unresolved_pronouns(sheet, beat, previous=None): return out +# LEAVING THE FRAME -- the transition out of the picture, which nothing had. +# +# Who a keyframe shows was read off the TEXT: the people the previous shot described. +# But a shot that stops describing somebody does not take them out of the picture it +# starts from. "Dan and Crystal sit at the table", then "Crystal laughs" -- Dan is +# still sitting there, undescribed. The next beat about Dan then read as Dan "back +# after a shot away", and the node sent a recovered frame of him as a reference while +# the keyframe still had him in it: two pictures of Dan, and a second Dan drawn. +# +# Not _MOVES_OFF. That one ends a LOOK, and walking to the counter ends a look without +# taking anybody out of the picture. A false exit here is a second picture of somebody +# still standing in the frame, so only words that take a person OUT count -- leaving, +# exiting, walking out/off/away, disappearing. "Steps out of the shower" and "runs out +# of patience" go nowhere. +_EXIT_ROOMS = "|".join(p for p in engine.PLACES.split("|") + if p not in {"shower", "showers", "pool", "sauna", "van", "truck", + "elevator", "cell", "steps", "stairs", "court"}) +_EXIT_OUT_OF = (r"(?:(?:the|this|that|his|her|their|our)\s+)?(?:frame|shot|view|sight)" + r"|(?:the|this|that|his|her|their|our)\s+(?:[\w-]+\s+)?(?:" + _EXIT_ROOMS + + r"|house|home|building|apartment|flat|door|front\s+door|gate)") +_EXIT = re.compile( + r"\b(?:leaves?|left|leaving)(?=\s*(?:[.,;:!?]|$)" + r"|\s+(?:again|together|without|through|by|via|for|with|and|then|now|quietly|alone)\b" + r"|\s+(?:the|this|that|his|her|their|our)\s+(?:[\w-]+\s+)?(?:" + _EXIT_ROOMS + + r"|house|home|building|apartment|flat)\b)" + r"|\bexit(?:s|ed|ing)?\b" + r"|\b(?:walk(?:s|ed|ing)?|go(?:es|ing)?|went|head(?:s|ed|ing)?|step(?:s|ped|ping)?|" + r"run(?:s|ning)?|ran|storm(?:s|ed|ing)?|hurr(?:y|ies|ied|ying)|slip(?:s|ped|ping)?|" + r"wander(?:s|ed|ing)?|strid(?:e|es|ing)|strode|march(?:es|ed|ing)?|" + r"rush(?:es|ed|ing)?|back(?:s|ed|ing)?|driv(?:e|es|ing)|drove|sneak(?:s|ed|ing)?|" + r"snuck|dash(?:es|ed|ing)?|bolt(?:s|ed|ing)?|stomp(?:s|ed|ing)?|limp(?:s|ed|ing)?)" + r"\s+(?:\w+ly\s+)?" + r"(?:out\b(?!\s+of\s+(?!" + _EXIT_OUT_OF + r"))|off\b(?!\s+(?:the|a|an|his|her|their)\b)" + r"|away\b(?!\s+from\b)|outside\b|home\b)" + r"|\b(?:disappear|vanish)(?:s|es|ed|ing)?\b" + r"|\bout\s+of\s+(?:(?:the|this|that|his|her|their)\s+)?(?:frame|shot|view|sight)\b", + re.I) +# Where a new predicate can take its own subject. "Crystal hands Dan the keys and +# leaves" is Crystal leaving -- Dan is an object -- and "...and he leaves" is Dan. +_CLAUSE_OPEN = re.compile(r"(?:^|[,;:]|\b(?:and|then|but|while|as|when|before|after|so)\b)\s*$", + re.I) + + +def _movers(rx, beat, sheet, pool, alone_is_it=False): + """The people a movement in this beat belongs to -- the subject of each match of rx. + + A name or a subject pronoun opening the clause, reached back across "and" to the + predicate it continues: "Crystal hands Dan the keys and leaves" is Crystal, and + "...and he leaves" is Dan. A pronoun resolves only to one person in `pool` who + declares it. A movement pinned on nobody is the sole person in the pool's when + `alone_is_it`, and otherwise nobody's.""" + text = engine.staged_text(beat or "") + rows = [(n, ln) for n, ln in sheet_lines(sheet) if n] + names = [n for n, _ in rows] + pool = [n for n in (pool or []) if n] + out = [] + for sentence in re.split(r"(?<=[.!?])\s+", text): + for m in rx.finditer(sentence): + before = sentence[:m.start()] + subj = [] + spots = [] + for n in names: + spots += [(k.start(), k.end(), [n]) + for k in re.finditer(r"\b" + re.escape(n) + r"\b", before)] + for k in re.finditer(r"\b(she|he|they)\b", before, re.I): + word = k.group(1).lower() + if word == "they" and not any(sheet_pronoun(ln) == "they" for _, ln in rows): + who = list(pool) + else: + who = [n for n, ln in rows if n in pool and sheet_pronoun(ln) == word] + spots.append((k.start(), k.end(), who if len(who) == 1 or word == "they" + else [])) + spots.sort() + # The last spot that opens a clause is the subject; one joined to it by + # "and" or a comma is the same subject -- "Dan and Crystal leave". + # ...unless somebody stands right against the verb: "Crystal watches Dan + # walk away" is Dan walking. + for idx in range(len(spots) - 1, -1, -1): + s, e, who = spots[idx] + if (not _CLAUSE_OPEN.search(before[:s]) + and not (idx == len(spots) - 1 + and re.fullmatch(r"\s+(?:\w+ly\s+)?", before[e:]))): + continue + subj = list(who) + j = idx + while (j > 0 and re.fullmatch(r"\s*(?:,|and|,\s*and)\s*", + before[spots[j - 1][1]:spots[j][0]], re.I)): + j -= 1 + subj = list(spots[j][2]) + subj + if j != idx and not _CLAUSE_OPEN.search(before[:spots[j][0]]): + subj = list(who) + break + else: + subj = list(pool) if (alone_is_it and len(pool) == 1) else [] + out += [n for n in subj if n not in out] + return out + + +def leaves_in(beat, sheet, present=()): + """The people this beat takes OUT of the frame -- see _EXIT. + + A leaving the beat does not pin on anybody is the one person in the frame's, or + nobody's: keeping somebody in the picture costs a reference, and taking out + somebody who is still there costs a second copy of them.""" + return _movers(_EXIT, beat, sheet, present, alone_is_it=True) + + +# COMING IN -- narrower than _ENTRANCE on purpose. That one decides whether a newcomer +# can walk into the keyframe, and "walks over", "follows" and "joins" are fine there. +# This one asks whether somebody ALREADY IN the frame is being staged arriving, and +# "Dan walks over to the sink" is not that. +_COMES_IN = re.compile( + r"\b(?:walk(?:s|ed|ing)?|com(?:e|es|ing)|came|step(?:s|ped|ping)?|run(?:s|ning)?|ran|" + r"hurr(?:y|ies|ied|ying)|burst(?:s|ing)?|barg(?:e|es|ed|ing)|slip(?:s|ped|ping)?|" + r"strid(?:e|es|ing)|strode|stroll(?:s|ed|ing)?|wander(?:s|ed|ing)?|rush(?:es|ed|ing)?|" + r"storm(?:s|ed|ing)?|march(?:es|ed|ing)?|sneak(?:s|ed|ing)?|snuck|limp(?:s|ed|ing)?|" + r"stagger(?:s|ed|ing)?)\s+(?:\w+ly\s+)?(?:back\s+)?" + r"(?:in\b(?!\s+(?:the|a|an|his|her|their)\b)|inside\b" + r"|into\s+(?:the|this|that|a)\s+(?:[\w-]+\s+)?(?:" + _EXIT_ROOMS + + r"|house|building|apartment|flat)\b)" + r"|\benter(?:s|ed|ing)?\b(?!\s+(?:the|a|his|her)\s+(?:code|number|password|data|pin)\b)" + r"|\barriv(?:e|es|ed|ing)\b" + r"|\b(?:com(?:e|es|ing)|came)\s+back\b(?=\s*(?:[.,;:!?]|$)" + r"|\s+(?:in|into|inside|home|with|and|carrying|holding)\b)" + r"|\breturn(?:s|ed|ing)?\b(?!\s+(?:the|a|an|his|her|their|it|them|to\s+(?:the|his|her|their)\s+" + r"(?:table|desk|couch|sofa|chair|bed|seat|work|book|screen|sink|stove|counter)))", + re.I) + + +def comes_in(beat, sheet): + """The people this beat stages ARRIVING in the frame -- see _COMES_IN.""" + return _movers(_COMES_IN, beat, sheet, [n for n, _ in sheet_lines(sheet) if n]) + + +_SHE_NOUNS = {"woman", "girl", "lady", "female", "mother", "wife", "sister", "daughter", + "aunt", "grandmother", "niece"} +_HE_NOUNS = {"man", "boy", "guy", "gentleman", "male", "father", "husband", "brother", + "son", "uncle", "grandfather", "nephew"} +_PERSON_NOUN = re.compile( + r"^(?:(?:a|an|the)\s+)?(?:[\w-]+\s+){0,2}(" + "|".join(sorted(_SHE_NOUNS | _HE_NOUNS)) + + r")\b(?!['\u2019])", re.I) _PRONOUN_SET = {"she": {"she", "her", "hers"}, "he": {"he", "him", "his"}, "they": {"they", "them", "their", "theirs"}} @@ -564,19 +773,127 @@ def sheet_pronoun(line): "her coat" in a beat be resolved to Maya rather than to whoever was in the last shot.""" body = (line or "").split(":", 1)[-1] - for group, words in _PRONOUN_SET.items(): - if any(re.search(r"\b" + w + r"\b", body, re.I) for w in words): - return group + # THE DECLARED ONE, NOT THE FIRST GROUP WITH A WORD ANYWHERE. The groups were + # checked she-then-he-then-they, so "Owen: he, 42, blue shirt, carries her photo + # in his wallet" was a "she" -- "her" is in his description -- and every "he" in + # the script stopped reaching him while every "she" could. The pronoun standing + # alone as an item ("he", "she", "they") is the declaration; failing that, the + # earliest pronoun in the entry. + group_of = {w: g for g, words in _PRONOUN_SET.items() for w in words} + for item in body.split(","): + word = item.strip().strip(".;").lower() + if word in _PRONOUN_SET: + return word + hits = [(m.start(), group_of[m.group(0).lower()]) + for m in re.finditer(r"\b(?:" + "|".join(group_of) + r")\b", body, re.I)] + if hits: + return min(hits)[1] + # ...AND A PERSON NOUN, WHERE NO PRONOUN IS WRITTEN. "Maya: 38, a woman with red + # hair" says who she is in the author's own word, and ignoring it left every "she" + # in the script with nobody to reach. Only as the head of an item describing the + # person -- "a tall man", "a young woman" -- never a possessive: "her brother's + # jacket" is not a brother. + for item in body.split(","): + m = _PERSON_NOUN.match(item.strip()) + if m: + return "she" if m.group(1).lower() in _SHE_NOUNS else "he" return None +ADULT_AGE = 18 # below this the node describes no body at all + +# HOW OLD THE SHEET SAYS SOMEBODY IS. Inert text until now: the age went to the model +# inside the author's own words and nothing here read it, so every clause this file +# writes about a body said only "a woman's body" -- and an attribute a prompt does not +# state is not left to the model, it is left to the model's PRIOR. The prior for an +# adult woman is a woman in her twenties whatever the sheet says, which is how a +# character written as 45 renders as 22. +# +# DELIBERATELY NARROW. A number is an age only where it stands as its own attribute in +# the list, or where an age word is attached to it. A sheet says "size 10 boots" and +# "5'7" and "a 9mm" and a tag, and reading any of those as an age would +# describe a body nobody asked for -- worse than describing none. +_AGE_WORD = r"(?:y\.?o\.?|yrs?|years?(?:\s+old)?|year-old)" +_AGE_AT = re.compile( + # "aged 24", "age 24", "24yo", "24 years old", "24-year-old" + r"\bage[d]?\s+(\d{1,3})\b" + r"|\b(\d{1,3})\s*-?\s*" + _AGE_WORD + r"\b" + # ...or a bare number alone between the commas of the attribute list. The list can + # END on it -- "Kate: she, 28." is how most entries are written -- so a full stop or + # a semicolon closes the attribute as well as a comma does. Requiring a comma read + # that entry as having no age at all, which is silent twice over: no body named, and + # the refusal below never fired either. + r"|(?:^|,)\s*(\d{1,3})\s*(?=[,;.]|$)", re.I) +# "in her forties", "early thirties", "mid-50s", "late 20s". The decade's MIDDLE, +# except where the qualifier says otherwise -- and it is read as an age only for the +# decades an adult has, because a bare "teens" names no single year and is not a +# licence to guess one. +_DECADE = {"twenties": 20, "thirties": 30, "forties": 40, "fifties": 50, + "sixties": 60, "seventies": 70, "eighties": 80} +_DECADE_AT = re.compile( + r"\b(early|mid|middle|late)?\s*-?\s*" + r"(?:(twenties|thirties|forties|fifties|sixties|seventies|eighties)" + r"|(\d0)\s*s)\b", re.I) + + +def age_in(line): + """The age this sheet entry declares, or 0 when it declares none. + + Read off the attribute list, never off a beat: a beat saying "twenty years later" + is not somebody's age, and the sheet is where the author states what is true of a + person for the whole film.""" + body = str(line or "").split(":", 1)[-1] + # The tag carries digits of its own, and they are a slot number. + body = re.sub(r"<\s*picture[\s_]*\d+\s*>", " ", body, flags=re.I) + m = _AGE_AT.search(body) + if m: + got = int(next(g for g in m.groups() if g)) + return got if 1 <= got <= 120 else 0 + m = _DECADE_AT.search(body) + if m: + base = _DECADE.get((m.group(2) or "").lower()) + if base is None and m.group(3): + base = int(m.group(3)) + if base in _DECADE.values(): + q = (m.group(1) or "").lower() + return base + (2 if q == "early" else 8 if q == "late" else 5) + return 0 + + _PRONOUN = re.compile(r"\b(?:she|he|her|hers|his|him|they|them|their|theirs)\b", re.I) # A determiner in front means the capitalised word DESCRIBES something rather than # doing something: "her Nike leggings" names a garment, not somebody in the room. _DETERMINER = frozenset("a an the her his its their our my your this that".split()) -_CAPITALISED = re.compile(r"\b([A-Z][a-z’'-]{1,24})\b") +_CAPITALISED = re.compile(r"\b([A-Z][a-z\u2019'-]{1,24})\b") +# A WORD THAT IS NEVER SOMEBODY'S NAME, however it is capitalised. +# +# The mid-sentence test was supposed to make this list unnecessary -- an ordinary word +# only opens a sentence, a name appears inside one -- and it is defeated by the +# commonest punctuation in a script: +# +# "Nearly there," The guard says. +# +# "The" follows a comma, so it IS mid-sentence, so it was reported as a character with +# no sheet entry: "shot(s) 4, 7, 8, 9 name The, who has no entry in the character +# sheet". Every pronoun reaches the same way out of a speech tag -- '"Wait," She says' +# -- and the warning then sends the author looking for a person who does not exist +# while saying nothing about the one who does. +# +# Only words that are never a name go in here. Grace, Will, Hope, Faith and May are +# names and are deliberately absent: this file has already been bitten by matching +# "will" and "grace" case-insensitively. +_NEVER_A_NAME = _DETERMINER | frozenset(""" +i we you he she it they me him us them myself yourself himself herself itself +themselves mine yours hers ours theirs +and but or nor so yet then than as at in on of off to into onto from with without +if when while because though although after before until once since +there here what which who whom whose why how where whether +no not now never always again also just only even still both each either neither +one two three four five six seven eight nine ten first second next last another +yes ok okay oh ah well right left up down out over under across back forward +""".split()) def unknown_people(beats, sheet): @@ -606,6 +923,15 @@ def unknown_people(beats, sheet): for m in _CAPITALISED.finditer(beat or ""): # "Jon's kitchen" is Jon. The apostrophe is in the class for O'Neill. word = re.sub(r"['’]s$", "", m.group(1)) + # Never a name, however the punctuation capitalised it. + # + # NAMED _NEVER_A_NAME, not _NOT_A_NAME: that one already exists further + # down as a regex STRING, and shadowing it turned this membership test into + # a silent substring match against a regex -- "one" passed because it + # appears inside the pattern and "the" failed because the pattern spells it + # "The". The test caught it; `in` on a string never raises. + if word.lower() in _NEVER_A_NAME: + continue before = (beat[:m.start()]).rstrip() prev = re.search(r"([\w’'-]+)\W*$", before) if prev and prev.group(1).lower() in _DETERMINER: @@ -698,7 +1024,141 @@ def revealed_by(covers, gone): _REGION_OF = engine._REGION_RX -def bare_clause(gone, covers=None, worn=""): +def body_of(pronoun, age=0): + """The body the sheet's declared pronoun and age mean. "" where nothing is declared. + + AN UNSPECIFIED BODY IS FILLED FROM THE PRIOR, which is the lesson this file + already recorded for the chest -- "this said 'The arms and shoulders are bare' and + stopped there, so the one region a bra occupies was unspecified, and an unspecified + region is filled by the model's own prior". The hip-down clause had the same gap + and it was never closed: "The legs are bare from the hip down" names the region and + says nothing about whose body it is, so the anatomy at the hip came from the prior + too. Reported as the wrong anatomy on a female character. + + Read from the pronoun the author DECLARED, which the README already requires for + every entry, so this asserts nothing the sheet does not already say. `they` returns + nothing: an undeclared body is not a licence to guess one. + + ...AND THE AGE THEY DECLARED, for the same reason one step further on. "A woman's + body" is true of a woman of 22 and a woman of 62, so it settles nothing between + them, and what fills the gap is the prior -- which is a woman in her twenties + whatever the sheet says. Reported as a character written at one age rendering at + another. The age is the author's own word, already in the sheet and already going + to the model inside it; this only stops it being the one attribute nothing here + reads. + + NO BODY IS DESCRIBED FOR A DECLARED AGE UNDER 18. Not a softer description -- none, + and this returns "" so every clause built on it stays silent. An age the author + states is the one fact here that is not a guess, and a generator has no business + composing anatomy for a child. See also the refusal in _prepare: a script that + declares a minor and stages nudity or sex does not render at all.""" + who = {"she": "woman", "he": "man"}.get(str(pronoun or "").strip().lower(), "") + if not who: + return "" + age = int(age or 0) + if age and age < ADULT_AGE: + return "" + return f"a {who}'s body" if not age else f"the body of a {who} of {age}" + + +# HOW AN ADULT CHEST DIFFERS WITH AGE. Plain physical description -- fullness, where it +# sits, how firm, what the skin does -- because those are the facts that separate one +# adult decade from another, and the prior collapses all of them onto the twenties. +# +# Asked for directly: "Breast development should also be correct, given the age of a +# person." The clause is scoped hard. It is said only where the chest is ALREADY being +# described as bare, so it adds nothing to a clothed shot; only for a declared age of +# 18 or over, with no entry below that; and only for a sheet that declares "she", +# because the request was about breasts and a pronoun this file was not given is not a +# licence to guess an anatomy. +_FIGURE = ( + (18, 24, "grown and firm, sitting high on the chest"), + (25, 34, "fully grown and full, sitting a little lower than in her early twenties"), + (35, 44, "full and softer, settled lower with the weight of middle age"), + # No "skin" in these two: the sentence they join already ends on "the skin itself + # the outermost surface there", and the word arriving twice in one clause reads as + # two different things being described. + (45, 54, "mature and heavier, softened and lower again, with less tension in them"), + (55, 120, "older and slacker, hanging low and soft, loose and lined"), +) + + +# A SHEET THAT DECLARES A CHILD AND A SCRIPT THAT STAGES SEX DO NOT RENDER TOGETHER. +# +# This file reads an age now, and the age drives anatomy -- see body_of and figure_of, +# which describe no body at all below ADULT_AGE. That floor is necessary and it is not +# sufficient: withholding the node's own clauses does nothing about a script whose own +# words stage nudity or sex, and those words reach the model verbatim. So the two +# together are refused outright, before anything is sampled. +# +# Read off the SHEET for the age, because that is where an author states a person's +# age, and off the whole script for the staging. Deliberately blunt: no attempt to work +# out who the nudity is about. A film that declares a minor anywhere and stages this +# anywhere is refused whole, and a legitimate scene with a child in it -- which this +# node will render, with no body described for them -- does not contain either. +_SEXUAL_STAGING = re.compile( + r"\b(?:sex|sexual|fucks?|fucking|fucked|intercourse|penetrat\w*|blow\s?job|" + r"handjob|masturbat\w*|orgasms?|orgasmic|climax(?:es|ed|ing)?|cums?|cumming|" + r"aroused|arousal|horny|erotic\w*|nipples?|genitals?|vagina\w*|penis\w*|" + r"cocks?|dicks?|pussy|clit\w*|erections?|foreplay|straddl\w*|" + r"topless|bottomless|naked|nude|nudity|undress\w*|strips?\s+(?:off|naked|bare)|" + r"moans?|moaning|moaned)\b", re.I) + + +def minor_with_sexual_staging(sheet, script): + """A refusal message when a sheet declares a minor and the script stages sex. "" otherwise. + + Both halves required. An age under 18 on its own renders -- children exist in + films -- and gets no body described for them by anything here. Sexual staging on + its own renders, which is what this node is for.""" + named = [(n, age_in(ln)) for n, ln in sheet_lines(sheet or "") if n] + minors = sorted({n for n, a in named if 0 < a < ADULT_AGE}) + if not minors: + return "" + m = _SEXUAL_STAGING.search(str(script or "")) + if not m: + return "" + return (f"REFUSED, and nothing was rendered. The character sheet declares " + f"{_join_names(minors)} as under {ADULT_AGE}, and the script stages sexual " + f"or nude content -- it contains {m.group(0)!r}. This node will not " + f"generate that combination, whichever character the wording is about and " + f"whatever was intended by it. Nothing here tried to work out who: a film " + f"holding both is refused whole.\n\n" + f"If an age is a typo, fix the sheet and run again -- an adult age renders " + f"normally. If the character is an adult, state an adult age. A scene with " + f"a child in it and no sexual or nude content renders as it always did, " + f"and no body is described for them by this node.") + + +def _pron_age(sheet, name): + """(pronoun, age) off one person's own sheet entry. ("" , 0) when it has neither.""" + line = dict(sheet_lines(sheet)).get(name, "") + return sheet_pronoun(line), age_in(line) + + +def figure_of(pronoun, age=0): + """Age-consistent adult chest description, or "". See _FIGURE and body_of. + + Returns nothing at all without BOTH a declared "she" and a declared adult age: + with no age there is nothing to be consistent with, and the old silence is better + than a guess.""" + if str(pronoun or "").strip().lower() != "she": + return "" + age = int(age or 0) + if age < ADULT_AGE: + return "" + for lo, hi, said in _FIGURE: + if lo <= age <= hi: + # THE AGE IS NOT REPEATED HERE. body_of already states it in the same + # sentence, and "the breasts those of a woman of 45 ... on the body of a + # woman of 45" says one fact twice -- which is the vice this file spends + # most of its comments on. figure_of is only ever reached through a + # declared "she", so the body phrase is always there to carry it. + return f"the breasts {said}" + return "" + + +def bare_clause(gone, covers=None, worn="", body="", figure=""): """Say the uncovered region is BARE, when the sheet names nothing under it. A removal clause is emphatic -- off the body, dropped out of frame -- and then @@ -721,10 +1181,10 @@ def bare_clause(gone, covers=None, worn=""): r = engine.region_of(item) if r and r not in regions: regions.append(r) - return bare_hold(regions, covers, worn, gone) + return bare_hold(regions, covers, worn, gone, body=body, figure=figure) -def bare_hold(regions, covers=None, worn="", gone=(), whose=""): +def bare_hold(regions, covers=None, worn="", gone=(), whose="", body="", figure=""): """Say those regions are bare -- from STATE, so it outlives its beat. The same suppression as the removal beat, because it is the same sentence: @@ -739,15 +1199,33 @@ def bare_hold(regions, covers=None, worn="", gone=(), whose=""): inventing one, and the keyframe then carried the invention forward.""" if not regions: return "" - under = {str(u).lower() for u in (covers or {})} + spoke = [] # the regions this clause actually speaks about + # ...AND ONLY WHILE IT IS STILL ON. `covers` is read off the SHEET, and the + # sheet is never edited, so a thong listed under a skirt went on suppressing + # this clause long after the thong had come off as well -- and a full strip is + # the one case this clause matters most in. The hips then had no sentence at + # all, an unspecified region is filled by the model's own prior, the prior for + # a hip is underwear, and the keyframe carried what it invented into every + # later shot. Reported as a thong restored a beat after she undressed. + # + # It cost the bra half too, which is the report this function was written for: + # a sheet that layered the bra under a shirt suppressed the chest clause by + # this same line, so the fix only ever worked for a sheet that did not. + # + # reveal_clause already filters itself the same way -- it is silent for an + # under-layer coming off in the same breath -- so the two still never both + # speak, which is the only thing this suppression was for. + under = {str(u).lower() for u in (covers or {}) if not names_any(u, gone)} said, out = set(), [] for _region in regions: for rx, region, sentence in _REGION_OF: if region != _region or region in said: continue # Something else still on the body covers this region: not bare. - if any(rx.search(w) for w in (worn or "").split(",") - if not names_any(w, gone)): + # Per GARMENT, not per comma entry: "coat over a grey sweater" is one entry + # with the coat gone and the sweater still covering. See entry_parts. + if any(rx.search(t) for w in (worn or "").split(",") + for _sep, t in entry_parts(w) if not names_any(t, gone)): said.add(region) break # The sheet named a layer underneath: reveal_clause has this one, and @@ -763,6 +1241,7 @@ def bare_hold(regions, covers=None, worn="", gone=(), whose=""): break said.add(region) out.append(sentence) + spoke.append(region) break if not out: return "" @@ -778,7 +1257,30 @@ def bare_hold(regions, covers=None, worn="", gone=(), whose=""): if whose: joined = f"{whose}'s " + joined[4:] if joined.startswith("The ") else \ f"{whose}: " + joined - return " " + joined + ", with nothing else worn there." + # ...AND WHOSE BODY IT IS. A bare region with no body named is anatomy left to the + # prior, and at cfg 1 nothing later takes back what the prior draws. See body_of. + # POSITIVELY PHRASED, to the last clause. This ended ", with nothing else worn + # there" -- a negation, in the one sentence whose whole purpose is to stop the + # model filling a region from its own prior, and at cfg 1 there is no negative + # prompt to carry it: "nothing else worn" offers the word worn and no picture. + # It went unnoticed because the clause could only reach a shot whose sheet put + # no layer under the garment, and the suite that checks every guard sentence for + # a negation had no such scene until the suppression was fixed. + # + # What replaces it says the same thing as a surface, which is what a model + # renders: the skin is the outermost thing on that part of the body. Same move + # under_clause made for the cover it describes. + # ...AND WHAT THAT PART OF THE BODY IS LIKE AT THE AGE THE SHEET STATES, but only + # where the chest is one of the regions this sentence actually reached. A clause + # about bare legs that describes a chest is describing a region it was not asked + # about, and `out` is capped at two, so "torso was in `regions`" is not the same + # question as "torso got said". See figure_of. + # AFTER the body, not before it: the body phrase is what the figure is a fact + # about, and "the breasts ..., on the body of a woman of 45" puts the attribute + # ahead of the thing it belongs to. + said_fig = f", {figure}" if (figure and "torso" in spoke[:2]) else "" + return " " + joined + (f", on {body}" if body else "") + said_fig + \ + ", the skin itself the outermost surface there." def defer_tag_for(text, items): @@ -896,34 +1398,7 @@ def reveal_clause(items): said = " and ".join(f"the {i}" for i in items[:2]) plural = len(items) > 1 or items[0].endswith("s") return (f" {said[0].upper()}{said[1:]} underneath {'are' if plural else 'is'} what " - f"shows there now, still on and unchanged.") - - -_TAGGED_FRAGMENT = re.compile(r"<\s*Picture\s*\d+\s*>", re.I) - - -def tagged_items(sheet): - """Head nouns of sheet entries that carry a of their own. - - An item the author has attached a reference to is one they have said, as - plainly as this node allows, that they want drawn. It is also identity - wiring: the tag is how ref_image_N reaches the shot, and a reference no text - claims is read as an extra subject -- the worst failure this node has. - - So a tagged item is never held back as merely HIDDEN. Reported as a chastity - belt with a picture reference disappearing out of the character memory: it - was inferred to be under the jeans, went into `covered` with the ordinary - under-layers, and scrub_removed dropped the fragment -- taking - with it, so the image was loaded, counted in the report as going "where - tagged", and tagged nowhere.""" - out = set() - for _n, line in sheet_lines(sheet or ""): - for frag in str(line).split(","): - if not _TAGGED_FRAGMENT.search(frag): - continue - for head in entry_heads(frag): - out.add(head) - return out + f"shows there now, on and unchanged.") def merge_sheets(*sources): @@ -940,13 +1415,35 @@ def merge_sheets(*sources): no line repeated. The earlier source wins, so character_memory overrides a sheet left in the prompt.""" seen_names, seen_lines, out, dupes = set(), set(), [], [] + # ...AND ONE PERSON UNDER TWO FORMS OF THE NAME. "Maya Brooks" in + # character_memory and "Maya:" in the prompt matched as two keys, so both entries + # went into every shot -- one woman in a green sweater and one in a red coat, + # under "There is one person in the shot". A name made of words the other name + # already has is the same person, unless the two entries say otherwise: a + # different pronoun or a different age is somebody else ("May: she, 24" and + # "Aunt May: she, 60"). + seen_rows = [] + def _same_person(name, line): + words = set(name.lower().split()) + for other, other_line in seen_rows: + theirs = set(other.lower().split()) + if not (words <= theirs or theirs <= words): + continue + p1, p2 = sheet_pronoun(line), sheet_pronoun(other_line) + a1, a2 = age_in(line), age_in(other_line) + if (p1 and p2 and p1 != p2) or (a1 and a2 and a1 != a2): + continue + return True + return False for src in sources: for name, line in sheet_lines(src): key = name.lower() if name else None - if key and key in seen_names: + if key and (key in seen_names or _same_person(name, line)): if name not in dupes: dupes.append(name) continue + if key: + seen_rows.append((name, line)) if line in seen_lines: continue if key: @@ -985,6 +1482,96 @@ def build_scene(anchor, first_para, character_memory, sheet): return "\n".join(terminate_lines(p) for p in parts if p) +# THE OPENING PARAGRAPH NAMES PEOPLE TOO, AND IT IS IN EVERY SHOT. +# +# sheet_for_beat scopes the sheet to the people a beat involves, because "describing +# EVERYONE in every shot puts everyone in every shot". The anchor and the opening +# paragraph ride into every shot beside it and were never scoped the same way, so an +# opening written the ordinary way -- "Maya and Owen wait in a train station." -- +# put both names in a shot the node had cut down to one: +# +# Maya and Owen wait in a train station. Owen checks the departure board. +# Owen: he, 42, blue shirt. There is one person in the shot: one body, one face. +# +# Two names, one description, one body: the model is told a second person stands +# there and given nobody to draw but Owen, which is how a character is rendered +# twice. So a sentence there that names someone NOT in the shot gives up its +# setting and loses the person; with no setting to give up, it goes. +# A place that ENCLOSES is preferred over a spot beside a thing: "On the couch in a +# dark living room." reads as somebody on the couch, where "In a dark living room." +# is only the room. +_SETTING_PHRASE = re.compile( + r"\b(?:in|inside|outside|at)\s+(?:a|an|the|this|that)\b", re.I) +_SPOT_PHRASE = re.compile( + r"\b(?:on|by|near|beside|under|behind)\s+(?:a|an|the|this|that)\b", re.I) +_PERSON_WORD = re.compile(r"\b(?:her|his|their|him|them|herself|himself)\b", re.I) +_LEADING_PRONOUN = re.compile(r"^\s*(?:she|he|they|her|his|their)\b", re.I) + + +def _name_forms(name): + """A name as a script writes it: whole, and a two-part name by either part.""" + forms = {name} + parts = [p for p in name.split() if len(p) >= 3 and p[:1].isupper()] + if len(parts) > 1: + forms.update(parts) + return forms + + +def _setting_of(sentence): + """Where a sentence happens, without who is there. "" when it names no place. + + From the first "in a / at the / on the ..." to the end of the sentence, so + "Maya sits at her desk in an office" gives "In an office" -- "at her desk" is + hers, not the room's, and is passed over because it is not "at a" or "at the".""" + m = _SETTING_PHRASE.search(sentence or "") or _SPOT_PHRASE.search(sentence or "") + if not m: + return "" + phrase = sentence[m.start():].strip().rstrip(".!?;, ") + if not phrase or _PERSON_WORD.search(phrase): + return "" + return phrase[0].upper() + phrase[1:] + "." + + +def static_for_shot(static, sheet, shot_sheet): + """The anchor and opening paragraph for one shot: nobody named who is not in it. + + Names are the sheet's, matched case-sensitively as sheet_for_beat matches them, + so "will" is never Will. A sentence that opens on a pronoun straight after one + that was cut goes with it -- "Maya sits at her desk. She types." in a shot + without Maya leaves no "She" behind to be drawn.""" + if not (static or "").strip() or not (sheet or "").strip(): + return static + here = {n for n, _ in sheet_lines(shot_sheet or "") if n} + absent = set() + for name, _ in sheet_lines(sheet): + if name and name not in here: + absent |= _name_forms(name) + for name in here: + absent -= _name_forms(name) + if not absent: + return static + named = re.compile(r"(? and # as special tokens, alongside a caption channel (<|caption_start|>...) and a lyrics @@ -1024,6 +1611,41 @@ _CLAUSE_SPLIT = re.compile( r"|,\s+(?=\w+(?:ing|es|s|ed)\b))") +def travel_spaces(beat): + """How many distinct spaces this beat shows on screen. 0 when it goes nowhere. + + THE WALK IS THE EXPENSIVE PART OF A TRANSIT, AND IT WAS INVISIBLE TO THE SIZING. + beat_seconds counts ACTION CLAUSES, so the grammar of the sentence set the time + and the ground covered did not: "McKenna walks down the hallway to the living + room" is one verb phrase, so it was sized for one action -- 3.0s, the floor, the + SHORTEST shot in its script -- and then told to show three rooms inside it, while + "gets up and comes out of her bedroom" got 5.2s to stand up in one room. The beats + doing the most spatial work were getting the least time to do it. + + A model handed 73 frames, a bedroom keyframe and instructions to reach a living + room cannot TRAVEL, so it blends the two into one hybrid space -- which is a + living room with a bed in it, the third route to a bug already fixed twice in the + text. _CLAUSE_SPLIT's own comment names this failure exactly, "a walk down a + hallway arriving as a cut to the far end", and fixed it only for comma lists. + + AN INTRA-ROOM WALK IS NOT THIS and must stay short: test_pace measured "Maya walks + to the window" as under two seconds of real movement and the constants were tuned + down for it. A window is not a place, so it crosses nothing here. Only a beat that + actually ARRIVES somewhere counts, which is the same test travel_anchor applies + before it will say a journey happened at all. + + The origin counts even when the beat does not name it: the shot opens in the room + it was already in, that room is on screen at frame one, and it has to be left.""" + text = _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", str(beat or ""))) + frm, via, to = travel_legs(text) + if not to: + # A place the list cannot name still has to be walked to, and getting there + # still costs screen time. Origin plus destination. See moved_to. + return 2 if moved_to(text) else 0 + named = [p for p in (frm, via, to) if p] + return len(named) + (0 if frm else 1) + + def beat_seconds(beat): """Roughly how much screen time this beat's content asks for. @@ -1034,7 +1656,14 @@ def beat_seconds(beat): text = _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", beat or "")) text = _REMOVE_LINE.sub("", _ADD_LINE.sub("", text)) clauses = [p for p in _CLAUSE_SPLIT.split(text) if p and len(p.split()) >= 2] - action = (BEAT_BASE_SEC + SECONDS_PER_ACTION * len(clauses)) if clauses else 0.0 + # A ROOM BOUNDARY CROSSED ON SCREEN COSTS WHAT A STAGED ACTION COSTS. Rooms have + # to be established to be left, and crossing into one is work the grammar of the + # sentence does not show: one verb phrase can move somebody through three rooms. + # See travel_spaces. Reused constant rather than a new one, because this IS the + # same quantity -- screen time that something has to happen in. + crossings = max(0, travel_spaces(text) - 1) + action = (BEAT_BASE_SEC + SECONDS_PER_ACTION * (len(clauses) + crossings)) \ + if (clauses or crossings) else 0.0 spoken = sum(len(q.split()) for q in _QUOTED.findall(beat or "")) \ + sum(len(q.split()) for q in _DIALOGUE_TAG.findall(beat or "")) return max(action, (spoken / WORDS_PER_SEC + 1.0) if spoken else 0.0) @@ -1116,7 +1745,7 @@ def pace_clause(need, have): if need <= 0 or (have - need) < 2.5 or have <= need * 1.25: return "" return (" What the beat stages runs at an even pace across the whole shot, " - "beginning at the first frame and still finishing on the last.") + "beginning at the first frame and finishing on the last.") def thin_beats(beats, seconds): @@ -1168,8 +1797,8 @@ _EFFORT_OBJ = (r"(?:her|his|their|the)\s+(?:backs?|hips?|thighs?|shoulders?|arms # # THE TRADE RUNS THE OTHER WAY FROM WHAT I ASSUMED. A wrong OPEN branch costs # moaning text, a free mouth and a babbling stream that drags the framing with it; a -# wrong CLOSED one costs a silent shot, and built foley now covers part of even -# that. So these must corroborate, never fire alone. +# wrong CLOSED one costs a silent shot -- and that is the whole cost again now that +# nothing is built to cover it. So these must corroborate, never fire alone. # # `clench` is gone except standing alone: a clenched jaw or fist is silent tension, # which is the opposite of a sound cue. @@ -1205,7 +1834,7 @@ def exertion_in(beat): # Sound the text asks for. H3 is joint, so the same prose conditions the audio # branch -- a scene is scored by describing it, not by a setting. _SOUND_CUE = re.compile( - r"\b(?:sounds?|noises?|echo(?:e?s|ing)?|silence|rattl(?:e|es|ing)|clank(?:s|ing)?|" + r"\b(?:sounds?|noises?|echo(?:e?s|ing)?|rattl(?:e|es|ing)|clank(?:s|ing)?|" r"clink(?:s|ing)?|creak(?:s|ing)?|scrap(?:e|es|ing)|thud(?:s|ding)?|bang(?:s|ing)?|" r"slam(?:s|ming)?|clatter(?:s|ing)?|jingl(?:e|es|ing)|squeak(?:s|ing)?|" r"footsteps?|breath(?:s|es|ing)?|pant(?:s|ing)?|gasp(?:s|ing)?|sigh(?:s|ing)?|" @@ -1223,6 +1852,10 @@ _SOUND_CUE = re.compile( # "she is quiet" and "a faint smile" are the absence of one or nothing to do with # one. Opening the branch on those is a free branch with no line in the shot, # which is where an invented voice comes from. + # `silence` sat at the head of this list as a noun and did exactly that: "she + # sits in silence" read as a sound being asked for, opened the branch, and the + # one word that most plainly asks for a pinned shot was the one that unpinned + # it. A beat that names silence names nothing to make; it gets the default. r"loud(?:ly)?|quietly|faintly|audible|noisy|deafening|" r"scuff(?:s|ing|ed)?|crunch(?:es|ing|ed)?|thump(?:s|ing|ed)?|" r"patter(?:s|ing)?|whirr?(?:s|ing)?|whine(?:s|d)?|whining|rumbl(?:e|es|ing)|" @@ -1420,6 +2053,25 @@ MAX_SOUNDS = 3 # a shot's audio needs a cue, not an inventory # and "thrashes" both fired and a shot came back listing "unsteady breathing, with # gasps and moans of effort AND breathing". _VOCAL_RETIRES = ("unsteady breathing, with gasps and moans of effort", "breathing") +# WHAT IS HAPPENING BETWEEN THE MOANS. +# +# A named vocal opens the audio branch on purpose -- it is meant to be heard -- and +# then sounds_for said nothing at all, because the vocal was the whole list and the +# beat already carries it. Reported as babble between the moans, and that is exactly +# where it came from: a moan is INTERMITTENT, the branch is open for the whole shot, +# and nothing described the gaps. An open branch on a joint model fills itself, and +# what it fills itself with, next to a face, is speech. +# +# There is no way to ask for the absence of speech -- cfg is 1, there is no negative +# prompt, and naming it would ask for it. The only move is to say what IS there, and +# between moans what is there is breath. It is continuous where the vocal is not, +# which is the whole point: it gives the gaps something to be. +# +# This is why a lone vocal no longer returns nothing. The old reasoning was that the +# node would be restating the author to the author -- true of the vocal, and the +# vocal is still not restated for its own sake; what is added is the half the author +# did not write and the branch cannot do without. +_VOCAL_BETWEEN = "breathing" # The six above, as a set: see the tail of sounds_for for why they are special-cased. _NAMED_VOCALS = frozenset(("whimpering", "sobbing", "moaning", "groaning", "screaming", "whining")) @@ -1675,9 +2327,30 @@ _SAYS = (r"says?|said|asks?|asked|whispers?|whispered|shouts?|shouted|calls?|" r"breathes?|breathed|hisses|hissed") +# HOW FAR A SUBJECT REACHES TO ITS VERB -- and the asymmetry that gave a woman's +# line to the phone in her hand. +# +# _DEVICE_SAYS reaches THREE words to find its speech verb. This reached two. So +# "Mara picks up the phone and says" -- four words between the person and the verb, +# one between the phone and it -- read as the phone talking, and every beat of the +# shape "somebody handles a machine, then speaks" hit it. On a joint model that is +# the worst reading of the beat available: the branch opens because there IS a line, +# the mouth guard shuts the only face in frame because the line is judged not hers, +# and her own words play out of the object she just picked up. +# +# This file already states the rule -- if there is any chance a person has the line, +# the person keeps it -- so the person's reach is now the wider one. Bounded by the +# SENTENCE, never past it: 'Mara sits on the sofa. The TV says: "..."' is a real +# device line, and a gap that crossed the full stop would take it straight back off +# the television. +_TO_VERB = r"[^.!?\n]{0,80}?" + +# A possessive is not a speaker. "Dana's phone says" is the phone talking. +_NOT_POSSESSIVE = r"(?!['\u2019]s\b)" + _PERSON_SAYS = re.compile( - r"\b(?:he|she|they|i|we|you|" + _NOT_A_NAME + r"[A-Z][\w-]+)\s+" - r"(?:[\w,']+\s+){0,2}?(?:" + _SAYS + r")\b") + r"\b(?:he|she|they|i|we|you|" + _NOT_A_NAME + r"[A-Z][\w-]+)\b" + _NOT_POSSESSIVE + + _TO_VERB + r"\s(?:" + _SAYS + r")\b") def speech_is_a_devices(beat, sheet=""): @@ -1688,13 +2361,20 @@ def speech_is_a_devices(beat, sheet=""): b = beat or "" if not has_speech(b) or not _DEVICE_SAYS.search(b): return False - if _PERSON_SAYS.search(b): + # WHO IS TALKING IS SETTLED OUTSIDE THE QUOTE. What a machine SAYS is not + # evidence about who said it -- an answerphone playing "Mara, Dan called you + # back." names two people and a speech verb, and reading the line's own + # contents as an attribution handed the message back to whoever it mentioned. + # Strip the spoken spans and attribute what is left. This is also what lets the + # reach above be widened safely: the only text it can now cross is narration. + outside = _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", b)) + if _PERSON_SAYS.search(outside): return False # A name from the sheet with a speech verb after it, which the pattern above # only catches when the name happens to be capitalised in the beat. for n, _ in sheet_lines(sheet): - if n and re.search(r"\b" + re.escape(n) + r"\b(?:\s+[\w,']+){0,2}?\s+" - r"(?:" + _SAYS + r")\b", b, re.I): + if n and re.search(r"\b" + re.escape(n) + r"\b" + _NOT_POSSESSIVE + _TO_VERB + + r"\s(?:" + _SAYS + r")\b", outside, re.I): return False return True @@ -1707,8 +2387,39 @@ def device_voice_clause(beat): # As the author spelled it. Lowercasing turned "TV" into "tv", and a set is not # improved by the node correcting its capitalisation. thing = re.sub(r"\s+", " ", m.group(0)) + # "hold still" was a freeze on everybody in the room, and a television scene is + # mostly people watching one. What this clause has to buy is that no face in the + # room is given the machine's line -- that is a closed mouth, not a still body. return (f" The voice in this shot is the {thing}'s, coming out of it across the " - f"room, and the people listening hold still and let it play.") + f"room, and the people listening let it play, their own mouths closed.") + + +# SAYING NOTHING IS NOT SAYING SOMETHING. +# +# "Mara says nothing" matched Name-then-speech-verb and credited her with a line. +# That is bad on its own and worse in context: with both people counted as +# speakers, nobody was left silent, so the lock clause -- which is only emitted +# when there IS somebody to hold -- was cancelled outright. A negation switching +# the guard off is the worst available reading of it. +_SAYS_NOTHING = re.compile( + r"\b(?:says?|said|speaks?|spoke)\s+(?:absolutely\s+|almost\s+)?" + r"(?:nothing|not\s+a\s+word|no\s+more|none)\b" + r"|\b(?:does|do|did|would|will|could)\s*n[o']?t\s+(?:say|speak|answer|reply)\b" + r"|\bnever\s+(?:says?|said|speaks?|spoke)\b" + r"|\b(?:stays?|stayed|remains?|remained|keeps?|kept)\s+(?:quiet|silent)\b" + r"|\bin\s+silence\b|\bwithout\s+(?:a\s+word|speaking|answering)\b", re.I) + + +def _in_beat_order(names, beat): + """The names sorted by where the BEAT first mentions them. + + speakers_in walks the sheet, so it returned sheet order -- and the lock clause + now says "Dan speaks first, then Mara", which is a claim about the beat.""" + b = beat or "" + def at(n): + m = re.search(r"\b" + re.escape(n) + r"\b", b, re.I) + return m.start() if m else len(b) + return sorted([n for n in names if n], key=at) def speakers_in(beat, sheet=""): @@ -1719,6 +2430,14 @@ def speakers_in(beat, sheet=""): speaker's. That is the commonest scene there is, and the lip-sync problem the guard exists for lands squarely on the person saying nothing.""" b, out = beat or "", [] + # A DENIAL OF SPEECH CANCELS THE CLAUSE IT SITS IN, not the whole beat: "Dan + # says: 'Wait.' Mara says nothing." has one speaker and one person who + # explicitly does not speak, and both halves have to survive. So the beat is + # split on sentence boundaries and only the denying halves are dropped. + # The terminator is usually INSIDE the quote -- `says: "Wait here."` ends on a + # quote mark, not a full stop -- so the closing quote counts as a boundary too. + b = " ".join(part for part in re.split(r"(?<=[.!?\"\u201d>])\s+", b) + if not _SAYS_NOTHING.search(part)) for n, _ in sheet_lines(sheet): if not n: continue @@ -1784,7 +2503,7 @@ def speakers_in(beat, sheet=""): at[n] = pm.start() if at: out.append(min(at, key=at.get)) - return out + return _in_beat_order(out, beat) # The mouth half AND the voice half. This said only that the other mouths stay @@ -1938,20 +2657,141 @@ def told_hold(listeners): # ONE naming each. A described person is a person the model draws, and naming # somebody twice in one shot is what put a second copy of them in frame. if len(who) == 1: - return f" {who[0]} listens, still, wearing what the sheet already lists." + # NOT "listens, still". This clause exists to give the listener something to BE + # DOING -- its own docstring says so -- and what it gave them was an instruction + # to be motionless, set off in commas so it could only be read as the adjective. + # It lands on the reaction shot, which is where acting happens. `listens` is the + # activity; the comma was doing the opposite of the clause's whole purpose. + return f" {who[0]} listens, wearing what the sheet already lists." said = ", ".join(who[:-1]) + " and " + who[-1] - return f" {said} listen, still, wearing what the sheet already lists." + return f" {said} listen, wearing what the sheet already lists." -MOUTH_HOLD_OTHERS = (" Only {who} speaks; every other mouth in the shot stays " - "closed, jaws still.") +# The tail both voice guards end on, defined once so they cannot drift apart. +MOUTH_HOLD_REST = "every other mouth in the shot stays closed, those expressions moving" +MOUTH_HOLD_OTHERS = " Only {who} speaks; " + MOUTH_HOLD_REST + "." + + +# A VOCAL BELONGS TO SOMEBODY. +# +# Reported: "her whimpering is opening up his ability to babble. Dialogue is not +# being localized to the characters." +# +# _voiced is exertion_in(body) -- a SHOT-LEVEL flag with no owner -- and both mouth +# guards stand down on it, for everybody in the shot. The comment says exactly why +# they stand down: "straining is vocal and that mouth should be open." THAT mouth. +# Not every mouth. So "McKenna sobs while Dan watches" left Dan's mouth as free as +# hers, on a shot whose audio branch her sob had just opened -- which is precisely +# the machinery the speech guard exists to stop, switched off by the one kind of +# beat that opens the branch without giving anybody words. +# +# Measured on a six-shot scene of a woman gagged in a van: not one shot carried any +# mouth guard at all. +# +# Attribution table of its own, NOT _VOCAL_FROM. That one feeds the sound clause and +# is the six vocals the node will name as a sound; this is about whose face moves, +# which is a wider list and must not change what the shot is heard as. +_VOCAL_SOURCE = ( + (r"whimper(?:s|ing|ed)?", "whimpering"), (r"sob(?:s|bing|bed)?", "sobbing"), + (r"moan(?:s|ing|ed)?", "moaning"), (r"groan(?:s|ing|ed)?", "groaning"), + (r"scream(?:s|ing|ed)?", "screaming"), (r"whin(?:e|es|ing|ed)", "whining"), + (r"gasp(?:s|ing|ed)?", "gasping"), (r"pant(?:s|ing|ed)?", "panting"), + (r"cr(?:y|ies|ying|ied)", "crying"), (r"sigh(?:s|ing|ed)?", "sighing"), + (r"shriek(?:s|ing|ed)?", "shrieking"), (r"yelp(?:s|ing|ed)?", "yelping"), + (r"grunt(?:s|ing|ed)?", "grunting"), (r"weep(?:s|ing)?", "weeping"), + (r"wail(?:s|ing|ed)?", "wailing"), (r"laugh(?:s|ing|ed)?", "laughing"), +) + + +def vocal_sources_in(beat, sheet=""): + """Who this beat says is making a non-speech vocal, and what it is. + + [(name, phrase)], in sheet order. Same shape as speakers_in, including its + conjunction guard: "Dan holds the door and McKenna sobs" must not credit Dan, + because `and` starts a new predicate with its own subject -- and crediting the + wrong person here is worse than crediting nobody, since the shot would then hold + the mouth of whoever is actually making the noise.""" + b, out = beat or "", [] + for n, _ in sheet_lines(sheet): + if not n: + continue + for pat, phrase in _VOCAL_SOURCE: + if re.search(r"\b" + re.escape(n) + r"\b" + r"(?:\s+(?!and\b|but\b|then\b|who\b|,\s*who\b)[\w,']+){0,2}?\s+" + r"(?:" + pat + r")\b", b, re.I): + out.append((n, phrase)) + break + # A COMPOUND SUBJECT IS TWO SOURCES, NOT ONE. + # + # "Mia and Tess laugh over breakfast" credited only Tess, so the shot said + # "the laughing is Tess's; every other mouth in the shot stays closed" -- + # holding Mia's mouth shut in a beat that says she laughs. Reported as the + # acting not matching the scene. + # + # The conjunction guard above is RIGHT about "Dan holds the door and + # McKenna sobs", where `and` starts a new predicate with its own subject. + # What it cannot tell apart is two names sharing ONE verb, and the + # difference is whether a verb intervenes: here nothing stands between the + # names and the verb they share. Same reading posture_in already uses for + # "Kate and Sam sit down", which seats both. + if re.search(r"\b" + re.escape(n) + r"\b(?:\s*,\s*[\w'\u2019-]+)*" + r"\s+and\s+[\w'\u2019-]+\s+" + r"(?:" + pat + r")\b", b, re.I): + out.append((n, phrase)) + break + return out + + +def _joined(names): + """'Dan', 'Dan and Sam', 'Dan, Sam and Mara'.""" + ns = [n for n in (names or []) if n] + if len(ns) < 2: + return ns[0] if ns else "" + return ", ".join(ns[:-1]) + " and " + ns[-1] + + +def voice_sources(talkers, vocal, vocalisers, silent): + """Say whose voice is whose, and close the mouths that are neither. + + Two jobs, and the second is the reported one. Closing the rest stops the + listener babbling on a branch somebody else's sob opened. NAMING THE SOURCES + stops the model swapping them -- two voices in one shot with nothing saying + which is which is a shot where he can be given her whimper and she his line. + + So the sentence is emitted for two DIFFERENT sources even when nobody is left to + hold: with one source and nobody silent there is nothing to disambiguate and + nothing to close, and the shot is left alone.""" + parts = [] + if len(talkers or []) == 1: + parts.append(f"only {talkers[0]} speaks") + elif talkers: + # TWO LINES, TWO MOUTHS, AND NOTHING SAYING WHICH IS WHICH. A beat with two + # speakers left the shot free to put either line on either face. Said in the + # order the BEAT gives them, which is the only ordering there is. + parts.append(f"{talkers[0]} speaks first, then " + + ", then ".join(talkers[1:])) + if vocalisers and vocal: + parts.append(f"the {vocal} is {_joined(vocalisers)}'s") + # Emitted for two DIFFERENT sources even when nobody is left to hold, and for + # two speakers for the same reason: the ordering is the whole point of it. + if not parts or (len(parts) == 1 and not silent and len(talkers or []) < 2): + return "" + if silent: + parts.append(MOUTH_HOLD_REST) + said = "; ".join(parts) + return f" {said[0].upper()}{said[1:]}." # ...and when the line has no name on it. Two people, one line, nobody named: the # speaker cannot be identified, so neither mouth could be held and BOTH were free # to move -- which on a joint model is two voices in the stream and the second one # is the babble. Saying how many voices there are does not require knowing whose. -ONE_VOICE = (" Only the person speaking has their mouth moving; every other jaw " - "in the shot stays still.") +# A JAW THAT "STAYS STILL" IS A FROZEN FACE. The guarantee here is one voice, and a +# closed mouth delivers it -- lip-sync needs lips to part. "Stays still" asked for +# something stronger than the guarantee needs and put it on the listener, which is +# the face the audience is watching. Same wording as MOUTH_HOLD_OTHERS now, because +# they are the same situation with and without a name to put on it. +ONE_VOICE = (" Only the person speaking has their mouth moving; every other mouth " + "in the shot stays closed, those expressions moving.") # H3'S OWN DIALOGUE MARKER. and are special tokens the model was trained @@ -2109,23 +2949,6 @@ def check_audio_vae_loaded(audio_vae): "the Comfy-Org release; rendering with this one produces noise, not speech.") -def ref_image_canvas(w, h, gen_w, gen_h, mode="match"): - """Pure: the (width, height) a reference image is encoded at. - - 'match' scales it (DOWN only, aspect kept) to the generation's pixel area, so a - reference costs about as much as one frame of the shot. 'max' goes to the - reference pipeline's 2048 short edge for the best identity fidelity, which on a - long chain is several times slower because the rows are re-attended every step - of every shot. Never upscales: a small reference stays small.""" - w, h = max(1, int(w)), max(1, int(h)) - if mode == "max": - scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h)) - else: - scale = min(1.0, math.sqrt((int(gen_w) * int(gen_h)) / float(w * h))) - snap = lambda v: max(CANVAS_MULTIPLE, round(v * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) - return snap(w), snap(h) - - def shot_latent_cells(w, h, frames, fps): """Latent cells in one shot: what sampling VRAM actually scales with. @@ -2163,810 +2986,6 @@ def model_fingerprint(model): return None - -# --- H3 plumbing, carried over unchanged: these were arrived at against the real -# model and the real VAEs, and none of it is prompt logic. - -def _resize(image, width, height, crop): - s = image[..., :3].movedim(-1, 1) - s = comfy.utils.common_upscale(s, width, height, "lanczos", crop) - return s.movedim(1, -1) - - -def _empty_av_latent(width, height, length, fps, batch_size=1): - fc, lt, at = temporal_shape(length, fps) - video = torch.zeros([batch_size, 24, lt, height // 16, width // 16], device=mm.intermediate_device()) - audio = torch.zeros([batch_size, 32, 2, at], device=mm.intermediate_device()) - return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, fc - - -def _auto_tile_t(n_latent_frames, requested=None): - """Temporal tile for a tiled decode. An explicit value wins. - - The decode_tile_frames widget is gone, so this is where the value comes from - now. It has to come from somewhere: ComfyUI's decode_tiled_3d defaults tile_t - to 999, i.e. SPATIAL tiles only, and expanding the whole clip's time axis at - once is the single largest allocation in a run. A "tiled" decode that keeps the - full temporal extent barely lowers the peak, so the OOM retry that switches - tiling on was, without this, retrying with almost the same footprint.""" - if requested: - return int(requested) - n = int(n_latent_frames or 0) - return AUTO_TILE_T if n > AUTO_TILE_T else None - - -def _decode_video(vae, out_latent, tiled, free_first=None, tile_t=None, tile_xy=None, - keep=()): - """Decode the video latent. - - `free_first` is the diffusion model: sampling is finished, and the video VAE - needs the room for THIS decode -- the free runs immediately before it, not to - make room for the next shot. On a card where the DiT is most of the VRAM, the - decode does not fit until it goes. - - `keep` is what must NOT be evicted on the way. It was `keep_loaded=[]`, which - unloaded every resident model -- including the video VAE, which ComfyUI then - reloaded three lines later to run the decode. An evict-and-reload of the thing - about to be used, once per shot, on every card. Peak VRAM is identical either - way, since the VAE has to be resident to decode; the round trip was pure cost. - - memory_required is ASKED FOR HONESTLY, which it was not. It was 1e30, and - free_memory computes `memory_to_free = memory_required - get_free_memory(device)` - (model_management.py:887), so 1e30 means "unload everything not in keep_loaded", - every shot, in full -- skipping partially_unload entirely. - - What that evicts is the DiT, three lines before the next shot needs it again. On - a machine whose RAM is already full of finished frames there is nowhere for it to - go but disk, so the reload is a read from the drive, once per shot. Reported as - thrashing that slows the preload, and it is exactly that: the same weights being - read back at every boundary. - - The VAE knows what its own decode costs -- ComfyUI sizes it with - memory_used_decode and uses that number everywhere else. Asked for that instead, - a card with headroom frees NOTHING and the DiT simply stays. A card without - headroom frees what it needs and no more, which is what partially_unload is for. - 1e30 remains the fallback for a VAE that cannot estimate itself.""" - latent = out_latent["samples"] - if latent.is_nested: - latent = latent.unbind()[0] - if free_first is not None: - try: - mm.free_memory(_decode_headroom(vae, latent), mm.get_torch_device(), - keep_loaded=_resident(keep or (vae,))) - except Exception: - pass - # A VAE THAT ALREADY TILES DOES NOT NEED TO BE ASKED TO, AND ASKING COSTS 3x. - # - # MiniMaxH3VideoVAE.decode_tiled is, in full: - # - # def decode_tiled(self, z, **kwargs): - # return self.decode(z) - # - # Every tile_t/overlap_t/tile_x/tile_y this function computes is discarded, so - # the tiling the widget promises is not happening here -- the model tiles - # internally either way (256px spatial, 17-frame temporal), which is why - # comfy/sd.py sets handles_tiling on it. - # - # What the detour costs is the OUTPUT BUFFER. comfy's VAE.decode preallocates - # ONE result at vae_output_dtype and hands it to the model as output_buffer=, - # and MiniMaxH3VideoVAE.decode_temporal writes finalized chunks straight into - # it. Going through decode_tiled instead reaches _decode_tiled_owned, which - # calls the model with output_buffer=None -- so decode_temporal allocates its - # own at torch.float32 -- and then makes an fp16 `copy=True` of that. Two - # buffers, the larger of them at double width: - # - # tiled : fp32 2.60GB + fp16 copy 1.30GB = 3.90GB per shot - # decode: one preallocated fp16 = 1.30GB per shot - # - # at 362 frames of 1056x608. Every shot, on the node's own default. - # - # So: when the VAE owns its tiling AND can be written into, the un-tiled call IS - # the tiled one, minus the copies. Anything else keeps the old path -- this is a - # detour around a detour, not a claim that tiling is useless. - _owns_tiling = bool(getattr(vae, "handles_tiling", False) and getattr( - getattr(vae, "first_stage_model", None), "comfy_has_chunked_io", False)) - if tiled and _owns_tiling: - imgs = vae.decode(latent) - elif tiled: - # Temporal + spatial tiling. Without tile_t the VAE expands the WHOLE latent - # clip at once, which on a 243-frame 1344x768 shot is the single largest - # allocation in the run -- and on an unpruned checkpoint that is already - # streaming, it is what tips the card over. Decoding in temporal chunks - # trades a little speed for a much lower peak; None keeps ComfyUI's defaults. - args = {} - tile_t = _auto_tile_t(latent.shape[2] if latent.ndim >= 5 else 0, tile_t) - if tile_t: - args["tile_t"] = int(tile_t) - args["overlap_t"] = max(1, int(tile_t) // 8) - if tile_xy: - args["tile_x"] = int(tile_xy) - args["tile_y"] = int(tile_xy) - try: - imgs = vae.decode_tiled(latent, **args) if args else vae.decode_tiled(latent) - except TypeError: - imgs = vae.decode_tiled(latent) # older signature without tile_t - else: - imgs = vae.decode(latent) - if len(imgs.shape) == 5: - imgs = imgs.reshape(-1, imgs.shape[-3], imgs.shape[-2], imgs.shape[-1]) - return imgs - - -def _decode_audio(audio_vae, out_latent): - latent = out_latent["samples"] - if latent.is_nested: - latent = latent.unbind()[-1] - audio = audio_vae.decode(latent).movedim(-1, 1) - std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 - std[std < 1.0] = 1.0 - audio = audio / std - sr = getattr(audio_vae, "audio_sample_rate_output", getattr(audio_vae, "audio_sample_rate", 44100)) - return {"waveform": audio, "sample_rate": sr} - - -# SYNTHESISING THE BED, from the description the node already read off the scene. -# -# No file to wire and no second model pass. Room tone is physically shaped noise -- -# air, rumble, plant, a mains hum -- so it can be built rather than fetched, and -# built noise cannot speak, which is the whole problem with getting ambience out of -# a joint model. -# -# Each recipe is: spectral tilt (0 white, 1 pink, 2 brown), a low-pass corner, an -# optional high-pass, an optional tonal hum with its harmonic, and an optional slow -# amplitude movement. Ordered, first match wins, most specific first. -# -# HONEST LIMIT: this makes TONE, not events. "birdsong", "cutlery and moving chairs" -# and "a monitor somewhere down the corridor" get the ROOM those things are in, not -# the things -- synthesising a convincing bird is not something a noise shaper does, -# and a bad one is worse than the room alone. `info` says when that has happened. -_BED_EVENTFUL = ("birdsong", "cutlery", "monitor somewhere", "corridor beyond") -# Target level for a built bed, before ambient_level scales it. -22 dBFS RMS, so -# the default 0.25 lands near -34 dBFS: present, and well under a spoken line. -_BED_RMS = 0.08 -_BED_RECIPE = ( - (r"\brain\b", dict(tilt=0.8, cut=9000, hp=250, mod=(0.30, 0.18))), - (r"\bstorm\b|\bthunder", dict(tilt=1.7, cut=700, mod=(0.13, 0.40))), - (r"\bwind\b|\btrees\b", dict(tilt=1.2, cut=2600, mod=(0.18, 0.42))), - (r"\bsea\b|\bocean\b", dict(tilt=1.3, cut=1700, mod=(0.11, 0.50))), - (r"\btraffic\b", dict(tilt=1.7, cut=900, mod=(0.07, 0.22))), - (r"\bengine\b", dict(tilt=1.5, cut=520, hum=(60.0, 0.30), - mod=(0.09, 0.12))), - (r"\bpipes\b|\bwater\b", dict(tilt=1.3, cut=1250, mod=(0.55, 0.45))), - (r"\bclock\b|\bticking", dict(tilt=1.6, cut=800, tick=(1.0, 0.22))), - # The hum family: a fridge, a fan, a strip light, a monitor. Tonal, not noise. - (r"\bhum(?:ming|s)?\b|\bfan\b|\bfridge\b|\bstrip light\b|\bmonitor\b", - dict(tilt=1.4, cut=1500, hum=(100.0, 0.22))), - (r"\btiled\b|\bringing\b", dict(tilt=0.9, cut=6000, hp=180)), - (r"\bhard walls\b|\bgiving the sound back\b", dict(tilt=1.6, cut=950)), - (r"\bopen air\b|\bno walls close\b|\bbirdsong\b", dict(tilt=1.0, cut=7000)), - (r"\bhollow quiet\b|\bhallway\b|\bcorridor\b|\blarge empty room\b|\blong tail\b", - dict(tilt=1.6, cut=700)), - (r"\bcutlery\b|\bchairs\b", dict(tilt=1.1, cut=4500)), - (r"\bsoft room\b|\blittle echo\b", dict(tilt=1.8, cut=520)), - (r"\bnight\b|\bbedroom\b|\bhouse\b|\bquiet\b", dict(tilt=1.9, cut=380)), -) - - -def bed_recipe(phrase): - """How to build the bed this phrase describes. The neutral room if none match.""" - p = str(phrase or "").lower() - for pat, rec in _BED_RECIPE: - if re.search(pat, p): - return dict(rec) - return dict(tilt=1.8, cut=420) - - -def synth_ambient(phrase, n, sr, seed=0, channels=2): - """Build `n` samples of the ambience `phrase` describes. [C, n], or None. - - Shaped in the FREQUENCY domain -- white noise, an envelope, back again -- which - gives exact spectral control in one pass and, unlike a per-sample filter, does - not walk a million-sample loop in Python. - - Generated at the FULL length of the film, so unlike a wired file there is no - loop and therefore no join to hide. - - Defensive like everything else on this path: any failure returns None and the - soundtrack goes out as the model made it.""" - try: - n, sr = int(n), int(sr) - if n < 64 or sr <= 0: - return None - rec = bed_recipe(phrase) - g = torch.Generator().manual_seed(int(seed) & 0x7fffffff) - w = torch.randn((int(channels), n), generator=g) - f = torch.fft.rfftfreq(n, d=1.0 / sr).clamp(min=1.0) - # Amplitude goes as f^(-tilt/2), so POWER goes as f^-tilt: tilt 1 is pink, - # 2 is brown. Then a gentle low-pass, and a high-pass where the recipe wants - # the bottom out of it. - env = f.pow(-float(rec.get("tilt", 1.8)) / 2.0) - env = env / (1.0 + (f / float(rec.get("cut", 420))) ** 2) - if rec.get("hp"): - env = env * (f / (f + float(rec["hp"]))) - y = torch.fft.irfft(torch.fft.rfft(w, dim=-1) * env, n=n, dim=-1) - t = torch.arange(n, dtype=torch.float32) / sr - # Slow movement, so a bed does not sit perfectly still and read as a hiss. - if rec.get("mod"): - rate, depth = rec["mod"] - y = y * (1.0 + float(depth) * torch.sin(2 * math.pi * float(rate) * t)) - # A tonal hum is a TONE, not noise: a fridge and a strip light are pitched. - if rec.get("hum"): - hz, amp = rec["hum"] - hum = (torch.sin(2 * math.pi * float(hz) * t) - + 0.35 * torch.sin(2 * math.pi * float(hz) * 2 * t)) - y = y + float(amp) * hum.unsqueeze(0) - if rec.get("tick"): - rate, amp = rec["tick"] - step = max(1, int(sr / max(float(rate), 0.01))) - click = torch.zeros(n) - idx = torch.arange(0, n, step) - click[idx] = 1.0 - decay = torch.exp(-torch.arange(min(step, int(sr * 0.05)), - dtype=torch.float32) / (sr * 0.004)) - click = torch.nn.functional.conv1d( - click.view(1, 1, -1), decay.flip(0).view(1, 1, -1), - padding=decay.numel() - 1)[0, 0, :n] - y = y + float(amp) * (click * torch.randn(n, generator=g)).unsqueeze(0) - # NORMALISE BY RMS, NOT PEAK. Peak-normalising made the loudness depend on - # the recipe's crest factor rather than on the setting: measured across the - # beds, a strip-light hum came out at -8.2 dBFS and a ticking clock at - # -34.1, a 26 dB spread from one ambient_level. RMS puts them all at the - # same subjective level, so the widget means the same thing in every room. - rms = float(y.pow(2).mean().sqrt()) - if not (rms > 0.0) or not torch.isfinite(y).all(): - return None - y = y * (_BED_RMS / rms) - # ...then hold the peak down, because a peaky recipe (the clock) would - # otherwise reach 2.8 at that RMS and clip before the mix even sees it. - peak = float(y.abs().max()) - if peak > 0.95: - y = y * (0.95 / peak) - return y - except Exception: - return None # a bed is a nicety, a render is not - - -# FOLEY: the sounds an action MAKES, built and mixed rather than asked of the model. -# -# auto_sound already reads these out of the beat, but only as TEXT in the prompt -- -# and text can never open a shot's audio branch, because an open branch on a joint -# model invents a voice. So a wordless shot staging cuffs going on was pinned to -# silence and the cue was dropped: the one shot whose whole point is a sound made -# none, and the only way to get it was to write the sound into the beat by hand. -# -# Mixing solves that the same way the ambient bed does. A built sound asks nothing -# of the model, so it cannot babble, and it goes into THAT SHOT'S span of the -# soundtrack rather than under the whole film. -# -# HONEST LIMIT, and it is worth stating rather than discovering: this is synthesis, -# not a recording. It reads as a click, a rattle, a rustle -- serviceable and in the -# right place, not a foley stage. Wire a recording to ambient_audio, or write the -# sound into the beat and let the model make it, where that is not enough. -# -# NOTHING VOCAL IS EVER BUILT. Breathing and effort are in the sound table too, and -# they are a VOICE: the one thing this file must not manufacture. They are absent -# from the recipes below on purpose, and a phrase with no recipe is simply skipped. -# A struck object rings at SEVERAL frequencies at once, and they are not a -# harmonic series -- a bar or a shell has inharmonic modes, which is exactly why a -# cuff reads as metal and not as a note. One resonator is one tone colour, and one -# tone colour over a whole train of hits is the sound of a filter rather than the -# sound of a thing. -# -# Ratios are deliberately irrational-ish. Integer multiples would make a pitched -# tone, which is a different and worse kind of fake. Higher modes get less gain and -# a lower Q, because in a real object they are both weaker and more damped. -# -# The upper-mode gains are a MEASURED TRADE, not a guess. Swept against modal -# density (count of spectral peaks) and against how far the cluster drags the -# centroid off what each recipe was tuned to as a single resonator: -# -# gain scale 1.00 0.75 0.60 0.50 0.40 0.30 -# modes 478 381 326 284 238 208 (was 200) -# centroid 1.61x 1.52x 1.45x 1.40x 1.35x 1.28x -# -# 0.60 keeps about 1.6x the spectral density of the single resonator while moving -# the centre 1.45x rather than 1.61x. Density is the realism; the centroid shift is -# a change to a character that was already tuned, so it is spent, not maximised. -_MODES = ((1.00, 1.000, 1.00), (1.48, 0.270, 0.75), - (2.13, 0.132, 0.55), (3.31, 0.060, 0.40)) - - -def _band(x, sr, f0, q=4.0, order=3): - """Resonant filter by spectral envelope: a mode cluster around f0, one pass. - - ORDER 3, which was measured. A single resonator's skirt falls off as 1/f, and - against noise -- equal energy per Hz, spread over 20 kHz -- enough survives above - the centre that the result is bright whatever f0 says: footsteps aimed at 130 Hz - came back with a spectral centroid of 3.6 kHz, and every recipe sounded like the - same hiss. Cubing the response is what makes f0 mean something. - - The f0/q interface is unchanged, so every recipe gets the mode cluster without - being rewritten -- this is the one place all 21 of them pass through.""" - n = int(x.shape[-1]) - X = torch.fft.rfft(x) - f = torch.fft.rfftfreq(n, d=1.0 / sr).clamp(min=1.0) - resp = torch.zeros_like(f) - for ratio, gain, qs in _MODES: - fc = float(f0) * ratio - if fc >= sr * 0.45: # past Nyquist is not a mode, it is aliasing - continue - qq = max(0.7, float(q) * qs) - # BANDWIDTH COMPENSATION, and it is not optional. A resonator's absolute - # bandwidth is fc/Q, so a mode an octave up passes twice the noise for the - # same gain -- and these are excited by noise, which has equal energy per - # Hz. Uncompensated, the cluster came out about 2x brighter across every - # recipe and put a footstep at 428 Hz against the 130 it is aimed at, which - # is the "a footstep is a hiss" failure the order-3 skirt was fixed for. - # Energy through a mode goes as gain^2 * fc / Q, so scaling the gain by - # sqrt(Q/fc) makes the numbers above mean the loudness they look like. - g_i = float(gain) * math.sqrt(float(qs) / float(ratio)) - # ...and the cluster itself scales with Q, because Q IS how much the thing - # rings. Metal at q 5-8 has strong upper modes; a footstep at q 1.6 is a - # broadband thud on a floor and has almost none. Applied only above the - # fundamental, so a low-Q recipe collapses back to the single resonator it - # was tuned as -- which is what keeps a footstep at 130 Hz a footstep. - if ratio > 1.0: - g_i *= min(1.0, float(q) / 4.0) - resp = resp + g_i * (1.0 / torch.sqrt( - 1.0 + (qq * (f / fc - fc / f)) ** 2)) ** int(order) - return torch.fft.irfft(X * resp, n=n) - - -def _hits(n, sr, g, times, decay, amp=1.0): - """Decaying noise bursts at the given times (seconds). The excitation for a - click, a rattle, a footfall -- everything percussive here is this plus a band. - - EVERY HIT DIFFERS. They used to be identical -- same level, same decay, same - everything -- and thirty-three identical clicks is not a chain, it is a machine. - Nothing gives a synthetic sound away faster: the ear is far better at spotting a - repeat than at judging a timbre, so a rattle whose links are all the same reads - as fake even when each single link sounds right. - - Level varies about +/-5 dB and decay by about a third, which is the spread a - real repeated contact has from hitting at a different point and angle.""" - x = torch.zeros(n) - for t in times: - i = int(t * sr) - if i < 0 or i >= n: - continue - a = float(amp) * float(torch.exp((torch.rand(1, generator=g) - 0.5) * 1.1)) - d = float(decay) * float(1.0 + (torch.rand(1, generator=g) - 0.5) * 0.7) - L = max(4, int(d * sr)) - m = min(L, n - i) - env = torch.exp(-torch.arange(m, dtype=torch.float32) - / max(d * sr / 4.0, 1.0)) - x[i:i + m] += torch.randn(m, generator=g) * env * a - return x - - -def _room(x, sr, secs=0.11, wet=0.16, seed=0): - """A small room around the sound. Convolution with a decaying noise tail plus - three early reflections. - - The dryness was the loudest tell. Every one of these was rendered anechoic -- - no reflections, no tail -- and nothing in the physical world sounds like that; - the ear reads a bone-dry impact as "not in a place" before it judges anything - else about it. The tail is rolled off above 2.2 kHz because a real room absorbs - highs faster than lows, and a bright tail is its own kind of wrong. - - Linear convolution, not circular: the transform is padded past n + L so a tail - cannot wrap round and appear before the hit that caused it.""" - n = int(x.shape[-1]) - L = max(8, int(float(secs) * sr)) - if n < 8 or not (float(wet) > 0.0): - return x - g = torch.Generator().manual_seed(int(seed) & 0x7fffffff) - t = torch.arange(L, dtype=torch.float32) - ir = torch.randn(L, generator=g) * torch.exp(-t / max(L / 5.0, 1.0)) - ir[0] = 0.0 - for d, a in ((0.0071, 0.50), (0.0133, 0.34), (0.0211, 0.23)): - i = int(d * sr) - if i < L: - ir[i] += a - m = 1 - while m < n + L: - m <<= 1 - F = torch.fft.rfftfreq(m, d=1.0 / sr).clamp(min=1.0) - IR = torch.fft.rfft(ir, n=m) / (1.0 + F / 2200.0) - wet_sig = torch.fft.irfft(torch.fft.rfft(x, n=m) * IR, n=m)[:n] - p, q = float(wet_sig.abs().max()), float(x.abs().max()) - if not (p > 0.0) or not torch.isfinite(wet_sig).all(): - return x - wet_sig = wet_sig * (q / p) - return x * (1.0 - float(wet)) + wet_sig * float(wet) - - -def _even(start, count, gap, jitter, g): - """Click times, with a little jitter so a rattle is not a drum machine.""" - j = (torch.rand(int(count), generator=g) - 0.5) * 2.0 * float(jitter) - return [float(start + i * gap + j[i]) for i in range(int(count))] - - -# phrase -> how to build it. `secs` is the shot length, so a rattle runs the shot -# while a ratchet is one event placed a third of the way in. -_FOLEY = { - "cuffs ratcheting closed": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.33, 9, 0.030, 0.004, g), - 0.020), sr, 3200, 6.0), - "cuffs knocking": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.25, 4, 0.22, 0.06, g), - 0.035), sr, 2600, 5.0), - "chain links dragging": - lambda n, sr, g, secs: _band(_hits(n, sr, g, - _even(0.05, max(4, int(secs * 11)), 0.09, 0.035, g), - 0.028), sr, 4200, 7.0), - "restraints pulling taut": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 3, 0.35, 0.10, g), - 0.30), sr, 700, 2.5), - "rope creaking as it goes tight": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 4, 0.28, 0.09, g), - 0.28), sr, 620, 2.5), - "a lock snapping shut": - lambda n, sr, g, secs: _band(_hits(n, sr, g, [secs * 0.5], 0.045), sr, 2100, 5.0), - "a metal bolt sliding": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.4, 6, 0.035, 0.010, g), - 0.030), sr, 1800, 4.0), - "keys on a ring": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 7, 0.055, 0.025, g), - 0.030), sr, 5200, 8.0), - "a zip running": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.35, 70, 0.0065, 0.0012, g), - 0.006), sr, 4800, 5.0), - "velcro tearing open": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.35, 120, 0.004, 0.0015, g), - 0.005), sr, 3000, 1.6), - "tape pulling off": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 90, 0.007, 0.002, g), - 0.008), sr, 2400, 2.0), - "fabric rustling": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(0.1, max(3, int(secs * 3)), 0.30, - 0.12, g), 0.10), sr, 2800, 1.8), - "blades through fabric": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 5, 0.18, 0.05, g), - 0.09), sr, 3600, 2.2), - # A slow rhythm of frame creaks. Low and wooden, and the rate is deliberately - # unhurried: the point is that the room is not silent, not that the shot has a - # metronome in it. - "a bed frame working": - lambda n, sr, g, secs: _band(_hits(n, sr, g, - _even(0.15, max(3, int(secs * 1.6)), 0.62, - 0.05, g), 0.16), sr, 240, 3.0), - "a buckle and leather creaking": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 4, 0.20, 0.07, g), - 0.12), sr, 1200, 3.0), - "footsteps": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(0.25, max(2, int(secs / 0.55)), - 0.55, 0.05, g), 0.10), sr, 130, 1.6), - "something dragging on the floor": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.2, max(6, int(secs * 8)), - 0.12, 0.05, g), 0.14), sr, 420, 1.4), - "something landing": - lambda n, sr, g, secs: _band(_hits(n, sr, g, [secs * 0.5], 0.14), sr, 110, 1.5), - "a sharp impact": - lambda n, sr, g, secs: _band(_hits(n, sr, g, [secs * 0.45], 0.07), sr, 900, 1.5), - "a door on its hinges": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(secs * 0.3, 8, 0.09, 0.03, g), - 0.13), sr, 780, 6.0), - "water": - lambda n, sr, g, secs: _band(_hits(n, sr, g, _even(0.05, max(8, int(secs * 14)), 0.07, - 0.03, g), 0.06), sr, 1400, 1.5), -} - - -def foley_for(phrase, n, sr, seed=0): - """Build the sound `phrase` names, `n` samples long. None when there is no - recipe -- which includes every vocal phrase, deliberately.""" - try: - n, sr = int(n), int(sr) - make = _FOLEY.get(str(phrase or "")) - if make is None or n < 64 or sr <= 0: - return None - g = torch.Generator().manual_seed(int(seed) & 0x7fffffff) - y = make(n, sr, g, n / float(sr)) - # The room goes on LAST and on everything, which is what a room does: it is - # a property of the place, not of the prop. Applied here rather than in the - # recipes so all 21 get it and none can forget it. - y = _room(y, sr, seed=int(seed) + 977) - peak = float(y.abs().max()) - if not (peak > 0.0) or not torch.isfinite(y).all(): - return None - return y * (0.7 / peak) - except Exception: - return None - - -def plain_bed(n, sr, seed=0, channels=2): - """The last-resort bed: noise and a moving average, and nothing else. - - synth_ambient is defensive, so it can return None -- and a built bed that comes - back empty leaves the output with no ambience at all. Wiring a file is NOT the - remedy for that: the built bed is the feature, and a file is only ever an - override for a real location. So there is a floor under it. - - Deliberately primitive. No FFT, no envelope, no recipe -- a cumulative-sum box - filter over white noise, which is a rumble, and which cannot fail on any input - the caller can hand it. It is not as good as the shaped bed and does not try to - be; it is the difference between a quiet room and nothing at all.""" - try: - n, sr, channels = int(n), int(sr), max(1, int(channels)) - if n < 8 or sr <= 0: - return None - g = torch.Generator().manual_seed(int(seed) & 0x7fffffff) - y = torch.randn((channels, n), generator=g) - # Box filter by cumulative sum: out[i] = mean(w[i-k:i]). k sets the corner. - # - # CASCADED THREE TIMES, which was measured rather than assumed. One pass is - # a sinc, whose first sidelobe is only -13 dB -- against white noise, which - # has equal energy per Hz, enough leaks through the whole top of the band to - # put the spectral centroid at 3.3 kHz. That is a hiss, not the rumble this - # is meant to be. Three passes is sinc^3, and the centroid lands where the - # description says. - k = max(2, min(n // 4, int(sr / 200))) # ~200 Hz - for _ in range(3): - c = torch.cumsum(torch.nn.functional.pad(y, (k, 0)), dim=-1) - y = (c[..., k:] - c[..., :-k])[..., :n] / float(k) - rms = float(y.pow(2).mean().sqrt()) - if not (rms > 0.0) or not torch.isfinite(y).all(): - return None - y = y * (_BED_RMS / rms) - peak = float(y.abs().max()) - return y * (0.95 / peak) if peak > 0.95 else y - except Exception: - return None - - -def _seamless_loop(x, n, sr): - """[C, M] -> [C, n], looped with a crossfade so the join does not click. - - Plain tiling puts a discontinuity at every repeat, once per loop length. In a - bed that is meant to sit under everything unnoticed, a regular click is the one - thing that gets noticed -- the same objection that made the silence latent - ping-pong its interior rather than tile it. Here the material is real audio - being PLAYED rather than a latent being conditioned on, so it cannot be - reversed: a room tone read backwards is fine, but footsteps are not. Crossfade - instead, which works on both.""" - m = int(x.shape[-1]) - if m <= 0: - return None - if m >= n: - return x[..., :n] - fade = min(int(0.25 * sr), m // 4) - if fade < 1: - reps = -(-n // m) - return x.repeat(1, reps)[..., :n] - # OVERLAP-ADD the tail onto the head, and shorten the unit by the overlap. The - # unit then runs x[m-fade] .. x[m-fade-1], so tiling it steps between samples - # that were adjacent in the source and there is no discontinuity anywhere. - # - # Measured, because the obvious construction is wrong: appending the crossfade - # to the END of a full-length unit leaves it finishing on x[fade-1] while the - # next repeat starts on x[0], which are not adjacent -- a 2s tone that does not - # divide evenly gave a 64x jump at the join, worse than plain tiling's 41x. - t = torch.linspace(0.0, 1.0, fade, dtype=x.dtype, device=x.device) - head = x[..., :fade] * t + x[..., m - fade:] * (1.0 - t) - unit = torch.cat([head, x[..., fade:m - fade]], dim=-1) - if int(unit.shape[-1]) < 1: - reps = -(-n // m) - return x.repeat(1, reps)[..., :n] - reps = -(-n // int(unit.shape[-1])) - return unit.repeat(1, reps)[..., :n] - - -def mix_ambient(audio, sr, bed, level): - """Lay an ambient bed UNDER a finished soundtrack. -> (waveform, note). - - The bed is PLAYED, not conditioned on: it is the file, at the level asked for, - under whatever the model generated. That is the whole reason to do it here - rather than in the sampler -- ambience needs no cooperation from a joint model, - has nothing to lip-sync to, and so cannot put a voice in a wordless shot. The - conditioning path can only steer the branch toward something bed-LIKE, and on a - shot with a line it competes with the line. - - Defensive throughout, like the silence latent: any failure returns the audio - untouched with a note saying so, because a bed is a nicety and a render is not. - """ - try: - if audio is None or bed is None or float(level or 0.0) <= 0.0: - return audio, "" - w = bed.get("waveform") if isinstance(bed, dict) else None - if w is None or not int(getattr(w, "ndim", 0)): - return audio, ("ambient_audio is wired but carries no waveform, so nothing " - "was laid under the soundtrack") - w = w[0] if w.dim() == 3 else w # [B, C, M] -> [C, M] - if w.dim() != 2 or w.shape[-1] < 2: - return audio, ("ambient_audio is too short to loop, so nothing was laid " - "under the soundtrack") - w = w.detach().to(dtype=audio.dtype, device=audio.device) - b_sr = int((bed.get("sample_rate") if isinstance(bed, dict) else 0) or 0) - # RESAMPLE, or the bed plays at the wrong speed and pitch. Linear is coarse - # for music and inaudible on a room tone, which is what this input is for. - resampled = "" - if b_sr > 0 and b_sr != int(sr): - want = max(2, int(round(w.shape[-1] * float(sr) / float(b_sr)))) - w = torch.nn.functional.interpolate( - w.unsqueeze(0), size=want, mode="linear", align_corners=False)[0] - resampled = f", resampled from {b_sr} Hz" - ch = int(audio.shape[1]) - if int(w.shape[0]) != ch: - w = (w.mean(dim=0, keepdim=True).repeat(ch, 1) if int(w.shape[0]) > ch - else w[:1].repeat(ch, 1)) - n = int(audio.shape[-1]) - loop = _seamless_loop(w, n, int(sr)) - if loop is None: - return audio, "" - out = audio + loop.unsqueeze(0) * float(level) - # NORMALISE rather than clip. Clipping a bed that pushed a loud line over - # the top distorts the LINE, which is the thing worth keeping. - peak = float(out.abs().max()) - gain = "" - if peak > 1.0: - out = out / peak - gain = f", and the mix was scaled by {1.0 / peak:.2f} to stop it clipping" - secs = w.shape[-1] / float(sr) - return out, (f"an ambient bed was laid under the whole soundtrack at level " - f"{float(level):.2f} -- {secs:.1f}s of audio{resampled}, looped " - f"with a crossfade so the join does not click{gain}. It is your " - f"file, played under what the model generated: it conditions " - f"nothing, so it cannot put a voice in a wordless shot the way " - f"an inferred bed did. Shots pinned to silence keep their silent " - f"conditioning and get the bed on top, which is what makes a " - f"wordless shot sound like a room instead of a mute") - except Exception as exc: - return audio, (f"the ambient bed could not be mixed ({type(exc).__name__}), so " - f"the soundtrack is unchanged") - - -def _is_oom(e): - return isinstance(e, torch.cuda.OutOfMemoryError) or "out of memory" in str(e).lower() - - -def _deep_cleanup(): - """Release VRAM + RAM between shots so a long chain doesn't accumulate and OOM. - Runs a Python GC pass (frees dereferenced tensors / CPU buffers), then empties - the CUDA allocator's cached blocks and IPC handles. Cheap relative to sampling; - called once per beat. - - It unloads NOTHING. soft_empty_cache(force) ignores `force` in current ComfyUI - (model_management.py:2050) -- the body only reaches empty_cache() and - ipc_collect() -- so this drops cached blocks, not models. The `True` is kept - only for older builds that read it; the older comment here claimed this took an - unload_all_models path, and it does not.""" - gc.collect() - try: - mm.soft_empty_cache(True) - except TypeError: - mm.soft_empty_cache() - try: - if torch.cuda.is_available(): - torch.cuda.empty_cache() - torch.cuda.ipc_collect() - except Exception: - pass - - -DECODE_HEADROOM = 1.25 # over ComfyUI's own estimate, for working allocations -SAMPLE_HEADROOM = 1.35 # likewise for sampling, which is the longer stretch - - -def _decode_headroom(vae, latent): - """VRAM this decode actually needs, by the VAE's own estimate. 1e30 if unknown. - - ComfyUI sizes every VAE with memory_used_decode and uses that number itself, so - it is the honest figure to hand free_memory. The alternative -- and what was here - -- is 1e30, which means "unload everything" and evicts the DiT before every - decode, three lines before the next shot reloads it. - - 1e30 on failure rather than 0: a bad estimate that frees too little turns a slow - render into an OOM, and a wrong guess should fall back to the behaviour that has - been running, not to no freeing at all.""" - try: - dtype = getattr(vae, "vae_dtype", None) or latent.dtype - need = float(vae.memory_used_decode(tuple(latent.shape), dtype)) - if need > 0: - return need * DECODE_HEADROOM - except Exception: - pass - return 1e30 - - -def _resident(models): - """The LoadedModel entries ComfyUI currently holds for `models`. - - That is the form free_memory's keep_loaded wants: it compares against the - entries in current_loaded_models, not against the ModelPatcher objects a node - is holding. Anything not matched is simply not kept, so a model that is not - resident costs nothing here.""" - out = [] - for lm in list(getattr(mm, "current_loaded_models", [])): - for m in models or (): - if m is None: - continue - try: - if lm.model is m or getattr(lm, "model", None) is getattr(m, "model", None): - if lm not in out: - out.append(lm) - except Exception: - pass - return out - - -def _image_out_dtype(): - """The dtype ComfyUI itself hands between nodes on THIS install. - - The join used to end in a hard-coded .float(), commented "back to what every - downstream node expects". That was true when it was written and is not a - constant: ComfyUI has --fp16-intermediates, and on an install running it the - VAE's own decode already returns fp16 -- VAE.vae_output_dtype() IS - model_management.intermediate_dtype() (comfy/sd.py) -- as do EmptyLatentImage - and the rest of nodes.py. So on that install the node was taking frames the - VAE handed it in fp16, widening them to fp32 nothing had asked for, and - handing them to nodes whose own convention is fp16. - - It is the largest thing this node holds, so the widening is not free: the - 2580-frame chain costed at the join is 9.3GB as fp16 and 18.5GB as fp32, - against 44.6GB of staged weights on a 62GB machine -- which is the difference - between the render finishing and the OOM killer taking the server. Reported as - exactly that, twice. - - Asked, not assumed, and never widened: whatever ComfyUI says it wants between - nodes is what the chain is built in. An install with the flag off is told - float32 and gets float32, byte for byte what it got before. Older builds have - no intermediate_dtype at all, so the fallback is the old constant.""" - try: - return mm.intermediate_dtype() - except Exception: - return torch.float32 - - -def _evict_all_but(keep_model, latent=None): - """Unload every model EXCEPT the diffusion model from the GPU. - - This is the fix for VRAM ratcheting across a long chain. soft_empty_cache() - only drops the CUDA allocator's cached blocks -- it does NOT unload models, so - ComfyUI keeps the Qwen3-VL text encoder (~14.6GB) and both VAEs resident in - current_loaded_models alongside the DiT. Each shot re-encodes the prompt - (text encoder), encodes the handoff keyframe (video VAE), then samples (DiT), - so all three compete for the card. - - ComfyUI does free ahead of each load -- load_models_gpu() calls free_memory() - for what it is about to need (model_management.py:975), so the weight path is - not purely reactive. What it cannot size for is a long chain's ACTIVATIONS on - a card where the DiT is most of the VRAM. Freeing explicitly, right after - conditioning is built and before sampling, keeps only what the sampler needs. - - ASKED FOR HONESTLY, and this is the expensive one. free_memory computes - `memory_to_free = memory_required - get_free_memory(device)`, so 1e30 meant - "unload everything but the DiT" on every shot, unconditionally -- on a 48GB card - with room for all of it as readily as on a 16GB one. What it unloads is the - ~14.6GB text encoder and both VAEs, and the next shot re-encodes the prompt and - the handoff keyframe, so all three come straight back. On a machine whose RAM is - already full of finished frames they come back from DISK, once per shot, which is - the thrashing this was reported as. - - The DiT can size its own activations -- memory_required(shape) is what ComfyUI - itself calls before a load -- so ask for that. A card with room frees nothing and - keeps the encoder resident; a card without frees exactly as much as it must. - 1e30 stays the fallback, because a bad estimate that frees too little turns a - slow render into an OOM.""" - need = 1e30 - try: - if latent is not None: - shape = latent["samples"].shape if isinstance(latent, dict) else latent.shape - need = float(keep_model.model.memory_required(tuple(shape))) * SAMPLE_HEADROOM - if not (need > 0): - need = 1e30 - except Exception: - need = 1e30 - try: - mm.free_memory(need, mm.get_torch_device(), - keep_loaded=_resident([keep_model])) - except Exception: - try: - mm.soft_empty_cache(True) - except Exception: - pass - - def check_vae_wiring(vae, audio_vae): """Catch the commonest miswire -- the video VAE dropped into BOTH VAE inputs. Without this the run samples a whole shot, decodes the video fine, then dies @@ -3010,118 +3029,16 @@ def flush_for_model_change(model): pass # Never let a cleanup failure abort the run: the flush is best-effort hygiene, # and a partially-flushed card is still better than raising here. - for _ in range(2): # 2nd pass frees blocks released by the 1st - try: - _deep_cleanup() - except Exception: - pass + try: + _deep_cleanup() + except Exception: + pass old_fmt, _n, old_sz, _c = prev new_fmt = fp[0] return (f"model changed since last run ({old_fmt} ~{old_sz / GB:.1f}GB -> {new_fmt} " f"~{fp[2] / GB:.1f}GB): flushed all resident models and VRAM caches") -# Whether the silence conditioning ACTUALLY went on, per run. _silent_audio_latent -# is defensive by design -- every failure returns None so a render never dies for a -# nicety -- but the info note reported the silence_nonspeech FLAG, not the result. -# A shot whose latent could not be built was described as "conditioned on real -# silence" while its audio branch was wide open, which is a shot that babbles with -# no scripted line and nothing in the report saying why. Counted here so the note -# can say what happened instead of what was asked for. -_SILENCE_STATUS = {"asked": 0, "applied": 0, "why": ""} - - -# How much silence to encode, and how much of each end to throw away. The encoder -# pads at the edges, so the first and last few latent frames carry an artifact that -# is not silence: measured on the H3 audio VAE, the frame-to-frame delta runs 0.224 -# at the first join and 0.172 at the last against 0.002 in the interior. Four -# frames off each end clears it with room to spare. -_SILENT_SECONDS = 2 -_SILENT_EDGE = 4 - - -def _silent_audio_latent(audio_vae, frame_count, fps): - """A keyframe audio latent of actual SILENCE, or None if it cannot be made. - - H3 is a JOINT model: the mouth follows the audio branch. On a shot with no - scripted line the branch is otherwise unconditioned, and an unconditioned audio - branch invents a voice -- which the picture then lip-syncs to. The lips-closed - sentence is arguing with a stream that has already decided someone is talking. - - REBUILT 2026-09-05, from measurements against the real VAE rather than from - reasoning. The previous version encoded one second, kept a SINGLE interior - frame and repeated it, on the argument that silence is homogeneous. It is not, - in latent space: encoded silence has genuine frame-to-frame variation (delta - mean 0.002-0.004, max 0.021), and a repeated frame has a delta of exactly - 0.000000. That is a flat signal no encoder produces, and a model handed - conditioning outside its own distribution has every reason to disregard it -- - which is an audio branch back to inventing a voice, with the report saying - silence went on. - - The fix that version was avoiding is real too: tiling the whole encoded second - end to end leaves a 25x spike at each join (0.554 against 0.022), once per - second, which is a metronome in the conditioning of a joint model. - - So: encode two seconds, drop the padded ends, and PING-PONG the interior -- - forward, reversed, forward. Every join repeats a frame, so there is no seam, - and the interior statistics are the encoder's own. Measured over a 9s shot: - - one frame repeated peak 0.000686 delta mean 0.000000 max 0.000000 - whole 2s tiled peak 0.000314 delta mean 0.017451 max 0.554715 - interior ping-pong peak 0.000566 delta mean 0.002039 max 0.021159 - - where the encoder's own interior is mean 0.0021, max 0.0212. Decoded peak - 0.000566 on a +/-1.0 scale is about -65 dBFS: silence. - - Everything here stays defensive. Shapes are CHECKED against what the layout - expects rather than assumed, and any failure returns None so the shot falls - back to an unconditioned branch instead of breaking the render -- the caller - reports when that happens, so it is no longer a silent failure. - """ - try: - sr = int(getattr(audio_vae, "audio_sample_rate", 0) or 0) - if sr <= 0: - return None - _, _, want_t = temporal_shape(frame_count, fps) - if want_t <= 0: - return None - block = _SILENT_UNIT.get("lat") - if block is None: - # CHANNELS LAST. comfy.sd.VAE.encode() does `pixel_samples.movedim(-1, 1)` - # before handing off, so the audio VAE -- which wants [B, 2, L] -- must be - # given [B, L, 2]. Passing [B, 2, L] raises inside the encoder, and an - # early version did exactly that: swallowed by the guard below, so the - # whole layer silently did nothing. - # - # Two seconds, encoded ONCE and cached. Encoding a full 15s shot instead - # cost a VAE pass big enough to OOM mid-render on a 16GB card, where the - # failure again degraded silently to no conditioning at all. - enc = audio_vae.encode(torch.zeros((1, sr * _SILENT_SECONDS, 2))) - if enc is None or enc.dim() != 4 or enc.shape[1] != 32: - return None - if enc.shape[-1] <= 2 * _SILENT_EDGE + 1: - return None - block = enc[..., _SILENT_EDGE:-_SILENT_EDGE].detach().to("cpu").clone() - _SILENT_UNIT["lat"] = block - n = block.shape[-1] - if n < 1: - return None - # Forward, reversed, forward... Each join repeats a frame, so the seam that - # plain tiling leaves is gone while the interior variation is the encoder's. - pieces, have, i = [], 0, 0 - while have < want_t: - piece = block if i % 2 == 0 else torch.flip(block, dims=[-1]) - pieces.append(piece) - have += n - i += 1 - out = torch.cat(pieces, dim=-1)[..., :want_t].clone() - if out.shape[-1] != want_t: - return None - return out - except Exception: - return None # never fail a render for a nicety - - _POSTURE = re.compile( r"\b(?:lying|laying|lies|lays|kneel(?:s|ing)?|knelt|sit(?:s|ting)?|sat|" r"crouch(?:es|ing|ed)?|curled|sprawled|slumped|face[- ]?down|face[- ]?up|" @@ -3181,7 +3098,14 @@ def reference_note(n_refs, aug, has_first_frame): if not has_first_frame: note += (". Shot 1 has no keyframe, so the reference is its only picture and " "nothing competes with reproducing it -- that shot is where a " - "near-clean reference shows up as the opening frame") + "near-clean reference shows up as the opening frame, AND IT DOES NOT " + "STAY THERE: every later shot opens on the previous shot's last " + "frame, so whatever composition shot 1 settles on is handed down the " + "whole chain. A portrait reproduced at shot 1 is therefore a portrait " + "framing for the film, which is what 'the camera is fixated on her' " + "is. Wire a wide establishing frame into first_frame and shot 1 is " + "pinned to that composition instead -- it is the one input that " + "outranks a reference, because it IS frame one") return note @@ -3197,7 +3121,10 @@ def frame_detail(img): frame it runs on is the model's own output, so shot 11 is sampled from a picture that has been through ten decode/encode cycles. Softening that compounds is invisible shot to shot and obvious end to end -- so measure it.""" - x = img.float() + # This is diagnostic only. Sampling at most roughly 256 points per axis avoids + # allocating a full-resolution float32 copy of every shot's final frame. + step = max(1, max(int(img.shape[0]), int(img.shape[1])) // 256) + x = img[::step, ::step].float() if x.dim() == 3 and x.shape[-1] >= 3: x = x[..., :3].mean(dim=-1) elif x.dim() == 3: @@ -3209,99 +3136,81 @@ def frame_detail(img): return float((gx + gy) * 0.5), float(x.std()) -def detail_report(per_shot): - """One line saying whether the chain is softening, and by how much. +def levels_report(levels, shots): + """What hold_levels measured, and what it did about it. - per_shot is [(detail, contrast), ...] measured on each shot's last frame.""" - vals = [d for d, _ in per_shot if d > 0] - if len(vals) < 2: + Worth printing even when it corrected nothing: the measurement is the evidence that + the chain is or is not cooking, and a run that measured a drift too small to act on + is a different thing from a run that never looked.""" + if levels is None: return "" - first, last = vals[0], vals[-1] - drop = (first - last) / first * 100.0 if first else 0.0 - trend = " ".join(f"{d:.4f}" for d, _ in per_shot) - line = f"detail per shot (last frame): {trend}" - if drop >= 10.0: - line += (f" -- DOWN {drop:.0f}% from shot 1 to shot {len(vals)}. Each boundary " - f"decodes a shot, takes its LAST frame and re-encodes it as the next " - f"shot's keyframe, so the loss of one round trip is carried into the " - f"next and compounds. Break the chain to stop it accumulating: " - f"restart_after_removal starts a shot from the text instead of the " - f"previous frame, at the cost of a visible cut there") - elif drop <= -10.0: - line += f" -- UP {-drop:.0f}%, so the chain is not softening" + g, o = levels.estimate() + if g is None: + return "" + pct = "/".join(f"{(float(torch.exp(v)) - 1.0) * 100.0:+.1f}%" for v in g) + lvl = "/".join(f"{float(v):+.4f}" for v in o) + line = (f"hold_levels: measured the chain drifting {pct} of contrast and {lvl} of level " + f"per boundary, per R/G/B channel, from {len(levels._bg)} boundary(ies)") + n = len(levels.applied) + if not n: + line += (" -- below the 8-bit floor a handoff is quantised to, so nothing was " + "applied rather than claiming a correction that would be erased") else: - line += f" -- flat within {abs(drop):.0f}%" + last = levels.applied[-1][0] + line += (f", and took it back out of {n} handoff(s); the last gain applied was " + f"{'/'.join(f'{float(v):.3f}' for v in last)}. The contrast line above is " + f"measured on the corrected frames, so it is the residual, not the defect") return line -def _keyframe_latent(vae, hand_img): - """The keyframe latent for this shot: an ENCODE of the previous shot's last frame. +def detail_report(per_shot): + """Two lines: whether the chain is softening, and whether it is COOKING. - This was briefly an optimisation -- pass the previous shot's own latent straight - through and skip a VAE round trip per boundary. It was wrong, and it degraded - every shot after the first. + per_shot is [(detail, contrast), ...] measured on each shot's last frame. - A keyframe is ONE pixel frame, and H3's grid puts that at 5f -> TWO latent - frames. Slicing [:, :, -1:] off a finished shot hands over one. Worse, the video - VAE is causal: the last latent of a 72-frame sequence encodes its temporal - context, not a standalone opening frame, so even at the right count it does not - mean what a keyframe means. The spatial-size guard could not see either problem. - - The round trip is real but it is one lossy step on a correctly formed anchor, - which beats a cheap malformed one.""" - return vae.encode(hand_img) - - -def _build_ref_images(vae, images, gen_w, gen_h, mode="match"): - """(tokenizer items, DiT blocks) for a list of reference IMAGE tensors. - - The tokenizer labels each one `:` itself, in the order given here -- - so the roster the prompt refers to is decided by input order, not by anything - written in the prompt.""" - items, blocks = [], [] - for img in images: - if img is None: - continue - h, w = int(img.shape[1]), int(img.shape[2]) - tw, th = ref_image_canvas(w, h, gen_w, gen_h, mode) - resized = _resize(img[:1], tw, th, "disabled") - items.append({"type": "image", "data": resized}) - blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, - "latent": vae.encode(resized)}) - return items, blocks - - -def _sample_on_sigmas(model, seed, cfg, sampler_name, positive, negative, latent, sigmas): - """common_ksampler, driven by an EXTERNAL sigma schedule. - - common_ksampler derives its sigmas from (sampler_name, scheduler, steps, denoise) - and takes no schedule argument, so a schedule computed anywhere else cannot - reach it. Under PDD that is fatal rather than merely inconvenient: the heads - accept only their nine trained boundaries, and re-deriving the grid from - widgets means hitting it by coincidence and losing it again the moment a step - count changes. - - Mirrors nodes.common_ksampler's noise / mask / callback handling exactly -- the - only substitution is comfy.sample.sample_custom for comfy.sample.sample.""" - latent_image = latent["samples"] - latent_image = comfy.sample.fix_empty_latent_channels( - model, latent_image, - latent.get("downscale_ratio_spacial", None), - latent.get("downscale_ratio_temporal", None)) - noise = comfy.sample.prepare_noise(latent_image, seed, latent.get("batch_index")) - # `steps` here only sizes the progress bar -- the schedule is `sigmas`, whose - # step count is one less than its length (the trailing 0.0 is an endpoint). - callback = latent_preview.prepare_callback(model, max(len(sigmas) - 1, 1)) - samples = comfy.sample.sample_custom( - model, noise, cfg, comfy.samplers.sampler_object(sampler_name), sigmas, - positive, negative, latent_image, - noise_mask=latent.get("noise_mask"), callback=callback, - disable_pbar=not comfy.utils.PROGRESS_BAR_ENABLED, seed=seed) - out = latent.copy() - out.pop("downscale_ratio_spacial", None) - out.pop("downscale_ratio_temporal", None) - out["samples"] = samples - return out + Contrast used to be measured here and thrown away, which was the worst possible + arrangement: the surviving metric RISES with burn-in -- expanding contrast creates + neighbour differences -- so a chain visibly cooking printed "UP n%, so the chain is + not softening" and read as reassurance. The reported symptom was being measured on + exactly the right frame and never shown. Both trends are reported now, and the + detail line no longer pronounces on a rise it cannot explain by itself.""" + ds = [d for d, _ in per_shot if d > 0] + cs = [c for _, c in per_shot if c > 0] + if len(ds) < 2: + return "" + out = [] + drop = (ds[0] - ds[-1]) / ds[0] * 100.0 if ds[0] else 0.0 + line = "detail per shot (last frame): " + " ".join(f"{d:.4f}" for d, _ in per_shot) + if drop >= 10.0: + line += (f" -- DOWN {drop:.0f}% from shot 1 to shot {len(ds)}. Each boundary " + f"decodes a shot, takes its LAST frame and re-encodes it as the next " + f"shot's keyframe, so the loss of one round trip is carried into the " + f"next and compounds. Break the chain to stop it accumulating: " + f"restart_after_removal stops a shot opening on the previous frame, " + f"at the cost of a cut there") + elif drop <= -10.0: + line += (f" -- UP {-drop:.0f}%. Read the contrast line before taking that as good " + f"news: expanding contrast raises this number too") + else: + line += f" -- flat within {abs(drop):.0f}%" + out.append(line) + if len(cs) >= 2: + rise = (cs[-1] - cs[0]) / cs[0] * 100.0 if cs[0] else 0.0 + cl = "contrast per shot (last frame): " + " ".join(f"{c:.4f}" for _, c in per_shot) + if rise >= 10.0: + cl += (f" -- UP {rise:.0f}% from shot 1 to shot {len(cs)}, which is the chain " + f"COOKING: every shot is sampled from the previous shot's last frame, " + f"the model reproduces it with a little more contrast, and the VAE " + f"clamps the result to 0..1 -- so the headroom each pass spends is " + f"never given back, and it shows as crushed blacks and blown " + f"highlights rather than merely as more contrast. hold_levels takes " + f"the per-boundary part of it back out") + elif rise <= -10.0: + cl += f" -- DOWN {-rise:.0f}%, so the chain is flattening rather than cooking" + else: + cl += f" -- flat within {abs(rise):.0f}%" + out.append(cl) + return " | ".join(out) def _find_h3_sampling_node(): @@ -3359,6 +3268,78 @@ def _direct_model_sampling(model, shift_video, shift_audio): return m +# WHERE THE AUDIO BRANCH LANDS FROM, AND HOW TO SHORTEN THE FALL. +# +# Reported over and over as babble at the OPENING of a beat, and none of the prose +# in this file could touch it. Every clause here changes what the branch is TOLD. +# None of them changes how much noise it still has to clear when it stops. +# +# Computed from ComfyUI's own scheduler code at shift 12/3 -- the last AUDIO sigma +# before zero, which the final step has to clear in a single jump: +# +# scheduler 5 steps 8 steps +# simple 0.4286 0.3000 +# beta 0.2981 0.1559 +# normal 0.0348 0.0348 +# kl_optimal 0.0030 0.0030 +# exponential 0.0030 0.0030 +# +# 43% in one step against 0.3%. A branch resolving that much at once invents +# whatever is easiest to invent, and on a branch conditioned on "somebody speaks" +# that is a voice. It surfaces at the OPENING because that is where the branch has +# least conditioning to anchor it -- the line has not started. That is also why the +# prose fixes helped and did not solve it: they reduce the empty space the invention +# lands in; this reduces the capacity to invent. +# +# THE AUDIO BRANCH HAS NO SCHEDULE OF ITS OWN. comfy/ldm/minimax/model.py derives it +# per step -- sigma_a = time_shift_sigma(sigma_v, shift_v, shift_a) -- so the audio +# tail is decided by the VIDEO schedule, and choosing a scheduler for the audio +# means giving up the one chosen for the picture. Inserting ONE step does not: it +# splits the final jump and leaves every earlier sigma exactly where it was. +# +# The formula is comfy's, restated here rather than imported, for the same reason +# last_audio_sigma restates it: this has to work when comfy is not importable. +def audio_sigma_of(video_sigma, shift_video, shift_audio): + """The audio branch's sigma at a given video sigma. comfy's time_shift_sigma.""" + v, a, s = float(shift_video), float(shift_audio), float(video_sigma) + base = s / (v + s * (1.0 - v)) + return a * base / (1.0 + (a - 1.0) * base) + + +def video_sigma_for_audio(target_audio, shift_video, shift_audio): + """The video sigma that puts the audio branch on `target_audio`. The inverse.""" + v, a, t = float(shift_video), float(shift_audio), float(target_audio) + base = t / (a - t * (a - 1.0)) + return base * v / (1.0 - base + base * v) + + +def insert_audio_landing(sigmas, shift_video, shift_audio, + target=0.03, coarse=0.10): + """One extra step so the audio branch does not land from a great height. + + Returns a new list, or the input unchanged when there is nothing to do. This + runs inside the render path, so anything unexpected -- an empty schedule, no + trailing zero, a tail that is already soft -- returns the input rather than + raising. It never inserts twice: after one pass the tail is below `coarse`. + + `target` is 0.03 because that is roughly what `normal` achieves on its own, and + it is comfortably above the 0.003 kl_optimal leaves -- close enough to free, far + enough from zero that the extra step is doing work rather than nothing.""" + try: + out = [float(x) for x in (sigmas or [])] + except (TypeError, ValueError): + return sigmas + if len(out) < 3 or out[-1] != 0.0 or out[-2] <= 0.0: + return sigmas + if audio_sigma_of(out[-2], shift_video, shift_audio) <= coarse: + return sigmas + land = video_sigma_for_audio(target, shift_video, shift_audio) + # Strictly inside the final jump, or the schedule stops being monotonic. + if not (0.0 < land < out[-2]): + return sigmas + return out[:-1] + [land, 0.0] + + def last_audio_sigma(steps, shift_audio, scheduler="simple", shift_video=None): """How much audio noise is still left going into the FINAL sampling step. @@ -3421,14 +3402,51 @@ def scheduler_that_finishes_audio(steps, shift_audio, shift_video=None, current="simple", target=0.10): """The shipped scheduler that leaves the LEAST audio noise on the last step. - Returns (name, sigma) when a different one would get under `target` and beat - what is selected, else None. Named rather than silently switched: the schedule - shape changes the picture too, and that is the reader's call to make.""" + ONLY ONE THAT HONOURS shift_video, and that restriction is the whole of what + this function got wrong. Reported: switching to kl_optimal put watery waves on + the picture. + + comfy/samplers.py grades its schedulers by `use_ms`. A handler with use_ms True + is called as handler(model_sampling, steps) and sees the shift; one with use_ms + False is called as handler(n, sigma_min, sigma_max) and NEVER SEES IT. kl_optimal + exponential and karras are all in the second group, so recommending them threw + shift_video away silently. At 5 steps and shift 12 the difference is the whole + schedule: + + simple 1.0 0.9796 0.9474 0.8889 0.7500 <- stays high, as shift 12 asks + kl_optimal 1.0 0.6725 0.4212 0.2082 0.0119 <- shift discarded + + The video branch gets almost no time at high sigma, so structure never resolves + and the remaining steps polish detail with nothing underneath it. That is what + watery looks like. The audio tail WAS better; it was better because the schedule + had stopped being the one that was asked for. + + Returns (name, sigma) when a shift-honouring scheduler would get under `target` + and beat what is selected, else None. Named rather than silently switched: the + schedule shape changes the picture too, and that is the reader's call.""" try: import comfy.samplers as _cs - names = list(getattr(_cs.KSampler, "SCHEDULERS", []) or []) + names = [n for n in (getattr(_cs.KSampler, "SCHEDULERS", []) or []) + if getattr(_cs.SCHEDULER_HANDLERS.get(n, None), "use_ms", False)] except Exception: return None + # ...AND THE VIDEO SCHEDULE HAS TO SURVIVE IT. Honouring the shift is necessary + # and not sufficient: ddim_uniform honours it and still starts at 0.98 rather + # than 1.0, so the first step does not begin from full noise. Checking only the + # audio number is the mistake that recommended kl_optimal, so the candidate's + # own video schedule is read and has to start from noise and keep its high-sigma + # steps -- all but two of them above 0.5, which is what shift 12 is buying. + def _video_ok(nm): + try: + import comfy.model_sampling as _cms + _ms = _cms.ModelSamplingDiscreteFlow() + _ms.set_parameters(shift=float(shift_video or 12.0)) + sig = [float(x) for x in _cs.calculate_sigmas(_ms, nm, int(steps))] + except Exception: + return False + return (len(sig) >= 3 and sig[0] >= 0.999 + and sum(1 for x in sig if x > 0.5) >= max(1, int(steps) - 1)) + now = last_audio_sigma(steps, shift_audio, current, shift_video) best, best_s = None, now for nm in names: @@ -3438,7 +3456,7 @@ def scheduler_that_finishes_audio(steps, shift_audio, shift_video=None, sg = last_audio_sigma(steps, shift_audio, nm, shift_video) except Exception: continue - if sg > 0.0 and sg < best_s: + if sg > 0.0 and sg < best_s and _video_ok(nm): best, best_s = nm, sg return (best, best_s) if (best is not None and best_s <= target) else None @@ -3508,6 +3526,61 @@ def apply_h3_model_sampling(model, shift_video, shift_audio): "MXFP8/turbo profile and the audio sounds wrong") +def sparse_dit_patched(model): + """True when something upstream installed a per-block DiT replace patch, False when it + provably did not, None when this model cannot say. + + That patch is how ComfyUI's Model Sparse Attention node registers itself + (set_model_patch_replace -> model_options["transformer_options"]["patches_replace"] + ["dit"]), whatever method it was set to. None is distinct from False on purpose: a stub + or hand-built model carries no model_options at all, and the one caller of this turns a + True into a refusal, so "cannot tell" must never be read as "not there".""" + opts = getattr(model, "model_options", None) + if not isinstance(opts, dict): + return None + tops = opts.get("transformer_options") or {} + if not isinstance(tops, dict): + return None + return bool((tops.get("patches_replace") or {}).get("dit")) + + +def sparse_attention_allocator_abort(model): + """The one configuration that does not raise, it ABORTS. Returns why, or "". + + cudaMallocAsync is stream-ordered: a block allocated on one CUDA stream must be freed + consistently with that stream. comfy_kitchen's chunked sparse-attention producer is a + GENERATOR consumed from inside the kernel call, across the stream boundary that + --enable-dynamic-vram's prefetch machinery sets up, and freeing its per-chunk tensor + there returns CUDA_ERROR_INVALID_VALUE from cuMemFreeAsync. That throws out of a tensor + DESTRUCTOR, where there is no Python frame to catch it, so the process calls + std::terminate: a core dump, not an exception, taking the server and the rest of the + queue with it. + + Worth refusing rather than warning for exactly that reason -- there is nothing to + recover from an abort, and nothing downstream gets the chance to try. Losing one render + to a readable error is the better trade. + + Nobody chooses this, either. ComfyUI force-enables the allocator on any CUDA 13 torch + build and does not consult its own card blacklist on that path (cuda_malloc.py), so a + current install arrives here by default.""" + conf = str(os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") or "") + if "cudamallocasync" not in conf.lower(): + return "" + if sparse_dit_patched(model) is not True: + return "" + return ("this render would ABORT the ComfyUI process rather than fail: a sparse-attention " + "patch is on the model AND torch is using the cudaMallocAsync allocator " + f"(PYTORCH_CUDA_ALLOC_CONF={conf}). Freeing the attention producer's per-chunk " + "tensor under that allocator returns CUDA_ERROR_INVALID_VALUE from cuMemFreeAsync, " + "inside a tensor destructor where nothing can catch it -- so the process " + "core-dumps and the queue goes with it, which is why this stops here instead.\n\n" + "Either restart ComfyUI with --disable-cuda-malloc, which is the flag ComfyUI's " + "own cuda_malloc.py names for this failure, or take the Model Sparse Attention " + "node out of the graph. ComfyUI turns that allocator on by itself on every CUDA 13 " + "torch build without checking whether the card supports it, so this is the default " + "rather than anything you picked.") + + def sampling_oom_help(w, h, frames, fps, megapixels=0.0): """What to change, in this shot's own numbers, after a SAMPLING OOM. @@ -3598,6 +3671,35 @@ def strip_legacy_fields(text): _ADD_LINE = re.compile(r"^[ \t]*(?:add|wear|wearing)[ \t]*:[ \t]*(.+?)[ \t]*$", re.I | re.M) +# YOUR SENTENCE, IN THIS SHOT, UNTOUCHED. +# +# Everything else in a shot is either the author's text put through a reader -- scoped +# to this shot's people, scrubbed of what came off, reordered so the action leads -- +# or a clause this file wrote. Both are governed: on a short beat the node's own +# continuity clauses were measured at 76% of a shot against the beat's 8%, and a +# sentence competing with that cannot be relied on to survive intact. +# +# An `exact:` line is neither. It is placed straight after the beat, in the author's +# words, and nothing here reads it, scopes it, scrubs it or drops it: it is not a +# guard and has no budget to lose. Its cost is counted against the BEAT in the +# balance report, because that is whose text it is. +# +# NOTHING READS IT, and that is the contract rather than an oversight. A name in an +# exact line does not add that person to the shot, a garment in it removes nothing, +# and a door in it stages no change -- otherwise "say this exactly" would quietly +# mean "stage this too", and the one instruction guaranteed to reach the model +# verbatim would be the one with the least predictable side effects. +# NOT "say". A beat writes speech as `Mara says: "Wait here."` and a line could +# plausibly open with it, and a directive that swallows dialogue is worse than one +# word less convenient. +_EXACT_LINE = re.compile(r"^[ \t]*(?:exact|exactly|verbatim)[ \t]*:[ \t]*(.+?)[ \t]*$", + re.I | re.M) + + +def exact_lines(beat): + """[the author's verbatim sentences] for this beat, in the order written.""" + return [m.group(1).strip() for m in _EXACT_LINE.finditer(beat or "") if m.group(1).strip()] + # Prose that reads as taking something off. NOT used to remove anything -- inferring # removals from prose is what made the old node unpredictable. It is used only to # notice that a beat looks like a removal while the scene still describes the @@ -3627,6 +3729,23 @@ _TRAILING_VERB = (r"take[sn]?|took|taking|pull(?:s|ed|ing)?|peel(?:s|ed|ing)?|" # ...and verbs that are a removal on their own, needing no particle. # One definition, in the engine. See engine._UNDO_VERB. _UNDO_VERB = engine._UNDO_VERB +# Verbs that only take a garment off with the preposition that says so. Kept apart +# from _STRIP_VERB on purpose: that list also builds the DISPLACEMENT reader, and a +# bare "gets" or "pushes" there reads "gets down on her knees" and "pushes the door +# open" as garments being moved. Here they are only ever matched with "out of", +# "clear of", "off" or a destination, which is where the meaning lives. +_OUT_OF_VERB = (r"get(?:s|ting)?|got|shimm(?:y|ies|ied|ying)|squirm(?:s|ed|ing)?|" + r"climb(?:s|ed|ing)?|ease[sd]?|easing|back(?:s|ed|ing)?|" + r"step(?:s|ped|ping)?|wriggle[sd]?|wiggle[sd]?|struggle[sd]?") +_PUSH_VERB = (r"push(?:es|ed|ing)?|shove[sd]?|skim(?:s|med|ming)?|ease[sd]?|" + r"easing|roll(?:s|ed|ing)?|work(?:s|ed|ing)?") + +_OPENER_VERB = (r"unzip(?:s|ped)?|unbutton(?:s|ed)?|unfasten(?:s|ed)?|undo(?:es)?|undid|" + r"unhook(?:s|ed)?|unclasp(?:s|ed)?") +_FINISHES_REMOVAL = re.compile(r"\b(?:off|away|out\s+of|remove[sd]?|removing|drops?|" + r"dropped|discard(?:s|ed)?|sheds?|" + r"lets?\s+(?:it|them)\s+(?:fall|drop|slide)|" + r"falls?\s+(?:to|down|away|off))\b", re.I) _REMOVAL_PROSE = re.compile( r"\b(?:" + _UNDO_VERB + r")\b" @@ -3644,7 +3763,41 @@ _REMOVAL_PROSE = re.compile( # the garment sits between the verb and the particle -- "lifts her top over her # head" -- and the object span is read forward from the end of the match. r"|\b(?:" + _STRIP_VERB + r")\b" - r"(?=[^.;!?]{0,40}?\bover\s+(?:her|his|their|the)\s+head\b)", + r"(?=[^.;!?]{0,40}?\bover\s+(?:her|his|their|the)\s+head\b)" + # OUT OF IT, CLEAR OF IT, FREE OF IT. These verbs say nothing on their own -- + # "gets down", "eases back", "climbs up", "backs away" -- so they are kept out + # of _STRIP_VERB, which also feeds the DISPLACEMENT reader, where a bare "gets" + # would read every "gets down on her knees" as a garment being moved. With the + # preposition in front of a garment there is no second reading: you cannot get + # out of a thong and still have it on. The engine's own state reader has had + # `gets out of` since it was written; this one did not, so the state knew the + # garment was off while the text went on describing it as worn. + r"|\b(?:" + _OUT_OF_VERB + r")\s+(?:out|clear|free)\s+of\b" + # PUSHED OR SHOVED OFF. push and shove live in the displacement reader and not + # in the strip verbs, so "pushes the thong off her hips" was a displacement at + # best -- and in practice nothing at all, because the displacement pattern + # wants the direction word where this sentence puts a body part. The garment + # stayed described as worn in every later shot. + r"|\b(?:" + _PUSH_VERB + r")\s+(?:off|away)\b" + r"|\b(?:" + _PUSH_VERB + r")\b(?=[^.;!?]{0,40}?\b(?:off|away)\b)" + # DOWN PAST THE HIPS. "down" on its own stays a displacement, for exactly the + # reason the comment above gives: it leaves the garment ON, "around the thighs + # or the hips". Down her LEGS, her knees, her ankles, or down to the floor is + # the garment travelling past all of that, with nothing left holding it up. + # The two readings are separated by the part of the body named -- and the two + # positions this deliberately excludes are the two that comment names. + r"|\b(?:" + _STRIP_VERB + r"|" + _PUSH_VERB + r")\b" + r"(?=[^.;!?]{0,40}?\bdown\s+(?:(?:her|his|their|the)\s+" + r"(?:legs?|knees?|calves|shins?|ankles?|feet)\b|(?:and\s+)?(?:off|away)\b|" + + engine.TO_THE_FLOOR + r"))" + # ...AND ONTO THE FLOOR. A garment dropped, let fall, kicked or thrown onto the + # floor is off the body by the end of the sentence, whatever verb carried it + # there. `drop` and `let` are the RESTORE vocabulary as well -- that is the + # same ambiguity restored_garments resolves, and it resolves it the same way, + # on where the garment lands. See engine.FLOOR. + r"|\b(?:" + _STRIP_VERB + r"|" + _PUSH_VERB + r"|drop(?:s|ped|ping)?|" + r"let(?:s|ting)?|lob(?:s|bed)?|fling(?:s|ing)?|flung|discard(?:s|ed|ing)?)\b" + r"(?=[^.;!?]{0,40}?" + engine.TO_THE_FLOOR + r")", re.I) @@ -3846,7 +3999,7 @@ def off_by_last_frame(items, agent="", scene="", beat=""): # thing it forbids. It also names no garment, so it summons none. # About what is WORN, not about the body. "Everything else on the body stays # exactly as it is for the whole shot" reads as an instruction to hold still. - bound = "Everything else worn stays exactly as it is, untouched and still fastened." + bound = "Everything else worn stays exactly as it is, untouched and fastened." return " " + sentence[0].upper() + sentence[1:] + " " + bound @@ -3960,7 +4113,563 @@ RESTRAINT_HOLD_KEY = ("handcuffs cuffs chains rope ropes tape gag collar restrai # only way to satisfy it is to put a face in an empty frame. The AUDIO half has no # such limit -- an empty room still babbles -- so the two are separate conditions and # are gated separately below. -MOUTH_HOLD = " Mouths in the shot stay closed, jaws still." +# AND IT DOES NOT FREEZE THE FACE. "Mouths stay closed" is the whole of the +# lip-sync guarantee -- lip-sync needs lips to part, and a closed mouth cannot do +# it. "jaws still" was a stillness instruction riding along on that guarantee, +# landing on every quiet shot in the film, and `still` is not a quiet word to a +# video model: it damps motion wherever it is pointed. Reported as bad acting. +# +# So the second half now says what the face IS doing rather than what it is not, +# which is the same rule every other clause here follows -- at cfg 1 there is no +# negative prompt, and an unreacting face is exactly what you get by asking for +# nothing. It costs one word against a guard block already measured at 47% of the +# shot, and it is the only sentence in that block with anything to say about +# performance. +MOUTH_HOLD = " Mouths in the shot stay closed, the expressions moving." + + +# A MOUTH THE BEAT ITSELF PUTS TO WORK. +# +# The guard above is right for a face doing nothing. It was also landing on the +# beats that ARE the performance -- and contradicting them, in one case word for +# word: +# +# Dana grins, wide and mean. -> Mouths in the shot stay closed, jaws still. +# Dana's mouth falls open. -> Mouths in the shot stay closed, jaws still. +# Dana yawns. -> Mouths in the shot stay closed, jaws still. +# +# Only the VOCAL reactions stood down, because exertion_in covers laughing and +# sobbing. Every SILENT facial performance -- the ordinary currency of acting -- +# was answered with an instruction to freeze it, and the beat is the only +# performance direction a shot has. +# +# ONLY THE MOUTH. A stare, a frown, a wince is a face acting with its mouth shut, +# and the guard costs it nothing; standing down for those would free a mouth for no +# gain, and a free mouth on an open branch is where invented lip-sync lands. +# +# SEPARATE FROM _voiced, which is the same stand-down for EFFORT and also unpins +# the audio branch. A smile is silent. This frees the picture and leaves the branch +# exactly where it was -- no beat that was silent before this becomes audible -- +# because a silent expression is the most common beat in any script, and letting +# one open an audio branch would be the babble hole rebuilt at the widest point. +_MOUTH_WORKS = re.compile( + r"\b(?:smil(?:e|es|ed|ing)|grin(?:s|ned|ning)?|smirk(?:s|ed|ing)?|" + r"sneer(?:s|ed|ing)?|grimac(?:e|es|ed|ing)|pout(?:s|ed|ing)?|" + r"yawn(?:s|ed|ing)?|gape(?:s|d|ing)?|chew(?:s|ed|ing)?|" + r"kiss(?:es|ed|ing)?)\b" + # Spitting needs somewhere to spit. Bare `spits` is what an engine does. + r"|\bspits?\s+(?:it\s+)?(?:on|at|out|into|onto)\b" + # The rest need their object, because the bare verb is ordinary English: + # she bites her lip, not the dog bites; she licks her lips, not licks a stamp. + r"|\b(?:bite|bites|biting|bit)\s+(?:down\s+on\s+)?(?:her|his|their|the)\s+lips?\b" + r"|\blick(?:s|ed|ing)?\s+(?:her|his|their|the)\s+lips\b" + r"|\bpurs(?:e|es|ed|ing)\s+(?:her|his|their|the)\s+lips\b" + r"|\bbar(?:e|es|ed|ing)\s+(?:her|his|their|the)\s+teeth\b" + r"|\bmouth(?:s|ed|ing)?\s+(?:the\s+)?words?\b" + # "Dana's mouth falls open" is the same sentence as "her mouth falls open" and + # was the one this file contradicted word for word, so the possessive NAME has + # to be a determiner here too. + r"|\b(?:her|his|their|the|[\w-]+['\u2019]s)\s+(?:mouth|jaw)\s+" + r"(?:falls?|fell|drops?|dropped|hangs?|hung|opens?|opened)\b" + r"|\b(?:her|his|their|the|[\w-]+['\u2019]s)\s+lips?\s+(?:parts?|parted)\b", re.I) + + +# WHAT SILENCE IN THE PROMPT ACTUALLY ASKS FOR. +# +# Reported: she smiles at the camera in a situation of duress. Dumped, a four-shot +# scene of a woman handcuffed in the back of a van -- pulling at the cuffs, +# struggling, going limp -- carried NOT ONE WORD about anybody's face. Every clause +# in it was hardware, limbs, or mouths-closed. +# +# An unstated attribute is not a neutral one. The model fills it from its prior, and +# the prior for a named, described person is a PORTRAIT: facing the lens, pleasantly, +# because that is what photographs of people are. This file already knows that half +# of it about the EYES -- it is the entire reason gaze_hold exists -- but gaze_hold +# only fires where the beat NAMES something to look at, which most beats do not, and +# nothing in this node has ever spoken for the expression at all. +# +# So the shot says the one thing the scene has already established. NOT an invented +# emotion: hardware the sheet lists, or the author's own distress verbs. A shot +# staging neither gets nothing, because a node deciding how everybody feels is a node +# writing the film -- which is why this also has a switch. +# +# The distress list is _EXERTION's, minus the ones that are not distress. `laughs` is +# in that list because it is VOCAL, which is all _EXERTION is for; stamping strain on +# a face that was written laughing would be the mouth guard's bug again in a new +# place. `wakes up` is not duress either. +_DISTRESS = re.compile( + r"\b(?:thrash(?:es|ing|ed)?|struggl(?:e|es|ing|ed)|writh(?:e|es|ing|ed)|" + r"strain(?:s|ing|ed)?|squirm(?:s|ing|ed)?|kick(?:s|ing|ed)?|jerk(?:s|ing|ed)?|" + r"sob(?:s|bing|bed)?|cr(?:y|ies|ying|ied)|weep(?:s|ing)?|" + r"scream(?:s|ing|ed)?|shriek(?:s|ing|ed)?|whimper(?:s|ing|ed)?|" + r"beg(?:s|ging|ged)?|plead(?:s|ing|ed)?|" + r"flinch(?:es|ing|ed)?|winc(?:e|es|ing|ed)|recoil(?:s|ing|ed)?|" + r"trembl(?:e|es|ing|ed)|shiver(?:s|ing|ed)?|panic(?:s|king|ked)?|" + r"freak(?:s|ing)?\s+out)\b" + r"|\b(?:goes|went|going)\s+limp\b", re.I) + +# POSITIVELY PHRASED, like everything else here. At cfg 1 there is no negative +# prompt, so "not smiling" would name the smile -- and "unsmiling" is the same word +# with a prefix on it. A mouth that is SET is a thing the model can draw, and it is +# what the shot needs drawn. +# +# It names no camera. Naming one is asking for one, and the lens is exactly what +# this sentence is trying to get her to stop looking at. +# A FILM HAS A MOOD, AND IT DOES NOT BELONG TO ONE CHARACTER'S FACE. +# +# 8418805 gave the restrained person a face and stopped there. Reported back: "the +# last run had them smiling and thinking this was a happy scene, when indeed it was +# not." THEM -- plural. Measured on a five-shot kidnapping: +# +# shot 1 cast= -- NOTHING -- +# shot 2 cast=McKenna strain +# shot 3 cast=McKenna Dan strain <- impersonal, so Dan gets nothing +# shot 4 cast=Dan -- NOTHING -- +# shot 5 cast=Dan -- NOTHING -- +# +# Every shot the captor is alone in had no tone in it at all, and an unstated tone +# is filled from the prior the same way an unstated expression is: the prior for a +# man in a work coat is a pleasant one. The face clause could never reach him -- +# he is not the one under duress and should not look strained. What is wrong with +# those shots is not his expression, it is the whole frame. +# +# So the mood is read from the FILM and said in every shot, and the face clause +# rides on top of it where the person it describes is actually present. One word of +# tone conditions light, faces and framing together, which no per-face sentence can. +DURESS_MOOD = " The mood is grim." +DURESS_FACE = " The mood is grim; the face shows the strain of it, the mouth set." + + +# COERCION -- A KIDNAPPING IS NOT A DISTRESS VERB AND IT IS NOT IN THE SHEET. +# +# Reported: "she is still smiling in every beat, despite this being a kidnapping +# situation that was not defined in the anchor." That last clause is the bug. +# film_stages_duress read binding hardware from the CHARACTER SHEET and distress +# verbs from the beats, and an abduction is neither: nobody writes "McKenna: she, +# 26, handcuffs" for one -- the hardware goes ON during the film, in the beats -- +# and the beats use verbs that were nowhere in the distress list. Ten beats of an +# explicit abduction returned False, so the film had no mood, so every face came +# from the portrait prior, which is pleasant. +# +# THE VERBS NEED A PERSON. Grabbing, dragging, forcing and shoving are all ordinary +# things to do to an OBJECT -- a coffee, a case, a window -- and a film of those is +# not a grim film. So each one has to take a person: a pronoun, or a capitalised +# name. That single requirement is what separates "drags McKenna towards the van" +# from "drags the case to the door", and it is tested both ways. +# THE OBJECT HAS TO BE THE PERSON, NOT SOMETHING THEY OWN. +# +# "grabs her keys", "snatches her coat", "seizes her chance", "forces her way to the +# bar", "pins Ellie's painting to the fridge" -- all read as coercion, because `her` +# and `him` and a name are as often possessives as objects. Measured on 93 ordinary +# domestic beats, this was most of 65 false positives, and one of them is enough to +# stamp "The mood is grim" on a whole comedy. +# +# So the pronoun or name has to be the END of the object: a clause boundary, or one +# of the words that can only follow a completed object. "grabs her from behind" is +# coercion; "grabs her keys" is a Tuesday. +_OBJ_IS_THE_PERSON = ( + r"(?=\s*(?:[.,;:!?\"\u201d]|$)|\s+(?:into|out|off|from|down|up|towards?|to|" + r"against|across|onto|through|back|away|and|by|in|on|over|behind|while|as|" + r"before|after|until|so|but|with|without|aside|apart|hard|roughly|violently|" + r"bodily|clear|free|upright|sideways|forward|backwards?)\b)") + +_COERCION = re.compile( + r"\b(?:grab(?:s|bed|bing)?|drag(?:s|ged|ging)?|forc(?:e|es|ed|ing)|" + r"shov(?:e|es|ed|ing)|haul(?:s|ed|ing)?|bundl(?:e|es|ed|ing)|" + r"seiz(?:e|es|ed|ing)|snatch(?:es|ed|ing)?|pin(?:s|ned|ning)?|" + r"restrain(?:s|ed|ing)?|manhandl(?:e|es|ed|ing)|overpower(?:s|ed|ing)?|" + r"subdu(?:e|es|ed|ing)|wrestl(?:e|es|ed|ing))\s+" + # (?-i:) MATTERS. The whole pattern is case-insensitive, which turned [A-Z] + # into "any letter" and let "drags THE case" and "forces THE window" read as + # coercion. The capital is the only thing separating a name from a determiner. + r"(?:her|him|them|(?-i:[A-Z][\w-]+))" + _OBJ_IS_THE_PERSON + # ...and the phrases that carry it without a bare transitive verb. + + r"|\b(?:holds?|held|holding|pins?|pinned|forces?|forced)\s+" + r"(?:her|him|them|(?-i:[A-Z][\w-]+))\s+(?:down|still|against|into|in)\b" + r"|\bcover(?:s|ed|ing)?\s+(?:her|his|their|[\w-]+['\u2019]s)\s+mouth\b" + + r"|\bagainst\s+(?:her|his|their)\s+will\b" + # PASSIVE VOICE. "She was grabbed from behind", "is bundled into the back", + # "were hauled out of the church" -- the victim is the SUBJECT, so nothing + # follows the verb and every active pattern above misses. This was the single + # largest family of misses: an author writing an abduction reaches for the + # passive precisely because the victim is the one the sentence is about. + # + # The trailing preposition is what keeps "the photo is taken at noon" out: a + # person is taken FROM, INTO, OUT OF, AWAY. A thing is just taken. + r"|\b(?:was|were|is|are|been|being|got)\s+(?:\w+\s+){0,2}?" + r"(?:grabbed|dragged|forced|shoved|hauled|bundled|seized|snatched|pinned|" + r"restrained|manhandled|overpowered|subdued|taken|carried|marched|walked|" + r"loaded|bundled|driven|led)\s+" + r"(?:from|into|out|off|away|down|to|in|through|aboard|across|onto|with)\b" + # CAPTIVITY, which often has no verb of violence in it at all. + r"|\bheld\s+(?:captive|prisoner|hostage)\b" + r"|\b(?:captors?|hostages?|abduction|kidnapping)\b" + r"|\b(?:held|taken|kept)\s+captive\b" + r"|\blocked\s+(?:in|inside|up)\b" + r"|\bkidnap(?:s|ped|ping)?\b|\babduct(?:s|ed|ing|ion)?\b|\bhostage\b" + # Trying to get out is duress by definition. + r"|\btr(?:y|ies|ied|ying)\s+to\s+(?:get\s+away|get\s+out|escape|run|pull\s+free)\b" + r"|\b(?:break(?:s|ing)?|broke|pull(?:s|ed|ing)?)\s+free\b" + r"|\bescap(?:e|es|ed|ing)\b", re.I) + +# Hardware being APPLIED, in a beat. The same words as _BOUND_HARDWARE plus the +# forms an action uses -- a sheet says "tied", a beat says "ties" -- and each one +# still has to reach a person or a part of one, so taping a box shut is not an +# abduction. +# WHAT A RESTRAINT IS PUT ON: A BODY. +# +# The first version of this asked whether binding words appeared near a person, and +# a sweep of 512 beats showed what English does with those words when nobody is +# being restrained at all: +# +# She is bound for Lisbon on the early flight. +# At full time it is still tied at two apiece. +# She's tied up in meetings until four. +# He has been chained to that desk for eleven years. +# The ledger is bound in green cloth. +# The boat is tied up at the jetty. +# He gagged at the smell coming off the bins. +# He pulls the hood on his parka up against the drizzle. +# +# Every one read as STRONG evidence, and one strong beat is enough to stamp "The +# mood is grim" on a whole film. `bound`, `tied`, `chained` and `gagged` are all +# idioms before they are restraints. +# +# So the state forms are gone. Binding has to reach a BODY PART, or a person plus +# the furniture people actually get tied to. That is what a restraint is; the rest +# is a figure of speech. +_BINDABLE = (r"wrists?|ankles?|hands|feet|legs?|arms?|mouth|thumbs?|knees|elbows") +_TIE_TO = (r"chair|bed|bedframe|headboard|radiator|pipe|post|stake|banister|" + r"bannister|frame|hook|ring|beam|column|tree") +_BINDING_ACT = re.compile( + # ties her wrists, cuffs his ankles, gags her, tapes McKenna's mouth + r"\b(?:ties?|tying|tied|bind(?:s|ing)?|bound|cuff(?:s|ed|ing)?|" + r"shackl(?:e|es|ed|ing)|chain(?:s|ed|ing)?|zip-?ti(?:e|es|ed))\s+(?:up\s+)?" + r"(?:her|his|their|(?-i:[A-Z][\w-]+)'s)\s+(?:" + _BINDABLE + r")\b" + # taping is strapping unless it reaches a mouth, or wrists held together + r"|\btap(?:e|es|ed|ing)\s+(?:up\s+)?(?:her|his|their|(?-i:[A-Z][\w-]+)'s)\s+" + r"(?:mouth\b|(?:wrists?|ankles?|hands)\s+(?:together|behind|to)\b)" + # wrists cable-tied, ankles taped -- the participle fragment + r"|\b(?:" + _BINDABLE + r")\s+(?:\w+\s+){0,2}?" + r"(?:tied|taped|cuffed|bound|chained|shackled|zip-?tied|strapped)\b" + # tied TO the things people get tied to + r"|\b(?:tied|cuffed|bound|shackled|chained|strapped|handcuffed)\s+" + r"(?:her|him|them|(?-i:[A-Z][\w-]+)\s+)?(?:to|against)\s+" + r"(?:the|a|an|that|this|his|her|their)\s+(?:" + _TIE_TO + r")\b" + # hardware named as being ON a body + r"|\b(?:handcuffs?|cuffs|rope|ropes|cord|cords|chains?|shackles|zip\s*ties?|" + r"cable\s*ties?|duct\s*tape|tape|gag|blindfold)\s+(?:\w+\s+){0,2}?" + r"(?:on|around|round|over|across|behind)\s+" + r"(?:her|his|their|(?-i:[A-Z][\w-]+)'s|the)\s+(?:" + _BINDABLE + r"|head|eyes|face)\b" + # A person gagged -- the person, not a smell he gagged at. + r"|\bgag(?:s|ged|ging)\s+(?:her|him|them|(?-i:[A-Z][\w-]+))\b" + # A bag or hood put over SOMEBODY ELSE'S head. Bare `hooded` and `blindfolded` + # are out: a hooded parka, a hooded dressing gown, a hooded teenager and a + # blindfold wine tasting all read as abduction, and one strong beat is enough + # to call a whole film grim. The article is what carries it -- "a hood over her + # head" is done TO her, "her hood over her head" is her own coat in the rain. + r"|\b(?:a|the|another)\s+(?:bag|hood|sack|pillowcase)\s+over\s+" + r"(?:her|his|their|(?-i:[A-Z][\w-]+)'s|the)\s+head\b" + r"|\bblindfold(?:s|ed|ing)?\s+(?:her|him|them|(?-i:[A-Z][\w-]+))\b" + # A PERSON in a bound state. `tied` and `chained` are left out of this one + # deliberately -- "her hands are tied, politically speaking", "he has been + # chained to that desk for eleven years" -- and `bound` needs guarding against + # the commonest idiom of all, which is a departure board. + r"|\b(?:she|he|they|(?-i:[A-Z][\w-]+))\s+(?:\w+\s+){0,2}?" + r"(?:is|are|was|were|had\s+been|has\s+been|got)\s+(?:\w+\s+){0,2}?" + r"(?:bound(?!\s+for\b)|gagged|cuffed|handcuffed|shackled)\b", + re.I) + + +# STRONG EVIDENCE AND WEAK EVIDENCE, because English will not do better. +# +# Swept across 512 beats of six scenario families, the distress list alone produced +# 87 false positives, and they are not fixable by patching it: +# +# The children scream all the way down the waterslide. +# The baby cries in the next room. +# She strains to hear the platform announcement. +# She winces at the price and buys it anyway. +# She screams with laughter as the boat slaps down off the wake. +# +# `screams`, `cries`, `strains`, `winces`, `panics` and `begs` mean distress or they +# mean a good day out, and no pattern can tell which from the words alone. What CAN +# tell is the rest of the film. So the evidence is graded: +# +# STRONG -- says duress on its own and is almost never innocent: hardware on a +# body, a captor, a hostage, an abduction, being locked in, something +# done against somebody's will. +# WEAK -- an ambiguous verb: the distress words, and the ordinary coercion verbs +# that are equally at home in a garden centre. +# +# A film is grim if ANY beat is strong, or if TWO are weak. One ambiguous verb is +# not enough to stamp "The mood is grim" on somebody's comedy; two is little enough +# that a real abduction -- which is nothing but coercion verbs -- always lands. +# +# And a WEAK beat gets the face clause only in a film already established as grim. +# That is the point of grading: "she screams" is terror in an abduction and delight +# on a waterslide, and the film is the only thing that knows which. +_DURESS_STRONG = re.compile( + r"\bheld\s+(?:captive|prisoner|hostage)\b" + r"|\b(?:captors?|hostages?|abduction|kidnapping)\b" + r"|\b(?:held|taken|kept)\s+captive\b" + r"|\bkidnap(?:s|ped|ping)?\b|\babduct(?:s|ed|ing|ion)?\b" + r"|\blocked\s+(?:in|inside|up)\b" + r"|\bagainst\s+(?:her|his|their)\s+will\b", re.I) + + +def beat_duress_strength(beat): + """'' , 'weak' or 'strong'. See _DURESS_STRONG for why the grading exists.""" + b = beat or "" + if _DURESS_STRONG.search(b) or _BINDING_ACT.search(b): + return "strong" + if _DISTRESS.search(b) or _COERCION.search(b): + return "weak" + return "" + + +def beat_stages_duress(beat, film_duress=True): + """Does this BEAT stage duress? + + Strong evidence always counts. Weak evidence counts only where the FILM is + already grim, because that is the context that says which meaning an ambiguous + verb has. Defaults to True so a caller asking about a beat in isolation gets + the old, generous reading.""" + strength = beat_duress_strength(beat) + return strength == "strong" or (strength == "weak" and bool(film_duress)) + + +# THE AUTHOR CAN JUST SAY IT, AND THAT BEATS ANY AMOUNT OF GUESSING. +# +# Swept across 512 beats of six scenario families, inference alone does not work and +# the numbers say so plainly. Simulating 8-beat films: +# +# evidence needed duress films read grim ordinary films read grim +# 2 weak 73.5% 87.8% +# 3 weak 57.1% 82.5% +# strong only 44.5% 28.5% +# +# At every setting an ORDINARY film was as likely to be called grim as a duress one, +# because the words overlap: `screams` is a waterslide, `tied` is a boat, `bound` is +# a flight to Lisbon, `chained` is a desk job. That is not a pattern that needs more +# work, it is English, and no bag of patterns is going to separate them. +# +# So the ANCHOR is asked first. It is already the film-wide declaration -- "framing +# that belongs to the whole film" -- and a tone belongs there beside the lighting. +# Said there, it is authoritative in BOTH directions: a film declared warm is never +# given a grim mood however its beats read, and that is the escape hatch for every +# false positive above. +# +# Only where the anchor says nothing does this fall back to inference, and then only +# on STRONG evidence, because an unasked-for grim mood on somebody's comedy is a +# visible defect while a missing one is recoverable by typing six words. info says +# which of the three happened every run. +_MOOD_GRIM = re.compile( + r"\b(?:grim|bleak|tense|menacing|sinister|harrowing|distressing|brutal|" + r"frightening|terrifying|desperate|oppressive|claustrophobic|ominous|" + r"threatening|violent|grave|sombre|somber|dread|hostile|cruel|" + r"kidnap(?:ping)?|abduction|captivity|hostage|abusive|coercive)\b", re.I) +_MOOD_LIGHT = re.compile( + r"\b(?:warm|comic|comedy|cheerful|joyful|joyous|happy|light[-\s]?hearted|" + r"playful|romantic|tender|sunny|upbeat|gentle|affectionate|celebratory|" + r"whimsical|carefree|domestic\s+bliss|feel[-\s]?good)\b", re.I) + + +def mood_declared(anchor): + """'grim', 'light' or '' -- what the ANCHOR says the film's tone is. + + Both directions matter. A film declared warm must never be handed a grim mood + however its beats read, because that is the one reliable way out of a wrong + inference; and a film declared grim needs no inference at all.""" + a = anchor or "" + if _MOOD_LIGHT.search(a): + return "light" + if _MOOD_GRIM.search(a): + return "grim" + return "" + + +def film_stages_duress(beats, sheet="", anchor=""): + """Does this FILM stage duress anywhere -- binding hardware, or a distress verb? + + Read once, over the whole script, because a shot of the captor alone is grim on + account of what is on her wrists three beats ago. The same two signals the face + clause uses, and the same refusals: a collar alone is not duress, and a film + that stages neither is left alone in every shot. The node does not get to decide + that somebody's film is bleak.""" + said = mood_declared(anchor) + if said: + return said == "grim" + for _, ln in sheet_lines(sheet or ""): + if _BOUND_HARDWARE.search(ln or ""): + return True + return any(beat_duress_strength(b) == "strong" for b in (beats or [])) + + +# BINDING hardware, which is narrower than restraint hardware. restraint_present is +# right for the continuity holds -- a collar is a thing that must stay fastened and +# stay the object it was -- but it is not evidence of DURESS. A collar is worn in +# scenes that are not distressing at all, and stamping strain on a face in one of +# those is the same error as stamping a closed mouth on a grin. Cuffs, rope, chain, +# tape and a gag are not ambiguous that way. +_BOUND_HARDWARE = re.compile( + r"\b(?:handcuffs?|cuffs?|shackles?|manacles?|irons|" + r"ropes?|cords?|twine|zip\s*ties?|cable\s*ties?|" + r"chains?|chained|tape|taped|gag|gagged|bound|tied|bindings?)\b", re.I) + + +def duress_face(beat, wearers, described, film_duress=False): + """One short sentence about the face, on a shot whose scene already stages duress. + + IMPERSONAL, the choice gaze_hold already made and for the same reason: a named + person is a person the model draws, and naming somebody twice in one shot is what + put a second girl in frame at the moment of cuffing. On the shot where that could + be ambiguous -- two people, one of them restrained -- the hardware hold has + already said "Every restraint on Nora", so the shot is not short of an + attribution. It is short of a sentence about her face.""" + # THE AUTHOR'S OWN EMOTION WINS, and it is said back in THEIR word rather than + # the film's generic mood. "The mood is grim; ... the mouth set" was being stamped + # over "terrified", which is a different performance, and over a happy scene it + # fired not at all -- so the emotional register was only ever asserted in one + # direction and only ever generically. Said wherever the beat names a feeling, + # duress or not, which is why this sits ahead of every duress test below. + _emotion = emotion_in(beat) + if _emotion and described: + # ONE PERSON, NOBODY ELSE IT COULD BE. More than one and the feeling has to be + # pinned, or the sentence lands on every face in the shot -- and a feeling the + # beat pins on nobody holds nobody, exactly as a vocal does: guessing which of + # two faces wears it is how the captor came to look terrified. + if len(described) < 2: + return mood_face(_emotion) + _pairs = emotion_pairs(beat, described) + return mood_faces(_pairs) + # The author's own face beat wins, exactly as it does against the mouth guard. + # Where the beat says what the face is doing, the node has nothing to add. + if mouth_performs(beat): + return "" + if not described: + return "" + who = [n for n, ln in (wearers or []) if n and _BOUND_HARDWARE.search(ln or "")] + if not who and beat_stages_duress(beat, film_duress): + who = [n for n in (described or []) if n] + if who: + return DURESS_FACE + # Nobody under duress IN THIS SHOT, but the film is. The frame still is not a + # happy one, and saying nothing is what let the captor smile through it. + return DURESS_MOOD if film_duress else "" + + +# AN EMOTION THE AUTHOR STATED, in their own word. +# +# Reported: under duress she does not act or respond like it, and the same in scenes +# where she is supposed to be happy. Measured, the node was contradicting the beat in +# both directions at once. "Mia hugs Tess, beaming." came out with "Mouths in the shot +# stay closed" beside it, because beaming was in no list. "McKenna is terrified and +# shaking." came out with "The mood is grim; the face shows the strain of it, the mouth +# set" -- a generic, clenched, stoic face stamped over the specific word the author +# chose, and then a mouth guard on top of that. +# +# An emotion is performed largely WITH THE MOUTH: delight is a smile, terror is an open +# mouth, fury is bared teeth. A guard that closes the mouth closes the performance, and +# at cfg 1 the flat positive instruction wins over the adjective in the beat. +# +# Read as a stand-down and as a register, never as an invention: where the author names +# no feeling, nothing here fires and the film's own mood clause is untouched. +_EMOTION = re.compile( + r"\b(?:happy|happily|happiness|delighted|delight(?:ed)?|thrilled|overjoyed|" + r"joyful|joyous|elated|ecstatic|beaming|beams?|gleeful|glee|cheerful|cheery|" + r"pleased|excited|excitement|grateful|relieved|relief|proud|smug|amused|" + r"terrified|terror|frightened|afraid|scared|fearful|panicked|panicking|panic|" + r"furious|fury|angry|angrily|anger|enraged|livid|seething|indignant|" + r"desperate|desperation|distraught|devastated|grief|grieving|heartbroken|" + r"miserable|wretched|ashamed|shame|humiliated|mortified|disgusted|horrified|" + r"anguished|anguish|agony|bereft|despair(?:ing)?)\b", re.I) + + +def emotion_in(beat): + """The emotion this beat states, in the author's own word. "" when it states none.""" + m = _EMOTION.search(str(beat or "")) + return m.group(0).lower() if m else "" + + +def emotion_owner(beat, names, word): + """Whose feeling it is: the person the beat puts in front of it. "" if nobody. + + The shape subjects_for uses, conjunction guard included, so "Dan holds the door + and McKenna is terrified" does not hand the terror to Dan. Takes a name list + rather than a sheet because the caller already has the shot's cast.""" + b = str(beat or "") + for n in (names or []): + if n and re.search(r"\b" + re.escape(n) + r"\b" + r"(?:\s+(?!and\b|but\b|then\b|who\b|,\s*who\b)[\w,']+){0,2}?" + r"\s+(?:is|was|looks?|looked|seems?|feels?|felt|sounds?|" + r"becomes?|became|goes|went|turns?|gets?|got)?\s*" + + re.escape(word) + r"\b", b, re.I): + return n + return "" + + +def emotion_pairs(beat, names): + """[(who, feeling)] for the feelings this beat pins on people. Two at most. + + TWO PEOPLE CAN FEEL DIFFERENT THINGS IN ONE SHOT. "Dan is furious and McKenna is + terrified" gave only the first of them, so one face was performing and the other + was left to the prior -- the same half-fix as naming one of two speakers. Two at + most, like the layering clause: a shot carrying four feelings has stopped being + about its beat.""" + out, seen = [], set() + for m in _EMOTION.finditer(str(beat or "")): + word = m.group(0).lower() + who = emotion_owner(beat, names, word) + if who and who not in seen: + seen.add(who) + out.append((who, word)) + if len(out) >= 2: + break + return out + + +def mood_faces(pairs): + """Say whose feeling is whose, for one or two people. "" for none.""" + ps = [(w, e) for w, e in (pairs or []) if w and e] + if not ps: + return "" + if len(ps) == 1: + return mood_face(ps[0][1], ps[0][0]) + return (f" {ps[0][0]}'s face carries {ps[0][1]} and {ps[1][0]}'s carries " + f"{ps[1][1]}, each played in the eyes and the mouth.") + + +def mood_face(word, who=""): + """Say the face plays the feeling the author named. "" when they named none. + + Their word, not a synonym: "terrified" and "grim" are not the same performance, + and the generic one was replacing the specific one on every shot. + + NAMED ONCE A SECOND PERSON IS IN THE SHOT, which is the call gaze_hold already + makes for the same reason. Said impersonally, "the face carries it: the expression + is terrified" is a sentence about whoever is on screen -- so in a two-hander the + captor wore his victim's terror. Reported as actions being performed by all the + characters at once. With one person there is nobody else it could be, and naming + them again is a second mention of a person, which has its own cost.""" + if not word: + return "" + if who: + return (f" {who}'s face carries it: the expression is {word}, played in the " + f"eyes and the mouth together.") + return (f" The face carries it: the expression is {word}, played in the eyes and " + f"the mouth together.") + + +def mouth_performs(beat): + """Does the beat itself put the MOUTH to work? + + The beat has already said what the mouth does, so the guard has nothing to add + over the top -- and what it was adding contradicted it. Same shape as the LONE + vocal that sounds_for leaves alone: where the author wrote it, the node is + quiet.""" + return bool(_MOUTH_WORKS.search(beat or "")) _PERSON_WORD = re.compile( r"\b(?:he|she|they|him|her|hers|them|his|their|theirs|himself|herself|themselves|" @@ -4190,12 +4899,7 @@ def restraint_going_on(beat): return bool(_APPLY_NOW.search(b) or _APPLY_PHRASE.search(b)) -# COMPRESSED, 2026-09-05. This said "whole and closed", "fastened exactly as it was -# put on" and "still fastened at the last frame" -- three ways of saying closed -- -# and then the material clause on top. 32 words. Measured on a real scene the -# guards had reached 65% of the shot against a 12% beat, which is the number this -# node was rebuilt to escape and the number RESTRAINT_HOLD's own comment warns -# about. Every guarantee is still here; each is stated once. +# Keep this compact: it is repeated in every shot while restraints remain present. RESTRAINT_HOLD = (" Every restraint stays closed and fastened as it was put on") + FORM_HOLD @@ -4207,51 +4911,8 @@ def restraint_wearers(sheet): return [n for n, ln in sheet_lines(sheet) if n and restraint_present(ln)] -# A ceiling on continuity text, in words, relative to the beat it is standing next to. -# -# This node was rebuilt once because the guards had buried the action: the author's -# beat was under 4% of a 434-word prompt. It happened again by the ordinary route -- -# a clause per bug report, each one justified on its own, none of them counting the -# others. Measured on a real scene the guards were 65% against a 12% beat, and the -# symptom is not subtle: the shot stops doing what the beat says. Somebody does not -# sit in the chair they were told to sit in. -# -# So the clauses are ranked and the low-priority end is dropped when there is no room, -# rather than every clause being emitted because each was a good idea in isolation. -# The floor exists so a very short beat still gets its single most important guard. -# Set to catch RUNAWAY, not to trim routinely. Measured against the same scene before -# this session's clause work, a shot carried 71 words of prompt; merging the three -# hardware clauses into one and shortening the gaze and mouth lines brought the worst -# shot from 124 words back to 59, which is already under that baseline. A tight budget -# on top of that was dropping guards that exist because of real reports -- the mouth -# holds, the revealed layer, the limb anchor -- and trading one set of bugs for -# another. The ceiling is here so the next clause added without counting the others -# cannot quietly rebuild the pile; it is not the thing doing the work. -# TIGHTENING THIS WAS TRIED, MEASURED, AND REJECTED. Recorded here so it is not -# proposed again from the balance report alone -- the report says the guards -# outweigh the beat, which is true, and reads like slack, which it is not. -# -# Swept against the suites, which are the record of what was actually reported: -# -# 90/5 (this) worst shot 116 words beat 14% 0 suite failures -# 80/5 2 -- the fall/landing guard, the -# bound-fall wording -# 75/4 3 -- ...and the forced position -# 65/4 3 -# 60/4 6 -- ...and the rigid-metal hold -# 55/3 worst shot 87 words beat 16% 8 -- ...and both ends of applying -# -# Every clause the budget reaches is answering a report. There is no fat: two -# points of beat share cost the fall guard, the restraint holds and the posture -# hold, which is trading one set of bugs for another. With the sound clause ranked -# last (see _guards) a tighter floor drops THAT first instead, and on a shot whose -# audio branch is open the sound clause is the text half of the babble defence -- -# the failure reported more often than any other here, and one no test asserts, -# so the suites would have gone green on it. -# -# So the floor stays a runaway catcher and is not the thing doing the work. What -# changed is that the sound clause now SPENDS from it, so the next clause added -# without counting the others cannot rebuild the pile the way sound quietly did. +# Bound continuity text so it cannot overwhelm the authored beat. Clauses are +# ranked by the caller; the floor preserves essential guards for very short beats. GUARD_FLOOR_WORDS = 90 GUARD_WORDS_PER_BEAT_WORD = 5 @@ -4277,6 +4938,205 @@ def fit_guards(clauses, beat_words): return kept, dropped +# PEOPLE THE BEAT STAGES WHO ARE ON NOBODY'S SHEET. Extras: a crowd, dancers, +# other girls, two men at the bar. Plural nouns only, and deliberately not "both", +# "they" or "the two of them" -- those are group cues about the NAMED cast and +# group_beat already owns them. A singular "someone" is not here either: one more +# person is what the cast clause is already counting. +_EXTRA_PEOPLE = re.compile( + r"\b(?:crowds?|groups?|others|onlookers|bystanders|passers-?by|spectators|" + r"people|dancers|guests|customers|patrons|strangers|students|staff|tourists|" + r"girls|women|men|boys|guys|ladies|blondes|brunettes|figures|silhouettes)\b", + re.I) + + +# PEOPLE MENTIONED ARE NOT PEOPLE STAGED. "They hear people outside" puts nobody in +# the frame and "the others have gone" says the opposite of staging them -- and both +# used to count, which then stood the body-count clause down and let a random into +# every later shot. A plural noun in an absence or an offscreen phrase is not a crowd. +_NOT_STAGED = re.compile( + r"\b(?:gone|left|leaving|went|departed|vanished|absent|empty|alone|" + r"outside|elsewhere|away|upstairs|downstairs|next\s+door|beyond|" + r"no\s+one|no[- ]?body|none|without|hears?|heard|hearing|listens?|" + r"remembers?|imagines?|thinks?\s+of|expects?|waits?\s+for)\b", re.I) + + +# ...and what says they have GONE. The latch below needs an explicit way out, the way +# every other state in this file has one: a garment comes off, a restraint is unlocked, +# a room is left. Without it, extras staged once would suppress the body count for the +# rest of the film even after the script empties the room. +_ALONE = re.compile( + r"\b(?:alone|by\s+(?:her|him|them)self|on\s+(?:her|his|their)\s+own|" + r"empty|deserted|to\s+(?:her|him|them)self)\b", re.I) + + +def extras_in(beat): + """Does this beat stage people beyond the ones the sheet names, IN the frame?""" + b = str(beat or "") + if not _EXTRA_PEOPLE.search(b): + return False + return not _NOT_STAGED.search(b) + + +def extras_dismissed(beat): + """Does this beat say the people the sheet does not name are no longer there?""" + b = str(beat or "") + if _ALONE.search(b): + return True + return bool(_EXTRA_PEOPLE.search(b) and _NOT_STAGED.search(b)) + + +# WHO IS IN CONTACT WITH WHOM. +# +# Reported: girls kissing each other when they should be kissing boys. The beat said +# "Mia kisses Dan while Tess kisses Jon" and that is ALL the shot said about it -- one +# sentence among four appearance descriptions, and at cfg 1 the model reads the prompt +# as a bag of words and pairs by its own prior. This file names the owner of a gaze, a +# vocal, a feeling, a posture, a restraint and a body count; contact was the one +# relationship nothing restated. +# +# THE OBJECT HAS TO BE A NAME ON THE SHEET, which is what makes the verb list safe to +# be generous with: "holds the door" and "pulls the chain" name no person and yield no +# pair, so hold, pull, grab and take can all be here without reading furniture as a +# partner. +_CONTACT_SRC = ( + r"kiss(?:es|ed|ing)?|hug(?:s|ged|ging)?|embrac(?:e|es|ed|ing)|" + r"straddl(?:e|es|ed|ing)|mount(?:s|ed|ing)?|caress(?:es|ed|ing)?|" + r"strok(?:es|ed|ing)?|cuddl(?:e|es|ed|ing)|hold(?:s|ing)?|held|" + r"grab(?:s|bed|bing)?|touch(?:es|ed|ing)?|caught|catch(?:es|ing)?|" + r"pull(?:s|ed|ing)?|take[sn]?|took|taking|push(?:es|ed|ing)?|" + r"danc(?:e|es|ed|ing)\s+with|lean(?:s|ed|ing)?\s+(?:on|against|into)|" + r"press(?:es|ed|ing)?\s+(?:against|into)|sit(?:s|ting)?\s+on|" + r"wraps?\s+(?:her|his|their)\s+arms?\s+around|" + r"reach(?:es|ed|ing)?\s+for|undress(?:es|ed|ing)?") +_CONTACT_VERB = re.compile(r"(?:" + _CONTACT_SRC + r")", re.I) +# A clause boundary for contact: each pair gets its own, so "A kisses B while C kisses +# D" is read as two pairs rather than one four-way. +_CONTACT_SPLIT = re.compile(r"(?<=[.;!?])\s+|\s+\b(?:while|as|and|then)\b\s+|,\s+", re.I) + + +def contact_pairs(beat, names): + """[(who, whom)] the beat puts in physical contact. Two at most. + + Two, like the layering clause: a shot restating four pairings has stopped being + about its beat.""" + b = _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", str(beat or ""))) + people = [n for n in (names or []) if n] + out = [] + for part in _CONTACT_SPLIT.split(b): + for a in people: + m = re.search(r"\b" + re.escape(a) + r"\b\s+(?:\w+\s+){0,2}?(?:" + + _CONTACT_SRC + r")\b", part, re.I) + if not m: + continue + tail = part[m.end():] + for c in people: + if c == a: + continue + if re.match(r"\W{0,14}(?:the\s+)?" + re.escape(c) + r"\b", tail, re.I): + if not any({a, c} == set(p) for p in out): + out.append((a, c)) + break + if len(out) >= 2: + break + return out[:2] + + +def contact_hold(pairs): + """Say which body is with which. "" when the beat pairs nobody. + + Positive, like every other clause here: it says what the pairing IS, never that + anybody is not paired. Naming both sides is the point -- an unnamed "they kiss" in + a shot with four people is the sentence that let the model choose.""" + ps = [(a, b) for a, b in (pairs or []) if a and b] + if not ps: + return "" + if len(ps) == 1: + return f" The contact is {ps[0][0]} with {ps[0][1]}: those two bodies together." + return (f" The contact is {ps[0][0]} with {ps[0][1]}, and {ps[1][0]} with " + f"{ps[1][1]}: two pairs, each body with its own partner.") + + +# WHAT LORA IS ON THIS RUN, which nothing here could see before. +# +# Reported: character duplicates that survive every guard in this file. A LoRA is +# the one input to a shot the node does not write and cannot read out of the text, +# and it is invisible in the output: two runs whose prompts are identical render +# differently and nothing says why. +# +# ComfyUI keeps the patches on the patcher -- `patches` maps a weight name to the +# list of (strength, delta, ...) tuples applied to it, one entry per LoRA that +# touched that weight -- so how many are stacked, how strongly, and whether the +# TEXT ENCODER carries them too can all be read off the objects this node is +# handed. Names are not kept there; the last LoRA's safetensors metadata is, under +# the "lora_metadata" attachment, and that usually carries one. +def lora_facts(patcher): + """(stacked LoRAs, weights touched, [strengths]) for a model or a CLIP patcher.""" + patches = getattr(patcher, "patches", None) + if not isinstance(patches, dict) or not patches: + return (0, 0, []) + stacked = max((len(v) for v in patches.values() if isinstance(v, (list, tuple))), default=0) + strengths = [] + for entries in patches.values(): + for entry in entries if isinstance(entries, (list, tuple)) else []: + try: + value = round(float(entry[0]), 3) + except (TypeError, ValueError, IndexError): + continue + if value not in strengths: + strengths.append(value) + return (stacked, len(patches), sorted(strengths, reverse=True)) + + +def lora_name_of(patcher): + """The last-applied LoRA's own name, if its metadata carries one.""" + meta = getattr(patcher, "attachments", {}) or {} + meta = meta.get("lora_metadata") if isinstance(meta, dict) else None + if not isinstance(meta, dict): + return "" + for key in ("modelspec.title", "ss_output_name", "ss_session_id"): + value = str(meta.get(key) or "").strip() + if value: + return value + return "" + + +def cast_hold(names, beat="", extras=False): + """A positive body-count constraint for a one- or two-person composition. + + THE SOLO SHOT HAD NO COUNT AT ALL, and a solo shot is where a duplicate of the + one person in it has nothing standing against it. The pair case has been asserted + since this function was written; the one-person case returned "" with no recorded + reason for it, so the shot most at risk of being rendered as twins was the shot + that said nothing about how many bodies were in it. Reported, repeatedly, as + duplicate characters. + + STANDS DOWN WHERE THE BEAT STAGES EXTRAS. "There are two people in the shot, + with one body for each person" is exactly right against a duplicated character + and exactly wrong against "two women dance behind them": it forbids, as a + positive fact, the people the author just asked for. Reported as extras refusing + to appear. The author's words outrank anything inferred from them, which is this + file's standing rule, so a beat that puts more bodies in the frame keeps them and + the count goes unsaid.""" + people = list(dict.fromkeys(n for n in (names or []) if n)) + # `extras` is kept as a parameter so a caller can stand the count down explicitly. + # It is no longer LATCHED for the film: one plural word anywhere -- "the others + # have gone" included -- then silenced the count on every shot that followed, and + # this is the clause that keeps a duplicate or a stranger out of the frame. + if extras or extras_in(beat): + return "" + if len(people) == 1: + return " There is one person in the shot: one body, one face." + if len(people) == 2: + return " There are two people in the shot, with one body for each person." + # THREE OR MORE IS LEFT ALONE, as it always has been. The count comes from the + # cast this file decided is in the shot, and the more people that decision holds + # the likelier one of them is described without being in frame -- an assertion + # that there are four bodies is then a request for a fourth. One and two are the + # counts the duplicate reports are about. + return "" + + def restrained_by_beat(beat, cast): """Who this beat puts in the hardware. The agent is not the one wearing it. @@ -4487,7 +5347,7 @@ def restraint_sentence(item, wearers, described, anchor="", rigid=False, posed=F elif rigid: out += (f", {'their' if plural else 'its'} links keeping their size and the run " f"between them taut") - out += f", the same object in the same material." + out += ", the same object in the same material." if who: out += " Everyone else in the shot has on exactly what their own entry lists." return out @@ -4520,8 +5380,8 @@ def own_body(clause, who, described): body = re.sub(r"^The\s+", f"{subject}'s ", body) body = re.sub(r"^Everything worn\b", f"Everything {subject} is wearing", body) return (" " + body - + f" Everyone else in the shot keeps on exactly what their own entry " - f"lists.") + + " Everyone else in the shot keeps on exactly what their own entry " + "lists.") def own_hold(hold, wearers, described): @@ -4806,10 +5666,38 @@ _FORCED_POSE = re.compile( # carrying the position was the picture -- and the picture is the previous shot's last # frame, which a close shot crops the anchor point straight out of. Text is the only # thing that survives a tight frame. +# What makes a phrase describe a BODY rather than the room it is in: a limb, or a +# word for fastening one. Every entry below requires one of these within a few +# words of the position, because without it the table reads the set dressing -- +# "one bulb overhead" put the wrists above the head, "crates stacked to the sides" +# put the arms out to the sides, and "her legs spread wide" moved the arms to +# wherever the legs were. limb_anchor takes the FIRST pattern that matches, so an +# unguarded entry does not merely add a wrong reading, it outranks the right one +# written in the same sentence. +_LIMB_EV = (r"\b(?:hands?|wrists?|arms?|cuffed|handcuffed|bound|tied|shackled|" + r"manacled|strapped|secured|fastened|locked|pinned|chained|clasped|" + r"held|clipped|hooked)") + _LIMB_ANCHOR = ( - (r"(?:above|over)\s+(?:her|his|their|the)\s+head|overhead|" - r"stretched\s+(?:up|upward)", "above the head"), - (r"behind\s+(?:her|his|their|the)\s+back", "behind the back"), + # EVERY FORM HERE CARRIES ITS OWN EVIDENCE, the same rule the "behind" entries + # below already follow. It did not, and bare "overhead" and "stretched up" are + # scenery far more often than they are limbs: "one bulb overhead", "strip lights + # overhead", "the cable is stretched up the wall". limb_anchor takes the FIRST + # pattern that matches, and this is the first, so a light fitting in the scene + # line beat the wrists written in the same sentence -- a woman cuffed behind her + # back was told, in every shot, that both arms were raised above her head. + # Reported as the cuffs breaking and the arms coming round to the front, which is + # what a model does when the pose it is given contradicts the hardware. + (r"(?:cuffed|handcuffed|bound|tied|shackled|manacled|strapped|secured|" + r"fastened|locked|pinned|chained|clipped|hooked|suspended|hoisted)\s+" + r"(?:\w+\s+){0,3}?(?:above|over)\s+(?:her|his|their|the)\s+head|" + r"(?:hands?|wrists?|arms?)\s+(?:\w+\s+){0,4}?" + r"(?:above|over)\s+(?:her|his|their|the)\s+head|" + r"(?:hands?|wrists?|arms?)\s+(?:\w+\s+){0,3}?overhead|" + r"(?:hands?|wrists?|arms?)\s+(?:\w+\s+){0,2}?stretched\s+(?:up|upward)", + "above the head"), + (_LIMB_EV + r"\s+(?:\w+\s+){0,3}?behind\s+(?:her|his|their|the)\s+back", + "behind the back"), # THE SAME PLACE, WRITTEN THE WAYS PEOPLE WRITE IT. The line above needs the # literal word "back" after the possessive, so every one of these recorded # NOTHING -- and nothing here is not a smaller clause, it is pose_clause @@ -4834,9 +5722,15 @@ _LIMB_ANCHOR = ( r"behind\s+(?:her|his|their)\b", "behind the back"), (r"at\s+the\s+small\s+of\s+(?:her|his|their|the)\s+back", "behind the back"), (r"\b(?:hands?|wrists?|arms?)\s+behind\s+back\b", "behind the back"), - (r"in\s+front\s+of\s+(?:her|his|their)\s+(?:body|chest|waist)", "in front of the body"), - (r"(?:out\s+)?to\s+the\s+sides?|spread\s+wide", "out to the sides"), - (r"at\s+(?:her|his|their|the)\s+waist", "at the waist"), + (_LIMB_EV + r"\s+(?:\w+\s+){0,3}?in\s+front\s+of\s+(?:her|his|their)\s+" + r"(?:body|chest|waist)", "in front of the body"), + # "Her legs spread wide" was anchoring her ARMS out to the sides, and + # "crates stacked to the sides" did the same from the scenery. Legs are not + # arms and a crate is not a limb. + (_LIMB_EV + r"\s+(?:\w+\s+){0,3}?(?:(?:out\s+)?to\s+the\s+sides?|spread\s+wide)", + "out to the sides"), + (_LIMB_EV + r"\s+(?:\w+\s+){0,3}?at\s+(?:her|his|their|the)\s+waist", + "at the waist"), ) # What they are fastened TO. Named separately because a shot can state one, the other, # or both, and the clause reads correctly with whichever it has. @@ -4916,6 +5810,113 @@ _TIGHT_FRAME = re.compile( r"\bfills?\s+the\s+frame\b|\bmacro\b", re.I) +# A WHOLE BODY DOING SOMETHING. Not a face acting, and not a hand: these are the +# verbs whose action does not fit inside a portrait. +_WHOLE_BODY = re.compile( + r"\b(?:serves?|serving|throws?|throwing|kicks?|kicking|hits?|hitting|" + r"swings?|swinging|spikes?|blocks?|blocking|jumps?|jumping|runs?|running|" + r"sprints?|sprinting|dances?|dancing|plays?|playing|climbs?|climbing|" + r"lifts?|lifting|carries|carrying|pushes|pushing|pulls?|pulling|" + r"swims?|swimming|stretches|stretching|wrestles?|fights?|fighting)\b", re.I) +# A SHOT SIZE, which is the only kind of camera note that answers "how much of the +# person is in frame". This began as any camera word at all and that was wrong in the +# one way that mattered: the README tells you to put the camera in the anchor, so +# "Shot on 35mm, handheld" or "anamorphic lens" -- which say NOTHING about subject +# distance -- silenced the clause on every shot of a real script. Reported as the +# framing fix doing nothing whatsoever, and it was doing nothing: it never ran. +# +# Lens, stock, grade, mood and camera movement are not sizes and do not stand it +# down. A close-up does, because a close-up is a frame somebody asked for. +_FRAME_SIZE = re.compile( + r"\bclose[-\s]?ups?|\bclose\s+(?:shots?|on)\b|\btight\s+(?:shots?|on)\b|\bmacro\b|" + r"\bwide\s+(?:shots?|angle)\b|\bwide\b|\bestablishing\b|\blong\s+shots?\b|" + r"\bfull\s+(?:shots?|body|figure|length)\b|\bmedium\s+shots?\b|\bmid\s+shots?\b|" + r"\btwo[-\s]?shots?\b|\bover[-\s]the[-\s]shoulder\b|\bpov\b|" + r"\bhead\s+and\s+shoulders\b|\bportrait\b|\bwaist[-\s]up\b|" + r"\bknees?[-\s]up\b|\bhead\s+to\s+(?:toe|foot|feet)\b", re.I) + + +# THE CAMERA MOVING ON ITS OWN. +# +# Reported: the camera wanders -- a drift, a slow push, an orbit nobody asked for -- +# and it breaks the chain. Every shot opens on the previous shot's last frame, so a +# shot that ends on a viewpoint the shot never started from hands THAT viewpoint on, +# and the next shot inherits it and adds its own drift. The room is a different room +# by shot four, from a camera nobody placed. +# +# The text never said otherwise. An attribute a prompt does not state is not left to +# the model, it is left to the model's prior -- and for video that prior is MOTION: +# a still camera is the one thing a video model has no reason to produce unless the +# words ask for it. Every other picture guard here exists for the same reason. +# +# Silent where the author has said anything about the camera at all, in the beat or +# in the anchor: their words win, and a pan somebody asked for is not a defect. That +# includes asking for a still one -- this clause would only agree with it. +_CAMERA_ASKED = re.compile( + r"\bcameras?\b|\blens\b|\bshot\s+on\b|\bpans?\b|\bpanning\b|\btilts?\b|\btilting\b|" + r"\bdolly(?:ing)?\b|\btracking\s+shot\b|\btrucks?\s+(?:in|out|left|right)\b|" + r"\bzoom(?:s|ing|ed)?\b|\bpush(?:es|ing)?\s+in\b|\bpull(?:s|ing)?\s+(?:back|out)\b|" + r"\bcrane\b|\bjib\b|\bsteadicam\b|\bhand-?held\b|\bgimbal\b|\bdrone\b|" + r"\borbit(?:s|ing)?\b|\barc(?:s|ing)?\s+around\b|\bcircles?\s+around\b|" + r"\bwhip\s+pan\b|\brack\s+focus\b|\bfollow(?:s|ing)?\s+shot\b|\bpov\b|" + r"\blocked[-\s]off\b|\bstatic\s+(?:shot|frame|camera)\b|\bcrash\s+zoom\b", re.I) + + +def camera_hold(beat, anchor="", moving=False): + """One sentence holding the camera still, where nothing has placed it. + + `moving` stands it down for a shot that travels between places: a journey the + node has already asked to keep every step in frame is a shot whose camera has to + go with them, and telling it to stay put contradicts the beat.""" + if moving: + return "" + if _CAMERA_ASKED.search(str(beat or "")) or _CAMERA_ASKED.search(str(anchor or "")): + return "" + # IT NAMES NO CAMERA, for the reason the face guard gives: naming one is asking + # for one, and the lens is what the gaze guards spend their words getting people + # to stop looking at. A TAKE is the same fact from the other side -- one position + # for the length of the shot -- and it says the other half of what was reported + # too: no cut inside the shot. + # + # SHORT, and positive. Every clause competes for the same per-shot budget, so a + # long one evicts the body count or the gaze on a brief beat; and a guard says + # what IS, never what is not, because a bag of words at cfg 1 drops the "not" + # and keeps the verb it negates. + return " The shot is one unbroken take from one position, angle and distance." + + +def frame_hold(beat, anchor="", people=1): + """Say the frame holds a whole body, where nothing else says what the frame is. + + THE PORTRAIT IS WHAT AN UNSTATED FRAME BECOMES. This file already records the + reason: "an attribute a prompt does not state is not LEFT to the model, it is + left to the model's prior -- which for a named, described person is a PORTRAIT: + facing the lens, pleasantly, because that is what photographs of people are." + The sheet describes a face in every shot, because clothing continuity needs it, + and the mouth guard describes a mouth in every silent shot, because babble needs + it -- so the text is weighted towards a face and nothing in it says how much of + the person to show. Measured on a volleyball beat: 15 words of appearance and 9 + of mouth against 8 of action. Reported as the camera staying fixated on one + character, staring into the lens, with no reference image anywhere in the run. + + Only where the beat stages something a portrait cannot contain, and only where + the author has said nothing about the camera -- in the beat or in the anchor. + Their framing always wins, a close-up included, because a close-up is a frame + somebody asked for. Impersonal, like the other picture guards, and positively + phrased: it says what the frame holds, never what it is not.""" + b = str(beat or "") + if _FRAME_SIZE.search(b) or _FRAME_SIZE.search(str(anchor or "")): + return "" + if tight_framing(b) or tight_framing(str(anchor or "")): + return "" + if not (_WHOLE_BODY.search(b) or _TRAVEL_VERB.search(b)): + return "" + if int(people or 1) > 1: + return (" The frame holds every body in it whole, head to feet, with the room " + "around them.") + return (" The frame holds the whole body, head to feet, with the room around it.") + + def tight_framing(text): """Does this beat call for a frame close enough to lose the anchor point?""" return bool(_TIGHT_FRAME.search(text or "")) @@ -5016,7 +6017,64 @@ _NOT_A_TARGET = frozenset( "time moment thing things way".split()) -def look_target(beat): +# A LOOK AT A PERSON IS A LOOK. +# +# The old rule was "restating a pronoun says nothing the beat did not, and the other +# person is in frame to be looked at anyway". That is true against a neutral model +# and false against one whose prior is a portrait: the choice is not between the +# beat's word and a restatement of it, it is between the beat's word and the LENS. +# Reported as "she looks at the van in one beat and gazes at the camera in the +# next" -- and measured, "McKenna watches him" was the commonest way to lose it: +# the look moved, which cleared the latch correctly, and then nothing replaced it. +# +# Neither existing pattern can even see a person. Both require a determiner before +# the target -- the|a|an|her|his|their -- so "watches Dan" and "looks at Dan" match +# nothing at all. +_LOOK_AT_WHO = re.compile( + r"\b(?:look(?:s|ed|ing)?|star(?:e|es|ed|ing)|gaz(?:e|es|ed|ing)|" + r"glanc(?:e|es|ed|ing)|peer(?:s|ed|ing)?)\s+" + r"(?:back\s+|down\s+|up\s+|over\s+|round\s+|around\s+|straight\s+|right\s+)?" + + _GAZE_PREP + r"\s+([A-Z][\w-]+|him|her|them|he|she|they)\b" + r"|\b(?:watch(?:es|ed|ing)?|stud(?:y|ies|ied|ying)|examin(?:e|es|ed|ing))\s+" + r"([A-Z][\w-]+|him|her|them)\b", re.I) +# Which pronoun can be which. A gendered pronoun narrows the field, so a scene with +# a man and a woman resolves "him" without guessing; "them" does not narrow it and +# only lands where exactly one other person is there to land on. +_PRONOUN_SEX = {"him": "he", "he": "he", "her": "she", "she": "she"} + + +def _person_looked_at(beat, sheet="", described=()): + """The PERSON this beat says somebody is watching. '' when it is not resolvable. + + Only where it is unambiguous. Three people and a bare "him" resolves to nobody, + and guessing which one is worse than saying nothing: a shot told the wrong + sightline is a shot that has to be reshot, while a shot told none is only back + where it was.""" + m = _LOOK_AT_WHO.search(beat or "") + if not m: + return "" + raw = (m.group(1) or m.group(2) or "").strip() + if not raw: + return "" + rows = {n: ln for n, ln in sheet_lines(sheet or "") if n} + # A NAME, spelled as the sheet spells it. + for n in rows: + if raw.lower() == n.lower(): + return n + # A PRONOUN. Whoever else is in the shot, if that is one person -- and if the + # pronoun is gendered, only the people whose entry agrees with it. + want = _PRONOUN_SEX.get(raw.lower()) + here = [n for n in (described or []) if n in rows] + # The looker is not the one being looked at. + looker = subjects_for(beat, sheet, _LOOK_VERB_SRC) + here = [n for n in here if n not in set(looker)] + if want: + here = [n for n in here + if re.search(r"\b" + want + r"\b", rows.get(n, ""), re.I)] + return here[0] if len(here) == 1 else "" + + +def look_target(beat, sheet="", described=()): """What this beat says somebody is looking at. '' when it names nothing.""" for pat in (_LOOK_AT, _WATCH): m = pat.search(beat or "") @@ -5026,24 +6084,39 @@ def look_target(beat): if not target or target.lower() in _NOT_A_TARGET: continue return target - return "" + return _person_looked_at(beat, sheet, described) # Going somewhere ends a look. Held across it, "the eyes are on the TV" follows # somebody out of the room and into the next scene. -_MOVES_OFF = re.compile( - r"\b(?:walks?|walked|runs?|ran|steps?|stepped|moves?|moved|crosses|crossed|" - r"leaves?|left|exits?|exited|goes|went|heads?|headed|climbs?|climbed|" - r"follows?|followed)\b", re.I) +_MOVES_OFF_SRC = (r"walks?|walked|runs?|ran|steps?|stepped|moves?|moved|crosses|" + r"crossed|leaves?|left|exits?|exited|goes|went|heads?|headed|" + r"climbs?|climbed|follows?|followed") +_MOVES_OFF = re.compile(r"\b(?:" + _MOVES_OFF_SRC + r")\b", re.I) # The look VERBS on their own, with no target required. _LOOK_AT needs a nameable # object, so "looks at her" reads as no look at all -- and the latch then held a # television she had just turned away from. -_LOOK_VERB = re.compile( - r"\b(?:look(?:s|ed|ing)?|star(?:e|es|ed|ing)|gaz(?:e|es|ed|ing)|" - r"glanc(?:e|es|ed|ing)|peer(?:s|ed|ing)?|watch(?:es|ed|ing)?|" - r"stud(?:y|ies|ied|ying))\b", re.I) +_LOOK_VERB_SRC = (r"look(?:s|ed|ing)?|star(?:e|es|ed|ing)|gaz(?:e|es|ed|ing)|" + r"glanc(?:e|es|ed|ing)|peer(?:s|ed|ing)?|watch(?:es|ed|ing)?|" + r"stud(?:y|ies|ied|ying)") +_LOOK_VERB = re.compile(r"\b(?:" + _LOOK_VERB_SRC + r")\b", re.I) + + +def subjects_for(beat, sheet, verbs): + """Which people on the sheet this beat puts in front of one of these verbs. + + The shared shape behind speakers_in and vocal_sources_in, including the + conjunction guard: "Dan holds the door and McKenna looks away" must not credit + Dan, because `and` opens a new predicate with its own subject.""" + b, out = beat or "", [] + for n, _ in sheet_lines(sheet): + if n and re.search(r"\b" + re.escape(n) + r"\b" + r"(?:\s+(?!and\b|but\b|then\b|who\b|,\s*who\b)[\w,']+){0,2}?" + r"\s+(?:" + verbs + r")\b", b, re.I): + out.append(n) + return out def looks_somewhere(beat): @@ -5055,9 +6128,16 @@ def looks_somewhere(beat): return bool(_LOOK_VERB.search(beat or "")) -def gaze_hold(target): +def gaze_hold(target, who="", is_person=False): """One sentence putting the eyes and the head on the thing the beat named. + NAMED when the caller says to, which it does once a second person is in the + shot. Reported: "the girl is stuck gazing at a camera while the other character + does his part" -- a look staged by one person went on being said impersonally in + shots she was not in, so it landed on whoever was. Impersonal is still right with + one person in frame: naming somebody is a second mention of them, and a described + person is a person the model draws. + Impersonal, like the hardware placement clause: naming the person again is one more mention of a person, and that has its own cost. Says nothing about where the camera is -- the shot may be looking straight down the line of sight -- only that @@ -5067,7 +6147,37 @@ def gaze_hold(target): # SHORT. Nineteen words restating a nine-word beat is most of the shot spent # agreeing with it, and the guards crowding out the action is what "the # character did not do what I told it" looks like from the outside. - return f" The eyes and the head are turned to the {target}." + what = target if is_person else f"the {target}" + if who: + # A PRONOUN WHERE THE NAME IS ALREADY SPENT. A person is named once in a + # shot's guard text -- two clauses naming the same person is what put a + # second girl in frame at the moment of cuffing. But the clause that + # already named her is standing right beside this one, so "her eyes" has + # its antecedent and costs no second naming. Used only where no one else + # in the shot shares the pronoun. + return f" {who[0].upper()}{who[1:]} eyes and head are turned to {what}." + return f" The eyes and the head are turned to {what}." + + +def dialogue_gaze(n_people): + """One impersonal sentence turning speakers and listeners to each other. + + gaze_hold restates a look the beat named. A dialogue beat that names none + leaves both faces to the portrait prior, and the prior is the lens: reported + as "it looks like they are talking to a camera and not to each other". A + spoken line has an addressee whether or not the beat wrote one, and the + addressee is in the shot, so turning the faces to each other is the one thing + that can be said without inventing anything. + + Impersonal, like gaze_hold, and for the same reason: on a dialogue shot the + speaker's name is spent by the mouth guard and the listener's by told_hold, + and a third mention is a third person. Positively phrased -- at cfg 1 naming + the lens would ask for it. Says nothing about where the camera is.""" + if n_people < 2: + return "" + if n_people == 2: + return " They face each other, eyes on each other." + return " Eyes on whoever is speaking, faces turned to them." def forced_pose(text): @@ -5295,7 +6405,8 @@ def posture_in(beat, cast): # that followed it, and the beat registered no posture at all. hits = sorted(((m.start(), pose) for pose, rx in _POSTURE_OF for m in rx.finditer(part) - if not _in_a_request(b, base + m.start())), + if not _in_a_request(b, base + m.start()) + and not engine.denied_posture(part, m.start())), key=lambda h: h[0]) prev = 0 for at, pose in hits: @@ -5433,9 +6544,23 @@ def posture_hold(poses, described): if n in set(described or []) and p != "standing"] if not who: return "" + # "is still lying down" -- the adverb meaning "as before", which is not how a + # video model reads the token. This clause is LATCHED: it lands in every shot + # after the one that stages the pose, so a scene where somebody sat down once + # carried the word `still` beside their name for the rest of the film. Naming + # the pose is the entire guarantee; "as before" was never part of it. if len(who) == 1: - return f" {who[0][0]} is still {who[0][1]}." - said = "; ".join(f"{n} is still {p}" for n, p in who[:2]) + return f" {who[0][0]} is {who[0][1]}." + # ONE POSE SHARED BY EVERYBODY NEEDS NO NAMES AT ALL. "Dan is sitting; Crystal + # is sitting" spends a naming of each of them to say one thing about the pair, + # and this file's own rule is that naming somebody twice in one shot is what + # draws a second copy of them. Said impersonally it costs none, and nothing is + # lost: the pose is the whole guarantee, and every described person has it. + _poses = {p for _n, p in who} + if len(_poses) == 1 and len(who) == len(set(described or [])): + return (f" Both are {who[0][1]}." if len(who) == 2 + else f" Everyone in the shot is {who[0][1]}.") + said = "; ".join(f"{n} is {p}" for n, p in who[:2]) return f" {said}." @@ -5461,16 +6586,51 @@ _SHUTS = re.compile(r"(?:closes?|closed|closing|shuts?|shutting|slams?|slammed|" r"slamming|locks?|locked|locking|lowers?|lowered)\Z", re.I) +# A DIRECTION THE VERB DOES NOT CARRY, SAID BESIDE IT. "slides", "swings" and +# "pulls" go either way and get no anchor from the verb alone -- but "slides OPEN" +# is not ambiguous, and neither is "swings shut". Reported: a van whose side door +# slides open and is closed again halfway through the shot, which is the reversal +# the anchor exists to settle; the beat said which way and nothing read it. +# +# "back" is here because that is how a sliding door and a curtain open -- "slides +# back", "draws back". Nothing in this list means shut by accident: the shut words +# are the two that only ever mean shut. +_WAY_WORD = re.compile(r"\A(?:(open|wide|back|apart|aside)|(shut|closed))\b", re.I) +# ...and THE THING BEFORE THE VERB, which is the ordinary way to write it. Every +# reader here expected "opens the door" and the beat said "the door slides open", so +# a door that opens on its own -- which is what a van's side door does in a script -- +# was not a staged change at all: no anchor, and no clearing of a held state saying +# it was shut. A determiner in front is required for the same reason the forward +# reader demands one: "the closed door" is an adjective, "the door closed" is not. +_STATE_ACT_REV = re.compile( + r"\b(?:the|a|an|its|his|her|their|our|my|your|this|that|these|those|both|all|" + r"each|every|another|one|two|three|\w+'s)\s+((?:[\w']+\s+){0,2}?)(" + _STATE_THING + + r")\s+((?:[\w']+\s+){0,1}?)(" + _STATE_ACTS + r")\b", re.I) + + def state_changes(text): """[(thing, 'open'|'shut'|None)] for the scenery this text actually works. A state word sitting straight in front of its noun is an adjective describing the thing, not a verb acting on it: "the closed doors" says nothing happens. - The direction is None where the verb does not carry one.""" + The direction is None where neither the verb nor a word beside it carries one.""" out, seen = [], set() - for m in _STATE_ACT.finditer(text or ""): - verb, gap, thing = m.group(1), m.group(2), m.group(3) - if _adjectival(verb, gap): + found = [(m.group(1), m.group(2), m.group(3), m.end(), False) + for m in _STATE_ACT.finditer(text or "")] + found += [(m.group(4), m.group(3), m.group(2), m.end(), True) + for m in _STATE_ACT_REV.finditer(text or "")] + for verb, gap, thing, end, reverse in sorted(found, key=lambda f: f[3]): + # A STATE, NOT AN ACT. Read forwards that is the missing determiner ("closed + # rear doors"). Read backwards the word order cannot settle it -- "a van with + # its doors closed" and "the door closed" put the same two words in the same + # order -- so the backwards reader takes only words that are verbs and nothing + # else: "slides", "swings", "opens". A bare state word after its noun is left + # to stated_states, which puts it at the first frame. Anchoring it instead + # would ask for the change it says has already happened. + if reverse: + if re.fullmatch(_STATE_WORD, verb, re.I): + continue + elif _adjectival(verb, gap): continue key = _state_key(thing) if key in seen: @@ -5478,6 +6638,11 @@ def state_changes(text): seen.add(key) way = ("open" if _OPENS.match(verb) else "shut" if _SHUTS.match(verb) else None) + if way is None: + # The beat named the end even though the verb does not. + said = _WAY_WORD.match((text or "")[end:].lstrip()) + if said: + way = "open" if said.group(1) else "shut" out.append((thing.lower(), way)) return out @@ -5575,6 +6740,16 @@ _TRAVEL_VERB = re.compile( # omission from the other end. r"leave|leaves|left|leaving|enter|enters|entered|entering|" r"cross|crosses|crossed|crossing|exit|exits|exited|exiting|" + # TAKING SOMEBODY SOMEWHERE is a journey too, and these are the words a script + # uses for it. Reported: somebody escorted from a vehicle to a doorway got no + # travel clause at all -- "escorts" was in no list -- so the one shot that had to + # perform a walk was told nothing about performing it, and what it did instead + # was turn round and walk backwards. Safe to be generous: every reader here needs + # a DESTINATION as well as the verb, so "brings a cup" moves nobody. + r"escort|escorts|escorted|escorting|usher|ushers|ushered|ushering|" + r"march|marches|marched|marching|guide|guides|guided|guiding|" + r"bring|brings|brought|bringing|drag|drags|dragged|dragging|" + r"haul|hauls|hauled|hauling|" r"return|returns|returned|returning)\b", re.I) @@ -5600,6 +6775,27 @@ def travel_in(beat): return (frm, via, to) +def travel_legs(beat): + """(from, via, to) for a beat that moves somebody, with the one promotion that + the render and the SIZING have to agree on. + + A BEAT THAT TRAVELS ALONG A PLACE ENDS IN IT. "walks down the hallway" reads as a + via with no destination, and travel_anchor says nothing without one -- so that + shot was told nothing about where it was, the tracked room kept the bedroom, and + the NEXT beat opened "in the bedroom" on its way to the kitchen. Reported as a bed + in the hallway. The place travelled along is where the beat arrives, so the shot + opens where the last one left off and walks into it, in frame. + + Here rather than inline in the shot loop because travel_spaces reads the same + fact to size the shot. Kept in two places it would drift, and a transit rendered + as a walk while being sized as if it went nowhere is exactly the split that put a + three-room walk in a three-second shot.""" + frm, via, to = travel_in(beat) + if via and not to: + via, to = "", via + return frm, via, to + + def where_hold(here, scene): """Say which room the shot is in, once the film has left the one in the scene. @@ -5636,7 +6832,118 @@ def where_hold(here, scene): f"furniture are the {here}'s throughout.") -def travel_anchor(frm, via, to, here=""): +# THINGS THAT ARE IN A ROOM RATHER THAN BEING ONE. +# +# A closed list of place words cannot name every room a script invents -- a dungeon, +# a cargo bay, a stable, a chapel, a sauna, a morgue -- and a closed list is exactly +# why "they head to the locker room" was read as going nowhere and the set changed +# under the characters instead of being walked into. Adding room words one report at +# a time fixes one script each. +# +# So the generalisable side of the problem is the INVERSE: an unlisted destination is +# taken as a place unless it is one of these. Furniture, fittings, a body part, a +# position within a space, a vehicle, or a person. There are far fewer common object +# destinations in English than there are names for rooms, and this list does not have +# to grow when somebody writes a scene nobody has written before. +# +# "door" is the original of this whole failure and is named first: _PLACE once held +# it, so "Ana looks at the door" moved the camera into a door. +_NOT_A_DESTINATION = frozenset(""" +door doors doorknob handle window windows curtain curtains blind blinds mirror +sink basin bath tap taps table desk counter worktop bench chair seat stool sofa +couch armchair bed mattress headboard pillow cushion duvet quilt sheets blanket +cupboard cabinet drawer drawers shelf shelves wardrobe closet locker lockers +fridge freezer oven stove hob kettle microwave dishwasher washer machine +floor ground ceiling wall walls rail railing bannister bars bar post pole fence +light lights lamp switch socket screen tv television phone radio speaker camera +bag bags box crate case suitcase trunk basket bin sack tray bottle glass cup +car van truck bike motorbike trailer boat seat +edge middle centre center side sides end front back rear top bottom corner corners +spot position point row line queue +girl boy man woman lady guy person stranger guard nurse doctor teacher +hand hands arm arms elbow shoulder shoulders knee knees foot feet leg legs lap +face mouth lips chin neck throat hair head chest breast breasts stomach belly +waist hip hips thigh thighs wrist wrists ankle ankles bum butt crotch +""".split()) + +# Same shape as _GOES_TO, with an OPEN noun where that one has the place list. +# The destination STOPS at a conjunction. Without that guard the capture ran +# straight through one -- "walks to the bench and picks up a towel" produced the +# destination "bench and picks", whose last word is a verb, so the blocklist never +# saw the bench it was there to catch and the shot was told to travel "to the bench +# and picks". Every word of the destination is checked, not only the head, for the +# same reason. +_MOVES_TO_ANY = re.compile( + r"\b(?:to|into|toward|towards|inside|through\s+to|" + r"enters?|entered|entering|steps?\s+into|stepped\s+into)\s+" + + _DET_POSS + r"\s+((?:(?!(?:and|or|then|but|while|as|before|after|with|for|" + r"to|into|onto|from|at|on|in|of)\b)[A-Za-z][\w-]*\s+){0,2}" + r"(?!(?:and|or|then|but|while|as)\b)[A-Za-z][\w-]*)\b", re.I) + + +def moved_to(beat, people=()): + """Where this beat MOVES somebody, whatever the place is called. "" if nowhere. + + The place-list readers answer first and more richly, because a known room can be + named at both ends of the journey. This is what answers when they cannot: a + travel verb, a destination, and a head noun that is not furniture, a body part, + a vehicle or a person. It establishes NO room state -- it does not decide cuts, + it does not feed where_hold or the acoustics, and it never claims to know what + kind of space it is. All it does is make the arrival be PERFORMED, which is the + one thing the reported failure was missing.""" + b = _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", str(beat or ""))) + if not _TRAVEL_VERB.search(b): + return "" + names = {str(n).strip().lower() for n in (people or ()) if str(n).strip()} + for m in _MOVES_TO_ANY.finditer(b): + dest = re.sub(r"\s+", " ", m.group(1)).strip() + # THE HEAD NOUN DECIDES, not every word in the phrase. A locker is furniture + # and a locker ROOM is a room; so are an engine room and a boiler room. The + # conjunction guard in the pattern is what stops a run-on phrase reaching + # here with a verb for a head, which is what let "the bench and picks" past a + # blocklist that holds "bench". + head = dest.split()[-1].lower().strip("-") + if (len(head) < 3 or head in _NOT_A_DESTINATION or head in names + or dest.lower() in names or _EXTRA_PEOPLE.search(dest)): + continue + return dest.lower() + return "" + + +# A WALK THE MODEL CAN PLAY BACKWARDS. Reported: somebody escorted from a vehicle to +# a doorway turned round, walked the other way, and then walked BACKWARDS to the +# doorway. That is the same reversal the door anchor exists to settle -- time-flip +# augmentation teaches a model that a clip and its reverse are the same clip -- and +# the travel clause could not settle it, because everything it said is as true of the +# reversed walk as of the real one: a walk between two places, every step in frame, +# played out on screen. It named the two ends in SPACE and left the direction open. +# +# Two facts close it, and both are positive: which way the bodies face, and that the +# destination gets NEARER. A reversed render contradicts each of them. +# +# Not on a beat that walks backwards on purpose. The author's words win, as they do +# against every other inference here. +_GOES_BACKWARD = re.compile( + r"\b(?:backwards?|in\s+reverse|backs?\s+(?:away|out|up|off)|backing\s+(?:away|out|up)|" + r"retreats?|retreating|reverses?|reversing)\b", re.I) + + +def facing_phrase(beat=""): + """"each body facing the way it goes", unless the beat walks backwards.""" + return "" if _GOES_BACKWARD.search(str(beat or "")) else " each body facing the way it goes" + + +def move_clause(dest, beat=""): + """Perform an arrival the place list cannot name. "" when there is nowhere.""" + if not dest: + return "" + facing = facing_phrase(beat) + return (f" The shot travels to the {dest} on screen, the whole move played out " + f"from its first step to its last,{facing + ' and' if facing else ''} the " + f"{dest} nearer at the last frame than at the first.") + + +def travel_anchor(frm, via, to, here="", beat=""): """Say where the shot starts, what it passes, and where it ends. "" if nowhere. `here` is the room an earlier beat established, used when the beat names no @@ -5657,14 +6964,26 @@ def travel_anchor(frm, via, to, here=""): # Reported twice as an instant cut across a house, once after the destination # reader was fixed and the clause was demonstrably in the prompt. Say what the # shot DOES: the walk happens, on screen, in frame, the whole way. - walk = "the walk between them played out on screen, every step in frame." + facing = facing_phrase(beat) + walk = ("the walk between them played out on screen, every step in frame" + + (f",{facing}." if facing else ".")) if via: return (f" The shot opens in the {start}, carries along the {via}, and " f"arrives in the {to}, {walk}") if start else ( f" The shot carries along the {via} and arrives in the {to}, " f"{walk}") + # AN UNKNOWN ORIGIN IS STILL A JOURNEY. This returned nothing when the room + # they set out from was not on the list or had never been named -- so a beat + # walking out of a gym was told to walk nowhere, and with the previous shot's + # last frame as its keyframe the set simply changed under the characters. + # Reported as the scene shifting to the locker room instead of them walking into + # it. Naming no origin is fine; what the shot needs is that the arrival is + # PERFORMED. Positively phrased, like everything else at cfg 1: the way in, then + # the room. if not start: - return "" + return (f" The shot enters the {to} on screen: the way in first, then the " + f"{to} itself, the arrival played out, every step in frame" + + (f",{facing}." if facing else ".")) return f" The shot opens in the {start} and arrives in the {to}, {walk}" @@ -5704,10 +7023,233 @@ def first_place(text): return "" +# A PLACE SOMEBODY LOOKS AT IS NOT A PLACE THEY ARE IN. "Maya looks out of the window +# at the garden" matched "at the garden" and put the shot in the garden: "This shot +# takes place in the garden: the walls, floor, light and furniture are the garden's +# throughout", a fresh start with the kitchen thrown away, and the garden latched -- +# "Maya pours tea." two shots later was still in it. A kitchen that turns into a +# garden because somebody glanced out of its window is the set not staying the same. +# +# Read off the verb the preposition hangs on, with at most a particle and one "out of +# the window"-shaped phrase between them. "Watches TV at the bar" and "sits in the +# garden" have a different verb, or an object in between, and still place somebody. +_AIMED_AT = re.compile( + r"\b(?:look|stare|glance|gaze|peer|point|gestur|wave|nod|shout|call|yell|squint|" + r"glare|beckon|aim)\w*" + r"(?:\s+(?:out|over|up|down|back|across|around|round|through|away|off|in))*" + r"(?:\s+(?:of|through|from|across|over)\s+(?:the|a|an|her|his|their)\s+[\w-]+" + r"(?:\s+[\w-]+)?)?\s*$", re.I) + + def place_named(text): """The place this text says somebody is IN, without travelling. "" if none.""" - m = _IS_IN.search(str(text or "")) - return re.sub(r"\s+", " ", m.group(1)).strip().lower() if m else "" + text = str(text or "") + for m in _IS_IN.finditer(text): + if _AIMED_AT.search(text[:m.start()]): + continue + return re.sub(r"\s+", " ", m.group(1)).strip().lower() + return "" + + +def rooms_named(text): + """Every room this text names, lowercased. "" -> []. + + first_place returns only the FIRST one, which is what a tracked position needs. + A scene paragraph often names two -- "Her bedroom has an unmade bed. The kitchen + is small." -- and deciding whether a SENTENCE is about the room we are in means + accounting for all of them. + + The same two exclusions as first_place, for the same reasons: a word that is also + an ordinary verb cannot win in free text with no preposition in front of it, and a + bare "room" names nowhere unless the word in front qualifies it.""" + out = [] + s = str(text or "") + for m in _PLACE_WORD.finditer(s): + got = re.sub(r"\s+", " ", m.group(0)).strip().lower() + if got in _PLACE_ALSO_A_VERB: + continue + if got == "room": + before = re.search(r"(\w+)\s+$", s[:m.start()]) + word = before.group(1).lower() if before else "" + if word in _NOT_A_ROOM_MODIFIER or not word: + continue + got = word + " room" + if got not in out: + out.append(got) + return out + + +def split_sheet(scene, names=()): + """(everything that is not a character sheet entry, the sheet entries). + + WHAT LEADS A PROMPT DECIDES ITS COMPOSITION. This file already recorded that -- + "anatomy in the opening tokens is what a distilled LoRA settles composition on", + which is why the gaze clause was moved to follow the beat rather than lead it -- + and then left the biggest anatomy block in the prompt leading every shot: the + character sheet. "McKenna: she, 22, tall, long blonde hair, blue eyes, freckles" + is sixteen words of face, it has to be in every shot because clothing continuity + needs it there, and it sat in front of the action. + + Measured on a volleyball beat: 69% of the shot's words were in sentences about a + face, and turning off every face guard only took that to 63%, because the sheet is + most of it. Reported across many attempts as the camera fixated on one character + staring into the lens -- with no reference image, no LoRA and a pinned first frame, + none of which touched it, because none of them were what was leading the prompt. + + Splitting lets the scene keep the front, the beat follow it, and the appearance + come after the thing it is describing. The words are identical; only the order + changes, which is the one thing about this that was never tried.""" + cast = [str(n).strip() for n in (names or ()) if str(n).strip()] + rest, sheet = [], [] + for raw in str(scene or "").split("\n"): + for unit in re.split(r"(?<=[.!?])\s+", raw): + u = unit.strip() + if not u: + continue + if cast: + entry = any(re.match(r"^" + re.escape(n) + r"\s*:", u, re.I) for n in cast) + else: + entry = ":" in u + (sheet if entry else rest).append(u) + return " ".join(rest), " ".join(sheet) + + +# WHERE ONE CLAUSE ENDS. Full stops and semicolons both, because a scene paragraph +# describes a flat one room per clause and the clauses are as often joined as +# separated: "The living room has a red sofa; the kitchen has white tiles." Split on +# sentences alone that is ONE unit naming two rooms, which scene_for_here keeps -- +# correctly, by its own rule that a sentence naming both stays -- so every shot in +# the flat carried both rooms and the model was free to render either, or to change +# its mind halfway through the shot and render the other. +_CLAUSE_END = r"(?<=[.!?;])\s+" + + +def scene_for_here(scene, here, always="", names=(), beat=""): + """(text to send, rooms held back, True if it declined to hold anything). + + THE SCENE PARAGRAPH IS STAMPED INTO EVERY SHOT, and it has to be -- a removal + needs the text to have something to scrub, and where_hold's own comment says the + paragraph "still names the room they started in and is stamped into every shot". + But a paragraph that describes the opening ROOM describes its FURNITURE too, and + furniture does not travel. Reported: a flat whose scene paragraph read "Her + bedroom has an unmade bed and a lamp", a walk from the bedroom down the hallway + to the living room, and then A BED IN THE LIVING ROOM. where_hold had the room's + NAME right in every shot; the bed was in the text standing beside it, and at cfg 1 + there is no negative prompt that can take a named thing back. + + THE ROOM THE SHOT ENDS IN decides this, not every room it passes through, and the + difference is the whole fix. A walk out of the bedroom genuinely shows the bedroom + in its opening frames -- but that shot's LAST frame is the next shot's keyframe, so + a bed drawn at the end of the walk is inherited by the shot after it, which is the + second route the same bed took into the living room. Nothing is lost by holding it + there: the opening room arrives as a PICTURE regardless, because the keyframe is + the previous shot's last frame and that frame IS the room being left. So the words + describe where the shot ends and the frame carries where it began. + + A WITHHOLDING, NOT AN EDIT, exactly like the covered-garment deferral: the + author's paragraph is untouched, every reader inside this file still sees all of + it, this is only what the model is told for THIS shot, and a beat that walks back + into the bedroom gets the bed back in full. + + TWO GUARDS. A sentence carrying a LABEL -- "McKenna: she, 22, ..." -- is a + character sheet entry and is never touched whatever it names, because losing a + person's line is the failure hide_item exists to prevent. And if holding would + leave the shot no scene sentence at all, nothing is held: a sentence that welds + the film's own framing to one room's furniture ("A small flat at night, her + bedroom with an unmade bed") would otherwise take the night away with the bedroom, + and a shot with no scene is a bigger change than the bug. The caller reports that + case so the author can split the sentence. + + `always` IS THE ANCHOR AND IS NEVER HELD. build_scene fuses the anchor and the + scene paragraph into one string before either reaches a shot, and an anchor is + documented as what belongs to the WHOLE film -- "look, camera, lighting, + location". So an anchor reading "Shot on 35mm in a cramped kitchen" names a room, + and without this it was held on every shot outside that kitchen: the film lost its + stock and its lens to a rule about furniture. The anchor's sentences are spared by + text, which survives terminate_lines adding a full stop to them. + + `names` IS THE DECLARED CAST, and it is what identifies a sheet entry. A bare + colon test was the first guard and it had a hole both ways: "Her bedroom: an + unmade bed and a lamp." is the author describing a room, not a person, and it was + protected as though it were somebody's line -- so the bed survived in that + phrasing. Matching the LABEL against a name the sheet actually declares closes it + without ever risking a person: with no cast passed it falls back to protecting any + colon, because losing somebody's line is worse than a bed in one shot. + + A ROOM THE BEAT ITSELF NAMES IS NEVER HELD. `here` goes stale whenever the beat's + verb is not one the movement readers know -- "McKenna pads into the kitchen" moves + nobody as far as place_in is concerned -- and a stale room would hold the + description of the room the shot is actually IN. The beat's own words outrank + anything inferred from them, which is this file's standing rule, so a sentence + about a room the beat mentions stays whatever the tracked room says. + + Holding NOTHING returns the text unchanged, byte for byte, so a script that never + leaves one room is untouched and costs nothing.""" + text = str(scene or "") + room = (here or "").strip().lower() + if not text.strip() or not room: + return text, [], False, [] + cast = [str(n).strip() for n in (names or ()) if str(n).strip()] + + def _is_sheet_entry(unit): + u = unit.strip() + for n in cast: + if re.match(r"^" + re.escape(n) + r"\s*:", u, re.I): + return True + return bool(not cast and ":" in u) + + beat_rooms = set(rooms_named( + _DIALOGUE_TAG.sub(" ", _QUOTED.sub(" ", str(beat or ""))))) + spared = set() + for unit in re.split(_CLAUSE_END, str(always or "")): + u = unit.strip().rstrip(".!?; ").lower() + if u: + spared.add(u) + lines, held, held_text, survived = [], [], [], False + for raw in text.split("\n"): + kept, cut_here = [], False + for unit in re.split(_CLAUSE_END, raw): + # A sheet entry. Never touched. + if _is_sheet_entry(unit): + kept.append(unit) + continue + # ...nor anything the anchor said. It frames the whole film. + if unit.strip().rstrip(".!?; ").lower() in spared: + kept.append(unit) + survived = True + continue + named = rooms_named(unit) + # It names another room and not this one. A sentence naming BOTH stays -- + # it is partly about where we are, and keeping too much is the safe way to + # be wrong here. + if named and room not in named and not (beat_rooms & set(named)): + for r in named: + if r not in held: + held.append(r) + if unit.strip() not in held_text: + held_text.append(unit.strip()) + cut_here = True + continue + kept.append(unit) + survived = True + # Punctuation is repaired only on a line something was held BACK from. A + # clause that ended in a semicolon has lost what followed it, and one + # promoted out of a semicolon now opens a sentence. A line this held nothing + # from is the author's, spacing and semicolons included. + if cut_here: + mended = [] + for unit in (k for k in kept if k.strip()): + unit = re.sub(r";$", ".", unit.strip()) + if mended and mended[-1].endswith(".") and unit[:1].islower(): + unit = unit[0].upper() + unit[1:] + mended.append(unit) + kept = mended + lines.append(" ".join(k for k in kept if k.strip())) + if not held: + return text, [], False, [] + if not survived: + return text, held, True, held_text + return "\n".join(l for l in (s.strip() for s in lines) if l), held, False, held_text def direction_anchor(changes): @@ -5773,7 +7315,6 @@ def exits_vehicle(text): return bool(_EXIT_VEHICLE.search(text or "")) -_PICTURE_TAG = re.compile(r"<\s*picture[\s_\-]*(\d+)\s*>", re.I) def renumber_reference_tags(text, wired): @@ -5832,21 +7373,18 @@ def handoff_claim(n): def handoff_context_claim(first, last): - """Claim previous-shot tail frames carried as reference context. + """Claim passive tail frames from the previous shot as continuity context. - These are not keyframes. They are context stills that show the room, camera - path, and motion immediately before the hard handoff keyframe. They must be - named in the prompt because every reference image the VLM sees needs a job: - unnamed pictures are free to become extra subjects. + The actual keyframe chain remains one frame. These frames ride only as references, + so the text must say what they are or the model can treat them as new subjects. """ first, last = int(first), int(last) - if last <= first: - return (f" is a continuity context frame from immediately " - f"before this shot: same place, same camera path, same people, no " - f"new subject.") - return (f" through are continuity context " - f"frames from immediately before this shot: same place, same camera " - f"path, same people, no new subjects.") + label = f"" if last <= first else f" through " + return ( + f" Previous-shot continuity frames are shown in {label}; use them only as " + f"visual context for motion, camera continuity, room layout, lighting, and " + f"timing. They show the same moment leading into this shot and introduce no " + f"new subjects.") def room_claim(n, present, joining): @@ -5877,6 +7415,59 @@ def room_claim(n, present, joining): return said +def carried_people_claim(n, present, was_room="", now_room=""): + """Claim the previous frame carried as a reference across a cut to another room. + + The people come with it and the room does not. Naming both rooms is what keeps + the picture from pulling the old walls in: it says where the picture was taken + and where this shot is.""" + who = _join_names(present) + said = (f" is {who} a moment earlier" + f"{f', in the {was_room}' if was_room else ''}: the same " + f"{'faces, hair and clothes' if len(present) > 1 else 'face, hair and clothes'}.") + if now_room: + said += f" This shot is in the {now_room}." + return said + + +def returning_room_claim(n, room, present, arriving): + """Claim a frame of a room the film showed before and has come back to. + + Its own claim, not room_claim's: that one says "a moment earlier", and this + picture is from shots ago. It names who is in it, because an unclaimed person + in a picture is another person.""" + said = (f" is the {room} as the film last showed it" + f"{', where this shot arrives' if arriving else ''}: the same walls, floor, " + f"furniture and light.") + if present: + said += (f" {_join_names(present)} " + f"{'are the people' if len(present) > 1 else 'is the person'} in it.") + return said + + +def _join_names(names): + """"Nora", "Nora and Dan", "Nora, Dan and Mara" -- a list a reader can read.""" + names = [str(n) for n in (names or []) if str(n).strip()] + if len(names) < 2: + return names[0] if names else "" + return ", ".join(names[:-1]) + " and " + names[-1] + + +def plate_claim(n): + """Claim shot 1's first_frame when it is carried as the SET rather than frame one. + + room_claim cannot serve here and saying so is the point: it calls the picture + "this room a moment earlier" and names who was standing in it, and on shot 1 + there is no earlier and nobody was. A plate is a picture of a place with no + people in it, and the claim has to say exactly that -- an unclaimed picture is + read as another subject, and a picture of an empty room claimed as a person is + how a figure gets invented to stand in it.""" + return (f" is the set this shot takes place in: the same walls, " + f"floor, furniture and light, from the same camera. It is a picture of " + f"the place only, with nobody in it -- the people in this shot are the " + f"ones named above, standing where the text puts them.") + + def state_hold(pairs): """One sentence putting those states at the first frame instead of in the action. @@ -6000,12 +7591,52 @@ _OBJECT_END = re.compile(r"(?:,|;|\.|\bexposing\b|\brevealing\b|\bshowing\b|\ble # taken off -- "the tight and the her and the back come off during this shot". _NOT_A_GARMENT = frozenset(""" the a an and or her his its their our your this that these those +""" +# PRONOUNS, AND ONE OF THEM DESTROYED THE CHARACTER ENTRY. The object span runs to +# the next clause boundary and "and" is not one -- deliberately, so that "unzips her +# jacket and pulls it off" reads as one removal -- which puts the SUBJECT of the +# next clause inside the span. "Kate pulls off the jumper and she sits down" offered +# "she", and a sheet declares its pronoun exactly the way it lists a garment +# ("Kate: she, 28, a wool jumper"), so the positional entry-head test said yes. +# +# The shot then said "The wool jumper and the she come off during this shot", and +# the scrub drops the whole comma-separated entry it matched -- so "Kate: she, 28, a +# wool jumper, a denim skirt." became "28, a denim skirt." in every later shot. Name +# gone, pronoun gone, person gone: shots with nobody described in them, which is the +# cost this file already records for scrubbing a sheet line. + """ +she he him them they us we you one both each either neither +herself himself themselves myself yourself itself +somebody someone anybody anyone nobody everybody everyone off from over under onto into out down up away through across behind front side left right rest way bit end edge back neck chest waist hips hip wrist wrists ankle ankles arm arms hand hands leg legs thigh thighs knee knees foot feet shoulder shoulders head face mouth lips hair skin body torso stomach belly chin jaw eyes ear ears floor ground wall room air +""" +# FIXTURES AND FURNITURE. A garment is recognised by POSITION here, not by +# vocabulary -- see the note above infer_removals, and the reason is good: an +# author writes garments this file has never heard of, and a vocabulary would +# drop them silently. The cost is that position cannot tell a shower from a +# shirt. "Kate steps out of the shower" is the same shape as "Kate steps out of +# the thong", and the scene paragraph lists the shower the same way a sheet +# lists a skirt -- so the shower was taken off her and scrubbed out of every +# later shot, in a bathroom scene, which is a room that quietly stops existing. +# "Kate kicks the stool away" took the stool. +# +# So the list is the other way round: not what a garment IS, but the handful of +# things a person can step out of, get off, drop onto or kick away that are +# plainly not worn. Anything not named here still reaches the positional test, +# which is what keeps an unheard-of garment working. + """ +shower showers bath baths bathtub tub tubs basin sink sinks toilet loo cubicle +stall stalls bed beds sofa sofas couch couches chair chairs stool stools bench +seat seats armchair table tables desk desks counter shelf shelves cupboard +cabinet drawer drawers door doors doorway window windows mirror curtain +car cars cab taxi van truck lift elevator stairs step steps +kitchen bathroom bedroom hallway corridor landing garden street pavement +water pool puddle steam tiles tile mat mats rug rugs carpet basket hamper """.split()) # Where a scene's wardrobe entry ENDS. A garment word is the HEAD of its phrase -- @@ -6018,10 +7649,9 @@ _ENTRY_END = re.compile(r"^\s*(?:[,;.!?]|$|(?:and|over|under|beneath|above|with| # Hardware, not clothing. Inference never takes a restraint off: the standing rule is # that once one goes on it stays on, and an explicit `remove:` is the only thing that # clears it. A beat that cuts a rope must not silently unlock the cuffs as well. -_RESTRAINT_WORD = re.compile( - r"^(?:handcuffs?|cuffs?|shackles?|manacles?|chains?|ropes?|cords?|straps?|" - r"collars?|gags?|blindfolds?|restraints?|bindings?|tape|ties?|harness|" - r"straitjacket|spreader|hogtie|clamps?|clips?)$", re.I) +# One definition, in the engine, where it is called _NOT_CLOTHING. Two copies of a +# vocabulary drift apart, which this file has recorded more than once. +_RESTRAINT_WORD = engine._NOT_CLOTHING # A immediately after a word, so the entry-end test can look past an @@ -6096,7 +7726,9 @@ def displaced_hold(items): if not items: return "" said = ", ".join(f"the {thing} {how}" for thing, how in items[:2]) - return f" Still on the body and {said}, left exactly where the beat put them." + # Sentence-initial "Still" meaning "nevertheless". Nothing else in the prompt + # opens on that word, and "On the body" carries the fact by itself. + return f" On the body and {said}, left exactly where the beat put them." # A REQUEST is not the thing happening. "McKenna asks Dan to take the chastity belt @@ -6168,6 +7800,39 @@ def _in_a_request(text, at): return at <= (start + stop.start() if stop else len(text or "")) +# The object of a removal verb when the beat has already named the garment: "and +# steps out of it". Anchored at the start of the object span, so a pronoun further +# along the sentence is not mistaken for the object. +_PRONOUN_OBJECT = re.compile(r"\s*(?:it|them|these|those)\b", re.I) +_SENTENCE_BREAK = re.compile(r"[.;!?]\s+") + + +def _sentence_before(beat, at): + """The sentence `at` is in, up to `at`. The pronoun's antecedent lives here. + + A beat is a paragraph and can hold several sentences. "It" reaches back across + a comma or an "and", not across a full stop.""" + cut = max((m.end() for m in _SENTENCE_BREAK.finditer(beat[:at])), default=0) + return beat[cut:at] + + +# Garments people call by each other's names. Families, not synonyms: a beat saying +# "shoes" means whatever is on her feet, and the sheet's word is the one to act on. +_GARMENT_FAMILIES = ( + ("shoes", "boots", "sneakers", "trainers", "heels", "sandals", "loafers", "slippers", + "flats", "pumps", "clogs", "brogues", "moccasins", "espadrilles", "wedges"), + ("sweater", "jumper", "sweatshirt", "hoodie", "pullover", "cardigan"), + ("coat", "jacket", "parka", "blazer", "overcoat", "raincoat", "anorak", "windbreaker", + "peacoat"), + ("top", "shirt", "blouse", "tee", "t-shirt", "tshirt", "camisole"), + ("trousers", "pants", "jeans", "slacks", "chinos", "joggers", "sweatpants"), + ("hat", "cap", "beanie", "beret"), + ("gloves", "mittens"), +) +_GARMENT_KIN = {word: tuple(w for w in family if w != word) + for family in _GARMENT_FAMILIES for word in family} + + def infer_removals(beat, scene): """Garments this beat takes off, read from its own prose. [] when none. @@ -6186,6 +7851,15 @@ def infer_removals(beat, scene): # Asked for is not done. See _in_a_request. if _in_a_request(beat, m.start()): continue + # OPENING IS NOT TAKING OFF. "unzips his jacket" leaves the jacket on, and read + # as a removal it was scrubbed from every later shot with the chest called bare. + # Unless the same sentence finishes the job ("unzips her jacket and takes it + # off"), or what is being undone is hardware, which comes off by being undone. + if re.fullmatch(_OPENER_VERB, m.group(0), re.I): + _rest = re.split(r"[.;!?]", beat[m.end():])[0] + if not (_FINISHES_REMOVAL.search(_rest) or restraint_present(_rest)): + continue + _before = len(found) tail = beat[m.end():] cut = _OBJECT_END.search(tail) span = tail[:cut.start()] if cut else tail @@ -6209,7 +7883,12 @@ def infer_removals(beat, scene): continue span = span[:part.start()] for word in re.findall(r"\b[\w-]{3,}\b", span): - low = word.lower().strip("-") + # THE TOKEN THE SHEET WROTE. "take off their skirts" gives "skirts" and the + # sheet says "a denim skirt", so the entry-head test below found nothing and + # the removal did nothing -- on every beat where more than one person + # undressed. One owner for the normalisation, in the engine, because this + # reader and garment_words both look the token up in the same sheet. + low = engine.singular_garment(word) if not low or low in found: continue # Grammar, prepositions and anatomy are not garments. @@ -6220,8 +7899,17 @@ def infer_removals(beat, scene): continue # It has to be worn: the HEAD of something the scene lists, not a # modifier inside it and not half of a hyphenated compound. - if not _is_entry_head(word, scene): - continue + if not _is_entry_head(low, scene): + # ...OR THE ONE THING IT CAN MEAN. "takes off her shoes" beside a sheet + # saying "brown leather boots" named nothing the sheet lists, so the + # boots stayed described as on while the beat took them off: the shot + # drew them half-removed and the next one put them back. People call a + # garment by its family's everyday word. Only when exactly one member of + # that family is on the sheet -- two candidates is a guess. + _kin = [k for k in _GARMENT_KIN.get(low, ()) if _is_entry_head(k, scene)] + if len(_kin) != 1 or _kin[0] in found: + continue + low = _kin[0] # "her jeans shorts" is ONE garment. "jeans" there is a modifier, but it # is also the head of Dan's own entry, so it matched his line and took # HIS trousers off in a beat that never mentions him -- and they stayed @@ -6233,6 +7921,37 @@ def infer_removals(beat, scene): scene, re.I): continue found.append(low) + # "...AND STEPS OUT OF IT." The object is a pronoun, and the garment was + # named one clause earlier -- which is how most undressing is actually + # written: the hands arrive first ("hooks her thumbs in the thong"), the + # removal second, and by then the thing has a pronoun. The word loop above + # cannot see a pronoun at all; it skips anything under three letters. So the + # removal verb matched, the span held nothing it recognised, and NOTHING came + # off -- the sheet went on dressing her in the garment in every later shot, + # which is the author's removal silently reversed. + # + # Resolved the way the unnamed restore is: only when there is exactly one + # thing it can mean. The candidate has to be a garment by vocabulary AND an + # entry the sheet dresses somebody in, and it is read from THIS SENTENCE only + # -- a garment mentioned in an earlier sentence of the same beat is not what + # "it" refers to, and guessing across a full stop is how a coat comes off in + # a beat about a towel. + if len(found) == _before and _PRONOUN_OBJECT.match(span): + _near = [] + for _g in garments_in(_sentence_before(beat, m.start())): + _low = engine.singular_garment(_g) or _g + if (_low in _NOT_A_GARMENT or _RESTRAINT_WORD.match(_low) + or not _is_entry_head(_low, scene) or _low in _near): + continue + _near.append(_low) + # Compared on the garment KEY, not the word. The earlier clause names + # the thing in full ("the chastity belt") while the word loop recorded + # its head ("belt"), so a plain membership test read them as two + # garments and took the same one off twice. + if len(_near) == 1 and not any(engine._garment_key(x) + == engine._garment_key(_near[0]) + for x in found): + found.append(_near[0]) # A garment the beat says is EXPOSED cannot also be one it takes off. "Pulls off # her coat to show the jumper underneath" ran the removal verb's object span past # "to show" and took the jumper with it -- so the one garment the beat exists to @@ -6277,8 +7996,14 @@ _NAKED_CUE = re.compile( r"\bnaked\b(?!\s+(?:eye|flame))" r"|\bnude\b|\bin\s+the\s+nude\b" r"|\bundress(?:es|ed|ing)?\b" - r"|\bstrips?\s+(?:out\s+of|off|down|naked|bare)\b|\bstripp(?:ed|ing)\s+" - r"(?:out\s+of|off|down|naked|bare)\b" + # "strips off" and "strips out of" only when nothing specific follows. "She strips + # off her coat" named a coat and read as naked: coat, sweater, jeans and boots all + # came off, and the next shot called her bare. A named garment is handled as that + # garment; "strips off." and "strips off her clothes" still undress. + r"|\bstrips?\s+(?:down|naked|bare)\b|\bstripp(?:ed|ing)\s+(?:down|naked|bare)\b" + r"|\b(?:strips?|stripp(?:ed|ing))\s+(?:out\s+of|off)\b" + r"(?=\s*(?:[.,;!?]|$)|\s+(?:and|then|while|as)\b|\s+(?:everything|it\s+all|all\s+of\s+it)\b" + r"|\s+(?:(?:his|her|their|all\s+(?:his|her|their))\s+)?(?:clothes|clothing|garments|things|kit|outfit|gear)\b)" r"|\btakes?\s+(?:everything|it\s+all|all\s+of\s+it|the\s+lot)\s+off\b" # A GENERIC garment word as the object. "Sam takes off his clothes" is the # commonest way anybody writes this, and it named no garment the sheet lists, @@ -6401,14 +8126,38 @@ def extract_directives(beat): added.append(phrase) return "" - body = _ADD_LINE.sub(take_added, _REMOVE_LINE.sub(take_removed, beat or "")) + # `exact:` lines come OUT here and go back in downstream, untouched. Taking them + # out at the same point as the other directives is what keeps every reader in + # this file from seeing them -- see _EXACT_LINE. + body = _EXACT_LINE.sub("", _ADD_LINE.sub(take_added, _REMOVE_LINE.sub(take_removed, beat or ""))) return re.sub(r"\n{2,}", "\n", body).strip(), removed, added -def extract_removals(beat): - """Back-compatible shim: (body, removed tokens).""" - body, removed, _ = extract_directives(beat) - return body, removed +# What makes a garment-less fragment read as CONTINUING the item before it. A +# print cue, a quoted span, a pronoun pointing back, a fragment that opens with +# the preposition that would have followed the noun -- or a capitalised word +# placed ON the garment. Capitals alone are not enough: "PVC mini-skirt" is a +# material, and a first version took the skirt with the belt in front of it. +# "red lipstick" or "a tattoo across the lower back" has none of these and +# stands on its own; "BRAT across the back" and "with a bow at the hip" do not. +_PRINT_WORDS = re.compile( + r"\b(?:print(?:ed|s)?|lettering|letter(?:s|ed)?|text|reads?|reading|says|" + r"written|writing|embroider(?:ed|y)|emblazoned|stitched|stamped|logo|slogan|" + r"motto|monogram(?:med)?|words?|font|spell(?:s|ed|ing)?|its|it)\b", re.I) +_QUOTED_SPAN = re.compile(r'["“][^"”]+["”]') +_CAPS_WORD = re.compile(r"\b[A-Z]{2,}\b") # case matters +_ON_GARMENT = re.compile( + r"\b(?:across|on|along|down|over)\s+(?:the|its|her|his|their)\s+" + r"(?:front|back|chest|waistband|hem|seat|rear|crotch|straps?|cups?|hips?|" + r"bum|butt)\b", re.I) +_CONTINUES = re.compile(r"^\s*(?:with|across|along|down|over|on|at|bearing|" + r"reading|printed|lettered|emblazoned)\b", re.I) + + +def _continues_item(unit): + return bool(_PRINT_WORDS.search(unit) or _QUOTED_SPAN.search(unit) + or _CONTINUES.search(unit) + or (_CAPS_WORD.search(unit) and _ON_GARMENT.search(unit))) def hide_item(text, items): @@ -6425,18 +8174,27 @@ def hide_item(text, items): 22") never disappears, whatever else is in it.""" if not text or not items: return text + pats = [re.compile(r"(?:\b\w+[\w-]*\s+){0,3}?\b" + re.escape(str(i).strip()) + r"\b", + re.I) for i in items if str(i).strip()] out_lines = [] for line in str(text).split("\n"): frags, kept = line.split(","), [] - for n, frag in enumerate(frags): - new = frag - for item in items: - if not str(item).strip(): - continue - # The item, plus any adjectives sitting directly in front of it. - new = re.sub(r"(?:\b\w+[\w-]*\s+){0,3}?\b" - + re.escape(str(item).strip()) + r"\b", - "", new, flags=re.I) + trailing = False # the unit just before this one went with its garment + entry = ":" in line # a labelled sheet entry: where attribute lists live + for frag in frags: + # A UNIT IS A SENTENCE, not only a comma-fragment. A fragment holding + # "denim shorts. She wears a black thong. BRAT is printed across the + # back." kept all of it because the shorts were still in it, and + # shipped "She . BRAT is printed across the back." -- a stub and a + # stranded print. Each sentence is judged alone, and the ones kept + # are put back with the single space that separated them. + units, kept_units = re.split(r"(?<=[.!?])\s+", frag), [] + for unit in units: + new = unit + for p in pats: + # The item, plus any adjectives sitting directly in front of it. + new = p.sub("", new) + removed = new != unit # THE PRINT ON A COVERED GARMENT GOES WITH THE GARMENT. # # Reported: a thong under shorts, lettering on the thong, and the @@ -6457,23 +8215,43 @@ def hide_item(text, items): # which is the case hide_item exists to protect -- and never when the # fragment carries the person's LABEL, which would take their name out # of the sheet with it. - if (new != frag and ":" not in frag - and not garments_in(new) and re.search(r"\w", new)): - continue + if (removed and ":" not in unit + and not garments_in(new) and re.search(r"\w", new)): + trailing = True + continue # An article left standing alone ("a", "the") is not a garment, # so the fragment goes. A fragment carrying the person's LABEL # never reaches this test empty -- the removal takes the item and # leaves the name -- which is why there is no separate guard for # it. One was written; a disable-check showed it never fired, and # a guard that looks protective and is not is worse than none. - if not re.sub(r"\b(?:a|an|the|and|with|in)\b|[\s,.;]", "", new): - continue - kept.append(new) + if not re.sub(r"\b(?:a|an|the|and|with|in)\b|[\s,.;]", "", new): + if removed: + trailing = True + continue + # THE PRINT IN ITS OWN FRAGMENT GOES TOO. The rule above catches a + # print written inside the garment's fragment; one written after + # the comma -- "a black thong, BRAT across the back, denim shorts" + # -- had nothing removed from it, so it stayed, now sitting right + # before the shorts with no garment to carry it. Reported as the + # thong's lettering on the shorts, again. A garment-less unit that + # reads as continuing the one just dropped goes with it; anything + # else stands on its own and ends the chain. + if (entry and not removed and trailing and ":" not in unit + and not garments_in(unit) and _continues_item(unit)): + continue + kept_units.append(new) + trailing = False + if kept_units: + kept.append(" ".join(kept_units)) joined = ",".join(kept) # Tidy the seams the removal leaves: doubled commas and spaces. joined = re.sub(r"\s*,\s*,+", ",", joined) joined = re.sub(r"\s{2,}", " ", joined).strip() joined = re.sub(r",\s*([.;]|$)", r"\1", joined) + # A dropped sentence can leave the next fragment's comma sitting right + # after the previous full stop: "A bright beach., on the sand". + joined = re.sub(r"([.!?])\s*,\s*", r"\1 ", joined) # The seams a removal leaves at the LABEL. "Ana: chastity belt, jeans" # becomes "Ana: , jeans" and "Ana: a chastity belt" becomes "Ana: ." # Both are malformed, and a sheet entry the reader cannot parse is @@ -6491,6 +8269,71 @@ def hide_item(text, items): return "\n".join(out_lines) +def strippers_in(beat, sheet): + """Who this beat says takes something off. [] when it does not say. + + subjects_for with the compound subject vocal_sources_in already needed: "McKenna + and Tess take off their shirts" shares ONE verb between two names, and the + conjunction guard -- right about "Dan holds the door and McKenna undresses", where + `and` opens a new predicate -- cannot tell that apart on its own. Getting this + wrong in the narrowing direction would leave a garment on somebody who took it + off, so both names are kept.""" + verbs = engine._STRIP_VERB + "|" + engine._UNDO_VERB + b = str(beat or "") + out = list(subjects_for(b, sheet, verbs)) + for n, _ln in sheet_lines(sheet): + if not n or n in out: + continue + if re.search(r"\b" + re.escape(n) + r"\b(?:\s*,\s*[\w'\u2019-]+)*" + r"\s+and\s+[\w'\u2019-]+\s+(?:" + verbs + r")\b", b, re.I): + out.append(n) + return out + + +# ONE LIST ENTRY, SEVERAL GARMENTS. A sheet lists what somebody wears between commas, +# and an entry often holds more than one thing: "long red coat over a grey sweater", +# "a white shirt under a navy jacket", "a grey coat and black boots". Taking off the +# coat dropped the whole entry, so the sweater went with it -- and the removal shot +# then called her chest bare, because nothing left in the text covered it. One layer +# of clothing described three ways across one cut: on, gone, and skin. +# +# ...and ONE GARMENT, SEVERAL WORDS. "a denim jacket with rolled sleeves and a hood" +# split on its "and" left "a hood" behind when the jacket came off -- a hood with no +# garment under it. An "and" inside a "with" phrase joins parts of the SAME garment, +# unless what follows is a garment of its own ("a grey coat with a fur collar and black +# boots" is still two things). +_ENTRY_SEP = re.compile(r"(\s+(?:over|under|beneath|underneath|on\s+top\s+of|and)\s+)", re.I) +_LAYER_SEP = re.compile(r"^\s+(?:over|under|beneath|underneath|on\s+top\s+of)\s+$", re.I) +_GARMENT_PART = {"sleeve", "sleeves", "hood", "collar", "lapel", "lapels", "pocket", + "pockets", "button", "buttons", "zip", "zipper", "lining", "trim", + "fringe", "laces", "hem", "neckline", "print", "logo", "stripes", + "pattern", "cuffs", "cuff", "straps", "strap", "buckle", "badge"} + + +def entry_parts(frag): + """[(separator before it, text)] -- the garments one comma entry names, in order.""" + bits = _ENTRY_SEP.split(frag or "") + parts = [("", bits[0])] + for i in range(1, len(bits) - 1, 2): + sep, text = bits[i], bits[i + 1] + head = (re.findall(r"[a-z]+", text.lower()) or [""])[-1] + joins_with = (not _LAYER_SEP.match(sep) and re.search(r"\bwith\b", parts[-1][1], re.I) + and head in _GARMENT_PART) + if joins_with: + parts[-1] = (parts[-1][0], parts[-1][1] + sep + text) + else: + parts.append((sep, text)) + return parts + + +def join_entry_parts(parts): + """The entry again from the parts kept, the first one losing its separator.""" + out = "" + for i, (sep, text) in enumerate(parts): + out += (text if i == 0 else sep + text) + return out.strip() + + def scrub_removed(text, tokens): """Drop the parts of `text` that name a removed item. @@ -6542,9 +8385,12 @@ def scrub_removed(text, tokens): # and black boots". Dropping it whole takes the innocent one with # it, and an undescribed garment is one the model re-invents. So # drop only the side that names the removed item. - sides = re.split(r"\s+\band\b\s+", frag, flags=re.I) - gone = [s for s in sides if any(p.search(s) for p in pats)] - keep = [s for s in sides if s not in gone] if len(sides) > 1 else [] + # ...and "over"/"under" join layers the same way. See entry_parts. + _parts = entry_parts(frag) + gone = [t for _sep, t in _parts if any(p.search(t) for p in pats)] + _kept_parts = ([(sep, t) for sep, t in _parts if t not in gone] + if len(_parts) > 1 else []) + keep = [join_entry_parts(_kept_parts)] if _kept_parts else [] # A PERSON's tag must not leave with a garment that happened to share # its fragment -- losing it costs that shot its identity reference. # An OBJECT's tag is the opposite case: "a silver locket " @@ -6659,181 +8505,6 @@ def scrub_removed(text, tokens): return " ".join(kept).strip() -# --- upscaling --------------------------------------------------------------- - -# How many frames go through one resize call. The whole chain used to go in one, -# which is what made this the largest allocation in the node -- see below. -RESIZE_CHUNK = 32 - - -def _stream_chunks(total): - """A collector that writes upscaled chunks into ONE destination as they land. - - Both chunk loops in _upscale_frames used `out.append(...)` then - `frames = torch.cat(out, dim=0)`. That is the shape the finished-chain join was - rebuilt to stop, at a LARGER size: the list holds the whole upscaled chain and - the cat allocates a second one, both live at the cat, and `out` is a local that - is never cleared -- so it survives the cat, survives the trailing resize, and is - still bound at the return. Meanwhile the CALLER's pre-upscale chain cannot be - dropped either, because `part = frames[s:s+batch]` is a view into it. - - At 2580 frames of 1056x608 that is 9.26GB per copy per doubling: 37GB x2 at 2x, - and 148GB x2 with the RealESRGAN_x4plus that is sitting in models/upscale_models. - Preallocating from the first chunk and copying into it removes exactly one of - those two, and drops the list at the same time. - - The destination is sized from the FIRST chunk, so the model's scale factor does - not have to be known in advance, and the frame count is the caller's own -- an - upscaler changes width and height, never the number of frames.""" - state = {"dst": None, "at": 0} - - def put(piece): - if state["dst"] is None: - state["dst"] = torch.empty((int(total),) + tuple(piece.shape[1:]), - dtype=piece.dtype, device=piece.device) - k = int(piece.shape[0]) - end = min(state["at"] + k, state["dst"].shape[0]) - if end > state["at"]: - state["dst"][state["at"]:end].copy_(piece[:end - state["at"]]) - state["at"] = end - - def done(): - d, at = state["dst"], state["at"] - if d is None: - return None - return d if at == d.shape[0] else d[:at] - - return put, done - - -def _resize_short_edge(frames, target, method="lanczos", chunk=0): - """Resize a [B,H,W,C] frame batch so its short edge == target (keeping aspect, - snapped to /32). Plain high-quality resize -- enlarges, doesn't add detail. - - IN CHUNKS, BECAUSE LANCZOS IS FOUR FULL-LENGTH COPIES. The whole chain went - into one common_upscale call, and comfy.utils.lanczos is three successive list - comprehensions over every frame at once: - - images = [Image.fromarray(...) for image in samples] # N at source size - images = [image.resize(...) for image in images] # N at target size - images = [torch.from_numpy(np.array(im).astype(np.float32)/255.) ...] - result = torch.stack(images) - return result.to(samples.device, samples.dtype) - - A comprehension builds the new list completely before rebinding the name, so at - each rebind BOTH are live; then torch.stack allocates a full copy while its list - still exists, and .to() allocates the result while the stack still exists. Note - the astype(np.float32): the input is fp16 but the two largest transients are at - DOUBLE its width. At 2580 frames to a 1080 short edge that peaked around 147GB - to produce a 29GB result, and it fires on a DOWNSCALE too. - - Chunked, the peak is the result plus one chunk's worth of that machinery. It is - bit-identical: PIL resizes each frame independently, so per-chunk and per-chain - give the same pixels. The early return for an already-correct size is kept, so - the common no-op case still allocates nothing.""" - b, h, w, c = frames.shape - if min(h, w) == target: - return frames - if h <= w: - nh = target; nw = max(32, int(round(target * w / h / 32) * 32)) - else: - nw = target; nh = max(32, int(round(target * h / w / 32) * 32)) - step = max(1, int(chunk) or RESIZE_CHUNK) - out = torch.empty((b, nh, nw, c), dtype=frames.dtype, device=frames.device) - for i in range(0, b, step): - part = comfy.utils.common_upscale( - frames[i:i + step].movedim(-1, 1), nw, nh, method, "disabled") - out[i:i + step].copy_(part.movedim(1, -1)) - del part - return out - - -def _upscale_frames(frames, mode, model_name, target_short_edge, batch=4): - """Optional post-pass upscale of the finished frames (on CPU). - mode 'model' : run a ComfyUI upscale model (Real-ESRGAN/UltraSharp class) - via the registered loader+apply nodes, chunked with cleanup - so 2000+ frames don't OOM; then fit to target short edge. - mode 'rtx' : NVIDIA RTX Video Super Resolution (Tensor Cores; fastest, - best quality for video -- needs Nvidia_RTX_Nodes_ComfyUI). - mode 'lanczos' : plain high-quality resize to the target short edge. - Any failure falls back to lanczos (or the raw frames), so it never breaks a - render. Returns (frames, note). NOTE: this SHARPENS/ENLARGES; it does not - reconstruct video detail the way a second-model (LTX 2.3) pass does.""" - if mode == "off" or frames is None or getattr(frames, "shape", [0])[0] == 0: - return frames, "" - note = "" - if mode == "rtx": - # NVIDIA RTX Video Super Resolution (Comfy-Org/Nvidia_RTX_Nodes_ComfyUI). - # Runs on RTX Tensor Cores -- far faster than ESRGAN-class models and - # generally cleaner on video, though like them it enhances/enlarges rather - # than reconstructing detail (an LTX 2.3 re-generation does that). - try: - rtx = (_find_node(["rtx", "video", "super"]) or _find_node(["rtxvideosuperresolution"]) - or _find_node(["rtx", "upscale"])) - if rtx is None: - raise RuntimeError("RTX node not installed (Nvidia_RTX_Nodes_ComfyUI)") - scale = 2 - if target_short_edge and int(target_short_edge) > 0: - cur = min(frames.shape[1], frames.shape[2]) - if cur > 0: - scale = max(1, min(4, int(round(int(target_short_edge) / cur)))) - _put, _done = _stream_chunks(frames.shape[0]) - n = frames.shape[0] - step = max(1, int(batch)) - for st in range(0, n, step): - part = frames[st:st + step] - res = None - for kw in ({"image": part, "scale": scale}, {"images": part, "scale": scale}, - {"image": part, "scale_factor": scale}, {"image": part}): - try: - res = _invoke_node(rtx, **kw); break - except TypeError: - continue - if res is None: - raise RuntimeError("RTX node signature not recognized") - _put(res.detach().to("cpu")) - del res, part - _deep_cleanup() - frames = _done() - note = f"RTX Video Super Resolution x{scale}" - if target_short_edge and int(target_short_edge) > 0: - frames = _resize_short_edge(frames, int(target_short_edge)) - note += f"; fit to {int(target_short_edge)}px short edge" - return frames, note - except Exception as e: - mode = "model" - note = f"RTX upscale unavailable ({e}); fell back to model/lanczos" - if mode == "model" and model_name and model_name != "none": - try: - loader = _find_node(["upscale", "model", "load"]) or _find_node(["loadupscalemodel"]) - applier = _find_node(["imageupscale", "model"]) or _find_node(["upscaleimageusingmodel"]) - if loader is None or applier is None: - raise RuntimeError("upscale-model nodes not found") - up_model = _invoke_node(loader, model_name=model_name) - _put, _done = _stream_chunks(frames.shape[0]) - n = frames.shape[0] - for s in range(0, n, max(1, int(batch))): - part = frames[s:s + max(1, int(batch))] - res = _invoke_node(applier, upscale_model=up_model, image=part) - _put(res.detach().to("cpu")) - del res, part - _deep_cleanup() - frames = _done() - note = f"upscaled with {model_name}" - except Exception as e: - mode = "lanczos" - note = f"model upscale unavailable ({e}); used lanczos" - if target_short_edge and int(target_short_edge) > 0: - try: - frames = _resize_short_edge(frames, int(target_short_edge)) - note = (note + "; " if note else "") + f"fit to {int(target_short_edge)}px short edge" - except Exception as e: - note = (note + "; " if note else "") + f"resize failed ({e})" - elif mode == "lanczos" and not note: - note = "lanczos selected but no target set -> unchanged" - return frames, note - - def _upscale_model_list(): """Filenames in models/upscale_models, plus 'none'. Read fresh at INPUT_TYPES time so newly-added models show up on a graph reload.""" @@ -6906,156 +8577,48 @@ def _latent_upscale_model_list(): return ["off"] + names -def _find_node(substrings): - """Find a registered node whose key contains all of `substrings` (lowercased).""" - maps = getattr(nodes, "NODE_CLASS_MAPPINGS", {}) or {} - for k, v in maps.items(): - kl = k.lower() - if all(s in kl for s in substrings): - return v - return None - - -def _invoke_node(cls, **kwargs): - """Call a registered ComfyUI node (V1 FUNCTION or V3 execute) with kwargs and - return its first output. Used to reuse ComfyUI's own upscale-model loader/apply - so we don't reimplement spandrel loading or tiled scaling.""" - inst = cls() - fn = None - if getattr(cls, "FUNCTION", None) and hasattr(inst, cls.FUNCTION): - fn = getattr(inst, cls.FUNCTION) - else: - for cand in ("execute", "upscale", "load_model", "load"): - if hasattr(inst, cand): - fn = getattr(inst, cand); break - if fn is None: - raise RuntimeError("no callable entrypoint") - out = fn(**kwargs) - out = getattr(out, "result", out) - return out[0] if isinstance(out, (tuple, list)) else out - - def latent_upscaler_node(): return _find_node(["minimaxh3latentupscaler", "3d"]) or _find_node(["minimaxh3latentupscaler"]) -# --- one shot's conditioning ------------------------------------------------ +def landing_schedule(model, scheduler, steps, shift_video, shift_audio): + """The sigmas this shot will run on, with the audio landing added. None if not. -def build_conditioning(clip, vae, audio_vae, prompt, width, height, length, - handoff=None, refs=None, - ref_noise_aug=0.999, silent=False, ref_image_size="match", - handoff_as_ref=False): - """Text + references + keyframe for a single shot. - - THE ONE RULE from H3's layout: a shot's conditioning rows are packed in the - order the tokenizer is given them, and tokenize_with_weights is either/or -- - passing minimax_ref_items makes it ignore `images` outright. So a reference and - a keyframe cannot be handed over separately; whatever the encoder is to see goes - in one list, numbered by position. - - So one roster, and it has to be readable under ONE format. A shot with a keyframe - is fl2va -- the keyframe is -- and reference images are dropped for - that shot, because on fl2va slot 2 means the LAST frame rather than a second - subject. See the comments below. - """ - latent, fc = _empty_av_latent(width, height, length, H3_FPS) - refs = [r for r in (refs or []) if r is not None] - - hand_img = None - if handoff is not None: - hand_img = _resize(handoff[:1], width, height, "disabled") - - # REFERENCES AND THE KEYFRAME RIDE TOGETHER. This is the arrangement the node - # had before I broke it, and the reason is in ComfyUI's own layout: - # - # model_base.py:2183-2191 cond_video_latents = keyframe latents THEN ref latents - # model.py PackedLayout emits keyframe "cond" segments THEN ref "ref_img" ones - # - # The two orders agree, so both channels coexist. A shot takes its references AND - # a real keyframe: the keyframe ANCHORS the first frame, which is what continuity - # needs, while a reference only supplies identity. They are not alternatives. - # - # I had read "" as MEANING the first frame on fl2va, and rearranged the - # roster around that. It does not. Which image is the first frame is decided by - # resolved_frame_index in minimax_keyframes, not by a label's number -- the labels - # are only how the images are shown to the VLM, and what they have to line up with - # is the tags in the prompt. - # - # So references come FIRST and keep slots 1..N, which is what a sheet line's - # `Name: , ...` points at, and the handoff is appended AFTER them where - # it disturbs no numbering. It has to be in the list at all because - # tokenize_with_weights is either/or: passing minimax_ref_items makes it ignore - # `images` outright, so leaving the handoff out means the VLM is never shown where - # the shot left off and re-imagines the scenery -- same place, new room. - keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG - # One aug covers every visual condition row, references AND the keyframe. Below - # KEYFRAME_SAFE_AUG the keyframe latent would be noised and labelled at the wrong - # timestep, so the handoff stops being an anchor and rides as an extra reference - # instead: weaker continuity, but nothing pretending to anchor while carrying noise. - # ...or because the caller asked for it. A shot that introduces somebody already - # in position wants the room this picture carries and NOT the first frame it - # would force, and that is a demotion the aug knows nothing about. - carry_as_ref = bool(hand_img is not None - and (handoff_as_ref or (refs and not keyframe_ok))) - - enc_refs = refs + ([hand_img] if carry_as_ref else []) - items, blocks = ([], []) - if enc_refs: - items, blocks = _build_ref_images(vae, enc_refs, width, height, ref_image_size) - if hand_img is not None and not carry_as_ref: - items = items + [{"type": "image", "data": hand_img}] - - if items: - tokens = clip.tokenize(prompt, minimax_ref_items=items) - else: - tokens = clip.tokenize(prompt) - cond = clip.encode_from_tokens_scheduled(tokens) - - vals = {} - if blocks: - vals["minimax_refs"] = blocks - # How CLEAN the references are shown. One aug covers every conditioning - # latent, keyframe included -- which is why softening references below - # KEYFRAME_SAFE_AUG would soften the anchor too. - if ref_noise_aug is not None: - vals["minimax_visual_cond_noise_aug"] = float(ref_noise_aug) - - kfs = [] - if hand_img is not None and not carry_as_ref: - kfs.append({"resolved_frame_index": 0, - "latent": _keyframe_latent(vae, hand_img)}) - # Silence on the audio branch for a shot with no scripted line. H3 is joint: - # an unconditioned audio stream invents a voice and the picture lip-syncs to it, - # and no sentence in the prompt outvotes a stream that has already decided - # someone is talking. PackedLayout emits a video segment only when a keyframe - # carries a `latent`, so an audio-only keyframe is legal and costs no frame. - if silent: - _SILENCE_STATUS["asked"] += 1 - if audio_vae is None: - _SILENCE_STATUS["why"] = "no audio VAE is wired to the node" - else: - sil = _silent_audio_latent(audio_vae, fc, H3_FPS) - if sil is None: - _SILENCE_STATUS["why"] = ("the audio VAE would not encode a silent " - "second -- the wrong VAE is on the " - "audio_vae input") - else: - kfs.append({"resolved_frame_index": 0, "audio_latent": sil}) - _SILENCE_STATUS["applied"] += 1 - if kfs: - vals["minimax_keyframes"] = kfs - if vals: - cond = node_helpers.conditioning_set_values(cond, vals) - return cond, latent, fc, carry_as_ref + None means "take the ordinary path and change nothing", and it is returned for + every reason there is: comfy not reachable, a schedule that already lands + softly, anything unexpected. The schedule is built the way KSampler builds it -- + calculate_sigmas(model_sampling, scheduler, steps) at denoise 1.0 -- so what is + handed back is the shot's own schedule with one step spliced into the end, + never a different one.""" + try: + import comfy.samplers as _cs + _ms = model.get_model_object("model_sampling") + base = [float(x) for x in _cs.calculate_sigmas(_ms, str(scheduler), int(steps))] + landed = insert_audio_landing(base, shift_video, shift_audio) + if len(landed) == len(base): + return None + return torch.tensor(landed, dtype=torch.float32) + except Exception: + return None def sample_shot(model, cond, negative, latent, seed, steps, cfg, sampler_name, - scheduler, sigmas=None): + scheduler, sigmas=None, shift_video=None, shift_audio=None, + soft_landing=False): """One sampling pass. denoise is fixed at 1.0: partial denoise desyncs the joint audio/video schedule.""" if sigmas is not None and len(sigmas): return _sample_on_sigmas(model, seed, cfg, sampler_name, cond, negative, latent, sigmas) + # THE AUDIO BRANCH'S LANDING. Only where the caller has established that this + # schedule drops the audio from a height, and only when the node is the one + # setting the shift -- with apply_model_sampling off, the shifts this is + # computed from are not the shifts the model is using. See insert_audio_landing. + if soft_landing: + _own = landing_schedule(model, scheduler, steps, shift_video, shift_audio) + if _own is not None: + return _sample_on_sigmas(model, seed, cfg, sampler_name, cond, negative, + latent, _own) (out,) = nodes.common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, cond, negative, latent, denoise=1.0) return out @@ -7080,6 +8643,9 @@ _WIDGET_RANGE = { "pace": (1.0, 0.25, 2.0, float), "ambient_level": (0.25, 0.0, 1.0, float), "foley_level": (0.35, 0.0, 1.0, float), + "speech_lead_seconds": (0.5, 0.0, 2.0, float), + "speech_tail_seconds": (2.0, 0.0, 10.0, float), + "hold_levels": (0.8, 0.0, 1.0, float), "handoff_frames": (1, 1, MAX_FRAMES, int), } @@ -7123,7 +8689,7 @@ def alignment_error(bad): shown = "; ".join(f"{n} = {v!r}, which is not one of {c[:3]}" + ("..." if len(c) > 3 else "") for n, v, c in bad[:3]) return ( - "H3 Long Videos: this node's saved widget values are out of position. " + "H3-LongVideos: this node's saved widget values are out of position. " + shown + ".\n\n" "Widget values are restored by POSITION, with no names stored, so converting " "a widget to an input -- or adding or removing one -- slides every value after " @@ -7200,7 +8766,19 @@ class H3LongVideos: "Every paragraph after it is one beat = one shot.\n\n" "Nothing is rewritten. What you type is what the shot is told, " "plus the scene line. Put a quoted \"line of dialogue\" in a beat " - "and that shot keeps its audio; beats without one are silenced."}), + "and that shot keeps its audio; beats without one are silenced.\n\n" + "A LINE THAT MUST REACH THE MODEL WORD FOR WORD goes on its own " + "line in the beat:\n" + " exact: her wrists stay behind her back the whole way\n\n" + "It is placed straight after the beat in your words, and nothing " + "in this node reads, scopes, scrubs, reorders or drops it. On a " + "short beat the node's own continuity clauses can be 70% of a " + "shot and the beat 8%, and this is the one instruction that does " + "not compete with them for room.\n\n" + "Nothing reads it either, on purpose: a name in it puts nobody in " + "the shot, a garment in it removes nothing, and a door in it " + "stages no change. Write what must be SAID; let the beat stage " + "what happens. `exactly:` and `verbatim:` do the same thing."}), "resolution": (list(NATIVE_RES), {"default": "16:9", "tooltip": "Aspect ratio. megapixels sets the size."}), "megapixels": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, @@ -7208,10 +8786,9 @@ class H3LongVideos: "faster and leaner; 0 keeps the preset's own dimensions. Cost " "scales with latent cells and attention is quadratic in them."}), "shot_seconds": ("FLOAT", {"default": 10.0, "min": 1.0, "max": 15.0, "step": 0.5, - "tooltip": "Length of EVERY shot. Uniform on purpose: noise is drawn to the " - "latent's shape, so shots of different lengths get unrelated noise " - "from the same seed and the grain resets at every cut. Snapped to " - "H3's 17k+5 frame grid."}), + "tooltip": "Maximum shot length. With 'from the beat', each shot is sized " + "independently up to this cap; with 'fixed', every shot uses this " + "length. Snapped to H3's 17k+5 frame grid."}), "steps": ("INT", {"default": 8, "min": 1, "max": 100, "tooltip": "6-8 with a turbo/distill LoRA; 20+ without one."}), "cfg": ("FLOAT", {"default": 1.0, "min": 1.0, "max": 20.0, "step": 0.1, @@ -7238,7 +8815,18 @@ class H3LongVideos: "It pins the WHOLE frame, so give it a composed frame of the shot you want: " "subject, pose, framing, background. A head-and-shoulders portrait wired here " "makes shot 1 a head-and-shoulders portrait. An identity portrait belongs on " - "ref_image_1, which says who the person is without dictating the frame."}), + "ref_image_1, which says who the person is without dictating the frame.\n\n" + "OR GIVE IT THE SET, with nobody in it, and the node will read it that way: " + "when beat 1 PLACES the cast rather than staging an entrance, and every one of " + "them already has a reference of their own, this picture carries " + "the room, the light and the furniture as a reference and never becomes frame " + "one. That is the difference between a set and an opening frame -- pinned as " + "frame one, a picture with nobody in it makes the cast appear out of nothing " + "during shot 1, which is the same reason a later shot refuses the previous " + "frame when it introduces somebody in position.\n\n" + "Both readings are reported in info, so you can see which one you got. To " + "force the pinned reading, put the cast in the frame and drop their " + " tags, or write the entrance into beat 1."}), "ref_image_1": ("IMAGE", {"tooltip": "Identity reference, applied to every shot unless the prompt places it with a " " tag. Kept on every shot on purpose: it is the only fixed anchor a " @@ -7345,8 +8933,8 @@ class H3LongVideos: "'remove:' line still works and is added to whatever is " "inferred."}), "restart_after_removal": ("BOOLEAN", {"default": True, - "tooltip": "After a shot with a 'remove:', start the NEXT shot fresh " - "instead of continuing from that shot's last frame.\n\n" + "tooltip": "After a shot that takes something off, the NEXT shot does " + "not open on that shot's last frame.\n\n" "Every shot is anchored to the previous shot's last frame. If " "the model does not finish taking the garment off inside its " "own shot, that frame still shows it -- and a keyframe is a " @@ -7354,9 +8942,12 @@ class H3LongVideos: "later shot inherits it too, with no wording able to undo it. " "This breaks that inheritance at the one boundary where the " "state changes.\n\n" - "The cost is a visible cut there, and that shot re-deriving its " - "pose and framing from the text. Turn it off if your removals do " - "complete on screen and you would rather keep the continuity."}), + "The frame still rides as a REFERENCE, so the room, the faces " + "and the clothes carry across; only when nobody is left in it, or " + "somebody in it also has a portrait riding the next shot, is " + "nothing carried. The cost is a cut there, with that shot re-deriving its " + "pose and framing. Turn it off if your removals do complete on " + "screen and you would rather keep the continuity."}), "hold_restraints": ("BOOLEAN", {"default": True, "tooltip": "Once a restraint is put on, keep it whole. From the shot " "that applies it onward, every shot carries one sentence: " @@ -7525,94 +9116,203 @@ class H3LongVideos: "'watching', 'studies'. It says nothing about where the " "camera is, so a shot looking straight down the line of " "sight is unaffected. Looking at a PERSON is left alone: " - "restating a pronoun says nothing the beat did not."}), + "restating a pronoun says nothing the beat did not.\n\n" + "A LINE WITH NOBODY NAMED TO LOOK AT turns the faces to " + "each other. Reported: two people talking to the camera " + "instead of each other. With no look staged, both faces " + "fall to the same portrait prior, and a line has an " + "addressee whether or not the beat wrote one. Said once, " + "impersonally, only with two or more people in the shot; " + "a beat that names a look is never argued with.\n\n" + "IT ALSO SPEAKS FOR THE EXPRESSION, because that is the " + "same pull. Reported: she smiles at the camera in a " + "scene of duress. A four-shot scene of a woman " + "handcuffed in a van -- pulling at the cuffs, " + "struggling, going limp -- had not one word in it about " + "anybody's face, and an attribute the prompt leaves out " + "is not left to the model, it is left to the model's " + "prior: a portrait, facing the lens, pleasantly. So a " + "shot whose sheet lists BINDING hardware on somebody in " + "it, or whose beat uses your own distress verbs, gets " + "one sentence -- the face shows the strain of it, the " + "mouth set. A collar alone does not trigger it, a shot " + "staging neither gets nothing, and a beat that already " + "says what the face does is never argued with. Picture " + "only: it can never open the audio branch."}), # APPENDED, like every widget before it. Saved workflows restore # widget values by POSITION with no names stored. "ambient_audio": ("AUDIO", {"tooltip": - "OPTIONAL OVERRIDE. Leave it empty and the bed is BUILT from the " - "scene -- the node has already read what the room sounds like, " - "and room tone is physically shaped noise, so it can be made " - "rather than fetched. No file needed and no second model pass.\n\n" - "Wire a recording here only when you want that recording: a real " - "location, or the events a synthesiser cannot make. Building " - "produces TONE -- air, rumble, plant, a mains hum, water, a " - "clock -- so a scene whose ambience is birdsong or a room full " - "of cutlery gets the room those things are in, not the things. " - "info says when that has happened.\n\n" - "Either way this is a MIX, not conditioning: it plays under what " - "the model generated, at the level you set. That is the " - "difference that makes it work -- ambience has nothing to " - "lip-sync to and asks nothing of the model, so it cannot put a " - "voice in a wordless shot.\n\n" - "Ambience derived in the prompt CANNOT do this. To score a " - "silent shot from text, the audio branch has to be left open, " - "and an open branch on a joint model fills itself -- at 4-8 " - "steps the last audio step resolves 50%-30% of its denoising in " - "one jump, and what it invents there is a voice. Wordless shots " - "keep their silent conditioning and get this bed on top instead, " - "which is what makes them sound like a room rather than a mute.\n\n" - "Looped with a crossfade to the length of the video, resampled if " - "it does not match, and downmixed or spread to match the " - "channels. Anything shorter than the film is fine."}), + "Wire a recording to play UNDER the finished soundtrack. Empty " + "means no bed at all.\n\n" + "This used to be an override on a bed the node BUILT out of the " + "scene's own wording. That builder is gone -- reported as sounding " + "horrid -- so the soundtrack is the model's, and this is the one " + "way to put a room under it.\n\n" + "It is PLAYED, not conditioned on, and that is the point: ambience " + "needs no cooperation from a joint model, has nothing to lip-sync " + "to, and so cannot put a voice in a wordless shot. It is resampled " + "and looped with a crossfade to the length of the film. " + "ambient_level sets how loud."}), "ambient_level": ("FLOAT", {"default": 0.25, "min": 0.0, "max": 1.0, "step": 0.01, - "tooltip": "How loud the bed sits under everything, and the " - "switch that turns it on: above 0 a bed is built " - "from the scene even with nothing wired to " - "ambient_audio. 0 turns it off entirely.\n\n" - "A built bed is normalised to a fixed RMS first, so " - "this means the same thing in every room -- the " - "default 0.25 lands near -34 dBFS, present and well " - "under a spoken line. 0.15-0.3 is a bed you notice " - "only when it stops.\n\n" + "tooltip": "How loud the recording wired to ambient_audio plays " + "under the finished soundtrack. With nothing wired this " + "does nothing -- the bed the node used to BUILD from " + "the scene is gone, reported as sounding horrid, and " + "the audio is the model's.\n\n" + "0.15-0.3 is a bed you notice only when it stops.\n\n" "If the sum would clip, the whole mix is scaled down " "rather than clipped, because clipping distorts the " "line, which is the part worth keeping."}), # APPENDED. Saved workflows restore widget values by position. "foley_level": ("FLOAT", {"default": 0.35, "min": 0.0, "max": 1.0, "step": 0.01, - "tooltip": "Build the sound an action makes, on shots that have " - "no line.\n\n" - "auto_sound already reads those sounds out of the beat " - "-- cuffs, a chain, a zip, footsteps -- but only as " - "TEXT in the prompt, and text can never open a shot's " - "audio branch, because an open branch on a joint model " - "invents a voice. So a wordless shot staging cuffs " - "going on was pinned to silence and the cue was " - "dropped: the one shot whose point is a sound made " - "none, and the only fix was writing the sound into the " - "beat by hand.\n\n" - "This builds it and mixes it into THAT SHOT'S span " - "instead. It asks nothing of the model, so it cannot " - "babble. Shots that already have a line, or a sound " - "you wrote yourself, are left alone -- their branch is " - "open and making that sound from the same prose, and " - "building over it would double every footfall.\n\n" - "A shot staging EFFORT is the exception and does get " - "built sound, even though its branch is open. It " - "opened to make a VOICE, and a voice is not a bed " - "frame or a chain -- so what is built there is the " - "non-vocal half the model will not make. Lower this " - "if anything doubles.\n\n" - "0.35 puts it about 37 dB below full scale: well clear " - "on a silenced shot, which sits near -65, and about 23 " - "dB under a spoken one. Raise it towards 0.6-0.7 if " - "you want it audible under a voice.\n\n" - "It is synthesis, not a recording: a click, a rattle, " - "a rustle, in the right place. Nothing vocal is ever " - "built. 0 turns it off; needs auto_sound on."}), + "tooltip": "DOES NOTHING. Kept only so saved workflows keep " + "loading: widget values are restored by POSITION with " + "no names stored, so deleting this one would load the " + "wrong number into the four widgets after it.\n\n" + "It used to set how loud the sounds this node BUILT " + "were -- a click, a rattle, a rustle, mixed into the " + "shots whose audio branch is pinned to silence, which " + "cannot get audio from the model at all because prompt " + "text never opens a branch. Removed on the report that " + "it sounded horrid; the soundtrack is the model's now, " + "whole.\n\n" + "WHAT THAT COSTS, said plainly: a shot with no line " + "and no sound you described is pinned to silence and " + "is SILENT. The pin stays -- it is what stops a free " + "branch filling itself with a voice and the face " + "lip-syncing to the babble. To put sound in such a " + "shot, write the sound into that beat, which opens its " + "branch on purpose and lets the model make it; or wire " + "a track to ambient_audio; or lay one under the " + "finished video outside the node."}), # APPENDED. Saved workflows restore widget values by position. + "speech_lead_seconds": ("FLOAT", {"default": 0.5, "min": 0.0, + "max": 2.0, "step": 0.1, + "tooltip": "Pin generated audio to encoded silence at the start of each " + "dialogue shot. This stops pre-babble and keeps the joint " + "model's mouth still during that span. 0 disables it; a long " + "lead can trim the first word."}), + "speech_tail_seconds": ("FLOAT", {"default": 2.0, "min": 0.0, + "max": 10.0, "step": 0.5, + "tooltip": "Free audio kept AFTER a dialogue shot's line, in seconds. The " + "line's length is estimated from its words; past lead + line + " + "this margin the audio is pinned to encoded silence, the way " + "the lead-in pins the opening. A short line in a long shot " + "otherwise leaves seconds of open branch the model fills with " + "more speech -- babble, or the line again. The model chooses " + "WHEN to speak, so a small margin can clip the last word: raise " + "it if it does. 0 disables it."}), + # APPENDED. Saved workflows restore widget values by position. + "beat_leads": ("BOOLEAN", {"default": True, + "tooltip": "Put the BEAT in front of the character sheet.\n\n" + "The sheet has to be in every shot, because clothing " + "continuity is read out of it. But it is a description of a " + "FACE -- 'she, 22, tall, long blonde hair, blue eyes' -- and " + "it was sitting in the opening tokens of every shot, ahead of " + "the action. Measured: 69% of a shot's words were in " + "sentences about a face, and turning every face guard off " + "only reached 63%, because the sheet is most of it.\n\n" + "What leads a prompt decides its composition: anatomy in the " + "opening tokens is what a distilled model settles the frame " + "on, which at cfg 1 no later sentence outvotes. On, the order " + "is scene, then what happens, then who it happens to. The " + "words are identical and none are rewritten -- only the " + "order changes.\n\n" + "Off restores the old order, so the two can be compared in " + "one render."}), + # APPENDED. Saved workflows restore widget values by position. + "hold_levels": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 1.0, + "step": 0.05, + "tooltip": "Take the grade the chain adds to itself back out of each " + "handoff.\n\n" + "Every shot after the first is sampled from the previous " + "shot's last frame. The model reproduces that frame " + "faithfully -- which is what continuity needs -- so it " + "inherits whatever is already in it, and it SYNTHESISES the " + "opening frame rather than copying it, so its own bias lands " + "on top. The VAE then clamps every decode to 0..1, which " + "makes the expansion a ratchet: headroom spent is not given " + "back. Eleven shots of that is crushed blacks, blown " + "highlights and lurid colour, invisible shot to shot and " + "obvious end to end.\n\n" + "What makes this correctable without knowing anything about " + "your scene: at every boundary the render holds two pictures " + "that are supposed to be the SAME frame -- the handoff it " + "gave the shot, and the opening frame that came back. " + "Nothing was asked to change between them, so everything " + "separating them is the chain's doing and none of it is " + "yours. That difference is what is measured, per colour " + "channel, per boundary, and the median across boundaries is " + "what is taken back out.\n\n" + "It does NOT aim at a target and never compares a shot to " + "shot 1, so a beat that walks into a darker room stays " + "darker: measured, a deliberate lighting step keeps about " + "98% of its size. The correction is a capped fraction per " + "boundary rather than a reset, because shot N's frames reach " + "the video ungraded while N+1 is sampled from a corrected " + "keyframe -- an uncapped correction would trade burn-in for " + "a pop at every cut.\n\n" + "1.0 flattens the trend hardest; lower leaves more of the " + "look alone. 0 is off. Watch the contrast line in info: if " + "it still says UP, raise this. It cannot undo clipping that " + "earlier shots already baked in, and it corrects levels " + "only -- not softening, and nothing spatial."}), + # APPENDED. Saved workflows restore widget values by position. + "hold_camera": ("BOOLEAN", {"default": True, + "tooltip": "Say the camera does not move, on every shot that does not " + "ask it to.\n\n" + "Reported as the camera moving on its own and breaking " + "continuity, and it is the chain that makes it expensive: " + "every shot opens on the PREVIOUS shot's last frame, so a " + "shot that drifts away from the viewpoint it started on " + "hands the drifted one forward. The next shot inherits it " + "and adds its own, and by shot four the room is a room " + "nobody framed.\n\n" + "An attribute the text does not state is left to the " + "model's prior, and for a video model that prior is " + "movement: a still camera is the one thing it has no reason " + "to produce unless the words ask. So one sentence asks: " + "one unbroken take, from one position, angle and distance. " + "It names no camera -- naming one is asking for one, and the " + "lens is what the gaze guards are trying to get people to " + "stop looking at -- and a take is the same fact from the " + "other side, which also says no cut inside the shot.\n\n" + "YOUR WORDS WIN. Any camera note in the beat or the anchor " + "-- a pan, a push in, handheld, a lens, 'shot on' -- stands " + "it down for that shot, and a journey between places keeps " + "its moving camera, because the node has already asked for " + "every step of it in frame."}), + # APPENDED. Saved workflows restore widget values by position. + "verbatim": ("BOOLEAN", {"default": False, + "tooltip": "Send your text and NOTHING this node writes.\n\n" + "On, a shot is your scene paragraph, your beat and the " + "character sheet entries for the people it names -- and that " + "is all. Every continuity clause goes: the body count, the " + "mouth guard, the camera take, the two ends of a door or a " + "walk, posture, gaze, bare regions, held states, sound " + "direction.\n\n" + "WHAT COMES BACK WITH THEM is every failure each one answers: " + "duplicate characters, a face lip-syncing to invented speech, " + "the camera drifting until the room is a different room, a " + "door that opens and shuts itself, a walk played backwards, a " + "garment that returns after it came off. Each was added for a " + "reported failure, and info still lists what it would have " + "said on every shot.\n\n" + "The MECHANISMS stay: the keyframe chain, the reference " + "claims, silence pinning, shot sizing, and the scoping that " + "decides which of your own sentences a shot gets. This switch " + "is about sentences the node WROTE.\n\n" + "Use it to see your prompt on its own, or to prove whether a " + "problem is the node's doing or the model's."}), + # APPENDED. Saved workflows restore widgets by position. "handoff_frames": ("INT", {"default": 1, "min": 1, "max": MAX_FRAMES, - "tooltip": "How many frames from the previous shot are used to " - "condition the next one. 1 is the upstream default: " - "the previous final frame becomes the next shot's " - "keyframe. Values above 1 keep that final-frame " - "keyframe and append the earlier tail frames as " - "claimed reference context, so the next beat can see " - "more of the incoming motion and room continuity. " - "The extra frames are references, not extra " - "keyframes, so seam trimming still removes only the " - "duplicated opening frame."}), + "tooltip": "How many frames from the previous shot are used to condition " + "the next one. 1 is the upstream default: the previous final " + "frame becomes the next shot's keyframe. Values above 1 keep " + "that final-frame keyframe and append the earlier tail frames " + "as claimed reference context for the next beat."}), }, } @@ -7639,6 +9339,76 @@ class H3LongVideos: character_guard=True, pace=1.0, auto_sound=True, hold_scene_state=True, mouths_shut_when_no_line=True, hold_gaze=True, ambient_audio=None, ambient_level=0.25, foley_level=0.35, + speech_lead_seconds=0.5, speech_tail_seconds=2.0, beat_leads=True, + hold_levels=0.8, hold_camera=True, verbatim=False, + handoff_frames=1, + **_removed): + # **_removed: a workflow saved with the old `save_defaults` widget still sends + # it. Swallowed rather than raising, so an existing workflow keeps loading. + + # An interrupt arrives as a BaseException (model_management.py:2133), so it is + # NOT caught by the `except Exception` handlers in this file and must not be -- + # stopping a run has to stop it. What it does skip is every `del` in the render + # loop, leaving a multi-gigabyte frame buffer to be freed by the collector in + # its own order, after ComfyUI has already started unloading the models it was + # sized against. Reported as an illegal memory access on stopping a run, thrown + # from cuMemFreeAsync inside a tensor destructor rather than from any line of + # Python. Dropping it here makes that free happen at a known point, before the + # unwind; the interrupt is then re-raised untouched. + self._frames = None + prepared = self._prepare( + model=model, clip=clip, vae=vae, + audio_vae=audio_vae, prompt=prompt, resolution=resolution, + megapixels=megapixels, shot_seconds=shot_seconds, steps=steps, + cfg=cfg, sampler_name=sampler_name, scheduler=scheduler, + seed=seed, first_frame=first_frame, ref_image_1=ref_image_1, + ref_image_2=ref_image_2, ref_image_3=ref_image_3, ref_image_4=ref_image_4, + negative=negative, sigmas=sigmas, shift_video=shift_video, + shift_audio=shift_audio, apply_model_sampling=apply_model_sampling, silence_nonspeech=silence_nonspeech, + trim_seam=trim_seam, ref_noise_aug=ref_noise_aug, tiled_decode=tiled_decode, + cleanup_between_shots=cleanup_between_shots, plan_only=plan_only, latent_upscale=latent_upscale, + latent_upscale_scale=latent_upscale_scale, upscale=upscale, upscale_model=upscale_model, + upscale_target_short_edge=upscale_target_short_edge, upscale_batch=upscale_batch, shot_length=shot_length, + hold_restraints=hold_restraints, restart_after_removal=restart_after_removal, auto_remove=auto_remove, + anchor=anchor, character_memory=character_memory, character_guard=character_guard, + pace=pace, auto_sound=auto_sound, hold_scene_state=hold_scene_state, + mouths_shut_when_no_line=mouths_shut_when_no_line, hold_gaze=hold_gaze, ambient_audio=ambient_audio, + ambient_level=ambient_level, foley_level=foley_level, speech_lead_seconds=speech_lead_seconds, + speech_tail_seconds=speech_tail_seconds, beat_leads=beat_leads, + hold_levels=hold_levels, hold_camera=hold_camera, verbatim=verbatim, + handoff_frames=handoff_frames, + **_removed) + if isinstance(prepared, PreparedVideo): + try: + return self._render(prepared) + except BaseException: + _f, self._frames = self._frames, None + if _f is not None: + try: + _f.release() + except Exception: + pass # teardown must not mask the interrupt + raise + finally: + self._frames = None + return prepared + + def _prepare(self, model, clip, vae, audio_vae, prompt, resolution, megapixels, shot_seconds, + steps, cfg, sampler_name, scheduler, seed, + first_frame=None, ref_image_1=None, ref_image_2=None, ref_image_3=None, + ref_image_4=None, negative=None, sigmas=None, + shift_video=12.0, shift_audio=3.0, apply_model_sampling=True, + silence_nonspeech=True, trim_seam=True, ref_noise_aug=0.999, + tiled_decode=True, cleanup_between_shots=True, plan_only=False, + latent_upscale="off", latent_upscale_scale=2.0, + upscale="off", upscale_model="none", upscale_target_short_edge=0, + upscale_batch=4, shot_length="from the beat", hold_restraints=True, + restart_after_removal=True, auto_remove=True, anchor="", character_memory="", + character_guard=True, pace=1.0, auto_sound=True, hold_scene_state=True, + mouths_shut_when_no_line=True, hold_gaze=True, + ambient_audio=None, ambient_level=0.25, foley_level=0.35, + speech_lead_seconds=0.5, speech_tail_seconds=2.0, beat_leads=True, + hold_levels=0.8, hold_camera=True, verbatim=False, handoff_frames=1, **_removed): # **_removed: a workflow saved with the old `save_defaults` widget still sends @@ -7665,6 +9435,9 @@ class H3LongVideos: ref_noise_aug=ref_noise_aug, latent_upscale_scale=latent_upscale_scale, upscale_target_short_edge=upscale_target_short_edge, upscale_batch=upscale_batch, pace=pace, + ambient_level=ambient_level, foley_level=foley_level, + speech_lead_seconds=speech_lead_seconds, + speech_tail_seconds=speech_tail_seconds, hold_levels=hold_levels, handoff_frames=handoff_frames)) megapixels, shot_seconds = _fixed["megapixels"], _fixed["shot_seconds"] steps, cfg = _fixed["steps"], _fixed["cfg"] @@ -7673,6 +9446,10 @@ class H3LongVideos: latent_upscale_scale = _fixed["latent_upscale_scale"] upscale_target_short_edge = _fixed["upscale_target_short_edge"] upscale_batch, pace = _fixed["upscale_batch"], _fixed["pace"] + ambient_level, foley_level = _fixed["ambient_level"], _fixed["foley_level"] + speech_lead_seconds = _fixed["speech_lead_seconds"] + speech_tail_seconds = _fixed["speech_tail_seconds"] + hold_levels = _fixed["hold_levels"] handoff_frames = _fixed["handoff_frames"] notes.extend(_fixnotes) # means ref_image_N, the socket. Everything downstream works on @@ -7703,7 +9480,19 @@ class H3LongVideos: swap = flush_for_model_change(model) if swap: notes.append(swap) + # Before anything is sampled, because the failure mode is an abort and an abort + # cannot be reported from inside the render. + _abort = sparse_attention_allocator_abort(model) + if _abort: + raise RuntimeError(_abort) check_vae_wiring(vae, audio_vae) + # Before anything else reads the script, for the same reason the abort above is + # here: the answer is a refusal, and a refusal has to happen before work does. + _refuse = minor_with_sexual_staging( + "\n".join([(character_memory or ""), (prompt or "")]), "\n".join( + [(prompt or ""), (anchor or ""), (character_memory or "")])) + if _refuse: + raise RuntimeError(_refuse) prompt, n_legacy = strip_legacy_fields(prompt) if n_legacy: @@ -7724,10 +9513,43 @@ class H3LongVideos: # the scene, so it is re-stamped into EVERY shot -- which is what makes a # removal stick and what stops a later shot describing no clothing at all. beats, sheet = pull_character_sheets(beats) + # THE EXACT LINES COME OUT HERE, ONCE, and go back in where the shot text is + # assembled. Taking them out at the source is what makes "nothing reads it" + # true of every reader rather than of the ones that were remembered: the film + # mood, the shot sizing and the multi-line check all take the beats as they + # are, and a first version that stripped them further downstream had an exact: + # line about cuffs setting the mood of the whole film. See _EXACT_LINE. + _exact_all = [exact_lines(b) for b in beats] + beats = [_EXACT_LINE.sub("", b).strip() for b in beats] # The sheet is kept APART from the rest of the scene: it is the part that # varies per shot, because only the people a beat involves should be # described in it. Everything else is stamped on every shot unchanged. sheet, _dupes = merge_sheets((character_memory or "").strip(), sheet) + # Read ONCE, over the whole script, and AFTER character_memory is merged in + # -- that is where the wrists usually are. A shot of the captor alone is grim + # on account of what the sheet says three beats ago, so this cannot be a + # per-shot question. See film_stages_duress. + # A DECLARED AGE UNDER 18 GETS NO BODY DESCRIBED FOR IT, and the author is told + # so rather than left to wonder why one entry reads differently from the rest. + # The scene itself renders: children are in films. What is withheld is this + # node's own anatomy clauses, every one of them. See body_of and figure_of, and + # minor_with_sexual_staging for the case that does not render at all. + _minors = sorted({_n for _n, _ln in sheet_lines(sheet) + if _n and 0 < age_in(_ln) < ADULT_AGE}) + if _minors: + notes.append( + f"{_join_names(_minors)} " + f"{'are' if len(_minors) > 1 else 'is'} declared under {ADULT_AGE} on " + f"the sheet, so NO body is described for " + f"{'them' if len(_minors) > 1 else _minors[0]} by this node -- not a " + f"softer description, none. Every clause that would name a body, a bare " + f"region's anatomy or a figure stays silent for that entry, and the rest " + f"of the film is unaffected. The scene renders. Had the script also " + f"staged nudity or sex anywhere in it, nothing would have rendered at " + f"all. If the age is a typo, fix it and the entry behaves like any other" + ) + _film_mood = mood_declared(anchor) + _film_duress = film_stages_duress(beats, sheet, anchor) if _dupes: notes.append( f"{', '.join(_dupes)} described more than once -- character_memory and a " @@ -7735,8 +9557,27 @@ class H3LongVideos: f"using both put the person in every shot twice. A model told about one " f"person twice renders two of them. Kept the character_memory entry and " f"dropped the duplicate") + # NOBODY FOR A PRONOUN TO REACH. "He sits down at the table." against a sheet + # that declares no pronoun for Owen kept the previous shot's cast -- Maya alone + # -- so the shot described a woman for a beat about a man, and the model drew + # him from nothing beside her. Nothing here can know who "he" is without the + # sheet saying, and guessing from a name is not knowing. Said, so it is fixed + # where it can be: in the sheet. + _undeclared = [n for n, ln in sheet_lines(sheet) if n and not sheet_pronoun(ln)] + if _undeclared and any(re.search(r"\b(?:he|she|him|her|his|hers)\b", b or "", re.I) + for b in beats): + notes.append( + f"{_join_names(_undeclared)} {'have' if len(_undeclared) > 1 else 'has'} no " + f"pronoun on the sheet, and the script uses he/she -- so a beat that says " + f"\"he\" or \"she\" instead of a name cannot be resolved to " + f"{'them' if len(_undeclared) > 1 else 'that entry'}, and the shot keeps " + f"whoever the previous one described instead. Write the pronoun into each " + f"entry (\"Owen: he, 42, ...\")") static = build_scene(anchor, scene, "", "") scene = build_scene(anchor, scene, "", sheet) # the whole of it, for inference + # Which rooms the author actually DESCRIBES. A room the text only names is a + # room the model invents; this is what the warning below is read from. + _described_rooms = set(rooms_named(static)) if sheet: notes.append(f"folded {sheet.count(chr(10)) + 1} character-sheet line(s) into " f"the scene instead of spending a shot on them -- a sheet " @@ -7766,6 +9607,42 @@ class H3LongVideos: f"shots" + (f", {_sheets} folded in as character sheet(s)" if _sheets else "") + ("" if (anchor or "").strip() else ", 1 kept as the scene")) + # THE SCENE PARAGRAPH THAT QUIETLY BECAME A SHOT. + # + # Reported as a van changing direction between shots. Filling in `anchor` + # makes every paragraph a beat -- the anchor is then the scene -- and that is + # deliberate, and the widget's tooltip says so. What it MEANS is that a + # prompt whose first paragraph is scene text loses that text after shot 1: + # + # no anchor 4 shots van direction carried: yes yes yes yes + # with anchor 5 shots van direction carried: yes NO NO NO NO + # + # The heading of a vehicle, the location, the time of night -- stated once, + # spent on a shot of their own, and never said again. Nothing reported it at + # runtime, so what the author sees is the van turning round between takes, + # with no way to connect that to a widget they filled in for the camera. + # + # Read as "this paragraph puts nobody on screen", which is this file's own + # test for whether there is a person in a beat. An opening ACTION is a real + # beat and is left alone: telling somebody to move it into the anchor would + # be wrong, and the anchor branch exists precisely because the no-anchor path + # was making scenes out of actions. + _first_para = beats[0] if beats else "" + if ((anchor or "").strip() and len(beats) > 1 and _first_para + and not beat_puts_somebody_on_screen(_first_para, sheet)): + _quoted = " ".join(_first_para.split()) + notes.append( + f"the prompt's first paragraph describes a PLACE rather than staging " + f"anything, and with `anchor` filled in every paragraph is a beat -- so " + f"it is being spent as shot 1 and is not carried into any other shot: " + f"\"{_quoted[:100]}{'...' if len(_quoted) > 100 else ''}\". Whatever it " + f"establishes -- which way a vehicle faces, the room, the time of night " + f"-- is said once and then gone, and the shots after it are free to put " + f"it back differently. That is what a van changing direction between " + f"shots looks like from the outside. Move it into `anchor`, which is " + f"carried at the front of EVERY shot, or clear `anchor` and let the " + f"first paragraph be the scene as it is without one. Use one or the " + f"other: with both, all of the standing description belongs in `anchor`") # Paragraphs are separated by a BLANK line. Lines joined by a single newline # are ONE beat, so three actions written on three lines become one shot with # three actions in it, and two of them look like they were absorbed. @@ -7777,7 +9654,7 @@ class H3LongVideos: f"newline between them are one beat and share one shot. If those were " f"meant to be separate shots, put an empty line between them") if not beats: - raise RuntimeError("H3 Long Videos: no beat to render. Every paragraph after " + raise RuntimeError("H3-LongVideos: no beat to render. Every paragraph after " "the first is one shot; a character sheet ('Name: ...') " "is folded into the scene and does not count as one.") @@ -7787,13 +9664,16 @@ class H3LongVideos: # the scene stops describing a garment a beat has taken off. It applies to # the removing shot too: the keyframe already shows the garment on at the # start, and a description saying it is still worn is what puts it back. - shots, speech, gone, shown = [], [], [], [] - shot_events = [] # per shot: the sounds its action makes - sounded = [] # beats that ask for a sound of their own + plan = ShotPlan() + gone, shown = [], [] + # token -> who took it off, so the scrub reaches their entry and nobody + # else's. A token with nobody recorded stays unscoped. See scrub_removed. + gone_by = {} + _extras_seen = False # the film has staged people the sheet does not name + untracked_strip = [] # (shot, items) a group removal the sheet cannot hold # Of those, the ones open ONLY because the beat stages effort. The branch # is open on both, but for opposite reasons, and built sound has to tell - # them apart -- see the foley mix. - voiced_only = [] + # them apart. (It used to matter for the foley mix as well, which is gone.) inferred_sound = [] # shots given one derived from their action restrained = posed = rigid_latched = False # Has any BEAT stated a posture yet? The scene fallback for the weight @@ -7849,7 +9729,22 @@ class H3LongVideos: here = place_named(scene) or first_place(scene) # The film's ambient bed, read from the anchor and the scene rather # than typed into every beat. See scene_ambient. - ambient_bed = scene_ambient(anchor, scene) if auto_sound else "" + # THE OPENING BEAT IS THE FALLBACK, the same one room_tone has had all along + # and for the same stated reason: with `anchor` set there is no scene + # PARAGRAPH, and an anchor describes the camera rather than the room. Without + # it the film's bed died for everybody who filled in the widget the tooltips + # tell them to fill in -- and the bed is what this file's own answer to + # lead-in babble depends on: "a branch with a bed to lay down does not need + # to invent a voice to fill the space." Reported as babble at the opening of + # the beat, measured as three speaking shots with no sound clause at all. + # + # Read only when the scene names nothing, exactly as room_tone reads it, so + # a film whose scene DOES name a space is unchanged. + _opening = extract_directives(beats[0])[0] if beats else "" + ambient_bed = (scene_ambient(anchor, scene) + or scene_ambient(anchor, _opening)) if auto_sound else "" + _bed_src = ("the anchor and the scene" if scene_ambient(anchor, scene) + else "the opening beat") ambient_shots = [] # shots given the bed posture_shots = [] # shots told to keep a standing posture travel_shots = [] # shots that move between places @@ -7858,14 +9753,16 @@ class H3LongVideos: paced_shots = [] # shots told to spread their action staging_shots = set() # shots that MOVE a garment on screen bared_shots = [] # ...and shots that uncover skin - bare_held = [] # ...and shots told a region is STILL bare crowded = [] # (shot, clauses dropped for room) absent_hold = [] # shots where the wearer is not on screen exposed_by_beat = [] # (shot, garments the beat names while covered) named_shots = [] # shots reminded the thing is still there anchored_shots = [] # shots reminded of it gaze_shots = [] # shots told where the look goes - looking_at = "" # the target, held until it changes + dialogue_gaze_shots = [] # dialogue shots turned to face each other + scene_held = [] # (shot, rooms) whose scene description waited + scene_welded = [] # ...and shots where it could not be held + looking_at = {} # {name: target}, each held until it changes fall_shots = [] # shots told what takes the landing device_shots = [] # shots whose line belongs to a machine applied_shots = [] # shots that put the hardware on @@ -7889,8 +9786,22 @@ class H3LongVideos: stated_shots = [] # shots given a state put at the first frame turned_shots = [] # shots given both ends of a staged change mouth_shut = [] # shots told every mouth is closed + mouth_acting = [] # ...and the ones whose beat works the mouth + duress_shots = [] # shots told what the face is doing + vocal_shots = [] # shots where a vocal was given an owner muted_sound = [] # shots whose written sound was given up for it stripped_shots = set() # 0-based shots that took something off + cut_shots = set() # 0-based shots opening in a room the keyframe is not in + shot_rooms = {} # 0-based shot -> (room it opens in, room it ends in) + hardware_changed = set() # 1-based shots that put hardware on or take it off + _undescribed = [] # rooms the film enters that the prompt never describes + open_moves = [] # (shot, where) moves to a place the list cannot name + frame_shots = [] # shots told what the frame holds + exact_shots = [] # shots carrying an exact: line of the author's + camera_shots = [] # shots told the camera holds still + named_often = [] # (shot, name, times named, times this node named them) + contact_shots = [] # shots told which body is with which + led_shots = [] # shots whose beat was put ahead of the sheet restarted = [] # shots started fresh after a removal restored = [] # garments an add: put back on wearing_shots = [] # shots that put one back on, given both ends @@ -7951,15 +9862,13 @@ class H3LongVideos: # The acoustic of the space, read once: it is the same room in every shot. # The opening beat is the fallback: with `anchor` set there is no scene # paragraph, and an anchor describes the camera rather than the room. - _opening = extract_directives(beats[0])[0] if beats else "" _room = room_tone(scene, _opening) if auto_sound else "" _room_src = "the scene" if room_tone(scene) else "the opening beat" - # The same two readings, kept for the MIX and not gated on auto_sound. - # auto_sound governs what goes in the PROMPT, which is a conditioning-side - # question -- the mixed bed conditions nothing, so turning the prompt-side - # inference off should not also silence the room. - _mix_bed = scene_ambient(anchor, scene) - _mix_room = room_tone(scene, _opening) + # The two readings that used to be kept for the MIX as well -- ungated by + # auto_sound, because the built bed conditioned nothing -- are gone with the + # builder that consumed them. The readings themselves still run for the + # PROMPT, a few lines down, which is the conditioning side and the only side + # left. See the note at the top of audio.py. if _room: notes.append(f"room tone read from {_room_src}: {_room}. It goes under the " f"shots whose audio branch is already open -- ones with a line, " @@ -7971,13 +9880,25 @@ class H3LongVideos: f"conditioning says is not there. That is what stops the mouth " f"moving. H3 is joint, so a free branch fills itself with a " f"voice and the face lip-syncs to the babble, and no wording " - f"suppresses that -- only the silent keyframe does, and it pins " + f"suppresses that -- only the audio denoise mask does, and it pins " f"the whole shot rather than just its opening") active = [] # the people the previous beat involved _seen_before = set() # everyone a shot has described so far _returns = [] # (shot, names back after a shot away) + _in_frame = [] # who the previous shot's last frame shows, described or not + shot_frames = {} # 0-based shot -> (who its frames show, who is still there at its end) + reentry_shots = {} # 0-based shot -> who walks in while the keyframe still has them _placed_shots = {} # 0-based shot -> who it introduces in position - shot_cast = [] # the names each shot describes + # WHOSE FACE IS ALREADY COVERED BY A PICTURE OF THEIR OWN. A sheet line + # carrying for a slot that actually has an image connected -- a + # tag pointing at an empty socket covers nobody. + _have_slot = {_i + 1 for _i, _r in enumerate( + (ref_image_1, ref_image_2, ref_image_3, ref_image_4)) if _r is not None} + _portrait_of = {_n for _n, _ln in sheet_lines(sheet) + if _n and (set(picture_tags(_ln)) & _have_slot)} + # ...and whether shot 1's first_frame is a SET rather than an opening frame. + # See the decision below. + _first_is_plate = False guard_words = beat_words = total_words = sound_words = 0 # THE PROMPT ENGINE. One state, read beat by beat, rendered once per shot. # It replaces the continuity guards that used to be derived independently @@ -7997,6 +9918,12 @@ class H3LongVideos: _sheet_hw = {c for c, _p, _w, _a in engine.hardware_spans(sheet or "")} for b in beats: body, toks, adds = extract_directives(b) + # The author's own sentences for this shot, held aside until the text is + # assembled. See _EXACT_LINE. + _said = _exact_all[len(plan)] if len(plan) < len(_exact_all) else [] + _exact = (" " + " ".join(terminate_lines(x) for x in _said)) if _said else "" + if _said: + exact_shots.append(len(plan) + 1) # Quoted speech becomes H3'S OWN dialogue marker before anything else # reads it. and are special tokens the model was trained with, # and they say "this is spoken" where quotation marks say nothing at @@ -8005,7 +9932,7 @@ class H3LongVideos: # the quotation marks are exchanged. Reported below. _marked = mark_dialogue(body) if _marked != body: - dialogue_marked.append(len(shots) + 1) + dialogue_marked.append(len(plan) + 1) body = _marked # THE ENGINE READS FIRST, before anything downstream asks it what is # true. It was reading further down at one point, after the hardware @@ -8018,12 +9945,14 @@ class H3LongVideos: # cuffs on her from shot 1 -- reported as a handcuff on her arm # before she is handcuffed. _later_for_state = {c for c, at in _staged_at.items() - if at > len(shots) + 1} + if at > len(plan) + 1} for _n, _line in sheet_lines(sheet): if _n: _state.declare(_n, _line, staged_later=_later_for_state) _ch = _state.read(body, cast=[n for n, _ in sheet_lines(sheet) if n], - shot=len(shots) + 1) + shot=len(plan) + 1) + if _ch.get("applied") or _ch.get("released"): + hardware_changed.add(len(plan) + 1) # Who this beat involves, decided BEFORE the removals: a beat that # undresses somebody names no garment, so the wardrobe to clear is read # off their sheet entries -- and only theirs. Undressing one person must @@ -8032,10 +9961,11 @@ class H3LongVideos: # further down, to keep saying what is bare about somebody the keyframe # still carries, and that has nothing to do with the guard being on. _was = list(active) + _back_cands = [] if character_guard: shot_sheet, active = sheet_for_beat(sheet, body, active) if len(sheet_lines(sheet)) > len(sheet_lines(shot_sheet)): - notes.append(f"shot {len(shots) + 1} describes only " + notes.append(f"shot {len(plan) + 1} describes only " f"{', '.join(active) or 'the scene'} -- the rest of the " f"sheet is held back, because a person the text " f"describes is a person the model draws") @@ -8046,8 +9976,13 @@ class H3LongVideos: # not. This is what "walks out of frame and comes back looking # different" is. for _grp, _who_all in unresolved_pronouns(sheet, body, _was): + # Only when it really is neither. A shot that keeps them anyway -- + # somebody walking in on both of them -- describes them, and saying + # otherwise sends the author to fix a beat that is not broken. + if any(n in (active or []) for n in _who_all): + continue notes.append( - f"shot {len(shots) + 1} says '{_grp}' and " + f"shot {len(plan) + 1} says '{_grp}' and " f"{' and '.join(_who_all)} all answer to it, so the guard could " f"not tell which -- and it describes NEITHER rather than both, " f"because naming somebody the beat did not is how an extra " @@ -8056,10 +9991,62 @@ class H3LongVideos: # First appearance, with the beat saying where they ARE rather than # staging them arriving. See the handoff decision in the render loop. _new = [n for n in active if n not in _seen_before] - if _new and not arrives_in(body) and shots: - _placed_shots[len(shots)] = list(_new) + # SHOT 1 HAS THE SAME PROBLEM AND COULD NOT REACH THE SAME ANSWER. + # + # The branch below is the one that matters here, and for years it + # carried `and plan` -- which excludes the FIRST shot, because there + # is no previous frame to demote. True, until first_frame exists: wire + # one and shot 1 has a keyframe like any other, and if that picture is + # a SET rather than a composed opening frame then nobody in the script + # is in it. Which is this branch's whole subject: "that frame does not + # have them in it, and a keyframe is a picture, so they would have to + # appear out of nothing and travel to the spot the beat describes". + # + # Measured, with a plate wired and beat 1 placing her in position: + # shot 1 took it as a HARD KEYFRAME, while the identical case one beat + # later was correctly refused. Reported as the girl not looking the + # same in the first beat and fine in the rest -- she is inserted into a + # frame that lacks her during shot 1, and shot 2 onward inherits the + # settled version from its handoff, which is why only the first is off. + # + # A PLATE IS TOLD FROM AN OPENING FRAME BY THE SCRIPT, not by looking + # at the pixels. Two conditions, both required: + # * the beat PLACES the cast rather than staging an entrance. An + # entrance genuinely wants a frame they are absent from. + # * every one of them already has a portrait of their own. Their + # appearance is carried by that picture, so this one has nothing + # left to contribute but the room -- and an author who gives both a + # composed opening frame AND a portrait of the same person has + # described that person twice, which is its own hazard here. + # Without portraits the frame is the only picture of them there is, and + # it stays frame one. + if (_new and not arrives_in(body) and not plan + and first_frame is not None + and all(_n in _portrait_of for _n in _new)): + _first_is_plate = True notes.append( - f"shot {len(shots) + 1} introduces {', '.join(_new)} in " + f"first_frame is being read as the SET, not as shot 1's opening " + f"frame, so it carries the room while " + f"{_join_names(_new)} {'are' if len(_new) > 1 else 'is'} placed by " + f"the text and held by " + f"{'their own reference images' if len(_new) > 1 else 'a reference image of their own'}" + f". Beat 1 puts " + f"{'them' if len(_new) > 1 else _new[0]} " + f"in position rather than staging an entrance, and every one of " + f"them already has a of their own -- so this picture " + f"has nothing left to say about who they are, only about where " + f"they are. Pinned as frame one it would be a picture they are " + f"not in, and they would have to appear out of nothing during " + f"shot 1: that is the same reason a later shot refuses the " + f"previous frame when it introduces somebody in position. It is " + f"NOT discarded -- the room, the light and the furniture come " + f"with it as a reference. To pin frame one exactly instead, put " + f"the cast IN that frame and take their tags off the " + f"sheet, or write the entrance into beat 1") + if _new and not arrives_in(body) and plan: + _placed_shots[len(plan)] = list(_new) + notes.append( + f"shot {len(plan) + 1} introduces {', '.join(_new)} in " f"position rather than arriving, so the previous shot's last " f"frame stops being this shot's FIRST frame -- that frame does " f"not have them in it, and a keyframe is a picture, so they " @@ -8068,12 +10055,49 @@ class H3LongVideos: f"reference, so the room comes with it. Write the entrance -- " f"'walks in', 'steps through' -- if you would rather they " f"arrive on screen and keep the frame as the anchor") - _back = [n for n in active if n not in _was and n in _seen_before] - if _back: - _returns.append((len(shots) + 1, list(_back))) + # Only a CANDIDATE here: whether the keyframe still has them in it is + # decided below, once the shot is known to be a cut or not. + _back_cands = [n for n in active if n not in _was and n in _seen_before] _seen_before.update(active) else: shot_sheet = sheet + # WHERE THIS SHOT IS AND WHO ITS FRAME CARRIES, decided here rather than + # further down, because the TEXT is assembled in between and both answers + # belong in it. The room settles whether the chain breaks; the carry + # settles who is in the picture without being named by the beat. + _frm, _via, _to = travel_legs(body) + _is_travel = bool(travel_anchor(_frm, _via, _to, here, body)) + _room_before = here + _place_now = _to or _frm or place_named(body) or here + _opens_in = _frm or (_room_before if _is_travel else _place_now) + _is_cut = bool(len(plan) and _opens_in and _room_before + and _opens_in != _room_before) + # The render's own fresh starts: after a removal, or for somebody introduced + # in position, the frame before rides as a reference unless it cannot. + _prev_stays = shot_frames.get(len(plan) - 1, ([], []))[1] + _no_carry = not _cond_module.may_carry_frame( + _prev_stays, active, + {n for n, ln in sheet_lines(sheet) if n and picture_tags(ln)}) + _fresh = (_is_cut + or (restart_after_removal and (len(plan) - 1) in stripped_shots + and _no_carry) + or (len(plan) in _placed_shots and _no_carry) + or bool(_ALONE.search(engine.staged_text(body)))) + _kept = [] if _fresh else list(_in_frame) + # SOMEBODY STILL IN THE FRAME, STAGED WALKING IN. "Dan sits at the table", + # "Crystal walks in", "Dan walks in with the mugs": nothing walked Dan out, + # so the frame this shot opens on still has him sitting there, and the text + # brings in another one. That is a second Dan, and no wording undoes a + # picture. The shot starts fresh instead, the same trade a room change makes. + # Only for somebody the previous shot did not describe -- a person it staged + # at the door walks in from the door -- and never on a walk between rooms, + # whose keyframe is the room being left. + _again = [n for n in comes_in(body, sheet) + if n in _kept and n not in _was] if (plan and not _is_travel) else [] + if _again: + reentry_shots[len(plan)] = _again + _kept = [] + _carry = [n for n in _kept if n not in active] # Read the removal out of the beat itself. Explicit 'remove:' lines still # win and are added to whatever is inferred. if auto_remove: @@ -8104,7 +10128,7 @@ class H3LongVideos: inferred.append(_hw) if inferred: toks = list(toks) + inferred - notes.append(f"shot {len(shots) + 1}: read '{', '.join(inferred)}' as " + notes.append(f"shot {len(plan) + 1}: read '{', '.join(inferred)}' as " f"coming off, from the beat's own wording") # "...strip out of their clothes, becoming naked" names nothing, so every # other path had nothing to take off and the scene went on listing the @@ -8125,7 +10149,7 @@ class H3LongVideos: if stripped: toks = list(toks) + stripped notes.append( - f"shot {len(shots) + 1} reads as undressing " + f"shot {len(plan) + 1} reads as undressing " f"{', '.join(active) if character_guard and active else 'the cast'}" f" completely, and the beat names no garment -- so the wardrobe was " f"read off the character sheet and all of it taken off: " @@ -8133,7 +10157,7 @@ class H3LongVideos: f"still described as on; name it in a 'remove:' line if so") elif not gone: notes.append( - f"shot {len(shots) + 1} reads as undressing completely, but no " + f"shot {len(plan) + 1} reads as undressing completely, but no " f"garment was recognised in the character sheet, so nothing was " f"taken off and every later shot still describes the clothes. Add " f"a 'remove:' line naming them") @@ -8144,13 +10168,46 @@ class H3LongVideos: revived = [t for t in gone if names_any(body, [t])] if revived: notes.append( - f"shot {len(shots) + 1} names {', '.join(revived)} in its own text, and " + f"shot {len(plan) + 1} names {', '.join(revived)} in its own text, and " f"that came off earlier. Beats are sent to the model word for word, so " f"naming it puts it back on -- the scene no longer mentions it, but this " f"beat does. Reword the beat if it should stay off") if toks: - stripped_shots.add(len(shots)) + stripped_shots.add(len(plan)) gone.extend(t for t in toks if t not in gone) + # WHOSE garment it was. Without this the scrub took "shirt" out of + # every entry that had one, so a second woman in the same shirt lost + # hers while still wearing it -- the text and the keyframe then + # disagree, which renders as a garment half present. + # + # THE WEARER, NOT THE REMOVER, and the difference is the whole rule: + # "Dan unlocks the chastity belt" is Dan removing McKenna's, so scoping + # to whoever the beat names would strand it on her for ever. The + # candidates are the entries that LIST the garment; the beat only picks + # between them when it names one of them, which is what tells + # self-undressing ("McKenna takes off her shirt", two shirts on the + # sheet) from somebody being undressed. + # + # Read off the WHOLE sheet, not this shot's: a shot describing only the + # person doing the unlocking has no entry for the one wearing it, which + # is the case the hardware path already had to solve. + # A REMOVAL THAT INCLUDES PEOPLE THE SHEET DOES NOT NAME reaches only + # the ones it does. There is no entry to scrub for an unnamed woman and + # no state to carry her bare region, so her skirt persists on the + # keyframe alone and comes back the moment the keyframe stops showing + # it off. Reported as some of the skirts still being on when all of + # them should have come off. Said rather than left to be discovered. + if extras_in(body): + untracked_strip.append((len(plan) + 1, list(toks))) + _took = strippers_in(body, shot_sheet if shot_sheet else sheet) + for _t in toks: + _wears = [n for n, _wl in sheet_lines(sheet) + if n and re.search(r"\b" + re.escape(_t) + r"\b", + _wl or "", re.I)] + # No entry lists it -- it came out of the scene paragraph, and the + # scrub stays unscoped, exactly as it was. + gone_by.setdefault(_t, set()).update( + [n for n in _took if n in _wears] or _wears) # An added layer is subject to removal too: once the shirt comes off, # the phrase that introduced it goes with it, or the scene keeps # describing a garment that is no longer there. Retired HERE, at the @@ -8159,7 +10216,7 @@ class H3LongVideos: _retired = [a for a in shown if names_any(a, toks)] if _retired: shown = [a for a in shown if a not in _retired] - notes.append(f"shot {len(shots) + 1} takes off something an earlier " + notes.append(f"shot {len(plan) + 1} takes off something an earlier " f"'add:' had put on, so that line retires with it: " + "; ".join(_retired)) # Reported with the SHEET's words, not the head-noun keys. The @@ -8167,14 +10224,31 @@ class H3LongVideos: # bare "shorts" here for a sheet saying "blue jeans shorts" reads # as the node having lost the description -- which is exactly the # bug it had, so the report has to be able to show it is gone. - notes.append(f"removed from the scene from shot {len(shots) + 1} on: " + notes.append(f"removed from the scene from shot {len(plan) + 1} on: " + ", ".join(scene_name_for(t, scene) or t for t in toks)) maybe = missing_removals(body, scene, gone) if not auto_remove else [] if maybe: - notes.append(f"shot {len(shots) + 1} reads as taking something off, but the " + notes.append(f"shot {len(plan) + 1} reads as taking something off, but the " f"scene still describes {', '.join(maybe)} and there is no " f"'remove:' line for it -- so every shot keeps saying it is worn. " f"Add 'remove: {maybe[0]}' to that beat") + # PUT BACK ON, IN PROSE. "Maya puts her coat back on" was read as nothing: + # the removal had scrubbed the coat, and only an `add:` line brought a + # garment back -- so from that beat on she was put in a coat on screen and + # described without one, and the coat was whatever the model made of it. + # A garment that came off earlier and that this beat puts on is the same + # thing an `add:` says, under the sheet's own name for it. + if auto_remove and gone: + for _g in list(gone): + if _g in restored or any(names_any(a, [_g]) for a in (adds or [])): + continue + _head = str(_g).lower().split()[-1] + if not (names_any(body, [_head]) and beat_stages_wearing(body, _head)): + continue + _name = scene_name_for(_head, sheet or scene) or _g + adds = list(adds or []) + [_name] + notes.append(f"shot {len(plan) + 1}: read '{_name}' as put back on, " + f"the way an 'add:' line would say it") _wearing = "" # the both-ends clause for a garment going on _staged_add = [] # ...and the phrases it covers, held out of # this shot's static wardrobe @@ -8197,7 +10271,7 @@ class H3LongVideos: if _back: restored.extend(_back) notes.append( - f"shot {len(shots) + 1} puts " + ", ".join(_back) + f"shot {len(plan) + 1} puts " + ", ".join(_back) + " back on, so anything it covers is hidden again from " "here. A garment coming back has to un-cover as well as " "re-cover, or the layer under it stays described for the " @@ -8218,8 +10292,8 @@ class H3LongVideos: if _worn_now: _wearing = wearing_clause(_worn_now) _staged_add = list(_worn_now) - wearing_shots.append(len(shots) + 1) - notes.append(f"added to the scene from shot {len(shots) + 1} on: " + wearing_shots.append(len(plan) + 1) + notes.append(f"added to the scene from shot {len(plan) + 1} on: " + "; ".join(adds)) # The scrub applies to the removing shot too -- but only because that # shot's KEYFRAME already shows the garment on at the start, so the text @@ -8238,7 +10312,7 @@ class H3LongVideos: # text of the very shot that removes it, so the shot said it is not # worn AND to take it off, and it was gone a beat early with nothing # anchoring it on. Reported exactly that way. - i_shot = len(shots) + i_shot = len(plan) _anchoring = (ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG) has_keyframe = ((i_shot > 0 or first_frame is not None) @@ -8302,7 +10376,7 @@ class H3LongVideos: _said = [g for g in covered if re.search(r"\b" + re.escape(g) + r"\b", body or "", re.I)] if _said: - exposed_by_beat.append((len(shots) + 1, _said)) + exposed_by_beat.append((len(plan) + 1, _said)) # The shot that UNCOVERS one says so. Reported: the shorts come off and # the render goes straight to bare skin, past the underwear the sheet # named. The removal clause is emphatic and specific -- off the body, @@ -8317,7 +10391,7 @@ class H3LongVideos: _revealed = reveal_clause([u for u in revealed_by(covers, toks) if u not in visible and not names_any(u, toks)]) if _revealed: - revealed_shots.append(len(shots) + 1) + revealed_shots.append(len(plan) + 1) # ...and when the sheet names NOTHING underneath, say the region is bare. # Otherwise the shot says a garment is gone and leaves the space it left # unspecified, which is where the model's own prior fills in -- legwear @@ -8326,8 +10400,20 @@ class H3LongVideos: # nothing is. # ...and not beside BARE_HOLD, which already says everything comes off. # Both firing said it twice and attributed it twice. + # The body is named from the sheet's own declared pronoun, and only where + # one person is described -- with two, bare_hold's per-person path below + # carries it and naming it here would attach it to whichever of them the + # reader reached first. + # ...and the AGE off the same entry, so the body named is the age the sheet + # states rather than whatever the prior supplies. See body_of and figure_of. + _one_line = dict(sheet_lines(shot_sheet)).get((active or [""])[0], "") + _one_pron = sheet_pronoun(_one_line) + _one_age = age_in(_one_line) + _one_body = (body_of(_one_pron, _one_age) if len(active or []) == 1 else "") + _one_fig = (figure_of(_one_pron, _one_age) if len(active or []) == 1 else "") _bare = ("" if (_revealed or bare) - else bare_clause(toks, covers, shot_sheet)) + else bare_clause(toks, covers, shot_sheet, body=_one_body, + figure=_one_fig)) # ...and on EVERY shot after it, from state, for as long as the # region has nothing on it. Said only on the uncovering beat, the # region went unspecified from the next shot on -- and the model @@ -8381,12 +10467,25 @@ class H3LongVideos: _name_it = (len(_rows) > 1 or len(_who_here or []) > 1 or any(_n in _carried_on for _n, _r, _o in _rows)) _bare = "".join( - bare_hold(_rg, covers, _on, whose=(_n if _name_it else "")) + bare_hold(_rg, covers, _on, + # WHAT HAS COME OFF, so a layer the sheet puts + # underneath stops suppressing the clause once it + # has come off too. Cumulative, not this beat's: + # this path exists to speak on the shots AFTER the + # removal. Minus anything an `add:` put back on, + # which is worn again and is covering again. + [g for g in gone if g not in restored], + whose=(_n if _name_it else ""), + # PER PERSON here, unlike the single-cast path above: + # this loop already runs once for each of them, so each + # body and each figure is read off that person's OWN + # entry. One age applied to two people is the bug the + # `whose` argument exists to prevent, one attribute over. + body=body_of(*_pron_age(shot_sheet, _n)), + figure=figure_of(*_pron_age(shot_sheet, _n))) for _n, _rg, _on in _rows) - if _bare: - bare_held.append(len(shots) + 1) if _bare: - bared_shots.append(len(shots) + 1) + bared_shots.append(len(plan) + 1) # Terminated, or the last sheet line welds onto the beat -- "grey coat # Maya lies still" -- and a name fused to the end of an attribute list is # read as one more item in it. @@ -8426,9 +10525,40 @@ class H3LongVideos: and (cover_owner.get(u) in _here or u not in cover_owner)] _hidden = [u for u in covered if u not in _worn_under] - shot_scene = scrub_removed( - "\n".join(terminate_lines(p) for p in (static, shot_sheet) if p.strip()), - visible + _hidden) + # A REMOVAL TAKES THE GARMENT OFF THE PERSON WHO REMOVED IT, AND NOBODY + # ELSE. + # + # Reported: shirts looking half missing. Two women in white shirts, one + # takes hers off, and the token "shirt" was scrubbed from the WHOLE text -- + # so the other one's entry lost her shirt while the keyframe still showed + # her wearing it. That is the contradiction this file already describes: + # "the shot then says it is not worn, take it off, and the thing under it + # is already showing. The model renders that contradiction as a garment + # half present -- open, or partly cut". The strip-bare path has been scoped + # to "THEIR OWN entry" since it was written; the prose-removal path was not. + # + # Scoped HERE rather than inside scrub_removed: that function applies a + # second, whole-text sweep after its per-sentence pass, and threading an + # owner through both would mean restructuring 186 lines whose comments + # record a dozen separate fixes. Calling it once per entry gets the same + # answer and leaves it untouched. + # + # gone_by maps a token to who took it off. A token nobody is recorded for + # stays unscoped, which is what keeps every other removal behaving as it did. + _toks_all = visible + _hidden + # Scoped to this shot's people first: see static_for_shot. + _static_here = static_for_shot(static, sheet, shot_sheet) + _scrubbed = ([scrub_removed(terminate_lines(_static_here), _toks_all)] + if _static_here.strip() else []) + for _ln in (terminate_lines(shot_sheet).split("\n") + if shot_sheet.strip() else []): + _m = re.match(r"\s*([A-Za-z][\w'\u2019-]*)\s*:", _ln) + _who = _m.group(1).lower() if _m else "" + _allow = [t for t in _toks_all + if not (_who and gone_by.get(t) and _who not in + {str(x).lower() for x in gone_by[t]})] + _scrubbed.append(scrub_removed(_ln, _allow)) + shot_scene = "\n".join(p for p in _scrubbed if p.strip()) # A NAMED CLOSE FRAME STOPS DESCRIBING WHAT IT CANNOT HOLD. # # Applied HERE, last, on the finished text: the layer, removal and @@ -8477,7 +10607,7 @@ class H3LongVideos: shot_scene = defer_tag_for(shot_scene, _worn_under) shot_scene = hide_item(shot_scene, _worn_under) if _deferred and len(shot_scene) >= 0: - deferred_shots.append((len(shots) + 1, list(_worn_under))) + deferred_shots.append((len(plan) + 1, list(_worn_under))) _under = under_clause( [(u, covers.get(u, ""), cover_owner.get(u, "") if len(_here) > 1 else "") @@ -8491,7 +10621,7 @@ class H3LongVideos: # assertion that had to stop, as against the author's description, # which did not. _sheet_says_early = [c for c, at in _staged_at.items() - if c in _sheet_hw and at > len(shots) + 1] + if c in _sheet_hw and at > len(plan) + 1] _scene_for_state = (scrub_removed(shot_scene, _sheet_says_early) if _sheet_says_early else shot_scene) # "ALREADY ON" MEANS BEFORE THIS SHOT. The applying test asks whether @@ -8501,7 +10631,7 @@ class H3LongVideos: # shot that stages the fastening, and it gets the standing hold: a lie # about its first frame. _sheet_says_now_or_later = [c for c, at in _staged_at.items() - if c in _sheet_hw and at >= len(shots) + 1] + if c in _sheet_hw and at >= len(plan) + 1] _scene_before_now = ( scrub_removed(shot_scene, _sheet_says_now_or_later) if _sheet_says_now_or_later else shot_scene) @@ -8517,8 +10647,28 @@ class H3LongVideos: # that made it appear at the first frame. It joins the static wardrobe # from the NEXT shot on, exactly as a removal scrubs from its own. live = [a for a in shown if a not in _staged_add] - if live: - tail = ". ".join(a.rstrip(".") for a in live) + "." + # ...ON SOMEBODY. A garment put back went in as a sentence of its own -- + # "A hallway. Long red coat. Maya opens the front door." -- a coat in the + # room with nobody in it, which a model is free to hang on a hook or on + # the wrong person. When one sheet entry is the garment's owner it is said + # on them, and in a shot they are not in it is not said at all. With no + # clear owner it stays the sentence it was. + _here_names = {n for n, _ in sheet_lines(shot_sheet or "") if n} + _said = [] + for a in live: + _head = (re.findall(r"[a-z]+", a.lower()) or [""])[-1] + _owners = [n for n, ln in sheet_lines(sheet or "") + if n and _head and names_any(ln.split(":", 1)[-1], [_head])] + if len(_owners) == 1 and _here_names: + if _owners[0] not in _here_names: + continue + _bare_name = re.sub(r"^(?:a|an|the|her|his|their|its)\s+", "", + a.strip().rstrip("."), flags=re.I) + _said.append(f"{_owners[0]} is wearing the {_bare_name}") + else: + _said.append(a.rstrip(".")) + if _said: + tail = ". ".join(_said) + "." tail = tail[0].upper() + tail[1:] shot_scene = f"{shot_scene} {tail}".strip() if shot_scene else tail # The removal has to FINISH inside this shot, because its last frame is @@ -8542,8 +10692,22 @@ class H3LongVideos: # with a second person in the shot the model gives that second removal to # him. The action happens twice, once by each of them. _who_sheet = shot_sheet if sheet_lines(shot_sheet) else scene - _wearer = next((n for n, ln in sheet_lines(_who_sheet) - if n and names_any(ln, toks)), None) + # ...AND WHEN TWO ENTRIES LIST THE SAME KIND OF GARMENT, the first entry + # was taken. "Lena takes off her sweater" with Maya also in a sweater put + # the bare chest on Maya -- "Maya's chest, shoulders and arms are bare + # skin" beside Maya's own entry still listing her green sweater, one woman + # described both clothed and bare, which is a woman drawn twice. The sheet + # stays the answer when it is unambiguous (it is what gets "she asks Dan" + # right); a tie goes to whoever the scene state recorded taking it off + # this beat, then to whoever the beat names. + _listed = [n for n, ln in sheet_lines(_who_sheet) + if n and names_any(ln, toks)] + if len(_listed) > 1: + _by_state = [w for w, _g in (_ch.get("removed") or []) if w in _listed] + _by_beat = engine.names_in(body, _listed) + _wearer = (_by_state or _by_beat or _listed)[0] + else: + _wearer = _listed[0] if _listed else None # WHOSE body is bare. Unattributed in a two-person shot this reads as # an instruction about everyone on screen, and the second character # undresses alongside the first. Done HERE because _wearer is what @@ -8678,8 +10842,6 @@ class H3LongVideos: # genuinely already on, the sheet check is doing its job, and # overriding it there cost the cuffs their standing hold. The veto is # lifted only where this node created the conflict. - _stages_now = any(at == len(shots) + 1 and canon in _sheet_hw - for canon, at in _staged_at.items()) _applying = bool(restrained and not _was_restrained and not restraint_present(_scene_before_now) and restraint_going_on(body)) @@ -8694,7 +10856,7 @@ class H3LongVideos: # once -- it is one authoring decision, not one per shot. if (not early_hardware and restraint_going_on(body) and restraint_present(_scene_for_state)): - early_hardware.append(len(shots) + 1) + early_hardware.append(len(plan) + 1) # Rigidity latches like the hardware itself. Steel locked on in shot 1 is # still steel in shot 5, and a beat that does not happen to say "chain" # does not mean the chain became rope -- but tested per shot, that is @@ -8714,27 +10876,6 @@ class H3LongVideos: # them SHUT and says nothing about position, so the only thing carrying # it was the picture -- and a close shot crops the anchor point straight # out of frame, which is the reported failure exactly. - # Where the beat says somebody is looking, said once more as a fact - # about the eyes and the head. One mention in the beat loses to a - # near-clean reference asking for the portrait's pose, and the - # portrait looks at the lens because photographs of people do. - # LATCHED, like every other state here. A look was stated once and then - # dropped, so somebody watching a screen across four shots was told - # where their eyes were in the first one only -- and the portrait pull - # that made this necessary does not stop after one shot. - # - # Cleared by a beat that moves the look somewhere else, or one that - # moves the person: walking away ends it, and holding a stale target - # across that would be worse than saying nothing. - _look_now = look_target(body) if hold_gaze else "" - if _look_now: - looking_at = _look_now - elif (looks_somewhere(body) or arrives_in(body) or falls_in(body) - or turns_in(body, cast) or _MOVES_OFF.search(body or "")): - looking_at = "" - _gaze = gaze_hold(looking_at) if (hold_gaze and looking_at) else "" - if _gaze: - gaze_shots.append(len(shots) + 1) # POSTURE, latched the way the gaze is. A beat that sits somebody down # ends its shot with them seated; the next beat says nothing about it, # so the shot was free to stand them back up -- reported as the end of @@ -8760,19 +10901,71 @@ class H3LongVideos: shot_length == "from the beat", pace)[0][0] / H3_FPS _pace = pace_clause(beat_seconds(body), _have) if _pace: - paced_shots.append(len(shots) + 1) - _frm, _via, _to = travel_in(body) - _travel = travel_anchor(_frm, _via, _to, here) + paced_shots.append(len(plan) + 1) + # travel_legs, not travel_in: the promotion of a bare via to the + # destination is read by the SIZING too, and a transit rendered as a walk + # while sized as if it went nowhere is how a three-room walk ended up in a + # three-second shot. See travel_legs and travel_spaces. + _travel = travel_anchor(_frm, _via, _to, here, body) if _travel: - travel_shots.append(len(shots) + 1) + travel_shots.append(len(plan) + 1) + else: + # The place list could not name either end. Perform the arrival + # anyway: a move nobody is told to make is a move the model cuts to. + # See moved_to -- this establishes no room state at all. + _open_to = moved_to(body, active) + _travel = move_clause(_open_to, body) + if _travel: + open_moves.append((len(plan) + 1, _open_to)) # The room the next beat starts from: where this one ended, or where it - # simply says everyone is. - here = _to or _frm or place_named(body) or here + # simply says everyone is. Both decided above, before the text was written. + here = _place_now + # A ROOM THE KEYFRAME IS NOT IN IS A CUT. + # + # Every shot is anchored to the previous shot's last frame, and a keyframe + # is a PICTURE, which outvotes any sentence -- the reasoning + # restart_after_removal is already built on. So a shot that OPENS in a + # different room from the one the shot before ended in has a first frame + # showing the wrong room, and the model reconciles the two by blending + # them. Reported as a living room turning into a bathroom, which is exactly + # what a kitchen frame and the words "living room" have in common: tiles, a + # sink, cabinets. Breaking the chain costs a cut where a cut belongs. + # + # A WALK IS NOT THIS. A travel beat opens in the room it is leaving, so that + # frame is the right one -- which is why the test is on where the shot + # OPENS, not on whether the room changed. A beat naming an origin of its own + # is judged on that origin, so "walks from the bedroom to the bathroom" + # after a kitchen shot is still a cut. + if _is_cut: + cut_shots.add(len(plan)) + shot_rooms[len(plan)] = (_opens_in or "", here or "") + # WHO THE FRAMES SHOW, which is not who the text describes. A shot that + # stops describing somebody does not take them out of the picture it starts + # from: they stay in it until a beat walks them out, the camera goes to a + # room they are not in, or the chain breaks. Read by the render wherever a + # frame is used as a picture of the people in it. See _EXIT. + _carry = [n for n in _kept if n not in active] + _shows = list(active) + _carry + # A walk to another room leaves behind whoever it does not describe. + _ends_with = list(active) + ([] if (_to and _to != _room_before) else _carry) + + # BACK AFTER A SHOT AWAY means not in the keyframe -- not merely undescribed + # in the shot before. Dan sitting at the table through "Crystal laughs" is + # still in the frame "Dan smiles" opens on, and a recovered picture of him + # there is a second Dan. + _back = [n for n in _back_cands if n not in _kept] + if _back: + _returns.append((len(plan) + 1, list(_back))) + _gone = leaves_in(body, sheet, _shows) + _in_frame = [n for n in _ends_with if n not in _gone] + shot_frames[len(plan)] = (_shows, list(_in_frame)) + if here and here not in _described_rooms and here not in _undescribed: + _undescribed.append(here) # ...and say so on later shots, because the scene paragraph still # names the room they started in and is stamped into every shot. _where = where_hold(here, scene) if not _travel else "" if _where: - where_shots.append(len(shots) + 1) + where_shots.append(len(plan) + 1) # ...and the ACOUSTIC follows them. Both were read ONCE, before the # loop, out of the scene -- so a film that walks into a tiled bathroom # went on being told it sounds like the carpeted living room it left. @@ -8788,7 +10981,7 @@ class H3LongVideos: _bed_now = ((scene_ambient(here) or ambient_bed) if (auto_sound and _where) else ambient_bed) if _where and auto_sound and (_room_now != _room or _bed_now != ambient_bed): - acoustic_shots.append((len(shots) + 1, here)) + acoustic_shots.append((len(plan) + 1, here)) _pose_now = posture_in(body, active if character_guard and active else [n for n, _ in sheet_lines(_who_sheet) if n]) # ...and let go of any the beat contradicts. A pose that survives an @@ -8805,7 +10998,7 @@ class H3LongVideos: active if character_guard else [n for n, _ in sheet_lines(_who_sheet) if n])) if _posture: - posture_shots.append(len(shots) + 1) + posture_shots.append(len(plan) + 1) poses.update(_pose_now) _anchor_now = limb_anchor(body) if restrained else "" if _anchor_now: @@ -8819,9 +11012,9 @@ class H3LongVideos: # on -- so those key off the latch rather than off a clause. _holding = bool(restrained and anchored and not _anchor_now) if _holding: - anchored_shots.append(len(shots) + 1) + anchored_shots.append(len(plan) + 1) if _holding and (_anchor_tight or tight_framing(body)): - tight_shots.append(len(shots) + 1) + tight_shots.append(len(plan) + 1) # A turn shows a surface the keyframe never pinned, and the model fills # it from a clothed prior. Only on shots that turn, and only once there # is something to hold -- a removal already made, or hardware on. @@ -8840,7 +11033,7 @@ class H3LongVideos: fall = (FALL_HOLD if (restrained and _falls) else FALL_HOLD_FREE if _falls else "") if fall: - fall_shots.append(len(shots) + 1) + fall_shots.append(len(plan) + 1) # Steel is not rope. Without being told, the model draws a chain slack -- # sagging, stretching to wherever a limb is going, allowing movement the # hardware does not allow. Only where such hardware is actually named. @@ -8858,7 +11051,36 @@ class H3LongVideos: notes.append(f"shot {i_shot + 1} names hardware with no body part beside " f"it, so the shot says where it sits: " f"{anchors.split(': ', 1)[1].rstrip('.')}") - line = f"{shot_scene} {body}".strip() if shot_scene else body + # The scene's description of a room this shot does not END in waits here. + # The paragraph is stamped into every shot, and a paragraph that describes + # a room describes its furniture too -- which is how a bed reached a living + # room two beats after she left the bedroom. See scene_for_here. Only the + # text SENT changes: shot_scene itself is left alone, so every reader above + # and below this line keeps its full view of the scene. + _scene_sent, _held_rooms, _held_blocked, _held_text = scene_for_here( + shot_scene, here, anchor, + [n for n, _ in sheet_lines(shot_sheet) if n], body) + if _held_rooms and _held_blocked: + scene_welded.append((len(plan) + 1, list(_held_rooms))) + elif _held_rooms: + scene_held.append((len(plan) + 1, list(_held_rooms), list(_held_text))) + # WHAT LEADS DECIDES THE FRAME. See split_sheet: the appearance block goes + # after the action it describes, so the opening tokens are the place and + # what happens in it rather than sixteen words of face. + # NOT ON A SHOT CARRYING A PICTURE TAG. is numbered by the + # order the tags APPEAR in the shot, and the number is the image's place in + # that shot's reference list -- so moving the sheet past the beat renumbers + # them, and a renumbered reference is the wrong face on the wrong person. + # That is the oldest and worst bug in this file and it is not worth a + # composition gain. A shot with no tag has no numbering to disturb. + if beat_leads and _scene_sent and not picture_tags(f"{_scene_sent} {body}"): + _scene_part, _sheet_part = split_sheet( + _scene_sent, [n for n, _ in sheet_lines(shot_sheet) if n]) + line = " ".join(p for p in (_scene_part, body, _sheet_part) if p).strip() + if _sheet_part: + led_shots.append(len(plan) + 1) + else: + line = f"{_scene_sent} {body}".strip() if _scene_sent else body # A state the text asserts but does not stage. Read from the whole line, # because the van usually stands in the scene paragraph rather than in # the beat -- and suppressed for anything this beat is actually working, @@ -8876,9 +11098,9 @@ class H3LongVideos: # continuity, and four such sentences is a shot about its own continuity. _state_clause = state_hold(_pairs[:max(0, 2 - _turn.count("first frame"))]) + _turn if _pairs: - stated_shots.append(len(shots) + 1) + stated_shots.append(len(plan) + 1) if _turn: - turned_shots.append(len(shots) + 1) + turned_shots.append(len(plan) + 1) # The beat and the hold asking for opposite things. Reported three times # running as "the doors keep opening", and every time the node text was # by then correct -- it was the beat staging an exit the doors have to @@ -8886,7 +11108,7 @@ class H3LongVideos: if _pairs and exits_vehicle(body) and any( _state_key(t) in ("door",) for t, _ in _pairs): notes.append( - f"shot {len(shots) + 1} says somebody gets OUT of a vehicle and also " + f"shot {len(plan) + 1} says somebody gets OUT of a vehicle and also " f"says the doors are closed. Those are opposite instructions and the " f"beat wins: a person leaving a van opens a door to do it, so the " f"doors open however firmly the text says they are shut. If they are " @@ -8984,7 +11206,7 @@ class H3LongVideos: if _applying else chain if chain else (RESTRAINT_HOLD if restrained else "")) if _applying: - applied_shots.append(len(shots) + 1) + applied_shots.append(len(plan) + 1) # Name the thing on shots that do not. The hold says a restraint stays # fastened and never says WHAT, so a shot after the applying one is told # a restraint exists with no object to draw -- which renders as the @@ -9000,7 +11222,7 @@ class H3LongVideos: # The shot that STAGES a displacement -- the garment is being moved # on screen in it. Recorded because the render loop must not capture # a subject reference from it: moved_shots starts the shot AFTER. - staging_shots.add(len(shots) + 1) + staging_shots.add(len(plan) + 1) for _g, _how in _staged_here: _was = displaced.get(_g, "") # Put back up again is a restore, not a new displacement. @@ -9035,7 +11257,7 @@ class H3LongVideos: if not re.search(r"\b" + re.escape(g.split()[-1]) + r"\b", _body_low)]) if _moved: - moved_shots.append(len(shots) + 1) + moved_shots.append(len(plan) + 1) # ...and say WHOSE. Unattributed, "every restraint stays fastened" is an # instruction about whoever is on screen, so hardware locked onto one @@ -9045,6 +11267,106 @@ class H3LongVideos: if not character_guard or n in active] _described = (active if character_guard else [n for n, _ in sheet_lines(shot_sheet) if n]) + # LATCHED, WITH AN EXPLICIT WAY OUT. + # + # This went both ways before settling here. Latched on any plural word it + # stood the body count down for the whole film -- "the others have gone" + # included -- and that clause is what keeps a duplicate or a stranger out + # of the frame. Read per beat instead, a shot whose beat simply stops + # mentioning the extras got "There is one person in the shot: one body, + # one face" while five women were standing in it, which asserts four of + # them out of existence. + # + # Both faults were the same missing piece: extras are STATE, and state + # needs a transition out. extras_in is now absence-aware, so the latch no + # longer fires on a sentence saying they left, and extras_dismissed is the + # way out -- "she is alone now", "the others have gone". Background people + # do not leave because a sentence stopped mentioning them, and they do not + # stay for ever either. + if extras_in(body): + _extras_seen = True + elif extras_dismissed(body): + _extras_seen = False + # COUNTED FROM THE PICTURE, not only the text. "Crystal laughs" opening on a + # frame with Dan beside her was told there is one person in the shot: one + # body, one face -- a sentence against a keyframe with two people in it, + # which the model can only reconcile by merging them. The same reason the + # count stands down while extras are still in the room. See shot_frames. + _cast_hold = cast_hold(list(_described or []) + _carry, body, _extras_seen) + + # Where the beat says somebody is looking, said once more as a fact + # about the eyes and the head. One mention in the beat loses to a + # near-clean reference asking for the portrait's pose, and the + # portrait looks at the lens because photographs of people do. + # LATCHED, like every other state here. A look was stated once and then + # dropped, so somebody watching a screen across four shots was told + # where their eyes were in the first one only -- and the portrait pull + # that made this necessary does not stop after one shot. + # + # Cleared by a beat that moves the look somewhere else, or one that + # moves the person: walking away ends it, and holding a stale target + # across that would be worse than saying nothing. + # + # PER PERSON, because a look is one. It was a single string with no + # owner, held across shots and said impersonally, so a look staged by + # one character went on being said in shots she was not in -- and + # landed on whoever was: "McKenna looks at the lane" in shot 3, and + # shot 4, describing only Dan, was told "the eyes and the head are + # turned to the lane behind them." That is his head on her sightline, + # and it is the same defect as the vocal flag -- a per-person fact kept + # in a shot-level variable. + _look_now = (look_target(body, shot_sheet, _described) + if hold_gaze else "") + _lookers = (subjects_for(body, shot_sheet, _LOOK_VERB_SRC) + if hold_gaze else []) + _look_is_person = bool(_look_now) and any( + _look_now == _n for _n, _ in sheet_lines(shot_sheet)) + if _look_now: + # Whoever the beat says is looking. If it names nobody, everybody it + # describes -- which is what the single string did for everyone. + for _n in (_lookers or (_described or [])): + looking_at[_n] = (_look_now, _look_is_person) + elif (looks_somewhere(body) or arrives_in(body) or falls_in(body) + or turns_in(body, cast) or _MOVES_OFF.search(body or "")): + # Clear only the people this beat actually moved or turned. When it + # cannot be pinned on anybody, clear all of it: a stale target is + # worse than none, which is why this branch exists at all. + _ends = (_lookers + or subjects_for(body, shot_sheet, _MOVES_OFF_SRC)) + for _n in (_ends or list(looking_at)): + looking_at.pop(_n, None) + # ...and said for people this shot describes, PLUS anybody who was in + # the previous shot and whom this beat has not moved off. She is still + # in the van when the beat is about him: the next shot starts from a + # picture with her in it, and dropping her from the text is what leaves + # her with nothing to do but face the lens. ONE shot of memory -- the + # node knows she was in the last picture, not where she is now. + _carried = [n for n in _was + if n not in set(_described or []) and looking_at.get(n) + and n not in set(subjects_for(body, sheet, _MOVES_OFF_SRC))] + _gazers = [n for n in (_described or []) if looking_at.get(n)] + _carried + _gaze = "" + _faces = "" # the eye-line inferred for a dialogue shot + # What the frame holds, where the beat and the anchor both leave it open. + # An unstated frame becomes the prior, and the prior for a described + # person is a portrait facing the lens. See frame_hold. + # WHO IS WITH WHOM. Only where it could be read wrong: with two people in + # the shot there is nobody else to pair with, and naming them again costs a + # mention each. Three or more and an unnamed pairing is the model's to + # choose -- reported as girls kissing each other instead of the boys. + _contact = (contact_hold(contact_pairs(body, _described)) + if len(_described or []) > 2 else "") + if _contact: + contact_shots.append(len(plan) + 1) + _frame = frame_hold(body, anchor, len(_described or []) or 1) + if _frame: + frame_shots.append(len(plan) + 1) + # ...and where the camera IS, which nothing said either. A travel beat + # keeps its moving camera: the node has already asked for every step of + # the journey in frame. See camera_hold. + _camera = camera_hold(body, anchor, moving=bool(_travel)) if hold_camera else "" + if _camera: + camera_shots.append(len(plan) + 1) # ONE sentence for the hardware. The hold, the name of the thing and # where it holds were three separate clauses written for three separate # reports, each naming the same object again -- 53 words about one pair @@ -9065,7 +11387,7 @@ class H3LongVideos: # draws the person that sentence implies -- which is the duplicate. # It latches, so the shot they come back in has it again. hold = "" - absent_hold.append(len(shots) + 1) + absent_hold.append(len(plan) + 1) elif not _applying and restrained: hold = restraint_sentence( worn_item if not _named_item else "", @@ -9076,7 +11398,7 @@ class H3LongVideos: rigid=bool(rigid), posed=bool(posed), part=held_part(worn_items or ([worn_item] if worn_item else []))) if worn_item and not _named_item: - named_shots.append(len(shots) + 1) + named_shots.append(len(plan) + 1) else: hold = own_hold(hold, _wearers, _described) # What you wrote wins: a beat that already describes its own sound is left @@ -9102,14 +11424,22 @@ class H3LongVideos: # what was written, and finding that out from the render is worse than # reading it here. if not _own and not _speaks and _BREATH_PREP.search(body): - _breath_shots.append(len(shots) + 1) + _breath_shots.append(len(plan) + 1) # A beat staging EFFORT or vocal reaction is asking for a voice, and that # is read from the author's own verbs -- "thrashes", "writhes", "moans" -- # so it belongs with a quoted line and a written sound, not with the things # this file infers. Silencing it says the person makes no sound, and a # person making no sound is rendered still: it is the flat, unreacting # face, and it is why a body under effort came out mute. - _voiced = exertion_in(body) + # A VOCAL THE BEAT NAMES IS ASKING FOR AUDIO, as much as an effort verb is. + # This was exertion_in alone, and four of the six vocals passed only by + # ACCIDENT -- whimper, sob, moan and scream happen to sit in the effort table + # too. groan and whine do not, so "She groans." was sound_described with + # nothing to keep it open: _mute_written fired, _will_silence fired, and the + # shot was pinned to silence. The groan the author wrote never happened, and + # the clause naming it was never emitted either. Reading the vocal directly + # makes all six behave the way the four already did. + _voiced = bool(exertion_in(body) or named_vocals_in(body)) # A shot where nobody speaks but the author wrote a SOUND kept its branch # open, and an open branch invents a voice the face lip-syncs to. That is # the hole: "a low hum off the strip light" is nobody talking, and it was @@ -9142,7 +11472,7 @@ class H3LongVideos: # wordless shots, THIS is the first thing to turn off: auto_sound. _bed = _bed_now if auto_sound and _bed_now else "" if _bed: - ambient_shots.append(len(shots) + 1) + ambient_shots.append(len(plan) + 1) _mute_written = bool(mouths_shut_when_no_line and _own and not _speaks and not _voiced) # The bed no longer defeats this. It is the one thing this file infers @@ -9151,7 +11481,7 @@ class H3LongVideos: _will_silence = bool(silence_nonspeech and not _speaks and not _voiced and (not _own or _mute_written)) if _mute_written and _will_silence: - muted_sound.append(len(shots) + 1) + muted_sound.append(len(plan) + 1) # The picture side -- and ONLY where the shot actually describes somebody. # A mouth sentence on a scenery beat describes a person who is not there, # and the one way to satisfy it is to draw a face in an empty frame. That @@ -9173,9 +11503,24 @@ class H3LongVideos: # the voice is given back to the thing it came out of. _device_line = (mouths_shut_when_no_line and speech_is_a_devices(body, sheet)) + # The beat's own mouth. Read here, used ONLY on the picture guard + # below -- never on the audio decision, which is what keeps a smile + # silent. See mouth_performs. + # THE FACE. Built here because _wearers and _described are what say who + # is under duress and who else is in the frame; used only on the picture + # side, like the mouth guard beside it. See duress_face. + _duress = (duress_face( + body, + [(n, ln) for n, ln in sheet_lines(shot_sheet) if n in set(_wearers)], + _described, _film_duress) if hold_gaze else "") + if _duress: + duress_shots.append(len(plan) + 1) + # A STATED EMOTION PUTS THE MOUTH TO WORK. Delight is a smile, terror is + # an open mouth; holding it closed holds the performance. See _EMOTION. + _mouth_busy = bool(mouth_performs(body) or emotion_in(body)) _mouth = MOUTH_HOLD if (mouths_shut_when_no_line and _has_people and (not _speaks or _device_line) - and not _voiced) else "" + and not _voiced and not _mouth_busy) else "" # One of two people speaking still leaves the OTHER one's mouth free. The # shot is a speaking shot, so the guard stood down for everybody in it -- # and the listener is exactly who the invented lip-sync lands on. Name the @@ -9186,24 +11531,51 @@ class H3LongVideos: # then cannot tell a silenced shot from one where the speaker is named, # which are opposite situations. _mouth_from_silence = bool(_mouth) - if (not _mouth and mouths_shut_when_no_line and _speaks and not _voiced - and not _device_line): - _talkers = speakers_in(body, shot_sheet) - _silent = [n for n in (_described or []) if n not in _talkers] - if _talkers and _silent: - _mouth = MOUTH_HOLD_OTHERS.format( - who=_talkers[0] if len(_talkers) == 1 - else ", ".join(_talkers[:-1]) + " and " + _talkers[-1]) - elif not _talkers and len(_described or []) > 1: + # WHOSE VOICE IS WHOSE. A vocal used to switch this whole block off -- + # _voiced is a shot-level flag and both guards stood down on it, for + # everybody -- so her sob opened the branch and freed his mouth with it. + # The vocal gets an owner instead, and only the mouths that own neither a + # line nor a sound are closed. See voice_sources. + _vocal_src = vocal_sources_in(body, shot_sheet) if _voiced else [] + _voicers = [n for n, _ in _vocal_src] + _vocal_word = _vocal_src[0][1] if _vocal_src else "" + # A vocal this file cannot pin on anybody leaves every mouth alone, the + # way an unattributed line does: closing mouths on a guess could close + # the mouth of whoever is making the noise, and muting a real sound is + # worse than a mouth moving. + if (not _mouth and mouths_shut_when_no_line and (_speaks or _voicers) + and not _mouth_busy and not _device_line + and not (_voiced and not _voicers)): + _talkers = speakers_in(body, shot_sheet) if _speaks else [] + _open = set(_talkers) | set(_voicers) + # WHOSE MOUTH THERE IS TO HOLD. Not only the people this beat + # names: a beat naming just the speaker does not empty the room, + # and the person it leaves out is standing in the picture this + # shot starts from. That is the mouth an invented voice lands on, + # and "Dan says: ..." on its own is the commonest beat there is. + # One shot of memory, and not for anybody the beat walks off. + _here_too = [n for n in _was + if n not in set(_described or []) + and n not in set(subjects_for(body, sheet, + _MOVES_OFF_SRC))] + _silent = [n for n in list(_described or []) + _here_too + if n not in _open] + _mouth = voice_sources(_talkers, _vocal_word, _voicers, _silent) + if _mouth and _voicers: + vocal_shots.append(len(plan) + 1) + if (not _mouth and _speaks and not _talkers and not _voicers + and len(_described or []) > 1): # A line with no name on it, and more than one person who could # be saying it. Whose mouth to hold is unknowable, but how many # voices there are is not -- and leaving it unsaid is what let # the listener talk too. _mouth = ONE_VOICE - unattributed.append(len(shots) + 1) + unattributed.append(len(plan) + 1) if _mouth: (mouth_shut if _mouth_from_silence - else mouth_named).append(len(shots) + 1) + else mouth_named).append(len(plan) + 1) + elif _mouth_busy and mouths_shut_when_no_line and _has_people: + mouth_acting.append(len(plan) + 1) # A shot with a line is told what language it is in. Every shot with a # line, not only the ones with a listener to hold: a single speaker can # deliver the line in whatever language the model picks. @@ -9229,19 +11601,19 @@ class H3LongVideos: _described if character_guard else [n for n, _ in sheet_lines(_who_sheet) if n])) if _speaks else "" if _told: - told_shots.append(len(shots) + 1) + told_shots.append(len(plan) + 1) if _lang: - language_shots.append(len(shots) + 1) + language_shots.append(len(plan) + 1) # How much of this shot the line actually fills. A short line in a long # shot leaves the audio branch with time and nothing to put in it, and # what it puts there is more speech -- the line again. Counted here # where the beat is; judged against the shot length further down. _said_words = len(engine.spoken_text(body).split()) if _said_words: - _spoken_words[len(shots) + 1] = _said_words + _spoken_words[len(plan) + 1] = _said_words _device = device_voice_clause(body) if (_device_line and _has_people) else "" if _device: - device_shots.append(len(shots) + 1) + device_shots.append(len(plan) + 1) # The held scenery goes in, so the shot is not asked to keep the doors # shut and to sound like a door swinging in the same breath. heard = ([] if (not auto_sound or _own) @@ -9268,6 +11640,22 @@ class H3LongVideos: # different path and reports itself. if _own: heard = [v for v in named_vocals_in(body) if v not in heard] + heard + # AND WHAT IS HAPPENING BETWEEN THEM. A vocal is intermittent and the + # branch is open for the whole shot, so a list naming nothing but vocals + # describes the peaks and leaves the troughs blank -- and a blank trough on + # a joint model, next to a face, fills itself with speech. Reported as + # babble between the moans. + # + # Room tone does not answer it, even though the bed appends two continuous + # phrases below: the gap is a PERSON's audio presence, and a soft room with + # little echo is not a person. Breath is -- non-verbal, continuous where + # the vocal is not, and true of anybody making any of these six sounds. + # + # Only when the list is ALL vocal. A beat whose sound is already part + # non-vocal has something in the troughs, and the sound budget exists to + # stop inventories. + if heard and all(v in _NAMED_VOCALS for v in heard): + heard = heard + [_VOCAL_BETWEEN] if _will_silence: # The audio is pinned to silence for this shot's whole length, so a # sentence saying what it sounds like would describe an acoustic the @@ -9278,34 +11666,69 @@ class H3LongVideos: elif auto_sound and _room_now: heard = heard + [_room_now] if heard: - inferred_sound.append(len(shots) + 1) + inferred_sound.append(len(plan) + 1) # The branch is free on this shot, so SOMETHING fills it. Naming the sound # as the only thing heard leaves nothing for a voice to be -- it is not # the guard, the silence is, but it is what shapes a branch that is # legitimately open. Positively phrased: "the only sound is X" says what # IS there, where "nobody speaks" asks the model to render an absence. _sound = sound_clause(heard, only=not _speaks) - # RANKED, and cut to fit. Each of these was a good idea on its own and - # none of them counted the others; together they had reached 65% of the - # shot against a 12% beat, which is the state this node was rebuilt to - # escape. What the beat itself stages ranks above what merely persists. - # THE ENGINE DECIDES THE FACTS; THESE SENTENCES SAY THEM. - # - # The rewrite kept the half that was wrong and kept the half that was - # right. What was wrong was the DERIVATION: sixty readers each - # searching the beat alone, so nothing could notice that "neck" and - # "behind the back" contradicted, or that a beat naming two items had - # recorded one. That is now engine.SceneState -- one state, read once - # per beat, and the source of truth for what is on whom, where it - # holds, what it is anchored to and which room this is. - # - # What was RIGHT was the prose. Every clause below is worded the way - # it is because a specific render came back wrong: "both ends" exists - # because a garment came off a beat early, "dropped out of frame" - # because it reappeared, "the same object in the same material" - # because tape drifted into the nearest commoner object. Throwing that - # away would have cost more than the derivations ever did, so the - # builders stay and the engine feeds them. + # Gaze is resolved after the other guards so a character is never named + # twice. Person targets need no looker's name; object targets do when + # several people are present. + if hold_gaze and _gazers: + _g = _gazers[0] + _target, _is_person = looking_at[_g] + _elsewhere = " ".join([ + hold, _posture, _pose, _travel, _where, _told, turn, _duress, + _mouth, _revealed, _under, _bare, _wearing, tail, _moved, + anchors, _state_clause, _device, _sound, _pace, fall]) + + def _named_already(_n): + return bool(re.search(r"\b" + re.escape(_n) + r"\b", _elsewhere)) + + def _their_pronoun(_n): + """'her'/'his'/'their', if nobody else in the shot shares it.""" + _rows = {a: b for a, b in sheet_lines(shot_sheet) if a} + _m = re.search(r"\b(she|he|they)\b", _rows.get(_n, ""), re.I) + if not _m: + return "" + _sex = _m.group(1).lower() + for _o in (_described or []): + if _o != _n and re.search(r"\b" + _sex + r"\b", + _rows.get(_o, ""), re.I): + return "" + return {"she": "her", "he": "his", "they": "their"}[_sex] + + if _is_person: + # Nobody turns their eyes to themselves, so an impersonal + # sentence naming the TARGET can only be the other person's + # eyes. It spends no naming on the looker at all. + if not _named_already(_target): + _gaze = gaze_hold(_target, "", True) + elif len(_described or []) >= 2 or _g not in set(_described or []): + _who = "" if _named_already(_g) else f"{_g}'s" + if not _who: + _who = _their_pronoun(_g) + if _who: + _gaze = gaze_hold(_target, _who) + else: + _gaze = gaze_hold(_target) + if _gaze: + gaze_shots.append(len(plan) + 1) + # A LINE SAID, NO LOOK STAGED. gaze_hold restates what the beat named; + # a dialogue beat that names no look leaves both faces to the portrait + # prior, which is the lens. Reported as two people talking to the + # camera instead of each other. The addressee is in the shot, so the + # faces are turned to each other -- impersonally, both names here + # being already spent. Not for a voice from a device: somebody on the + # phone is not facing the room. A look the beat stages, even a pronoun + # one gaze_hold declines to restate, is never argued with. + if (hold_gaze and not _gaze and _speaks and not _look_now + and not _device_line and len(_described or []) >= 2): + _faces = dialogue_gaze(len(_described)) + if _faces: + dialogue_gaze_shots.append(len(plan) + 1) _guards = [ (1, "removal", tail), # the beat's own action, completing (1, "wearing", _wearing), # ...and its mirror, a garment going on @@ -9327,11 +11750,45 @@ class H3LongVideos: # a fact about metal. See pose_clause. (3, "pose", _pose), (11, "gaze", _gaze), + # Beside the gaze, because they answer the same pull: with nothing + # said about the eyes or the face, both come from the portrait prior. + # + # RANKED BELOW IT, though, and measured. At 11 it tied the gaze and + # won on list position, and on a short beat the budget then dropped + # the gaze clause from the very shot that staged the look while a + # stale copy survived on the shot after. Where the two compete, the + # spatial fact the beat itself stated goes first. + (12, "duress", _duress), (12, "mouth", _mouth), (12, "language", _lang), # ...and in which language + # The eye-line INFERRED for a dialogue shot. Reads after the mouth + # guard it belongs with. Rank 15, below even sound: it is a guess + # about where the eyes go, and at rank 11 -- the staged look's rank + # -- it took the budget from "Only Dan speaks" on a seven-word beat. + # An inference is cut before anything the author's own words imply. + # WHO IS WITH WHOM, ranked with the holds rather than the inferences: + # it restates a pairing the author WROTE, the way the gaze clause + # restates a look they wrote, and a wrong pairing is a gross error + # rather than a missing nicety. + (3, "contact", _contact), + (15, "faces", _faces), + # The frame, where nothing else says what it is. Ranked with the + # other inferred picture guards and below everything the author's + # own words imply: it is a guess about the camera, and the camera is + # the author's to state. See frame_hold. + (15, "frame", _frame), + # THE CAMERA STAYING PUT. An inference like the frame above it, and + # ranked above it, because this one does not stop at its own shot: the + # next shot opens on whatever viewpoint this one drifts to, so a + # dropped clause here is inherited by every shot after it. Still below + # anything the author's words imply, and silent the moment they say + # anything about the camera at all. + (13, "camera", _camera), (6, "told", _told), # a listener given an order to ignore (13, "turn", turn), - # LAST in the list and LAST in the ranking, both on purpose. + # LAST in the list, and last in the ranking of anything the author's + # words imply -- only the inferred eye-line (15) is cut before it. + # Both on purpose. # # This was appended after fit_guards and so was the one piece of # node-written text no budget could reach -- unranked, uncuttable, @@ -9357,27 +11814,47 @@ class H3LongVideos: ] _kept, _dropped = fit_guards(_guards, len(body.split())) if _dropped: - crowded.append((len(shots) + 1, _dropped)) - shot_text = (line + _kept).strip() + crowded.append((len(plan) + 1, _dropped)) + # Body count is a composition invariant, not a continuity detail. It + # must not evict speaker, gaze, or ownership clauses from the bounded + # guard budget; doing so fixed the extra body by breaking who spoke. + # The exact lines ride between the beat and the node's own clauses: after + # the action they belong to, ahead of everything this file decided. + # + # ...and under `verbatim` there is nothing after them. The clauses are still + # WORKED OUT -- info reports what each shot would have been told, which is + # what makes this switch worth having as a diagnostic -- they are simply not + # sent. See the widget's tooltip for what comes back with them. + shot_text = ((line + _exact).strip() if verbatim + else (line + _exact + _cast_hold + _kept).strip()) + # HOW OFTEN ONE PERSON IS NAMED IN ONE SHOT, counted where the shot is + # finished. This file's own rule is that naming somebody twice in a shot is + # what draws a second copy of them -- every clause that owns a fact pays + # that price to say whose fact it is -- and nothing was watching the total. + # Reported, not enforced: the beat's own mentions are the author's, and the + # clauses that name people do it to stop a fact landing on the wrong one. + for _n in (_described or []): + _total = len(re.findall(r"\b" + re.escape(_n) + r"\b", shot_text)) + if _total >= 3: + _mine = _total - len(re.findall(r"\b" + re.escape(_n) + r"\b", + f"{_scene_sent} {body} {_exact}")) + named_often.append((len(plan) + 1, _n, _total, _mine)) # Sound direction is not a continuity guard -- it asks for something to # HAPPEN rather than for something to stay as it is -- so it is counted # apart, or the balance report blames the wrong text for crowding the beat. _sound_kept = "" if "sound" in _dropped else _sound sound_words += len(_sound_kept.split()) guard_words += (len(shot_text.split()) - len(_sound_kept.split()) - - len(f"{shot_scene} {body}".split())) - beat_words += len(body.split()) + - len(f"{_scene_sent} {body}".split()) - len(_exact.split())) + beat_words += len(body.split()) + len(_exact.split()) total_words += len(shot_text.split()) - shots.append(shot_text) - shot_cast.append(list(active) if character_guard else []) - speech.append(_speaks) # The event sounds this beat implies, kept per shot so they can be # BUILT and mixed into that shot's span later. `heard` is not it: # that one has the bed and the room tone folded in and is emptied # on a silenced shot, which is precisely the shot this is for. - shot_events.append(list(sounds_for(body, held=[_state_key(t) - for t, _ in _pairs])) - if auto_sound else []) + _events = (list(sounds_for(body, held=[_state_key(t) + for t, _ in _pairs])) + if auto_sound else []) # What the AUTHOR wrote, and nothing this file worked out. See above -- # effort counts, because the verb staging it is theirs. # @@ -9389,8 +11866,9 @@ class H3LongVideos: # in one jump, so what it fills with is a voice. Ambience everywhere and # silence are mutually exclusive by construction: the silence latent IS # the audio, and there is no room in it for a room tone. - sounded.append(_own or _voiced) - voiced_only.append(bool(_voiced and not _own)) + plan.add(shot_text, + list(active) if character_guard else [], + _speaks, _own or _voiced, _voiced and not _own, _events) # What share of a shot is the node talking rather than the script. Continuity # clauses all say some version of "this stays as it is", and enough of them @@ -9416,6 +11894,38 @@ class H3LongVideos: "they are named in, this one included" if _bare else ". All of them carry a reference tag, which is what pins them here")) + _lora_model = lora_facts(model) + _lora_clip = lora_facts(getattr(clip, "patcher", None)) + if _lora_model[0] or _lora_clip[0]: + _name = lora_name_of(model) or lora_name_of(getattr(clip, "patcher", None)) + _said = [] + if _lora_model[0]: + _said.append(f"{_lora_model[0]} on the model over {_lora_model[1]} weights at " + f"strength {', '.join(f'{v:g}' for v in _lora_model[2][:4])}") + if _lora_clip[0]: + _said.append(f"{_lora_clip[0]} on the TEXT ENCODER over {_lora_clip[1]} weights at " + f"strength {', '.join(f'{v:g}' for v in _lora_clip[2][:4])}") + notes.append( + f"LoRA: {'; '.join(_said)}" + + (f" -- last one applied: {_name}" if _name else "") + + ". Reported because it is the one input to a shot this node neither " + "writes nor can read out of your text: two runs whose prompts are " + "identical render differently and nothing else here says why") + if verbatim: + # FIRST in the list, because every note after it describes a clause this run + # did not send. They are kept rather than suppressed: what the node WOULD + # have said, shot by shot, is the whole diagnostic value of this switch. + notes.insert(0, + "VERBATIM is on: each shot was sent your scene, your beat and the sheet " + "entries for the people it names, and nothing this node writes -- no body " + "count, no mouth guard, no camera take, no two-ended anchor for a door or " + "a walk, no posture, gaze, bare region, held state or sound direction. " + "Every note below still reports what a clause WOULD have said, which is " + "what makes this worth running: it tells you whether something you are " + "looking at is the node's doing or the model's. The mechanisms are " + "untouched -- the keyframe chain, the reference claims, silence pinning, " + "shot sizing, and the scoping that decides which of your own sentences a " + "shot gets") if total_words: notes.append( f"prompt balance: the beat is {100 * beat_words / total_words:.0f}% of " @@ -9450,18 +11960,26 @@ class H3LongVideos: "thing is actually visible") lens, len_note = plan_lengths(beats, ceiling, shot_length == "from the beat", pace) + plan.set_frame_counts(lens) + plan.validate() # How much of a SPEAKING shot the line does not cover. The branch is free for # the whole shot, so whatever the line does not fill is unconditioned audio in # a shot the model knows somebody is talking in -- which is where invented # speech after the line comes from. Reported per shot, because the fix is the # author's: a longer line, or a shorter shot. _tail = [] + _tailpin = [] # (shot, seconds) pinned past the line's end for _i, _b in enumerate(beats): if _i >= len(lens) or not has_speech(_b): continue _words = (sum(len(q.split()) for q in _QUOTED.findall(_b)) + sum(len(q.split()) for q in _DIALOGUE_TAG.findall(_b))) _say = _words / WORDS_PER_SEC + plan.shots[_i].line_seconds = _say + _tf = ShotAudio(True, True, False, bool(silence_nonspeech), speech_lead_seconds, + AUDIO_LATENT_FPS, _say, speech_tail_seconds, lens[_i]).tail_frames + if _tf: + _tailpin.append((_i + 1, _tf / AUDIO_LATENT_FPS)) _shot = lens[_i] / H3_FPS if _shot - _say >= 3.0: _tail.append((_i + 1, _words, _say, _shot)) @@ -9490,10 +12008,10 @@ class H3LongVideos: "longer than its action is filled by performing it more slowly. Lower " "pace for brisker movement" if _per > 3.5 else "")) if len(set(lens)) == 1: - notes.append(f"{len(shots)} shot(s) x {lens[0]}f (~{lens[0] / H3_FPS:.1f}s) " + notes.append(f"{len(plan)} shot(s) x {lens[0]}f (~{lens[0] / H3_FPS:.1f}s) " f"at {w}x{h} = ~{sum(lens) / H3_FPS:.1f}s total") else: - notes.append(f"{len(shots)} shot(s) at {w}x{h}, sized per beat: " + notes.append(f"{len(plan)} shot(s) at {w}x{h}, sized per beat: " + ", ".join(f"{n}f/{n / H3_FPS:.1f}s" for n in lens) + f" = ~{sum(lens) / H3_FPS:.1f}s total") if len_note: @@ -9517,6 +12035,185 @@ class H3LongVideos: f"when, never how fast: 'slowly' is a style instruction and this is " f"not one. Give the beat more to do, or shorten the shot, and it " f"stops being needed") + if scene_held: + _rooms = sorted({r for _, rs, _ in scene_held for r in rs}) + _waited = sorted({t for _, _, ts in scene_held for t in ts}) + notes.append( + f"the scene paragraph describes {', '.join(_rooms)}, and a paragraph that " + f"describes a room describes its FURNITURE too -- so that description WAITS " + f"OUTSIDE it, on shot(s) {', '.join(str(n) for n, _, _ in scene_held)}. The " + f"paragraph is stamped into every shot, which is what gives a removal " + f"something to scrub, and the room's NAME was already right in every shot -- " + f"but the bed was in the text standing beside it, and at cfg 1 there is no " + f"negative prompt that can take a named thing back. Reported as a bed in the " + f"living room two beats after she left the bedroom. The room a shot ENDS in " + f"decides this, not every room it passes through: a walk out of the bedroom " + f"does show it in the opening frames, but that shot's LAST frame is the next " + f"shot's keyframe, so a bed drawn at the end of the walk is inherited by the " + f"shot after it -- and the room being left arrives as a PICTURE anyway, " + f"because the keyframe IS the previous shot's last frame. So the words say " + f"where the shot ends and the frame carries where it began. Your paragraph is " + f"not edited: this is per shot, and a beat that walks back in gets it back in " + f"full. A character sheet line is never touched, whatever it names, and " + f"neither is the anchor or a room your beat mentions. What waited: " + + "; ".join(f'"{t}"' for t in _waited[:3]) + + ". If one of those also carried the film's own hour or light, split it: " + "one sentence for the film, one for the room") + if scene_welded: + notes.append( + f"shot(s) {', '.join(str(n) for n, _ in scene_welded)} are not in the room the " + f"scene paragraph describes, and that description was KEPT anyway, because " + f"holding it would have left those shots no scene sentence at all. The " + f"paragraph welds the film's own framing to one room's furniture in a single " + f"sentence, so taking the room would take the lighting and the hour with it, " + f"and a shot with no scene is a bigger change than a bed in the wrong room. " + f"Split it in two -- one sentence for the film ('A small flat at night.') and " + f"one for the room ('Her bedroom has an unmade bed and a lamp.') -- and the " + f"room's half will wait outside that room on its own") + if reentry_shots: + notes.append( + f"START FRESH where somebody still in the frame is staged walking in -- " + + "; ".join(f"shot {k + 1}: {_join_names(v)}" + for k, v in sorted(reentry_shots.items())) + + ". Nothing walked them out, so the frame the shot opens on still has " + f"them in it, and the beat brings them in again: kept, that is two of " + f"them. Write them leaving first ('Dan goes out to the car') and the shot " + f"keeps its keyframe") + if cut_shots: + notes.append( + f"shot(s) {', '.join(str(n + 1) for n in sorted(cut_shots))} CUT, because " + f"they OPEN IN A DIFFERENT ROOM from the one the shot before ended in. Every " + f"shot is anchored to the previous shot's last frame, and a keyframe is a " + f"PICTURE, which outvotes any sentence -- so a living-room shot opening on a " + f"frame of the kitchen renders neither of them, it renders a blend, and a " + f"kitchen blended with the words 'living room' is a bathroom: tiles, a sink, " + f"cabinets. So that frame is not frame one there. It still rides as a " + f"reference for the PEOPLE in it wherever all of them are in the new shot, so " + f"they keep their faces and clothes across the cut. A WALK IS NOT " + f"THIS: a travel beat opens in the room it is leaving, so that frame is the " + f"right one and the shot keeps its keyframe -- write the move as a journey " + f"('she walks through to the kitchen') and you get the walk instead of a cut") + if led_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in led_shots)} put the BEAT in front of " + f"the character sheet. The sheet has to be in every shot -- clothing " + f"continuity is read out of it -- but it is a description of a FACE, and it " + f"was leading every prompt ahead of the action. Measured: 69% of a shot's " + f"words sat in sentences about a face, and turning every face guard off only " + f"reached 63%, because the sheet is most of it. What LEADS a prompt decides " + f"its composition -- anatomy in the opening tokens is what a distilled model " + f"settles the frame on, and at cfg 1 no later sentence outvotes it. Your " + f"words are identical and none are rewritten; only the order changed, which " + f"is the one thing about this that had never been tried. Off with beat_leads " + f"to compare the two in one render") + if contact_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in contact_shots)} have three or more " + f"people and a beat that puts two of them in contact, so the shot is told " + f"WHICH body is with which. Your beat already says it, and it was the only " + f"thing that did: one sentence among everyone's appearance, and at cfg 1 " + f"the model reads the prompt as a bag of words and pairs by its own prior. " + f"Reported as girls kissing each other when they should have been kissing " + f"the boys. Both sides are named, because an unnamed pairing in a shot with " + f"four people is the sentence that let it choose. Read from your own words " + f"only -- a beat that pairs nobody by name ('they kiss') gets nothing, " + f"because guessing which two is the bug. With two people in the shot " + f"nothing is said: there is nobody else to pair with") + if named_often: + _worst = sorted(named_often, key=lambda r: -r[2])[:6] + notes.append( + "named more than twice in one shot -- " + + "; ".join(f"shot {n}: {who} {times}x ({mine} from this node)" + for n, who, times, mine in _worst) + + ". Naming a person twice in one shot is what draws a second copy of " + "them, and every clause that owns a fact -- a pose, a look, whose " + "voice it is, who is wearing what -- pays a naming to say whose fact " + "it is. Worth reading when duplicates persist: the ones from this node " + "go away with the guard that writes them (hold_gaze, hold_scene_state, " + "auto_sound, or verbatim for all of them), and the ones from your beat " + "are yours to rewrite -- a pronoun costs nothing") + if camera_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in camera_shots)} say nothing about the " + f"camera, so each is told it is one unbroken TAKE from one position, angle " + f"and distance. " + f"Reported as the camera moving on its own and breaking continuity -- and " + f"the chain is what makes that expensive, because every shot opens on the " + f"PREVIOUS shot's last frame. A shot that drifts hands the drifted " + f"viewpoint on, the next adds its own, and the room stops being the room. " + f"An unstated attribute is left to the model's prior, and for a video model " + f"that prior is movement. Your words always win: any camera note in the " + f"beat or the anchor stands it down there, and a journey between places " + f"keeps its moving camera. Off with hold_camera") + if exact_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in exact_shots)} carry an exact: line. " + f"It is placed straight after the beat in your words, and nothing in this " + f"node reads, scopes, scrubs, reorders or drops it -- it is not a guard " + f"and has no budget to lose, which is what makes it the one instruction " + f"that reaches the model exactly as written. Nothing reads it either: a " + f"name in it puts nobody in the shot, a garment in it removes nothing and " + f"a door in it stages no change, so write what must be SAID there and let " + f"the beat stage what happens. Counted against the beat in the balance " + f"below, because it is your text") + if frame_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in frame_shots)} stage something a " + f"portrait cannot contain and say nothing about the camera, so they are " + f"told what the frame HOLDS: the whole body, head to feet, with the room " + f"around it. An attribute a prompt does not state is not left to the model, " + f"it is left to the model's PRIOR -- and the prior for a named, described " + f"person is a portrait facing the lens. The sheet describes a face in every " + f"shot because clothing continuity needs it there, and the mouth guard " + f"describes a mouth in every silent shot because babble needs it, so the " + f"text leans towards a face and nothing in it said how much of the person to " + f"show. Reported as the camera fixated on one character staring into the " + f"lens, with no reference image in the run at all. Your camera always wins: " + f"write any framing in the beat or the anchor -- a close-up included, since " + f"a close-up is a frame somebody asked for -- and this stands down. It is " + f"ranked below everything your own words imply, so a crowded shot drops it " + f"first") + if open_moves: + notes.append( + "shot(s) " + ", ".join(f"{n} (to the {w})" for n, w in open_moves[:6]) + + " move somewhere the place list cannot name, so the arrival is told to be " + "PERFORMED -- the whole move on screen, first step to last. A closed list " + "of room words can never cover a script nobody has written yet, and a move " + "nobody is told to make is a move the model CUTS to: reported as the set " + "changing under the characters instead of them walking into it. This reads " + "the destination from your own words and claims nothing else about it -- no " + "room state, no acoustic, no cut decision -- so a dungeon, a cargo bay or a " + "stable all work without being listed anywhere. Anything that is furniture, " + "a body part, a vehicle or a person is left alone") + if untracked_strip: + _items = sorted({t for _n, ts in untracked_strip for t in ts}) + notes.append( + f"shot(s) {', '.join(str(n) for n, _ in untracked_strip)} take " + f"{', '.join(_items)} off PEOPLE THE SHEET DOES NOT NAME, and the " + f"removal reaches only the ones it does name. A character sheet entry is " + f"what a removal scrubs and what carries the bare region into every later " + f"shot; an unnamed woman has neither, so her own words come off in the " + f"beat that says so and nothing holds them off afterwards -- the next shot " + f"says nothing about her, and what it opens on is a keyframe taken while " + f"she was still half in them. Reported as some of the skirts still being " + f"on when all of them should have come off. Give each of them an entry, " + f"however short -- 'Girl 1: she, 20, a denim skirt.' -- and their removals " + f"hold exactly like the named character's. Your words are never rewritten " + f"either way; this is about what the node can keep saying after the beat " + f"that said it") + if _undescribed: + _them = "them" if len(_undescribed) > 1 else "it" + notes.append( + f"the film enters {', '.join(_undescribed)}, and your prompt never " + f"describes {_them} -- a room the text only NAMES is a room the model " + f"invents, and what it invents from is the frame the shot opened on plus " + f"whatever the other rooms suggest. Reported as a living room turning into " + f"a bathroom: a kitchen frame and the words 'living room' share tiles, a " + f"sink and cabinets, and nothing in the text said otherwise. Give each " + f"room a sentence of its own -- 'The living room has a green sofa and a low " + f"table.' -- either in the beat that enters it or in the scene paragraph. A " + f"scene sentence that names a room is carried ONLY in the shots that are in " + f"that room, so it costs every other shot nothing") if where_shots: notes.append( f"shot(s) {', '.join(str(n) for n in where_shots)} are in a room the " @@ -9740,7 +12437,26 @@ class H3LongVideos: f"frame faces the camera unless something says otherwise, and a " f"near-clean reference asks for the portrait's pose -- which looks at " f"the lens, because photographs of people do. Nothing is said about " - f"where the camera is. Off with hold_gaze") + f"where the camera is. It is HELD until something moves it, and a " + f"look belongs to whoever is doing the looking -- so it is said only " + f"in shots that describe that person, and named once a second person " + f"is in frame with them. Reported as one character stuck gazing at the " + f"camera while the other does his part: the target was one string with " + f"no owner, said impersonally, so a look she staged went on being said " + f"in shots she was not in and landed on whoever was. Off with hold_gaze") + if dialogue_gaze_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in dialogue_gaze_shots)} carry a line " + f"and two or more people, and the beat names nothing to look at, so the " + f"faces are turned to each other. Reported as two people talking to the " + f"camera instead of each other: with no look staged, both faces fall to " + f"the model's prior -- a portrait, facing the lens -- and a near-clean " + f"reference asks for exactly that pose. A line has an addressee whether " + f"or not the beat wrote one, and the addressee is in the shot, so this is " + f"the one thing that can be said without inventing. One impersonal " + f"sentence: both names in the shot are already spent, and a third " + f"mention is a third person. Write 'looks at' or 'turns to' in the beat " + f"and that is said instead. Off with hold_gaze") if anchored_shots: notes.append( f"fastened limbs held in place on shot(s) {', '.join(str(n) for n in anchored_shots)}" @@ -9775,7 +12491,7 @@ class H3LongVideos: if ambient_shots: notes.append( f"shot(s) {', '.join(str(n) for n in ambient_shots)} were given an " - f"ambient bed read from the anchor and the scene -- \"{ambient_bed}\". " + f"ambient bed read from {_bed_src} -- \"{ambient_bed}\". " f"It goes under shots whose audio branch is ALREADY open: ones with a " f"line, or with a sound you wrote yourself. It can never open one. " f"AMBIENCE ON EVERY SHOT WAS TRIED AND DOES NOT WORK: the bed was " @@ -9926,6 +12642,89 @@ class H3LongVideos: f"lips-closed line loses to a stream that has decided somebody is " f"talking. Shots staging effort are left out on purpose -- straining is " f"vocal and that mouth should be open. Off with mouths_shut_when_no_line") + # WHICH OF THE THREE HAPPENED, every run, because the inference is weak and + # the author needs to know when it decided nothing. + if _film_mood == "grim": + notes.append( + "the anchor declares the film's tone, so every shot carries \"The mood " + "is grim.\" and no guessing is done. That is the reliable way to set " + "it: swept over 512 beats, inferring a mood from the beats alone " + "called an ORDINARY film grim as often as a duress one, because the " + "words overlap -- screams is a waterslide, tied is a boat, bound is a " + "flight to Lisbon, chained is a desk job") + elif _film_mood == "light": + notes.append( + "the anchor declares a light tone, so no grim mood is applied to any " + "shot whatever the beats say. That is the override for a wrong " + "reading, and it wins outright") + elif _film_duress: + notes.append( + "no tone is declared in the anchor, and the beats or the character " + "sheet carry UNAMBIGUOUS duress -- hardware on a body, a captor, an " + "abduction, being locked in -- so every shot carries \"The mood is " + "grim.\" Only unambiguous evidence counts here: ordinary coercion " + "verbs and distress words are not enough on their own, because " + "grabbing, dragging and screaming are as much a garden centre and a " + "waterslide as an abduction. Write the tone into the anchor to settle " + "it either way") + elif any(beat_duress_strength(b) for b in beats): + notes.append( + "some beats read as though they MIGHT stage duress -- coercion or " + "distress verbs -- but nothing unambiguous, so no mood was applied and " + "every face is left to the model, whose prior for a described person " + "is a pleasant posed portrait. If this film has a tone, write it into " + "the anchor: 'grim', 'tense', 'a kidnapping' and the like turn it on " + "for every shot, and 'warm' or 'comic' turn it off for good. Measured " + "over 512 beats, guessing from the beats alone is no better than a " + "coin toss, so it does not guess") + if vocal_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in vocal_shots)} have a vocal that " + f"belongs to somebody -- a whimper, a sob, a moan -- so the shot is " + f"told whose it is, and the mouths owning neither a line nor a sound " + f"are closed. Reported as one character's whimpering opening up " + f"another's ability to babble. A vocal opens the audio branch, which " + f"is right -- it is meant to be heard -- but the flag saying so was " + f"shot-level with no owner, and BOTH mouth guards stood down on it " + f"for everybody in the shot. The person straining should have an open " + f"mouth; the person watching them should not, and theirs was the face " + f"an invented voice landed on. Two sources in one shot are named " + f"separately for the same reason, so the line and the vocal cannot be " + f"swapped between them. A vocal the beat does not pin on anybody " + f"holds nobody: closing mouths on a guess could close the mouth " + f"making the noise. This changes which faces move and never what the " + f"audio is conditioned on. Off with mouths_shut_when_no_line") + if duress_shots: + notes.append( + f"shot(s) {', '.join(str(n) for n in duress_shots)} are told what the " + f"face is doing, because the scene already stages duress -- restraint " + f"hardware the sheet lists on somebody in the shot, or your own " + f"distress verbs in the beat. Reported as somebody smiling at the " + f"camera in a scene of duress: a four-shot scene of a woman handcuffed " + f"in a van had not one word in it about anybody's face, and an " + f"attribute a prompt does not state is not LEFT to the model, it is " + f"left to the model's prior -- which for a named, described person is " + f"a portrait, facing the lens, pleasantly, because that is what " + f"photographs of people are. The eyes have had a clause since " + f"hold_gaze; the expression never had one. It is one sentence, it " + f"names no camera, and it reads the staging rather than inventing a " + f"feeling -- a shot staging neither gets nothing, and a beat that " + f"already says what the face does is never argued with. Picture only: " + f"it can never open the audio branch") + if mouth_acting: + notes.append( + f"shot(s) {', '.join(str(n) for n in mouth_acting)} kept their mouths " + f"because the beat itself puts the mouth to work -- a grin, a yawn, a " + f"bitten lip, a jaw dropping. The guard holds mouths closed on every " + f"shot with nobody speaking, and against a face doing nothing that is " + f"right; against these it was countermanding the only performance " + f"direction the shot has, in one case word for word. The beat has said " + f"what the mouth does, so nothing is added over the top. This frees the " + f"PICTURE only: a smile is silent, and the audio branch is left exactly " + f"where it was, because a silent expression is the commonest beat there " + f"is and letting one open a branch would be the invented voice back at " + f"its widest point. A stare or a wince is a face acting with its mouth " + f"shut and is still held") if muted_sound: notes.append( f"shot(s) {', '.join(str(n) for n in muted_sound)} gave up the sound you " @@ -9963,9 +12762,9 @@ class H3LongVideos: # that two of eleven can babble and gave them no way to find out which two # -- and the whole point of the note is that the beat's own sound wording is # what opened it, which cannot be acted on without knowing the beat. - _open_br = [i + 1 for i, (s_, snd) in enumerate(zip(speech, sounded)) + _open_br = [i + 1 for i, (s_, snd) in enumerate((shot.speech, shot.sounded) for shot in plan.shots) if not s_ and snd] - _pinned = [i + 1 for i, (s_, snd) in enumerate(zip(speech, sounded)) + _pinned = [i + 1 for i, (s_, snd) in enumerate((shot.speech, shot.sounded) for shot in plan.shots) if not s_ and not snd] n_silent, n_kept = len(_pinned), len(_open_br) if silence_nonspeech and n_kept: @@ -9994,15 +12793,57 @@ class H3LongVideos: # the author's. See _voiced in the shot loop. if first_frame is None: - notes.append("no first_frame: shot 1 has nothing pinning its opening frame, so its " - "starting pose and framing come from the text and any reference") + # SAID AS AN ASYMMETRY, because that is what it is and the old wording hid + # it. This used to read "shot 1 has nothing pinning its opening frame, so + # its starting pose and framing come from the text and any reference" -- + # true, and it left out the half that matters: every OTHER shot IS pinned, + # by the previous shot's last frame, so shot 1 is the only shot in the film + # that is free. What that looks like from outside is not shot 1 drifting, + # it is shot 1 disagreeing with a chain that agrees with itself. + # + # Reported exactly that way -- "doesn't look the same from the first to + # last beat", "the remaining beats are fine" -- together with a hardware + # artefact in beat 1 alone, hair caught in a collar. Both are the same + # thing: an arrangement no picture settles is settled by the model, and + # from shot 2 on the keyframe settles it. + # + # And the note has to say which dial is NOT this one, because the report + # came with "I even have image reference strength set to 0.999". It cannot + # work. build_conditioning's own comment is plain about it: "the keyframe + # ANCHORS the first frame, which is what continuity needs, while a + # reference only supplies identity. They are not alternatives." Raising + # ref_noise_aug makes the reference cleaner; it does not give shot 1 a + # first frame, because there is no frame there to clean. + notes.append( + "NO first_frame IS WIRED, so shot 1 is the only shot in this film whose " + "opening frame is pinned by NOTHING. Every other shot opens on the " + "previous shot's last frame, which fixes its pose, its framing and the " + "arrangement of everything on the body; shot 1 has only the text and " + "any reference. So the chain agrees with itself and shot 1 is the one " + "that can disagree -- which from outside looks like the person changing " + "between the first beat and the rest, and is also where a one-shot-only " + "oddity comes from: hair sitting differently against a collar, a " + "garment hanging differently, a pose the beat did not ask for. " + "ref_noise_aug IS NOT THE DIAL FOR THIS and raising it cannot help: a " + "reference says WHO somebody is and a keyframe says what the opening " + "frame HOLDS, and they are not alternatives -- there is no frame on " + "shot 1 for a cleaner reference to sharpen. Wire first_frame to fix it" + + (", and see the ref_noise_aug note above for what to put in it -- it " + "pins the WHOLE frame, so a composed frame of the shot you want and " + "not an identity portrait" + if refs_all else + ". It pins the WHOLE frame, so give it a composed frame of the shot " + "you want: subject, pose, framing, background. The last frame of a " + "previous run, or any still matching how beat 1 should open") + + ". Leaving it empty is fine when beat 1 is meant to establish the " + "look and the rest follow it -- which is what is happening now") # Text in the frame. H3 draws letterforms when the prompt names them, and at # cfg 1 there is no negative prompt to take them back -- adding "no watermark" # to the positive only names it again, which is how a mention becomes a # presence cue. So: point at the words, and leave the decision to the author. # H3 has a caption channel of its own. A prompt carrying those tokens is # ASKING for text on the picture. - if any(_CAPTION_TOKEN.search(s) for s in shots): + if any(_CAPTION_TOKEN.search(s) for s in plan.prompts): notes.append("the prompt contains H3's caption/lyrics tokens " "(<|caption_start|> and friends) -- those request text ON the " "picture. Remove them unless you want subtitles burned in") @@ -10025,7 +12866,7 @@ class H3LongVideos: f"one of them IS a line, end it with punctuation or mark it " f"yourself with ... and it will be spoken rather than " f"drawn") - cued = sorted({m.group(0).lower() for s in shots for m in _TEXT_CUE.finditer(s)}) + cued = sorted({m.group(0).lower() for s in plan.prompts for m in _TEXT_CUE.finditer(s)}) if cued: notes.append(f"the prompt names on-screen text ({', '.join(cued)}) -- H3 draws " f"letterforms when asked, and at cfg 1 no negative prompt can take " @@ -10066,17 +12907,21 @@ class H3LongVideos: # a reason to start placing pictures everywhere. _written = "\n".join([scene or ""] + list(beats)) _tagged = bool(picture_tags(_written) - or any(picture_tags(s) for s in shots)) + or any(picture_tags(s) for s in plan.prompts)) _tagged_names = {n for n, ln in sheet_lines(sheet) if n and picture_tags(ln)} + _claimed_untagged, _held_untagged = [], [] if refs_all and not _tagged: notes.append( f"{len(refs_all)} reference image(s) connected and no tag " - f"anywhere, so they go on EVERY shot -- placing by tag would place them " - f"nowhere. To aim them, write the tag on the person they depict: 'Nora: " - f", 34, she, ...'. Each then travels with that person into " - f"the shots she is in, and only those") - shot_refs_all = [] - for _i, _s in enumerate(shots): + f"anywhere. A picture the text NAMES is that subject; one it never " + f"mentions is another subject standing beside them -- which is a second " + f"person no wording in this node can argue with, because it arrives as a " + f"picture. So an untagged reference is claimed where the claim is " + f"unambiguous -- one picture, one person described in the shot, the tag " + f"written onto their sheet entry -- and held back where it is not. TAG " + f"IT and neither happens: 'Nora: , 34, she, ...' sends it into " + f"the shots Nora is in, and only those") + for _i, _s in enumerate(plan.prompts): # The tag is the BINDING between a picture and the subject the prompt # describes, and it stays IN the text -- comfy_extras/nodes_minimax_h3.py: # "the prompt refers to them as ", "Use the same tags when @@ -10084,15 +12929,59 @@ class H3LongVideos: # order it receives images and a shot carrying only slot 2 receives that # image as . if not _tagged: - shot_refs_all.append(list(refs_all)) + # AN UNTAGGED REFERENCE WAS SENT WITH NOTHING NAMING IT, on every shot. + # + # That is this file's oldest rule broken in its commonest setup: "a + # picture the prompt refers to is that subject; one it never mentions + # is ANOTHER subject" -- and connecting a face to ref_image_1 without + # writing a tag is how most people wire one up. Reported as duplicate + # characters that survive every guard here, because no guard in this + # file can argue with a second subject arriving as a PICTURE. + # + # Claimed where the claim is unambiguous: one picture, and one person + # described in the shot. That person is who a lone face reference + # depicts in every real script, and the tag goes on their sheet entry + # exactly as a written one would. + # + # HELD where it is not. Two pictures, or two people in the shot, and + # the node would be guessing which picture is whom -- so the shot goes + # without, the same answer every other unclaimable picture here gets. + # A reference that does not ride costs likeness; one that rides + # unclaimed costs a second person, and the author is told to tag it. + _here = [n for n in plan.shots[_i].cast if n] + if not _here: + # NOBODY TO DUPLICATE. A shot with no person described in it cannot + # grow a second character, whatever the picture is of, so a look or + # a location reference rides as it always did. + plan.shots[_i].refs = list(refs_all) + elif len(refs_all) == 1 and f"{_here[0]}:" in _s and len(_here) == 1: + plan.shots[_i].prompt = _s.replace(f"{_here[0]}:", f"{_here[0]}: ,", 1) + plan.shots[_i].refs = list(refs_all) + _claimed_untagged.append(_i + 1) + else: + plan.shots[_i].refs = [] + _held_untagged.append(_i + 1) continue _s, _r, _missing = resolve_tags(_s, refs_all) - shots[_i] = _s - shot_refs_all.append(_r) + plan.shots[_i].prompt = _s + plan.shots[_i].refs = _r for _n in _missing: _msg = f" names a slot with no image connected" if _msg not in notes: notes.append(_msg) + if _claimed_untagged: + notes.append( + f"shot(s) {', '.join(str(n) for n in _claimed_untagged)} had the untagged " + f"reference claimed on the one person they describe, so the picture has a " + f"subject in the text instead of arriving as a stranger") + if _held_untagged: + notes.append( + f"shot(s) {', '.join(str(n) for n in _held_untagged)} were sent NO " + f"reference: more than one picture or more than one person is in them, and " + f"which picture is whom is not something this node can guess. Sent " + f"unclaimed it would be a second person in the shot; held back it costs " + f"likeness there. Tag the pictures -- 'Dan: , ...' -- and they " + f"ride every shot that names their subject, claimed") # ONE FACE, TWO PEOPLE. A shot that carries a picture for somebody AND # describes somebody else who has none gives the model a photographed face # and two faces to draw. A reference is the strongest identity signal in the @@ -10106,10 +12995,10 @@ class H3LongVideos: # of the fix -- a second reference, tagged onto the other person. _twinned = [] if refs_all and _tagged_names: - for _i, _s in enumerate(shots): + for _i, _s in enumerate(plan.prompts): if not picture_tags(_s): continue - _cast_here = shot_cast[_i] if _i < len(shot_cast) else [] + _cast_here = plan.shots[_i].cast _cast_here = [n for n in _cast_here if n] or [ n for n, _ in sheet_lines(sheet) if n] _bare = [n for n in _cast_here if n not in _tagged_names] @@ -10132,8 +13021,40 @@ class H3LongVideos: f"'{_who[0]}: , ...' -- so every shot with both of them " f"carries both faces. No wording fixes this: nothing in the text " f"outranks a photograph") + if not character_guard and len([n for n, _ in sheet_lines(sheet) if n]) > 1: + _wardrobes = [f"{n} ({', '.join(garments_in(ln)[:3])})" + for n, ln in sheet_lines(sheet) if n and garments_in(ln)] + notes.append( + "character_guard is OFF, so EVERY sheet line is in EVERY shot -- " + "including the wardrobe of everyone the beat does not involve. " + + ("With " + "; ".join(_wardrobes[:4]) + ", " if _wardrobes else "") + + "a shot about one person is also describing what the others have on, " + "and at cfg 1 the model reads the prompt as a bag of words before it " + "reads a label: a garment listed for one character lands on whichever " + "body is in frame. Reported as boys wearing stockings. Measured: with " + "the guard ON, a shot that names only the men carries no word of the " + "women's clothing at all, because only the people a beat involves are " + "described. Turn it on. For extras nobody has an entry for, write them " + "into the beat instead -- your words reach the model verbatim and an " + "unnamed person needs no entry, though a removal cannot be held for one") + if refs_all and _tagged and not character_guard: + notes.append( + f"character_guard is OFF and {len(refs_all)} reference image(s) are tagged -- " + f"the combination that fixes the camera on one person. Off, EVERY sheet line " + f"goes into every shot, including the line carrying , so the " + f"reference is named in every shot and rides all of them. At ref_noise_aug " + f"{float(ref_noise_aug):g} that asks the model to reproduce the PICTURE -- " + f"pose and framing, not only the face -- so the portrait's composition " + f"becomes every shot's composition, and anyone without a reference is placed " + f"relative to it. AND TURNING THE GUARD OFF ADDS NOBODY: it describes the " + f"people your SHEET already names, in every shot, whether the beat involves " + f"them or not. For extras nobody has a sheet entry for, leave the guard ON " + f"and write them into the beat -- your words reach the model verbatim and an " + f"unnamed person needs no entry. To loosen the framing instead, lower " + f"ref_noise_aug (try 0.95, then 0.90) or crop the reference to head and " + f"shoulders, so there is less composition in it to reproduce") if refs_all: - _named = sum(1 for s in shots if picture_tags(s)) + _named = sum(1 for s in plan.prompts if picture_tags(s)) notes.append( f"{len(refs_all)} reference image(s) supply IDENTITY, and they go WHERE " f"TAGGED: every shot whose text names carries the image on " @@ -10166,9 +13087,9 @@ class H3LongVideos: f"holding. A hybrid fl2va/ref2va checkpoint is trained for reference " f"conditioning and does not make this trade; on a plain fl2va one, " f"lowering ref_noise_aug is the dial") - if _named < len(shots): + if _named < len(plan): notes.append( - f"{len(shots) - _named} shot(s) name no at all, so they " + f"{len(plan) - _named} shot(s) name no at all, so they " f"carry no reference. Claim it on the person it depicts -- 'Nora: " f", 34, she, ...' -- and it travels with her into the shots " f"she is in, and only those. A picture the prompt never refers to is " @@ -10202,7 +13123,52 @@ class H3LongVideos: # Never advise RAISING it: the target is a ceiling on the last step, not a # setting to move towards from below. _fix_a = min(shift_audio_for(steps), float(shift_audio or 0.0) or 1.0) - if _last_a > 0.4: + # ...AND THE NODE CAN SHORTEN THE FALL ITSELF, without taking the scheduler + # away from the picture. The audio branch has no schedule of its own -- it is + # derived from the video sigma at every step -- so choosing a scheduler for + # the audio means giving up the one chosen for the video. One extra step does + # not: it splits the final jump and leaves every earlier sigma alone. + # + # Only when the node is the one setting the shift. With apply_model_sampling + # off, the shifts this is computed from are not the shifts the model uses, + # and a schedule built on the wrong ones would be worse than none. A wired + # `sigmas` input is the author's own schedule and is never touched. + _soft_landing = bool(apply_model_sampling + and not (sigmas is not None and len(sigmas))) + # WHETHER THE LANDING ACTUALLY FIRES. Used twice: to describe it, and to stop + # the older warning sending the reader off to do by hand the thing that has + # already been done for them. + _landing_on = bool(_soft_landing and _last_a > 0.10) + if _landing_on: + notes.append( + f"the audio branch was landing from sigma {_last_a:.3f} on its final " + f"step, so ONE extra step is spliced into the end of the schedule to " + f"put it down at about 0.030 instead. That step costs one model " + f"evaluation per shot and nothing else: every earlier sigma is exactly " + f"where '{scheduler}' put it, so the picture keeps the schedule you " + f"chose. This is the one lever prose cannot reach -- every clause in " + f"this node changes what the branch is TOLD, and none of them changes " + f"how much noise it still has to clear when it stops. A branch " + f"resolving 43% of its denoising in one jump invents whatever is " + f"easiest, which on a branch told somebody speaks is a voice, and it " + f"lands at the OPENING of the shot because that is where there is " + f"least conditioning to anchor it. Choosing a scheduler that already " + f"finishes the audio" + + (f" -- '{_alt_sched[0]}' leaves {_alt_sched[1]:.3f}" if _alt_sched + else "") + + " is still the better fix and costs no step; this one fires only " + "while the tail is steep. It lands at 0.030 whatever shift_audio is " + "set to -- 1, 3 and 5 all end up there -- so shift_audio does NOT " + "need tuning by hand for this any more, and the older advice to " + "lower it does not apply while this is on. Off by wiring your own " + "`sigmas`, or with apply_model_sampling") + # ...and NOT where the landing has already dealt with it. Both notes fired + # together at the shipped defaults and the second was false the moment the + # first was true: it said the branch "still has sigma 0.43 to clear on its + # FINAL step" when that final step had just been replaced, and then sent the + # reader off to lower shift_audio by hand. Asked directly whether the manual + # shift was still needed, which is the confusion this caused. + if _last_a > 0.4 and not _landing_on: notes.append( f"the audio branch still has sigma {_last_a:.2f} to clear on its FINAL " f"step at {int(steps)} steps with shift_audio {float(shift_audio):g} -- " @@ -10211,11 +13177,15 @@ class H3LongVideos: f"voice. It is the step where babble appears. shift_VIDEO does not " f"change this: time_shift_sigma inverts the video shift and re-applies " f"the audio one. " - + (f"The SCHEDULER is the biggest dial here and '{scheduler}' is not " - f"using it: '{_alt_sched[0]}' at these same {int(steps)} steps and " - f"the same shift_audio leaves {_alt_sched[1]:.3f} instead of " - f"{_last_a:.2f}, because it spends steps in the low-sigma tail " - f"where the fine detail of speech is resolved. Try that first. " + + (f"'{_alt_sched[0]}' at these same {int(steps)} steps and the same " + f"shift_audio leaves {_alt_sched[1]:.3f} instead of {_last_a:.2f}, " + f"and it honours shift_video, so the picture keeps the schedule " + f"shape you asked for. DO NOT reach for kl_optimal, exponential or " + f"karras for this: comfy grades schedulers by use_ms, those three " + f"are called with sigma_min and sigma_max ONLY and never see the " + f"shift at all, so the video schedule collapses off its high-sigma " + f"steps and the picture comes out watery. Reported exactly that " + f"way. " if _alt_sched else "") + f"Otherwise LOWER shift_audio or raise steps -- sigma rises with " f"shift_audio, so raising it makes this worse. shift_audio " @@ -10226,7 +13196,7 @@ class H3LongVideos: # Probed whenever silencing is ON, not only when a shot is silent today: # an ambient bed can cover every shot, and the answer still matters for # the moment one is not covered -- and for knowing the wiring is sound. - if silence_nonspeech: + if silence_nonspeech or speech_lead_seconds > 0 or speech_tail_seconds > 0: if audio_vae is None: notes.append( "SILENCE CANNOT BE APPLIED: no audio VAE is wired to the node's " @@ -10250,16 +13220,111 @@ class H3LongVideos: else: notes.append( f"silence can be applied: the audio VAE encodes silence, so the " - f"{n_silent} shot(s) above are pinned to it rather than merely " - f"told to be quiet") - sent_text = list(shots) - script = "\n---\n".join(f"[Shot {i}] {s}" for i, s in enumerate(shots, 1)) + f"{n_silent} line-free shot(s) above can be pinned to it rather " + f"than merely told to be quiet" + + (f", and dialogue gets a {speech_lead_seconds:g}s silent lead-in" + if speech_lead_seconds > 0 else "") + + ((", and a silent tail past the line on shot(s) " + + ", ".join(f"{n} (last {s:.1f}s)" for n, s in _tailpin) + + f" -- everything after lead + the line's estimate + " + f"{speech_tail_seconds:g}s is pinned, so the branch cannot carry on " + f"talking into the seconds the line does not fill. The model chooses " + f"when to speak: if a last word is clipped, raise speech_tail_seconds") + if _tailpin else "")) + script = "\n---\n".join(f"[Shot {i}] {s}" for i, s in enumerate(plan.prompts, 1)) info = " | ".join(notes) if plan_only: empty = torch.zeros((1, h, w, 3)) return (empty, {"waveform": torch.zeros((1, 2, 1)), "sample_rate": 44100}, "PLAN ONLY -- nothing rendered. " + info, script, - lens[0], 0, len(shots), 0.0) + lens[0], 0, len(plan), 0.0) + + return PreparedVideo( + _placed_shots=_placed_shots, _first_is_plate=_first_is_plate, + _returns=_returns, _soft_landing=_soft_landing, _tagged_names=_tagged_names, + ambient_audio=ambient_audio, ambient_level=ambient_level, apply_model_sampling=apply_model_sampling, + audio_vae=audio_vae, auto_sound=auto_sound, bared_shots=bared_shots, + cfg=cfg, cleanup_between_shots=cleanup_between_shots, clip=clip, + first_frame=first_frame, foley_level=foley_level, h=h, + latent_upscale=latent_upscale, latent_upscale_scale=latent_upscale_scale, + megapixels=megapixels, model=model, moved_shots=moved_shots, + negative=negative, notes=notes, plan=plan, + ref_noise_aug=ref_noise_aug, restart_after_removal=restart_after_removal, revealed_shots=revealed_shots, + sampler_name=sampler_name, scheduler=scheduler, seed=seed, + shift_audio=shift_audio, shift_video=shift_video, + sigmas=sigmas, silence_nonspeech=silence_nonspeech, + speech_lead_seconds=speech_lead_seconds, speech_tail_seconds=speech_tail_seconds, hold_levels=hold_levels, handoff_frames=handoff_frames, staging_shots=staging_shots, steps=steps, + stripped_shots=stripped_shots, cut_shots=cut_shots, + shot_rooms=shot_rooms, hardware_changed=hardware_changed, shot_frames=shot_frames, + reentry_shots=reentry_shots, + tiled_decode=tiled_decode, trim_seam=trim_seam, + upscale=upscale, upscale_batch=upscale_batch, upscale_model=upscale_model, + upscale_target_short_edge=upscale_target_short_edge, vae=vae, w=w, + ) + + def _render(self, prepared): + """Execute the prepared shots and assemble the video and soundtrack.""" + _placed_shots = prepared._placed_shots + _first_is_plate = prepared._first_is_plate + _returns = prepared._returns + _soft_landing = prepared._soft_landing + _tagged_names = prepared._tagged_names + shot_rooms = prepared.shot_rooms or {} + _shot_frames = prepared.shot_frames or {} + reentry_shots = prepared.reentry_shots or {} + hardware_changed = prepared.hardware_changed or set() + ambient_audio = prepared.ambient_audio + ambient_level = prepared.ambient_level + apply_model_sampling = prepared.apply_model_sampling + audio_vae = prepared.audio_vae + auto_sound = prepared.auto_sound + bared_shots = prepared.bared_shots + cfg = prepared.cfg + cleanup_between_shots = prepared.cleanup_between_shots + clip = prepared.clip + first_frame = prepared.first_frame + foley_level = prepared.foley_level + h = prepared.h + latent_upscale = prepared.latent_upscale + latent_upscale_scale = prepared.latent_upscale_scale + megapixels = prepared.megapixels + model = prepared.model + moved_shots = prepared.moved_shots + negative = prepared.negative + notes = prepared.notes + plan = prepared.plan + ref_noise_aug = prepared.ref_noise_aug + restart_after_removal = prepared.restart_after_removal + revealed_shots = prepared.revealed_shots + sampler_name = prepared.sampler_name + scheduler = prepared.scheduler + seed = prepared.seed + shift_audio = prepared.shift_audio + shift_video = prepared.shift_video + sigmas = prepared.sigmas + silence_nonspeech = prepared.silence_nonspeech + speech_lead_seconds = prepared.speech_lead_seconds + speech_tail_seconds = prepared.speech_tail_seconds + hold_levels = prepared.hold_levels + handoff_frames = prepared.handoff_frames + staging_shots = prepared.staging_shots + steps = prepared.steps + stripped_shots = prepared.stripped_shots + cut_shots = prepared.cut_shots + tiled_decode = prepared.tiled_decode + trim_seam = prepared.trim_seam + upscale = prepared.upscale + upscale_batch = prepared.upscale_batch + upscale_model = prepared.upscale_model + upscale_target_short_edge = prepared.upscale_target_short_edge + vae = prepared.vae + w = prepared.w + + def _frame_cast(k, last=False): + # Who shot k's frames show -- its described cast plus anybody the chain + # still carries. See shot_frames. + _c = [n for n in plan.shots[k].cast if n] + return list(_shot_frames.get(k, (_c, _c))[1 if last else 0]) if apply_model_sampling: model, ms_note = apply_h3_model_sampling(model, shift_video, shift_audio) @@ -10277,22 +13342,53 @@ class H3LongVideos: _aug_warned = False fresh = [] t_start = time.perf_counter() - vid_out, aud_out, sr = [], [], 44100 - # THE FINISHED CHAIN, ALLOCATED ONCE, BEFORE THE FIRST SHOT LANDS IN IT. - # vid_out survives only as the overflow path -- see the shot write below. - _dst, _at = None, 0 + aud_out, sr = [], 44100 + # AN UPPER BOUND, NOT AN ESTIMATE. This used to subtract one frame per seam on + # the assumption that trim_seam drops one from every shot after the first. It no + # longer does: a shot that opens on no keyframe keeps its first frame, and a + # room change or a removal makes such shots on purpose. The buffer then filled + # and FrameAccumulator fell through to its overflow list -- which, with + # cleanup_between_shots off, retains each shot's decoded frames ON THE GPU, + # uncopied, for the rest of the run. Higher peak VRAM under dynamic VRAM loading + # is where a bad free turns into an illegal access, and the whole point of the + # accumulator is that the final tensor is allocated ONCE. + # + # Over-allocating by at most one frame per seam is a rounding error against a + # chain of hundreds, and it is the difference between a bounded allocation and + # an unbounded list of live GPU tensors. + frame_capacity = sum(shot.frame_count for shot in plan.shots) + frames = FrameAccumulator(frame_capacity, _image_out_dtype(), cleanup_between_shots) + # Reachable from run(), so an interrupt can drop it before unwinding. See there. + self._frames = frames av_fix = 0 # samples of A/V drift corrected across the chain _captured = {} # name -> a frame from the last shot they were in _captured_from = {} # name -> which shot that frame came from + _captured_gen = {} # name -> the wardrobe generation that frame shows + _soft_cuts = [] # (shot, why) cuts that carried the frame as a reference _recovered = [] # (shot, name, source shot) actually pinned + _evened = [] # (shot, name, source shot) given a face beside a tagged one + _room_frames = {} # room -> [(last frame there, who was in it, wardrobe generation, shot)], newest first + _room_returns = [] # (shot, room, source shot) actually carried + _wardrobe_gen = 0 # bumped by every shot that changes what anybody wears or is held by _handoff_claimed = [] # shots whose demoted handoff was named in the text + _untrimmed = [] # shots that opened on no keyframe, so kept frame one + _plate_on = 0 # the shot whose first_frame rides as the SET _carried = [] # (shot, who was there, who joins) room carried on shot_detail = [] # (detail, contrast) per shot, on its last frame + # One per run, never reset at a chain break: the grade belongs to the FILM, and + # restarting it per segment would give a film one grade per segment, which is a + # worse-looking version of the same complaint. + _levels = HandoffLevels() _SILENCE_STATUS.update(asked=0, applied=0, why="") _deep_cleanup() - for i, shot_prompt in enumerate(shots): - silent = bool(silence_nonspeech and not speech[i] and not sounded[i]) + for i, shot in enumerate(plan.shots): + shot_prompt = shot.prompt + _audio = ShotAudio(plan.shots[i].speech, plan.shots[i].sounded, plan.shots[i].voiced_only, + bool(silence_nonspeech), speech_lead_seconds, + AUDIO_LATENT_FPS, shot.line_seconds, speech_tail_seconds, + shot.frame_count) + silent = _audio.pinned # A shot that follows a removal starts FRESH. Every shot is anchored to # the previous one's last frame, so if the model did not finish taking @@ -10303,9 +13399,57 @@ class H3LongVideos: # cut exactly where a cut belongs. shot_handoff = handoff _handoff_ref = False + # NEITHER CUT THROWS THE FRAME AWAY ANY MORE. Dropping it left the shot + # with no picture of anything: the room, the faces, the hair and whatever + # everybody still wears were all re-imagined from the text, which is a + # new scene with new people in it. Reported as shots cutting to a new + # scene and breaking character continuity. + # + # The frame is DEMOTED instead, the way _placed_shots demotes it: a + # reference supplies appearance without being frame one, so a garment the + # model left half off is not pinned into the opening frame, and a room + # change is not blended into the new room -- while who these people are + # and what they look like carries across. Only when it is safe: everybody + # in that frame is in this shot and none has a portrait of their own, + # because a picture of somebody the text does not account for is another + # person. Otherwise the old fresh start stands. + _prev_people = _frame_cast(i - 1, last=True) if i else [] + # Into ANOTHER room, only people who are all in this shot: the ones left + # behind are not here to be claimed. In the same room everybody still in it + # is claimed, described or not. + _carry_ok = bool(i and handoff is not None and i not in reentry_shots + and (_cond_module.may_carry_room if i in cut_shots + else _cond_module.may_carry_frame)( + _prev_people, plan.shots[i].cast, _tagged_names)) + _carry_rooms = None if restart_after_removal and (i - 1) in stripped_shots: + if _carry_ok: + _handoff_ref = True + _carry_rooms = (shot_rooms.get(i - 1, ("", ""))[1], + shot_rooms.get(i, ("", ""))[0]) + _soft_cuts.append((i + 1, "removal")) + else: + shot_handoff = None + fresh.append(i + 1) + # ...and so does a shot that OPENS IN A DIFFERENT ROOM. Before the + # _placed_shots branch on purpose: that one DEMOTES the frame to a + # reference claiming "this room a moment earlier", which is a lie when the + # room has changed -- so this one claims the PEOPLE and names both rooms. + # See cut_shots. + elif i in cut_shots and _carry_ok: + _handoff_ref = True + _carry_rooms = (shot_rooms.get(i - 1, ("", ""))[1], + shot_rooms.get(i, ("", ""))[0]) + _soft_cuts.append((i + 1, "room")) + elif i in cut_shots or i in reentry_shots: shot_handoff = None - fresh.append(i + 1) + # SHOT 1'S first_frame, READ AS THE SET. Same answer as the branch below + # and for the same reason -- a keyframe is a picture, and the people the + # beat places are not in this one -- reached separately because shot 1 has + # no previous shot to ask about. See where _first_is_plate is decided. + elif i == 0 and _first_is_plate and shot_handoff is not None: + _handoff_ref = True + _plate_on = i + 1 # ...and so does a shot that INTRODUCES somebody already in position. # # Same reasoning, same evidence. The keyframe is the previous shot's last @@ -10328,14 +13472,17 @@ class H3LongVideos: # already there without being frame one, so the newcomer is simply in # place instead of walking in from nowhere. # - # Only when everybody in that frame is named in this shot. The picture - # contains whoever was on screen when it was taken, and one the prompt - # cannot account for is the node's oldest bug: a picture nobody claims is - # another person. When it cannot be claimed, the old fresh start stands. + # Only when everybody in that frame can be CLAIMED. The picture contains + # whoever was on screen when it was taken, and one the prompt cannot + # account for is the node's oldest bug: a picture nobody claims is another + # person. The claim names all of them -- described in this shot or not, + # they are still in this room, and the count says so too (see + # shot_frames). It used to demand that the beat name every one of them, + # so "Crystal reads by the window" after a shot of Dan dropped the frame: + # the room was re-imagined and Dan vanished from it. elif i in _placed_shots: - _was_here = [n for n in (shot_cast[i - 1] if i - 1 < len(shot_cast) - else []) if n] - _here_now = shot_cast[i] if i < len(shot_cast) else [] + _was_here = _frame_cast(i - 1, last=True) + _here_now = plan.shots[i].cast # ...and NOT when somebody in that frame already has a portrait of # their own in this shot. Their identity is carried by that # picture; the carried frame would be a SECOND picture of the same @@ -10347,8 +13494,7 @@ class H3LongVideos: # # The room is lost on those shots, back to the fresh start it was # before. A re-imagined set is a smaller bug than a second person. - if _was_here and all(n in _here_now for n in _was_here) \ - and not any(n in _tagged_names for n in _was_here): + if _cond_module.may_carry_frame(_was_here, _here_now, _tagged_names): _handoff_ref = True _carried.append((i + 1, list(_was_here), list(_placed_shots[i]))) @@ -10372,13 +13518,20 @@ class H3LongVideos: # travels into every shot they are named in, and a second picture of the # same person is just a second picture. _extra = [] - _cast = shot_cast[i] if i < len(shot_cast) else [] - if len(_cast) == 1 and _cast[0] not in _tagged_names: - _who = _cast[0] - if any(n == i + 1 and _who in ws for n, ws in _returns) \ - and _captured.get(_who) is not None: - _extra = [_captured[_who]] - _recovered.append((i + 1, _who, _captured_from.get(_who, 0))) + _cast = plan.shots[i].cast + _returning = {w for n, ws in _returns if n == i + 1 for w in ws} + # ...and only a frame of what they wear NOW. A face captured before + # anybody changed clothes is a picture of the old wardrobe, and a + # reference puts that back -- the same rule the room frames keep. + _who = _cond_module.recoverable_subject( + _cast, _tagged_names, _returning, + {k: v for k, v in _captured.items() + if _captured_gen.get(k) == _wardrobe_gen}) + if _who and _carry_rooms is not None and _who in _prev_people: + _who = "" # the carried frame is already a picture of them + if _who: + _extra = [_captured[_who]] + _recovered.append((i + 1, _who, _captured_from.get(_who, 0))) # CLAIM IT IN THE PROSE. A picture the prompt refers to is that # subject; one it never mentions is ANOTHER subject. Sent # unclaimed, a recovered frame of somebody is read as a second @@ -10388,19 +13541,101 @@ class H3LongVideos: # Its number is its place in the roster: the shot's own references # first, this after them. The handoff follows and stays unclaimed, # which is H3's own first-frame shape. - _n = len(shot_refs_all[i]) + 1 - _tag = f"" - if f"{_who}:" in shot_prompt: - shot_prompt = shot_prompt.replace( - f"{_who}:", f"{_who}: {_tag},", 1) - else: - shot_prompt = f"{shot_prompt} {_who} is the person in {_tag}." + _n = len(shot.refs) + 1 + _tag = f"" + if f"{_who}:" in shot_prompt: + shot_prompt = shot_prompt.replace( + f"{_who}:", f"{_who}: {_tag},", 1) + else: + shot_prompt = f"{shot_prompt} {_who} is the person in {_tag}." + # ONE PHOTOGRAPHED FACE AND TWO PEOPLE TO DRAW. + # + # A shot that carries a reference for one person and describes another who + # has none is the node's oldest unanswered duplicate: a reference is the + # strongest identity signal in a prompt -- far stronger than "35, dark + # hair" -- so the one that exists gets used for both bodies, and the second + # character arrives as a copy of the first. Reported as two of the same + # person in a scene written for two, and this file's own note on it said + # the node could not stop it: there is no sentence that outranks a photo. + # + # There is no sentence, but there is a PICTURE. The node has been keeping + # one all along -- a frame from a shot that held that person alone, at the + # wardrobe they are wearing now, the same frames a returning face is + # recovered from. Sending it evens the shot up: two people, two pictures, + # neither one the only face in the prompt. + # + # Narrow, for the same reasons the recovered face is: one person short of a + # picture (with two, which frame is whose becomes a guess), a frame that + # shows them ALONE, and nothing else already recovered for this shot. + elif _tagged_names and len(_cast) > 1 and any(n in _tagged_names for n in _cast): + _short = [n for n in _cast + if n and n not in _tagged_names + and _captured.get(n) is not None + and _captured_gen.get(n) == _wardrobe_gen] + if len(_short) == 1 and f"{_short[0]}:" in shot_prompt: + _extra = [_captured[_short[0]]] + _evened.append((i + 1, _short[0], _captured_from.get(_short[0], 0))) + _tag = f"" + shot_prompt = shot_prompt.replace( + f"{_short[0]}:", f"{_short[0]}: {_tag},", 1) # The handoff, when it is demoted to a reference, is a picture like any # other and has to be claimed or it reads as a second person. Decided # here rather than inside build_conditioning because the claim is text, # and the text is assembled up here. - _shot_refs = list(shot_refs_all[i]) + _extra - _ctx_refs = [] + # A ROOM THE FILM COMES BACK TO, WITH NO PICTURE OF IT. + # + # A cut to a room opens fresh, and a walk into one opens on the room being + # left -- either way nothing pictorial says what the room looked like the + # last time it was on screen, so the sentence rebuilds it and the rebuild is + # a different room: the living room on shot 3 is not the living room of + # shot 1. Reported as locations and interiors not staying the same. The node + # rendered that room already; its last frame there is the picture. + # + # Carried only when it cannot bring anything else back with it. A frame is + # a picture of everyone in it, so everybody in it has to be named in this + # shot (and none of them carry a portrait of their own -- a second picture + # of one person is how a second one gets drawn). And a frame taken before + # anybody changed clothes or hardware is a picture of the old wardrobe, + # which a reference would put back: any such change since retires it. + _opens, _ends = shot_rooms.get(i, ("", "")) + _prev_end = shot_rooms.get(i - 1, ("", ""))[1] if i else "" + _back, _arriving = "", False + if (i in cut_shots or i in reentry_shots) and _opens in _room_frames: + _back = _opens + elif _ends and _ends != _prev_end and _ends != _opens and _ends in _room_frames: + _back, _arriving = _ends, True + if _back: + _cast_now = set(plan.shots[i].cast) + # ...and, on a WALK in, nobody who is also in the keyframe. A walk keeps + # the keyframe -- the room being left, with whoever is leaving it -- so a + # frame of the arrival room with the same person in it is a second + # picture of her, which is how a second one gets drawn. A cut drops the + # keyframe, so there the frame is the only picture and needs no such test. + _in_keyframe = (set(_frame_cast(i - 1, last=True)) + if (_arriving and i and shot_handoff is not None) else set()) + for _frame, _in_it, _gen, _from in _room_frames[_back]: + # ...and nobody whose face was recovered for this shot above: that + # is already a picture of them, and this would be the second. + if (_gen == _wardrobe_gen + and all(n in _cast_now for n in _in_it) + and not any(n in _tagged_names for n in _in_it) + and not any(n in _in_keyframe for n in _in_it) + and not (_who and _who in _in_it) + # With the previous frame carried as a reference, only a + # room frame showing ALL of its people -- which then carries + # the room and them, and replaces it -- or none of them. + and not (_carry_rooms is not None + and any(n in _prev_people for n in _in_it) + and not all(n in _in_it for n in _prev_people))): + if _carry_rooms is not None and any(n in _prev_people for n in _in_it): + _handoff_ref, shot_handoff, _carry_rooms = False, None, None + _soft_cuts.pop() + _extra.append(_frame) + shot_prompt = shot_prompt + returning_room_claim( + len(shot.refs) + len(_extra), _back, _in_it, _arriving) + _room_returns.append((i + 1, _back, _from)) + break + _shot_refs = list(shot.refs) + _extra _keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG if (handoff_frames > 1 and shot_handoff is not None and handoff_context is not None and not _handoff_ref and _keyframe_ok): @@ -10410,11 +13645,25 @@ class H3LongVideos: except Exception: _ctx_refs = [] if _ctx_refs: - _first = len(_shot_refs) + 1 - _last = _first + len(_ctx_refs) - 1 - shot_prompt = shot_prompt + handoff_context_claim(_first, _last) + first = len(_shot_refs) + 1 _shot_refs.extend(_ctx_refs) - if _handoff_ref: + shot_prompt = shot_prompt + handoff_context_claim(first, len(_shot_refs)) + if _handoff_ref and _plate_on == i + 1: + # A SET, not a room a moment earlier. See plate_claim. + shot_prompt = shot_prompt + plate_claim(len(_shot_refs) + 1) + _handoff_claimed.append(i + 1) + elif _carry_rooms is not None: + # A cut that kept its frame as a reference. The same room is "this room + # a moment earlier"; another room claims the people and names both. + _was_room, _now_room = _carry_rooms + if _was_room and _now_room and _was_room != _now_room: + shot_prompt = shot_prompt + carried_people_claim( + len(_shot_refs) + 1, _prev_people, _was_room, _now_room) + else: + shot_prompt = shot_prompt + room_claim(len(_shot_refs) + 1, + _prev_people, []) + _handoff_claimed.append(i + 1) + elif _handoff_ref: # Carried for the ROOM, with somebody new in the shot -- so the # standing claim is exactly wrong here ("joined by anybody new") and # this one names the room, who was in it, and who is also here. @@ -10426,13 +13675,16 @@ class H3LongVideos: shot_prompt = shot_prompt + handoff_claim(len(_shot_refs) + 1) _handoff_claimed.append(i + 1) # Whatever this shot ends up being, that is what `script` reports. - sent_text[i] = shot_prompt + shot.prompt = shot_prompt cond, latent, fc, demoted = build_conditioning( - clip, vae, audio_vae, shot_prompt, w, h, lens[i], + clip, vae, audio_vae, shot_prompt, w, h, shot.frame_count, handoff=shot_handoff, refs=_shot_refs, ref_noise_aug=ref_noise_aug, silent=silent, - handoff_as_ref=_handoff_ref) - if demoted and not _aug_warned: + handoff_as_ref=_handoff_ref, + speech_lead_seconds=(_audio.lead_frames / AUDIO_LATENT_FPS), + speech_tail_frames=_audio.tail_frames) + if (demoted and not _aug_warned and ref_noise_aug is not None + and float(ref_noise_aug) < KEYFRAME_SAFE_AUG): _aug_warned = True notes.append( f"ref_noise_aug is {float(ref_noise_aug):g}, below {KEYFRAME_SAFE_AUG:g} -- " @@ -10445,13 +13697,14 @@ class H3LongVideos: try: _t0 = time.perf_counter() out = sample_shot(model, cond, negative, latent, seed, steps, cfg, - sampler_name, scheduler, sigmas) + sampler_name, scheduler, sigmas, + shift_video, shift_audio, _soft_landing) t_sample += time.perf_counter() - _t0 except (torch.cuda.OutOfMemoryError, RuntimeError) as e: if not _is_oom(e): raise raise RuntimeError( - f"H3 Long Videos: shot {i + 1} of {len(shots)} ran out of VRAM while " + f"H3-LongVideos: shot {i + 1} of {len(plan)} ran out of VRAM while " f"sampling. " + sampling_oom_help(w, h, fc, H3_FPS, megapixels)) from e # The video latent, for the latent upscale below. NOT used as the next @@ -10502,6 +13755,31 @@ class H3LongVideos: hand_src = tail except Exception: pass # fall back to the upscaled frames + # MEASURE FIRST, on the uncorrected frames. shot_handoff is the keyframe this + # shot was given and imgs[0] is what came back in its place -- two pictures of + # the same frame, so what separates them is the chain and not the author. The + # last two arguments put the pre-upscale handoff and the post-upscale output in + # one frame of reference; with latent_upscale off they are the same frame and + # the term is zero. A demoted handoff is skipped: it rode as a reference, so + # imgs[0] was never asked to reproduce it. + try: + if (shot_handoff is not None and not demoted and imgs is not None + and imgs.shape[0] > 1 and hand_src is not None and hand_src.shape[0]): + _levels.observe(shot_handoff, imgs[0], imgs[-1], hand_src[-1]) + except Exception: + pass + # Then correct, on a REBINDING -- imgs itself is untouched, so the frames the + # viewer sees are the ones the model made. Everything that leaves this shot for + # a later one comes off hand_src, so the handoff and any captured face take the + # same grade from the same call. + try: + if hold_levels > 0 and hand_src is not None and hand_src.shape[0]: + _lg, _lo = _levels.gains(hold_levels) + if _lg is not None: + hand_src = apply_levels(hand_src, _lg, _lo) + _levels.note(_lg, _lo) # recorded for the end-of-run report + except Exception: + pass # Clamp before it becomes a keyframe. A decode can land slightly outside # 0..1, and feeding that back in to be re-encoded every boundary is a # drift that accumulates rather than cancels. @@ -10553,20 +13831,63 @@ class H3LongVideos: or _n in bared_shots or _n in staging_shots) try: - if (hand_src.shape[0] and shot_cast and i < len(shot_cast) - and len(shot_cast[i]) == 1 and _wardrobe_normal): + # ONE PERSON IN THE FRAME, not in the text: a shot describing only + # Crystal while Dan sits beside her is a picture of both of them. + if (hand_src.shape[0] and len(plan.shots[i].cast) == 1 + and len(_frame_cast(i)) == 1 and _wardrobe_normal): _mid = hand_src.shape[0] // 2 _keep = hand_src[_mid:_mid + 1].detach().clamp(0.0, 1.0).to( "cpu", copy=True) - for _who in shot_cast[i]: + for _who in plan.shots[i].cast: _captured[_who] = _keep _captured_from[_who] = i + 1 + _captured_gen[_who] = _wardrobe_gen except Exception: pass # a recovered frame is a nicety, not the render + # The room this shot ENDS in, from its last frame -- the end, because a + # walk is in the room it arrives in by then. From an ordinary shot only, + # like the face above, and with hardware counted as wardrobe: a picture + # from before the cuffs went on would take them off again. + if not _wardrobe_normal or _n in hardware_changed: + _wardrobe_gen += 1 + else: + _room_end = shot_rooms.get(i, ("", ""))[1] + try: + if _room_end and hand_src.shape[0]: + # A few per room, newest first, so a walk back can find one + # without the person walking. Three bounds the memory. + _room_frames[_room_end] = ([( + hand_src[-1:].detach().clamp(0.0, 1.0).to("cpu", copy=True), + _frame_cast(i, last=True), _wardrobe_gen, i + 1)] + + _room_frames.get(_room_end, []))[:3] + except Exception: + pass # a carried room is a nicety, not the render del hand_src - if trim_seam and i > 0: + # TRIM ONLY WHERE THERE WAS A KEYFRAME TO DUPLICATE. + # + # The first frame of a shot is dropped because it is "the model's own + # reproduction of the keyframe, so it is a duplicate" -- and that is true + # only of a shot that OPENED on one. Three paths above leave a shot with + # no keyframe: restart_after_removal breaks the chain after a garment + # comes off, a character introduced already in position demotes the + # handoff to a reference, and the same case unclaimable drops it. On those + # shots the first frame is not a reproduction of anything -- it is the + # genuine opening frame of a deliberate cut -- and trimming it threw away + # real footage AND removed the one frame nearest the shot before it. + # Reported as the last frame and the first frame of the next beat not + # matching up, which is exactly what it looks like: the bridge frame is + # gone and what meets the cut is frame two. + # + # `demoted` is build_conditioning's own answer to "did the handoff ride as + # a reference instead of anchoring frame one", so this asks the question + # of the code that decided it rather than re-deriving the three cases and + # drifting from them. The audio trim moves with the video trim or the two + # come apart by a frame. + if trim_seam and i > 0 and shot_handoff is not None and not demoted: imgs = imgs[1:] wav["waveform"] = wav["waveform"][..., max(0, round(sr / H3_FPS)):] + elif trim_seam and i > 0: + _untrimmed.append(i + 1) # Make the sound exactly as long as the picture it belongs to. # # The audio latent count is round(frames / 24 * 40), which lands exactly @@ -10589,77 +13910,37 @@ class H3LongVideos: [wav["waveform"], torch.zeros(shape, dtype=wav["waveform"].dtype, device=wav["waveform"].device)], dim=-1) av_fix += have - want - # Measured on the frame that becomes the next shot's keyframe, because - # that is the one whose losses are inherited. + # Measured on the frame that becomes the next shot's keyframe, because that is + # the one whose losses are inherited -- which means the CORRECTED handoff, not + # imgs[-1]. Measured on imgs[-1] the line would report the defect for ever and + # never show whether the correction worked. try: - if imgs is not None and imgs.shape[0]: + if handoff is not None and handoff.shape[0]: + shot_detail.append(frame_detail(handoff[0])) + elif imgs is not None and imgs.shape[0]: shot_detail.append(frame_detail(imgs[-1])) except Exception: pass - # HALF PRECISION IN RAM. The finished shots are the largest thing this node - # holds, and they compete with the weights for system memory -- ComfyUI - # offloads models to RAM rather than discarding them, so a shot boundary is - # a PCIe copy only while that RAM is there. Once the frames crowd the - # weights out, the "reload" becomes a disk read, and on a chain that is - # once per shot per model. - # - # A 107s chain at 1056x608 is ~2580 frames, 18.5GB as float32 and 9.3GB as - # float16, against ~39GB of weights on a 64GB machine. That 9GB is the - # difference between the weights staying resident and not. - # - # Free, not a trade: fp16 carries ~3 decimal digits over 0..1, and the - # output is 8-bit. Converted back at the join, so nothing downstream sees - # a different dtype. - # STRAIGHT INTO THE FINISHED CHAIN, not into a list to be joined later. - # - # The per-shot list existed because the total length was not known until - # the loop ended -- and it IS known: plan_lengths fixed `lens` before the - # first shot sampled, every entry is on H3's 17k+5 grid, and trim_seam - # only ever REMOVES a frame, so sum(lens) is a hard upper bound. With the - # destination allocated up front each shot is written where it belongs - # and the join has nothing left to do. - # - # That deletes the last double-hold in the node. Even after the join was - # rewritten to drain the list, both were still fully live at the moment - # it started: 9.26GB of destination beside 9.26GB of pieces, 18.51GB of - # chain on top of 44.64GB of staged weights, which is where the render - # was being killed. Now the chain is one copy from first shot to return. - # - # It also drops the copy=True. That was duplicating a whole shot (1.30GB) - # purely to detach it from the decode buffer; copy_ into the destination - # detaches it just the same, and converts device and dtype on the way, so - # one copy does what two did. - # - # ON AN fp32 INSTALL THIS TRADES SUSTAINED FOR PEAK, deliberately. The - # list was fp16 while the render ran and widened only at the join, so a - # 107s chain sat at 9.26GB and spiked to 27.77GB; the destination is the - # OUTPUT dtype throughout, so it sits at 18.51GB and never spikes. Peak - # is what the OOM killer reads, and on an --fp16-intermediates install -- - # where the output dtype is fp16 anyway -- both numbers improve. - # - # The overflow branch is not reachable on the real VAE, which decodes a - # shot to exactly the length it was planned at. It exists because "not - # reachable" is a claim about somebody else's code, and a wrong frame - # count should cost a slower path, not a crash. - _k = int(imgs.shape[0]) - if _dst is None and _k: - _dst = torch.empty( - (max(_k, int(sum(lens))),) + tuple(imgs.shape[1:]), - dtype=_image_out_dtype(), - device=(torch.device("cpu") if cleanup_between_shots - else imgs.device)) - if _dst is not None and _at + _k <= _dst.shape[0]: - _dst[_at:_at + _k].copy_(imgs) - _at += _k - else: - vid_out.append(imgs.to("cpu", torch.float16, copy=True) - if cleanup_between_shots else imgs) + frames.add(imgs) aud_out.append(wav["waveform"].to("cpu", copy=True) if cleanup_between_shots else wav["waveform"]) del imgs, wav if cleanup_between_shots: _deep_cleanup() + if _untrimmed: + notes.append( + f"shot(s) {', '.join(str(n) for n in _untrimmed)} kept their FIRST frame " + f"even though trim_seam is on, because they did not open on a keyframe. " + f"The trim exists to drop a duplicate -- the model's own reproduction of " + f"the frame it was handed -- and these shots were handed none: the chain " + f"is broken deliberately after a removal, and a character introduced " + f"already in position gets the previous frame as a REFERENCE rather than " + f"as frame one. Their first frame is the real opening frame of a cut, so " + f"trimming it threw away footage and left frame TWO meeting the shot " + f"before -- reported as the last frame and the first frame of the next " + f"beat not matching up. The audio is trimmed with the picture or not at " + f"all, so the two cannot come apart") if _handoff_claimed: notes.append( f"ref_noise_aug is below {KEYFRAME_SAFE_AUG:g}, so on shot(s) " @@ -10672,6 +13953,16 @@ class H3LongVideos: f"appearing on the later shots because those are the ones with both a " f"handoff and a reference. Raising ref_noise_aug to {KEYFRAME_SAFE_AUG:g} " f"or above keeps the handoff a keyframe and the question does not arise") + if _room_returns: + notes.append( + "carried a room back: " + + "; ".join(f"the {room} on shot {n}, from shot {src}" + for n, room, src in _room_returns) + + ". The film returns to a room it already showed, and without a picture " + "the words rebuild it as a different room. The last frame from the shot " + "that was last there went in as a reference, claimed as that room. Only " + "where everybody in that frame is in the shot and nobody's clothes or " + "hardware have changed since, or the picture would carry the old ones back") if _recovered: notes.append( "recovered a face for " @@ -10691,51 +13982,10 @@ class H3LongVideos: "prompt never refers to is read as another subject, so an " "unclaimed one would arrive as a second person with the same " "face and the same clothes. `script` is written before the render, so it does not show that tag") - # JOIN WITHOUT HOLDING THE CHAIN TWICE. torch.cat allocates the whole chain - # a second time and .float() a third -- at fp32, so double again -- while the - # per-shot fp16 pieces the loop spent a copy each to make are still sitting in - # vid_out. On the 107s chain costed above that peaks at 9.3 + 9.3 + 18.5 = - # 37GB, and vid_out was never dropped afterwards, so 27.8GB stayed held for - # the rest of the run. The fp16 saving above was being spent here twice over. - # - # Allocate the fp32 output once and fill it shot by shot, releasing each piece - # as it lands: the peak is the output plus whatever is left of vid_out, and - # the pieces are gone by the end. Same tensor, same dtype, same device, same - # contract downstream. Measured on an 8-shot chain: 20.47GB peak -> 11.11GB, - # and at the 2580-frame size costed above, 37.9GB -> 20.6GB. That is 17GB off - # the peak (three copies became one) and 9.3GB no longer held afterwards. - # device= matters: with cleanup_between_shots off the pieces are still on the - # GPU and cat/float would have returned a GPU tensor, so this must too. - # THERE IS NO JOIN LEFT. Every shot was written into _dst as it was decoded, - # so the chain is already assembled and this is a view onto it -- zero new - # bytes at the moment that used to be the peak of the whole render. - # - # _at is short of the capacity by exactly one frame per seam that trim_seam - # removed, so the slice keeps a few frames of slack allocated rather than - # copying the chain to reclaim them: shots-1 frames against a copy of the - # whole thing is not a trade worth making. - if vid_out: - # OVERFLOW ONLY -- a VAE that decoded a shot longer than it was planned - # at. Assemble both halves the old way, which costs the extra copy this - # rewrite exists to remove, on a path the real VAE never takes. - _extra = sum(int(_t.shape[0]) for _t in vid_out) - _ref = _dst if _dst is not None else vid_out[0] - video = torch.empty((_at + _extra,) + tuple(_ref.shape[1:]), - dtype=_image_out_dtype(), device=_ref.device) - if _dst is not None and _at: - video[:_at].copy_(_dst[:_at]) - _dst = None - _w = _at - while vid_out: - _piece = vid_out.pop(0) - _k2 = int(_piece.shape[0]) - video[_w:_w + _k2].copy_(_piece) - _w += _k2 - del _piece - elif _dst is not None: - video = _dst if _at == _dst.shape[0] else _dst[:_at] - else: - video = torch.cat(vid_out, dim=0) # empty: fail exactly as before + # FrameAccumulator writes each decoded shot directly into the finished chain. + # Its overflow path covers malformed VAE output without making the normal + # path allocate and concatenate a second full copy. + video = frames.finish() # PIXEL upscale, once, on the finished chain. After the latent pass and after # the join, so a model-based upscaler sees whole frames and the seam is not # upscaled twice. @@ -10762,125 +14012,52 @@ class H3LongVideos: audio = torch.cat(aud_out, dim=-1) if audio.dtype != torch.float32: audio = audio.float() - # ...and the ambient bed goes on last, over the joined soundtrack rather than - # per shot, so the loop runs continuously through the cuts instead of + # ...and a wired ambient file goes on last, over the joined soundtrack rather + # than per shot, so it runs continuously through the cuts instead of # restarting at each one. A bed that resets every shot is a bed you can hear. - # THE BED IS BUILT, not fetched, unless something is wired to ambient_audio. - # The node has already read what the room sounds like off the scene -- that - # is what auto_sound puts in the prompt -- so the same phrase can be turned - # into the sound itself. No file, no second model pass, and shaped noise is - # the one source of ambience that physically cannot produce a voice. - _bed_in, _built = ambient_audio, "" + # + # A WIRED FILE IS ALL THAT GOES ON NOW. The node used to BUILD this, out of + # the scene's own wording, and build the shot-by-shot foley too -- see the + # note at the top of audio.py for what that was and why it is gone. Removed + # on the report: "Just get rid of the ambient sounds all together. They sound + # horrid. Go back to the model's natural audio." + # + # What is played here is the user's own recording, which is a different thing + # from ambience the node invents: it is not synthesised, it conditions + # nothing, and it cannot put a voice in a wordless shot. + _bed_in = ambient_audio if _bed_in is None and float(ambient_level or 0.0) > 0.0: - _phrase = " ".join(p for p in (_mix_bed, _mix_room) if p) - _synth = synth_ambient(_phrase, int(audio.shape[-1]), int(sr), - seed=seed, channels=int(audio.shape[1])) - _fell_back = False - if _synth is None: - # A shaped bed that will not build falls back to a plain one rather - # than to nothing. Wiring a file is NOT the remedy: the built bed is - # the feature and a file is only ever an override, so the floor has - # to be here. - _synth, _fell_back = plain_bed(int(audio.shape[-1]), int(sr), seed, - int(audio.shape[1])), True - if _synth is None: - # SAID, not swallowed. Both builders are defensive so a render never - # dies for a bed, and that would otherwise turn a failure into an - # output with no ambience and nothing anywhere saying why -- the - # exact hole _SILENCE_STATUS exists to close on the other branch. - notes.append( - f"AMBIENT LEVEL IS {float(ambient_level):.2f} BUT NO BED WENT ON. " - f"Both the shaped bed and the plain fallback failed to build, " - f"which should not be possible on a soundtrack this node just " - f"produced -- please report it") - else: - _bed_in = {"waveform": _synth.unsqueeze(0), "sample_rate": int(sr)} - _built = (f"built from the scene, not a file: \"{_phrase}\". " - if _phrase else "built as a neutral room tone. ") - if _fell_back: - _built += ("The SHAPED bed would not build, so this is the plain " - "fallback -- a rumble rather than the acoustic the " - "scene describes. Worth reporting: it should not " - "happen. ") - # Said plainly rather than left to disappoint: this shapes TONE. - if any(w in _phrase for w in _BED_EVENTFUL): - _built += ("That description names EVENTS, and this builds tone " - "-- so what went under is the room those things are " - "in, not the things themselves. Wire a recording to " - "ambient_audio if you want the events. ") - # FOLEY, into each shot's own span. Only shots pinned to SILENCE: an open - # branch is already making its own sound from the same prose, and building - # over that would double every footfall. These are the shots that had - # nothing -- a wordless beat staging cuffs going on, silent because opening - # its branch is what babbles. - _foley_on = [] - if auto_sound and float(foley_level or 0.0) > 0.0 and shot_events: - _at = 0 - for _i, _w in enumerate(aud_out): - _len = int(_w.shape[-1]) - _lo, _hi, _at = _at, _at + _len, _at + _len - if _i >= len(shot_events) or _i >= len(speech): - continue - _pinned = bool(silence_nonspeech and not speech[_i] - and not (sounded[_i] if _i < len(sounded) else False)) - # ...OR open only because the beat stages EFFORT. The skip above - # exists so built sound does not double what an open branch is - # already making out of the same prose. That is true when the - # AUTHOR wrote the sound, and false for effort: THAT branch opened - # to make a voice, and a voice is not a bed frame, a chain or a - # cuff. Every recipe here is non-vocal by construction, so on such - # a shot the vocal phrase simply has no recipe and drops out on its - # own -- what is left is exactly the half the model will not make. - # - # Missing this undid the recipes in the same commit that added - # them: the nine effort verbs became _voiced, _voiced unpins the - # shot, and an unpinned shot skips the mix. So "a bed frame - # working" and "restraints pulling taut" were read from the beat, - # written into the prompt, and then never built -- on precisely the - # beats they exist for. Reported as hearing nothing. - # - # Gated on silence_nonspeech with everything else. Turning that off - # says "pin nothing, let the model sound every shot" -- and then - # there is no shot the model cannot make, which is the entire reason - # anything is built here. Without this the effort shots kept their - # built layer while the model was also sounding them from the same - # prose, which is the doubling this whole gate exists to avoid. - _voice_open = bool(silence_nonspeech and _i < len(voiced_only) - and voiced_only[_i]) - if not (_pinned or _voice_open) or _len < 64: - continue - _made = [] - for _ph in shot_events[_i]: - _fx = foley_for(_ph, _len, int(sr), seed=int(seed) + _i) - if _fx is None: - continue - audio[..., _lo:_hi] = (audio[..., _lo:_hi] - + _fx.to(audio.dtype).unsqueeze(0) - * float(foley_level)) - _made.append(_ph) - if _made: - _foley_on.append((_i + 1, _made, _voice_open)) - if _foley_on: - _eff = [n for n, _, v in _foley_on if v] notes.append( - "sound built into the shot itself on " - + "; ".join(f"shot {n}: {', '.join(m)}" for n, m, _v in _foley_on) - + (f". Shot(s) {', '.join(str(n) for n in _eff)} stage effort, so " - f"their branch IS open and the model is making the voice -- what " - f"is built there is only the non-vocal half it will not make, the " - f"frame and the metal. Lower foley_level if it doubles anything. " - if _eff else ". ") - + "The rest have no line, so their audio branch is pinned to " - "silence and the model cannot make these -- auto_sound puts them in " - "the prompt, and prompt text can never open a branch, so the cue was " - "being dropped on exactly the shots whose point is a sound. Built and " - "mixed instead, which asks nothing of the model and so cannot babble. " - "It is synthesis, not a recording: it reads as a click, a rattle, a " - "rustle, in the right place. Nothing vocal is ever built. " - "foley_level sets how loud, 0 turns it off") + f"ambient_level is {float(ambient_level):.2f} and nothing is wired to " + f"ambient_audio, so no bed went under the soundtrack -- and that is " + f"now the only way to get one. The node used to BUILD a room tone out " + f"of the scene's wording, and a layer of foley into every shot pinned " + f"to silence; both are gone, because they were reported as sounding " + f"horrid and synthesis that measures right and sounds wrong is the end " + f"of that road. The audio is the model's, whole. This widget still " + f"sets the level for a recording you wire yourself, which is played " + f"under the finished track and conditions nothing") + if auto_sound and float(foley_level or 0.0) > 0.0: + notes.append( + f"foley_level is {float(foley_level):.2f} and does nothing any more. " + f"It set how loud the sounds this node BUILT were -- a click, a " + f"rattle, a rustle, mixed into the shots whose audio branch is pinned " + f"to silence, because prompt text can never open a branch and those " + f"shots could not make their own. That is removed: the soundtrack is " + f"the model's. The widget stays at this position because saved " + f"workflows restore values by position and shifting it would load the " + f"wrong number into every widget after it. " + f"THE CONSEQUENCE, said rather than left to be found: a shot with no " + f"line and no sound you described is pinned to silence and is SILENT. " + f"The pin is deliberate and untouched -- it is what stops a free " + f"branch filling itself with a voice and the face lip-syncing to the " + f"babble. To put sound in such a shot, write the sound into that beat, " + f"which opens its branch on purpose and lets the model make it; or " + f"wire a track to ambient_audio; or lay one under the finished video " + f"outside the node") audio, _bed_note = mix_ambient(audio, sr, _bed_in, ambient_level) if _bed_note: - notes.append(_built + _bed_note if _built else _bed_note) + notes.append(_bed_note) total = video.shape[0] # The finished chain is the largest thing this node holds, and it competes with # the MODELS for system RAM: ComfyUI offloads weights to RAM rather than @@ -10907,12 +14084,37 @@ class H3LongVideos: f"and join the parts outside the node), a lower megapixels, or a " f"smaller diffusion quant -- every GB of weights is a GB not " f"available to hold the render") + if _evened: + notes.append( + "; ".join(f"shot {n} gave {who} a face of their own, from shot {src}" + for n, who, src in _evened) + + " -- each of those shots carried a reference for somebody else and " + "described them with none, which is one photographed face and two " + "people to draw. A reference is the strongest identity signal in a " + "prompt, so the one that exists gets used for both bodies and the " + "second character arrives as a copy of the first. The frame sent is " + "one this run rendered, from a shot that held them alone in the " + "clothes they are wearing now, and it is claimed on their own sheet " + "entry. Tagging them with a of their own does the same " + "thing from the first shot instead of the second" + ) + if _soft_cuts: + notes.append( + "carried the previous frame as a REFERENCE across a cut -- " + + "; ".join(f"shot {n} ({'something came off in the shot before' if why == 'removal' else 'it opens in another room'})" + for n, why in _soft_cuts) + + ". Not as frame one, so a garment left half off is not pinned into the " + "opening and the old room is not blended into a new one, but the faces, " + "hair and clothes come with it instead of being re-imagined from the text") if fresh: notes.append( f"shot(s) {', '.join(str(n) for n in fresh)} start fresh, because the shot " - f"before each took something off -- continuing from a frame that may still " - f"show the garment is how it comes back, and a picture outvotes the text. " - f"That costs a cut there. Turn restart_after_removal off to keep the " + f"before each took something off and its last frame could not ride as a " + f"reference -- nobody is left in it to claim, or somebody in it has a " + f"portrait of their own riding the next shot. Continuing from a frame that " + f"may still show the garment is how " + f"it comes back, and a picture outvotes the text. That costs a cut there, " + f"with nothing carried. Turn restart_after_removal off to keep the " f"continuity instead") if _carried: notes.append( @@ -10926,7 +14128,7 @@ class H3LongVideos: + " -- a keyframe is frame one and a reference is not, which is what " "lets a shot introduce somebody without re-imagining the room") wall = time.perf_counter() - t_start - n = max(1, len(shots)) + n = max(1, len(plan)) other = max(0.0, wall - t_sample - t_decode) notes.append( f"rendered {total} frames (~{total / H3_FPS:.1f}s) in {wall:.0f}s -- " @@ -10935,10 +14137,10 @@ class H3LongVideos: f"other {other:.0f}s ({100 * other / wall:.0f}%); " f"per shot {t_sample / n:.1f}s + {t_decode / n:.1f}s") if av_fix: - per_shot = abs(av_fix) / sr * 1000 / max(1, len(shots)) + per_shot = abs(av_fix) / sr * 1000 / max(1, len(plan)) notes.append( f"audio realigned to the picture by ~{abs(av_fix) / sr * 1000:.0f} ms " - f"across {len(shots)} shot(s), {per_shot:.1f} ms each. H3's audio latent " + f"across {len(plan)} shot(s), {per_shot:.1f} ms each. H3's audio latent " f"runs at {AUDIO_LATENT_FPS}/s against {H3_FPS} fps video, so a shot's " f"sound lands exactly only when its frame count divides by 3 -- otherwise " f"it is up to 8.3 ms out, with the same sign every time when the shots " @@ -10950,12 +14152,15 @@ class H3LongVideos: _detail = detail_report(shot_detail) if _detail: notes.append(_detail) + _lvl = levels_report(_levels, len(plan.shots)) + if _lvl: + notes.append(_lvl) if t_decode > t_sample: notes.append("decode is costing more than sampling here -- latent_upscale " "trades cheaper sampling for a 4x more expensive decode, so it " "is the wrong way round at this step count. megapixels is the " "lever that lowers both") - script = "\n---\n".join(f"[Shot {i}] {s}" for i, s in enumerate(sent_text, 1)) + script = "\n---\n".join(f"[Shot {i}] {s}" for i, s in enumerate(plan.prompts, 1)) # Whether the silence conditioning ACTUALLY went on. Reported from the # result, not from the flag: every failure inside _silent_audio_latent # returns None on purpose so a render never dies for a nicety, but that @@ -10963,13 +14168,13 @@ class H3LongVideos: # on real silence" -- and a shot with no scripted line babbled with nothing # in the report saying why. This is the one note that has to come after the # loop, because before it there is no result to report. - if silence_nonspeech and _SILENCE_STATUS["asked"]: + if _SILENCE_STATUS["asked"]: _missed = _SILENCE_STATUS["asked"] - _SILENCE_STATUS["applied"] if _missed > 0: notes.append( f"SILENCE WAS ASKED FOR ON {_SILENCE_STATUS['asked']} shot(s) AND " f"WENT ON {_SILENCE_STATUS['applied']}: {_missed} shot(s) have no " - f"line and an audio branch that is NOT pinned, because " + f"working audio lock, because " f"{_SILENCE_STATUS['why'] or 'the silent latent could not be built'}" f". H3 is joint, so an unconditioned branch invents a voice and the " f"picture lip-syncs to it -- a shot babbling with nothing scripted " @@ -10978,12 +14183,14 @@ class H3LongVideos: else: notes.append( f"silence went on all {_SILENCE_STATUS['applied']} shot(s) that " - f"asked for it -- their audio branch is pinned to encoded silence, " - f"not merely told to be quiet") + f"asked for it -- the requested full shot or dialogue lead-in is " + f"pinned to encoded silence, not merely told to be quiet") return (video, {"waveform": audio, "sample_rate": sr}, " | ".join(notes), script, - lens[0], total, len(shots), round(total / H3_FPS, 2)) + plan.shots[0].frame_count, total, len(plan), round(total / H3_FPS, 2)) -NODE_CLASS_MAPPINGS = {"H3LongVideos": H3LongVideos} -NODE_DISPLAY_NAME_MAPPINGS = {"H3LongVideos": "H3 Long Videos"} +_NODE_IDS = ("H3LongVideos", "H3LongVideosFL2VA", "H3LongVideosV1", + "H3LongVideosREF2VA") +NODE_CLASS_MAPPINGS = {name: H3LongVideos for name in _NODE_IDS} +NODE_DISPLAY_NAME_MAPPINGS = {name: "H3-LongVideos" for name in _NODE_IDS} __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/tests/test_dumas_h3_longvideos.py b/tests/test_dumas_h3_longvideos.py index 8e128e4..99ae35a 100644 --- a/tests/test_dumas_h3_longvideos.py +++ b/tests/test_dumas_h3_longvideos.py @@ -23,6 +23,16 @@ class DumasH3LongVideosUpstreamWrapperTests(unittest.TestCase): "folder_paths", "dumas_h3_longvideos", "dumas_h3_longvideos_upstream", + "dumas_h3_longvideos_engine", + "dumas_h3_longvideos_shot_plan", + "dumas_h3_longvideos_runtime", + "dumas_h3_longvideos_audio", + "dumas_h3_longvideos_conditioning", + "h3_engine", + "h3_shot_plan", + "h3_runtime", + "h3_audio", + "h3_conditioning", ) } @@ -112,6 +122,9 @@ class DumasH3LongVideosUpstreamWrapperTests(unittest.TestCase): self.assertIn("first_frame", schema["optional"]) self.assertIn("ref_image_1", schema["optional"]) self.assertIn("latent_upscale", schema["optional"]) + self.assertIn("speech_tail_seconds", schema["optional"]) + self.assertIn("hold_camera", schema["optional"]) + self.assertIn("verbatim", schema["optional"]) self.assertIn("handoff_frames", schema["optional"]) self.assertEqual(schema["optional"]["handoff_frames"][1]["default"], 1) self.assertEqual(node_cls.RETURN_NAMES[0:4], ("images", "audio", "info", "script"))