311 lines
16 KiB
Python
311 lines
16 KiB
Python
# H3-LongVideos -- https://github.com/Smite79/MiniMax-H3-LongVideos
|
|
# Copyright (c) 2026 Smite79. All rights reserved.
|
|
# Redistribution, in whole or in part, requires written permission.
|
|
# This notice may not be removed or altered. See LICENSE.
|
|
"""Decisions about which pictures may condition a shot."""
|
|
|
|
import torch
|
|
import node_helpers
|
|
from h3_runtime import (H3_FPS, AUDIO_LATENT_FPS, _empty_av_latent, _resize, ref_image_canvas,
|
|
frame_levels)
|
|
from h3_audio import _SILENCE_STATUS, _silent_audio_latent, _pin_audio_silence
|
|
|
|
|
|
def may_carry_room(previous_cast, current_cast, tagged_names):
|
|
"""A previous frame is safe as a reference only when it adds no subject."""
|
|
previous = [name for name in (previous_cast or ()) if name]
|
|
current = set(current_cast or ())
|
|
tagged = set(tagged_names or ())
|
|
return bool(previous) and all(name in current for name in previous) \
|
|
and not any(name in tagged for name in previous)
|
|
|
|
|
|
def may_carry_frame(previous_cast, current_cast, tagged_names):
|
|
"""A previous frame of the SAME room is safe as a reference claimed with everyone in it.
|
|
|
|
Unlike may_carry_room, somebody this shot does not describe may be in it: the claim
|
|
names them, and they are still in that room. Refused only for an empty frame, or
|
|
one holding somebody whose own portrait also rides this shot -- two pictures of one
|
|
person is how a second one gets drawn."""
|
|
previous = [name for name in (previous_cast or ()) if name]
|
|
current = set(current_cast or ())
|
|
tagged = set(tagged_names or ())
|
|
return bool(previous) and not any(name in tagged and name in current
|
|
for name in previous)
|
|
|
|
|
|
def recoverable_subject(cast, tagged_names, returning_names, captured):
|
|
"""Return the sole safe recovered subject, or an empty string."""
|
|
people = [name for name in (cast or ()) if name]
|
|
if len(people) != 1:
|
|
return ""
|
|
name = people[0]
|
|
return name if (name not in set(tagged_names or ())
|
|
and name in set(returning_names or ())
|
|
and captured.get(name) is not None) else ""
|
|
|
|
|
|
KEYFRAME_SAFE_AUG = 0.99 # below this, a ref aug would soften the keyframe too
|
|
|
|
# What ONE boundary is allowed to claim it measured. Wider than any real per-pass drift,
|
|
# narrow enough that a bad frame -- a flash, a cut to black, a frame the model lost --
|
|
# cannot swing the estimate. The median across boundaries does the real rejecting.
|
|
LEVEL_GAIN_CAP = 0.12 # in log-gain, so +-12.7% of contrast
|
|
LEVEL_OFFSET_CAP = 0.05
|
|
# The within-shot term is believed only when boundaries AGREE on its sign, and even then
|
|
# only this far: within-shot change is often the author's (a light switched off), so it is
|
|
# the half of the signal that cannot be trusted on its own.
|
|
LEVEL_SHOT_GAIN_CAP = 0.015
|
|
LEVEL_SHOT_OFFSET_CAP = 0.010
|
|
LEVEL_AGREE = 2.0 / 3.0
|
|
LEVEL_MIN_OBS = 3
|
|
# What the correction may do to one handoff, whatever it measured. A cut should not carry
|
|
# a visible grade step: shot N's last frame reaches the video uncorrected while N+1 is
|
|
# sampled from a corrected keyframe, so an uncapped correction trades burn-in for a pop at
|
|
# every join -- the same class of complaint, differently shaped.
|
|
LEVEL_GAIN_LO, LEVEL_GAIN_HI = 0.80, 1.25
|
|
LEVEL_OFFSET_BOUND = 0.02
|
|
# Below this a frame is too flat for a contrast RATIO to mean anything.
|
|
LEVEL_MIN_SIGMA = 0.01
|
|
|
|
|
|
class HandoffLevels:
|
|
"""Takes the grade the chain adds to itself back out of the handoff.
|
|
|
|
THE MEASUREMENT, which is the whole reason this needs no scene list. At every
|
|
boundary the render holds two pictures that are SUPPOSED to be the same frame: K,
|
|
the handoff it gave the shot, and R, frame one of what came back -- the model's own
|
|
reproduction of K, from a keyframe labelled sigma 0.001. Nothing was asked to change
|
|
between them, so everything separating them is the chain's own doing and none of it
|
|
is the author's intent. That is the one difference in the loop that can be corrected
|
|
without guessing at anybody's lighting, and R costs nothing to look at: it is the
|
|
frame trim_seam throws away.
|
|
|
|
A beat that walks into a darker room moves K, and R follows it there. So the level is
|
|
never anchored, never compared to shot 1, and never compared to a target -- only K
|
|
against its own reproduction, boundary by boundary.
|
|
|
|
WHAT IT WILL NOT FIX. Clipping already baked into earlier shots, because the VAE
|
|
clamps every decode and headroom spent is gone. Softening, which is a different
|
|
measurement and a different cause. Anything spatial -- ghosting, local burn, identity
|
|
drift. A tone curve with a knee in it, since this is affine per channel; the residual
|
|
in the report is how that would show itself. The first boundary, which has nothing to
|
|
measure yet. And a deliberate monotone move -- a film that dims every single beat --
|
|
loses a bounded, reported fraction of itself."""
|
|
|
|
def __init__(self):
|
|
self._bg, self._bo = [], [] # per boundary: K -> R, the chain's own drift
|
|
self._sg, self._so = [], [] # per shot: R -> last frame, believed only on agreement
|
|
self.applied = [] # (gain, offset) actually used, for the report
|
|
|
|
def observe(self, given, repro, last=None, pre_up_last=None):
|
|
"""Record one boundary. given is the keyframe this shot got, repro is frame one
|
|
of what it produced, last is its final frame, pre_up_last the handoff it hands on.
|
|
|
|
last/pre_up_last are how the pre-upscale handoff and the post-upscale output are
|
|
put in the same frame of reference: their difference IS the pipeline's own offset,
|
|
measured on one frame that went through both, so it can be subtracted from the
|
|
K->R reading instead of being mistaken for drift. With latent_upscale off they are
|
|
the same frame and the term is zero."""
|
|
gm, gs = frame_levels(given)
|
|
rm, rs = frame_levels(repro)
|
|
if gm is None or rm is None:
|
|
return False
|
|
if float(gs.min()) < LEVEL_MIN_SIGMA or float(rs.min()) < LEVEL_MIN_SIGMA:
|
|
return False
|
|
ug = torch.zeros(3)
|
|
uo = torch.zeros(3)
|
|
lm, ls = frame_levels(last) if last is not None else (None, None)
|
|
if pre_up_last is not None and lm is not None:
|
|
pm, ps = frame_levels(pre_up_last)
|
|
if pm is not None and float(ps.min()) >= LEVEL_MIN_SIGMA:
|
|
ug = torch.log(ls / ps)
|
|
uo = lm - pm
|
|
self._bg.append((torch.log(rs / gs) - ug).clamp(-LEVEL_GAIN_CAP, LEVEL_GAIN_CAP))
|
|
self._bo.append((rm - gm - uo).clamp(-LEVEL_OFFSET_CAP, LEVEL_OFFSET_CAP))
|
|
if lm is not None and float(ls.min()) >= LEVEL_MIN_SIGMA:
|
|
self._sg.append(torch.log(ls / rs))
|
|
self._so.append(lm - rm)
|
|
return True
|
|
|
|
def _agreed(self, rows, cap):
|
|
"""The median of rows, but only per channel where at least LEVEL_AGREE of them
|
|
share its sign. A within-shot change the boundaries disagree about is content, not
|
|
drift, and content must not be corrected."""
|
|
out = torch.zeros(3)
|
|
if len(rows) < LEVEL_MIN_OBS:
|
|
return out
|
|
st = torch.stack(rows)
|
|
med = st.median(dim=0).values
|
|
agree = ((st * med.sign().unsqueeze(0)) > 0).float().mean(dim=0)
|
|
keep = agree >= LEVEL_AGREE
|
|
return torch.where(keep, med.clamp(-cap, cap), out)
|
|
|
|
def estimate(self):
|
|
"""(gain_log, offset) the chain is drifting by per boundary, per channel."""
|
|
if not self._bg:
|
|
return None, None
|
|
g = torch.stack(self._bg).median(dim=0).values + self._agreed(self._sg, LEVEL_SHOT_GAIN_CAP)
|
|
o = torch.stack(self._bo).median(dim=0).values + self._agreed(self._so, LEVEL_SHOT_OFFSET_CAP)
|
|
return g, o
|
|
|
|
def gains(self, strength):
|
|
"""(gain, offset) as 3-vectors, or (None, None) when there is nothing worth doing.
|
|
|
|
Separate from note() because more than one frame leaves a shot -- the handoff,
|
|
and any face captured for a return several shots later -- and they have to carry
|
|
the SAME grade. A recovered face arriving at a different exposure from the shot
|
|
around it would be a new bug of exactly the kind this is fixing."""
|
|
g, o = self.estimate()
|
|
if g is None or strength <= 0:
|
|
return None, None
|
|
gain = torch.exp(-float(strength) * g).clamp(LEVEL_GAIN_LO, LEVEL_GAIN_HI)
|
|
off = (-float(strength) * o).clamp(-LEVEL_OFFSET_BOUND, LEVEL_OFFSET_BOUND)
|
|
# The next thing this frame meets is an 8-bit quantisation, so a correction under
|
|
# 1/255 would be erased on the way there. Claiming it would be worse than silence.
|
|
if float((gain - 1.0).abs().max()) < 1e-3 and float(off.abs().max()) < 1.0 / 255.0:
|
|
return None, None
|
|
return gain, off
|
|
|
|
def note(self, gain, off):
|
|
"""Record what was applied, and say it in one clause."""
|
|
self.applied.append((gain.clone(), off.clone()))
|
|
return (f"gain {'/'.join(f'{float(v):.3f}' for v in gain)} "
|
|
f"level {'/'.join(f'{float(v):+.4f}' for v in off)}")
|
|
|
|
def _keyframe_latent(vae, hand_img):
|
|
"""The keyframe latent for this shot: an ENCODE of the previous shot's last frame.
|
|
|
|
This was briefly an optimisation -- pass the previous shot's own latent straight
|
|
through and skip a VAE round trip per boundary. It was wrong, and it degraded
|
|
every shot after the first.
|
|
|
|
A keyframe is ONE pixel frame, and H3's grid puts that at 5f -> TWO latent
|
|
frames. Slicing [:, :, -1:] off a finished shot hands over one. Worse, the video
|
|
VAE is causal: the last latent of a 72-frame sequence encodes its temporal
|
|
context, not a standalone opening frame, so even at the right count it does not
|
|
mean what a keyframe means. The spatial-size guard could not see either problem.
|
|
|
|
The round trip is real but it is one lossy step on a correctly formed anchor,
|
|
which beats a cheap malformed one."""
|
|
return vae.encode(hand_img)
|
|
|
|
|
|
def _build_ref_images(vae, images, gen_w, gen_h, mode="match"):
|
|
"""(tokenizer items, DiT blocks) for a list of reference IMAGE tensors.
|
|
|
|
The tokenizer labels each one `<Picture N>:` itself, in the order given here --
|
|
so the roster the prompt refers to is decided by input order, not by anything
|
|
written in the prompt."""
|
|
items, blocks = [], []
|
|
for img in images:
|
|
if img is None:
|
|
continue
|
|
h, w = int(img.shape[1]), int(img.shape[2])
|
|
tw, th = ref_image_canvas(w, h, gen_w, gen_h, mode)
|
|
resized = _resize(img[:1], tw, th, "disabled")
|
|
items.append({"type": "image", "data": resized})
|
|
blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16,
|
|
"latent": vae.encode(resized)})
|
|
return items, blocks
|
|
|
|
|
|
def build_conditioning(clip, vae, audio_vae, prompt, width, height, length,
|
|
handoff=None, refs=None,
|
|
ref_noise_aug=0.999, silent=False, ref_image_size="match",
|
|
handoff_as_ref=False, speech_lead_seconds=0.0, speech_tail_frames=0):
|
|
"""Encode prompt, identity references, keyframe, and audio constraints for a shot."""
|
|
latent, fc = _empty_av_latent(width, height, length, H3_FPS)
|
|
refs = [r for r in (refs or []) if r is not None]
|
|
|
|
hand_img = None
|
|
if handoff is not None:
|
|
hand_img = _resize(handoff[:1], width, height, "disabled")
|
|
|
|
# REFERENCES AND THE KEYFRAME RIDE TOGETHER. This is the arrangement the node
|
|
# had before I broke it, and the reason is in ComfyUI's own layout:
|
|
#
|
|
# model_base.py:2183-2191 cond_video_latents = keyframe latents THEN ref latents
|
|
# model.py PackedLayout emits keyframe "cond" segments THEN ref "ref_img" ones
|
|
#
|
|
# The two orders agree, so both channels coexist. A shot takes its references AND
|
|
# a real keyframe: the keyframe ANCHORS the first frame, which is what continuity
|
|
# needs, while a reference only supplies identity. They are not alternatives.
|
|
#
|
|
# I had read "<Picture 1>" as MEANING the first frame on fl2va, and rearranged the
|
|
# roster around that. It does not. Which image is the first frame is decided by
|
|
# resolved_frame_index in minimax_keyframes, not by a label's number -- the labels
|
|
# are only how the images are shown to the VLM, and what they have to line up with
|
|
# is the <Picture N> tags in the prompt.
|
|
#
|
|
# So references come FIRST and keep slots 1..N, which is what a sheet line's
|
|
# `Name: <Picture 1>, ...` points at, and the handoff is appended AFTER them where
|
|
# it disturbs no numbering. It has to be in the list at all because
|
|
# tokenize_with_weights is either/or: passing minimax_ref_items makes it ignore
|
|
# `images` outright, so leaving the handoff out means the VLM is never shown where
|
|
# the shot left off and re-imagines the scenery -- same place, new room.
|
|
keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG
|
|
# One aug covers every visual condition row, references AND the keyframe. Below
|
|
# KEYFRAME_SAFE_AUG the keyframe latent would be noised and labelled at the wrong
|
|
# timestep, so the handoff stops being an anchor and rides as an extra reference
|
|
# instead: weaker continuity, but nothing pretending to anchor while carrying noise.
|
|
# ...or because the caller asked for it. A shot that introduces somebody already
|
|
# in position wants the room this picture carries and NOT the first frame it
|
|
# would force, and that is a demotion the aug knows nothing about.
|
|
carry_as_ref = bool(hand_img is not None
|
|
and (handoff_as_ref or (refs and not keyframe_ok)))
|
|
|
|
enc_refs = refs + ([hand_img] if carry_as_ref else [])
|
|
items, blocks = ([], [])
|
|
if enc_refs:
|
|
items, blocks = _build_ref_images(vae, enc_refs, width, height, ref_image_size)
|
|
if hand_img is not None and not carry_as_ref:
|
|
items = items + [{"type": "image", "data": hand_img}]
|
|
|
|
if items:
|
|
tokens = clip.tokenize(prompt, minimax_ref_items=items)
|
|
else:
|
|
tokens = clip.tokenize(prompt)
|
|
cond = clip.encode_from_tokens_scheduled(tokens)
|
|
|
|
vals = {}
|
|
if blocks:
|
|
vals["minimax_refs"] = blocks
|
|
# How CLEAN the references are shown. One aug covers every conditioning
|
|
# latent, keyframe included -- which is why softening references below
|
|
# KEYFRAME_SAFE_AUG would soften the anchor too.
|
|
if ref_noise_aug is not None:
|
|
vals["minimax_visual_cond_noise_aug"] = float(ref_noise_aug)
|
|
|
|
kfs = []
|
|
if hand_img is not None and not carry_as_ref:
|
|
kfs.append({"resolved_frame_index": 0,
|
|
"latent": _keyframe_latent(vae, hand_img)})
|
|
# Audio keyframes are extra conditioning rows in H3's PackedLayout. Pin the
|
|
# generated target stream instead, so the joint model also sees a quiet mouth.
|
|
# A dialogue shot pins its opening (the lead) and, past the line's estimated end,
|
|
# its close (the tail); the span between is the model's.
|
|
if silent or float(speech_lead_seconds or 0.0) > 0.0 or int(speech_tail_frames or 0) > 0:
|
|
_SILENCE_STATUS["asked"] += 1
|
|
if audio_vae is None:
|
|
_SILENCE_STATUS["why"] = "no audio VAE is wired to the node"
|
|
else:
|
|
sil = _silent_audio_latent(audio_vae, fc, H3_FPS)
|
|
if sil is None:
|
|
_SILENCE_STATUS["why"] = ("the audio VAE would not encode a silent "
|
|
"second -- the wrong VAE is on the "
|
|
"audio_vae input")
|
|
else:
|
|
lead = None if silent else round(float(speech_lead_seconds) *
|
|
AUDIO_LATENT_FPS)
|
|
tail = 0 if silent else int(speech_tail_frames or 0)
|
|
if _pin_audio_silence(latent, sil, lead, tail):
|
|
_SILENCE_STATUS["applied"] += 1
|
|
else:
|
|
_SILENCE_STATUS["why"] = "the silent latent did not match the shot"
|
|
if kfs:
|
|
vals["minimax_keyframes"] = kfs
|
|
if vals:
|
|
cond = node_helpers.conditioning_set_values(cond, vals)
|
|
return cond, latent, fc, carry_as_ref
|