Add handoff context frames
This commit is contained in:
@@ -38,6 +38,7 @@
|
|||||||
- The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine directly so it can track the source project again.
|
- The local Dumas prompt-engineering fork has been removed from this node. Long Videos now wraps the upstream sampler/engine directly so it can track the source project again.
|
||||||
- Upstream compatibility keys `H3LongVideos`, `H3LongVideosFL2VA`, `H3LongVideosV1`, and `H3LongVideosREF2VA` are also registered to the same class.
|
- Upstream compatibility keys `H3LongVideos`, `H3LongVideosFL2VA`, `H3LongVideosV1`, and `H3LongVideosREF2VA` are also registered to the same class.
|
||||||
- The old Dumas browser widget grouping script is disabled for this node because it targeted controls that no longer exist on the upstream sampler.
|
- The old Dumas browser widget grouping script is disabled for this node because it targeted controls that no longer exist on the upstream sampler.
|
||||||
|
- `handoff_frames` extends the upstream last-frame handoff: `1` keeps the current single keyframe behavior; higher values keep that final-frame keyframe and add earlier tail frames from the previous shot as claimed reference context for the next beat.
|
||||||
- Upstream license text is included in [`H3_LONGVIDEOS_UPSTREAM_LICENSE.txt`](./H3_LONGVIDEOS_UPSTREAM_LICENSE.txt).
|
- Upstream license text is included in [`H3_LONGVIDEOS_UPSTREAM_LICENSE.txt`](./H3_LONGVIDEOS_UPSTREAM_LICENSE.txt).
|
||||||
|
|
||||||
- `Dumas H3 Latent Upscale Params`
|
- `Dumas H3 Latent Upscale Params`
|
||||||
|
|||||||
@@ -5831,6 +5831,24 @@ def handoff_claim(n):
|
|||||||
f"anybody new.")
|
f"anybody new.")
|
||||||
|
|
||||||
|
|
||||||
|
def handoff_context_claim(first, last):
|
||||||
|
"""Claim previous-shot tail frames carried as reference context.
|
||||||
|
|
||||||
|
These are not keyframes. They are context stills that show the room, camera
|
||||||
|
path, and motion immediately before the hard handoff keyframe. They must be
|
||||||
|
named in the prompt because every reference image the VLM sees needs a job:
|
||||||
|
unnamed pictures are free to become extra subjects.
|
||||||
|
"""
|
||||||
|
first, last = int(first), int(last)
|
||||||
|
if last <= first:
|
||||||
|
return (f" <Picture {first}> is a continuity context frame from immediately "
|
||||||
|
f"before this shot: same place, same camera path, same people, no "
|
||||||
|
f"new subject.")
|
||||||
|
return (f" <Picture {first}> through <Picture {last}> are continuity context "
|
||||||
|
f"frames from immediately before this shot: same place, same camera "
|
||||||
|
f"path, same people, no new subjects.")
|
||||||
|
|
||||||
|
|
||||||
def room_claim(n, present, joining):
|
def room_claim(n, present, joining):
|
||||||
"""Claim a handoff carried as a reference because somebody NEW is in the shot.
|
"""Claim a handoff carried as a reference because somebody NEW is in the shot.
|
||||||
|
|
||||||
@@ -7062,6 +7080,7 @@ _WIDGET_RANGE = {
|
|||||||
"pace": (1.0, 0.25, 2.0, float),
|
"pace": (1.0, 0.25, 2.0, float),
|
||||||
"ambient_level": (0.25, 0.0, 1.0, float),
|
"ambient_level": (0.25, 0.0, 1.0, float),
|
||||||
"foley_level": (0.35, 0.0, 1.0, float),
|
"foley_level": (0.35, 0.0, 1.0, float),
|
||||||
|
"handoff_frames": (1, 1, MAX_FRAMES, int),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -7582,6 +7601,18 @@ class H3LongVideos:
|
|||||||
"It is synthesis, not a recording: a click, a rattle, "
|
"It is synthesis, not a recording: a click, a rattle, "
|
||||||
"a rustle, in the right place. Nothing vocal is ever "
|
"a rustle, in the right place. Nothing vocal is ever "
|
||||||
"built. 0 turns it off; needs auto_sound on."}),
|
"built. 0 turns it off; needs auto_sound on."}),
|
||||||
|
# APPENDED. Saved workflows restore widget values by position.
|
||||||
|
"handoff_frames": ("INT", {"default": 1, "min": 1, "max": MAX_FRAMES,
|
||||||
|
"tooltip": "How many frames from the previous shot are used to "
|
||||||
|
"condition the next one. 1 is the upstream default: "
|
||||||
|
"the previous final frame becomes the next shot's "
|
||||||
|
"keyframe. Values above 1 keep that final-frame "
|
||||||
|
"keyframe and append the earlier tail frames as "
|
||||||
|
"claimed reference context, so the next beat can see "
|
||||||
|
"more of the incoming motion and room continuity. "
|
||||||
|
"The extra frames are references, not extra "
|
||||||
|
"keyframes, so seam trimming still removes only the "
|
||||||
|
"duplicated opening frame."}),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7608,6 +7639,7 @@ class H3LongVideos:
|
|||||||
character_guard=True, pace=1.0, auto_sound=True, hold_scene_state=True,
|
character_guard=True, pace=1.0, auto_sound=True, hold_scene_state=True,
|
||||||
mouths_shut_when_no_line=True, hold_gaze=True,
|
mouths_shut_when_no_line=True, hold_gaze=True,
|
||||||
ambient_audio=None, ambient_level=0.25, foley_level=0.35,
|
ambient_audio=None, ambient_level=0.25, foley_level=0.35,
|
||||||
|
handoff_frames=1,
|
||||||
**_removed):
|
**_removed):
|
||||||
# **_removed: a workflow saved with the old `save_defaults` widget still sends
|
# **_removed: a workflow saved with the old `save_defaults` widget still sends
|
||||||
# it. Swallowed rather than raising, so an existing workflow keeps loading.
|
# it. Swallowed rather than raising, so an existing workflow keeps loading.
|
||||||
@@ -7632,7 +7664,8 @@ class H3LongVideos:
|
|||||||
shift_video=shift_video, shift_audio=shift_audio,
|
shift_video=shift_video, shift_audio=shift_audio,
|
||||||
ref_noise_aug=ref_noise_aug, latent_upscale_scale=latent_upscale_scale,
|
ref_noise_aug=ref_noise_aug, latent_upscale_scale=latent_upscale_scale,
|
||||||
upscale_target_short_edge=upscale_target_short_edge,
|
upscale_target_short_edge=upscale_target_short_edge,
|
||||||
upscale_batch=upscale_batch, pace=pace))
|
upscale_batch=upscale_batch, pace=pace,
|
||||||
|
handoff_frames=handoff_frames))
|
||||||
megapixels, shot_seconds = _fixed["megapixels"], _fixed["shot_seconds"]
|
megapixels, shot_seconds = _fixed["megapixels"], _fixed["shot_seconds"]
|
||||||
steps, cfg = _fixed["steps"], _fixed["cfg"]
|
steps, cfg = _fixed["steps"], _fixed["cfg"]
|
||||||
shift_video, shift_audio = _fixed["shift_video"], _fixed["shift_audio"]
|
shift_video, shift_audio = _fixed["shift_video"], _fixed["shift_audio"]
|
||||||
@@ -7640,6 +7673,7 @@ class H3LongVideos:
|
|||||||
latent_upscale_scale = _fixed["latent_upscale_scale"]
|
latent_upscale_scale = _fixed["latent_upscale_scale"]
|
||||||
upscale_target_short_edge = _fixed["upscale_target_short_edge"]
|
upscale_target_short_edge = _fixed["upscale_target_short_edge"]
|
||||||
upscale_batch, pace = _fixed["upscale_batch"], _fixed["pace"]
|
upscale_batch, pace = _fixed["upscale_batch"], _fixed["pace"]
|
||||||
|
handoff_frames = _fixed["handoff_frames"]
|
||||||
notes.extend(_fixnotes)
|
notes.extend(_fixnotes)
|
||||||
# <Picture N> means ref_image_N, the socket. Everything downstream works on
|
# <Picture N> means ref_image_N, the socket. Everything downstream works on
|
||||||
# the packed roster instead, so translate once, here, before anything has
|
# the packed roster instead, so translate once, here, before anything has
|
||||||
@@ -10234,6 +10268,7 @@ class H3LongVideos:
|
|||||||
negative = clip.encode_from_tokens_scheduled(clip.tokenize(""))
|
negative = clip.encode_from_tokens_scheduled(clip.tokenize(""))
|
||||||
|
|
||||||
handoff = first_frame
|
handoff = first_frame
|
||||||
|
handoff_context = None
|
||||||
# Where the time actually goes. Sampling and decode trade off against each
|
# Where the time actually goes. Sampling and decode trade off against each
|
||||||
# other -- latent_upscale buys cheaper sampling and pays for it at decode,
|
# other -- latent_upscale buys cheaper sampling and pays for it at decode,
|
||||||
# and which side wins depends on `steps`. Reported so the trade is a
|
# and which side wins depends on `steps`. Reported so the trade is a
|
||||||
@@ -10365,6 +10400,20 @@ class H3LongVideos:
|
|||||||
# here rather than inside build_conditioning because the claim is text,
|
# here rather than inside build_conditioning because the claim is text,
|
||||||
# and the text is assembled up here.
|
# and the text is assembled up here.
|
||||||
_shot_refs = list(shot_refs_all[i]) + _extra
|
_shot_refs = list(shot_refs_all[i]) + _extra
|
||||||
|
_ctx_refs = []
|
||||||
|
_keyframe_ok = ref_noise_aug is None or float(ref_noise_aug) >= KEYFRAME_SAFE_AUG
|
||||||
|
if (handoff_frames > 1 and shot_handoff is not None and handoff_context is not None
|
||||||
|
and not _handoff_ref and _keyframe_ok):
|
||||||
|
try:
|
||||||
|
_ctx_refs = [handoff_context[j:j + 1]
|
||||||
|
for j in range(int(handoff_context.shape[0]))]
|
||||||
|
except Exception:
|
||||||
|
_ctx_refs = []
|
||||||
|
if _ctx_refs:
|
||||||
|
_first = len(_shot_refs) + 1
|
||||||
|
_last = _first + len(_ctx_refs) - 1
|
||||||
|
shot_prompt = shot_prompt + handoff_context_claim(_first, _last)
|
||||||
|
_shot_refs.extend(_ctx_refs)
|
||||||
if _handoff_ref:
|
if _handoff_ref:
|
||||||
# Carried for the ROOM, with somebody new in the shot -- so the
|
# Carried for the ROOM, with somebody new in the shot -- so the
|
||||||
# standing claim is exactly wrong here ("joined by anybody new") and
|
# standing claim is exactly wrong here ("joined by anybody new") and
|
||||||
@@ -10380,7 +10429,7 @@ class H3LongVideos:
|
|||||||
sent_text[i] = shot_prompt
|
sent_text[i] = shot_prompt
|
||||||
cond, latent, fc, demoted = build_conditioning(
|
cond, latent, fc, demoted = build_conditioning(
|
||||||
clip, vae, audio_vae, shot_prompt, w, h, lens[i],
|
clip, vae, audio_vae, shot_prompt, w, h, lens[i],
|
||||||
handoff=shot_handoff, refs=list(shot_refs_all[i]) + _extra,
|
handoff=shot_handoff, refs=_shot_refs,
|
||||||
ref_noise_aug=ref_noise_aug, silent=silent,
|
ref_noise_aug=ref_noise_aug, silent=silent,
|
||||||
handoff_as_ref=_handoff_ref)
|
handoff_as_ref=_handoff_ref)
|
||||||
if demoted and not _aug_warned:
|
if demoted and not _aug_warned:
|
||||||
@@ -10457,6 +10506,16 @@ class H3LongVideos:
|
|||||||
# 0..1, and feeding that back in to be re-encoded every boundary is a
|
# 0..1, and feeding that back in to be re-encoded every boundary is a
|
||||||
# drift that accumulates rather than cancels.
|
# drift that accumulates rather than cancels.
|
||||||
handoff = hand_src[-1:].detach().clamp(0.0, 1.0).to("cpu", copy=True)
|
handoff = hand_src[-1:].detach().clamp(0.0, 1.0).to("cpu", copy=True)
|
||||||
|
handoff_context = None
|
||||||
|
if handoff_frames > 1:
|
||||||
|
try:
|
||||||
|
available = max(0, int(hand_src.shape[0]) - 1)
|
||||||
|
want = min(max(0, int(handoff_frames) - 1), available)
|
||||||
|
if want > 0:
|
||||||
|
handoff_context = hand_src[-(want + 1):-1].detach().clamp(
|
||||||
|
0.0, 1.0).to("cpu", copy=True)
|
||||||
|
except Exception:
|
||||||
|
handoff_context = None
|
||||||
# Keep a frame for the shot they come back on -- but ONLY from a shot that
|
# Keep a frame for the shot they come back on -- but ONLY from a shot that
|
||||||
# was theirs alone.
|
# was theirs alone.
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -112,8 +112,17 @@ class DumasH3LongVideosUpstreamWrapperTests(unittest.TestCase):
|
|||||||
self.assertIn("first_frame", schema["optional"])
|
self.assertIn("first_frame", schema["optional"])
|
||||||
self.assertIn("ref_image_1", schema["optional"])
|
self.assertIn("ref_image_1", schema["optional"])
|
||||||
self.assertIn("latent_upscale", schema["optional"])
|
self.assertIn("latent_upscale", schema["optional"])
|
||||||
|
self.assertIn("handoff_frames", schema["optional"])
|
||||||
|
self.assertEqual(schema["optional"]["handoff_frames"][1]["default"], 1)
|
||||||
self.assertEqual(node_cls.RETURN_NAMES[0:4], ("images", "audio", "info", "script"))
|
self.assertEqual(node_cls.RETURN_NAMES[0:4], ("images", "audio", "info", "script"))
|
||||||
|
|
||||||
|
def test_handoff_context_claim_names_reference_range(self):
|
||||||
|
upstream = importlib.import_module("dumas_h3_longvideos_upstream")
|
||||||
|
|
||||||
|
self.assertIn("<Picture 2> through <Picture 22>", upstream.handoff_context_claim(2, 22))
|
||||||
|
self.assertIn("no new subjects", upstream.handoff_context_claim(2, 22))
|
||||||
|
self.assertIn("<Picture 5>", upstream.handoff_context_claim(5, 5))
|
||||||
|
|
||||||
|
|
||||||
class _NullContext:
|
class _NullContext:
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user