Files
DumasNodes/tests/test_layout_adapter.py
T
chris.dumas bf8f76505f Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum
git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
2026-10-20 13:43:38 +00:00

189 lines
7.0 KiB
Python

from types import SimpleNamespace
import torch
from ComfyUI_H3_Continuum_Join.constants import (
MARK_AUDIO_CONTEXT,
MARK_AUDIO_END_FRAME,
MARK_AUDIO_OVERHANG,
MARK_CONTEXT_FRAMES,
MARK_VIDEO_CONTEXT,
)
from ComfyUI_H3_Continuum_Join.layout_adapter import (
normalize_condition_latents,
patch_layout_in_place,
)
def _video_times(origin: float):
offsets = (0, 1, 5, 9, 13, 17, 18)
return [origin + value * 5.0 / 3.0 for value in offsets]
def _fake_layout(*, with_last_keyframe=False, audio_steps=37):
# text=3, optional one stock keyframe, a 22-frame video/audio context ref,
# target audio=10 steps, target video=7 slots x 2 rows.
row = 0
segments = [(row, row + 3, "text")]
row += 3
if with_last_keyframe:
segments.append((row, row + 2, "cond"))
row += 2
audio_ref = (row, row + audio_steps * 2)
segments.append((*audio_ref, "ref_audio"))
row += audio_steps * 2
video_ref = (row, row + 14)
segments.append((*video_ref, "ref_img"))
row += 14
target_audio = (row, row + 20)
segments.append((*target_audio, "audio"))
row += 20
target_video = (row, row + 14)
segments.append((*target_video, "video"))
row += 14
pos = torch.zeros(row, 3, dtype=torch.float64)
pos[:3, 0] = torch.arange(3, dtype=torch.float64)
if with_last_keyframe:
# Stock last keyframe is text-relative before refs shift the target.
pos[3:5, 0] = 3.0 + 35.0
# Stock context ref lives in its own pre-target coordinate span.
a0, a1 = audio_ref
pos[a0 : a0 + audio_steps, 0] = (
torch.arange(audio_steps, dtype=torch.float64) + 3.0
)
pos[a0 + audio_steps : a1, 0] = (
torch.arange(audio_steps, dtype=torch.float64) + 3.0
)
v0, _v1 = video_ref
for slot, value in enumerate(_video_times(3.0)):
pos[v0 + slot * 2 : v0 + (slot + 1) * 2, 0] = value
# Target origin follows Core's longest reference span.
target_origin = 3.0 + max(float(audio_steps), 22.0 * 5.0 / 3.0)
ta0, ta1 = target_audio
pos[ta0 : ta0 + 10, 0] = torch.arange(10, dtype=torch.float64) + target_origin
pos[ta0 + 10 : ta1, 0] = (
torch.arange(10, dtype=torch.float64) + target_origin
)
tv0, _tv1 = target_video
for slot, value in enumerate(_video_times(target_origin)):
pos[tv0 + slot * 2 : tv0 + (slot + 1) * 2, 0] = value
# Give every target video row distinct h/w values so the full-coordinate
# copy (not only time) is tested.
pos[tv0 : tv0 + 14, 1] = torch.arange(14, dtype=torch.float64)
pos[tv0 : tv0 + 14, 2] = torch.arange(14, dtype=torch.float64) + 100
return SimpleNamespace(
segments=segments,
position_ids=pos,
signature=(3, 7, 2, 2, 10),
), video_ref, target_video
def _context_ref(video, audio):
return {
"kind": "video_audio",
"latent_t": 7,
"latent_h": 2,
"latent_w": 2,
"ref_audio_t": 37,
"latent": video,
"audio_latent": audio,
MARK_VIDEO_CONTEXT: True,
MARK_CONTEXT_FRAMES: 22,
MARK_AUDIO_CONTEXT: True,
MARK_AUDIO_END_FRAME: 22.0,
MARK_AUDIO_OVERHANG: 1.0 / 3.0,
}
def test_layout_patches_video_audio_ref_in_place_and_is_idempotent():
layout, video_ref_span, target_video_span = _fake_layout()
tensor_id = id(layout.position_ids)
video = torch.zeros(1, 24, 7, 2, 2)
audio = torch.zeros(1, 32, 2, 37)
payload = {"layout": layout, "keyframes": [], "refs": [_context_ref(video, audio)]}
normalize_condition_latents(payload)
result = patch_layout_in_place(payload)
assert result["video_refs"] == 1
assert result["audio_windows"] == 1
assert id(layout.position_ids) == tensor_id
rv0, rv1 = video_ref_span
tv0, tv1 = target_video_span
assert torch.equal(layout.position_ids[rv0:rv1], layout.position_ids[tv0:tv1])
# 22 frames (36 2/3 audio steps) plus source overhang 1/3 = 37,
# so the audio context begins exactly at target origin 40.
audio_start = next(a for a, _b, kind in layout.segments if kind == "ref_audio")
assert torch.isclose(
layout.position_ids[audio_start, 0], torch.tensor(40.0, dtype=torch.float64)
)
before = layout.position_ids.clone()
second = patch_layout_in_place(payload)
assert second["status"] == "already_patched"
assert torch.equal(before, layout.position_ids)
assert payload["cond_video_latents"] == [video]
assert payload["cond_audio_latents"] == [audio]
def test_existing_last_keyframe_is_shifted_with_context_ref():
layout, _video_ref_span, target_video_span = _fake_layout(with_last_keyframe=True)
target_origin = float(layout.position_ids[target_video_span[0], 0])
image = torch.zeros(1, 24, 1, 2, 2)
video = torch.zeros(1, 24, 7, 2, 2)
audio = torch.zeros(1, 32, 2, 37)
payload = {
"layout": layout,
"keyframes": [{"resolved_frame_index": 140, "latent": image}],
"refs": [_context_ref(video, audio)],
}
normalize_condition_latents(payload)
patch_layout_in_place(payload)
cond_start = next(a for a, _b, kind in layout.segments if kind == "cond")
expected = target_origin + 140.0 * 5.0 / 3.0
assert torch.allclose(
layout.position_ids[cond_start : cond_start + 2, 0],
torch.full((2,), expected, dtype=torch.float64),
)
assert payload["cond_video_latents"] == [image, video]
def test_negative_audio_grid_offset_places_context_before_target_origin():
layout, _video_ref_span, _target_video_span = _fake_layout()
video = torch.zeros(1, 24, 7, 2, 2)
audio = torch.zeros(1, 32, 2, 37)
ref = _context_ref(video, audio)
ref[MARK_AUDIO_OVERHANG] = -1.0 / 3.0
payload = {"layout": layout, "keyframes": [], "refs": [ref]}
normalize_condition_latents(payload)
patch_layout_in_place(payload)
audio_start = next(a for a, _b, kind in layout.segments if kind == "ref_audio")
# target origin is 40; 22 frames = 36 2/3 audio steps, signed offset -1/3,
# and the retained window is 37 steps, so it starts 2/3 step before origin.
assert torch.isclose(
layout.position_ids[audio_start, 0],
torch.tensor(40.0 - 2.0 / 3.0, dtype=torch.float64),
)
def test_audio40_research_window_is_end_aligned_without_layout_changes():
layout, _video_ref_span, target_video_span = _fake_layout(audio_steps=40)
video = torch.zeros(1, 24, 7, 2, 2)
audio = torch.zeros(1, 32, 2, 40)
ref = _context_ref(video, audio)
ref["ref_audio_t"] = 40
payload = {"layout": layout, "keyframes": [], "refs": [ref]}
normalize_condition_latents(payload)
patch_layout_in_place(payload)
audio_start = next(a for a, _b, kind in layout.segments if kind == "ref_audio")
target_origin = float(layout.position_ids[target_video_span[0], 0])
# The extra three ticks precede the ordinary 37T window; both end at the
# same source-grid phase beside the 22-frame video context.
assert torch.isclose(
layout.position_ids[audio_start, 0],
torch.tensor(target_origin - 3.0, dtype=torch.float64),
)