Files
ComfyUI-H3-Continuum-Tiled/tests/test_v34_driving_audio.py
T

299 lines
9.0 KiB
Python

from __future__ import annotations
import pytest
import torch
from ComfyUI_H3_Continuum_Join.driving_audio import (
DrivingAudioAssets,
DrivingAudioSource,
attach_driving_audio,
encode_driving_audio,
prepare_driving_audio_source,
slice_driving_audio_latent,
)
from ComfyUI_H3_Continuum_Join.run_storage import automatic_project_key
from ComfyUI_H3_Continuum_Join.reference_video import (
REFERENCE_VIDEO_SIZE_BALANCED,
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
REFERENCE_VIDEO_SIZE_OPTIONS,
_resolved_size,
)
from ComfyUI_H3_Continuum_Join.v3.assembly import (
H3ContinuumAssembleSeamExperimental,
)
from ComfyUI_H3_Continuum_Join.v3.driving_nodes import (
H3ContinuumAssembleSeamV34,
H3ContinuumSamplerV34,
)
def test_v34_video_reference_schema_uses_image_batch():
schema = H3ContinuumSamplerV34.INPUT_TYPES()
assert schema["optional"]["reference_video_1"][0] == "IMAGE"
size_schema = schema["required"]["video_reference_size"]
assert size_schema[0] == REFERENCE_VIDEO_SIZE_OPTIONS
assert size_schema[1]["default"] == REFERENCE_VIDEO_SIZE_EFFICIENT
def test_v34_requests_one_chunk_video_reference(monkeypatch):
import ComfyUI_H3_Continuum_Join.reference_video as reference_video
captured = {}
class _Captured(RuntimeError):
pass
def capture_target(
_video,
*,
target_frames,
output_width,
output_height,
size_mode,
):
captured["target_frames"] = target_frames
captured["output_size"] = (output_width, output_height)
captured["size_mode"] = size_mode
raise _Captured
monkeypatch.setattr(
reference_video,
"prepare_reference_video_source",
capture_target,
)
with pytest.raises(_Captured):
H3ContinuumSamplerV34().run(
reference_video_1=torch.zeros((243, 8, 8, 3)),
driving_audio=None,
audio_vae=None,
chunks=2,
chunk_seconds=5.0,
width=800,
height=800,
video_reference_size=REFERENCE_VIDEO_SIZE_BALANCED,
)
assert captured["target_frames"] == 124
assert captured["output_size"] == (800, 800)
assert captured["size_mode"] == REFERENCE_VIDEO_SIZE_BALANCED
def test_video_reference_size_modes_preserve_source_and_avoid_upscale():
assert _resolved_size(
768,
768,
output_width=800,
output_height=800,
size_mode=REFERENCE_VIDEO_SIZE_EFFICIENT,
) == (640, 640)
assert _resolved_size(
768,
768,
output_width=800,
output_height=800,
size_mode=REFERENCE_VIDEO_SIZE_BALANCED,
) == (768, 768)
assert _resolved_size(
768,
768,
output_width=800,
output_height=800,
size_mode=REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
) == (768, 768)
def test_video_reference_match_output_preserves_source_aspect_ratio():
width, height = _resolved_size(
1920,
1080,
output_width=1344,
output_height=768,
size_mode=REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
)
assert width % 32 == 0
assert height % 32 == 0
assert width <= 1920
assert height <= 1080
assert abs((width / height) - (1920 / 1080)) < 0.06
class _AudioVAE:
audio_sample_rate = 10
def __init__(self):
self.calls = []
def encode(self, waveform):
self.calls.append(waveform)
return torch.arange(400, dtype=torch.float32).reshape(1, 1, 1, 400)
def _audio(seconds: int, *, tail_value: float = 0.0):
waveform = torch.zeros((1, 2, seconds * 10), dtype=torch.float32)
if seconds > 15:
waveform[..., 150:] = tail_value
return {"waveform": waveform, "sample_rate": 10}
def test_v34_public_schema_replaces_reference_audio_with_driving_audio():
schema = H3ContinuumSamplerV34.INPUT_TYPES()
assert "driving_audio" in schema["optional"]
assert "audio_vae" in schema["optional"]
assert "reference_audio_1" not in schema["optional"]
assert "reference_audio_vae" not in schema["optional"]
from ComfyUI_H3_Continuum_Join import nodes as root_nodes
assert root_nodes.NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumSamplerV34"] == (
"H3 Continuum Sampler V3.4"
)
assert root_nodes.NODE_DISPLAY_NAME_MAPPINGS[
"H3ContinuumSamplerTimelineVideo"
].startswith("[Legacy]")
def test_unused_long_audio_tail_does_not_change_contract_or_output():
vae = _AudioVAE()
first = prepare_driving_audio_source(
_audio(30, tail_value=1.0), vae, target_frames=360, fps=24
)
second = prepare_driving_audio_source(
_audio(30, tail_value=9.0), vae, target_frames=360, fps=24
)
assert first.combined_hash == second.combined_hash
assert first.source_audio["waveform"].shape[-1] == 150
assert second.source_audio["waveform"].shape[-1] == 150
def test_short_driving_audio_is_accepted_without_padding():
source = prepare_driving_audio_source(
_audio(4), _AudioVAE(), target_frames=360, fps=24
)
assert source.source_audio["waveform"].shape == (1, 2, 40)
assert source.resampled_waveform.shape == (1, 2, 40)
def test_encode_is_core_layout_and_absolute_slices_are_drift_free():
vae = _AudioVAE()
source = prepare_driving_audio_source(
_audio(30), vae, target_frames=360, fps=24
)
assets = encode_driving_audio(source, vae)
assert len(vae.calls) == 1
assert vae.calls[0].shape == (1, 150, 2)
first = slice_driving_audio_latent(
assets,
cumulative_retained_before=0,
total_frames=124,
trim_frames=0,
fps=24,
)
second = slice_driving_audio_latent(
assets,
cumulative_retained_before=124,
total_frames=141,
trim_frames=22,
fps=24,
)
third = slice_driving_audio_latent(
assets,
cumulative_retained_before=243,
total_frames=141,
trim_frames=22,
fps=24,
)
assert first.flatten()[0].item() == 0
assert second.flatten()[0].item() == round(102 * 40 / 24)
assert third.flatten()[0].item() == round(221 * 40 / 24)
def test_driving_audio_uses_keyframe_payload_not_audio_reference_tokens():
conditioning = [[torch.zeros((1, 2, 3)), {"minimax_ref_items": []}]]
latent = torch.zeros((1, 32, 2, 10))
output = attach_driving_audio(conditioning, latent)
keyframe = output[0][1]["minimax_keyframes"][-1]
assert keyframe == {"resolved_frame_index": 0, "audio_latent": latent}
assert output[0][1]["minimax_ref_items"] == []
def test_audio_vae_topology_is_ignored_when_driving_audio_is_disconnected():
base = {
"1": {"class_type": "H3ContinuumSamplerV34", "inputs": {}},
"2": {"class_type": "VAELoader", "inputs": {}},
}
connected_vae_only = {
**base,
"1": {
"class_type": "H3ContinuumSamplerV34",
"inputs": {"audio_vae": ["2", 0]},
},
}
assert automatic_project_key(base, "1") == automatic_project_key(
connected_vae_only, "1"
)
def test_v34_assembler_selects_original_driving_audio_and_bypasses_seam(
monkeypatch,
):
images = torch.zeros((10, 8, 8, 3))
generated = {"waveform": torch.ones((1, 2, 20)), "sample_rate": 10}
driving = {"waveform": torch.zeros((1, 2, 20)), "sample_rate": 10}
monkeypatch.setattr(
H3ContinuumAssembleSeamExperimental,
"assemble",
lambda self, *args, **kwargs: (images, generated, "base report"),
)
output_images, output_audio, report = H3ContinuumAssembleSeamV34().assemble(
images=[],
audio=[],
assembly_plan={"target_frames": 10},
exact_total_duration=False,
audio_seam="Auto",
video_seam="Auto",
diagnostics="Basic",
driving_audio=driving,
)
assert output_images is images
assert torch.equal(output_audio["waveform"], driving["waveform"])
assert "Audio Seam bypassed" in report
def test_v34_assembly_plan_audio_wins_when_direct_input_is_generated_audio(
monkeypatch,
):
images = torch.zeros((10, 8, 8, 3))
generated = {
"waveform": torch.zeros((1, 2, 320000), dtype=torch.float32),
"sample_rate": 32000,
}
original = {
"waveform": torch.linspace(-1.0, 1.0, 480000, dtype=torch.float32)
.reshape(1, 1, -1)
.repeat(1, 2, 1),
"sample_rate": 48000,
}
plan = {
"target_frames": 10,
"_h3_continuum_driving_audio_v1": original,
}
monkeypatch.setattr(
H3ContinuumAssembleSeamExperimental,
"assemble",
lambda self, *args, **kwargs: (images, generated, "base report"),
)
_, output_audio, report = H3ContinuumAssembleSeamV34().assemble(
images=[],
audio=[],
assembly_plan=[plan],
exact_total_duration=False,
driving_audio=[generated],
)
assert output_audio["sample_rate"] == 48000
assert torch.equal(output_audio["waveform"], original["waveform"])
assert output_audio["waveform"].data_ptr() != original["waveform"].data_ptr()
assert "selected from assembly plan" in report