Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
This commit is contained in:
@@ -0,0 +1,555 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import pytest
|
||||
|
||||
import ComfyUI_H3_Continuum_Join.v3.second_pass as second_pass_module
|
||||
from ComfyUI_H3_Continuum_Join.nodes import NODE_CLASS_MAPPINGS
|
||||
from ComfyUI_H3_Continuum_Join.v3.second_pass import (
|
||||
SecondPassContractError,
|
||||
derive_refine_seed,
|
||||
run_second_pass_groups,
|
||||
)
|
||||
from ComfyUI_H3_Continuum_Join.v3.refine_context import (
|
||||
MAGIC as REFINE_CONTEXT_MAGIC,
|
||||
RefineConditioningAdaptationError,
|
||||
)
|
||||
from ComfyUI_H3_Continuum_Join.v3.refine_schedule import make_tail_schedule
|
||||
from ComfyUI_H3_Continuum_Join.v3.second_pass_nodes import (
|
||||
H3ContinuumSecondPassV35,
|
||||
)
|
||||
|
||||
|
||||
def _latent(samples: torch.Tensor) -> dict[str, torch.Tensor]:
|
||||
return {"samples": samples}
|
||||
|
||||
|
||||
def _plan(groups: list[dict]) -> dict:
|
||||
return {
|
||||
"width": 640,
|
||||
"height": 640,
|
||||
"second_pass_contract": {
|
||||
"version": 1,
|
||||
"physical_groups": groups,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _group(group_id: int, *, temporal: int, prompt: str, terminal: bool = False) -> dict:
|
||||
return {
|
||||
"group_id": group_id,
|
||||
"logical_chunks": [group_id + 1] if not terminal else [2, 3],
|
||||
"physical_prompt": prompt,
|
||||
"prompt_policy": "single" if not terminal else "paired_timeline_v1",
|
||||
"physical_frames": 124 if not terminal else 260,
|
||||
"trim_prefix_frames": 0 if not terminal else 22,
|
||||
"terminal_merged": terminal,
|
||||
"source_width": 640,
|
||||
"source_height": 640,
|
||||
"source_batch": 1,
|
||||
"latent_channels": 24,
|
||||
"source_latent_t": temporal,
|
||||
"source_latent_h": 40,
|
||||
"source_latent_w": 40,
|
||||
"source_audio_shape": [1, 8, temporal * 4],
|
||||
}
|
||||
|
||||
|
||||
def _run_fixture(groups: list[dict]):
|
||||
videos = [
|
||||
_latent(torch.zeros((1, 24, group["source_latent_t"], 48, 48)))
|
||||
for group in groups
|
||||
]
|
||||
audios = [
|
||||
_latent(torch.full(tuple(group["source_audio_shape"]), float(index + 1)))
|
||||
for index, group in enumerate(groups)
|
||||
]
|
||||
calls: list[dict] = []
|
||||
|
||||
def encode_prompt(_clip, prompt, **kwargs):
|
||||
assert kwargs["first_image"] is None
|
||||
assert kwargs["last_image"] is None
|
||||
calls.append({"prompt": prompt})
|
||||
return [prompt]
|
||||
|
||||
def build_latent(video, audio):
|
||||
return {"video": video, "audio": audio}
|
||||
|
||||
def clone_model(model, **kwargs):
|
||||
calls[-1]["clone"] = kwargs
|
||||
return model
|
||||
|
||||
def sample(**kwargs):
|
||||
calls[-1]["seed"] = kwargs["seed"]
|
||||
calls[-1]["conditioning"] = kwargs["conditioning"]
|
||||
calls[-1]["sigmas"] = kwargs["sigmas"]
|
||||
nested = kwargs["latent"]
|
||||
return {
|
||||
"video": nested["video"] + 1,
|
||||
"audio": nested["audio"] + 99,
|
||||
}
|
||||
|
||||
def extract(sampled):
|
||||
return sampled["video"], sampled["audio"]
|
||||
|
||||
result = run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=videos,
|
||||
audio_latents=audios,
|
||||
assembly_plan=_plan(groups),
|
||||
refine_seed=42,
|
||||
encode_prompt_fn=encode_prompt,
|
||||
latent_builder=build_latent,
|
||||
sample_fn=sample,
|
||||
stream_extractor=extract,
|
||||
clone_model_fn=clone_model,
|
||||
)
|
||||
return videos, audios, calls, result
|
||||
|
||||
|
||||
def test_second_pass_uses_plan_prompts_one_seed_per_physical_group_and_audio_passthrough():
|
||||
groups = [
|
||||
_group(0, temporal=31, prompt="first physical prompt"),
|
||||
_group(1, temporal=65, prompt="terminal physical prompt", terminal=True),
|
||||
]
|
||||
videos, audios, calls, (refined, output_audio, updated, report) = _run_fixture(groups)
|
||||
|
||||
assert [call["prompt"] for call in calls] == [
|
||||
"first physical prompt",
|
||||
"terminal physical prompt",
|
||||
]
|
||||
assert [call["seed"] for call in calls] == [
|
||||
derive_refine_seed(42, 0),
|
||||
derive_refine_seed(42, 1),
|
||||
]
|
||||
assert [call["clone"] for call in calls] == [
|
||||
{
|
||||
"strict": False,
|
||||
"debug": False,
|
||||
"chunk_index": 1,
|
||||
"context_frames": None,
|
||||
},
|
||||
{
|
||||
"strict": False,
|
||||
"debug": False,
|
||||
"chunk_index": 2,
|
||||
"context_frames": 22,
|
||||
},
|
||||
]
|
||||
assert all(after is before for before, after in zip(audios, output_audio, strict=True))
|
||||
assert all(torch.equal(item["samples"], torch.ones_like(item["samples"])) for item in refined)
|
||||
assert [item["samples"].shape[2] for item in refined] == [31, 65]
|
||||
assert updated["width"] == 768
|
||||
assert updated["height"] == 768
|
||||
assert updated["second_pass_contract"]["audio_output"] == (
|
||||
"bit_exact_first_pass_passthrough"
|
||||
)
|
||||
assert updated["second_pass_contract"]["execution"] == (
|
||||
"t2va_physical_groups_audio_locked_v2"
|
||||
)
|
||||
assert updated["second_pass_contract"]["audio_sampling"] == (
|
||||
"zero_noise_mask_locked"
|
||||
)
|
||||
schedule = updated["second_pass_contract"]["refine_schedule"]
|
||||
assert schedule["mode"] == "external"
|
||||
assert schedule["evaluation_count"] == 1
|
||||
assert updated["second_pass_contract"]["refine_schedule_identity"] == (
|
||||
schedule["schedule_hash"]
|
||||
)
|
||||
assert all(call["sigmas"] is calls[0]["sigmas"] for call in calls)
|
||||
assert "temporary_audio_discarded=true" in report
|
||||
assert "audio zero noise/mask=0" in report
|
||||
assert "refine_seed=" in report
|
||||
assert videos[1]["samples"].shape[2] == 65
|
||||
|
||||
|
||||
def test_internal_tail_schedule_passes_the_exact_suffix_and_records_identity():
|
||||
group = _group(0, temporal=31, prompt="physical prompt")
|
||||
video = _latent(torch.zeros((1, 24, 31, 48, 48)))
|
||||
audio = _latent(torch.ones(tuple(group["source_audio_shape"])))
|
||||
base_sigmas = torch.tensor([1.0, 0.75, 0.5, 0.25, 0.0])
|
||||
tail = make_tail_schedule(base_sigmas, evaluation_count=2)
|
||||
captured = {}
|
||||
|
||||
def sample(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return kwargs["latent"]
|
||||
|
||||
_video, output_audio, updated, report = run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=base_sigmas,
|
||||
video_latents=[video],
|
||||
audio_latents=[audio],
|
||||
assembly_plan=_plan([group]),
|
||||
refine_seed=17,
|
||||
refine_schedule=tail,
|
||||
encode_prompt_fn=lambda *_args, **_kwargs: ["conditioning"],
|
||||
latent_builder=lambda video_samples, audio_samples: {
|
||||
"video": video_samples,
|
||||
"audio": audio_samples,
|
||||
},
|
||||
sample_fn=sample,
|
||||
stream_extractor=lambda sampled: (sampled["video"], sampled["audio"]),
|
||||
clone_model_fn=lambda source, **_kwargs: source,
|
||||
)
|
||||
|
||||
assert captured["sigmas"] is tail.sigmas
|
||||
assert torch.equal(captured["sigmas"], base_sigmas[2:])
|
||||
schedule = updated["second_pass_contract"]["refine_schedule"]
|
||||
assert schedule["mode"] == "tail"
|
||||
assert schedule["evaluation_count"] == 2
|
||||
assert schedule["start_index"] == 2
|
||||
assert schedule["noise_mode"] == "video_random_mask_1_audio_zero_mask_0"
|
||||
assert schedule["audio_policy"] == "locked_first_pass_bit_exact_passthrough"
|
||||
assert output_audio[0] is audio
|
||||
assert "mode=tail" in report
|
||||
|
||||
|
||||
def test_second_pass_defaults_to_audio_locked_refine_sampler(monkeypatch):
|
||||
group = _group(0, temporal=31, prompt="physical prompt")
|
||||
video = _latent(torch.zeros((1, 24, 31, 48, 48)))
|
||||
audio = _latent(torch.ones(tuple(group["source_audio_shape"])))
|
||||
captured = {}
|
||||
|
||||
def fake_refine_sample(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return kwargs["latent"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
second_pass_module,
|
||||
"sample_refine_chunk",
|
||||
fake_refine_sample,
|
||||
)
|
||||
run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=[video],
|
||||
audio_latents=[audio],
|
||||
assembly_plan=_plan([group]),
|
||||
refine_seed=17,
|
||||
encode_prompt_fn=lambda *_args, **_kwargs: ["conditioning"],
|
||||
latent_builder=lambda video_samples, audio_samples: {
|
||||
"video": video_samples,
|
||||
"audio": audio_samples,
|
||||
},
|
||||
stream_extractor=lambda sampled: (sampled["video"], sampled["audio"]),
|
||||
clone_model_fn=lambda source, **_kwargs: source,
|
||||
)
|
||||
|
||||
assert captured["latent"]["video"] is video["samples"]
|
||||
assert captured["latent"]["audio"] is audio["samples"]
|
||||
assert captured["seed"] == derive_refine_seed(17, 0)
|
||||
|
||||
|
||||
def test_complete_refine_context_skips_prompt_encode_and_clones_each_group_model():
|
||||
groups = [
|
||||
_group(0, temporal=31, prompt="unused first prompt"),
|
||||
_group(1, temporal=65, prompt="unused terminal prompt", terminal=True),
|
||||
]
|
||||
videos = [
|
||||
_latent(torch.zeros((1, 24, group["source_latent_t"], 48, 48)))
|
||||
for group in groups
|
||||
]
|
||||
audios = [
|
||||
_latent(torch.ones(tuple(group["source_audio_shape"]))) for group in groups
|
||||
]
|
||||
context_groups = (
|
||||
{
|
||||
"group_id": 0,
|
||||
"logical_chunks": (1,),
|
||||
"physical_clip_index": 4,
|
||||
"context_frames": 0,
|
||||
},
|
||||
{
|
||||
"group_id": 1,
|
||||
"logical_chunks": (2, 3),
|
||||
"physical_clip_index": 5,
|
||||
"context_frames": 22,
|
||||
},
|
||||
)
|
||||
refine_context = {
|
||||
"magic": REFINE_CONTEXT_MAGIC,
|
||||
"complete": True,
|
||||
"groups": context_groups,
|
||||
}
|
||||
adapted_calls = []
|
||||
clone_calls = []
|
||||
sample_calls = []
|
||||
video_vae = object()
|
||||
|
||||
def unexpected_prompt_encode(*_args, **_kwargs):
|
||||
raise AssertionError("complete refine_context must bypass prompt encoding")
|
||||
|
||||
def validate_context(value, *, assembly_plan):
|
||||
assert value is refine_context
|
||||
assert assembly_plan is plan
|
||||
return value
|
||||
|
||||
def adapt(group, **kwargs):
|
||||
adapted_calls.append((group, kwargs))
|
||||
return [f"captured-{group['group_id']}"], {
|
||||
"keyframes_resized": group["group_id"],
|
||||
"keyframes_reencoded": 0,
|
||||
"context_refs_resized": 0,
|
||||
"warnings": (),
|
||||
}
|
||||
|
||||
def clone(source, **kwargs):
|
||||
clone_calls.append((source, kwargs))
|
||||
return f"group-model-{kwargs['chunk_index']}"
|
||||
|
||||
def sample(**kwargs):
|
||||
sample_calls.append(kwargs)
|
||||
return kwargs["latent"]
|
||||
|
||||
plan = _plan(groups)
|
||||
refined, output_audio, updated, report = run_second_pass_groups(
|
||||
model="source-model",
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=videos,
|
||||
audio_latents=audios,
|
||||
assembly_plan=plan,
|
||||
refine_seed=42,
|
||||
refine_context=refine_context,
|
||||
video_vae=video_vae,
|
||||
encode_prompt_fn=unexpected_prompt_encode,
|
||||
latent_builder=lambda video, audio: {"video": video, "audio": audio},
|
||||
sample_fn=sample,
|
||||
stream_extractor=lambda value: (value["video"], value["audio"]),
|
||||
clone_model_fn=clone,
|
||||
validate_refine_context_fn=validate_context,
|
||||
adapt_group_conditioning_fn=adapt,
|
||||
)
|
||||
|
||||
assert [call[0] for call in adapted_calls] == list(context_groups)
|
||||
assert all(call[1] == {
|
||||
"target_latent_h": 48,
|
||||
"target_latent_w": 48,
|
||||
"video_vae": video_vae,
|
||||
"upscale_method": "bilinear",
|
||||
} for call in adapted_calls)
|
||||
assert [call[1] for call in clone_calls] == [
|
||||
{
|
||||
"strict": False,
|
||||
"debug": False,
|
||||
"chunk_index": 4,
|
||||
"context_frames": None,
|
||||
},
|
||||
{
|
||||
"strict": False,
|
||||
"debug": False,
|
||||
"chunk_index": 5,
|
||||
"context_frames": 22,
|
||||
},
|
||||
]
|
||||
assert [call["model"] for call in sample_calls] == [
|
||||
"group-model-4",
|
||||
"group-model-5",
|
||||
]
|
||||
assert [call["conditioning"] for call in sample_calls] == [
|
||||
["captured-0"],
|
||||
["captured-1"],
|
||||
]
|
||||
assert all(before is after for before, after in zip(audios, output_audio, strict=True))
|
||||
assert all(
|
||||
item["samples"] is before["samples"]
|
||||
for item, before in zip(refined, videos, strict=True)
|
||||
)
|
||||
assert updated["second_pass_contract"]["execution"] == (
|
||||
"context_aware_physical_groups_audio_locked_v3"
|
||||
)
|
||||
assert "conditioning_source=refine_context" in report
|
||||
|
||||
|
||||
def test_unadaptable_complete_context_warns_and_uses_prompt_only_fallback():
|
||||
group = _group(0, temporal=31, prompt="fallback prompt")
|
||||
video = _latent(torch.zeros((1, 24, 31, 48, 48)))
|
||||
audio = _latent(torch.ones(tuple(group["source_audio_shape"])))
|
||||
refine_context = {
|
||||
"magic": REFINE_CONTEXT_MAGIC,
|
||||
"complete": True,
|
||||
"groups": ({"group_id": 0, "logical_chunks": (1,)},),
|
||||
}
|
||||
prompts = []
|
||||
|
||||
def unavailable(*_args, **_kwargs):
|
||||
raise RefineConditioningAdaptationError("target conversion unavailable")
|
||||
|
||||
_refined, output_audio, updated, report = run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=[video],
|
||||
audio_latents=[audio],
|
||||
assembly_plan=_plan([group]),
|
||||
refine_seed=5,
|
||||
refine_context=refine_context,
|
||||
encode_prompt_fn=lambda _clip, prompt, **_kwargs: prompts.append(prompt) or [prompt],
|
||||
latent_builder=lambda video_samples, audio_samples: {
|
||||
"video": video_samples,
|
||||
"audio": audio_samples,
|
||||
},
|
||||
sample_fn=lambda **kwargs: kwargs["latent"],
|
||||
stream_extractor=lambda value: (value["video"], value["audio"]),
|
||||
clone_model_fn=lambda source, **_kwargs: source,
|
||||
validate_refine_context_fn=lambda value, **_kwargs: value,
|
||||
adapt_group_conditioning_fn=unavailable,
|
||||
)
|
||||
|
||||
assert prompts == ["fallback prompt"]
|
||||
assert output_audio[0] is audio
|
||||
assert updated["second_pass_contract"]["execution"] == (
|
||||
"t2va_physical_groups_audio_locked_v2"
|
||||
)
|
||||
assert "target adaptation was unavailable" in report
|
||||
assert "conditioning_source=prompt_only_fallback" in report
|
||||
|
||||
|
||||
def test_incomplete_refine_context_warns_and_preserves_legacy_prompt_path():
|
||||
group = _group(0, temporal=31, prompt="legacy prompt")
|
||||
video = _latent(torch.zeros((1, 24, 31, 48, 48)))
|
||||
audio = _latent(torch.ones(tuple(group["source_audio_shape"])))
|
||||
incomplete_context = {
|
||||
"magic": REFINE_CONTEXT_MAGIC,
|
||||
"complete": False,
|
||||
"groups": (),
|
||||
}
|
||||
prompts = []
|
||||
|
||||
def unexpected_adaptation(*_args, **_kwargs):
|
||||
raise AssertionError("incomplete context must not be adapted")
|
||||
|
||||
_refined, output_audio, updated, report = run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=[video],
|
||||
audio_latents=[audio],
|
||||
assembly_plan=_plan([group]),
|
||||
refine_seed=5,
|
||||
refine_context=incomplete_context,
|
||||
encode_prompt_fn=lambda _clip, prompt, **_kwargs: prompts.append(prompt) or [prompt],
|
||||
latent_builder=lambda video_samples, audio_samples: {
|
||||
"video": video_samples,
|
||||
"audio": audio_samples,
|
||||
},
|
||||
sample_fn=lambda **kwargs: kwargs["latent"],
|
||||
stream_extractor=lambda value: (value["video"], value["audio"]),
|
||||
clone_model_fn=lambda source, **_kwargs: source,
|
||||
validate_refine_context_fn=lambda value, **_kwargs: value,
|
||||
adapt_group_conditioning_fn=unexpected_adaptation,
|
||||
)
|
||||
|
||||
assert prompts == ["legacy prompt"]
|
||||
assert output_audio[0] is audio
|
||||
assert updated["second_pass_contract"]["execution"] == (
|
||||
"t2va_physical_groups_audio_locked_v2"
|
||||
)
|
||||
assert "capture is incomplete" in report
|
||||
|
||||
|
||||
def test_corrupt_typed_refine_context_is_a_contract_error():
|
||||
group = _group(0, temporal=31, prompt="prompt")
|
||||
with pytest.raises(SecondPassContractError, match="refine_context contract"):
|
||||
run_second_pass_groups(
|
||||
model=object(),
|
||||
clip=object(),
|
||||
sampler=object(),
|
||||
sigmas=torch.tensor([0.3, 0.0]),
|
||||
video_latents=[_latent(torch.zeros((1, 24, 31, 48, 48)))],
|
||||
audio_latents=[_latent(torch.ones(tuple(group["source_audio_shape"])))],
|
||||
assembly_plan=_plan([group]),
|
||||
refine_seed=5,
|
||||
refine_context={"magic": REFINE_CONTEXT_MAGIC, "version": 999},
|
||||
)
|
||||
|
||||
|
||||
def test_experimental_node_unwraps_single_inputs_but_preserves_physical_lists(monkeypatch):
|
||||
captured = {}
|
||||
sentinel = (["video"], ["audio"], {"plan": True}, "status")
|
||||
|
||||
def fake_run(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(
|
||||
"ComfyUI_H3_Continuum_Join.v3.second_pass_nodes.run_second_pass_groups",
|
||||
fake_run,
|
||||
)
|
||||
node = H3ContinuumSecondPassV35()
|
||||
videos = [{"samples": torch.zeros((1, 24, 31, 40, 40))}]
|
||||
audios = [{"samples": torch.zeros((1, 8, 124))}]
|
||||
result = node.refine(
|
||||
["model"],
|
||||
["clip"],
|
||||
["sampler"],
|
||||
[torch.tensor([0.3, 0.0])],
|
||||
videos,
|
||||
audios,
|
||||
[{"second_pass_contract": {}}],
|
||||
[7],
|
||||
)
|
||||
assert result == sentinel
|
||||
assert captured["video_latents"] is videos
|
||||
assert captured["audio_latents"] is audios
|
||||
assert captured["refine_seed"] == 7
|
||||
assert captured["refine_context"] is None
|
||||
assert captured["video_vae"] is None
|
||||
|
||||
|
||||
def test_second_pass_node_forwards_optional_refine_context_and_video_vae(monkeypatch):
|
||||
captured = {}
|
||||
sentinel = ([], [], {"plan": True}, "status")
|
||||
|
||||
def fake_run(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr(
|
||||
"ComfyUI_H3_Continuum_Join.v3.second_pass_nodes.run_second_pass_groups",
|
||||
fake_run,
|
||||
)
|
||||
context = {"context": True}
|
||||
video_vae = object()
|
||||
schedule = object()
|
||||
result = H3ContinuumSecondPassV35().refine(
|
||||
["model"],
|
||||
["clip"],
|
||||
["sampler"],
|
||||
[torch.tensor([0.3, 0.0])],
|
||||
[{"samples": torch.zeros((1, 24, 31, 40, 40))}],
|
||||
[{"samples": torch.zeros((1, 8, 124))}],
|
||||
[{"second_pass_contract": {}}],
|
||||
[7],
|
||||
[context],
|
||||
[video_vae],
|
||||
schedule,
|
||||
)
|
||||
|
||||
assert result == sentinel
|
||||
assert captured["refine_context"] is context
|
||||
assert captured["video_vae"] is video_vae
|
||||
assert captured["refine_schedule"] is schedule
|
||||
|
||||
|
||||
def test_second_pass_node_is_registered_and_not_deprecated():
|
||||
node_class = NODE_CLASS_MAPPINGS["H3ContinuumSecondPassV35"]
|
||||
assert node_class is H3ContinuumSecondPassV35
|
||||
assert node_class.DEPRECATED is False
|
||||
assert node_class.INPUT_IS_LIST is True
|
||||
assert node_class.OUTPUT_IS_LIST == (True, True, False, False)
|
||||
assert node_class.INPUT_TYPES()["optional"] == {
|
||||
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
|
||||
"video_vae": ("VAE",),
|
||||
}
|
||||
Reference in New Issue
Block a user