feat: expose Reference Audio in V3.5 sampler

This commit is contained in:
ukr8b3g-cmyk
2026-08-24 17:30:43 +09:00
parent 3830cd66f6
commit a704aa961f
5 changed files with 340 additions and 40 deletions
+65
View File
@@ -277,6 +277,15 @@ class _VideoVAE:
self.upscale_ratio = 16
class _AudioVAE:
def __init__(self):
self.first_stage_model = torch.nn.Linear(2, 2)
self.patcher = _Patcher(self.first_stage_model)
self.vae_dtype = torch.float32
self.audio_sample_rate = 32000
self.latent_channels = 32
def _sampling_contract(
*, model=None, clip=None, video_vae=None, sampler=None, reroll_nonce=0,
conditioning_mode="i2va", first_frame_hash="a" * 64,
@@ -284,6 +293,8 @@ def _sampling_contract(
upstream_graph_contract=None, upstream_graph_safe=None,
upstream_graph_reasons=None,
execution_semantics=None,
reference_audio_contract=None,
reference_audio_vae=None,
):
return build_sampling_contract(
model=model or _Model(),
@@ -314,9 +325,63 @@ def _sampling_contract(
upstream_graph_safe=upstream_graph_safe,
upstream_graph_reasons=upstream_graph_reasons,
execution_semantics=execution_semantics,
reference_audio_contract=reference_audio_contract,
reference_audio_vae=reference_audio_vae,
)
def test_reference_audio_revision_separates_changes_and_restores_on_disconnect():
model = _Model()
clip = _Clip()
sampler = _Sampler()
audio_vae = _AudioVAE()
common = {
"model": model,
"clip": clip,
"sampler": sampler,
"conditioning_mode": "t2va",
"first_frame_hash": "",
}
disconnected, safe, reasons = _sampling_contract(
**common,
reference_audio_vae=audio_vae,
)
assert safe, reasons
assert "reference_audio" not in disconnected["global"]
assert "reference_audio_vae" not in disconnected["global"]
first_contract = {
"reference_audio_contract_version": 1,
"source_sha256": "a" * 64,
"resolved_vae_sample_rate": 32000,
}
first, safe, reasons = _sampling_contract(
**common,
reference_audio_contract=first_contract,
reference_audio_vae=audio_vae,
)
assert safe, reasons
assert first["global"]["reference_audio"] == first_contract
assert "reference_audio_vae" in first["global"]
changed_contract = dict(first_contract, source_sha256="b" * 64)
changed, safe, reasons = _sampling_contract(
**common,
reference_audio_contract=changed_contract,
reference_audio_vae=audio_vae,
)
assert safe, reasons
restored, safe, reasons = _sampling_contract(
**common,
reference_audio_vae=audio_vae,
)
assert safe, reasons
assert revision_identity(disconnected) != revision_identity(first)
assert revision_identity(first) != revision_identity(changed)
assert revision_identity(restored) == revision_identity(disconnected)
def test_terminal_seed_v2_revision_is_scoped_and_distinct():
old_contract, old_safe, old_reasons = _sampling_contract(
conditioning_mode="fl2va",
+154
View File
@@ -0,0 +1,154 @@
from __future__ import annotations
import inspect
import json
from pathlib import Path
import pytest
import torch
from ComfyUI_H3_Continuum_Join.reference_audio import (
ReferenceAudioError,
prepare_reference_audio_source,
)
from ComfyUI_H3_Continuum_Join.run_storage import automatic_project_key
from ComfyUI_H3_Continuum_Join.v3 import driving_nodes
from ComfyUI_H3_Continuum_Join.v3.driving_nodes import (
H3ContinuumSamplerV34,
H3ContinuumSamplerV35,
)
from ComfyUI_H3_Continuum_Join.v3.nodes import H3ContinuumSamplerProduction
ROOT = Path(__file__).resolve().parents[1]
def test_v35_appends_reference_audio_sockets_without_changing_v34_schema():
v34_optional = H3ContinuumSamplerV34.INPUT_TYPES()["optional"]
v35_optional = H3ContinuumSamplerV35.INPUT_TYPES()["optional"]
assert "reference_audio_1" not in v34_optional
assert "reference_audio_vae" not in v34_optional
assert list(v35_optional)[:-2] == list(v34_optional)
assert list(v35_optional)[-2:] == ["reference_audio_1", "reference_audio_vae"]
assert v35_optional["reference_audio_1"][0] == "AUDIO"
assert v35_optional["reference_audio_1"][1]["display_name"] == (
"Reference Audio"
)
assert v35_optional["reference_audio_vae"][0] == "VAE"
assert v35_optional["reference_audio_vae"][1]["display_name"] == (
"Reference Audio VAE"
)
assert "Driving Audio" in v35_optional["reference_audio_1"][1]["tooltip"]
def test_v34_internal_reference_audio_parameters_are_appended_and_forwarded(
monkeypatch,
):
parameters = list(inspect.signature(H3ContinuumSamplerV34.run).parameters)
assert parameters[-4:] == [
"capture_refine_context",
"reference_audio_1",
"reference_audio_vae",
"kwargs",
]
captured = {}
reference_audio = object()
reference_audio_vae = object()
monkeypatch.setattr(
driving_nodes,
"prepare_driving_audio_source",
lambda *_args, **_kwargs: None,
)
monkeypatch.setattr(
"ComfyUI_H3_Continuum_Join.reference_video.prepare_reference_video_source",
lambda *_args, **_kwargs: None,
)
def production_run(_self, **kwargs):
captured.update(kwargs)
return [], [], {}, "status"
monkeypatch.setattr(H3ContinuumSamplerProduction, "run", production_run)
outputs = H3ContinuumSamplerV34().run(
reference_audio_1=reference_audio,
reference_audio_vae=reference_audio_vae,
chunks=1,
chunk_seconds=5.0,
width=640,
height=640,
)
assert captured["reference_audio_1"] is reference_audio
assert captured["reference_audio_vae"] is reference_audio_vae
assert outputs[-1] is None
def test_v35_run_forwards_reference_audio_without_duplicate_preparation(monkeypatch):
captured = {}
reference_audio = object()
reference_audio_vae = object()
def v34_run(_self, *args, **kwargs):
captured.update(kwargs)
return [], [], {}, "status", None, {"groups": ()}
monkeypatch.setattr(H3ContinuumSamplerV34, "run", v34_run)
H3ContinuumSamplerV35().run(
reference_audio_1=reference_audio,
reference_audio_vae=reference_audio_vae,
)
assert captured["reference_audio_1"] is reference_audio
assert captured["reference_audio_vae"] is reference_audio_vae
assert captured["capture_refine_context"] is True
assert captured["memory_attribution"] is True
def test_reference_audio_vae_is_ignored_without_audio_and_required_with_audio():
vae = object()
assert prepare_reference_audio_source(None, vae) is None
audio = {
"waveform": torch.zeros((1, 2, 32000), dtype=torch.float32),
"sample_rate": 32000,
}
with pytest.raises(ReferenceAudioError, match="reference_audio_vae is required"):
prepare_reference_audio_source(audio, None)
def test_unused_reference_audio_vae_does_not_change_automatic_project_key():
base = {
"1": {"class_type": "H3ContinuumSamplerV35", "inputs": {}},
"2": {"class_type": "VAELoader", "inputs": {}},
}
vae_only = {
**base,
"1": {
"class_type": "H3ContinuumSamplerV35",
"inputs": {"reference_audio_vae": ["2", 0]},
},
}
assert automatic_project_key(base, "1") == automatic_project_key(vae_only, "1")
def test_published_v35_template_keeps_existing_sampler_socket_routes():
template = ROOT / "examples/workflows/MiniMax_H3_Continuum_V35.json"
if not template.exists():
pytest.skip("published workflow template is not installed in this runtime copy")
workflow = json.loads(
template.read_text(encoding="utf-8")
)
sampler = next(
node for node in workflow["nodes"] if node["type"] == "H3ContinuumSamplerV35"
)
routes = {item["name"]: item.get("link") for item in sampler["inputs"]}
assert routes["reference_image_1"] == 321
assert routes["reference_image_2"] == 320
assert routes["reference_image_3"] == 318
assert routes["reference_video_1"] == 322
assert routes["driving_audio"] == 294
assert routes["audio_vae"] == 295
assert "reference_audio_1" not in routes
assert "reference_audio_vae" not in routes
+51
View File
@@ -69,6 +69,7 @@ def _capture_sequence(
monkeypatch,
*,
terminal: bool,
reference_audio: bool = False,
session=None,
diagnostics=DIAGNOSTICS_BASIC,
memory_attribution=False,
@@ -101,6 +102,27 @@ def _capture_sequence(
"clone_model_for_chunk",
lambda model, **_kwargs: model,
)
reference_audio_source = None
reference_audio_vae = None
if reference_audio:
from ComfyUI_H3_Continuum_Join import reference_audio as reference_audio_module
reference_audio_source = SimpleNamespace(
combined_hash="reference-audio-hash",
contract={
"reference_audio_contract_version": 1,
"source_sha256": "a" * 64,
},
)
reference_audio_vae = object()
reference_audio_assets = SimpleNamespace(
audio_latent=torch.arange(64, dtype=torch.float32).reshape(1, 32, 2, 1)
)
monkeypatch.setattr(
reference_audio_module,
"encode_reference_audio",
lambda _vae, _source: reference_audio_assets,
)
chunks = 3 if terminal else 2
mode = PROMPT_MODE_LIST if terminal else PROMPT_MODE_FIXED
script = "first\n---\nmiddle\n---\nlast" if terminal else "same prompt"
@@ -133,6 +155,8 @@ def _capture_sequence(
debug=False,
session=session,
latent_only=True,
reference_audio_source=reference_audio_source,
reference_audio_vae=reference_audio_vae,
capture_refine_context=True,
memory_attribution=memory_attribution,
)
@@ -275,6 +299,33 @@ def test_capture_records_terminal_pair_as_one_physical_group(monkeypatch):
assert validate_refine_context(context, assembly_plan) is context
def test_reference_audio_is_captured_in_every_normal_physical_group(monkeypatch):
outputs = _capture_sequence(
monkeypatch,
terminal=False,
reference_audio=True,
)
groups = outputs[4]["groups"]
assert [group["logical_chunks"] for group in groups] == [(1,), (2,)]
for group in groups:
refs = group["conditioning"][0][1]["minimax_refs"]
assert [ref["kind"] for ref in refs].count("audio") == 1
def test_reference_audio_is_captured_in_terminal_physical_group(monkeypatch):
outputs = _capture_sequence(
monkeypatch,
terminal=True,
reference_audio=True,
)
groups = outputs[4]["groups"]
assert [group["logical_chunks"] for group in groups] == [(1,), (2, 3)]
assert groups[1]["prompt_policy"] == "paired_timeline_v1"
for group in groups:
refs = group["conditioning"][0][1]["minimax_refs"]
assert [ref["kind"] for ref in refs].count("audio") == 1
def test_reused_session_returns_incomplete_context_without_stopping(monkeypatch):
first_outputs = _capture_sequence(monkeypatch, terminal=False)
reused_outputs = _capture_sequence(