"""MiniMax H3 standalone audio-reference adapter for the V3 sampler.""" from __future__ import annotations from dataclasses import dataclass import hashlib import json import re from typing import Any import torch REFERENCE_AUDIO_CONTRACT_VERSION = 1 REFERENCE_AUDIO_PREPROCESS_VERSION = 1 _AUDIO_TAG = re.compile(r"") class ReferenceAudioError(ValueError): pass @dataclass(frozen=True) class ReferenceAudioSource: waveform: torch.Tensor source_sample_rate: int source_shape: tuple[int, ...] source_dtype: str source_sha256: str resolved_vae_sample_rate: int resampled_shape: tuple[int, ...] resampled_dtype: str resampled_sha256: str combined_hash: str @property def contract(self) -> dict[str, Any]: return { "reference_audio_contract_version": REFERENCE_AUDIO_CONTRACT_VERSION, "source_sample_rate": self.source_sample_rate, "source_shape": list(self.source_shape), "source_dtype": self.source_dtype, "source_sha256": self.source_sha256, "resolved_vae_sample_rate": self.resolved_vae_sample_rate, "resampled_shape": list(self.resampled_shape), "resampled_dtype": self.resampled_dtype, "resampled_sha256": self.resampled_sha256, "preprocess_version": REFERENCE_AUDIO_PREPROCESS_VERSION, "combined_hash": self.combined_hash, } @dataclass(frozen=True) class ReferenceAudioAssets: source: ReferenceAudioSource audio_latent: torch.Tensor @property def combined_hash(self) -> str: return self.source.combined_hash def _canonical_hash(value: Any) -> str: payload = json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=True ).encode("utf-8") return hashlib.sha256(payload).hexdigest() def _tensor_hash(value: torch.Tensor) -> str: tensor = value.detach().to(device="cpu").contiguous() digest = hashlib.sha256() digest.update(str(tuple(int(v) for v in tensor.shape)).encode("ascii")) digest.update(str(tensor.dtype).encode("ascii")) digest.update(tensor.view(torch.uint8).numpy().tobytes(order="C")) return digest.hexdigest() def _validate_waveform(name: str, waveform: Any) -> torch.Tensor: if not torch.is_tensor(waveform) or waveform.ndim != 3: raise ReferenceAudioError(f"{name} waveform must have shape [B, C, L]") if int(waveform.shape[0]) != 1: raise ReferenceAudioError(f"{name} waveform batch size must be exactly 1") if int(waveform.shape[1]) < 1 or int(waveform.shape[2]) < 1: raise ReferenceAudioError(f"{name} waveform must contain channels and samples") if not waveform.is_floating_point(): raise ReferenceAudioError(f"{name} waveform must use a floating-point dtype") if not bool(torch.isfinite(waveform).all().item()): raise ReferenceAudioError(f"{name} waveform contains NaN or infinity") return waveform.detach().to(device="cpu").contiguous() def prepare_reference_audio_source( reference_audio_1: Any, reference_audio_vae: Any, ) -> ReferenceAudioSource | None: if reference_audio_1 is None: return None if reference_audio_vae is None: raise ReferenceAudioError( "reference_audio_vae is required when reference_audio_1 is connected" ) if not isinstance(reference_audio_1, dict): raise ReferenceAudioError("Reference Audio 1 must be a ComfyUI AUDIO value") waveform = _validate_waveform( "Reference Audio 1", reference_audio_1.get("waveform") ) sample_rate_value = reference_audio_1.get("sample_rate") if isinstance(sample_rate_value, bool): raise ReferenceAudioError("Reference Audio 1 sample_rate must be positive") try: source_sample_rate = int(sample_rate_value) except (TypeError, ValueError) as exc: raise ReferenceAudioError( "Reference Audio 1 sample_rate must be positive" ) from exc if source_sample_rate <= 0: raise ReferenceAudioError("Reference Audio 1 sample_rate must be positive") resolved_rate = int(getattr(reference_audio_vae, "audio_sample_rate", 32000)) if resolved_rate <= 0: raise ReferenceAudioError("Reference Audio VAE sample rate must be positive") resampled = waveform if source_sample_rate != resolved_rate: try: import torchaudio resampled = torchaudio.functional.resample( waveform, source_sample_rate, resolved_rate ).contiguous() except Exception as exc: raise ReferenceAudioError( "Reference Audio 1 could not be resampled for the Audio VAE" ) from exc if not bool(torch.isfinite(resampled).all().item()): raise ReferenceAudioError( "Reference Audio 1 contains NaN or infinity after resampling" ) source_shape = tuple(int(value) for value in waveform.shape) resampled_shape = tuple(int(value) for value in resampled.shape) source_sha256 = _tensor_hash(waveform) resampled_sha256 = _tensor_hash(resampled) contract_base = { "reference_audio_contract_version": REFERENCE_AUDIO_CONTRACT_VERSION, "source_sample_rate": source_sample_rate, "source_shape": list(source_shape), "source_dtype": str(waveform.dtype), "source_sha256": source_sha256, "resolved_vae_sample_rate": resolved_rate, "resampled_shape": list(resampled_shape), "resampled_dtype": str(resampled.dtype), "resampled_sha256": resampled_sha256, "preprocess_version": REFERENCE_AUDIO_PREPROCESS_VERSION, } return ReferenceAudioSource( waveform=resampled, source_sample_rate=source_sample_rate, source_shape=source_shape, source_dtype=str(waveform.dtype), source_sha256=source_sha256, resolved_vae_sample_rate=resolved_rate, resampled_shape=resampled_shape, resampled_dtype=str(resampled.dtype), resampled_sha256=resampled_sha256, combined_hash=_canonical_hash(contract_base), ) def encode_reference_audio( reference_audio_vae: Any, source: ReferenceAudioSource, ) -> ReferenceAudioAssets: try: latent = reference_audio_vae.encode(source.waveform.movedim(1, -1)) except Exception as exc: raise ReferenceAudioError("Reference Audio VAE Encode failed") from exc if ( not torch.is_tensor(latent) or latent.ndim != 4 or int(latent.shape[0]) != 1 or int(latent.shape[1]) != 32 or int(latent.shape[2]) != 2 or int(latent.shape[3]) < 1 ): raise ReferenceAudioError( "Reference Audio VAE must produce latent shape [1, 32, 2, T]" ) if not bool(torch.isfinite(latent).all().item()): raise ReferenceAudioError("Reference Audio latent contains NaN or infinity") return ReferenceAudioAssets(source=source, audio_latent=latent) def validate_reference_audio_prompts( prompts: list[str], source: ReferenceAudioSource | None ) -> str: found: set[int] = set() for prompt in prompts: found.update(int(value) for value in _AUDIO_TAG.findall(str(prompt))) warnings: list[str] = [] active_count = 1 if source is not None else 0 unavailable = sorted( value for value in found if source is None or value != 1 ) for value in unavailable: warnings.append( f"Warning: prompt references unavailable