git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
193 lines
6.0 KiB
Python
193 lines
6.0 KiB
Python
"""Core-aligned Driving Audio preparation for H3 Continuum V3.4."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import json
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from .audio_compat import resample_audio
|
|
|
|
|
|
DRIVING_AUDIO_CONTRACT_VERSION = 1
|
|
DRIVING_AUDIO_PREPROCESS_VERSION = 1
|
|
H3_AUDIO_LATENT_FPS = 40
|
|
|
|
|
|
def _tensor_sha256(value: torch.Tensor) -> str:
|
|
tensor = value.detach().to("cpu").contiguous()
|
|
digest = hashlib.sha256()
|
|
digest.update(str(tuple(int(v) for v in tensor.shape)).encode("ascii"))
|
|
digest.update(str(tensor.dtype).encode("ascii"))
|
|
digest.update(tensor.numpy().tobytes(order="C"))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _canonical_sha256(value: Any) -> str:
|
|
payload = json.dumps(
|
|
value,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=True,
|
|
).encode("utf-8")
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DrivingAudioSource:
|
|
source_audio: dict[str, Any]
|
|
resampled_waveform: torch.Tensor
|
|
resolved_vae_sample_rate: int
|
|
contract: dict[str, Any]
|
|
combined_hash: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DrivingAudioAssets:
|
|
source: DrivingAudioSource
|
|
audio_latent: torch.Tensor
|
|
|
|
|
|
def slice_review_source_audio(audio: dict[str, Any] | None, assembly_plan):
|
|
"""Cut the selected full-source PCM once; do not alter the plan or source."""
|
|
from .v3.plan import validate_review_audio_projection
|
|
from .media import validate_audio
|
|
projection = validate_review_audio_projection(assembly_plan)
|
|
if audio is None or projection is None:
|
|
return audio
|
|
waveform, sample_rate = validate_audio(audio)
|
|
samples = int(waveform.shape[-1])
|
|
start = min(samples, round(projection["source_start_frame"] * sample_rate / projection["fps"]))
|
|
stop = min(samples, round(projection["source_stop_frame"] * sample_rate / projection["fps"]))
|
|
return {**audio, "waveform": waveform[..., start:stop], "sample_rate": sample_rate}
|
|
|
|
|
|
def prepare_driving_audio_source(
|
|
audio: dict[str, Any] | None,
|
|
audio_vae: Any,
|
|
*,
|
|
target_frames: int,
|
|
fps: int,
|
|
) -> DrivingAudioSource | None:
|
|
"""Prepare only the effective target-duration prefix; never pad a short source."""
|
|
if audio is None:
|
|
return None
|
|
if audio_vae is None:
|
|
raise ValueError("anchoring guide audio needs the audio_vae input")
|
|
|
|
waveform = audio["waveform"]
|
|
sample_rate = int(audio["sample_rate"])
|
|
if not isinstance(waveform, torch.Tensor):
|
|
raise TypeError("AUDIO waveform must be a torch.Tensor")
|
|
|
|
target_samples = max(0, round(int(target_frames) * sample_rate / int(fps)))
|
|
effective_waveform = waveform[..., : min(int(waveform.shape[-1]), target_samples)]
|
|
source_audio = {
|
|
"waveform": effective_waveform,
|
|
"sample_rate": sample_rate,
|
|
}
|
|
|
|
resolved_rate = int(getattr(audio_vae, "audio_sample_rate", 32000))
|
|
resampled = effective_waveform
|
|
if sample_rate != resolved_rate:
|
|
resampled = resample_audio(
|
|
effective_waveform,
|
|
sample_rate,
|
|
resolved_rate,
|
|
)
|
|
|
|
contract = {
|
|
"driving_audio_contract_version": DRIVING_AUDIO_CONTRACT_VERSION,
|
|
"source_sample_rate": sample_rate,
|
|
"source_shape": list(effective_waveform.shape),
|
|
"source_dtype": str(effective_waveform.dtype),
|
|
"source_sha256": _tensor_sha256(effective_waveform),
|
|
"resolved_vae_sample_rate": resolved_rate,
|
|
"resampled_shape": list(resampled.shape),
|
|
"resampled_dtype": str(resampled.dtype),
|
|
"resampled_sha256": _tensor_sha256(resampled),
|
|
"preprocess_version": DRIVING_AUDIO_PREPROCESS_VERSION,
|
|
}
|
|
return DrivingAudioSource(
|
|
source_audio=source_audio,
|
|
resampled_waveform=resampled,
|
|
resolved_vae_sample_rate=resolved_rate,
|
|
contract=contract,
|
|
combined_hash=_canonical_sha256(contract),
|
|
)
|
|
|
|
|
|
def encode_driving_audio(
|
|
source: DrivingAudioSource | None,
|
|
audio_vae: Any,
|
|
) -> DrivingAudioAssets | None:
|
|
if source is None:
|
|
return None
|
|
latent = audio_vae.encode(source.resampled_waveform[:1].movedim(1, -1))
|
|
return DrivingAudioAssets(source=source, audio_latent=latent)
|
|
|
|
|
|
def slice_driving_audio_latent(
|
|
assets: DrivingAudioAssets | None,
|
|
*,
|
|
cumulative_retained_before: int,
|
|
total_frames: int,
|
|
trim_frames: int,
|
|
fps: int,
|
|
) -> torch.Tensor | None:
|
|
if assets is None:
|
|
return None
|
|
source_start_frame = max(
|
|
0,
|
|
int(cumulative_retained_before) - int(trim_frames),
|
|
)
|
|
source_end_frame = source_start_frame + int(total_frames)
|
|
latent_start = round(source_start_frame * H3_AUDIO_LATENT_FPS / int(fps))
|
|
latent_end = round(source_end_frame * H3_AUDIO_LATENT_FPS / int(fps))
|
|
latent_length = int(assets.audio_latent.shape[-1])
|
|
latent_start = min(max(0, latent_start), latent_length)
|
|
latent_end = min(max(latent_start, latent_end), latent_length)
|
|
if latent_end <= latent_start:
|
|
return None
|
|
return assets.audio_latent[..., latent_start:latent_end].clone()
|
|
|
|
|
|
def combine_driving_audio_identity(
|
|
visual_identity_hash: str,
|
|
source: DrivingAudioSource | None,
|
|
) -> str:
|
|
if source is None:
|
|
return visual_identity_hash
|
|
return _canonical_sha256(
|
|
{
|
|
"driving_audio_identity_version": 1,
|
|
"visual_identity_hash": visual_identity_hash,
|
|
"driving_audio_hash": source.combined_hash,
|
|
}
|
|
)
|
|
|
|
|
|
def attach_driving_audio(
|
|
conditioning: list,
|
|
audio_latent: torch.Tensor | None,
|
|
) -> list:
|
|
"""Attach a Core-compatible guide-audio keyframe without tokenizer refs."""
|
|
if audio_latent is None:
|
|
return conditioning
|
|
output = []
|
|
for tokens, metadata in conditioning:
|
|
copied = dict(metadata)
|
|
keyframes = list(copied.get("minimax_keyframes", []))
|
|
keyframes.append(
|
|
{
|
|
"resolved_frame_index": 0,
|
|
"audio_latent": audio_latent,
|
|
}
|
|
)
|
|
copied["minimax_keyframes"] = keyframes
|
|
output.append([tokens, copied])
|
|
return output
|