chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: cache director audio sources and trim timeline scans

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-09 19:58:15 +00:00
parent b9c935466f
commit 58469ccb4b
2 changed files with 135 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
ltx_director.py
View File

@@ -174,6 +174,17 @@ def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) ->
return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
def _safe_json_loads(raw_value, default=None):
if isinstance(raw_value, dict):
return raw_value
if not raw_value:
return {} if default is None else default
try:
return json.loads(raw_value)
except Exception:
return {} if default is None else default
def _read_audio_bytes(container, stream, resampler) -> bytes:
audio_bytes = bytearray()
for frame in container.decode(stream):
@@ -210,6 +221,54 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
return None, None
def _get_audio_segment_source(seg: dict, override_audio: bool = False) -> tuple[str | None, str | _io.BytesIO | None]:
file_key = "videoFile" if override_audio else "audioFile"
source_path = seg.get(file_key)
if source_path:
file_path = _resolve_input_path(source_path)
if file_path:
return f"file:{file_path}", file_path
if not override_audio and seg.get("audioB64"):
b64 = seg.get("audioB64")
if "," in b64:
b64 = b64.split(",", 1)[1]
try:
audio_bytes = base64.b64decode(b64)
cache_key = f"b64:{len(b64)}:{b64[:64]}"
return cache_key, _io.BytesIO(audio_bytes)
except Exception:
return None, None
return None, None
def _decode_audio_waveform(buffer, target_sr: int) -> torch.Tensor | None:
clip_arrays = []
with av.open(buffer) as container:
if not container.streams.audio:
return None
stream = container.streams.audio[0]
resampler = av.AudioResampler(
format="fltp",
layout="stereo",
rate=target_sr,
)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
clip_arrays.append(resampled_frame.to_ndarray())
for resampled_frame in resampler.resample(None):
clip_arrays.append(resampled_frame.to_ndarray())
if not clip_arrays:
return None
waveform_np = clip_arrays[0] if len(clip_arrays) == 1 else np.concatenate(clip_arrays, axis=1)
return torch.from_numpy(waveform_np)
def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int,
total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]:
total_sec = total_frames / frame_rate
@@ -905,13 +964,10 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
return empty_audio
try:
if isinstance(timeline_data_value, str):
data = json.loads(timeline_data_value)
else:
data = timeline_data_value
data = _safe_json_loads(timeline_data_value)
is_retake = data.get("retakeMode", False)
if is_retake and data.get("retakeVideo"):
retake_vid = data.get("retakeVideo")
retake_vid = data.get("retakeVideo")
audio_segs = [{
"videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"),
"audioFile": retake_vid.get("imageFile") or retake_vid.get("fileName"),
@@ -929,68 +985,29 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
if not audio_segs:
return empty_audio
out_waveform = torch.zeros((2, total_samples), dtype=torch.float32)
for seg in audio_segs:
buffer = None
file_key = "videoFile" if override_audio else "audioFile"
if seg.get(file_key):
file_path = _resolve_input_path(seg[file_key])
if file_path:
buffer = file_path
if not override_audio and not buffer and seg.get("audioB64"):
b64 = seg.get("audioB64")
if "," in b64:
b64 = b64.split(",", 1)[1]
try:
audio_bytes = base64.b64decode(b64)
buffer = _io.BytesIO(audio_bytes)
except:
pass
if not buffer:
continue
try:
clip_arrays = []
# Use PyAV to decode directly from memory buffer
with av.open(buffer) as container:
if not container.streams.audio:
continue
stream = container.streams.audio[0]
# Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar
resampler = av.AudioResampler(
format='fltp',
layout='stereo',
rate=target_sr,
)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
# to_ndarray() on fltp gives shape (channels, samples)
arr = resampled_frame.to_ndarray()
clip_arrays.append(arr)
# Flush the resampler to get any remaining samples
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
clip_arrays.append(arr)
if not clip_arrays:
out_waveform = torch.zeros((2, total_samples), dtype=torch.float32)
decoded_waveform_cache = {}
for seg in audio_segs:
cache_key, buffer = _get_audio_segment_source(seg, override_audio=override_audio)
if not buffer:
continue
try:
waveform = decoded_waveform_cache.get(cache_key)
if waveform is None:
waveform = _decode_audio_waveform(buffer, target_sr)
if waveform is None:
continue
if cache_key:
decoded_waveform_cache[cache_key] = waveform
if waveform.shape[1] <= 0:
continue
if len(clip_arrays) == 1:
waveform_np = clip_arrays[0]
else:
waveform_np = np.concatenate(clip_arrays, axis=1)
waveform = torch.from_numpy(waveform_np) # Shape: [2, total_clip_samples]
# Calculate interactive trim boundaries
trim_start_frames = float(seg.get("trimStart", 0))
# Calculate interactive trim boundaries
trim_start_frames = float(seg.get("trimStart", 0))
length_frames = float(seg.get("length", 1))
start_frames = float(seg.get("start", 0))
@@ -1778,10 +1795,10 @@ class LTXDirector(io.ComfyNode):
# Parse timeline data
try:
tdata = json.loads(timeline_data) if timeline_data else {}
tdata = _safe_json_loads(timeline_data)
except Exception as e:
log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
tdata = {}
log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
tdata = {}
# --- Automation overrides (connection-only inputs, in SECONDS) ---
# When wired, these take precedence over the panel/timeline values.