chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: cache director audio sources and trim timeline scans

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-09 19:58:15 +00:00
parent b9c935466f
commit 58469ccb4b
2 changed files with 135 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
js/ltx_director.js
View File

@@ -55,6 +55,12 @@ function stripAudioSegmentTransient(s) {
function dedupeById(items = []) {
return items.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id));
}
function findActiveSegmentAtFrame(segments = [], targetFrame, type) {
return (segments || []).find(
(seg) => seg?.type === type && targetFrame >= seg.start && targetFrame < seg.start + seg.length
);
}
function hideWidget(w) {
if (!w) return;
@@ -1502,7 +1508,7 @@ class TimelineEditor {
return;
}
const seg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length);
const seg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video");
if (seg) {
this._ensureVideoEl(seg);
if (seg.videoEl) {
@@ -1511,7 +1517,7 @@ class TimelineEditor {
}
}
const motionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length);
const motionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video");
if (motionSeg) {
this._ensureVideoEl(motionSeg);
if (motionSeg.videoEl) {
@@ -1604,7 +1610,7 @@ class TimelineEditor {
for (let i = 0; i < numFrames; i++) {
// Check if the file/segment is still active in the current timeline
const exists = this._getSegmentsByFileKey(fileKey).length > 0;
const exists = this._hasMatchingSegment(fileKey);
if (!exists) break;
const time = (i / numFrames) * duration;
@@ -1625,10 +1631,9 @@ class TimelineEditor {
thumbs.push({ time, img });
// Propagate the partial progress live to all active segments sharing this file
const matchingSegs = this._getSegmentsByFileKey(fileKey);
for (const ms of matchingSegs) {
this._forEachMatchingSegment(fileKey, (ms) => {
ms.thumbnails = thumbs;
}
});
this.render();
}
@@ -1653,26 +1658,24 @@ class TimelineEditor {
const thumbs = await extractPromise;
this._thumbnailCache.set(fileKey, thumbs);
const matchingSegs = this._getSegmentsByFileKey(fileKey);
for (const ms of matchingSegs) {
this._forEachMatchingSegment(fileKey, (ms) => {
ms.thumbnails = thumbs;
ms._extractingThumbs = false;
// If fileKey is a blob URL, and the segment now has a server file path, cache under that path too
if (fileKey.startsWith("blob:")) {
// If fileKey is a blob URL, and the segment now has a server file path, cache under that path too
if (fileKey.startsWith("blob:")) {
const serverKey = ms.imageFile || ms.videoFile;
if (serverKey) {
this._thumbnailCache.set(serverKey, thumbs);
}
}
}
if (serverKey) {
this._thumbnailCache.set(serverKey, thumbs);
}
}
});
} catch (err) {
console.error("Extraction error:", err);
const matchingSegs = this._getSegmentsByFileKey(fileKey);
for (const ms of matchingSegs) {
this._forEachMatchingSegment(fileKey, (ms) => {
ms._extractingThumbs = false;
}
} finally {
});
} finally {
this._thumbnailPromises.delete(fileKey);
this.render();
}
@@ -1978,14 +1981,34 @@ class TimelineEditor {
return !!seg && (seg.imageFile === fileKey || seg.videoFile === fileKey || seg._blobUrl === fileKey);
}
_getSegmentsByFileKey(fileKey) {
const matches = [
...(this.timeline.segments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)),
...(this.timeline.motionSegments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)),
];
_forEachMatchingSegment(fileKey, visitor) {
const visitList = (segments) => {
for (const seg of segments || []) {
if (this._segmentMatchesFileKey(seg, fileKey)) visitor(seg);
}
};
visitList(this.timeline.segments);
visitList(this.timeline.motionSegments);
if (this._segmentMatchesFileKey(this.timeline.retakeVideo, fileKey)) {
matches.push(this.timeline.retakeVideo);
visitor(this.timeline.retakeVideo);
}
}
_hasMatchingSegment(fileKey) {
let found = false;
this._forEachMatchingSegment(fileKey, () => {
found = true;
});
return found;
}
_getSegmentsByFileKey(fileKey) {
const matches = [];
this._forEachMatchingSegment(fileKey, (seg) => {
matches.push(seg);
});
return matches;
}
@@ -5932,11 +5955,11 @@ class TimelineEditor {
if (this.retakeMode && this.timeline.retakeVideo) {
this._ensureVideoEl(this.timeline.retakeVideo);
} else {
const activeSeg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length);
const activeSeg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video");
if (activeSeg) this._ensureVideoEl(activeSeg);
if (this.timeline.motionSegments) {
const activeMotionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length);
const activeMotionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video");
if (activeMotionSeg) this._ensureVideoEl(activeMotionSeg);
}
}

151
ltx_director.py
View File

@@ -174,6 +174,17 @@ def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) ->
return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
def _safe_json_loads(raw_value, default=None):
if isinstance(raw_value, dict):
return raw_value
if not raw_value:
return {} if default is None else default
try:
return json.loads(raw_value)
except Exception:
return {} if default is None else default
def _read_audio_bytes(container, stream, resampler) -> bytes:
audio_bytes = bytearray()
for frame in container.decode(stream):
@@ -210,6 +221,54 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
return None, None
def _get_audio_segment_source(seg: dict, override_audio: bool = False) -> tuple[str | None, str | _io.BytesIO | None]:
file_key = "videoFile" if override_audio else "audioFile"
source_path = seg.get(file_key)
if source_path:
file_path = _resolve_input_path(source_path)
if file_path:
return f"file:{file_path}", file_path
if not override_audio and seg.get("audioB64"):
b64 = seg.get("audioB64")
if "," in b64:
b64 = b64.split(",", 1)[1]
try:
audio_bytes = base64.b64decode(b64)
cache_key = f"b64:{len(b64)}:{b64[:64]}"
return cache_key, _io.BytesIO(audio_bytes)
except Exception:
return None, None
return None, None
def _decode_audio_waveform(buffer, target_sr: int) -> torch.Tensor | None:
clip_arrays = []
with av.open(buffer) as container:
if not container.streams.audio:
return None
stream = container.streams.audio[0]
resampler = av.AudioResampler(
format="fltp",
layout="stereo",
rate=target_sr,
)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
clip_arrays.append(resampled_frame.to_ndarray())
for resampled_frame in resampler.resample(None):
clip_arrays.append(resampled_frame.to_ndarray())
if not clip_arrays:
return None
waveform_np = clip_arrays[0] if len(clip_arrays) == 1 else np.concatenate(clip_arrays, axis=1)
return torch.from_numpy(waveform_np)
def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int,
total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]:
total_sec = total_frames / frame_rate
@@ -905,13 +964,10 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
return empty_audio
try:
if isinstance(timeline_data_value, str):
data = json.loads(timeline_data_value)
else:
data = timeline_data_value
data = _safe_json_loads(timeline_data_value)
is_retake = data.get("retakeMode", False)
if is_retake and data.get("retakeVideo"):
retake_vid = data.get("retakeVideo")
retake_vid = data.get("retakeVideo")
audio_segs = [{
"videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"),
"audioFile": retake_vid.get("imageFile") or retake_vid.get("fileName"),
@@ -929,68 +985,29 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
if not audio_segs:
return empty_audio
out_waveform = torch.zeros((2, total_samples), dtype=torch.float32)
for seg in audio_segs:
buffer = None
file_key = "videoFile" if override_audio else "audioFile"
if seg.get(file_key):
file_path = _resolve_input_path(seg[file_key])
if file_path:
buffer = file_path
if not override_audio and not buffer and seg.get("audioB64"):
b64 = seg.get("audioB64")
if "," in b64:
b64 = b64.split(",", 1)[1]
try:
audio_bytes = base64.b64decode(b64)
buffer = _io.BytesIO(audio_bytes)
except:
pass
if not buffer:
continue
try:
clip_arrays = []
# Use PyAV to decode directly from memory buffer
with av.open(buffer) as container:
if not container.streams.audio:
continue
stream = container.streams.audio[0]
# Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar
resampler = av.AudioResampler(
format='fltp',
layout='stereo',
rate=target_sr,
)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
# to_ndarray() on fltp gives shape (channels, samples)
arr = resampled_frame.to_ndarray()
clip_arrays.append(arr)
# Flush the resampler to get any remaining samples
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
clip_arrays.append(arr)
if not clip_arrays:
out_waveform = torch.zeros((2, total_samples), dtype=torch.float32)
decoded_waveform_cache = {}
for seg in audio_segs:
cache_key, buffer = _get_audio_segment_source(seg, override_audio=override_audio)
if not buffer:
continue
try:
waveform = decoded_waveform_cache.get(cache_key)
if waveform is None:
waveform = _decode_audio_waveform(buffer, target_sr)
if waveform is None:
continue
if cache_key:
decoded_waveform_cache[cache_key] = waveform
if waveform.shape[1] <= 0:
continue
if len(clip_arrays) == 1:
waveform_np = clip_arrays[0]
else:
waveform_np = np.concatenate(clip_arrays, axis=1)
waveform = torch.from_numpy(waveform_np) # Shape: [2, total_clip_samples]
# Calculate interactive trim boundaries
trim_start_frames = float(seg.get("trimStart", 0))
# Calculate interactive trim boundaries
trim_start_frames = float(seg.get("trimStart", 0))
length_frames = float(seg.get("length", 1))
start_frames = float(seg.get("start", 0))
@@ -1778,10 +1795,10 @@ class LTXDirector(io.ComfyNode):
# Parse timeline data
try:
tdata = json.loads(timeline_data) if timeline_data else {}
tdata = _safe_json_loads(timeline_data)
except Exception as e:
log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
tdata = {}
log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
tdata = {}
# --- Automation overrides (connection-only inputs, in SECONDS) ---
# When wired, these take precedence over the panel/timeline values.