perf: cache director audio sources and trim timeline scans

js/ltx_director.js

@@ -56,6 +56,12 @@ function dedupeById(items = []) {
   return items.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id));
 }
 
+function findActiveSegmentAtFrame(segments = [], targetFrame, type) {
+  return (segments || []).find(
+    (seg) => seg?.type === type && targetFrame >= seg.start && targetFrame < seg.start + seg.length
+  );
+}
+
 function hideWidget(w) {
   if (!w) return;
 
@@ -1502,7 +1508,7 @@ class TimelineEditor {
       return;
     }
 
-    const seg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length);
+    const seg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video");
     if (seg) {
       this._ensureVideoEl(seg);
       if (seg.videoEl) {
@@ -1511,7 +1517,7 @@ class TimelineEditor {
       }
     }
 
-    const motionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length);
+    const motionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video");
     if (motionSeg) {
       this._ensureVideoEl(motionSeg);
       if (motionSeg.videoEl) {
@@ -1604,7 +1610,7 @@ class TimelineEditor {
 
         for (let i = 0; i < numFrames; i++) {
           // Check if the file/segment is still active in the current timeline
-          const exists = this._getSegmentsByFileKey(fileKey).length > 0;
+          const exists = this._hasMatchingSegment(fileKey);
           if (!exists) break;
 
           const time = (i / numFrames) * duration;
@@ -1625,10 +1631,9 @@ class TimelineEditor {
           thumbs.push({ time, img });
 
           // Propagate the partial progress live to all active segments sharing this file
-          const matchingSegs = this._getSegmentsByFileKey(fileKey);
+          this._forEachMatchingSegment(fileKey, (ms) => {
-          for (const ms of matchingSegs) {
             ms.thumbnails = thumbs;
-          }
+          });
 
           this.render();
         }
@@ -1653,8 +1658,7 @@ class TimelineEditor {
       const thumbs = await extractPromise;
       this._thumbnailCache.set(fileKey, thumbs);
 
-      const matchingSegs = this._getSegmentsByFileKey(fileKey);
+      this._forEachMatchingSegment(fileKey, (ms) => {
-      for (const ms of matchingSegs) {
         ms.thumbnails = thumbs;
         ms._extractingThumbs = false;
 
@@ -1665,13 +1669,12 @@ class TimelineEditor {
             this._thumbnailCache.set(serverKey, thumbs);
           }
         }
-      }
+      });
     } catch (err) {
       console.error("Extraction error:", err);
-      const matchingSegs = this._getSegmentsByFileKey(fileKey);
+      this._forEachMatchingSegment(fileKey, (ms) => {
-      for (const ms of matchingSegs) {
         ms._extractingThumbs = false;
-      }
+      });
     } finally {
       this._thumbnailPromises.delete(fileKey);
       this.render();
@@ -1978,14 +1981,34 @@ class TimelineEditor {
     return !!seg && (seg.imageFile === fileKey || seg.videoFile === fileKey || seg._blobUrl === fileKey);
   }
 
-  _getSegmentsByFileKey(fileKey) {
+  _forEachMatchingSegment(fileKey, visitor) {
-    const matches = [
+    const visitList = (segments) => {
-      ...(this.timeline.segments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)),
+      for (const seg of segments || []) {
-      ...(this.timeline.motionSegments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)),
+        if (this._segmentMatchesFileKey(seg, fileKey)) visitor(seg);
-    ];
+      }
+    };
+
+    visitList(this.timeline.segments);
+    visitList(this.timeline.motionSegments);
+
     if (this._segmentMatchesFileKey(this.timeline.retakeVideo, fileKey)) {
-      matches.push(this.timeline.retakeVideo);
+      visitor(this.timeline.retakeVideo);
     }
+  }
+
+  _hasMatchingSegment(fileKey) {
+    let found = false;
+    this._forEachMatchingSegment(fileKey, () => {
+      found = true;
+    });
+    return found;
+  }
+
+  _getSegmentsByFileKey(fileKey) {
+    const matches = [];
+    this._forEachMatchingSegment(fileKey, (seg) => {
+      matches.push(seg);
+    });
     return matches;
   }
 
@@ -5932,11 +5955,11 @@ class TimelineEditor {
     if (this.retakeMode && this.timeline.retakeVideo) {
       this._ensureVideoEl(this.timeline.retakeVideo);
     } else {
-      const activeSeg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length);
+      const activeSeg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video");
       if (activeSeg) this._ensureVideoEl(activeSeg);
 
       if (this.timeline.motionSegments) {
-        const activeMotionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length);
+        const activeMotionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video");
         if (activeMotionSeg) this._ensureVideoEl(activeMotionSeg);
       }
     }

ltx_director.py

@@ -174,6 +174,17 @@ def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) ->
     return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
 
 
+def _safe_json_loads(raw_value, default=None):
+    if isinstance(raw_value, dict):
+        return raw_value
+    if not raw_value:
+        return {} if default is None else default
+    try:
+        return json.loads(raw_value)
+    except Exception:
+        return {} if default is None else default
+
+
 def _read_audio_bytes(container, stream, resampler) -> bytes:
     audio_bytes = bytearray()
     for frame in container.decode(stream):
@@ -210,6 +221,54 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
         return None, None
 
 
+def _get_audio_segment_source(seg: dict, override_audio: bool = False) -> tuple[str | None, str | _io.BytesIO | None]:
+    file_key = "videoFile" if override_audio else "audioFile"
+    source_path = seg.get(file_key)
+    if source_path:
+        file_path = _resolve_input_path(source_path)
+        if file_path:
+            return f"file:{file_path}", file_path
+
+    if not override_audio and seg.get("audioB64"):
+        b64 = seg.get("audioB64")
+        if "," in b64:
+            b64 = b64.split(",", 1)[1]
+        try:
+            audio_bytes = base64.b64decode(b64)
+            cache_key = f"b64:{len(b64)}:{b64[:64]}"
+            return cache_key, _io.BytesIO(audio_bytes)
+        except Exception:
+            return None, None
+
+    return None, None
+
+
+def _decode_audio_waveform(buffer, target_sr: int) -> torch.Tensor | None:
+    clip_arrays = []
+    with av.open(buffer) as container:
+        if not container.streams.audio:
+            return None
+        stream = container.streams.audio[0]
+        resampler = av.AudioResampler(
+            format="fltp",
+            layout="stereo",
+            rate=target_sr,
+        )
+
+        for frame in container.decode(stream):
+            for resampled_frame in resampler.resample(frame):
+                clip_arrays.append(resampled_frame.to_ndarray())
+
+        for resampled_frame in resampler.resample(None):
+            clip_arrays.append(resampled_frame.to_ndarray())
+
+    if not clip_arrays:
+        return None
+
+    waveform_np = clip_arrays[0] if len(clip_arrays) == 1 else np.concatenate(clip_arrays, axis=1)
+    return torch.from_numpy(waveform_np)
+
+
 def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int,
                                   total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]:
     total_sec = total_frames / frame_rate
@@ -905,10 +964,7 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
         return empty_audio
 
     try:
-        if isinstance(timeline_data_value, str):
+        data = _safe_json_loads(timeline_data_value)
-            data = json.loads(timeline_data_value)
-        else:
-            data = timeline_data_value
         is_retake = data.get("retakeMode", False)
         if is_retake and data.get("retakeVideo"):
             retake_vid = data.get("retakeVideo")
@@ -931,64 +987,25 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
         return empty_audio
 
     out_waveform = torch.zeros((2, total_samples), dtype=torch.float32)
+    decoded_waveform_cache = {}
 
     for seg in audio_segs:
-        buffer = None
+        cache_key, buffer = _get_audio_segment_source(seg, override_audio=override_audio)
-        file_key = "videoFile" if override_audio else "audioFile"
-        if seg.get(file_key):
-            file_path = _resolve_input_path(seg[file_key])
-            if file_path:
-                buffer = file_path
-        
-        if not override_audio and not buffer and seg.get("audioB64"):
-            b64 = seg.get("audioB64")
-            if "," in b64:
-                b64 = b64.split(",", 1)[1]
-            try:
-                audio_bytes = base64.b64decode(b64)
-                buffer = _io.BytesIO(audio_bytes)
-            except:
-                pass
-                
         if not buffer:
             continue
 
         try:
-            clip_arrays = []
+            waveform = decoded_waveform_cache.get(cache_key)
-            
+            if waveform is None:
-            # Use PyAV to decode directly from memory buffer
+                waveform = _decode_audio_waveform(buffer, target_sr)
-            with av.open(buffer) as container:
+                if waveform is None:
-                if not container.streams.audio:
                     continue
-                stream = container.streams.audio[0]
+                if cache_key:
+                    decoded_waveform_cache[cache_key] = waveform
 
-                # Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar
+            if waveform.shape[1] <= 0:
-                resampler = av.AudioResampler(
-                    format='fltp',
-                    layout='stereo',
-                    rate=target_sr,
-                )
-                
-                for frame in container.decode(stream):
-                    for resampled_frame in resampler.resample(frame):
-                        # to_ndarray() on fltp gives shape (channels, samples)
-                        arr = resampled_frame.to_ndarray()
-                        clip_arrays.append(arr)
-                
-                # Flush the resampler to get any remaining samples
-                for resampled_frame in resampler.resample(None):
-                    arr = resampled_frame.to_ndarray()
-                    clip_arrays.append(arr)
-
-            if not clip_arrays:
                 continue
 
-            if len(clip_arrays) == 1:
-                waveform_np = clip_arrays[0]
-            else:
-                waveform_np = np.concatenate(clip_arrays, axis=1)
-            waveform = torch.from_numpy(waveform_np)  # Shape: [2, total_clip_samples]
-
             # Calculate interactive trim boundaries
             trim_start_frames = float(seg.get("trimStart", 0))
             length_frames = float(seg.get("length", 1))
@@ -1778,7 +1795,7 @@ class LTXDirector(io.ComfyNode):
 
         # Parse timeline data
         try:
-            tdata = json.loads(timeline_data) if timeline_data else {}
+            tdata = _safe_json_loads(timeline_data)
         except Exception as e:
             log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
             tdata = {}