diff --git a/js/ltx_director.js b/js/ltx_director.js index 380c082..fba2cb9 100644 --- a/js/ltx_director.js +++ b/js/ltx_director.js @@ -55,6 +55,12 @@ function stripAudioSegmentTransient(s) { function dedupeById(items = []) { return items.filter((seg, index, self) => index === self.findIndex((s) => s.id === seg.id)); } + +function findActiveSegmentAtFrame(segments = [], targetFrame, type) { + return (segments || []).find( + (seg) => seg?.type === type && targetFrame >= seg.start && targetFrame < seg.start + seg.length + ); +} function hideWidget(w) { if (!w) return; @@ -1502,7 +1508,7 @@ class TimelineEditor { return; } - const seg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length); + const seg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video"); if (seg) { this._ensureVideoEl(seg); if (seg.videoEl) { @@ -1511,7 +1517,7 @@ class TimelineEditor { } } - const motionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length); + const motionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video"); if (motionSeg) { this._ensureVideoEl(motionSeg); if (motionSeg.videoEl) { @@ -1604,7 +1610,7 @@ class TimelineEditor { for (let i = 0; i < numFrames; i++) { // Check if the file/segment is still active in the current timeline - const exists = this._getSegmentsByFileKey(fileKey).length > 0; + const exists = this._hasMatchingSegment(fileKey); if (!exists) break; const time = (i / numFrames) * duration; @@ -1625,10 +1631,9 @@ class TimelineEditor { thumbs.push({ time, img }); // Propagate the partial progress live to all active segments sharing this file - const matchingSegs = this._getSegmentsByFileKey(fileKey); - for (const ms of matchingSegs) { + this._forEachMatchingSegment(fileKey, (ms) => { ms.thumbnails = thumbs; - } + }); this.render(); } @@ -1653,26 +1658,24 @@ class TimelineEditor { const thumbs = await extractPromise; this._thumbnailCache.set(fileKey, thumbs); - const matchingSegs = this._getSegmentsByFileKey(fileKey); - for (const ms of matchingSegs) { + this._forEachMatchingSegment(fileKey, (ms) => { ms.thumbnails = thumbs; ms._extractingThumbs = false; - - // If fileKey is a blob URL, and the segment now has a server file path, cache under that path too - if (fileKey.startsWith("blob:")) { + + // If fileKey is a blob URL, and the segment now has a server file path, cache under that path too + if (fileKey.startsWith("blob:")) { const serverKey = ms.imageFile || ms.videoFile; - if (serverKey) { - this._thumbnailCache.set(serverKey, thumbs); - } - } - } + if (serverKey) { + this._thumbnailCache.set(serverKey, thumbs); + } + } + }); } catch (err) { console.error("Extraction error:", err); - const matchingSegs = this._getSegmentsByFileKey(fileKey); - for (const ms of matchingSegs) { + this._forEachMatchingSegment(fileKey, (ms) => { ms._extractingThumbs = false; - } - } finally { + }); + } finally { this._thumbnailPromises.delete(fileKey); this.render(); } @@ -1978,14 +1981,34 @@ class TimelineEditor { return !!seg && (seg.imageFile === fileKey || seg.videoFile === fileKey || seg._blobUrl === fileKey); } - _getSegmentsByFileKey(fileKey) { - const matches = [ - ...(this.timeline.segments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)), - ...(this.timeline.motionSegments || []).filter(s => this._segmentMatchesFileKey(s, fileKey)), - ]; + _forEachMatchingSegment(fileKey, visitor) { + const visitList = (segments) => { + for (const seg of segments || []) { + if (this._segmentMatchesFileKey(seg, fileKey)) visitor(seg); + } + }; + + visitList(this.timeline.segments); + visitList(this.timeline.motionSegments); + if (this._segmentMatchesFileKey(this.timeline.retakeVideo, fileKey)) { - matches.push(this.timeline.retakeVideo); + visitor(this.timeline.retakeVideo); } + } + + _hasMatchingSegment(fileKey) { + let found = false; + this._forEachMatchingSegment(fileKey, () => { + found = true; + }); + return found; + } + + _getSegmentsByFileKey(fileKey) { + const matches = []; + this._forEachMatchingSegment(fileKey, (seg) => { + matches.push(seg); + }); return matches; } @@ -5932,11 +5955,11 @@ class TimelineEditor { if (this.retakeMode && this.timeline.retakeVideo) { this._ensureVideoEl(this.timeline.retakeVideo); } else { - const activeSeg = this.timeline.segments.find(s => s.type === "video" && targetFrame >= s.start && targetFrame < s.start + s.length); + const activeSeg = findActiveSegmentAtFrame(this.timeline.segments, targetFrame, "video"); if (activeSeg) this._ensureVideoEl(activeSeg); if (this.timeline.motionSegments) { - const activeMotionSeg = this.timeline.motionSegments.find(s => s.type === "motion_video" && targetFrame >= s.start && targetFrame < s.start + s.length); + const activeMotionSeg = findActiveSegmentAtFrame(this.timeline.motionSegments, targetFrame, "motion_video"); if (activeMotionSeg) this._ensureVideoEl(activeMotionSeg); } } diff --git a/ltx_director.py b/ltx_director.py index 32ba2ce..61392a6 100644 --- a/ltx_director.py +++ b/ltx_director.py @@ -174,6 +174,17 @@ def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist() +def _safe_json_loads(raw_value, default=None): + if isinstance(raw_value, dict): + return raw_value + if not raw_value: + return {} if default is None else default + try: + return json.loads(raw_value) + except Exception: + return {} if default is None else default + + def _read_audio_bytes(container, stream, resampler) -> bytes: audio_bytes = bytearray() for frame in container.decode(stream): @@ -210,6 +221,54 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]: return None, None +def _get_audio_segment_source(seg: dict, override_audio: bool = False) -> tuple[str | None, str | _io.BytesIO | None]: + file_key = "videoFile" if override_audio else "audioFile" + source_path = seg.get(file_key) + if source_path: + file_path = _resolve_input_path(source_path) + if file_path: + return f"file:{file_path}", file_path + + if not override_audio and seg.get("audioB64"): + b64 = seg.get("audioB64") + if "," in b64: + b64 = b64.split(",", 1)[1] + try: + audio_bytes = base64.b64decode(b64) + cache_key = f"b64:{len(b64)}:{b64[:64]}" + return cache_key, _io.BytesIO(audio_bytes) + except Exception: + return None, None + + return None, None + + +def _decode_audio_waveform(buffer, target_sr: int) -> torch.Tensor | None: + clip_arrays = [] + with av.open(buffer) as container: + if not container.streams.audio: + return None + stream = container.streams.audio[0] + resampler = av.AudioResampler( + format="fltp", + layout="stereo", + rate=target_sr, + ) + + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + clip_arrays.append(resampled_frame.to_ndarray()) + + for resampled_frame in resampler.resample(None): + clip_arrays.append(resampled_frame.to_ndarray()) + + if not clip_arrays: + return None + + waveform_np = clip_arrays[0] if len(clip_arrays) == 1 else np.concatenate(clip_arrays, axis=1) + return torch.from_numpy(waveform_np) + + def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int, total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]: total_sec = total_frames / frame_rate @@ -905,13 +964,10 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames return empty_audio try: - if isinstance(timeline_data_value, str): - data = json.loads(timeline_data_value) - else: - data = timeline_data_value + data = _safe_json_loads(timeline_data_value) is_retake = data.get("retakeMode", False) if is_retake and data.get("retakeVideo"): - retake_vid = data.get("retakeVideo") + retake_vid = data.get("retakeVideo") audio_segs = [{ "videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"), "audioFile": retake_vid.get("imageFile") or retake_vid.get("fileName"), @@ -929,68 +985,29 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames if not audio_segs: return empty_audio - - out_waveform = torch.zeros((2, total_samples), dtype=torch.float32) - - for seg in audio_segs: - buffer = None - file_key = "videoFile" if override_audio else "audioFile" - if seg.get(file_key): - file_path = _resolve_input_path(seg[file_key]) - if file_path: - buffer = file_path - - if not override_audio and not buffer and seg.get("audioB64"): - b64 = seg.get("audioB64") - if "," in b64: - b64 = b64.split(",", 1)[1] - try: - audio_bytes = base64.b64decode(b64) - buffer = _io.BytesIO(audio_bytes) - except: - pass - - if not buffer: - continue - - try: - clip_arrays = [] - - # Use PyAV to decode directly from memory buffer - with av.open(buffer) as container: - if not container.streams.audio: - continue - stream = container.streams.audio[0] - - # Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar - resampler = av.AudioResampler( - format='fltp', - layout='stereo', - rate=target_sr, - ) - - for frame in container.decode(stream): - for resampled_frame in resampler.resample(frame): - # to_ndarray() on fltp gives shape (channels, samples) - arr = resampled_frame.to_ndarray() - clip_arrays.append(arr) - - # Flush the resampler to get any remaining samples - for resampled_frame in resampler.resample(None): - arr = resampled_frame.to_ndarray() - clip_arrays.append(arr) - if not clip_arrays: + out_waveform = torch.zeros((2, total_samples), dtype=torch.float32) + decoded_waveform_cache = {} + + for seg in audio_segs: + cache_key, buffer = _get_audio_segment_source(seg, override_audio=override_audio) + if not buffer: + continue + + try: + waveform = decoded_waveform_cache.get(cache_key) + if waveform is None: + waveform = _decode_audio_waveform(buffer, target_sr) + if waveform is None: + continue + if cache_key: + decoded_waveform_cache[cache_key] = waveform + + if waveform.shape[1] <= 0: continue - if len(clip_arrays) == 1: - waveform_np = clip_arrays[0] - else: - waveform_np = np.concatenate(clip_arrays, axis=1) - waveform = torch.from_numpy(waveform_np) # Shape: [2, total_clip_samples] - - # Calculate interactive trim boundaries - trim_start_frames = float(seg.get("trimStart", 0)) + # Calculate interactive trim boundaries + trim_start_frames = float(seg.get("trimStart", 0)) length_frames = float(seg.get("length", 1)) start_frames = float(seg.get("start", 0)) @@ -1778,10 +1795,10 @@ class LTXDirector(io.ComfyNode): # Parse timeline data try: - tdata = json.loads(timeline_data) if timeline_data else {} + tdata = _safe_json_loads(timeline_data) except Exception as e: - log.error(f"[LTXDirector] execute timeline_data parse error: {e}") - tdata = {} + log.error(f"[LTXDirector] execute timeline_data parse error: {e}") + tdata = {} # --- Automation overrides (connection-only inputs, in SECONDS) --- # When wired, these take precedence over the panel/timeline values.