From a1a4be4fc39f6eda49358d81b70d074dc0268d8c Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 9 Jul 2026 14:17:30 +0000 Subject: [PATCH] refactor: dedupe director audio loading helpers --- js/ltx_director.js | 302 ++++++++++++++++++++------------------------- 1 file changed, 137 insertions(+), 165 deletions(-) diff --git a/js/ltx_director.js b/js/ltx_director.js index 4231792..41d7b72 100644 --- a/js/ltx_director.js +++ b/js/ltx_director.js @@ -1900,21 +1900,11 @@ class TimelineEditor { _extractAudioOnClient(file, audSegId, blobUrl) { (async () => { try { - const arrayBuffer = await file.arrayBuffer(); - const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); - const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); - const channelData = audioBuffer.getChannelData(0); - const peaks = []; - const numPeaks = 200; - const step = Math.floor(channelData.length / numPeaks); - for (let i = 0; i < numPeaks; i++) { - let max = 0; - for (let j = 0; j < step; j++) { - const val = Math.abs(channelData[i * step + j]); - if (val > max) max = val; - } - peaks.push(max); - } + const arrayBuffer = await file.arrayBuffer(); + const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); + const channelData = audioBuffer.getChannelData(0); + const peaks = this._computeWaveformPeaks(channelData); for (let s of this.timeline.audioSegments) { if (s.id === audSegId || (blobUrl && s._blobUrl === blobUrl)) { s.waveformPeaks = peaks; @@ -1935,22 +1925,112 @@ class TimelineEditor { })(); } - _isAudioDecodingAllowed(seg) { - if (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(wav|mp3|ogg|flac|m4a)$/)) { - return true; + _isAudioDecodingAllowed(seg) { + if (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(wav|mp3|ogg|flac|m4a)$/)) { + return true; } const isVideo = (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/)) || (!seg.audioFile && seg._blobUrl); if (isVideo) { const isSmall = seg.fileSize && seg.fileSize <= 100 * 1024 * 1024; return !!isSmall; - } - return true; - } - - async _preloadAudioSegment(seg) { - if (seg._audioBuffer || seg._decoding) return; - if (!seg.audioFile && !seg._blobUrl) return; + } + return true; + } + + _ensureAudioContext() { + if (!this.audioContext) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + } + return this.audioContext; + } + + _getAudioCacheKey(seg) { + if (seg.audioFile) return seg.audioFile; + if (seg._blobUrl) return seg._blobUrl; + if (seg.audioB64) return `b64:${seg.audioB64.length}:${seg.audioB64.slice(0, 64)}`; + return null; + } + + _getAudioUrl(seg) { + if (seg._blobUrl) return seg._blobUrl; + const parts = (seg.audioFile || "").split(/[/\\\\]/); + const filename = parts.pop() || ""; + const subfolder = parts.join("/"); + return api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + } + + async _decodeBase64Audio(audioB64) { + const binaryString = window.atob(audioB64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return await this._ensureAudioContext().decodeAudioData(bytes.buffer); + } + + _computeWaveformPeaks(channelData, numPeaks = 200) { + if (!channelData || !channelData.length) return Array(numPeaks).fill(0); + const step = Math.max(1, Math.floor(channelData.length / numPeaks)); + const peaks = []; + for (let i = 0; i < numPeaks; i++) { + let max = 0; + for (let j = 0; j < step; j++) { + const idx = i * step + j; + if (idx >= channelData.length) break; + const val = Math.abs(channelData[idx]); + if (val > max) max = val; + } + peaks.push(max); + } + return peaks; + } + + async _loadAudioBuffer(seg) { + if (!seg.audioFile && !seg._blobUrl && !seg.audioB64) return null; + + const audioContext = this._ensureAudioContext(); + this._audioBufferCache = this._audioBufferCache || new Map(); + this._audioBufferPromises = this._audioBufferPromises || new Map(); + const cacheKey = this._getAudioCacheKey(seg); + + if (cacheKey && this._audioBufferCache.has(cacheKey)) { + return this._audioBufferCache.get(cacheKey); + } + if (cacheKey && this._audioBufferPromises.has(cacheKey)) { + return await this._audioBufferPromises.get(cacheKey); + } + + const decodePromise = (async () => { + if (seg.audioB64 && !seg.audioFile && !seg._blobUrl) { + return await this._decodeBase64Audio(seg.audioB64); + } + const resp = await fetch(this._getAudioUrl(seg)); + const arrayBuffer = await resp.arrayBuffer(); + return await audioContext.decodeAudioData(arrayBuffer); + })(); + + if (cacheKey) { + this._audioBufferPromises.set(cacheKey, decodePromise); + } + + try { + const audioBuffer = await decodePromise; + if (cacheKey) { + this._audioBufferCache.set(cacheKey, audioBuffer); + } + return audioBuffer; + } finally { + if (cacheKey) { + this._audioBufferPromises.delete(cacheKey); + } + } + } + + async _preloadAudioSegment(seg) { + if (seg._audioBuffer || seg._decoding) return; + if (!seg.audioFile && !seg._blobUrl) return; seg._decoding = true; if (!this._isDragging) this.render(); @@ -1958,46 +2038,15 @@ class TimelineEditor { try { await this._getOrExtractAudio(seg); - if (!this._isAudioDecodingAllowed(seg)) { - seg._decoding = false; - return; - } - - if (!this.audioContext) { - this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); - } - - const parts = (seg.audioFile || "").split(/[/\\\\]/); - const filename = parts.pop() || ''; - const subfolder = parts.join('/'); - const audioUrl = seg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); - - this._audioBufferCache = this._audioBufferCache || new Map(); - this._audioBufferPromises = this._audioBufferPromises || new Map(); - const cacheKey = seg.audioFile || audioUrl; - - let audioBuffer; - if (this._audioBufferCache.has(cacheKey)) { - audioBuffer = this._audioBufferCache.get(cacheKey); - } else if (this._audioBufferPromises.has(cacheKey)) { - audioBuffer = await this._audioBufferPromises.get(cacheKey); - } else { - const decodePromise = (async () => { - const resp = await fetch(audioUrl); - const arrayBuffer = await resp.arrayBuffer(); - return await this.audioContext.decodeAudioData(arrayBuffer); - })(); - this._audioBufferPromises.set(cacheKey, decodePromise); - try { - audioBuffer = await decodePromise; - this._audioBufferCache.set(cacheKey, audioBuffer); - } finally { - this._audioBufferPromises.delete(cacheKey); - } - } - - const matchingSegs = this.timeline.audioSegments.filter(s => s.audioFile === seg.audioFile || s._blobUrl === seg._blobUrl); - for (const s of matchingSegs) { + if (!this._isAudioDecodingAllowed(seg)) { + seg._decoding = false; + return; + } + + const audioBuffer = await this._loadAudioBuffer(seg); + + const matchingSegs = this.timeline.audioSegments.filter(s => s.audioFile === seg.audioFile || s._blobUrl === seg._blobUrl); + for (const s of matchingSegs) { s._audioBuffer = audioBuffer; s._decoding = false; } @@ -2025,46 +2074,15 @@ class TimelineEditor { await this._getOrExtractAudio(mockSeg); - if (!this._isAudioDecodingAllowed(mockSeg)) { - seg._decodingAudio = false; - return; - } - - if (!this.audioContext) { - this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); - } - - const parts = (mockSeg.audioFile || "").split(/[/\\\\]/); - const filename = parts.pop() || ''; - const subfolder = parts.join('/'); - const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); - - this._audioBufferCache = this._audioBufferCache || new Map(); - this._audioBufferPromises = this._audioBufferPromises || new Map(); - const cacheKey = mockSeg.audioFile || audioUrl; - - let audioBuffer; - if (this._audioBufferCache.has(cacheKey)) { - audioBuffer = this._audioBufferCache.get(cacheKey); - } else if (this._audioBufferPromises.has(cacheKey)) { - audioBuffer = await this._audioBufferPromises.get(cacheKey); - } else { - const decodePromise = (async () => { - const resp = await fetch(audioUrl); - const arrayBuffer = await resp.arrayBuffer(); - return await this.audioContext.decodeAudioData(arrayBuffer); - })(); - this._audioBufferPromises.set(cacheKey, decodePromise); - try { - audioBuffer = await decodePromise; - this._audioBufferCache.set(cacheKey, audioBuffer); - } finally { - this._audioBufferPromises.delete(cacheKey); - } - } - seg._audioBuffer = audioBuffer; - } catch (e) { - console.warn("Failed to preload motion audio segment:", e); + if (!this._isAudioDecodingAllowed(mockSeg)) { + seg._decodingAudio = false; + return; + } + + const audioBuffer = await this._loadAudioBuffer(mockSeg); + seg._audioBuffer = audioBuffer; + } catch (e) { + console.warn("Failed to preload motion audio segment:", e); } finally { seg._decodingAudio = false; } @@ -4687,21 +4705,11 @@ class TimelineEditor { const arrayBuffer = await file.arrayBuffer(); const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); - const clipDurationSecs = audioBuffer.duration; - const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); - - const channelData = audioBuffer.getChannelData(0); - const peaks = []; - const numPeaks = 200; - const step = Math.floor(channelData.length / numPeaks); - for (let i = 0; i < numPeaks; i++) { - let max = 0; - for (let j = 0; j < step; j++) { - const val = Math.abs(channelData[i * step + j]); - if (val > max) max = val; - } - peaks.push(max); - } + const clipDurationSecs = audioBuffer.duration; + const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); + + const channelData = audioBuffer.getChannelData(0); + const peaks = this._computeWaveformPeaks(channelData); let newLength = clipFrames; let newStart = targetFrameStart; @@ -11471,51 +11479,15 @@ class TimelineEditor { return; } - // Build audio buffer - let audioBuffer = item.originalSeg._audioBuffer; - if (!audioBuffer) { - if (mockSeg.audioFile || mockSeg._blobUrl) { - const parts = (mockSeg.audioFile || "").split(/[/\\\\]/); - const filename = parts.pop() || ''; - const subfolder = parts.join('/'); - const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); - - this._audioBufferCache = this._audioBufferCache || new Map(); - this._audioBufferPromises = this._audioBufferPromises || new Map(); - const cacheKey = mockSeg.audioFile || audioUrl; - - if (this._audioBufferCache.has(cacheKey)) { - audioBuffer = this._audioBufferCache.get(cacheKey); - } else if (this._audioBufferPromises.has(cacheKey)) { - audioBuffer = await this._audioBufferPromises.get(cacheKey); - } else { - const decodePromise = (async () => { - const resp = await fetch(audioUrl); - const arrayBuffer = await resp.arrayBuffer(); - return await this.audioContext.decodeAudioData(arrayBuffer); - })(); - this._audioBufferPromises.set(cacheKey, decodePromise); - try { - audioBuffer = await decodePromise; - this._audioBufferCache.set(cacheKey, audioBuffer); - } finally { - this._audioBufferPromises.delete(cacheKey); - } - } - item.originalSeg._audioBuffer = audioBuffer; - } else if (mockSeg.audioB64) { - const binaryString = window.atob(mockSeg.audioB64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - audioBuffer = await this.audioContext.decodeAudioData(bytes.buffer); - item.originalSeg._audioBuffer = audioBuffer; - } else { - return; - } - } + // Build audio buffer + let audioBuffer = item.originalSeg._audioBuffer; + if (!audioBuffer) { + if (!mockSeg.audioFile && !mockSeg._blobUrl && !mockSeg.audioB64) { + return; + } + audioBuffer = await this._loadAudioBuffer(mockSeg); + item.originalSeg._audioBuffer = audioBuffer; + } if (this._currentPlayId !== playId || !this.isPlaying) return;