chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: dedupe director audio loading helpers

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-09 14:17:30 +00:00
parent 5df6ed6669
commit a1a4be4fc3
1 changed files with 137 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

302
js/ltx_director.js
View File

@@ -1900,21 +1900,11 @@ class TimelineEditor {
_extractAudioOnClient(file, audSegId, blobUrl) { _extractAudioOnClient(file, audSegId, blobUrl) {
(async () => { (async () => {
try { try {
const arrayBuffer = await file.arrayBuffer(); const arrayBuffer = await file.arrayBuffer();
const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
const channelData = audioBuffer.getChannelData(0); const channelData = audioBuffer.getChannelData(0);
const peaks = []; const peaks = this._computeWaveformPeaks(channelData);
const numPeaks = 200;
const step = Math.floor(channelData.length / numPeaks);
for (let i = 0; i < numPeaks; i++) {
let max = 0;
for (let j = 0; j < step; j++) {
const val = Math.abs(channelData[i * step + j]);
if (val > max) max = val;
}
peaks.push(max);
}
for (let s of this.timeline.audioSegments) { for (let s of this.timeline.audioSegments) {
if (s.id === audSegId || (blobUrl && s._blobUrl === blobUrl)) { if (s.id === audSegId || (blobUrl && s._blobUrl === blobUrl)) {
s.waveformPeaks = peaks; s.waveformPeaks = peaks;
@@ -1935,22 +1925,112 @@ class TimelineEditor {
})(); })();
} }
_isAudioDecodingAllowed(seg) { _isAudioDecodingAllowed(seg) {
if (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(wav|mp3|ogg|flac|m4a)$/)) { if (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(wav|mp3|ogg|flac|m4a)$/)) {
return true; return true;
} }
const isVideo = (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/)) || const isVideo = (seg.audioFile && seg.audioFile.toLowerCase().match(/\.(mp4|webm|mkv|avi|mov|m4v|flv|wmv)$/)) ||
(!seg.audioFile && seg._blobUrl); (!seg.audioFile && seg._blobUrl);
if (isVideo) { if (isVideo) {
const isSmall = seg.fileSize && seg.fileSize <= 100 * 1024 * 1024; const isSmall = seg.fileSize && seg.fileSize <= 100 * 1024 * 1024;
return !!isSmall; return !!isSmall;
} }
return true; return true;
} }
async _preloadAudioSegment(seg) { _ensureAudioContext() {
if (seg._audioBuffer || seg._decoding) return; if (!this.audioContext) {
if (!seg.audioFile && !seg._blobUrl) return; this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
}
return this.audioContext;
}
_getAudioCacheKey(seg) {
if (seg.audioFile) return seg.audioFile;
if (seg._blobUrl) return seg._blobUrl;
if (seg.audioB64) return `b64:${seg.audioB64.length}:${seg.audioB64.slice(0, 64)}`;
return null;
}
_getAudioUrl(seg) {
if (seg._blobUrl) return seg._blobUrl;
const parts = (seg.audioFile || "").split(/[/\\\\]/);
const filename = parts.pop() || "";
const subfolder = parts.join("/");
return api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`);
}
async _decodeBase64Audio(audioB64) {
const binaryString = window.atob(audioB64);
const len = binaryString.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return await this._ensureAudioContext().decodeAudioData(bytes.buffer);
}
_computeWaveformPeaks(channelData, numPeaks = 200) {
if (!channelData || !channelData.length) return Array(numPeaks).fill(0);
const step = Math.max(1, Math.floor(channelData.length / numPeaks));
const peaks = [];
for (let i = 0; i < numPeaks; i++) {
let max = 0;
for (let j = 0; j < step; j++) {
const idx = i * step + j;
if (idx >= channelData.length) break;
const val = Math.abs(channelData[idx]);
if (val > max) max = val;
}
peaks.push(max);
}
return peaks;
}
async _loadAudioBuffer(seg) {
if (!seg.audioFile && !seg._blobUrl && !seg.audioB64) return null;
const audioContext = this._ensureAudioContext();
this._audioBufferCache = this._audioBufferCache || new Map();
this._audioBufferPromises = this._audioBufferPromises || new Map();
const cacheKey = this._getAudioCacheKey(seg);
if (cacheKey && this._audioBufferCache.has(cacheKey)) {
return this._audioBufferCache.get(cacheKey);
}
if (cacheKey && this._audioBufferPromises.has(cacheKey)) {
return await this._audioBufferPromises.get(cacheKey);
}
const decodePromise = (async () => {
if (seg.audioB64 && !seg.audioFile && !seg._blobUrl) {
return await this._decodeBase64Audio(seg.audioB64);
}
const resp = await fetch(this._getAudioUrl(seg));
const arrayBuffer = await resp.arrayBuffer();
return await audioContext.decodeAudioData(arrayBuffer);
})();
if (cacheKey) {
this._audioBufferPromises.set(cacheKey, decodePromise);
}
try {
const audioBuffer = await decodePromise;
if (cacheKey) {
this._audioBufferCache.set(cacheKey, audioBuffer);
}
return audioBuffer;
} finally {
if (cacheKey) {
this._audioBufferPromises.delete(cacheKey);
}
}
}
async _preloadAudioSegment(seg) {
if (seg._audioBuffer || seg._decoding) return;
if (!seg.audioFile && !seg._blobUrl) return;
seg._decoding = true; seg._decoding = true;
if (!this._isDragging) this.render(); if (!this._isDragging) this.render();
@@ -1958,46 +2038,15 @@ class TimelineEditor {
try { try {
await this._getOrExtractAudio(seg); await this._getOrExtractAudio(seg);
if (!this._isAudioDecodingAllowed(seg)) { if (!this._isAudioDecodingAllowed(seg)) {
seg._decoding = false; seg._decoding = false;
return; return;
} }
if (!this.audioContext) { const audioBuffer = await this._loadAudioBuffer(seg);
this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
} const matchingSegs = this.timeline.audioSegments.filter(s => s.audioFile === seg.audioFile || s._blobUrl === seg._blobUrl);
for (const s of matchingSegs) {
const parts = (seg.audioFile || "").split(/[/\\\\]/);
const filename = parts.pop() || '';
const subfolder = parts.join('/');
const audioUrl = seg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`);
this._audioBufferCache = this._audioBufferCache || new Map();
this._audioBufferPromises = this._audioBufferPromises || new Map();
const cacheKey = seg.audioFile || audioUrl;
let audioBuffer;
if (this._audioBufferCache.has(cacheKey)) {
audioBuffer = this._audioBufferCache.get(cacheKey);
} else if (this._audioBufferPromises.has(cacheKey)) {
audioBuffer = await this._audioBufferPromises.get(cacheKey);
} else {
const decodePromise = (async () => {
const resp = await fetch(audioUrl);
const arrayBuffer = await resp.arrayBuffer();
return await this.audioContext.decodeAudioData(arrayBuffer);
})();
this._audioBufferPromises.set(cacheKey, decodePromise);
try {
audioBuffer = await decodePromise;
this._audioBufferCache.set(cacheKey, audioBuffer);
} finally {
this._audioBufferPromises.delete(cacheKey);
}
}
const matchingSegs = this.timeline.audioSegments.filter(s => s.audioFile === seg.audioFile || s._blobUrl === seg._blobUrl);
for (const s of matchingSegs) {
s._audioBuffer = audioBuffer; s._audioBuffer = audioBuffer;
s._decoding = false; s._decoding = false;
} }
@@ -2025,46 +2074,15 @@ class TimelineEditor {
await this._getOrExtractAudio(mockSeg); await this._getOrExtractAudio(mockSeg);
if (!this._isAudioDecodingAllowed(mockSeg)) { if (!this._isAudioDecodingAllowed(mockSeg)) {
seg._decodingAudio = false; seg._decodingAudio = false;
return; return;
} }
if (!this.audioContext) { const audioBuffer = await this._loadAudioBuffer(mockSeg);
this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); seg._audioBuffer = audioBuffer;
} } catch (e) {
console.warn("Failed to preload motion audio segment:", e);
const parts = (mockSeg.audioFile || "").split(/[/\\\\]/);
const filename = parts.pop() || '';
const subfolder = parts.join('/');
const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`);
this._audioBufferCache = this._audioBufferCache || new Map();
this._audioBufferPromises = this._audioBufferPromises || new Map();
const cacheKey = mockSeg.audioFile || audioUrl;
let audioBuffer;
if (this._audioBufferCache.has(cacheKey)) {
audioBuffer = this._audioBufferCache.get(cacheKey);
} else if (this._audioBufferPromises.has(cacheKey)) {
audioBuffer = await this._audioBufferPromises.get(cacheKey);
} else {
const decodePromise = (async () => {
const resp = await fetch(audioUrl);
const arrayBuffer = await resp.arrayBuffer();
return await this.audioContext.decodeAudioData(arrayBuffer);
})();
this._audioBufferPromises.set(cacheKey, decodePromise);
try {
audioBuffer = await decodePromise;
this._audioBufferCache.set(cacheKey, audioBuffer);
} finally {
this._audioBufferPromises.delete(cacheKey);
}
}
seg._audioBuffer = audioBuffer;
} catch (e) {
console.warn("Failed to preload motion audio segment:", e);
} finally { } finally {
seg._decodingAudio = false; seg._decodingAudio = false;
} }
@@ -4687,21 +4705,11 @@ class TimelineEditor {
const arrayBuffer = await file.arrayBuffer(); const arrayBuffer = await file.arrayBuffer();
const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
const clipDurationSecs = audioBuffer.duration; const clipDurationSecs = audioBuffer.duration;
const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate));
const channelData = audioBuffer.getChannelData(0); const channelData = audioBuffer.getChannelData(0);
const peaks = []; const peaks = this._computeWaveformPeaks(channelData);
const numPeaks = 200;
const step = Math.floor(channelData.length / numPeaks);
for (let i = 0; i < numPeaks; i++) {
let max = 0;
for (let j = 0; j < step; j++) {
const val = Math.abs(channelData[i * step + j]);
if (val > max) max = val;
}
peaks.push(max);
}
let newLength = clipFrames; let newLength = clipFrames;
let newStart = targetFrameStart; let newStart = targetFrameStart;
@@ -11471,51 +11479,15 @@ class TimelineEditor {
return; return;
} }
// Build audio buffer // Build audio buffer
let audioBuffer = item.originalSeg._audioBuffer; let audioBuffer = item.originalSeg._audioBuffer;
if (!audioBuffer) { if (!audioBuffer) {
if (mockSeg.audioFile || mockSeg._blobUrl) { if (!mockSeg.audioFile && !mockSeg._blobUrl && !mockSeg.audioB64) {
const parts = (mockSeg.audioFile || "").split(/[/\\\\]/); return;
const filename = parts.pop() || ''; }
const subfolder = parts.join('/'); audioBuffer = await this._loadAudioBuffer(mockSeg);
const audioUrl = mockSeg._blobUrl || api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); item.originalSeg._audioBuffer = audioBuffer;
}
this._audioBufferCache = this._audioBufferCache || new Map();
this._audioBufferPromises = this._audioBufferPromises || new Map();
const cacheKey = mockSeg.audioFile || audioUrl;
if (this._audioBufferCache.has(cacheKey)) {
audioBuffer = this._audioBufferCache.get(cacheKey);
} else if (this._audioBufferPromises.has(cacheKey)) {
audioBuffer = await this._audioBufferPromises.get(cacheKey);
} else {
const decodePromise = (async () => {
const resp = await fetch(audioUrl);
const arrayBuffer = await resp.arrayBuffer();
return await this.audioContext.decodeAudioData(arrayBuffer);
})();
this._audioBufferPromises.set(cacheKey, decodePromise);
try {
audioBuffer = await decodePromise;
this._audioBufferCache.set(cacheKey, audioBuffer);
} finally {
this._audioBufferPromises.delete(cacheKey);
}
}
item.originalSeg._audioBuffer = audioBuffer;
} else if (mockSeg.audioB64) {
const binaryString = window.atob(mockSeg.audioB64);
const len = binaryString.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
audioBuffer = await this.audioContext.decodeAudioData(bytes.buffer);
item.originalSeg._audioBuffer = audioBuffer;
} else {
return;
}
}
if (this._currentPlayId !== playId || !this.isPlaying) return; if (this._currentPlayId !== playId || !this.isPlaying) return;