chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: cache reused guide and audio work

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-09 13:44:23 +00:00
parent b8cedad997
commit 51bd3b4968
1 changed files with 113 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
ltx_director.py
View File

@@ -210,6 +210,18 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
return None, None
def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int,
total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]:
total_sec = total_frames / frame_rate
start_sec = max(0.0, rel_start_frames - start_frame) / frame_rate
end_sec = max(start_sec, start_sec + (rel_length_frames / frame_rate))
start_idx = int((start_sec / total_sec) * latent_frames)
end_idx = int((end_sec / total_sec) * latent_frames)
start_idx = max(0, min(latent_frames, start_idx))
end_idx = max(0, min(latent_frames, end_idx))
return start_idx, end_idx
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
"""Invisibly swaps out @character1/@char1 tags with their high-fidelity VLM descriptions."""
gp = global_prompt or ""
@@ -942,7 +954,7 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
continue
try:
clip_frames = []
clip_arrays = []
# Use PyAV to decode directly from memory buffer
with av.open(buffer) as container:
@@ -961,18 +973,21 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames
for resampled_frame in resampler.resample(frame):
# to_ndarray() on fltp gives shape (channels, samples)
arr = resampled_frame.to_ndarray()
clip_frames.append(torch.from_numpy(arr))
clip_arrays.append(arr)
# Flush the resampler to get any remaining samples
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
clip_frames.append(torch.from_numpy(arr))
clip_arrays.append(arr)
if not clip_frames:
if not clip_arrays:
continue
# Concatenate all frame blocks along the samples dimension (dim 1)
waveform = torch.cat(clip_frames, dim=1) # Shape: [2, total_clip_samples]
if len(clip_arrays) == 1:
waveform_np = clip_arrays[0]
else:
waveform_np = np.concatenate(clip_arrays, axis=1)
waveform = torch.from_numpy(waveform_np) # Shape: [2, total_clip_samples]
# Calculate interactive trim boundaries
trim_start_frames = float(seg.get("trimStart", 0))
@@ -1296,6 +1311,18 @@ class LTXDirector(io.ComfyNode):
start=None, end=None, duration=None) -> io.NodeOutput:
input_dir = folder_paths.get_input_directory()
image_cache = {}
processed_image_cache = {}
def prepare_tensor_image(tensor: torch.Tensor, width: int, height: int) -> torch.Tensor:
cache_key = (id(tensor), width, height, resize_method, divisible_by, img_compression)
cached = processed_image_cache.get(cache_key)
if cached is not None:
return cached
processed = _resize_image(tensor, width, height, resize_method, divisible_by)
if img_compression > 0:
processed = _compress_image(processed, img_compression)
processed_image_cache[cache_key] = processed
return processed
# Parse timeline data
try:
@@ -1320,6 +1347,7 @@ class LTXDirector(io.ComfyNode):
# Derive the window length from start..end when duration is not explicitly wired.
duration_frames = max(1, end_frame - start_frame)
duration_seconds = duration_frames / _auto_fps
frame_rate_f = float(frame_rate)
is_retake_mode = tdata.get("retakeMode", False)
is_retake_active = is_retake_mode and tdata.get("retakeVideo") is not None
@@ -1408,7 +1436,7 @@ class LTXDirector(io.ComfyNode):
seg_for_load = dict(seg)
seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset
seg_for_load["length"] = max(1, seg_length - offset)
tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir)
tensor = _load_video_tensor(seg_for_load, frame_rate_f, input_dir=input_dir)
else:
tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir)
@@ -1509,6 +1537,8 @@ class LTXDirector(io.ComfyNode):
latent_w = max(32, (derived_w // 32) * 32)
latent_h = max(32, (derived_h // 32) * 32)
latent_grid_h = latent_h // 32
latent_grid_w = latent_w // 32
# Clean (visible) region: latent frames and the matching pixel frames.
clean_latent_frames = ((ltxv_length - 1) // 8) + 1
clean_pixel_frames = int(ltxv_length)
@@ -1546,7 +1576,7 @@ class LTXDirector(io.ComfyNode):
log.info("[LTXDirector] MSR slideshow subjects: referenced slots=%s, images=%d", _referenced_slots, len(_selected))
identity_images = [
_resize_image(c, latent_w, latent_h, resize_method, divisible_by) for c in _selected
prepare_tensor_image(c, latent_w, latent_h) for c in _selected
]
# Scene background = first timeline image (already in guide_data), else black.
@@ -1581,7 +1611,7 @@ class LTXDirector(io.ComfyNode):
injected_lengths = f"{clean_lengths},{tail_pixels}"
dummy_full = {"samples": torch.zeros(
[1, 128, total_latent_frames, latent_h // 32, latent_w // 32], device=_dev,
[1, 128, total_latent_frames, latent_grid_h, latent_grid_w], device=_dev,
)}
patched, conditioning = _encode_relay(
model, clip, dummy_full, global_prompt, injected_local, injected_lengths, epsilon,
@@ -1590,15 +1620,15 @@ class LTXDirector(io.ComfyNode):
# Clean base latent (the true visible region); the guide node pads + appends keyframes per stage.
if optional_latent is None:
latent = {
"samples": torch.zeros([1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev),
"noise_mask": torch.ones((1, 1, clean_latent_frames, latent_h // 32, latent_w // 32), dtype=torch.float32, device=_dev),
"samples": torch.zeros([1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev),
"noise_mask": torch.ones((1, 1, clean_latent_frames, latent_grid_h, latent_grid_w), dtype=torch.float32, device=_dev),
}
else:
latent = optional_latent
# Hand the RAW full-res references to the guide node via guide_data["msr"].
guide_data = {
"images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate),
"images": [], "insert_frames": [], "strengths": [], "frame_rate": frame_rate_f,
"msr": {
"slideshow": slideshow_video,
"keyframes": keyframe_images,
@@ -1641,9 +1671,7 @@ class LTXDirector(io.ComfyNode):
# Build the reference list now (so the guide can encode + anchor them), but keep
# the emitted latent clean. Each reference anchors at (clean_latent_frames + i).
for i, single_ref in enumerate(ghost_refs):
ref_tensor = _resize_image(single_ref, latent_w, latent_h, resize_method, divisible_by)
if img_compression > 0:
ref_tensor = _compress_image(ref_tensor, img_compression)
ref_tensor = prepare_tensor_image(single_ref, latent_w, latent_h)
guide_data["images"].append(ref_tensor)
guide_data["insert_frames"].append((clean_latent_frames + i) * 8)
guide_data["strengths"].append(float(reference_strength))
@@ -1661,7 +1689,7 @@ class LTXDirector(io.ComfyNode):
injected_local = f"{local_part} | {global_prompt}"
injected_lengths = f"{clean_lengths},{tail_pixels}"
dummy_full = {"samples": torch.zeros(
[1, 128, total_latent_frames, latent_h // 32, latent_w // 32], device=_dev,
[1, 128, total_latent_frames, latent_grid_h, latent_grid_w], device=_dev,
)}
patched, conditioning = _encode_relay(
model, clip, dummy_full, global_prompt, injected_local, injected_lengths, epsilon,
@@ -1669,8 +1697,8 @@ class LTXDirector(io.ComfyNode):
if optional_latent is None:
latent = {
"samples": torch.zeros([1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev),
"noise_mask": torch.ones((1, 1, clean_latent_frames, latent_h // 32, latent_w // 32), dtype=torch.float32, device=_dev),
"samples": torch.zeros([1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev),
"noise_mask": torch.ones((1, 1, clean_latent_frames, latent_grid_h, latent_grid_w), dtype=torch.float32, device=_dev),
}
else:
latent = optional_latent
@@ -1685,7 +1713,7 @@ class LTXDirector(io.ComfyNode):
# OFF: plain clean latent + relay.
if optional_latent is None:
samples = torch.zeros(
[1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev,
[1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev,
)
latent = {"samples": samples}
log.info(
@@ -1700,7 +1728,7 @@ class LTXDirector(io.ComfyNode):
)
# --- Build Audio Output ---
audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, frame_rate_f, override_audio=override_audio)
# --- Audio Latent Generation ---
audio_latent = {}
@@ -1712,7 +1740,7 @@ class LTXDirector(io.ComfyNode):
inner = getattr(audio_vae, "first_stage_model", audio_vae)
z_channels = audio_vae.latent_channels
audio_freq = inner.latent_frequency_bins
num_audio_latents = inner.num_of_latents_from_frames(ltxv_length, float(frame_rate))
num_audio_latents = inner.num_of_latents_from_frames(ltxv_length, frame_rate_f)
audio_latents = torch.zeros(
(1, z_channels, num_audio_latents, audio_freq),
device=comfy.model_management.intermediate_device(),
@@ -1761,17 +1789,9 @@ class LTXDirector(io.ComfyNode):
if overlap_end > overlap_start:
rel_start = overlap_start - start_frame
rel_len = overlap_end - overlap_start
start_sec = rel_start / float(frame_rate)
len_sec = rel_len / float(frame_rate)
total_sec = ltxv_length / float(frame_rate)
start_idx = int((start_sec / total_sec) * F_len)
end_idx = int(((start_sec + len_sec) / total_sec) * F_len)
start_idx = max(0, min(F_len, start_idx))
end_idx = max(0, min(F_len, end_idx))
start_idx, end_idx = _time_range_to_latent_indices(
rel_start, rel_len, 0, ltxv_length, frame_rate_f, F_len
)
gap_mask[:, start_idx:end_idx, :] = 1.0
else:
gap_mask = torch.ones((B, F_len, H_len), dtype=torch.float32, device=latent_samples.device)
@@ -1791,13 +1811,9 @@ class LTXDirector(io.ComfyNode):
offset = max(0, start_frame - seg_start)
seg_len = max(1.0, seg_len - offset)
seg_start = max(0, seg_start - start_frame)
start_sec = seg_start / float(frame_rate)
len_sec = seg_len / float(frame_rate)
total_sec = ltxv_length / float(frame_rate)
start_idx = int((start_sec / total_sec) * F_len)
end_idx = int(((start_sec + len_sec) / total_sec) * F_len)
start_idx, end_idx = _time_range_to_latent_indices(
seg_start, seg_len, 0, ltxv_length, frame_rate_f, F_len
)
gap_mask[:, start_idx:end_idx, :] = 0.0
if inpaint_audio:
@@ -1829,7 +1845,7 @@ class LTXDirector(io.ComfyNode):
raise e
# --- Motion guide output from timeline video segments ---
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
motion_guide_data = {"segments": [], "frame_rate": frame_rate_f, "duration_frames": int(duration_frames), "resize_method": resize_method}
try:
if use_custom_motion:
motion_segments = tdata.get("motionSegments", [])
@@ -1865,7 +1881,7 @@ class LTXDirector(io.ComfyNode):
guide_data["duration_frames"] = duration_frames
guide_data["resize_method"] = resize_method
return io.NodeOutput(patched, conditioning, conditioning_neg, latent, audio_latent, guide_data, motion_guide_data, float(frame_rate), audio_out, int(clean_latent_frames), int(clean_pixel_frames))
return io.NodeOutput(patched, conditioning, conditioning_neg, latent, audio_latent, guide_data, motion_guide_data, frame_rate_f, audio_out, int(clean_latent_frames), int(clean_pixel_frames))
NODE_CLASS_MAPPINGS = {