From 51bd3b4968b8b7f4ac02ef3c90b0008b3e4ed808 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 9 Jul 2026 13:44:23 +0000 Subject: [PATCH] perf: cache reused guide and audio work --- ltx_director.py | 210 ++++++++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 97 deletions(-) diff --git a/ltx_director.py b/ltx_director.py index acb1cd7..eaef23d 100644 --- a/ltx_director.py +++ b/ltx_director.py @@ -208,6 +208,18 @@ def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]: return stream.width or stream.codec_context.width, stream.height or stream.codec_context.height except Exception: return None, None + + +def _time_range_to_latent_indices(rel_start_frames: float, rel_length_frames: float, start_frame: int, + total_frames: int, frame_rate: float, latent_frames: int) -> tuple[int, int]: + total_sec = total_frames / frame_rate + start_sec = max(0.0, rel_start_frames - start_frame) / frame_rate + end_sec = max(start_sec, start_sec + (rel_length_frames / frame_rate)) + start_idx = int((start_sec / total_sec) * latent_frames) + end_idx = int((end_sec / total_sec) * latent_frames) + start_idx = max(0, min(latent_frames, start_idx)) + end_idx = max(0, min(latent_frames, end_idx)) + return start_idx, end_idx def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""): @@ -942,7 +954,7 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames continue try: - clip_frames = [] + clip_arrays = [] # Use PyAV to decode directly from memory buffer with av.open(buffer) as container: @@ -958,21 +970,24 @@ def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames ) for frame in container.decode(stream): - for resampled_frame in resampler.resample(frame): - # to_ndarray() on fltp gives shape (channels, samples) - arr = resampled_frame.to_ndarray() - clip_frames.append(torch.from_numpy(arr)) + for resampled_frame in resampler.resample(frame): + # to_ndarray() on fltp gives shape (channels, samples) + arr = resampled_frame.to_ndarray() + clip_arrays.append(arr) # Flush the resampler to get any remaining samples - for resampled_frame in resampler.resample(None): - arr = resampled_frame.to_ndarray() - clip_frames.append(torch.from_numpy(arr)) - - if not clip_frames: - continue - - # Concatenate all frame blocks along the samples dimension (dim 1) - waveform = torch.cat(clip_frames, dim=1) # Shape: [2, total_clip_samples] + for resampled_frame in resampler.resample(None): + arr = resampled_frame.to_ndarray() + clip_arrays.append(arr) + + if not clip_arrays: + continue + + if len(clip_arrays) == 1: + waveform_np = clip_arrays[0] + else: + waveform_np = np.concatenate(clip_arrays, axis=1) + waveform = torch.from_numpy(waveform_np) # Shape: [2, total_clip_samples] # Calculate interactive trim boundaries trim_start_frames = float(seg.get("trimStart", 0)) @@ -1296,6 +1311,18 @@ class LTXDirector(io.ComfyNode): start=None, end=None, duration=None) -> io.NodeOutput: input_dir = folder_paths.get_input_directory() image_cache = {} + processed_image_cache = {} + + def prepare_tensor_image(tensor: torch.Tensor, width: int, height: int) -> torch.Tensor: + cache_key = (id(tensor), width, height, resize_method, divisible_by, img_compression) + cached = processed_image_cache.get(cache_key) + if cached is not None: + return cached + processed = _resize_image(tensor, width, height, resize_method, divisible_by) + if img_compression > 0: + processed = _compress_image(processed, img_compression) + processed_image_cache[cache_key] = processed + return processed # Parse timeline data try: @@ -1316,13 +1343,14 @@ class LTXDirector(io.ComfyNode): if duration is not None: duration_seconds = float(duration) duration_frames = max(1, int(round(duration_seconds * _auto_fps))) - elif end is not None: - # Derive the window length from start..end when duration is not explicitly wired. - duration_frames = max(1, end_frame - start_frame) - duration_seconds = duration_frames / _auto_fps - - is_retake_mode = tdata.get("retakeMode", False) - is_retake_active = is_retake_mode and tdata.get("retakeVideo") is not None + elif end is not None: + # Derive the window length from start..end when duration is not explicitly wired. + duration_frames = max(1, end_frame - start_frame) + duration_seconds = duration_frames / _auto_fps + frame_rate_f = float(frame_rate) + + is_retake_mode = tdata.get("retakeMode", False) + is_retake_active = is_retake_mode and tdata.get("retakeVideo") is not None # Extract global_prompt from timeline_data if not connected/empty if not global_prompt: @@ -1408,7 +1436,7 @@ class LTXDirector(io.ComfyNode): seg_for_load = dict(seg) seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset seg_for_load["length"] = max(1, seg_length - offset) - tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir) + tensor = _load_video_tensor(seg_for_load, frame_rate_f, input_dir=input_dir) else: tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir) @@ -1507,11 +1535,13 @@ class LTXDirector(io.ComfyNode): # This ensures we get AT LEAST the requested frames, snapped to LTXV's requirements. ltxv_length = int(math.ceil((duration_frames - 1) / 8.0) * 8) + 1 - latent_w = max(32, (derived_w // 32) * 32) - latent_h = max(32, (derived_h // 32) * 32) - # Clean (visible) region: latent frames and the matching pixel frames. - clean_latent_frames = ((ltxv_length - 1) // 8) + 1 - clean_pixel_frames = int(ltxv_length) + latent_w = max(32, (derived_w // 32) * 32) + latent_h = max(32, (derived_h // 32) * 32) + latent_grid_h = latent_h // 32 + latent_grid_w = latent_w // 32 + # Clean (visible) region: latent frames and the matching pixel frames. + clean_latent_frames = ((ltxv_length - 1) // 8) + 1 + clean_pixel_frames = int(ltxv_length) # Negative conditioning emitted on the "negative" output (slot 2, right under positive). # The Director no longer takes a negative input; it emits a neutral EMPTY negative so the @@ -1543,20 +1573,20 @@ class LTXDirector(io.ComfyNode): _selected.extend(char_slot_images[_slot]) if not _selected: _selected = char_images - log.info("[LTXDirector] MSR slideshow subjects: referenced slots=%s, images=%d", _referenced_slots, len(_selected)) - - identity_images = [ - _resize_image(c, latent_w, latent_h, resize_method, divisible_by) for c in _selected - ] + log.info("[LTXDirector] MSR slideshow subjects: referenced slots=%s, images=%d", _referenced_slots, len(_selected)) + + identity_images = [ + prepare_tensor_image(c, latent_w, latent_h) for c in _selected + ] # Scene background = first timeline image (already in guide_data), else black. scene_images = list(guide_data["images"]) if scene_images: - bg_slide = scene_images[0] - if bg_slide.shape[1] != latent_h or bg_slide.shape[2] != latent_w: - bg_slide = _resize_image(bg_slide, latent_w, latent_h, resize_method, divisible_by) - else: - bg_slide = torch.zeros((1, latent_h, latent_w, 3), dtype=torch.float32) + bg_slide = scene_images[0] + if bg_slide.shape[1] != latent_h or bg_slide.shape[2] != latent_w: + bg_slide = _resize_image(bg_slide, latent_w, latent_h, resize_method, divisible_by) + else: + bg_slide = torch.zeros((1, latent_h, latent_w, 3), dtype=torch.float32) # Build the slideshow: characters then background, distributed across MSR_PREFIX_FRAMES. slideshow_sources = identity_images + [bg_slide] @@ -1578,28 +1608,28 @@ class LTXDirector(io.ComfyNode): local_part = local_prompts.strip() if local_prompts.strip() else global_prompt clean_lengths = segment_lengths.strip() if segment_lengths.strip() else str(duration_frames) injected_local = f"{local_part} | {global_prompt}" - injected_lengths = f"{clean_lengths},{tail_pixels}" - - dummy_full = {"samples": torch.zeros( - [1, 128, total_latent_frames, latent_h // 32, latent_w // 32], device=_dev, - )} + injected_lengths = f"{clean_lengths},{tail_pixels}" + + dummy_full = {"samples": torch.zeros( + [1, 128, total_latent_frames, latent_grid_h, latent_grid_w], device=_dev, + )} patched, conditioning = _encode_relay( model, clip, dummy_full, global_prompt, injected_local, injected_lengths, epsilon, ) # Clean base latent (the true visible region); the guide node pads + appends keyframes per stage. if optional_latent is None: - latent = { - "samples": torch.zeros([1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev), - "noise_mask": torch.ones((1, 1, clean_latent_frames, latent_h // 32, latent_w // 32), dtype=torch.float32, device=_dev), - } + latent = { + "samples": torch.zeros([1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev), + "noise_mask": torch.ones((1, 1, clean_latent_frames, latent_grid_h, latent_grid_w), dtype=torch.float32, device=_dev), + } else: latent = optional_latent - # Hand the RAW full-res references to the guide node via guide_data["msr"]. - guide_data = { - "images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate), - "msr": { + # Hand the RAW full-res references to the guide node via guide_data["msr"]. + guide_data = { + "images": [], "insert_frames": [], "strengths": [], "frame_rate": frame_rate_f, + "msr": { "slideshow": slideshow_video, "keyframes": keyframe_images, "prefix_latents": int(prefix_latents), @@ -1640,11 +1670,9 @@ class LTXDirector(io.ComfyNode): if is_ghost: # Build the reference list now (so the guide can encode + anchor them), but keep # the emitted latent clean. Each reference anchors at (clean_latent_frames + i). - for i, single_ref in enumerate(ghost_refs): - ref_tensor = _resize_image(single_ref, latent_w, latent_h, resize_method, divisible_by) - if img_compression > 0: - ref_tensor = _compress_image(ref_tensor, img_compression) - guide_data["images"].append(ref_tensor) + for i, single_ref in enumerate(ghost_refs): + ref_tensor = prepare_tensor_image(single_ref, latent_w, latent_h) + guide_data["images"].append(ref_tensor) guide_data["insert_frames"].append((clean_latent_frames + i) * 8) guide_data["strengths"].append(float(reference_strength)) # Tell the guide to pre-extend the (clean) latent by this many frames before it @@ -1658,20 +1686,20 @@ class LTXDirector(io.ComfyNode): tail_pixels = n_refs * 8 local_part = local_prompts.strip() if local_prompts.strip() else global_prompt clean_lengths = segment_lengths.strip() if segment_lengths.strip() else str(duration_frames) - injected_local = f"{local_part} | {global_prompt}" - injected_lengths = f"{clean_lengths},{tail_pixels}" - dummy_full = {"samples": torch.zeros( - [1, 128, total_latent_frames, latent_h // 32, latent_w // 32], device=_dev, - )} + injected_local = f"{local_part} | {global_prompt}" + injected_lengths = f"{clean_lengths},{tail_pixels}" + dummy_full = {"samples": torch.zeros( + [1, 128, total_latent_frames, latent_grid_h, latent_grid_w], device=_dev, + )} patched, conditioning = _encode_relay( model, clip, dummy_full, global_prompt, injected_local, injected_lengths, epsilon, ) if optional_latent is None: - latent = { - "samples": torch.zeros([1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev), - "noise_mask": torch.ones((1, 1, clean_latent_frames, latent_h // 32, latent_w // 32), dtype=torch.float32, device=_dev), - } + latent = { + "samples": torch.zeros([1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev), + "noise_mask": torch.ones((1, 1, clean_latent_frames, latent_grid_h, latent_grid_w), dtype=torch.float32, device=_dev), + } else: latent = optional_latent @@ -1684,9 +1712,9 @@ class LTXDirector(io.ComfyNode): else: # OFF: plain clean latent + relay. if optional_latent is None: - samples = torch.zeros( - [1, 128, clean_latent_frames, latent_h // 32, latent_w // 32], device=_dev, - ) + samples = torch.zeros( + [1, 128, clean_latent_frames, latent_grid_h, latent_grid_w], device=_dev, + ) latent = {"samples": samples} log.info( "[PromptRelay] Auto-generated LTXV latent: %dx%d, %d pixel frames (%d latent frames)", @@ -1700,7 +1728,7 @@ class LTXDirector(io.ComfyNode): ) # --- Build Audio Output --- - audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio) + audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, frame_rate_f, override_audio=override_audio) # --- Audio Latent Generation --- audio_latent = {} @@ -1712,7 +1740,7 @@ class LTXDirector(io.ComfyNode): inner = getattr(audio_vae, "first_stage_model", audio_vae) z_channels = audio_vae.latent_channels audio_freq = inner.latent_frequency_bins - num_audio_latents = inner.num_of_latents_from_frames(ltxv_length, float(frame_rate)) + num_audio_latents = inner.num_of_latents_from_frames(ltxv_length, frame_rate_f) audio_latents = torch.zeros( (1, z_channels, num_audio_latents, audio_freq), device=comfy.model_management.intermediate_device(), @@ -1758,21 +1786,13 @@ class LTXDirector(io.ComfyNode): overlap_start = max(start_frame, retake_start) overlap_end = min(start_frame + ltxv_length, retake_start + retake_len) - if overlap_end > overlap_start: - rel_start = overlap_start - start_frame - rel_len = overlap_end - overlap_start - - start_sec = rel_start / float(frame_rate) - len_sec = rel_len / float(frame_rate) - total_sec = ltxv_length / float(frame_rate) - - start_idx = int((start_sec / total_sec) * F_len) - end_idx = int(((start_sec + len_sec) / total_sec) * F_len) - - start_idx = max(0, min(F_len, start_idx)) - end_idx = max(0, min(F_len, end_idx)) - - gap_mask[:, start_idx:end_idx, :] = 1.0 + if overlap_end > overlap_start: + rel_start = overlap_start - start_frame + rel_len = overlap_end - overlap_start + start_idx, end_idx = _time_range_to_latent_indices( + rel_start, rel_len, 0, ltxv_length, frame_rate_f, F_len + ) + gap_mask[:, start_idx:end_idx, :] = 1.0 else: gap_mask = torch.ones((B, F_len, H_len), dtype=torch.float32, device=latent_samples.device) @@ -1788,17 +1808,13 @@ class LTXDirector(io.ComfyNode): if seg_start + seg_len <= start_frame or seg_start >= start_frame + ltxv_length: continue - offset = max(0, start_frame - seg_start) - seg_len = max(1.0, seg_len - offset) - seg_start = max(0, seg_start - start_frame) - - start_sec = seg_start / float(frame_rate) - len_sec = seg_len / float(frame_rate) - total_sec = ltxv_length / float(frame_rate) - - start_idx = int((start_sec / total_sec) * F_len) - end_idx = int(((start_sec + len_sec) / total_sec) * F_len) - gap_mask[:, start_idx:end_idx, :] = 0.0 + offset = max(0, start_frame - seg_start) + seg_len = max(1.0, seg_len - offset) + seg_start = max(0, seg_start - start_frame) + start_idx, end_idx = _time_range_to_latent_indices( + seg_start, seg_len, 0, ltxv_length, frame_rate_f, F_len + ) + gap_mask[:, start_idx:end_idx, :] = 0.0 if inpaint_audio: # Generate new audio in the gaps, preserve custom audio segments @@ -1829,7 +1845,7 @@ class LTXDirector(io.ComfyNode): raise e # --- Motion guide output from timeline video segments --- - motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method} + motion_guide_data = {"segments": [], "frame_rate": frame_rate_f, "duration_frames": int(duration_frames), "resize_method": resize_method} try: if use_custom_motion: motion_segments = tdata.get("motionSegments", []) @@ -1865,7 +1881,7 @@ class LTXDirector(io.ComfyNode): guide_data["duration_frames"] = duration_frames guide_data["resize_method"] = resize_method - return io.NodeOutput(patched, conditioning, conditioning_neg, latent, audio_latent, guide_data, motion_guide_data, float(frame_rate), audio_out, int(clean_latent_frames), int(clean_pixel_frames)) + return io.NodeOutput(patched, conditioning, conditioning_neg, latent, audio_latent, guide_data, motion_guide_data, frame_rate_f, audio_out, int(clean_latent_frames), int(clean_pixel_frames)) NODE_CLASS_MAPPINGS = {