diff --git a/ltx_director.py b/ltx_director.py index ff700a8..acb1cd7 100644 --- a/ltx_director.py +++ b/ltx_director.py @@ -11,13 +11,14 @@ import torch.nn.functional as F import av from PIL import Image -import os -import platform -import subprocess -import folder_paths -import comfy.model_management -from server import PromptServer -from aiohttp import web +import os +import platform +import subprocess +from urllib.parse import parse_qs, urlparse +import folder_paths +import comfy.model_management +from server import PromptServer +from aiohttp import web from comfy_api.latest import io @@ -68,11 +69,145 @@ MotionGuideData = io.Custom("MOTION_GUIDE_DATA") # --- Licon MSR engine fixed parameters (were node widgets; hardcoded for a clean UI) --- # Slideshow length. 41 is the Licon training default (valid: 17 / 25 / 33 / 41). -MSR_PREFIX_FRAMES = 41 +MSR_PREFIX_FRAMES = 41 # latent_downscale_factor for the IC-LoRA reference frames. Tied to how the MSR LoRA was # trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any # multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream. -MSR_LATENT_DOWNSCALE = 1.0 +MSR_LATENT_DOWNSCALE = 1.0 +EMPTY_IMAGE_SHAPE = (1, 512, 512, 3) +PEAK_BUCKETS = 200 + + +def _empty_image_tensor() -> torch.Tensor: + return torch.zeros(EMPTY_IMAGE_SHAPE, dtype=torch.float32) + + +def _normalise_b64_payload(payload: str) -> str: + if payload and "," in payload: + return payload.split(",", 1)[1] + return payload or "" + + +def _image_to_tensor(img: Image.Image) -> torch.Tensor: + arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def _resolve_input_path(path: str, input_dir: str = None) -> str | None: + if not path: + return None + + input_dir = input_dir or folder_paths.get_input_directory() + clean_path = path.replace("\\", "/") + candidates = [ + os.path.join(input_dir, clean_path), + os.path.join(input_dir, "whatdreamscost", os.path.basename(clean_path)), + os.path.join(input_dir, os.path.basename(clean_path)), + ] + + for candidate in candidates: + if os.path.exists(candidate): + return candidate + return None + + +def _load_image_reference(b64_or_url: str = "", filename: str = None, cache: dict | None = None, + input_dir: str = None) -> torch.Tensor: + """Load a reference image from a base64 string, a Comfy /view URL, or an input filename.""" + if not b64_or_url and not filename: + return _empty_image_tensor() + + input_dir = input_dir or folder_paths.get_input_directory() + cache = cache if cache is not None else {} + + file_path = None + if b64_or_url and "view?" in b64_or_url: + try: + parsed = urlparse(b64_or_url) + q = parse_qs(parsed.query) + fname = q.get("filename", [None])[0] + subfolder = q.get("subfolder", [""])[0] + if fname: + file_path = _resolve_input_path(os.path.join(subfolder, fname), input_dir) + except Exception as e: + log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}") + + if file_path is None and filename: + file_path = _resolve_input_path(filename, input_dir) + + if file_path: + cache_key = ("file", file_path) + cached = cache.get(cache_key) + if cached is not None: + return cached + try: + tensor = _image_to_tensor(Image.open(file_path)) + cache[cache_key] = tensor + return tensor + except Exception as e: + log.debug(f"[PromptRelay] File image loading failed for {file_path}: {e}") + + if b64_or_url: + try: + b64_str = _normalise_b64_payload(b64_or_url) + cache_key = ("b64", b64_str) + cached = cache.get(cache_key) + if cached is not None: + return cached + img_bytes = base64.b64decode(b64_str) + tensor = _image_to_tensor(Image.open(_io.BytesIO(img_bytes))) + cache[cache_key] = tensor + return tensor + except Exception as e: + log.debug(f"[PromptRelay] Base64 decoding failed: {e}") + + return _empty_image_tensor() + + +def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> list[float]: + if samples.size == 0: + return [0.0] * num_peaks + + step = max(1, int(math.ceil(samples.size / num_peaks))) + padded = np.pad(samples, (0, step * num_peaks - samples.size), mode="constant") + chunks = padded.reshape(num_peaks, step) + return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist() + + +def _read_audio_bytes(container, stream, resampler) -> bytes: + audio_bytes = bytearray() + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + audio_bytes.extend(resampled_frame.to_ndarray().tobytes()) + for resampled_frame in resampler.resample(None): + audio_bytes.extend(resampled_frame.to_ndarray().tobytes()) + return bytes(audio_bytes) + + +def _target_resize_dims(src_w: int, src_h: int, custom_width: int, custom_height: int, divisible_by: int) -> tuple[int, int, str]: + def snap(val, div): + return max(div, (val // div) * div) + + if custom_width > 0 and custom_height > 0: + return custom_width, custom_height, None + if custom_width > 0: + tgt_w = snap(custom_width, divisible_by) + tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) + return tgt_w, tgt_h, "stretch to fit" + if custom_height > 0: + tgt_h = snap(custom_height, divisible_by) + tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) + return tgt_w, tgt_h, "stretch to fit" + return src_w, src_h, "maintain aspect ratio" + + +def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]: + try: + with av.open(file_path) as container: + stream = container.streams.video[0] + return stream.width or stream.codec_context.width, stream.height or stream.codec_context.height + except Exception: + return None, None def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""): @@ -111,47 +246,9 @@ def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", return gp, " | ".join(processed_locals) -def _load_image_source(b64_or_url: str, filename: str = None) -> torch.Tensor: - """Load a reference image from a base64 string, a Comfy /view URL, or an input filename.""" - if not b64_or_url and not filename: - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) - - if b64_or_url and "view?" in b64_or_url: - try: - from urllib.parse import urlparse, parse_qs - parsed = urlparse(b64_or_url) - q = parse_qs(parsed.query) - fname = q.get("filename", [None])[0] - subfolder = q.get("subfolder", [""])[0] - if fname: - file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname) - if os.path.exists(file_path): - img = Image.open(file_path).convert("RGB") - arr = np.array(img, dtype=np.float32) / 255.0 - return torch.from_numpy(arr).unsqueeze(0) - except Exception as e: - log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}") - - if filename: - file_path = os.path.join(folder_paths.get_input_directory(), filename) - if os.path.exists(file_path): - img = Image.open(file_path).convert("RGB") - arr = np.array(img, dtype=np.float32) / 255.0 - return torch.from_numpy(arr).unsqueeze(0) - - if b64_or_url: - try: - b64_str = b64_or_url - if "," in b64_str: - b64_str = b64_str.split(",", 1)[1] - img_bytes = base64.b64decode(b64_str) - img = Image.open(_io.BytesIO(img_bytes)).convert("RGB") - arr = np.array(img, dtype=np.float32) / 255.0 - return torch.from_numpy(arr).unsqueeze(0) - except Exception as e: - log.debug(f"[PromptRelay] Base64 decoding failed: {e}") - - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) +def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None, + input_dir: str = None) -> torch.Tensor: + return _load_image_reference(b64_or_url=b64_or_url, filename=filename, cache=cache, input_dir=input_dir) def _execute_comfy_node(node_class, **kwargs): @@ -392,26 +489,15 @@ async def unload_ollama_endpoint(request): return web.json_response({"status": "error", "message": str(e)}) -def read_wav_peaks(wav_path): - import wave - peaks = [] - with wave.open(wav_path, 'rb') as w: - n_frames = w.getnframes() - if n_frames > 0: - frames_bytes = w.readframes(n_frames) - samples = np.frombuffer(frames_bytes, dtype=np.int16) - num_peaks = 200 - step = max(1, len(samples) // num_peaks) - for i in range(num_peaks): - chunk = samples[i * step : (i + 1) * step] - if len(chunk) > 0: - max_val = np.max(np.abs(chunk)) / 32767.0 - peaks.append(float(max_val)) - else: - peaks.append(0.0) - else: - peaks = [0.0] * 200 - return peaks +def read_wav_peaks(wav_path): + import wave + with wave.open(wav_path, 'rb') as w: + n_frames = w.getnframes() + if n_frames <= 0: + return [0.0] * PEAK_BUCKETS + frames_bytes = w.readframes(n_frames) + samples = np.frombuffer(frames_bytes, dtype=np.int16) + return _compute_signal_peaks(samples) def extract_audio_from_video(video_path): @@ -445,20 +531,9 @@ def extract_audio_from_video(video_path): rate=44100, ) - audio_bytes = bytearray() - - for frame in container.decode(stream): - for resampled_frame in resampler.resample(frame): - arr = resampled_frame.to_ndarray() - audio_bytes.extend(arr.tobytes()) - - # Flush resampler - for resampled_frame in resampler.resample(None): - arr = resampled_frame.to_ndarray() - audio_bytes.extend(arr.tobytes()) - - if not audio_bytes: - return None, None + audio_bytes = _read_audio_bytes(container, stream, resampler) + if not audio_bytes: + return None, None # Write WAV file with wave.open(output_wav, 'wb') as w: @@ -468,17 +543,8 @@ def extract_audio_from_video(video_path): w.writeframes(audio_bytes) # Calculate peaks - peaks = [] - samples = np.frombuffer(audio_bytes, dtype=np.int16) - num_peaks = 200 - step = max(1, len(samples) // num_peaks) - for i in range(num_peaks): - chunk = samples[i * step : (i + 1) * step] - if len(chunk) > 0: - max_val = np.max(np.abs(chunk)) / 32767.0 - peaks.append(float(max_val)) - else: - peaks.append(0.0) + samples = np.frombuffer(audio_bytes, dtype=np.int16) + peaks = _compute_signal_peaks(samples) input_dir = folder_paths.get_input_directory() rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/') @@ -509,30 +575,11 @@ def get_audio_peaks(audio_path): layout='mono', rate=8000, ) - audio_bytes = bytearray() - for frame in container.decode(stream): - for resampled_frame in resampler.resample(frame): - arr = resampled_frame.to_ndarray() - audio_bytes.extend(arr.tobytes()) - for resampled_frame in resampler.resample(None): - arr = resampled_frame.to_ndarray() - audio_bytes.extend(arr.tobytes()) - - if not audio_bytes: - return None - - peaks = [] - samples = np.frombuffer(audio_bytes, dtype=np.int16) - num_peaks = 200 - step = max(1, len(samples) // num_peaks) - for i in range(num_peaks): - chunk = samples[i * step : (i + 1) * step] - if len(chunk) > 0: - max_val = np.max(np.abs(chunk)) / 32767.0 - peaks.append(float(max_val)) - else: - peaks.append(0.0) - return peaks + audio_bytes = _read_audio_bytes(container, stream, resampler) + if not audio_bytes: + return None + samples = np.frombuffer(audio_bytes, dtype=np.int16) + return _compute_signal_peaks(samples) except Exception as e: print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}") return None @@ -657,38 +704,24 @@ async def ltx_director_upload_chunk(request): -def _load_image_tensor(seg: dict) -> torch.Tensor: - """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 - to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" - if seg.get("imageFile"): - file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"]) - if os.path.exists(file_path): - img = Image.open(file_path).convert("RGB") - arr = np.array(img, dtype=np.float32) / 255.0 - return torch.from_numpy(arr).unsqueeze(0) - - b64_str = seg.get("imageB64", "") - if not b64_str or b64_str.startswith("/view?"): - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) - - if "," in b64_str: - b64_str = b64_str.split(",", 1)[1] - - try: - img_bytes = base64.b64decode(b64_str) - img = Image.open(_io.BytesIO(img_bytes)).convert("RGB") - arr = np.array(img, dtype=np.float32) / 255.0 - return torch.from_numpy(arr).unsqueeze(0) - except: - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) - -def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor: - """Extracts a sequence of frames from a video file based on the segment's trim parameters, - and returns them as an [N, H, W, 3] float32 tensor.""" - file_path = os.path.join(folder_paths.get_input_directory(), seg.get("imageFile", "")) - - if not os.path.exists(file_path): - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) +def _load_image_tensor(seg: dict, cache: dict | None = None, input_dir: str = None) -> torch.Tensor: + """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 + to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" + return _load_image_reference( + b64_or_url=seg.get("imageB64", ""), + filename=seg.get("imageFile"), + cache=cache, + input_dir=input_dir, + ) + + +def _load_video_tensor(seg: dict, frame_rate: float, input_dir: str = None) -> torch.Tensor: + """Extracts a sequence of frames from a video file based on the segment's trim parameters, + and returns them as an [N, H, W, 3] float32 tensor.""" + file_path = _resolve_input_path(seg.get("imageFile", ""), input_dir) + + if not file_path: + return _empty_image_tensor() trim_start_frames = float(seg.get("trimStart", 0)) length_frames = float(seg.get("length", 1)) @@ -723,11 +756,11 @@ def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor: if len(frames) >= int(length_frames): break - except Exception as e: - log.warning(f"[PromptRelay] Video extract error: {e}") - - if not frames: - return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + except Exception as e: + log.warning(f"[PromptRelay] Video extract error: {e}") + + if not frames: + return _empty_image_tensor() frames_np = np.array(frames, dtype=np.float32) / 255.0 return torch.from_numpy(frames_np) @@ -848,21 +881,24 @@ def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor: return tensor -def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict: - """Parses timeline JSON, loads/trims audio directly from memory using PyAV, - and aligns to a global timeline yielding ComfyUI's format. - Output length explicitly mimics the timeline's duration_frames length.""" - target_sr = 44100 - total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr))) - empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr} - - if not timeline_data_str: - return empty_audio - - try: - data = json.loads(timeline_data_str) - is_retake = data.get("retakeMode", False) - if is_retake and data.get("retakeVideo"): +def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict: + """Parses timeline JSON, loads/trims audio directly from memory using PyAV, + and aligns to a global timeline yielding ComfyUI's format. + Output length explicitly mimics the timeline's duration_frames length.""" + target_sr = 44100 + total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr))) + empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr} + + if not timeline_data_value: + return empty_audio + + try: + if isinstance(timeline_data_value, str): + data = json.loads(timeline_data_value) + else: + data = timeline_data_value + is_retake = data.get("retakeMode", False) + if is_retake and data.get("retakeVideo"): retake_vid = data.get("retakeVideo") audio_segs = [{ "videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"), @@ -888,17 +924,9 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra buffer = None file_key = "videoFile" if override_audio else "audioFile" if seg.get(file_key): - file_path = os.path.join(folder_paths.get_input_directory(), seg[file_key]) - if not os.path.exists(file_path): - # Try fallback under whatdreamscost subfolder - basename = os.path.basename(seg[file_key]) - fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) - if os.path.exists(fallback_path): - file_path = fallback_path - - if os.path.exists(file_path): - with open(file_path, "rb") as f: - buffer = _io.BytesIO(f.read()) + file_path = _resolve_input_path(seg[file_key]) + if file_path: + buffer = file_path if not override_audio and not buffer and seg.get("audioB64"): b64 = seg.get("audioB64") @@ -917,7 +945,7 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra clip_frames = [] # Use PyAV to decode directly from memory buffer - with av.open(buffer) as container: + with av.open(buffer) as container: if not container.streams.audio: continue stream = container.streams.audio[0] @@ -1258,19 +1286,21 @@ class LTXDirector(io.ComfyNode): ) @classmethod - def execute(cls, model, clip, start_second, end_second, duration_seconds, start_frame, end_frame, duration_frames, - timeline_data, local_prompts, segment_lengths, global_prompt="", guide_strength="", epsilon=1e-3, - frame_rate=24, display_mode="seconds", - custom_width=768, custom_height=512, resize_method="maintain aspect ratio", - divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None, - use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False, - vae=None, reference_strength=1.0, ref_images=None, - start=None, end=None, duration=None) -> io.NodeOutput: - - # Parse timeline data - try: - tdata = json.loads(timeline_data) if timeline_data else {} - except Exception as e: + def execute(cls, model, clip, start_second, end_second, duration_seconds, start_frame, end_frame, duration_frames, + timeline_data, local_prompts, segment_lengths, global_prompt="", guide_strength="", epsilon=1e-3, + frame_rate=24, display_mode="seconds", + custom_width=768, custom_height=512, resize_method="maintain aspect ratio", + divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None, + use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False, + vae=None, reference_strength=1.0, ref_images=None, + start=None, end=None, duration=None) -> io.NodeOutput: + input_dir = folder_paths.get_input_directory() + image_cache = {} + + # Parse timeline data + try: + tdata = json.loads(timeline_data) if timeline_data else {} + except Exception as e: log.error(f"[LTXDirector] execute timeline_data parse error: {e}") tdata = {} @@ -1327,14 +1357,14 @@ class LTXDirector(io.ComfyNode): if legacy_b64 and not images_list: images_list = [{"b64": legacy_b64, "name": char_info.get("fileName", "")}] - slot_tensors = [] - for img_info in images_list: - image_b64 = img_info.get("b64", "") - file_name = img_info.get("name", "") - tensor = _load_image_source(image_b64, file_name) - char_images.append(tensor) - slot_tensors.append(tensor) - char_slot_images.append(slot_tensors) + slot_tensors = [] + for img_info in images_list: + image_b64 = img_info.get("b64", "") + file_name = img_info.get("name", "") + tensor = _load_image_source(image_b64, file_name, cache=image_cache, input_dir=input_dir) + char_images.append(tensor) + slot_tensors.append(tensor) + char_slot_images.append(slot_tensors) except Exception as e: log.warning("[LTXDirector] Could not process character slot inputs: %s", e) @@ -1367,40 +1397,33 @@ class LTXDirector(io.ComfyNode): if guide_strength.strip(): strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()] - for idx, seg in enumerate(img_segs): - seg_start = int(seg.get("start", 0)) - offset = max(0, start_frame - seg_start) - - if seg.get("type") == "video": - if offset > 0: - seg["trimStart"] = float(seg.get("trimStart", 0)) + offset - seg["length"] = max(1, int(seg.get("length", 1)) - offset) - tensor = _load_video_tensor(seg, float(frame_rate)) - else: - tensor = _load_image_tensor(seg) - - # Apply resize - src_h, src_w = tensor.shape[1], tensor.shape[2] - - def snap(val, div): - return max(div, (val // div) * div) - - if custom_width > 0 and custom_height > 0: - # Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR) - tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) - elif custom_width > 0: - # Width only — scale height from AR, snap both, then resize to exact dimensions - tgt_w = snap(custom_width, divisible_by) - tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) - tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) - elif custom_height > 0: - # Height only — scale width from AR, snap both, then resize to exact dimensions - tgt_h = snap(custom_height, divisible_by) - tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) - tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) - else: - # Both zero — keep original dimensions, just snap to divisible_by - tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by) + for idx, seg in enumerate(img_segs): + seg_start = int(seg.get("start", 0)) + offset = max(0, start_frame - seg_start) + seg_length = int(seg.get("length", 1)) + seg_for_load = seg + + if seg.get("type") == "video": + if offset > 0: + seg_for_load = dict(seg) + seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset + seg_for_load["length"] = max(1, seg_length - offset) + tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir) + else: + tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir) + + # Apply resize + src_h, src_w = tensor.shape[1], tensor.shape[2] + tgt_w, tgt_h, resize_override = _target_resize_dims( + src_w, src_h, custom_width, custom_height, divisible_by + ) + tensor = _resize_image( + tensor, + tgt_w, + tgt_h, + resize_override or resize_method, + divisible_by, + ) # Apply compression @@ -1408,15 +1431,15 @@ class LTXDirector(io.ComfyNode): tensor = _compress_image(tensor, img_compression) # Record dimensions of the first processed image for latent generation - if idx == 0: - derived_h = tensor.shape[1] - derived_w = tensor.shape[2] - - if seg.get("isEndFrame"): - insert_frame = max(0, seg_start + int(seg.get("length", 1)) - 1 - start_frame) - else: - insert_frame = max(0, seg_start - start_frame) - strength = strengths[idx] if idx < len(strengths) else 1.0 + if idx == 0: + derived_h = tensor.shape[1] + derived_w = tensor.shape[2] + + if seg.get("isEndFrame"): + insert_frame = max(0, seg_start + seg_length - 1 - start_frame) + else: + insert_frame = max(0, seg_start - start_frame) + strength = strengths[idx] if idx < len(strengths) else 1.0 guide_data["images"].append(tensor) guide_data["insert_frames"].append(insert_frame) guide_data["strengths"].append(float(strength)) @@ -1424,75 +1447,49 @@ class LTXDirector(io.ComfyNode): # If no images were loaded from the timeline, create a dummy image at strength 0 # to prevent artifacts in text-to-video mode. if not guide_data["images"] and optional_latent is None: - src_w = derived_w if derived_w > 0 else 768 - src_h = derived_h if derived_h > 0 else 512 - - # If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling - tdata_motion = json.loads(timeline_data) if timeline_data else {} - found_dims = False - - # Check for retake base video first - is_retake = tdata_motion.get("retakeMode", False) - retake_vid = tdata_motion.get("retakeVideo") or {} - retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else "" - if is_retake and retake_file: - r_path = os.path.join(folder_paths.get_input_directory(), retake_file) - if not os.path.exists(r_path): - basename = os.path.basename(retake_file) - fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) - if os.path.exists(fallback_path): - r_path = fallback_path - if os.path.exists(r_path): - try: - with av.open(r_path) as container: - stream = container.streams.video[0] - src_w = stream.width or stream.codec_context.width - src_h = stream.height or stream.codec_context.height - found_dims = True - except: - pass - - # Fallback to normal motion segments - if not found_dims: - for mseg in tdata_motion.get("motionSegments", []): - v_file = mseg.get("videoFile") - if v_file: - v_path = os.path.join(folder_paths.get_input_directory(), v_file) - if not os.path.exists(v_path): - basename = os.path.basename(v_file) - fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) - if os.path.exists(fallback_path): - v_path = fallback_path - if os.path.exists(v_path): - try: - with av.open(v_path) as container: - stream = container.streams.video[0] - src_w = stream.width or stream.codec_context.width - src_h = stream.height or stream.codec_context.height - found_dims = True - break - except: - pass - - # Create a dummy tensor of the exact source dimensions - tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32) - - def snap(val, div): - return max(div, (val // div) * div) - - # Route the dummy tensor through the exact same resizing pipeline - if custom_width > 0 and custom_height > 0: - tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) - elif custom_width > 0: - tgt_w = snap(custom_width, divisible_by) - tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) - tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) - elif custom_height > 0: - tgt_h = snap(custom_height, divisible_by) - tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) - tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) - else: - tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by) + src_w = derived_w if derived_w > 0 else 768 + src_h = derived_h if derived_h > 0 else 512 + + # If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling + found_dims = False + + # Check for retake base video first + is_retake = tdata.get("retakeMode", False) + retake_vid = tdata.get("retakeVideo") or {} + retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else "" + if is_retake and retake_file: + r_path = _resolve_input_path(retake_file, input_dir) + if r_path: + src_dims = _extract_video_dimensions(r_path) + if all(src_dims): + src_w, src_h = src_dims + found_dims = True + + # Fallback to normal motion segments + if not found_dims: + for mseg in tdata.get("motionSegments", []): + v_file = mseg.get("videoFile") + if v_file: + v_path = _resolve_input_path(v_file, input_dir) + if v_path: + src_dims = _extract_video_dimensions(v_path) + if all(src_dims): + src_w, src_h = src_dims + found_dims = True + break + + # Create a dummy tensor of the exact source dimensions + tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32) + tgt_w, tgt_h, resize_override = _target_resize_dims( + src_w, src_h, custom_width, custom_height, divisible_by + ) + tensor = _resize_image( + tensor, + tgt_w, + tgt_h, + resize_override or resize_method, + divisible_by, + ) guide_data["images"].append(tensor) guide_data["insert_frames"].append(0) @@ -1703,7 +1700,7 @@ class LTXDirector(io.ComfyNode): ) # --- Build Audio Output --- - audio_out = _build_combined_audio(timeline_data, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio) + audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio) # --- Audio Latent Generation --- audio_latent = {} @@ -1831,13 +1828,12 @@ class LTXDirector(io.ComfyNode): log.error("[PromptRelay] Could not generate empty audio latent: %s", e) raise e - # --- Motion guide output from timeline video segments --- - motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method} - try: - tdata = json.loads(timeline_data) if timeline_data else {} - if use_custom_motion: - motion_segments = tdata.get("motionSegments", []) - else: + # --- Motion guide output from timeline video segments --- + motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method} + try: + if use_custom_motion: + motion_segments = tdata.get("motionSegments", []) + else: motion_segments = [] for seg in motion_segments: seg_start = int(seg.get("start", 0)) @@ -1878,4 +1874,4 @@ NODE_CLASS_MAPPINGS = { NODE_DISPLAY_NAME_MAPPINGS = { "PromptRelayEncodeTimeline": "Prompt Relay Encode (Timeline)", -} \ No newline at end of file +}