chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: reduce redundant media work in director

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-09 13:29:55 +00:00
parent 6695b1a1aa
commit b8cedad997
1 changed files with 321 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

646
ltx_director.py
View File

@@ -11,13 +11,14 @@ import torch.nn.functional as F
import av
from PIL import Image
import os
import platform
import subprocess
import folder_paths
import comfy.model_management
from server import PromptServer
from aiohttp import web
import os
import platform
import subprocess
from urllib.parse import parse_qs, urlparse
import folder_paths
import comfy.model_management
from server import PromptServer
from aiohttp import web
from comfy_api.latest import io
@@ -68,11 +69,145 @@ MotionGuideData = io.Custom("MOTION_GUIDE_DATA")
# --- Licon MSR engine fixed parameters (were node widgets; hardcoded for a clean UI) ---
# Slideshow length. 41 is the Licon training default (valid: 17 / 25 / 33 / 41).
MSR_PREFIX_FRAMES = 41
MSR_PREFIX_FRAMES = 41
# latent_downscale_factor for the IC-LoRA reference frames. Tied to how the MSR LoRA was
# trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any
# multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream.
MSR_LATENT_DOWNSCALE = 1.0
MSR_LATENT_DOWNSCALE = 1.0
EMPTY_IMAGE_SHAPE = (1, 512, 512, 3)
PEAK_BUCKETS = 200
def _empty_image_tensor() -> torch.Tensor:
return torch.zeros(EMPTY_IMAGE_SHAPE, dtype=torch.float32)
def _normalise_b64_payload(payload: str) -> str:
if payload and "," in payload:
return payload.split(",", 1)[1]
return payload or ""
def _image_to_tensor(img: Image.Image) -> torch.Tensor:
arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
def _resolve_input_path(path: str, input_dir: str = None) -> str | None:
if not path:
return None
input_dir = input_dir or folder_paths.get_input_directory()
clean_path = path.replace("\\", "/")
candidates = [
os.path.join(input_dir, clean_path),
os.path.join(input_dir, "whatdreamscost", os.path.basename(clean_path)),
os.path.join(input_dir, os.path.basename(clean_path)),
]
for candidate in candidates:
if os.path.exists(candidate):
return candidate
return None
def _load_image_reference(b64_or_url: str = "", filename: str = None, cache: dict | None = None,
input_dir: str = None) -> torch.Tensor:
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename."""
if not b64_or_url and not filename:
return _empty_image_tensor()
input_dir = input_dir or folder_paths.get_input_directory()
cache = cache if cache is not None else {}
file_path = None
if b64_or_url and "view?" in b64_or_url:
try:
parsed = urlparse(b64_or_url)
q = parse_qs(parsed.query)
fname = q.get("filename", [None])[0]
subfolder = q.get("subfolder", [""])[0]
if fname:
file_path = _resolve_input_path(os.path.join(subfolder, fname), input_dir)
except Exception as e:
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
if file_path is None and filename:
file_path = _resolve_input_path(filename, input_dir)
if file_path:
cache_key = ("file", file_path)
cached = cache.get(cache_key)
if cached is not None:
return cached
try:
tensor = _image_to_tensor(Image.open(file_path))
cache[cache_key] = tensor
return tensor
except Exception as e:
log.debug(f"[PromptRelay] File image loading failed for {file_path}: {e}")
if b64_or_url:
try:
b64_str = _normalise_b64_payload(b64_or_url)
cache_key = ("b64", b64_str)
cached = cache.get(cache_key)
if cached is not None:
return cached
img_bytes = base64.b64decode(b64_str)
tensor = _image_to_tensor(Image.open(_io.BytesIO(img_bytes)))
cache[cache_key] = tensor
return tensor
except Exception as e:
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
return _empty_image_tensor()
def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> list[float]:
if samples.size == 0:
return [0.0] * num_peaks
step = max(1, int(math.ceil(samples.size / num_peaks)))
padded = np.pad(samples, (0, step * num_peaks - samples.size), mode="constant")
chunks = padded.reshape(num_peaks, step)
return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
def _read_audio_bytes(container, stream, resampler) -> bytes:
audio_bytes = bytearray()
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
for resampled_frame in resampler.resample(None):
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
return bytes(audio_bytes)
def _target_resize_dims(src_w: int, src_h: int, custom_width: int, custom_height: int, divisible_by: int) -> tuple[int, int, str]:
def snap(val, div):
return max(div, (val // div) * div)
if custom_width > 0 and custom_height > 0:
return custom_width, custom_height, None
if custom_width > 0:
tgt_w = snap(custom_width, divisible_by)
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
return tgt_w, tgt_h, "stretch to fit"
if custom_height > 0:
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
return tgt_w, tgt_h, "stretch to fit"
return src_w, src_h, "maintain aspect ratio"
def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
try:
with av.open(file_path) as container:
stream = container.streams.video[0]
return stream.width or stream.codec_context.width, stream.height or stream.codec_context.height
except Exception:
return None, None
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
@@ -111,47 +246,9 @@ def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="",
return gp, " | ".join(processed_locals)
def _load_image_source(b64_or_url: str, filename: str = None) -> torch.Tensor:
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename."""
if not b64_or_url and not filename:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
if b64_or_url and "view?" in b64_or_url:
try:
from urllib.parse import urlparse, parse_qs
parsed = urlparse(b64_or_url)
q = parse_qs(parsed.query)
fname = q.get("filename", [None])[0]
subfolder = q.get("subfolder", [""])[0]
if fname:
file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname)
if os.path.exists(file_path):
img = Image.open(file_path).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except Exception as e:
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
if filename:
file_path = os.path.join(folder_paths.get_input_directory(), filename)
if os.path.exists(file_path):
img = Image.open(file_path).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
if b64_or_url:
try:
b64_str = b64_or_url
if "," in b64_str:
b64_str = b64_str.split(",", 1)[1]
img_bytes = base64.b64decode(b64_str)
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except Exception as e:
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None,
input_dir: str = None) -> torch.Tensor:
return _load_image_reference(b64_or_url=b64_or_url, filename=filename, cache=cache, input_dir=input_dir)
def _execute_comfy_node(node_class, **kwargs):
@@ -392,26 +489,15 @@ async def unload_ollama_endpoint(request):
return web.json_response({"status": "error", "message": str(e)})
def read_wav_peaks(wav_path):
import wave
peaks = []
with wave.open(wav_path, 'rb') as w:
n_frames = w.getnframes()
if n_frames > 0:
frames_bytes = w.readframes(n_frames)
samples = np.frombuffer(frames_bytes, dtype=np.int16)
num_peaks = 200
step = max(1, len(samples) // num_peaks)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
else:
peaks = [0.0] * 200
return peaks
def read_wav_peaks(wav_path):
import wave
with wave.open(wav_path, 'rb') as w:
n_frames = w.getnframes()
if n_frames <= 0:
return [0.0] * PEAK_BUCKETS
frames_bytes = w.readframes(n_frames)
samples = np.frombuffer(frames_bytes, dtype=np.int16)
return _compute_signal_peaks(samples)
def extract_audio_from_video(video_path):
@@ -445,20 +531,9 @@ def extract_audio_from_video(video_path):
rate=44100,
)
audio_bytes = bytearray()
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
# Flush resampler
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
if not audio_bytes:
return None, None
audio_bytes = _read_audio_bytes(container, stream, resampler)
if not audio_bytes:
return None, None
# Write WAV file
with wave.open(output_wav, 'wb') as w:
@@ -468,17 +543,8 @@ def extract_audio_from_video(video_path):
w.writeframes(audio_bytes)
# Calculate peaks
peaks = []
samples = np.frombuffer(audio_bytes, dtype=np.int16)
num_peaks = 200
step = max(1, len(samples) // num_peaks)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
samples = np.frombuffer(audio_bytes, dtype=np.int16)
peaks = _compute_signal_peaks(samples)
input_dir = folder_paths.get_input_directory()
rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/')
@@ -509,30 +575,11 @@ def get_audio_peaks(audio_path):
layout='mono',
rate=8000,
)
audio_bytes = bytearray()
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
if not audio_bytes:
return None
peaks = []
samples = np.frombuffer(audio_bytes, dtype=np.int16)
num_peaks = 200
step = max(1, len(samples) // num_peaks)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
return peaks
audio_bytes = _read_audio_bytes(container, stream, resampler)
if not audio_bytes:
return None
samples = np.frombuffer(audio_bytes, dtype=np.int16)
return _compute_signal_peaks(samples)
except Exception as e:
print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}")
return None
@@ -657,38 +704,24 @@ async def ltx_director_upload_chunk(request):
def _load_image_tensor(seg: dict) -> torch.Tensor:
"""Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64
to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1]."""
if seg.get("imageFile"):
file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"])
if os.path.exists(file_path):
img = Image.open(file_path).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
b64_str = seg.get("imageB64", "")
if not b64_str or b64_str.startswith("/view?"):
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
if "," in b64_str:
b64_str = b64_str.split(",", 1)[1]
try:
img_bytes = base64.b64decode(b64_str)
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
"""Extracts a sequence of frames from a video file based on the segment's trim parameters,
and returns them as an [N, H, W, 3] float32 tensor."""
file_path = os.path.join(folder_paths.get_input_directory(), seg.get("imageFile", ""))
if not os.path.exists(file_path):
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
def _load_image_tensor(seg: dict, cache: dict | None = None, input_dir: str = None) -> torch.Tensor:
"""Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64
to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1]."""
return _load_image_reference(
b64_or_url=seg.get("imageB64", ""),
filename=seg.get("imageFile"),
cache=cache,
input_dir=input_dir,
)
def _load_video_tensor(seg: dict, frame_rate: float, input_dir: str = None) -> torch.Tensor:
"""Extracts a sequence of frames from a video file based on the segment's trim parameters,
and returns them as an [N, H, W, 3] float32 tensor."""
file_path = _resolve_input_path(seg.get("imageFile", ""), input_dir)
if not file_path:
return _empty_image_tensor()
trim_start_frames = float(seg.get("trimStart", 0))
length_frames = float(seg.get("length", 1))
@@ -723,11 +756,11 @@ def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
if len(frames) >= int(length_frames):
break
except Exception as e:
log.warning(f"[PromptRelay] Video extract error: {e}")
if not frames:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
except Exception as e:
log.warning(f"[PromptRelay] Video extract error: {e}")
if not frames:
return _empty_image_tensor()
frames_np = np.array(frames, dtype=np.float32) / 255.0
return torch.from_numpy(frames_np)
@@ -848,21 +881,24 @@ def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor:
return tensor
def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict:
"""Parses timeline JSON, loads/trims audio directly from memory using PyAV,
and aligns to a global timeline yielding ComfyUI's format.
Output length explicitly mimics the timeline's duration_frames length."""
target_sr = 44100
total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr)))
empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr}
if not timeline_data_str:
return empty_audio
try:
data = json.loads(timeline_data_str)
is_retake = data.get("retakeMode", False)
if is_retake and data.get("retakeVideo"):
def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict:
"""Parses timeline JSON, loads/trims audio directly from memory using PyAV,
and aligns to a global timeline yielding ComfyUI's format.
Output length explicitly mimics the timeline's duration_frames length."""
target_sr = 44100
total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr)))
empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr}
if not timeline_data_value:
return empty_audio
try:
if isinstance(timeline_data_value, str):
data = json.loads(timeline_data_value)
else:
data = timeline_data_value
is_retake = data.get("retakeMode", False)
if is_retake and data.get("retakeVideo"):
retake_vid = data.get("retakeVideo")
audio_segs = [{
"videoFile": retake_vid.get("imageFile") or retake_vid.get("fileName"),
@@ -888,17 +924,9 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
buffer = None
file_key = "videoFile" if override_audio else "audioFile"
if seg.get(file_key):
file_path = os.path.join(folder_paths.get_input_directory(), seg[file_key])
if not os.path.exists(file_path):
# Try fallback under whatdreamscost subfolder
basename = os.path.basename(seg[file_key])
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
if os.path.exists(fallback_path):
file_path = fallback_path
if os.path.exists(file_path):
with open(file_path, "rb") as f:
buffer = _io.BytesIO(f.read())
file_path = _resolve_input_path(seg[file_key])
if file_path:
buffer = file_path
if not override_audio and not buffer and seg.get("audioB64"):
b64 = seg.get("audioB64")
@@ -917,7 +945,7 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
clip_frames = []
# Use PyAV to decode directly from memory buffer
with av.open(buffer) as container:
with av.open(buffer) as container:
if not container.streams.audio:
continue
stream = container.streams.audio[0]
@@ -1258,19 +1286,21 @@ class LTXDirector(io.ComfyNode):
)
@classmethod
def execute(cls, model, clip, start_second, end_second, duration_seconds, start_frame, end_frame, duration_frames,
timeline_data, local_prompts, segment_lengths, global_prompt="", guide_strength="", epsilon=1e-3,
frame_rate=24, display_mode="seconds",
custom_width=768, custom_height=512, resize_method="maintain aspect ratio",
divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None,
use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False,
vae=None, reference_strength=1.0, ref_images=None,
start=None, end=None, duration=None) -> io.NodeOutput:
# Parse timeline data
try:
tdata = json.loads(timeline_data) if timeline_data else {}
except Exception as e:
def execute(cls, model, clip, start_second, end_second, duration_seconds, start_frame, end_frame, duration_frames,
timeline_data, local_prompts, segment_lengths, global_prompt="", guide_strength="", epsilon=1e-3,
frame_rate=24, display_mode="seconds",
custom_width=768, custom_height=512, resize_method="maintain aspect ratio",
divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None,
use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False,
vae=None, reference_strength=1.0, ref_images=None,
start=None, end=None, duration=None) -> io.NodeOutput:
input_dir = folder_paths.get_input_directory()
image_cache = {}
# Parse timeline data
try:
tdata = json.loads(timeline_data) if timeline_data else {}
except Exception as e:
log.error(f"[LTXDirector] execute timeline_data parse error: {e}")
tdata = {}
@@ -1327,14 +1357,14 @@ class LTXDirector(io.ComfyNode):
if legacy_b64 and not images_list:
images_list = [{"b64": legacy_b64, "name": char_info.get("fileName", "")}]
slot_tensors = []
for img_info in images_list:
image_b64 = img_info.get("b64", "")
file_name = img_info.get("name", "")
tensor = _load_image_source(image_b64, file_name)
char_images.append(tensor)
slot_tensors.append(tensor)
char_slot_images.append(slot_tensors)
slot_tensors = []
for img_info in images_list:
image_b64 = img_info.get("b64", "")
file_name = img_info.get("name", "")
tensor = _load_image_source(image_b64, file_name, cache=image_cache, input_dir=input_dir)
char_images.append(tensor)
slot_tensors.append(tensor)
char_slot_images.append(slot_tensors)
except Exception as e:
log.warning("[LTXDirector] Could not process character slot inputs: %s", e)
@@ -1367,40 +1397,33 @@ class LTXDirector(io.ComfyNode):
if guide_strength.strip():
strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()]
for idx, seg in enumerate(img_segs):
seg_start = int(seg.get("start", 0))
offset = max(0, start_frame - seg_start)
if seg.get("type") == "video":
if offset > 0:
seg["trimStart"] = float(seg.get("trimStart", 0)) + offset
seg["length"] = max(1, int(seg.get("length", 1)) - offset)
tensor = _load_video_tensor(seg, float(frame_rate))
else:
tensor = _load_image_tensor(seg)
# Apply resize
src_h, src_w = tensor.shape[1], tensor.shape[2]
def snap(val, div):
return max(div, (val // div) * div)
if custom_width > 0 and custom_height > 0:
# Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR)
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by)
elif custom_width > 0:
# Width only — scale height from AR, snap both, then resize to exact dimensions
tgt_w = snap(custom_width, divisible_by)
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
elif custom_height > 0:
# Height only — scale width from AR, snap both, then resize to exact dimensions
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
else:
# Both zero — keep original dimensions, just snap to divisible_by
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
for idx, seg in enumerate(img_segs):
seg_start = int(seg.get("start", 0))
offset = max(0, start_frame - seg_start)
seg_length = int(seg.get("length", 1))
seg_for_load = seg
if seg.get("type") == "video":
if offset > 0:
seg_for_load = dict(seg)
seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset
seg_for_load["length"] = max(1, seg_length - offset)
tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir)
else:
tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir)
# Apply resize
src_h, src_w = tensor.shape[1], tensor.shape[2]
tgt_w, tgt_h, resize_override = _target_resize_dims(
src_w, src_h, custom_width, custom_height, divisible_by
)
tensor = _resize_image(
tensor,
tgt_w,
tgt_h,
resize_override or resize_method,
divisible_by,
)
# Apply compression
@@ -1408,15 +1431,15 @@ class LTXDirector(io.ComfyNode):
tensor = _compress_image(tensor, img_compression)
# Record dimensions of the first processed image for latent generation
if idx == 0:
derived_h = tensor.shape[1]
derived_w = tensor.shape[2]
if seg.get("isEndFrame"):
insert_frame = max(0, seg_start + int(seg.get("length", 1)) - 1 - start_frame)
else:
insert_frame = max(0, seg_start - start_frame)
strength = strengths[idx] if idx < len(strengths) else 1.0
if idx == 0:
derived_h = tensor.shape[1]
derived_w = tensor.shape[2]
if seg.get("isEndFrame"):
insert_frame = max(0, seg_start + seg_length - 1 - start_frame)
else:
insert_frame = max(0, seg_start - start_frame)
strength = strengths[idx] if idx < len(strengths) else 1.0
guide_data["images"].append(tensor)
guide_data["insert_frames"].append(insert_frame)
guide_data["strengths"].append(float(strength))
@@ -1424,75 +1447,49 @@ class LTXDirector(io.ComfyNode):
# If no images were loaded from the timeline, create a dummy image at strength 0
# to prevent artifacts in text-to-video mode.
if not guide_data["images"] and optional_latent is None:
src_w = derived_w if derived_w > 0 else 768
src_h = derived_h if derived_h > 0 else 512
# If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling
tdata_motion = json.loads(timeline_data) if timeline_data else {}
found_dims = False
# Check for retake base video first
is_retake = tdata_motion.get("retakeMode", False)
retake_vid = tdata_motion.get("retakeVideo") or {}
retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else ""
if is_retake and retake_file:
r_path = os.path.join(folder_paths.get_input_directory(), retake_file)
if not os.path.exists(r_path):
basename = os.path.basename(retake_file)
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
if os.path.exists(fallback_path):
r_path = fallback_path
if os.path.exists(r_path):
try:
with av.open(r_path) as container:
stream = container.streams.video[0]
src_w = stream.width or stream.codec_context.width
src_h = stream.height or stream.codec_context.height
found_dims = True
except:
pass
# Fallback to normal motion segments
if not found_dims:
for mseg in tdata_motion.get("motionSegments", []):
v_file = mseg.get("videoFile")
if v_file:
v_path = os.path.join(folder_paths.get_input_directory(), v_file)
if not os.path.exists(v_path):
basename = os.path.basename(v_file)
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
if os.path.exists(fallback_path):
v_path = fallback_path
if os.path.exists(v_path):
try:
with av.open(v_path) as container:
stream = container.streams.video[0]
src_w = stream.width or stream.codec_context.width
src_h = stream.height or stream.codec_context.height
found_dims = True
break
except:
pass
# Create a dummy tensor of the exact source dimensions
tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32)
def snap(val, div):
return max(div, (val // div) * div)
# Route the dummy tensor through the exact same resizing pipeline
if custom_width > 0 and custom_height > 0:
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by)
elif custom_width > 0:
tgt_w = snap(custom_width, divisible_by)
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
elif custom_height > 0:
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
else:
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
src_w = derived_w if derived_w > 0 else 768
src_h = derived_h if derived_h > 0 else 512
# If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling
found_dims = False
# Check for retake base video first
is_retake = tdata.get("retakeMode", False)
retake_vid = tdata.get("retakeVideo") or {}
retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else ""
if is_retake and retake_file:
r_path = _resolve_input_path(retake_file, input_dir)
if r_path:
src_dims = _extract_video_dimensions(r_path)
if all(src_dims):
src_w, src_h = src_dims
found_dims = True
# Fallback to normal motion segments
if not found_dims:
for mseg in tdata.get("motionSegments", []):
v_file = mseg.get("videoFile")
if v_file:
v_path = _resolve_input_path(v_file, input_dir)
if v_path:
src_dims = _extract_video_dimensions(v_path)
if all(src_dims):
src_w, src_h = src_dims
found_dims = True
break
# Create a dummy tensor of the exact source dimensions
tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32)
tgt_w, tgt_h, resize_override = _target_resize_dims(
src_w, src_h, custom_width, custom_height, divisible_by
)
tensor = _resize_image(
tensor,
tgt_w,
tgt_h,
resize_override or resize_method,
divisible_by,
)
guide_data["images"].append(tensor)
guide_data["insert_frames"].append(0)
@@ -1703,7 +1700,7 @@ class LTXDirector(io.ComfyNode):
)
# --- Build Audio Output ---
audio_out = _build_combined_audio(timeline_data, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
# --- Audio Latent Generation ---
audio_latent = {}
@@ -1831,13 +1828,12 @@ class LTXDirector(io.ComfyNode):
log.error("[PromptRelay] Could not generate empty audio latent: %s", e)
raise e
# --- Motion guide output from timeline video segments ---
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
try:
tdata = json.loads(timeline_data) if timeline_data else {}
if use_custom_motion:
motion_segments = tdata.get("motionSegments", [])
else:
# --- Motion guide output from timeline video segments ---
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
try:
if use_custom_motion:
motion_segments = tdata.get("motionSegments", [])
else:
motion_segments = []
for seg in motion_segments:
seg_start = int(seg.get("start", 0))
@@ -1878,4 +1874,4 @@ NODE_CLASS_MAPPINGS = {
NODE_DISPLAY_NAME_MAPPINGS = {
"PromptRelayEncodeTimeline": "Prompt Relay Encode (Timeline)",
}
}