perf: reduce redundant media work in director

This commit is contained in:
OpenClaw Agent
2026-07-09 13:29:55 +00:00
parent 6695b1a1aa
commit b8cedad997

View File

@@ -14,6 +14,7 @@ from PIL import Image
import os import os
import platform import platform
import subprocess import subprocess
from urllib.parse import parse_qs, urlparse
import folder_paths import folder_paths
import comfy.model_management import comfy.model_management
from server import PromptServer from server import PromptServer
@@ -73,6 +74,140 @@ MSR_PREFIX_FRAMES = 41
# trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any # trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any
# multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream. # multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream.
MSR_LATENT_DOWNSCALE = 1.0 MSR_LATENT_DOWNSCALE = 1.0
EMPTY_IMAGE_SHAPE = (1, 512, 512, 3)
PEAK_BUCKETS = 200
def _empty_image_tensor() -> torch.Tensor:
return torch.zeros(EMPTY_IMAGE_SHAPE, dtype=torch.float32)
def _normalise_b64_payload(payload: str) -> str:
if payload and "," in payload:
return payload.split(",", 1)[1]
return payload or ""
def _image_to_tensor(img: Image.Image) -> torch.Tensor:
arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
def _resolve_input_path(path: str, input_dir: str = None) -> str | None:
if not path:
return None
input_dir = input_dir or folder_paths.get_input_directory()
clean_path = path.replace("\\", "/")
candidates = [
os.path.join(input_dir, clean_path),
os.path.join(input_dir, "whatdreamscost", os.path.basename(clean_path)),
os.path.join(input_dir, os.path.basename(clean_path)),
]
for candidate in candidates:
if os.path.exists(candidate):
return candidate
return None
def _load_image_reference(b64_or_url: str = "", filename: str = None, cache: dict | None = None,
input_dir: str = None) -> torch.Tensor:
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename."""
if not b64_or_url and not filename:
return _empty_image_tensor()
input_dir = input_dir or folder_paths.get_input_directory()
cache = cache if cache is not None else {}
file_path = None
if b64_or_url and "view?" in b64_or_url:
try:
parsed = urlparse(b64_or_url)
q = parse_qs(parsed.query)
fname = q.get("filename", [None])[0]
subfolder = q.get("subfolder", [""])[0]
if fname:
file_path = _resolve_input_path(os.path.join(subfolder, fname), input_dir)
except Exception as e:
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
if file_path is None and filename:
file_path = _resolve_input_path(filename, input_dir)
if file_path:
cache_key = ("file", file_path)
cached = cache.get(cache_key)
if cached is not None:
return cached
try:
tensor = _image_to_tensor(Image.open(file_path))
cache[cache_key] = tensor
return tensor
except Exception as e:
log.debug(f"[PromptRelay] File image loading failed for {file_path}: {e}")
if b64_or_url:
try:
b64_str = _normalise_b64_payload(b64_or_url)
cache_key = ("b64", b64_str)
cached = cache.get(cache_key)
if cached is not None:
return cached
img_bytes = base64.b64decode(b64_str)
tensor = _image_to_tensor(Image.open(_io.BytesIO(img_bytes)))
cache[cache_key] = tensor
return tensor
except Exception as e:
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
return _empty_image_tensor()
def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> list[float]:
if samples.size == 0:
return [0.0] * num_peaks
step = max(1, int(math.ceil(samples.size / num_peaks)))
padded = np.pad(samples, (0, step * num_peaks - samples.size), mode="constant")
chunks = padded.reshape(num_peaks, step)
return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
def _read_audio_bytes(container, stream, resampler) -> bytes:
audio_bytes = bytearray()
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
for resampled_frame in resampler.resample(None):
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
return bytes(audio_bytes)
def _target_resize_dims(src_w: int, src_h: int, custom_width: int, custom_height: int, divisible_by: int) -> tuple[int, int, str]:
def snap(val, div):
return max(div, (val // div) * div)
if custom_width > 0 and custom_height > 0:
return custom_width, custom_height, None
if custom_width > 0:
tgt_w = snap(custom_width, divisible_by)
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
return tgt_w, tgt_h, "stretch to fit"
if custom_height > 0:
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
return tgt_w, tgt_h, "stretch to fit"
return src_w, src_h, "maintain aspect ratio"
def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
try:
with av.open(file_path) as container:
stream = container.streams.video[0]
return stream.width or stream.codec_context.width, stream.height or stream.codec_context.height
except Exception:
return None, None
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""): def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
@@ -111,47 +246,9 @@ def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="",
return gp, " | ".join(processed_locals) return gp, " | ".join(processed_locals)
def _load_image_source(b64_or_url: str, filename: str = None) -> torch.Tensor: def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None,
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename.""" input_dir: str = None) -> torch.Tensor:
if not b64_or_url and not filename: return _load_image_reference(b64_or_url=b64_or_url, filename=filename, cache=cache, input_dir=input_dir)
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
if b64_or_url and "view?" in b64_or_url:
try:
from urllib.parse import urlparse, parse_qs
parsed = urlparse(b64_or_url)
q = parse_qs(parsed.query)
fname = q.get("filename", [None])[0]
subfolder = q.get("subfolder", [""])[0]
if fname:
file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname)
if os.path.exists(file_path):
img = Image.open(file_path).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except Exception as e:
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
if filename:
file_path = os.path.join(folder_paths.get_input_directory(), filename)
if os.path.exists(file_path):
img = Image.open(file_path).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
if b64_or_url:
try:
b64_str = b64_or_url
if "," in b64_str:
b64_str = b64_str.split(",", 1)[1]
img_bytes = base64.b64decode(b64_str)
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except Exception as e:
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
def _execute_comfy_node(node_class, **kwargs): def _execute_comfy_node(node_class, **kwargs):
@@ -394,24 +491,13 @@ async def unload_ollama_endpoint(request):
def read_wav_peaks(wav_path): def read_wav_peaks(wav_path):
import wave import wave
peaks = []
with wave.open(wav_path, 'rb') as w: with wave.open(wav_path, 'rb') as w:
n_frames = w.getnframes() n_frames = w.getnframes()
if n_frames > 0: if n_frames <= 0:
frames_bytes = w.readframes(n_frames) return [0.0] * PEAK_BUCKETS
samples = np.frombuffer(frames_bytes, dtype=np.int16) frames_bytes = w.readframes(n_frames)
num_peaks = 200 samples = np.frombuffer(frames_bytes, dtype=np.int16)
step = max(1, len(samples) // num_peaks) return _compute_signal_peaks(samples)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
else:
peaks = [0.0] * 200
return peaks
def extract_audio_from_video(video_path): def extract_audio_from_video(video_path):
@@ -445,18 +531,7 @@ def extract_audio_from_video(video_path):
rate=44100, rate=44100,
) )
audio_bytes = bytearray() audio_bytes = _read_audio_bytes(container, stream, resampler)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
# Flush resampler
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
if not audio_bytes: if not audio_bytes:
return None, None return None, None
@@ -468,17 +543,8 @@ def extract_audio_from_video(video_path):
w.writeframes(audio_bytes) w.writeframes(audio_bytes)
# Calculate peaks # Calculate peaks
peaks = []
samples = np.frombuffer(audio_bytes, dtype=np.int16) samples = np.frombuffer(audio_bytes, dtype=np.int16)
num_peaks = 200 peaks = _compute_signal_peaks(samples)
step = max(1, len(samples) // num_peaks)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
input_dir = folder_paths.get_input_directory() input_dir = folder_paths.get_input_directory()
rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/') rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/')
@@ -509,30 +575,11 @@ def get_audio_peaks(audio_path):
layout='mono', layout='mono',
rate=8000, rate=8000,
) )
audio_bytes = bytearray() audio_bytes = _read_audio_bytes(container, stream, resampler)
for frame in container.decode(stream):
for resampled_frame in resampler.resample(frame):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
for resampled_frame in resampler.resample(None):
arr = resampled_frame.to_ndarray()
audio_bytes.extend(arr.tobytes())
if not audio_bytes: if not audio_bytes:
return None return None
peaks = []
samples = np.frombuffer(audio_bytes, dtype=np.int16) samples = np.frombuffer(audio_bytes, dtype=np.int16)
num_peaks = 200 return _compute_signal_peaks(samples)
step = max(1, len(samples) // num_peaks)
for i in range(num_peaks):
chunk = samples[i * step : (i + 1) * step]
if len(chunk) > 0:
max_val = np.max(np.abs(chunk)) / 32767.0
peaks.append(float(max_val))
else:
peaks.append(0.0)
return peaks
except Exception as e: except Exception as e:
print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}") print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}")
return None return None
@@ -657,38 +704,24 @@ async def ltx_director_upload_chunk(request):
def _load_image_tensor(seg: dict) -> torch.Tensor: def _load_image_tensor(seg: dict, cache: dict | None = None, input_dir: str = None) -> torch.Tensor:
"""Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64
to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1]."""
if seg.get("imageFile"): return _load_image_reference(
file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"]) b64_or_url=seg.get("imageB64", ""),
if os.path.exists(file_path): filename=seg.get("imageFile"),
img = Image.open(file_path).convert("RGB") cache=cache,
arr = np.array(img, dtype=np.float32) / 255.0 input_dir=input_dir,
return torch.from_numpy(arr).unsqueeze(0) )
b64_str = seg.get("imageB64", "")
if not b64_str or b64_str.startswith("/view?"):
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
if "," in b64_str: def _load_video_tensor(seg: dict, frame_rate: float, input_dir: str = None) -> torch.Tensor:
b64_str = b64_str.split(",", 1)[1]
try:
img_bytes = base64.b64decode(b64_str)
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
arr = np.array(img, dtype=np.float32) / 255.0
return torch.from_numpy(arr).unsqueeze(0)
except:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
"""Extracts a sequence of frames from a video file based on the segment's trim parameters, """Extracts a sequence of frames from a video file based on the segment's trim parameters,
and returns them as an [N, H, W, 3] float32 tensor.""" and returns them as an [N, H, W, 3] float32 tensor."""
file_path = os.path.join(folder_paths.get_input_directory(), seg.get("imageFile", "")) file_path = _resolve_input_path(seg.get("imageFile", ""), input_dir)
if not os.path.exists(file_path): if not file_path:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32) return _empty_image_tensor()
trim_start_frames = float(seg.get("trimStart", 0)) trim_start_frames = float(seg.get("trimStart", 0))
length_frames = float(seg.get("length", 1)) length_frames = float(seg.get("length", 1))
@@ -727,7 +760,7 @@ def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
log.warning(f"[PromptRelay] Video extract error: {e}") log.warning(f"[PromptRelay] Video extract error: {e}")
if not frames: if not frames:
return torch.zeros((1, 512, 512, 3), dtype=torch.float32) return _empty_image_tensor()
frames_np = np.array(frames, dtype=np.float32) / 255.0 frames_np = np.array(frames, dtype=np.float32) / 255.0
return torch.from_numpy(frames_np) return torch.from_numpy(frames_np)
@@ -848,7 +881,7 @@ def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor:
return tensor return tensor
def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict: def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict:
"""Parses timeline JSON, loads/trims audio directly from memory using PyAV, """Parses timeline JSON, loads/trims audio directly from memory using PyAV,
and aligns to a global timeline yielding ComfyUI's format. and aligns to a global timeline yielding ComfyUI's format.
Output length explicitly mimics the timeline's duration_frames length.""" Output length explicitly mimics the timeline's duration_frames length."""
@@ -856,11 +889,14 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr))) total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr)))
empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr} empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr}
if not timeline_data_str: if not timeline_data_value:
return empty_audio return empty_audio
try: try:
data = json.loads(timeline_data_str) if isinstance(timeline_data_value, str):
data = json.loads(timeline_data_value)
else:
data = timeline_data_value
is_retake = data.get("retakeMode", False) is_retake = data.get("retakeMode", False)
if is_retake and data.get("retakeVideo"): if is_retake and data.get("retakeVideo"):
retake_vid = data.get("retakeVideo") retake_vid = data.get("retakeVideo")
@@ -888,17 +924,9 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
buffer = None buffer = None
file_key = "videoFile" if override_audio else "audioFile" file_key = "videoFile" if override_audio else "audioFile"
if seg.get(file_key): if seg.get(file_key):
file_path = os.path.join(folder_paths.get_input_directory(), seg[file_key]) file_path = _resolve_input_path(seg[file_key])
if not os.path.exists(file_path): if file_path:
# Try fallback under whatdreamscost subfolder buffer = file_path
basename = os.path.basename(seg[file_key])
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
if os.path.exists(fallback_path):
file_path = fallback_path
if os.path.exists(file_path):
with open(file_path, "rb") as f:
buffer = _io.BytesIO(f.read())
if not override_audio and not buffer and seg.get("audioB64"): if not override_audio and not buffer and seg.get("audioB64"):
b64 = seg.get("audioB64") b64 = seg.get("audioB64")
@@ -1266,6 +1294,8 @@ class LTXDirector(io.ComfyNode):
use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False, use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False,
vae=None, reference_strength=1.0, ref_images=None, vae=None, reference_strength=1.0, ref_images=None,
start=None, end=None, duration=None) -> io.NodeOutput: start=None, end=None, duration=None) -> io.NodeOutput:
input_dir = folder_paths.get_input_directory()
image_cache = {}
# Parse timeline data # Parse timeline data
try: try:
@@ -1331,7 +1361,7 @@ class LTXDirector(io.ComfyNode):
for img_info in images_list: for img_info in images_list:
image_b64 = img_info.get("b64", "") image_b64 = img_info.get("b64", "")
file_name = img_info.get("name", "") file_name = img_info.get("name", "")
tensor = _load_image_source(image_b64, file_name) tensor = _load_image_source(image_b64, file_name, cache=image_cache, input_dir=input_dir)
char_images.append(tensor) char_images.append(tensor)
slot_tensors.append(tensor) slot_tensors.append(tensor)
char_slot_images.append(slot_tensors) char_slot_images.append(slot_tensors)
@@ -1370,37 +1400,30 @@ class LTXDirector(io.ComfyNode):
for idx, seg in enumerate(img_segs): for idx, seg in enumerate(img_segs):
seg_start = int(seg.get("start", 0)) seg_start = int(seg.get("start", 0))
offset = max(0, start_frame - seg_start) offset = max(0, start_frame - seg_start)
seg_length = int(seg.get("length", 1))
seg_for_load = seg
if seg.get("type") == "video": if seg.get("type") == "video":
if offset > 0: if offset > 0:
seg["trimStart"] = float(seg.get("trimStart", 0)) + offset seg_for_load = dict(seg)
seg["length"] = max(1, int(seg.get("length", 1)) - offset) seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset
tensor = _load_video_tensor(seg, float(frame_rate)) seg_for_load["length"] = max(1, seg_length - offset)
tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir)
else: else:
tensor = _load_image_tensor(seg) tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir)
# Apply resize # Apply resize
src_h, src_w = tensor.shape[1], tensor.shape[2] src_h, src_w = tensor.shape[1], tensor.shape[2]
tgt_w, tgt_h, resize_override = _target_resize_dims(
def snap(val, div): src_w, src_h, custom_width, custom_height, divisible_by
return max(div, (val // div) * div) )
tensor = _resize_image(
if custom_width > 0 and custom_height > 0: tensor,
# Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR) tgt_w,
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) tgt_h,
elif custom_width > 0: resize_override or resize_method,
# Width only — scale height from AR, snap both, then resize to exact dimensions divisible_by,
tgt_w = snap(custom_width, divisible_by) )
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
elif custom_height > 0:
# Height only — scale width from AR, snap both, then resize to exact dimensions
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
else:
# Both zero — keep original dimensions, just snap to divisible_by
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
# Apply compression # Apply compression
@@ -1413,7 +1436,7 @@ class LTXDirector(io.ComfyNode):
derived_w = tensor.shape[2] derived_w = tensor.shape[2]
if seg.get("isEndFrame"): if seg.get("isEndFrame"):
insert_frame = max(0, seg_start + int(seg.get("length", 1)) - 1 - start_frame) insert_frame = max(0, seg_start + seg_length - 1 - start_frame)
else: else:
insert_frame = max(0, seg_start - start_frame) insert_frame = max(0, seg_start - start_frame)
strength = strengths[idx] if idx < len(strengths) else 1.0 strength = strengths[idx] if idx < len(strengths) else 1.0
@@ -1428,71 +1451,45 @@ class LTXDirector(io.ComfyNode):
src_h = derived_h if derived_h > 0 else 512 src_h = derived_h if derived_h > 0 else 512
# If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling # If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling
tdata_motion = json.loads(timeline_data) if timeline_data else {}
found_dims = False found_dims = False
# Check for retake base video first # Check for retake base video first
is_retake = tdata_motion.get("retakeMode", False) is_retake = tdata.get("retakeMode", False)
retake_vid = tdata_motion.get("retakeVideo") or {} retake_vid = tdata.get("retakeVideo") or {}
retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else "" retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else ""
if is_retake and retake_file: if is_retake and retake_file:
r_path = os.path.join(folder_paths.get_input_directory(), retake_file) r_path = _resolve_input_path(retake_file, input_dir)
if not os.path.exists(r_path): if r_path:
basename = os.path.basename(retake_file) src_dims = _extract_video_dimensions(r_path)
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) if all(src_dims):
if os.path.exists(fallback_path): src_w, src_h = src_dims
r_path = fallback_path found_dims = True
if os.path.exists(r_path):
try:
with av.open(r_path) as container:
stream = container.streams.video[0]
src_w = stream.width or stream.codec_context.width
src_h = stream.height or stream.codec_context.height
found_dims = True
except:
pass
# Fallback to normal motion segments # Fallback to normal motion segments
if not found_dims: if not found_dims:
for mseg in tdata_motion.get("motionSegments", []): for mseg in tdata.get("motionSegments", []):
v_file = mseg.get("videoFile") v_file = mseg.get("videoFile")
if v_file: if v_file:
v_path = os.path.join(folder_paths.get_input_directory(), v_file) v_path = _resolve_input_path(v_file, input_dir)
if not os.path.exists(v_path): if v_path:
basename = os.path.basename(v_file) src_dims = _extract_video_dimensions(v_path)
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename) if all(src_dims):
if os.path.exists(fallback_path): src_w, src_h = src_dims
v_path = fallback_path found_dims = True
if os.path.exists(v_path): break
try:
with av.open(v_path) as container:
stream = container.streams.video[0]
src_w = stream.width or stream.codec_context.width
src_h = stream.height or stream.codec_context.height
found_dims = True
break
except:
pass
# Create a dummy tensor of the exact source dimensions # Create a dummy tensor of the exact source dimensions
tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32) tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32)
tgt_w, tgt_h, resize_override = _target_resize_dims(
def snap(val, div): src_w, src_h, custom_width, custom_height, divisible_by
return max(div, (val // div) * div) )
tensor = _resize_image(
# Route the dummy tensor through the exact same resizing pipeline tensor,
if custom_width > 0 and custom_height > 0: tgt_w,
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) tgt_h,
elif custom_width > 0: resize_override or resize_method,
tgt_w = snap(custom_width, divisible_by) divisible_by,
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) )
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
elif custom_height > 0:
tgt_h = snap(custom_height, divisible_by)
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
else:
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
guide_data["images"].append(tensor) guide_data["images"].append(tensor)
guide_data["insert_frames"].append(0) guide_data["insert_frames"].append(0)
@@ -1703,7 +1700,7 @@ class LTXDirector(io.ComfyNode):
) )
# --- Build Audio Output --- # --- Build Audio Output ---
audio_out = _build_combined_audio(timeline_data, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio) audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
# --- Audio Latent Generation --- # --- Audio Latent Generation ---
audio_latent = {} audio_latent = {}
@@ -1834,7 +1831,6 @@ class LTXDirector(io.ComfyNode):
# --- Motion guide output from timeline video segments --- # --- Motion guide output from timeline video segments ---
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method} motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
try: try:
tdata = json.loads(timeline_data) if timeline_data else {}
if use_custom_motion: if use_custom_motion:
motion_segments = tdata.get("motionSegments", []) motion_segments = tdata.get("motionSegments", [])
else: else: