perf: reduce redundant media work in director
This commit is contained in:
426
ltx_director.py
426
ltx_director.py
@@ -14,6 +14,7 @@ from PIL import Image
|
|||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from urllib.parse import parse_qs, urlparse
|
||||||
import folder_paths
|
import folder_paths
|
||||||
import comfy.model_management
|
import comfy.model_management
|
||||||
from server import PromptServer
|
from server import PromptServer
|
||||||
@@ -73,6 +74,140 @@ MSR_PREFIX_FRAMES = 41
|
|||||||
# trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any
|
# trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any
|
||||||
# multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream.
|
# multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream.
|
||||||
MSR_LATENT_DOWNSCALE = 1.0
|
MSR_LATENT_DOWNSCALE = 1.0
|
||||||
|
EMPTY_IMAGE_SHAPE = (1, 512, 512, 3)
|
||||||
|
PEAK_BUCKETS = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_image_tensor() -> torch.Tensor:
|
||||||
|
return torch.zeros(EMPTY_IMAGE_SHAPE, dtype=torch.float32)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_b64_payload(payload: str) -> str:
|
||||||
|
if payload and "," in payload:
|
||||||
|
return payload.split(",", 1)[1]
|
||||||
|
return payload or ""
|
||||||
|
|
||||||
|
|
||||||
|
def _image_to_tensor(img: Image.Image) -> torch.Tensor:
|
||||||
|
arr = np.asarray(img.convert("RGB"), dtype=np.float32) / 255.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_input_path(path: str, input_dir: str = None) -> str | None:
|
||||||
|
if not path:
|
||||||
|
return None
|
||||||
|
|
||||||
|
input_dir = input_dir or folder_paths.get_input_directory()
|
||||||
|
clean_path = path.replace("\\", "/")
|
||||||
|
candidates = [
|
||||||
|
os.path.join(input_dir, clean_path),
|
||||||
|
os.path.join(input_dir, "whatdreamscost", os.path.basename(clean_path)),
|
||||||
|
os.path.join(input_dir, os.path.basename(clean_path)),
|
||||||
|
]
|
||||||
|
|
||||||
|
for candidate in candidates:
|
||||||
|
if os.path.exists(candidate):
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_image_reference(b64_or_url: str = "", filename: str = None, cache: dict | None = None,
|
||||||
|
input_dir: str = None) -> torch.Tensor:
|
||||||
|
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename."""
|
||||||
|
if not b64_or_url and not filename:
|
||||||
|
return _empty_image_tensor()
|
||||||
|
|
||||||
|
input_dir = input_dir or folder_paths.get_input_directory()
|
||||||
|
cache = cache if cache is not None else {}
|
||||||
|
|
||||||
|
file_path = None
|
||||||
|
if b64_or_url and "view?" in b64_or_url:
|
||||||
|
try:
|
||||||
|
parsed = urlparse(b64_or_url)
|
||||||
|
q = parse_qs(parsed.query)
|
||||||
|
fname = q.get("filename", [None])[0]
|
||||||
|
subfolder = q.get("subfolder", [""])[0]
|
||||||
|
if fname:
|
||||||
|
file_path = _resolve_input_path(os.path.join(subfolder, fname), input_dir)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
|
||||||
|
|
||||||
|
if file_path is None and filename:
|
||||||
|
file_path = _resolve_input_path(filename, input_dir)
|
||||||
|
|
||||||
|
if file_path:
|
||||||
|
cache_key = ("file", file_path)
|
||||||
|
cached = cache.get(cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
try:
|
||||||
|
tensor = _image_to_tensor(Image.open(file_path))
|
||||||
|
cache[cache_key] = tensor
|
||||||
|
return tensor
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"[PromptRelay] File image loading failed for {file_path}: {e}")
|
||||||
|
|
||||||
|
if b64_or_url:
|
||||||
|
try:
|
||||||
|
b64_str = _normalise_b64_payload(b64_or_url)
|
||||||
|
cache_key = ("b64", b64_str)
|
||||||
|
cached = cache.get(cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
img_bytes = base64.b64decode(b64_str)
|
||||||
|
tensor = _image_to_tensor(Image.open(_io.BytesIO(img_bytes)))
|
||||||
|
cache[cache_key] = tensor
|
||||||
|
return tensor
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
|
||||||
|
|
||||||
|
return _empty_image_tensor()
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> list[float]:
|
||||||
|
if samples.size == 0:
|
||||||
|
return [0.0] * num_peaks
|
||||||
|
|
||||||
|
step = max(1, int(math.ceil(samples.size / num_peaks)))
|
||||||
|
padded = np.pad(samples, (0, step * num_peaks - samples.size), mode="constant")
|
||||||
|
chunks = padded.reshape(num_peaks, step)
|
||||||
|
return (np.max(np.abs(chunks), axis=1) / 32767.0).astype(np.float32).tolist()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_audio_bytes(container, stream, resampler) -> bytes:
|
||||||
|
audio_bytes = bytearray()
|
||||||
|
for frame in container.decode(stream):
|
||||||
|
for resampled_frame in resampler.resample(frame):
|
||||||
|
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
|
||||||
|
for resampled_frame in resampler.resample(None):
|
||||||
|
audio_bytes.extend(resampled_frame.to_ndarray().tobytes())
|
||||||
|
return bytes(audio_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def _target_resize_dims(src_w: int, src_h: int, custom_width: int, custom_height: int, divisible_by: int) -> tuple[int, int, str]:
|
||||||
|
def snap(val, div):
|
||||||
|
return max(div, (val // div) * div)
|
||||||
|
|
||||||
|
if custom_width > 0 and custom_height > 0:
|
||||||
|
return custom_width, custom_height, None
|
||||||
|
if custom_width > 0:
|
||||||
|
tgt_w = snap(custom_width, divisible_by)
|
||||||
|
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
|
||||||
|
return tgt_w, tgt_h, "stretch to fit"
|
||||||
|
if custom_height > 0:
|
||||||
|
tgt_h = snap(custom_height, divisible_by)
|
||||||
|
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
|
||||||
|
return tgt_w, tgt_h, "stretch to fit"
|
||||||
|
return src_w, src_h, "maintain aspect ratio"
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_video_dimensions(file_path: str) -> tuple[int | None, int | None]:
|
||||||
|
try:
|
||||||
|
with av.open(file_path) as container:
|
||||||
|
stream = container.streams.video[0]
|
||||||
|
return stream.width or stream.codec_context.width, stream.height or stream.codec_context.height
|
||||||
|
except Exception:
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
|
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
|
||||||
@@ -111,47 +246,9 @@ def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="",
|
|||||||
return gp, " | ".join(processed_locals)
|
return gp, " | ".join(processed_locals)
|
||||||
|
|
||||||
|
|
||||||
def _load_image_source(b64_or_url: str, filename: str = None) -> torch.Tensor:
|
def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None,
|
||||||
"""Load a reference image from a base64 string, a Comfy /view URL, or an input filename."""
|
input_dir: str = None) -> torch.Tensor:
|
||||||
if not b64_or_url and not filename:
|
return _load_image_reference(b64_or_url=b64_or_url, filename=filename, cache=cache, input_dir=input_dir)
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
|
||||||
|
|
||||||
if b64_or_url and "view?" in b64_or_url:
|
|
||||||
try:
|
|
||||||
from urllib.parse import urlparse, parse_qs
|
|
||||||
parsed = urlparse(b64_or_url)
|
|
||||||
q = parse_qs(parsed.query)
|
|
||||||
fname = q.get("filename", [None])[0]
|
|
||||||
subfolder = q.get("subfolder", [""])[0]
|
|
||||||
if fname:
|
|
||||||
file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname)
|
|
||||||
if os.path.exists(file_path):
|
|
||||||
img = Image.open(file_path).convert("RGB")
|
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
|
||||||
return torch.from_numpy(arr).unsqueeze(0)
|
|
||||||
except Exception as e:
|
|
||||||
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
|
|
||||||
|
|
||||||
if filename:
|
|
||||||
file_path = os.path.join(folder_paths.get_input_directory(), filename)
|
|
||||||
if os.path.exists(file_path):
|
|
||||||
img = Image.open(file_path).convert("RGB")
|
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
|
||||||
return torch.from_numpy(arr).unsqueeze(0)
|
|
||||||
|
|
||||||
if b64_or_url:
|
|
||||||
try:
|
|
||||||
b64_str = b64_or_url
|
|
||||||
if "," in b64_str:
|
|
||||||
b64_str = b64_str.split(",", 1)[1]
|
|
||||||
img_bytes = base64.b64decode(b64_str)
|
|
||||||
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
|
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
|
||||||
return torch.from_numpy(arr).unsqueeze(0)
|
|
||||||
except Exception as e:
|
|
||||||
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
|
|
||||||
|
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
|
||||||
|
|
||||||
|
|
||||||
def _execute_comfy_node(node_class, **kwargs):
|
def _execute_comfy_node(node_class, **kwargs):
|
||||||
@@ -394,24 +491,13 @@ async def unload_ollama_endpoint(request):
|
|||||||
|
|
||||||
def read_wav_peaks(wav_path):
|
def read_wav_peaks(wav_path):
|
||||||
import wave
|
import wave
|
||||||
peaks = []
|
|
||||||
with wave.open(wav_path, 'rb') as w:
|
with wave.open(wav_path, 'rb') as w:
|
||||||
n_frames = w.getnframes()
|
n_frames = w.getnframes()
|
||||||
if n_frames > 0:
|
if n_frames <= 0:
|
||||||
|
return [0.0] * PEAK_BUCKETS
|
||||||
frames_bytes = w.readframes(n_frames)
|
frames_bytes = w.readframes(n_frames)
|
||||||
samples = np.frombuffer(frames_bytes, dtype=np.int16)
|
samples = np.frombuffer(frames_bytes, dtype=np.int16)
|
||||||
num_peaks = 200
|
return _compute_signal_peaks(samples)
|
||||||
step = max(1, len(samples) // num_peaks)
|
|
||||||
for i in range(num_peaks):
|
|
||||||
chunk = samples[i * step : (i + 1) * step]
|
|
||||||
if len(chunk) > 0:
|
|
||||||
max_val = np.max(np.abs(chunk)) / 32767.0
|
|
||||||
peaks.append(float(max_val))
|
|
||||||
else:
|
|
||||||
peaks.append(0.0)
|
|
||||||
else:
|
|
||||||
peaks = [0.0] * 200
|
|
||||||
return peaks
|
|
||||||
|
|
||||||
|
|
||||||
def extract_audio_from_video(video_path):
|
def extract_audio_from_video(video_path):
|
||||||
@@ -445,18 +531,7 @@ def extract_audio_from_video(video_path):
|
|||||||
rate=44100,
|
rate=44100,
|
||||||
)
|
)
|
||||||
|
|
||||||
audio_bytes = bytearray()
|
audio_bytes = _read_audio_bytes(container, stream, resampler)
|
||||||
|
|
||||||
for frame in container.decode(stream):
|
|
||||||
for resampled_frame in resampler.resample(frame):
|
|
||||||
arr = resampled_frame.to_ndarray()
|
|
||||||
audio_bytes.extend(arr.tobytes())
|
|
||||||
|
|
||||||
# Flush resampler
|
|
||||||
for resampled_frame in resampler.resample(None):
|
|
||||||
arr = resampled_frame.to_ndarray()
|
|
||||||
audio_bytes.extend(arr.tobytes())
|
|
||||||
|
|
||||||
if not audio_bytes:
|
if not audio_bytes:
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
@@ -468,17 +543,8 @@ def extract_audio_from_video(video_path):
|
|||||||
w.writeframes(audio_bytes)
|
w.writeframes(audio_bytes)
|
||||||
|
|
||||||
# Calculate peaks
|
# Calculate peaks
|
||||||
peaks = []
|
|
||||||
samples = np.frombuffer(audio_bytes, dtype=np.int16)
|
samples = np.frombuffer(audio_bytes, dtype=np.int16)
|
||||||
num_peaks = 200
|
peaks = _compute_signal_peaks(samples)
|
||||||
step = max(1, len(samples) // num_peaks)
|
|
||||||
for i in range(num_peaks):
|
|
||||||
chunk = samples[i * step : (i + 1) * step]
|
|
||||||
if len(chunk) > 0:
|
|
||||||
max_val = np.max(np.abs(chunk)) / 32767.0
|
|
||||||
peaks.append(float(max_val))
|
|
||||||
else:
|
|
||||||
peaks.append(0.0)
|
|
||||||
|
|
||||||
input_dir = folder_paths.get_input_directory()
|
input_dir = folder_paths.get_input_directory()
|
||||||
rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/')
|
rel_output = os.path.relpath(output_wav, input_dir).replace('\\', '/')
|
||||||
@@ -509,30 +575,11 @@ def get_audio_peaks(audio_path):
|
|||||||
layout='mono',
|
layout='mono',
|
||||||
rate=8000,
|
rate=8000,
|
||||||
)
|
)
|
||||||
audio_bytes = bytearray()
|
audio_bytes = _read_audio_bytes(container, stream, resampler)
|
||||||
for frame in container.decode(stream):
|
|
||||||
for resampled_frame in resampler.resample(frame):
|
|
||||||
arr = resampled_frame.to_ndarray()
|
|
||||||
audio_bytes.extend(arr.tobytes())
|
|
||||||
for resampled_frame in resampler.resample(None):
|
|
||||||
arr = resampled_frame.to_ndarray()
|
|
||||||
audio_bytes.extend(arr.tobytes())
|
|
||||||
|
|
||||||
if not audio_bytes:
|
if not audio_bytes:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
peaks = []
|
|
||||||
samples = np.frombuffer(audio_bytes, dtype=np.int16)
|
samples = np.frombuffer(audio_bytes, dtype=np.int16)
|
||||||
num_peaks = 200
|
return _compute_signal_peaks(samples)
|
||||||
step = max(1, len(samples) // num_peaks)
|
|
||||||
for i in range(num_peaks):
|
|
||||||
chunk = samples[i * step : (i + 1) * step]
|
|
||||||
if len(chunk) > 0:
|
|
||||||
max_val = np.max(np.abs(chunk)) / 32767.0
|
|
||||||
peaks.append(float(max_val))
|
|
||||||
else:
|
|
||||||
peaks.append(0.0)
|
|
||||||
return peaks
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}")
|
print(f"[LTXDirector] Failed to get audio peaks via PyAV: {e}")
|
||||||
return None
|
return None
|
||||||
@@ -657,38 +704,24 @@ async def ltx_director_upload_chunk(request):
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _load_image_tensor(seg: dict) -> torch.Tensor:
|
def _load_image_tensor(seg: dict, cache: dict | None = None, input_dir: str = None) -> torch.Tensor:
|
||||||
"""Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64
|
"""Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64
|
||||||
to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1]."""
|
to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1]."""
|
||||||
if seg.get("imageFile"):
|
return _load_image_reference(
|
||||||
file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"])
|
b64_or_url=seg.get("imageB64", ""),
|
||||||
if os.path.exists(file_path):
|
filename=seg.get("imageFile"),
|
||||||
img = Image.open(file_path).convert("RGB")
|
cache=cache,
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
input_dir=input_dir,
|
||||||
return torch.from_numpy(arr).unsqueeze(0)
|
)
|
||||||
|
|
||||||
b64_str = seg.get("imageB64", "")
|
|
||||||
if not b64_str or b64_str.startswith("/view?"):
|
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
|
||||||
|
|
||||||
if "," in b64_str:
|
def _load_video_tensor(seg: dict, frame_rate: float, input_dir: str = None) -> torch.Tensor:
|
||||||
b64_str = b64_str.split(",", 1)[1]
|
|
||||||
|
|
||||||
try:
|
|
||||||
img_bytes = base64.b64decode(b64_str)
|
|
||||||
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
|
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
|
||||||
return torch.from_numpy(arr).unsqueeze(0)
|
|
||||||
except:
|
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
|
||||||
|
|
||||||
def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
|
|
||||||
"""Extracts a sequence of frames from a video file based on the segment's trim parameters,
|
"""Extracts a sequence of frames from a video file based on the segment's trim parameters,
|
||||||
and returns them as an [N, H, W, 3] float32 tensor."""
|
and returns them as an [N, H, W, 3] float32 tensor."""
|
||||||
file_path = os.path.join(folder_paths.get_input_directory(), seg.get("imageFile", ""))
|
file_path = _resolve_input_path(seg.get("imageFile", ""), input_dir)
|
||||||
|
|
||||||
if not os.path.exists(file_path):
|
if not file_path:
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
return _empty_image_tensor()
|
||||||
|
|
||||||
trim_start_frames = float(seg.get("trimStart", 0))
|
trim_start_frames = float(seg.get("trimStart", 0))
|
||||||
length_frames = float(seg.get("length", 1))
|
length_frames = float(seg.get("length", 1))
|
||||||
@@ -727,7 +760,7 @@ def _load_video_tensor(seg: dict, frame_rate: float) -> torch.Tensor:
|
|||||||
log.warning(f"[PromptRelay] Video extract error: {e}")
|
log.warning(f"[PromptRelay] Video extract error: {e}")
|
||||||
|
|
||||||
if not frames:
|
if not frames:
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
return _empty_image_tensor()
|
||||||
|
|
||||||
frames_np = np.array(frames, dtype=np.float32) / 255.0
|
frames_np = np.array(frames, dtype=np.float32) / 255.0
|
||||||
return torch.from_numpy(frames_np)
|
return torch.from_numpy(frames_np)
|
||||||
@@ -848,7 +881,7 @@ def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor:
|
|||||||
return tensor
|
return tensor
|
||||||
|
|
||||||
|
|
||||||
def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict:
|
def _build_combined_audio(timeline_data_value, start_frame: int, duration_frames: int, frame_rate: float, override_audio: bool = False) -> dict:
|
||||||
"""Parses timeline JSON, loads/trims audio directly from memory using PyAV,
|
"""Parses timeline JSON, loads/trims audio directly from memory using PyAV,
|
||||||
and aligns to a global timeline yielding ComfyUI's format.
|
and aligns to a global timeline yielding ComfyUI's format.
|
||||||
Output length explicitly mimics the timeline's duration_frames length."""
|
Output length explicitly mimics the timeline's duration_frames length."""
|
||||||
@@ -856,11 +889,14 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
|
|||||||
total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr)))
|
total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr)))
|
||||||
empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr}
|
empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr}
|
||||||
|
|
||||||
if not timeline_data_str:
|
if not timeline_data_value:
|
||||||
return empty_audio
|
return empty_audio
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = json.loads(timeline_data_str)
|
if isinstance(timeline_data_value, str):
|
||||||
|
data = json.loads(timeline_data_value)
|
||||||
|
else:
|
||||||
|
data = timeline_data_value
|
||||||
is_retake = data.get("retakeMode", False)
|
is_retake = data.get("retakeMode", False)
|
||||||
if is_retake and data.get("retakeVideo"):
|
if is_retake and data.get("retakeVideo"):
|
||||||
retake_vid = data.get("retakeVideo")
|
retake_vid = data.get("retakeVideo")
|
||||||
@@ -888,17 +924,9 @@ def _build_combined_audio(timeline_data_str: str, start_frame: int, duration_fra
|
|||||||
buffer = None
|
buffer = None
|
||||||
file_key = "videoFile" if override_audio else "audioFile"
|
file_key = "videoFile" if override_audio else "audioFile"
|
||||||
if seg.get(file_key):
|
if seg.get(file_key):
|
||||||
file_path = os.path.join(folder_paths.get_input_directory(), seg[file_key])
|
file_path = _resolve_input_path(seg[file_key])
|
||||||
if not os.path.exists(file_path):
|
if file_path:
|
||||||
# Try fallback under whatdreamscost subfolder
|
buffer = file_path
|
||||||
basename = os.path.basename(seg[file_key])
|
|
||||||
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
|
|
||||||
if os.path.exists(fallback_path):
|
|
||||||
file_path = fallback_path
|
|
||||||
|
|
||||||
if os.path.exists(file_path):
|
|
||||||
with open(file_path, "rb") as f:
|
|
||||||
buffer = _io.BytesIO(f.read())
|
|
||||||
|
|
||||||
if not override_audio and not buffer and seg.get("audioB64"):
|
if not override_audio and not buffer and seg.get("audioB64"):
|
||||||
b64 = seg.get("audioB64")
|
b64 = seg.get("audioB64")
|
||||||
@@ -1266,6 +1294,8 @@ class LTXDirector(io.ComfyNode):
|
|||||||
use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False,
|
use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False,
|
||||||
vae=None, reference_strength=1.0, ref_images=None,
|
vae=None, reference_strength=1.0, ref_images=None,
|
||||||
start=None, end=None, duration=None) -> io.NodeOutput:
|
start=None, end=None, duration=None) -> io.NodeOutput:
|
||||||
|
input_dir = folder_paths.get_input_directory()
|
||||||
|
image_cache = {}
|
||||||
|
|
||||||
# Parse timeline data
|
# Parse timeline data
|
||||||
try:
|
try:
|
||||||
@@ -1331,7 +1361,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
for img_info in images_list:
|
for img_info in images_list:
|
||||||
image_b64 = img_info.get("b64", "")
|
image_b64 = img_info.get("b64", "")
|
||||||
file_name = img_info.get("name", "")
|
file_name = img_info.get("name", "")
|
||||||
tensor = _load_image_source(image_b64, file_name)
|
tensor = _load_image_source(image_b64, file_name, cache=image_cache, input_dir=input_dir)
|
||||||
char_images.append(tensor)
|
char_images.append(tensor)
|
||||||
slot_tensors.append(tensor)
|
slot_tensors.append(tensor)
|
||||||
char_slot_images.append(slot_tensors)
|
char_slot_images.append(slot_tensors)
|
||||||
@@ -1370,37 +1400,30 @@ class LTXDirector(io.ComfyNode):
|
|||||||
for idx, seg in enumerate(img_segs):
|
for idx, seg in enumerate(img_segs):
|
||||||
seg_start = int(seg.get("start", 0))
|
seg_start = int(seg.get("start", 0))
|
||||||
offset = max(0, start_frame - seg_start)
|
offset = max(0, start_frame - seg_start)
|
||||||
|
seg_length = int(seg.get("length", 1))
|
||||||
|
seg_for_load = seg
|
||||||
|
|
||||||
if seg.get("type") == "video":
|
if seg.get("type") == "video":
|
||||||
if offset > 0:
|
if offset > 0:
|
||||||
seg["trimStart"] = float(seg.get("trimStart", 0)) + offset
|
seg_for_load = dict(seg)
|
||||||
seg["length"] = max(1, int(seg.get("length", 1)) - offset)
|
seg_for_load["trimStart"] = float(seg.get("trimStart", 0)) + offset
|
||||||
tensor = _load_video_tensor(seg, float(frame_rate))
|
seg_for_load["length"] = max(1, seg_length - offset)
|
||||||
|
tensor = _load_video_tensor(seg_for_load, float(frame_rate), input_dir=input_dir)
|
||||||
else:
|
else:
|
||||||
tensor = _load_image_tensor(seg)
|
tensor = _load_image_tensor(seg, cache=image_cache, input_dir=input_dir)
|
||||||
|
|
||||||
# Apply resize
|
# Apply resize
|
||||||
src_h, src_w = tensor.shape[1], tensor.shape[2]
|
src_h, src_w = tensor.shape[1], tensor.shape[2]
|
||||||
|
tgt_w, tgt_h, resize_override = _target_resize_dims(
|
||||||
def snap(val, div):
|
src_w, src_h, custom_width, custom_height, divisible_by
|
||||||
return max(div, (val // div) * div)
|
)
|
||||||
|
tensor = _resize_image(
|
||||||
if custom_width > 0 and custom_height > 0:
|
tensor,
|
||||||
# Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR)
|
tgt_w,
|
||||||
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by)
|
tgt_h,
|
||||||
elif custom_width > 0:
|
resize_override or resize_method,
|
||||||
# Width only — scale height from AR, snap both, then resize to exact dimensions
|
divisible_by,
|
||||||
tgt_w = snap(custom_width, divisible_by)
|
)
|
||||||
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
|
|
||||||
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
|
|
||||||
elif custom_height > 0:
|
|
||||||
# Height only — scale width from AR, snap both, then resize to exact dimensions
|
|
||||||
tgt_h = snap(custom_height, divisible_by)
|
|
||||||
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
|
|
||||||
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
|
|
||||||
else:
|
|
||||||
# Both zero — keep original dimensions, just snap to divisible_by
|
|
||||||
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
|
|
||||||
|
|
||||||
|
|
||||||
# Apply compression
|
# Apply compression
|
||||||
@@ -1413,7 +1436,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
derived_w = tensor.shape[2]
|
derived_w = tensor.shape[2]
|
||||||
|
|
||||||
if seg.get("isEndFrame"):
|
if seg.get("isEndFrame"):
|
||||||
insert_frame = max(0, seg_start + int(seg.get("length", 1)) - 1 - start_frame)
|
insert_frame = max(0, seg_start + seg_length - 1 - start_frame)
|
||||||
else:
|
else:
|
||||||
insert_frame = max(0, seg_start - start_frame)
|
insert_frame = max(0, seg_start - start_frame)
|
||||||
strength = strengths[idx] if idx < len(strengths) else 1.0
|
strength = strengths[idx] if idx < len(strengths) else 1.0
|
||||||
@@ -1428,71 +1451,45 @@ class LTXDirector(io.ComfyNode):
|
|||||||
src_h = derived_h if derived_h > 0 else 512
|
src_h = derived_h if derived_h > 0 else 512
|
||||||
|
|
||||||
# If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling
|
# If there's an IC-LoRA video or retake base video on the timeline, extract its dimensions for accurate aspect ratio scaling
|
||||||
tdata_motion = json.loads(timeline_data) if timeline_data else {}
|
|
||||||
found_dims = False
|
found_dims = False
|
||||||
|
|
||||||
# Check for retake base video first
|
# Check for retake base video first
|
||||||
is_retake = tdata_motion.get("retakeMode", False)
|
is_retake = tdata.get("retakeMode", False)
|
||||||
retake_vid = tdata_motion.get("retakeVideo") or {}
|
retake_vid = tdata.get("retakeVideo") or {}
|
||||||
retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else ""
|
retake_file = retake_vid.get("imageFile", "") if isinstance(retake_vid, dict) else ""
|
||||||
if is_retake and retake_file:
|
if is_retake and retake_file:
|
||||||
r_path = os.path.join(folder_paths.get_input_directory(), retake_file)
|
r_path = _resolve_input_path(retake_file, input_dir)
|
||||||
if not os.path.exists(r_path):
|
if r_path:
|
||||||
basename = os.path.basename(retake_file)
|
src_dims = _extract_video_dimensions(r_path)
|
||||||
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
|
if all(src_dims):
|
||||||
if os.path.exists(fallback_path):
|
src_w, src_h = src_dims
|
||||||
r_path = fallback_path
|
|
||||||
if os.path.exists(r_path):
|
|
||||||
try:
|
|
||||||
with av.open(r_path) as container:
|
|
||||||
stream = container.streams.video[0]
|
|
||||||
src_w = stream.width or stream.codec_context.width
|
|
||||||
src_h = stream.height or stream.codec_context.height
|
|
||||||
found_dims = True
|
found_dims = True
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Fallback to normal motion segments
|
# Fallback to normal motion segments
|
||||||
if not found_dims:
|
if not found_dims:
|
||||||
for mseg in tdata_motion.get("motionSegments", []):
|
for mseg in tdata.get("motionSegments", []):
|
||||||
v_file = mseg.get("videoFile")
|
v_file = mseg.get("videoFile")
|
||||||
if v_file:
|
if v_file:
|
||||||
v_path = os.path.join(folder_paths.get_input_directory(), v_file)
|
v_path = _resolve_input_path(v_file, input_dir)
|
||||||
if not os.path.exists(v_path):
|
if v_path:
|
||||||
basename = os.path.basename(v_file)
|
src_dims = _extract_video_dimensions(v_path)
|
||||||
fallback_path = os.path.join(folder_paths.get_input_directory(), "whatdreamscost", basename)
|
if all(src_dims):
|
||||||
if os.path.exists(fallback_path):
|
src_w, src_h = src_dims
|
||||||
v_path = fallback_path
|
|
||||||
if os.path.exists(v_path):
|
|
||||||
try:
|
|
||||||
with av.open(v_path) as container:
|
|
||||||
stream = container.streams.video[0]
|
|
||||||
src_w = stream.width or stream.codec_context.width
|
|
||||||
src_h = stream.height or stream.codec_context.height
|
|
||||||
found_dims = True
|
found_dims = True
|
||||||
break
|
break
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Create a dummy tensor of the exact source dimensions
|
# Create a dummy tensor of the exact source dimensions
|
||||||
tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32)
|
tensor = torch.zeros((1, src_h, src_w, 3), dtype=torch.float32)
|
||||||
|
tgt_w, tgt_h, resize_override = _target_resize_dims(
|
||||||
def snap(val, div):
|
src_w, src_h, custom_width, custom_height, divisible_by
|
||||||
return max(div, (val // div) * div)
|
)
|
||||||
|
tensor = _resize_image(
|
||||||
# Route the dummy tensor through the exact same resizing pipeline
|
tensor,
|
||||||
if custom_width > 0 and custom_height > 0:
|
tgt_w,
|
||||||
tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by)
|
tgt_h,
|
||||||
elif custom_width > 0:
|
resize_override or resize_method,
|
||||||
tgt_w = snap(custom_width, divisible_by)
|
divisible_by,
|
||||||
tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by)
|
)
|
||||||
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
|
|
||||||
elif custom_height > 0:
|
|
||||||
tgt_h = snap(custom_height, divisible_by)
|
|
||||||
tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by)
|
|
||||||
tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by)
|
|
||||||
else:
|
|
||||||
tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by)
|
|
||||||
|
|
||||||
guide_data["images"].append(tensor)
|
guide_data["images"].append(tensor)
|
||||||
guide_data["insert_frames"].append(0)
|
guide_data["insert_frames"].append(0)
|
||||||
@@ -1703,7 +1700,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# --- Build Audio Output ---
|
# --- Build Audio Output ---
|
||||||
audio_out = _build_combined_audio(timeline_data, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
|
audio_out = _build_combined_audio(tdata, start_frame, ltxv_length, float(frame_rate), override_audio=override_audio)
|
||||||
|
|
||||||
# --- Audio Latent Generation ---
|
# --- Audio Latent Generation ---
|
||||||
audio_latent = {}
|
audio_latent = {}
|
||||||
@@ -1834,7 +1831,6 @@ class LTXDirector(io.ComfyNode):
|
|||||||
# --- Motion guide output from timeline video segments ---
|
# --- Motion guide output from timeline video segments ---
|
||||||
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
|
motion_guide_data = {"segments": [], "frame_rate": float(frame_rate), "duration_frames": int(duration_frames), "resize_method": resize_method}
|
||||||
try:
|
try:
|
||||||
tdata = json.loads(timeline_data) if timeline_data else {}
|
|
||||||
if use_custom_motion:
|
if use_custom_motion:
|
||||||
motion_segments = tdata.get("motionSegments", [])
|
motion_segments = tdata.get("motionSegments", [])
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user