Add files via upload
This commit is contained in:
1
example_workflows/LTX Director MSR_Ghost_V5.json
Normal file
1
example_workflows/LTX Director MSR_Ghost_V5.json
Normal file
File diff suppressed because one or more lines are too long
769
ltx_director.py
769
ltx_director.py
@@ -1,3 +1,5 @@
|
|||||||
|
# --- START OF FILE ltx_director.py ---
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
import base64
|
import base64
|
||||||
@@ -31,6 +33,14 @@ log = logging.getLogger(__name__)
|
|||||||
# Custom socket type shared with LTXSequencer
|
# Custom socket type shared with LTXSequencer
|
||||||
GuideData = io.Custom("GUIDE_DATA")
|
GuideData = io.Custom("GUIDE_DATA")
|
||||||
|
|
||||||
|
# --- Licon MSR engine fixed parameters (were node widgets; hardcoded for a clean UI) ---
|
||||||
|
# Slideshow length. 41 is the Licon training default (valid: 17 / 25 / 33 / 41).
|
||||||
|
MSR_PREFIX_FRAMES = 41
|
||||||
|
# latent_downscale_factor for the IC-LoRA reference frames. Tied to how the MSR LoRA was
|
||||||
|
# trained; Licon MSR V1 uses full-resolution references (1.0). This is INDEPENDENT of any
|
||||||
|
# multi-stage scale_by / x2 upscaling, which LTXDirectorGuide handles downstream.
|
||||||
|
MSR_LATENT_DOWNSCALE = 1.0
|
||||||
|
|
||||||
|
|
||||||
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
|
def _preprocess_prompts_with_characters(global_prompt, local_prompts, char1="", char2="", char3=""):
|
||||||
"""Invisibly swaps out @character1/@char1 tags with their high-fidelity VLM descriptions."""
|
"""Invisibly swaps out @character1/@char1 tags with their high-fidelity VLM descriptions."""
|
||||||
@@ -160,135 +170,173 @@ def _format_timeline_to_text(global_prompt, duration_frames, frame_rate, epsilon
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
# Register API endpoint for instant export from the JS UI if needed
|
def _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames):
|
||||||
try:
|
if not pixel_lengths:
|
||||||
import server
|
return []
|
||||||
from aiohttp import web
|
total_pixel = sum(pixel_lengths)
|
||||||
|
if total_pixel <= 0:
|
||||||
|
return [1] * len(pixel_lengths)
|
||||||
|
|
||||||
@server.PromptServer.instance.routes.post("/ltx_director/export_timeline")
|
naive_total = max(1, round(total_pixel / temporal_stride))
|
||||||
async def export_timeline_endpoint(request):
|
target_total = min(latent_frames, naive_total)
|
||||||
try:
|
if target_total >= latent_frames - 1:
|
||||||
data = await request.json()
|
target_total = latent_frames
|
||||||
global_prompt = data.get("global_prompt", "")
|
|
||||||
timeline_data = data.get("timeline_data", "{}")
|
|
||||||
duration_frames = int(data.get("duration_frames", 0))
|
|
||||||
frame_rate = float(data.get("frame_rate", 24))
|
|
||||||
epsilon = float(data.get("epsilon", 0.001))
|
|
||||||
custom_width = int(data.get("custom_width", 0))
|
|
||||||
custom_height = int(data.get("custom_height", 0))
|
|
||||||
resize_method = data.get("resize_method", "maintain aspect ratio")
|
|
||||||
local_prompts = data.get("local_prompts", "")
|
|
||||||
segment_lengths = data.get("segment_lengths", "")
|
|
||||||
guide_strength = data.get("guide_strength", "")
|
|
||||||
|
|
||||||
formatted_text = _format_timeline_to_text(
|
exact = [p * target_total / total_pixel for p in pixel_lengths]
|
||||||
global_prompt, duration_frames, frame_rate, epsilon,
|
result = [int(e) for e in exact]
|
||||||
custom_width, custom_height, resize_method,
|
diff = target_total - sum(result)
|
||||||
timeline_data, local_prompts, segment_lengths, guide_strength
|
if diff > 0:
|
||||||
|
order = sorted(range(len(exact)), key=lambda i: -(exact[i] - int(exact[i])))
|
||||||
|
for k in range(diff):
|
||||||
|
result[order[k % len(order)]] += 1
|
||||||
|
|
||||||
|
for i in range(len(result)):
|
||||||
|
if result[i] < 1:
|
||||||
|
max_idx = max(range(len(result)), key=lambda j: result[j])
|
||||||
|
if result[max_idx] > 1:
|
||||||
|
result[max_idx] -= 1
|
||||||
|
result[i] = 1
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _encode_relay(model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon):
|
||||||
|
for name, val in (("global_prompt", global_prompt),
|
||||||
|
("local_prompts", local_prompts),
|
||||||
|
("segment_lengths", segment_lengths)):
|
||||||
|
if val is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"PromptRelay: '{name}' arrived as None. "
|
||||||
|
"Likely causes: a stale workflow JSON saved with null, the timeline "
|
||||||
|
"editor's web extension failing to load, or an upstream node returning None. "
|
||||||
|
"Set the field to an empty string or fix the upstream connection."
|
||||||
)
|
)
|
||||||
|
|
||||||
out_dir = folder_paths.get_output_directory()
|
locals_list = [p.strip() for p in local_prompts.split("|")]
|
||||||
filename = f"ltx_director_prompts_{int(time.time())}.txt"
|
|
||||||
filepath = os.path.join(out_dir, filename)
|
|
||||||
|
|
||||||
with open(filepath, "w", encoding="utf-8") as f:
|
for p in locals_list:
|
||||||
f.write(formatted_text)
|
if not p:
|
||||||
|
raise ValueError("There is a segment on the timeline missing a prompt!")
|
||||||
|
|
||||||
return web.json_response({
|
if not locals_list or (len(locals_list) == 1 and not locals_list[0]):
|
||||||
"status": "success",
|
raise ValueError("At least one local prompt is required.")
|
||||||
"filepath": filepath,
|
|
||||||
"filename": filename,
|
|
||||||
"content": formatted_text
|
|
||||||
})
|
|
||||||
except Exception as e:
|
|
||||||
log.error(f"[PromptRelay] Export timeline endpoint error: {e}")
|
|
||||||
return web.json_response({"status": "error", "message": str(e)}, status=500)
|
|
||||||
except Exception as e:
|
|
||||||
log.warning(f"[PromptRelay] Could not register /ltx_director/export_timeline endpoint: {e}")
|
|
||||||
|
|
||||||
|
arch, patch_size, temporal_stride = detect_model_type(model)
|
||||||
|
|
||||||
# Register API endpoint for instant character analysis via local Ollama
|
samples = latent["samples"]
|
||||||
try:
|
latent_frames = samples.shape[2]
|
||||||
import server
|
tokens_per_frame = (samples.shape[3] // patch_size[1]) * (samples.shape[4] // patch_size[2])
|
||||||
from aiohttp import web
|
|
||||||
import aiohttp
|
|
||||||
|
|
||||||
@server.PromptServer.instance.routes.post("/ltx_director/analyze_character")
|
parsed_lengths = None
|
||||||
async def analyze_character_endpoint(request):
|
if segment_lengths.strip():
|
||||||
try:
|
pixel_lengths = [int(float(x.strip())) for x in segment_lengths.split(",") if x.strip()]
|
||||||
data = await request.json()
|
parsed_lengths = _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames)
|
||||||
image_b64 = data.get("image_b64", "")
|
|
||||||
char_index = int(data.get("char_index", 0))
|
|
||||||
|
|
||||||
if not image_b64:
|
raw_tokenizer = get_raw_tokenizer(clip)
|
||||||
return web.json_response({"status": "error", "message": "No image provided for analysis."})
|
full_prompt, token_ranges = map_token_indices(raw_tokenizer, global_prompt, locals_list)
|
||||||
|
|
||||||
b64_list = image_b64 if isinstance(image_b64, list) else [image_b64]
|
log.info("[PromptRelay] Global: tokens [0:%d] (%d tokens)", token_ranges[0][0], token_ranges[0][0])
|
||||||
cleaned_b64_list = []
|
for i, (s, e) in enumerate(token_ranges):
|
||||||
for b64 in b64_list:
|
log.info("[PromptRelay] Segment %d: tokens [%d:%d] (%d tokens)", i, s, e, e - s)
|
||||||
if "," in b64:
|
|
||||||
b64 = b64.split(",", 1)[1]
|
|
||||||
cleaned_b64_list.append(b64)
|
|
||||||
|
|
||||||
if not cleaned_b64_list:
|
conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(full_prompt))
|
||||||
return web.json_response({"status": "error", "message": "No valid base64 images decoded."})
|
|
||||||
|
|
||||||
# Configured to use huihui_ai/qwen3.5-abliterated:2b on Ollama
|
effective_lengths = distribute_segment_lengths(len(locals_list), latent_frames, parsed_lengths)
|
||||||
model_name = "huihui_ai/qwen3.5-abliterated:2b"
|
|
||||||
|
|
||||||
# Detailed visual prompt for high-fidelity descriptions
|
log.info(
|
||||||
prompt = (
|
"[PromptRelay] Latent: %d frames, %d tokens/frame, segments: %s",
|
||||||
"Describe the character's physical appearance in two concise sentences. "
|
latent_frames, tokens_per_frame, effective_lengths,
|
||||||
"Specify their hair color/style, face details, and their clothing type/color. "
|
|
||||||
"Keep the entire response very brief."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
log.info(f"[PromptRelay] Analyzing Character {char_index+1} with local Ollama model '{model_name}'...")
|
q_token_idx = build_segments(token_ranges, effective_lengths, epsilon, None)
|
||||||
|
mask_fn = create_mask_fn(q_token_idx, tokens_per_frame, latent_frames)
|
||||||
|
|
||||||
payload = {
|
patched = model.clone()
|
||||||
"model": model_name,
|
apply_patches(patched, arch, mask_fn)
|
||||||
"prompt": prompt,
|
|
||||||
"images": cleaned_b64_list,
|
|
||||||
"stream": False,
|
|
||||||
"keep_alive": 0 # Fixed: Immediately unloads the model from VRAM after generating the description
|
|
||||||
}
|
|
||||||
|
|
||||||
ollama_url = "http://127.0.0.1:11434/api/generate"
|
return patched, conditioning
|
||||||
|
|
||||||
# Send the request asynchronously to Ollama without blocking ComfyUI's main thread
|
|
||||||
async with aiohttp.ClientSession() as session:
|
def _execute_comfy_node(node_class, **kwargs):
|
||||||
|
"""Executes any ComfyUI node class programmatically by resolving its entrypoint dynamically."""
|
||||||
|
if hasattr(node_class, "execute"):
|
||||||
|
return node_class.execute(**kwargs)
|
||||||
|
|
||||||
|
node_instance = node_class()
|
||||||
|
func_name = getattr(node_class, "FUNCTION", None)
|
||||||
|
if func_name is None:
|
||||||
|
for candidate in ("generate", "encode", "process", "execute"):
|
||||||
|
if hasattr(node_instance, candidate):
|
||||||
|
func_name = candidate
|
||||||
|
break
|
||||||
|
|
||||||
|
if func_name is not None:
|
||||||
|
func = getattr(node_instance, func_name)
|
||||||
|
import inspect
|
||||||
|
sig = inspect.signature(func)
|
||||||
|
filtered_kwargs = {}
|
||||||
|
for param in sig.parameters.values():
|
||||||
|
if param.name in kwargs:
|
||||||
|
filtered_kwargs[param.name] = kwargs[param.name]
|
||||||
|
elif param.default is not inspect.Parameter.empty:
|
||||||
|
pass
|
||||||
|
elif param.kind == inspect.Parameter.VAR_KEYWORD:
|
||||||
|
filtered_kwargs.update(kwargs)
|
||||||
|
break
|
||||||
|
return func(**filtered_kwargs)
|
||||||
|
else:
|
||||||
|
raise AttributeError(f"Could not resolve entrypoint for node class {node_class.__name__}")
|
||||||
|
|
||||||
|
|
||||||
|
def _unpack(out):
|
||||||
|
"""Safely extracts positional returns from ComfyUI API outputs."""
|
||||||
try:
|
try:
|
||||||
async with session.post(ollama_url, json=payload, timeout=60) as response:
|
return out[0], out[1], out[2]
|
||||||
if response.status != 200:
|
except Exception:
|
||||||
err_txt = await response.text()
|
t = tuple(out)
|
||||||
return web.json_response({
|
return t[0], t[1], t[2]
|
||||||
"status": "error",
|
|
||||||
"message": f"Ollama returned HTTP {response.status}: {err_txt}"
|
|
||||||
})
|
|
||||||
|
|
||||||
resp_json = await response.json()
|
|
||||||
generated_text = resp_json.get("response", "").strip()
|
|
||||||
except aiohttp.ClientConnectorError:
|
|
||||||
return web.json_response({
|
|
||||||
"status": "error",
|
|
||||||
"message": (
|
|
||||||
"Could not connect to Ollama. Please ensure Ollama is installed, "
|
|
||||||
"running in the background, and that you have pulled the model "
|
|
||||||
"via 'ollama run huihui_ai/qwen3.5-abliterated:2b' in your terminal."
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
if "<think>" in generated_text:
|
def _load_image_source(b64_or_url: str, filename: str = None) -> torch.Tensor:
|
||||||
generated_text = generated_text.split("</think>")[-1].strip()
|
if not b64_or_url and not filename:
|
||||||
|
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
||||||
log.info(f"[PromptRelay] Analysis complete: {generated_text}")
|
|
||||||
return web.json_response({"status": "success", "description": generated_text})
|
|
||||||
|
|
||||||
|
if b64_or_url and "view?" in b64_or_url:
|
||||||
|
try:
|
||||||
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
parsed = urlparse(b64_or_url)
|
||||||
|
q = parse_qs(parsed.query)
|
||||||
|
fname = q.get("filename", [None])[0]
|
||||||
|
subfolder = q.get("subfolder", [""])[0]
|
||||||
|
if fname:
|
||||||
|
file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname)
|
||||||
|
if os.path.exists(file_path):
|
||||||
|
img = Image.open(file_path).convert("RGB")
|
||||||
|
arr = np.array(img, dtype=np.float32) / 255.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error(f"[PromptRelay] Failed to analyze character: {e}")
|
log.debug(f"[PromptRelay] URL parsing failed for {b64_or_url}: {e}")
|
||||||
return web.json_response({"status": "error", "message": str(e)}, status=500)
|
|
||||||
|
if filename:
|
||||||
|
file_path = os.path.join(folder_paths.get_input_directory(), filename)
|
||||||
|
if os.path.exists(file_path):
|
||||||
|
img = Image.open(file_path).convert("RGB")
|
||||||
|
arr = np.array(img, dtype=np.float32) / 255.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
|
|
||||||
|
if b64_or_url:
|
||||||
|
try:
|
||||||
|
b64_str = b64_or_url
|
||||||
|
if "," in b64_str:
|
||||||
|
b64_str = b64_str.split(",", 1)[1]
|
||||||
|
img_bytes = base64.b64decode(b64_str)
|
||||||
|
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
|
||||||
|
arr = np.array(img, dtype=np.float32) / 255.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning(f"[PromptRelay] Could not register /ltx_director/analyze_character endpoint: {e}")
|
log.debug(f"[PromptRelay] Base64 decoding failed: {e}")
|
||||||
|
|
||||||
|
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
||||||
|
|
||||||
|
|
||||||
def _load_image_tensor(seg: dict) -> torch.Tensor:
|
def _load_image_tensor(seg: dict) -> torch.Tensor:
|
||||||
@@ -300,6 +348,23 @@ def _load_image_tensor(seg: dict) -> torch.Tensor:
|
|||||||
return torch.from_numpy(arr).unsqueeze(0)
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
|
|
||||||
b64_str = seg.get("imageB64", "")
|
b64_str = seg.get("imageB64", "")
|
||||||
|
|
||||||
|
if b64_str and "view?" in b64_str:
|
||||||
|
try:
|
||||||
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
parsed = urlparse(b64_str)
|
||||||
|
q = parse_qs(parsed.query)
|
||||||
|
fname = q.get("filename", [None])[0]
|
||||||
|
subfolder = q.get("subfolder", [""])[0]
|
||||||
|
if fname:
|
||||||
|
file_path = os.path.join(folder_paths.get_input_directory(), subfolder, fname)
|
||||||
|
if os.path.exists(file_path):
|
||||||
|
img = Image.open(file_path).convert("RGB")
|
||||||
|
arr = np.array(img, dtype=np.float32) / 255.0
|
||||||
|
return torch.from_numpy(arr).unsqueeze(0)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug(f"[PromptRelay] URL parsing failed for {b64_str}: {e}")
|
||||||
|
|
||||||
if not b64_str or b64_str.startswith("/view?"):
|
if not b64_str or b64_str.startswith("/view?"):
|
||||||
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
return torch.zeros((1, 512, 512, 3), dtype=torch.float32)
|
||||||
|
|
||||||
@@ -317,7 +382,6 @@ def _load_image_tensor(seg: dict) -> torch.Tensor:
|
|||||||
|
|
||||||
def _resize_image(tensor: torch.Tensor, target_w: int, target_h: int, method: str, divisible_by: int) -> torch.Tensor:
|
def _resize_image(tensor: torch.Tensor, target_w: int, target_h: int, method: str, divisible_by: int) -> torch.Tensor:
|
||||||
from PIL import Image as _PilImage
|
from PIL import Image as _PilImage
|
||||||
import torchvision.transforms.functional as TF
|
|
||||||
|
|
||||||
def snap(val, div):
|
def snap(val, div):
|
||||||
return max(div, (val // div) * div)
|
return max(div, (val // div) * div)
|
||||||
@@ -447,21 +511,14 @@ def _build_combined_audio(timeline_data_str: str, duration_frames: int, frame_ra
|
|||||||
|
|
||||||
with av.open(buffer) as container:
|
with av.open(buffer) as container:
|
||||||
stream = container.streams.audio[0]
|
stream = container.streams.audio[0]
|
||||||
|
resampler = av.AudioResampler(format='fltp', layout='stereo', rate=target_sr)
|
||||||
resampler = av.AudioResampler(
|
|
||||||
format='fltp',
|
|
||||||
layout='stereo',
|
|
||||||
rate=target_sr,
|
|
||||||
)
|
|
||||||
|
|
||||||
for frame in container.decode(stream):
|
for frame in container.decode(stream):
|
||||||
for resampled_frame in resampler.resample(frame):
|
for resampled_frame in resampler.resample(frame):
|
||||||
arr = resampled_frame.to_ndarray()
|
clip_frames.append(torch.from_numpy(resampled_frame.to_ndarray()))
|
||||||
clip_frames.append(torch.from_numpy(arr))
|
|
||||||
|
|
||||||
for resampled_frame in resampler.resample(None):
|
for resampled_frame in resampler.resample(None):
|
||||||
arr = resampled_frame.to_ndarray()
|
clip_frames.append(torch.from_numpy(resampled_frame.to_ndarray()))
|
||||||
clip_frames.append(torch.from_numpy(arr))
|
|
||||||
|
|
||||||
if not clip_frames:
|
if not clip_frames:
|
||||||
continue
|
continue
|
||||||
@@ -509,91 +566,129 @@ def _build_combined_audio(timeline_data_str: str, duration_frames: int, frame_ra
|
|||||||
return {"waveform": out_waveform.unsqueeze(0), "sample_rate": target_sr}
|
return {"waveform": out_waveform.unsqueeze(0), "sample_rate": target_sr}
|
||||||
|
|
||||||
|
|
||||||
def _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames):
|
# Register API endpoint for instant export
|
||||||
if not pixel_lengths:
|
try:
|
||||||
return []
|
import server
|
||||||
total_pixel = sum(pixel_lengths)
|
from aiohttp import web
|
||||||
if total_pixel <= 0:
|
|
||||||
return [1] * len(pixel_lengths)
|
|
||||||
|
|
||||||
naive_total = max(1, round(total_pixel / temporal_stride))
|
@server.PromptServer.instance.routes.post("/ltx_director/export_timeline")
|
||||||
target_total = min(latent_frames, naive_total)
|
async def export_timeline_endpoint(request):
|
||||||
if target_total >= latent_frames - 1:
|
try:
|
||||||
target_total = latent_frames
|
data = await request.json()
|
||||||
|
global_prompt = data.get("global_prompt", "")
|
||||||
|
timeline_data = data.get("timeline_data", "{}")
|
||||||
|
duration_frames = int(data.get("duration_frames", 0))
|
||||||
|
frame_rate = float(data.get("frame_rate", 24))
|
||||||
|
epsilon = float(data.get("epsilon", 0.001))
|
||||||
|
custom_width = int(data.get("custom_width", 0))
|
||||||
|
custom_height = int(data.get("custom_height", 0))
|
||||||
|
resize_method = data.get("resize_method", "maintain aspect ratio")
|
||||||
|
local_prompts = data.get("local_prompts", "")
|
||||||
|
segment_lengths = data.get("segment_lengths", "")
|
||||||
|
guide_strength = data.get("guide_strength", "")
|
||||||
|
|
||||||
exact = [p * target_total / total_pixel for p in pixel_lengths]
|
formatted_text = _format_timeline_to_text(
|
||||||
result = [int(e) for e in exact]
|
global_prompt, duration_frames, frame_rate, epsilon,
|
||||||
diff = target_total - sum(result)
|
custom_width, custom_height, resize_method,
|
||||||
if diff > 0:
|
timeline_data, local_prompts, segment_lengths, guide_strength
|
||||||
order = sorted(range(len(exact)), key=lambda i: -(exact[i] - int(exact[i])))
|
|
||||||
for k in range(diff):
|
|
||||||
result[order[k % len(order)]] += 1
|
|
||||||
|
|
||||||
for i in range(len(result)):
|
|
||||||
if result[i] < 1:
|
|
||||||
max_idx = max(range(len(result)), key=lambda j: result[j])
|
|
||||||
if result[max_idx] > 1:
|
|
||||||
result[max_idx] -= 1
|
|
||||||
result[i] = 1
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_relay(model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon):
|
|
||||||
for name, val in (("global_prompt", global_prompt),
|
|
||||||
("local_prompts", local_prompts),
|
|
||||||
("segment_lengths", segment_lengths)):
|
|
||||||
if val is None:
|
|
||||||
raise ValueError(
|
|
||||||
f"PromptRelay: '{name}' arrived as None. "
|
|
||||||
"Likely causes: a stale workflow JSON saved with null, the timeline "
|
|
||||||
"editor's web extension failing to load, or an upstream node returning None. "
|
|
||||||
"Set the field to an empty string or fix the upstream connection."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
locals_list = [p.strip() for p in local_prompts.split("|")]
|
out_dir = folder_paths.get_output_directory()
|
||||||
|
filename = f"ltx_director_prompts_{int(time.time())}.txt"
|
||||||
|
filepath = os.path.join(out_dir, filename)
|
||||||
|
|
||||||
for p in locals_list:
|
with open(filepath, "w", encoding="utf-8") as f:
|
||||||
if not p:
|
f.write(formatted_text)
|
||||||
raise ValueError("There is a segment on the timeline missing a prompt!")
|
|
||||||
|
|
||||||
if not locals_list or (len(locals_list) == 1 and not locals_list[0]):
|
return web.json_response({
|
||||||
raise ValueError("At least one local prompt is required.")
|
"status": "success",
|
||||||
|
"filepath": filepath,
|
||||||
|
"filename": filename,
|
||||||
|
"content": formatted_text
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
return web.json_response({"status": "error", "message": str(e)}, status=500)
|
||||||
|
except Exception as e:
|
||||||
|
pass
|
||||||
|
|
||||||
arch, patch_size, temporal_stride = detect_model_type(model)
|
# Register API endpoint for instant character analysis via local Ollama
|
||||||
|
try:
|
||||||
|
import server
|
||||||
|
from aiohttp import web
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
samples = latent["samples"]
|
@server.PromptServer.instance.routes.post("/ltx_director/analyze_character")
|
||||||
latent_frames = samples.shape[2]
|
async def analyze_character_endpoint(request):
|
||||||
tokens_per_frame = (samples.shape[3] // patch_size[1]) * (samples.shape[4] // patch_size[2])
|
try:
|
||||||
|
data = await request.json()
|
||||||
|
image_b64 = data.get("image_b64", "")
|
||||||
|
char_index = int(data.get("char_index", 0))
|
||||||
|
|
||||||
parsed_lengths = None
|
if not image_b64:
|
||||||
if segment_lengths.strip():
|
return web.json_response({"status": "error", "message": "No image provided for analysis."})
|
||||||
pixel_lengths = [int(float(x.strip())) for x in segment_lengths.split(",") if x.strip()]
|
|
||||||
parsed_lengths = _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames)
|
|
||||||
|
|
||||||
raw_tokenizer = get_raw_tokenizer(clip)
|
b64_list = image_b64 if isinstance(image_b64, list) else [image_b64]
|
||||||
full_prompt, token_ranges = map_token_indices(raw_tokenizer, global_prompt, locals_list)
|
cleaned_b64_list = []
|
||||||
|
for b64 in b64_list:
|
||||||
|
if "," in b64:
|
||||||
|
b64 = b64.split(",", 1)[1]
|
||||||
|
cleaned_b64_list.append(b64)
|
||||||
|
|
||||||
log.info("[PromptRelay] Global: tokens [0:%d] (%d tokens)", token_ranges[0][0], token_ranges[0][0])
|
if not cleaned_b64_list:
|
||||||
for i, (s, e) in enumerate(token_ranges):
|
return web.json_response({"status": "error", "message": "No valid base64 images decoded."})
|
||||||
log.info("[PromptRelay] Segment %d: tokens [%d:%d] (%d tokens)", i, s, e, e - s)
|
|
||||||
|
|
||||||
conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(full_prompt))
|
# Restored your preferred model configuration
|
||||||
|
model_name = "huihui_ai/qwen3.5-abliterated:2b"
|
||||||
effective_lengths = distribute_segment_lengths(len(locals_list), latent_frames, parsed_lengths)
|
prompt = (
|
||||||
|
"Describe the character's physical appearance in two concise sentences. "
|
||||||
log.info(
|
"Specify their hair color/style, face details, and their clothing type/color. "
|
||||||
"[PromptRelay] Latent: %d frames, %d tokens/frame, segments: %s",
|
"Keep the entire response very brief."
|
||||||
latent_frames, tokens_per_frame, effective_lengths,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
q_token_idx = build_segments(token_ranges, effective_lengths, epsilon, None)
|
log.info(f"[PromptRelay] Analyzing Character {char_index+1} with local Ollama model '{model_name}'...")
|
||||||
mask_fn = create_mask_fn(q_token_idx, tokens_per_frame, latent_frames)
|
|
||||||
|
|
||||||
patched = model.clone()
|
payload = {
|
||||||
apply_patches(patched, arch, mask_fn)
|
"model": model_name,
|
||||||
|
"prompt": prompt,
|
||||||
|
"images": cleaned_b64_list,
|
||||||
|
"stream": False,
|
||||||
|
"keep_alive": 0
|
||||||
|
}
|
||||||
|
|
||||||
return patched, conditioning
|
ollama_url = "http://127.0.0.1:11434/api/generate"
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
try:
|
||||||
|
async with session.post(ollama_url, json=payload, timeout=60) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
err_txt = await response.text()
|
||||||
|
return web.json_response({
|
||||||
|
"status": "error",
|
||||||
|
"message": f"Ollama returned HTTP {response.status}: {err_txt}"
|
||||||
|
})
|
||||||
|
|
||||||
|
resp_json = await response.json()
|
||||||
|
generated_text = resp_json.get("response", "").strip()
|
||||||
|
except aiohttp.ClientConnectorError:
|
||||||
|
return web.json_response({
|
||||||
|
"status": "error",
|
||||||
|
"message": (
|
||||||
|
f"Could not connect to Ollama. Please ensure Ollama is installed and "
|
||||||
|
f"running, and that you have pulled the model via 'ollama run {model_name}'."
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
if "<think>" in generated_text:
|
||||||
|
generated_text = generated_text.split("</think>")[-1].strip()
|
||||||
|
|
||||||
|
log.info(f"[PromptRelay] Analysis complete: {generated_text}")
|
||||||
|
return web.json_response({"status": "success", "description": generated_text})
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"[PromptRelay] Failed to analyze character: {e}")
|
||||||
|
return web.json_response({"status": "error", "message": str(e)}, status=500)
|
||||||
|
except Exception as e:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class LTXDirector(io.ComfyNode):
|
class LTXDirector(io.ComfyNode):
|
||||||
@@ -613,6 +708,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
inputs=[
|
inputs=[
|
||||||
io.Model.Input("model"),
|
io.Model.Input("model"),
|
||||||
io.Clip.Input("clip"),
|
io.Clip.Input("clip"),
|
||||||
|
io.Conditioning.Input("negative", optional=True, tooltip="Optional. Connect your negative prompt conditioning here. Highly recommended for Licon MSR to prevent noise artifacts."),
|
||||||
io.Vae.Input("vae", optional=True, tooltip="Optional. Connect the LTX Autoencoder/VAE here to natively encode the MSR visual reference slideshow into the latent prefix."),
|
io.Vae.Input("vae", optional=True, tooltip="Optional. Connect the LTX Autoencoder/VAE here to natively encode the MSR visual reference slideshow into the latent prefix."),
|
||||||
io.Vae.Input("audio_vae", optional=True, tooltip="Optional. Connect an Audio VAE to generate audio latents."),
|
io.Vae.Input("audio_vae", optional=True, tooltip="Optional. Connect an Audio VAE to generate audio latents."),
|
||||||
io.Latent.Input("optional_latent", optional=True, tooltip="Optional. Connect a latent to override the auto-generated one."),
|
io.Latent.Input("optional_latent", optional=True, tooltip="Optional. Connect a latent to override the auto-generated one."),
|
||||||
@@ -632,14 +728,14 @@ class LTXDirector(io.ComfyNode):
|
|||||||
io.Combo.Input("resize_method", options=["maintain aspect ratio", "stretch to fit", "pad", "crop"], default="maintain aspect ratio", tooltip="How to resize image segments to fit the target dimensions."),
|
io.Combo.Input("resize_method", options=["maintain aspect ratio", "stretch to fit", "pad", "crop"], default="maintain aspect ratio", tooltip="How to resize image segments to fit the target dimensions."),
|
||||||
io.Int.Input("divisible_by", default=32, min=1, max=256, step=1, tooltip="Snap the final output image dimensions to be divisible by this number (e.g. 32 for LTX)."),
|
io.Int.Input("divisible_by", default=32, min=1, max=256, step=1, tooltip="Snap the final output image dimensions to be divisible by this number (e.g. 32 for LTX)."),
|
||||||
io.Int.Input("img_compression", default=18, min=0, max=100, step=1, tooltip="H.264 CRF compression to apply to each guide image. 0 = no compression, higher = more artefacts."),
|
io.Int.Input("img_compression", default=18, min=0, max=100, step=1, tooltip="H.264 CRF compression to apply to each guide image. 0 = no compression, higher = more artefacts."),
|
||||||
|
io.Combo.Input("reference_mode", options=["Ghost Mask (End)", "Licon MSR (Prefix)"], default="Ghost Mask (End)", tooltip="Choose whether to hide the references at the end with attention masks (Ghost Mask) or stack them sequentially at the beginning (Licon MSR Prefix) for the MSR LoRA."),
|
||||||
io.Boolean.Input("save_prompts_to_file", default=False, optional=True, tooltip="Save the timeline prompts and parameters to a text file in your ComfyUI output directory during execution."),
|
io.Boolean.Input("save_prompts_to_file", default=False, optional=True, tooltip="Save the timeline prompts and parameters to a text file in your ComfyUI output directory during execution."),
|
||||||
io.Float.Input("reference_strength", default=1.0, min=0.0, max=5.0, step=0.05, optional=True, tooltip="Guide strength for the reference images."),
|
io.Float.Input("reference_strength", default=1.0, min=0.0, max=5.0, step=0.05, optional=True, tooltip="Guide strength for the reference images."),
|
||||||
io.Combo.Input("reference_mode", options=["Ghost Mask (End)", "Licon MSR (Prefix)"], default="Ghost Mask (End)", tooltip="Choose whether to hide the references at the end with attention masks (Ghost Mask) or stack them sequentially at the beginning (Licon MSR Prefix) for the MSR LoRA."),
|
|
||||||
io.Int.Input("msr_prefix_frames", default=17, min=1, max=120, step=1, tooltip="The number of visual slideshow frames to generate at the start of the sequence for MSR LoRA (typically 17, 25, 33, or 41)."),
|
|
||||||
],
|
],
|
||||||
outputs=[
|
outputs=[
|
||||||
io.Model.Output(display_name="model"),
|
io.Model.Output(display_name="model"),
|
||||||
io.Conditioning.Output(display_name="positive"),
|
io.Conditioning.Output(display_name="positive"),
|
||||||
|
io.Conditioning.Output(display_name="negative"),
|
||||||
io.Latent.Output(display_name="video_latent"),
|
io.Latent.Output(display_name="video_latent"),
|
||||||
io.Latent.Output(display_name="audio_latent"),
|
io.Latent.Output(display_name="audio_latent"),
|
||||||
GuideData.Output(display_name="guide_data"),
|
GuideData.Output(display_name="guide_data"),
|
||||||
@@ -651,43 +747,33 @@ class LTXDirector(io.ComfyNode):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def execute(cls, model, clip, global_prompt, duration_frames, duration_seconds,
|
def execute(cls, model, clip, negative=None, vae=None, audio_vae=None, optional_latent=None,
|
||||||
timeline_data, local_prompts, segment_lengths, guide_strength="", epsilon=1e-3,
|
global_prompt="", duration_frames=120, duration_seconds=5.0,
|
||||||
frame_rate=24.0, display_mode="seconds",
|
timeline_data="", use_custom_audio=False, local_prompts="", segment_lengths="",
|
||||||
custom_width=768, custom_height=512, resize_method="maintain aspect ratio",
|
epsilon=1e-3, frame_rate=24.0, display_mode="seconds", guide_strength="",
|
||||||
divisible_by=32, img_compression=0, vae=None, audio_vae=None, optional_latent=None,
|
custom_width=0, custom_height=0, resize_method="maintain aspect ratio",
|
||||||
use_custom_audio=False, save_prompts_to_file=False,
|
divisible_by=32, img_compression=18,
|
||||||
reference_strength=1.0, reference_mode="Ghost Mask (End)", msr_prefix_frames=17) -> io.NodeOutput:
|
reference_mode="Ghost Mask (End)",
|
||||||
|
save_prompts_to_file=False, reference_strength=1.0) -> io.NodeOutput:
|
||||||
|
|
||||||
# Force Ollama to unload the vision model to maximize VRAM before generation begins
|
if isinstance(optional_latent, list):
|
||||||
try:
|
if len(optional_latent) > 0:
|
||||||
import requests
|
optional_latent = optional_latent[0]
|
||||||
unload_url = "http://127.0.0.1:11434/api/generate"
|
else:
|
||||||
unload_payload = {
|
optional_latent = None
|
||||||
"model": "huihui_ai/qwen3.5-abliterated:2b",
|
|
||||||
"keep_alive": 0
|
|
||||||
}
|
|
||||||
requests.post(unload_url, json=unload_payload, timeout=2.0)
|
|
||||||
log.info("[PromptRelay] Dispatched VRAM eviction request to Ollama to maximize memory for LTX generation.")
|
|
||||||
except Exception as e:
|
|
||||||
log.debug("[PromptRelay] Ollama VRAM eviction skipped: %s", e)
|
|
||||||
|
|
||||||
clean_pixel_frames = duration_frames + 1
|
clean_pixel_frames = duration_frames + 1
|
||||||
clean_latent_frames = ((clean_pixel_frames - 1) // 8) + 1
|
clean_latent_frames = ((clean_pixel_frames - 1) // 8) + 1
|
||||||
|
|
||||||
guide_data = {"images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate)}
|
guide_data = {"images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate)}
|
||||||
derived_w, derived_h = custom_width, custom_height
|
derived_w, derived_h = custom_width, custom_height
|
||||||
|
|
||||||
char_images = []
|
char_images = []
|
||||||
|
char_slot_images = [] # per-slot tensors, for MSR @char tag filtering
|
||||||
|
|
||||||
# Extract Timeline Character Descriptions (Fallback) and process dropped style images
|
char1_val, char2_val, char3_val = "", "", ""
|
||||||
char1_val = ""
|
|
||||||
char2_val = ""
|
|
||||||
char3_val = ""
|
|
||||||
try:
|
try:
|
||||||
tdata = json.loads(timeline_data) if timeline_data else {}
|
tdata = json.loads(timeline_data) if timeline_data else {}
|
||||||
characters = tdata.get("characters", [])
|
characters = tdata.get("characters", [])
|
||||||
|
|
||||||
if len(characters) > 0: char1_val = characters[0].get("description", "")
|
if len(characters) > 0: char1_val = characters[0].get("description", "")
|
||||||
if len(characters) > 1: char2_val = characters[1].get("description", "")
|
if len(characters) > 1: char2_val = characters[1].get("description", "")
|
||||||
if len(characters) > 2: char3_val = characters[2].get("description", "")
|
if len(characters) > 2: char3_val = characters[2].get("description", "")
|
||||||
@@ -698,17 +784,14 @@ class LTXDirector(io.ComfyNode):
|
|||||||
if legacy_b64 and not images_list:
|
if legacy_b64 and not images_list:
|
||||||
images_list = [{"b64": legacy_b64, "name": char_info.get("fileName", "")}]
|
images_list = [{"b64": legacy_b64, "name": char_info.get("fileName", "")}]
|
||||||
|
|
||||||
|
slot_tensors = []
|
||||||
for img_info in images_list:
|
for img_info in images_list:
|
||||||
image_b64 = img_info.get("b64", "")
|
image_b64 = img_info.get("b64", "")
|
||||||
if image_b64:
|
file_name = img_info.get("name", "")
|
||||||
if "," in image_b64:
|
tensor = _load_image_source(image_b64, file_name)
|
||||||
image_b64 = image_b64.split(",", 1)[1]
|
|
||||||
img_bytes = base64.b64decode(image_b64)
|
|
||||||
img = Image.open(_io.BytesIO(img_bytes)).convert("RGB")
|
|
||||||
|
|
||||||
arr = np.array(img, dtype=np.float32) / 255.0
|
|
||||||
tensor = torch.from_numpy(arr).unsqueeze(0)
|
|
||||||
char_images.append(tensor)
|
char_images.append(tensor)
|
||||||
|
slot_tensors.append(tensor)
|
||||||
|
char_slot_images.append(slot_tensors) # keep slot boundaries for @char filtering
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("[PromptRelay] Could not process character slot inputs: %s", e)
|
log.warning("[PromptRelay] Could not process character slot inputs: %s", e)
|
||||||
|
|
||||||
@@ -721,10 +804,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
and int(s.get("start", 0)) < duration_frames
|
and int(s.get("start", 0)) < duration_frames
|
||||||
]
|
]
|
||||||
img_segs.sort(key=lambda s: s["start"])
|
img_segs.sort(key=lambda s: s["start"])
|
||||||
|
strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()] if guide_strength.strip() else []
|
||||||
strengths = []
|
|
||||||
if guide_strength.strip():
|
|
||||||
strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()]
|
|
||||||
|
|
||||||
for idx, seg in enumerate(img_segs):
|
for idx, seg in enumerate(img_segs):
|
||||||
tensor = _load_image_tensor(seg)
|
tensor = _load_image_tensor(seg)
|
||||||
@@ -758,6 +838,7 @@ class LTXDirector(io.ComfyNode):
|
|||||||
guide_data["insert_frames"].append(int(seg["start"]))
|
guide_data["insert_frames"].append(int(seg["start"]))
|
||||||
guide_data["strengths"].append(float(strength))
|
guide_data["strengths"].append(float(strength))
|
||||||
|
|
||||||
|
# Prevent crashes if completely empty
|
||||||
if not guide_data["images"] and not char_images:
|
if not guide_data["images"] and not char_images:
|
||||||
w = derived_w if derived_w > 0 else 768
|
w = derived_w if derived_w > 0 else 768
|
||||||
h = derived_h if derived_h > 0 else 512
|
h = derived_h if derived_h > 0 else 512
|
||||||
@@ -774,110 +855,154 @@ class LTXDirector(io.ComfyNode):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning("[PromptRelay] Could not build guide_data: %s", e)
|
log.warning("[PromptRelay] Could not build guide_data: %s", e)
|
||||||
|
|
||||||
# Initialize dimension checks
|
|
||||||
derived_w = max(32, (derived_w // 32) * 32) if derived_w > 0 else 768
|
derived_w = max(32, (derived_w // 32) * 32) if derived_w > 0 else 768
|
||||||
derived_h = max(32, (derived_h // 32) * 32) if derived_h > 0 else 512
|
derived_h = max(32, (derived_h // 32) * 32) if derived_h > 0 else 512
|
||||||
|
|
||||||
# Dual Mode execution branches
|
if reference_mode == "Licon MSR (Prefix)":
|
||||||
|
processed_global, processed_local = global_prompt, local_prompts
|
||||||
|
else:
|
||||||
|
processed_global, processed_local = _preprocess_prompts_with_characters(
|
||||||
|
global_prompt, local_prompts, char1_val, char2_val, char3_val
|
||||||
|
)
|
||||||
|
|
||||||
|
# ====== LICON MSR DIRECTOR ENGINE (relay-aware) ======
|
||||||
|
# Axis separation, so the relay and IC-LoRA never fight over the same bookkeeping:
|
||||||
|
# - Character slots -> IDENTITY. Built into the IC-LoRA slideshow AND pinned as
|
||||||
|
# one identity keyframe each (the MSR LoRA's job).
|
||||||
|
# - Timeline images -> SCENE/background. The first seeds the slideshow's background
|
||||||
|
# slot; every timeline image is also pinned at its own
|
||||||
|
# frame_idx as a positional scene reference.
|
||||||
|
# - Timeline text -> the relay's per-segment attention mask over the clean region.
|
||||||
|
# The relay conditioning is encoded FIRST, then threaded THROUGH the IC-LoRA guide
|
||||||
|
# calls, so keyframe metadata + guide_attention_entries land on the real conditioning
|
||||||
|
# natively. The Director bakes everything it conditions on, so the relay mask length
|
||||||
|
# stays self-consistent with the latent and nothing downstream needs to add frames.
|
||||||
if reference_mode == "Licon MSR (Prefix)":
|
if reference_mode == "Licon MSR (Prefix)":
|
||||||
if vae is None:
|
if vae is None:
|
||||||
raise ValueError("reference_mode is set to 'Licon MSR (Prefix)' but no VAE is connected to the 'vae' input pin of LTX Director! Please connect your LTX VAE to encode the reference slideshow.")
|
raise ValueError("Licon MSR (Prefix) mode requires connecting the VAE to LTX Director!")
|
||||||
|
|
||||||
# Compile character reference images and first timeline background image
|
from nodes import NODE_CLASS_MAPPINGS
|
||||||
prepared_images = []
|
IC_Lora_Guide_Class = NODE_CLASS_MAPPINGS.get("LTXAddVideoICLoRAGuide")
|
||||||
|
if IC_Lora_Guide_Class is None:
|
||||||
|
raise ValueError("LTXAddVideoICLoRAGuide node not found! Please install the ComfyUI-LTXVideo repository.")
|
||||||
|
|
||||||
# Prepare characters
|
# 1. Identity references. In MSR the reference IMAGE drives identity, so we
|
||||||
for char_img in char_images:
|
# honour the @charN tags by selecting ONLY the slots the prompt references.
|
||||||
prepared = _resize_image(char_img, derived_w, derived_h, resize_method, divisible_by)
|
# @char1/@character1 -> slot 0, @char2 -> slot 1, @char3 -> slot 2.
|
||||||
prepared_images.append(prepared)
|
# No tag referenced (or referenced slots empty) -> use every filled slot.
|
||||||
|
_prompt_text = (global_prompt or "") + " " + (local_prompts or "")
|
||||||
|
_tag_pairs = [("@character1", "@char1"), ("@character2", "@char2"), ("@character3", "@char3")]
|
||||||
|
_referenced_slots = [i for i, tags in enumerate(_tag_pairs) if any(t in _prompt_text for t in tags)]
|
||||||
|
_selected = []
|
||||||
|
for _slot in _referenced_slots:
|
||||||
|
if _slot < len(char_slot_images):
|
||||||
|
_selected.extend(char_slot_images[_slot])
|
||||||
|
if not _selected:
|
||||||
|
_selected = char_images # fallback: no tags, or tagged slots had no image
|
||||||
|
log.info("[PromptRelay] MSR slideshow subjects: referenced slots=%s, images=%d", _referenced_slots, len(_selected))
|
||||||
|
identity_images = [
|
||||||
|
_resize_image(c, derived_w, derived_h, resize_method, divisible_by)
|
||||||
|
for c in _selected
|
||||||
|
]
|
||||||
|
|
||||||
# Prepare background scene
|
# 2. Scene references: every timeline image (already loaded/resized into guide_data
|
||||||
bg_tensor = None
|
# upstream). Snapshot them, then clear guide_data so the downstream LTXDirectorGuide
|
||||||
if img_segs:
|
# is a no-op -- the Director bakes them itself for a self-consistent latent length.
|
||||||
bg_tensor = _load_image_tensor(img_segs[0])
|
scene_images = list(guide_data["images"])
|
||||||
if bg_tensor is None:
|
scene_frames = list(guide_data["insert_frames"])
|
||||||
bg_tensor = torch.zeros((1, derived_h, derived_w, 3), dtype=torch.float32)
|
scene_strengths = list(guide_data["strengths"])
|
||||||
|
guide_data = {"images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate)}
|
||||||
|
|
||||||
prepared_bg = _resize_image(bg_tensor, derived_w, derived_h, resize_method, divisible_by)
|
# 3. Slideshow background slide = first timeline image, else black.
|
||||||
prepared_images.append(prepared_bg)
|
if scene_images:
|
||||||
|
bg_slide = scene_images[0]
|
||||||
# Expand frames
|
if bg_slide.shape[1] != derived_h or bg_slide.shape[2] != derived_w:
|
||||||
base_count = msr_prefix_frames // len(prepared_images)
|
bg_slide = _resize_image(bg_slide, derived_w, derived_h, resize_method, divisible_by)
|
||||||
remainder = msr_prefix_frames % len(prepared_images)
|
else:
|
||||||
|
bg_slide = torch.zeros((1, derived_h, derived_w, 3), dtype=torch.float32)
|
||||||
|
|
||||||
|
# 4. Build the slideshow exactly like Licon MSR: characters then background,
|
||||||
|
# distributed across MSR_PREFIX_FRAMES.
|
||||||
|
slideshow_sources = identity_images + [bg_slide]
|
||||||
|
base_count = MSR_PREFIX_FRAMES // len(slideshow_sources)
|
||||||
|
remainder = MSR_PREFIX_FRAMES % len(slideshow_sources)
|
||||||
slideshow_tensors = []
|
slideshow_tensors = []
|
||||||
for index, prepared_img in enumerate(prepared_images):
|
for index, src_img in enumerate(slideshow_sources):
|
||||||
repeats = base_count + (1 if index < remainder else 0)
|
repeats = base_count + (1 if index < remainder else 0)
|
||||||
for _ in range(repeats):
|
slideshow_tensors.extend([src_img[0]] * repeats)
|
||||||
slideshow_tensors.append(prepared_img[0])
|
|
||||||
|
|
||||||
slideshow_video = torch.stack(slideshow_tensors)
|
slideshow_video = torch.stack(slideshow_tensors)
|
||||||
|
|
||||||
# VAE encode slideshow_video
|
# 5. PER-STAGE reference handoff. In two-stage pipelines the references must be
|
||||||
log.info(f"[PromptRelay] Encoding {msr_prefix_frames} Licon-MSR slideshow frames...")
|
# (re)encoded at EACH stage's resolution; a footprint baked here at full res breaks
|
||||||
|
# when scale_by shrinks the latent downstream. So the Director no longer injects --
|
||||||
|
# it hands the raw full-res slideshow + keyframe images to LTXDirectorGuide, which
|
||||||
|
# encodes and injects them at its own resolution. The length budget is purely
|
||||||
|
# TEMPORAL (scale-invariant): LTXVCropGuides always trims the slideshow's temporal
|
||||||
|
# footprint (prefix_latents), so the guide node pads the generated region so that
|
||||||
|
# (pad + keyframes) == prefix_latents at whatever resolution it runs.
|
||||||
|
keyframe_images = slideshow_sources # chars + bg (same set as the slideshow)
|
||||||
|
num_keyframes = len(keyframe_images)
|
||||||
|
prefix_latents = ((MSR_PREFIX_FRAMES - 1) // 8) + 1
|
||||||
|
tail_latents = prefix_latents # pad + keyframes, filled in per stage
|
||||||
|
total_runtime_latents = clean_latent_frames + tail_latents
|
||||||
|
tail_pixels = tail_latents * 8
|
||||||
|
|
||||||
# Fixed: directly capture the output Tensor returned from standard vae.encode
|
# 6. Relay conditioning over the runtime length. This is temporal, so the mask is
|
||||||
msr_latent_samples = vae.encode(slideshow_video)
|
# valid at every stage regardless of scale_by: clean -> locals, tail -> global.
|
||||||
|
local_part = processed_local.strip() if processed_local.strip() else processed_global
|
||||||
|
clean_lengths = segment_lengths.strip() if segment_lengths.strip() else str(duration_frames)
|
||||||
|
injected_local = f"{local_part} | {processed_global}"
|
||||||
|
injected_lengths = f"{clean_lengths},{tail_pixels}"
|
||||||
|
|
||||||
# Ensure msr_latent_samples is 5D [B, C, F, H, W] to match LTX Video standard
|
dummy_full = {"samples": torch.zeros(
|
||||||
if msr_latent_samples.dim() == 4:
|
[1, 128, total_runtime_latents, derived_h // 32, derived_w // 32],
|
||||||
msr_latent_samples = msr_latent_samples.unsqueeze(2)
|
|
||||||
|
|
||||||
prefix_latent_frames = msr_latent_samples.shape[2]
|
|
||||||
|
|
||||||
if optional_latent is not None:
|
|
||||||
# Concatenate slideshow and optional_latent along temporal dimension (dim=2)
|
|
||||||
log.info("[PromptRelay] Prepending reference slideshow to incoming optional_latent...")
|
|
||||||
opt_samples = optional_latent["samples"]
|
|
||||||
if opt_samples.dim() == 4:
|
|
||||||
opt_samples = opt_samples.unsqueeze(2)
|
|
||||||
|
|
||||||
samples = torch.cat([msr_latent_samples.to(device=opt_samples.device, dtype=opt_samples.dtype), opt_samples], dim=2)
|
|
||||||
total_latents = samples.shape[2]
|
|
||||||
else:
|
|
||||||
total_latents = prefix_latent_frames + clean_latent_frames
|
|
||||||
# Initialize 5D samples
|
|
||||||
samples = torch.zeros(
|
|
||||||
[1, 128, total_latents, derived_h // 32, derived_w // 32],
|
|
||||||
device=comfy.model_management.intermediate_device(),
|
device=comfy.model_management.intermediate_device(),
|
||||||
|
)}
|
||||||
|
|
||||||
|
# FALLBACK (relay vs LoRA attention conflict): replace the next call with
|
||||||
|
# patched = model.clone()
|
||||||
|
# conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(processed_global))
|
||||||
|
patched, conditioning = _encode_relay(
|
||||||
|
model, clip, dummy_full, processed_global, injected_local, injected_lengths, epsilon
|
||||||
)
|
)
|
||||||
# Copy encoded slideshow to the beginning of our latent
|
|
||||||
samples[:, :, :prefix_latent_frames, :, :] = msr_latent_samples
|
|
||||||
|
|
||||||
# Protect prefix frames using a 5D noise mask [B, 1, F, H, W] matching LTX Video standard
|
if negative is None:
|
||||||
if optional_latent is not None and "noise_mask" in optional_latent and optional_latent["noise_mask"] is not None:
|
neg_tokens = clip.tokenize("worst quality, blurry, low resolution, jittery, low geometry, bad details")
|
||||||
opt_mask = optional_latent["noise_mask"]
|
conditioning_neg = clip.encode_from_tokens_scheduled(neg_tokens)
|
||||||
if opt_mask.dim() == 4:
|
|
||||||
opt_mask = opt_mask.unsqueeze(1) # Convert 4D to 5D [B, 1, F, H, W]
|
|
||||||
|
|
||||||
prefix_mask = torch.ones((1, 1, prefix_latent_frames, samples.shape[3], samples.shape[4]), dtype=torch.float32, device=samples.device)
|
|
||||||
mask = torch.cat([prefix_mask, opt_mask.to(device=samples.device, dtype=prefix_mask.dtype)], dim=2)
|
|
||||||
else:
|
else:
|
||||||
mask = torch.zeros(
|
conditioning_neg = negative
|
||||||
(1, 1, total_latents, samples.shape[3], samples.shape[4]),
|
|
||||||
dtype=torch.float32,
|
# 7. Clean base latent: the TRUE clean region only (C frames), at full target res.
|
||||||
device=samples.device
|
# LTXDirectorGuide scales it per stage, pads it, and appends the keyframes.
|
||||||
)
|
latent = {
|
||||||
mask[:, :, :prefix_latent_frames, :, :] = 1.0
|
"samples": torch.zeros([1, 128, clean_latent_frames, derived_h // 32, derived_w // 32], device=comfy.model_management.intermediate_device()),
|
||||||
|
"noise_mask": torch.ones((1, 1, clean_latent_frames, derived_h // 32, derived_w // 32), dtype=torch.float32, device=comfy.model_management.intermediate_device()),
|
||||||
|
}
|
||||||
|
|
||||||
|
# 8. Hand the RAW full-res references to the guide node via guide_data (no new socket).
|
||||||
|
# Each stage VAE-encodes them at its own scale, so Stage 2 sees full-res detail.
|
||||||
|
guide_data = {
|
||||||
|
"images": [], "insert_frames": [], "strengths": [], "frame_rate": float(frame_rate),
|
||||||
|
"msr": {
|
||||||
|
"slideshow": slideshow_video, # [F, H, W, 3] full-res montage
|
||||||
|
"keyframes": keyframe_images, # list of [1, H, W, 3] full-res refs
|
||||||
|
"prefix_latents": int(prefix_latents),
|
||||||
|
"strength": float(reference_strength),
|
||||||
|
"downscale": float(MSR_LATENT_DOWNSCALE),
|
||||||
|
"clean_latent_frames": int(clean_latent_frames),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
total_latents = total_runtime_latents # clean + prefix (what the sampler sees)
|
||||||
|
|
||||||
latent = {"samples": samples, "noise_mask": mask}
|
|
||||||
log.info(
|
log.info(
|
||||||
"[PromptRelay] Created Licon-MSR Latent Prefix: %dx%d, %d total latent frames (%d prefix, %d generated)",
|
"[PromptRelay] Licon MSR Engine (per-stage): clean=%d, tail=%d (prefix), "
|
||||||
derived_w, derived_h, total_latents, prefix_latent_frames, total_latents - prefix_latent_frames
|
"%d keyframes handed to LTXDirectorGuide for per-resolution injection.",
|
||||||
|
clean_latent_frames, tail_latents, num_keyframes,
|
||||||
)
|
)
|
||||||
else: # Standard "Ghost Mask (End)"
|
else: # Standard "Ghost Mask (End)"
|
||||||
total_latents = clean_latent_frames + len(char_images)
|
total_latents = clean_latent_frames + len(char_images)
|
||||||
|
|
||||||
if optional_latent is None:
|
|
||||||
samples = torch.zeros(
|
|
||||||
[1, 128, total_latents, derived_h // 32, derived_w // 32],
|
|
||||||
device=comfy.model_management.intermediate_device(),
|
|
||||||
)
|
|
||||||
latent = {"samples": samples}
|
|
||||||
else:
|
|
||||||
latent = optional_latent
|
|
||||||
|
|
||||||
# Process references as guide_data appended to the end of the video
|
|
||||||
if char_images:
|
if char_images:
|
||||||
for i, single_ref in enumerate(char_images):
|
for i, single_ref in enumerate(char_images):
|
||||||
ref_tensor = _resize_image(single_ref, derived_w, derived_h, resize_method, divisible_by)
|
ref_tensor = _resize_image(single_ref, derived_w, derived_h, resize_method, divisible_by)
|
||||||
@@ -888,14 +1013,26 @@ class LTXDirector(io.ComfyNode):
|
|||||||
guide_data["insert_frames"].append(insert_point)
|
guide_data["insert_frames"].append(insert_point)
|
||||||
guide_data["strengths"].append(float(reference_strength))
|
guide_data["strengths"].append(float(reference_strength))
|
||||||
|
|
||||||
processed_global, processed_local = _preprocess_prompts_with_characters(
|
dummy_latent = {"samples": torch.zeros([1, 128, total_latents, derived_h // 32, derived_w // 32], device=comfy.model_management.intermediate_device())}
|
||||||
global_prompt, local_prompts, char1_val, char2_val, char3_val
|
|
||||||
)
|
|
||||||
|
|
||||||
patched, conditioning = _encode_relay(
|
patched, conditioning = _encode_relay(
|
||||||
model, clip, latent, processed_global, processed_local, segment_lengths, epsilon,
|
model, clip, dummy_latent, processed_global, processed_local, segment_lengths, epsilon,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if optional_latent is None:
|
||||||
|
latent = {
|
||||||
|
"samples": torch.zeros([1, 128, total_latents, derived_h // 32, derived_w // 32], device=comfy.model_management.intermediate_device()),
|
||||||
|
"noise_mask": torch.ones((1, 1, total_latents, derived_h // 32, derived_w // 32), dtype=torch.float32, device=comfy.model_management.intermediate_device())
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
latent = optional_latent
|
||||||
|
|
||||||
|
if negative is None:
|
||||||
|
neg_tokens = clip.tokenize("worst quality, blurry, low resolution, jittery, low geometry, bad details")
|
||||||
|
conditioning_neg = clip.encode_from_tokens_scheduled(neg_tokens)
|
||||||
|
else:
|
||||||
|
conditioning_neg = negative
|
||||||
|
|
||||||
ltxv_length = ((total_latents - 1) * 8) + 1
|
ltxv_length = ((total_latents - 1) * 8) + 1
|
||||||
audio_out = _build_combined_audio(timeline_data, ltxv_length, float(frame_rate))
|
audio_out = _build_combined_audio(timeline_data, ltxv_length, float(frame_rate))
|
||||||
audio_latent = {}
|
audio_latent = {}
|
||||||
@@ -932,7 +1069,6 @@ class LTXDirector(io.ComfyNode):
|
|||||||
if latent_samples.numel() == 0:
|
if latent_samples.numel() == 0:
|
||||||
raise ValueError("Encoded audio latent is empty (0 elements).")
|
raise ValueError("Encoded audio latent is empty (0 elements).")
|
||||||
|
|
||||||
# Fix variable shadowing: renamed audio mask tensor to prevent overwriting video mask
|
|
||||||
audio_mask_tensor = torch.full(
|
audio_mask_tensor = torch.full(
|
||||||
(1, latent_samples.shape[-2], latent_samples.shape[-1]),
|
(1, latent_samples.shape[-2], latent_samples.shape[-1]),
|
||||||
0.0,
|
0.0,
|
||||||
@@ -948,34 +1084,19 @@ class LTXDirector(io.ComfyNode):
|
|||||||
else:
|
else:
|
||||||
raise ValueError("No audio waveform to encode.")
|
raise ValueError("No audio waveform to encode.")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error("[PromptRelay] Failed to generate custom audio latent: %s", e)
|
log.warning("[PromptRelay] Failed to generate custom audio latent: %s", e)
|
||||||
raise e
|
raise e
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
audio_latent = get_empty_latent()
|
audio_latent = get_empty_latent()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error("[PromptRelay] Could not generate empty audio latent: %s", e)
|
log.warning("[PromptRelay] Could not generate empty audio latent: %s", e)
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
if save_prompts_to_file:
|
|
||||||
try:
|
|
||||||
formatted_text = _format_timeline_to_text(
|
|
||||||
processed_global, duration_frames, float(frame_rate), epsilon,
|
|
||||||
custom_width, custom_height, resize_method,
|
|
||||||
timeline_data, processed_local, segment_lengths, guide_strength
|
|
||||||
)
|
|
||||||
out_dir = folder_paths.get_output_directory()
|
|
||||||
filename = f"ltx_director_prompts_{int(time.time())}.txt"
|
|
||||||
filepath = os.path.join(out_dir, filename)
|
|
||||||
with open(filepath, "w", encoding="utf-8") as f:
|
|
||||||
f.write(formatted_text)
|
|
||||||
log.info(f"[PromptRelay] Saved timeline prompts to {filepath}")
|
|
||||||
except Exception as e:
|
|
||||||
log.warning(f"[PromptRelay] Failed to save prompts to txt: {e}")
|
|
||||||
|
|
||||||
return io.NodeOutput(
|
return io.NodeOutput(
|
||||||
patched,
|
patched,
|
||||||
conditioning,
|
conditioning,
|
||||||
|
conditioning_neg,
|
||||||
latent,
|
latent,
|
||||||
audio_latent,
|
audio_latent,
|
||||||
guide_data,
|
guide_data,
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ class LTXDirectorGuide(LTXVAddGuide):
|
|||||||
GuideData.Input("guide_data", tooltip="Guide data produced by Prompt Relay Encode (Timeline)."),
|
GuideData.Input("guide_data", tooltip="Guide data produced by Prompt Relay Encode (Timeline)."),
|
||||||
io.Float.Input("scale_by", default=1.0, min=0.01, max=8.0, step=0.01, tooltip="Scale the latent by this factor."),
|
io.Float.Input("scale_by", default=1.0, min=0.01, max=8.0, step=0.01, tooltip="Scale the latent by this factor."),
|
||||||
io.Combo.Input("upscale_method", options=["nearest-exact", "bilinear", "area", "bicubic", "bislerp"], default="bicubic", tooltip="Method used to upscale/downscale the latent."),
|
io.Combo.Input("upscale_method", options=["nearest-exact", "bilinear", "area", "bicubic", "bislerp"], default="bicubic", tooltip="Method used to upscale/downscale the latent."),
|
||||||
|
io.Float.Input("msr_strength", default=0.0, min=0.0, max=1.0, step=0.05, tooltip="Licon MSR only: per-stage reference strength override. 0 = use the Director's value (full pull, right for stage 1). On a refinement/upscale stage set ~0.4 to hold detail without the references repainting the opening (fixes stage-2 mist/ghosting)."),
|
||||||
],
|
],
|
||||||
outputs=[
|
outputs=[
|
||||||
io.Conditioning.Output(display_name="positive"),
|
io.Conditioning.Output(display_name="positive"),
|
||||||
@@ -33,7 +34,7 @@ class LTXDirectorGuide(LTXVAddGuide):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def execute(cls, positive, negative, vae, latent, guide_data, scale_by=1.0, upscale_method="bicubic") -> io.NodeOutput:
|
def execute(cls, positive, negative, vae, latent, guide_data, scale_by=1.0, upscale_method="bicubic", msr_strength=0.0) -> io.NodeOutput:
|
||||||
scale_factors = vae.downscale_index_formula
|
scale_factors = vae.downscale_index_formula
|
||||||
|
|
||||||
# Clone latents to avoid mutating upstream nodes
|
# Clone latents to avoid mutating upstream nodes
|
||||||
@@ -68,6 +69,12 @@ class LTXDirectorGuide(LTXVAddGuide):
|
|||||||
|
|
||||||
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
||||||
|
|
||||||
|
# MSR mode: the Director handed us the raw references; inject them at THIS node's
|
||||||
|
# resolution (so a half-res Stage 1 and a full-res Stage 2 each stay self-consistent).
|
||||||
|
msr = guide_data.get("msr")
|
||||||
|
if msr is not None:
|
||||||
|
return cls._inject_msr(positive, negative, vae, latent_image, noise_mask, msr, msr_strength)
|
||||||
|
|
||||||
images = guide_data.get("images", [])
|
images = guide_data.get("images", [])
|
||||||
insert_frames = guide_data.get("insert_frames", [])
|
insert_frames = guide_data.get("insert_frames", [])
|
||||||
strengths = guide_data.get("strengths", [])
|
strengths = guide_data.get("strengths", [])
|
||||||
@@ -88,3 +95,60 @@ class LTXDirectorGuide(LTXVAddGuide):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
|
return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _inject_msr(cls, positive, negative, vae, latent_image, noise_mask, msr, msr_strength=0.0):
|
||||||
|
"""Inject the Licon MSR references at this node's current resolution."""
|
||||||
|
from nodes import NODE_CLASS_MAPPINGS
|
||||||
|
from .ltx_director import _execute_comfy_node, _unpack
|
||||||
|
|
||||||
|
IC = NODE_CLASS_MAPPINGS.get("LTXAddVideoICLoRAGuide")
|
||||||
|
AG = NODE_CLASS_MAPPINGS.get("LTXVAddGuide")
|
||||||
|
if IC is None or AG is None:
|
||||||
|
raise ValueError("MSR mode needs LTXAddVideoICLoRAGuide (ComfyUI-LTXVideo) and LTXVAddGuide (core LTXV nodes).")
|
||||||
|
|
||||||
|
def clamp01(v):
|
||||||
|
return max(0.0, min(1.0, float(v)))
|
||||||
|
|
||||||
|
slideshow = msr["slideshow"]
|
||||||
|
keyframes = msr["keyframes"]
|
||||||
|
prefix_latents = int(msr["prefix_latents"])
|
||||||
|
# Per-stage override: msr_strength > 0 wins (set it low on a refinement stage so the
|
||||||
|
# references hold detail without repainting the opening); 0 = use the Director's value.
|
||||||
|
strength = clamp01(msr_strength) if float(msr_strength) > 0.0 else clamp01(msr["strength"])
|
||||||
|
downscale = float(msr["downscale"])
|
||||||
|
|
||||||
|
# Pad the generated region so (pad + keyframes) == prefix_latents, so the downstream crop
|
||||||
|
# (which trims the slideshow's temporal footprint) lands on the true clean length.
|
||||||
|
pad_latents = max(0, prefix_latents - len(keyframes))
|
||||||
|
if pad_latents > 0:
|
||||||
|
B, C, F, H, W = latent_image.shape
|
||||||
|
latent_image = torch.cat(
|
||||||
|
[latent_image, torch.zeros((B, C, pad_latents, H, W), dtype=latent_image.dtype, device=latent_image.device)],
|
||||||
|
dim=2,
|
||||||
|
)
|
||||||
|
mb, mc, mf, mh, mw = noise_mask.shape
|
||||||
|
noise_mask = torch.cat(
|
||||||
|
[noise_mask, torch.ones((mb, mc, pad_latents, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
|
||||||
|
dim=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
latent = {"samples": latent_image, "noise_mask": noise_mask}
|
||||||
|
|
||||||
|
# CONDITIONING PATH: slideshow -> IC-LoRA -> keep conditioning, discard latent.
|
||||||
|
cp, cn, _ = _unpack(_execute_comfy_node(
|
||||||
|
IC, positive=positive, negative=negative, vae=vae, latent=latent, image=slideshow,
|
||||||
|
frame_idx=0, strength=strength, latent_downscale_factor=downscale,
|
||||||
|
crop="center", use_tiled_encode=False, tile_size=256, tile_overlap=64,
|
||||||
|
))
|
||||||
|
|
||||||
|
positive, negative = cp, cn
|
||||||
|
|
||||||
|
# LATENT PATH: one frozen keyframe per reference at frame 0 -> keep latent, drop cond.
|
||||||
|
for kf in keyframes:
|
||||||
|
_p, _n, latent = _unpack(_execute_comfy_node(
|
||||||
|
AG, positive=positive, negative=negative, vae=vae, latent=latent,
|
||||||
|
image=kf, frame_idx=0, strength=strength,
|
||||||
|
))
|
||||||
|
|
||||||
|
return io.NodeOutput(positive, negative, latent)
|
||||||
Reference in New Issue
Block a user