Remove character image analysis flow

This commit is contained in:
OpenClaw Agent
2026-08-07 11:09:30 +00:00
parent beb0892793
commit 37a9c020a3
5 changed files with 40 additions and 697 deletions
+1 -7
View File
@@ -50,13 +50,7 @@ All of my nodes are created with the help of AI, so there may or may not be redu
This is a Modded LTX director node from "WhatDreamsCost" with some extra options to help you create videos with references sheets This is a Modded LTX director node from "WhatDreamsCost" with some extra options to help you create videos with references sheets
This node uses Ollama Locally : you will need to install it (very lightweight) and - Character descriptions are entered manually in the MSR character fields and are appended to the director prompt as tagged character references.
- In your Ollama model folder run this command to install qwen 3.5 2b q4 (1.9gb) "ollama run huihui_ai/qwen3.5-abliterated:2B"
- The model won't eat memory while generating since there is an auto clear VRAM when you hit Run or after 5min.
- You can still enter your description manually if you don't want to install it but it works very well!
- Reference mode included: Licon MSR. - Reference mode included: Licon MSR.
File diff suppressed because one or more lines are too long
-22
View File
@@ -11352,28 +11352,6 @@ const APPENDED_WIDGET_DEFAULTS = [
app.registerExtension({ app.registerExtension({
name: "LTXDirectorCS", name: "LTXDirectorCS",
async setup() {
// On Run, ask Ollama to release the analysis model from VRAM so it doesn't
// compete with LTX generation. Fully tolerant: failures are swallowed.
if (app._ltxDirectorUnloadHookInstalled) return;
app._ltxDirectorUnloadHookInstalled = true;
const origQueuePrompt = app.queuePrompt;
app.queuePrompt = async function (...args) {
try {
const nodes = app.graph?._nodes || [];
const director = nodes.find(n => n && (n.comfyClass === "LTXDirectorCS" || n.type === "LTXDirectorCS"));
if (director) {
try {
await api.fetchApi("/ltx_director/unload_ollama", {
method: "POST",
body: JSON.stringify({ provider: "ollama" }),
});
} catch (e) {}
}
} catch (e) {}
return origQueuePrompt.apply(this, args);
};
},
async beforeRegisterNodeDef(nodeType, nodeData, app) { async beforeRegisterNodeDef(nodeType, nodeData, app) {
if (nodeData.name === "LTXDirectorCS") { if (nodeData.name === "LTXDirectorCS") {
+2 -302
View File
@@ -1,6 +1,5 @@
const { app } = window.comfyAPI.app; const { app } = window.comfyAPI.app;
const { api } = window.comfyAPI.api; const { api } = window.comfyAPI.api;
const DEFAULT_ANALYZE_PROMPT = "Describe the character's physical appearance in two concise sentences. Specify their hair color/style, face details, and their clothing type/color. Keep the entire response very brief.";
function findWidget(node, name) { function findWidget(node, name) {
return (node.widgets || []).find((widget) => widget.name === name); return (node.widgets || []).find((widget) => widget.name === name);
@@ -58,139 +57,6 @@ function resolveCandidateImageUrls(originNode) {
return urls; return urls;
} }
async function blobToOptimizedDataUrl(blob, maxDim = 1024, quality = 0.82) {
const objectUrl = URL.createObjectURL(blob);
try {
const image = await new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = () => reject(new Error("Failed to decode image blob"));
img.src = objectUrl;
});
const width = image.naturalWidth || image.width || 0;
const height = image.naturalHeight || image.height || 0;
const scale = width > 0 && height > 0 ? Math.min(1, maxDim / Math.max(width, height)) : 1;
const targetWidth = Math.max(1, Math.round(width * scale)) || width || 1;
const targetHeight = Math.max(1, Math.round(height * scale)) || height || 1;
const canvas = document.createElement("canvas");
canvas.width = targetWidth;
canvas.height = targetHeight;
const ctx = canvas.getContext("2d");
if (!ctx) {
throw new Error("Could not acquire canvas context");
}
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
const optimizedBlob = await new Promise((resolve, reject) => {
canvas.toBlob((result) => {
if (result) resolve(result);
else reject(new Error("Canvas toBlob failed"));
}, "image/jpeg", quality);
});
const dataUrl = await new Promise((resolve) => {
const reader = new FileReader();
reader.onloadend = () => resolve(reader.result);
reader.readAsDataURL(optimizedBlob);
});
return {
dataUrl,
originalWidth: width,
originalHeight: height,
outputWidth: targetWidth,
outputHeight: targetHeight,
outputBytes: optimizedBlob.size || 0,
};
} finally {
URL.revokeObjectURL(objectUrl);
}
}
async function imageInputToDataUrl(node, inputName) {
const originNode = getOriginNodeForInput(node, inputName);
const candidateUrls = resolveCandidateImageUrls(originNode);
if (!candidateUrls.length) {
return {
dataUrl: null,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls: [],
selectedUrl: null,
mimeType: null,
blobBytes: 0,
dataUrlLength: 0,
originalWidth: 0,
originalHeight: 0,
outputWidth: 0,
outputHeight: 0,
outputBytes: 0,
error: "No candidate image URLs found",
},
};
}
for (const imageUrl of candidateUrls) {
try {
const response = await fetch(imageUrl);
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const blob = await response.blob();
if (!blob.type.startsWith("image/")) {
throw new Error(`Unexpected blob type: ${blob.type || "unknown"}`);
}
const optimized = await blobToOptimizedDataUrl(blob);
const dataUrl = optimized.dataUrl;
return {
dataUrl,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls,
selectedUrl: imageUrl,
mimeType: blob.type || null,
blobBytes: blob.size || 0,
dataUrlLength: typeof dataUrl === "string" ? dataUrl.length : 0,
originalWidth: optimized.originalWidth,
originalHeight: optimized.originalHeight,
outputWidth: optimized.outputWidth,
outputHeight: optimized.outputHeight,
outputBytes: optimized.outputBytes,
error: null,
},
};
} catch (error) {
console.warn("[MSRCharacter] Failed candidate image source", imageUrl, error);
}
}
return {
dataUrl: null,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls,
selectedUrl: null,
mimeType: null,
blobBytes: 0,
dataUrlLength: 0,
originalWidth: 0,
originalHeight: 0,
outputWidth: 0,
outputHeight: 0,
outputBytes: 0,
error: "All candidate image URLs failed",
},
};
}
function setWidgetValue(node, widget, value) { function setWidgetValue(node, widget, value) {
if (!widget) return; if (!widget) return;
const previous = widget.value; const previous = widget.value;
@@ -220,9 +86,6 @@ function setWidgetValue(node, widget, value) {
function syncFormFromWidgets(node) { function syncFormFromWidgets(node) {
const description = findWidget(node, "description")?.value || ""; const description = findWidget(node, "description")?.value || "";
const alias = findWidget(node, "alias")?.value || ""; const alias = findWidget(node, "alias")?.value || "";
const provider = findWidget(node, "analyze_provider")?.value || "ollama";
const model = findWidget(node, "analyze_model")?.value || "";
const baseUrl = findWidget(node, "analyze_base_url")?.value || "";
if (node._msrAliasInput && node._msrAliasInput.value !== alias) { if (node._msrAliasInput && node._msrAliasInput.value !== alias) {
node._msrAliasInput.value = alias; node._msrAliasInput.value = alias;
@@ -235,8 +98,7 @@ function syncFormFromWidgets(node) {
alias alias
? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>` ? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>`
: "Alias: <span style=\"color:#666\">none</span>", : "Alias: <span style=\"color:#666\">none</span>",
`<div style="margin-top:4px;color:#888">Analyze: ${provider}${model ? ` / ${model}` : ""}${baseUrl ? ` / ${baseUrl}` : ""}</div>`, `<div style="margin-top:4px;color:#666">Description is stored in the field below and appended to the director prompt as a tagged character reference.</div>`,
`<div style="margin-top:4px;color:#666">Description is stored in the field below.</div>`,
].join(""); ].join("");
} }
} }
@@ -323,70 +185,6 @@ function buildCharacterUi(node) {
gap: "6px", gap: "6px",
}); });
const actionsRow = document.createElement("div");
Object.assign(actionsRow.style, {
display: "flex",
gap: "8px",
alignItems: "center",
flexWrap: "wrap",
});
const analyzeButton = document.createElement("button");
analyzeButton.type = "button";
analyzeButton.textContent = "Analyze";
Object.assign(analyzeButton.style, {
alignSelf: "flex-start",
background: "#2b4f38",
color: "#f3f3f3",
border: "1px solid #496d56",
borderRadius: "6px",
padding: "6px 10px",
fontSize: "11px",
cursor: "pointer",
});
analyzeButton.addEventListener("click", () => {
const widgetButton = (node.widgets || []).find((widget) => widget.msrAnalyze);
if (!widgetButton) {
alert("Analyze button widget is missing on this node.");
return;
}
analyzeCharacterNode(node, widgetButton);
});
const settingsButton = document.createElement("button");
settingsButton.type = "button";
settingsButton.textContent = "Analyze Settings";
Object.assign(settingsButton.style, {
alignSelf: "flex-start",
background: "#252525",
color: "#ddd",
border: "1px solid #444",
borderRadius: "6px",
padding: "6px 10px",
fontSize: "11px",
cursor: "pointer",
});
settingsButton.addEventListener("click", () => {
const providerWidget = findWidget(node, "analyze_provider");
const baseUrlWidget = findWidget(node, "analyze_base_url");
const modelWidget = findWidget(node, "analyze_model");
const promptWidget = findWidget(node, "analyze_prompt");
const provider = window.prompt("Analyze provider: ollama, lmstudio, custom, off", providerWidget?.value || "ollama");
if (provider == null) return;
const baseUrl = window.prompt("Analyze base URL (blank = default)", baseUrlWidget?.value || "");
if (baseUrl == null) return;
const model = window.prompt("Analyze model (blank = provider default)", modelWidget?.value || "");
if (model == null) return;
const prompt = window.prompt("Analyze prompt (blank = default)", promptWidget?.value || DEFAULT_ANALYZE_PROMPT);
if (prompt == null) return;
setWidgetValue(node, providerWidget, provider.trim() || "ollama");
setWidgetValue(node, baseUrlWidget, baseUrl.trim());
setWidgetValue(node, modelWidget, model.trim());
setWidgetValue(node, promptWidget, prompt.trim());
syncFormFromWidgets(node);
node.setDirtyCanvas?.(true, true);
});
const aliasLabel = document.createElement("label"); const aliasLabel = document.createElement("label");
aliasLabel.textContent = "Alias"; aliasLabel.textContent = "Alias";
Object.assign(aliasLabel.style, { Object.assign(aliasLabel.style, {
@@ -439,13 +237,10 @@ function buildCharacterUi(node) {
syncFormFromWidgets(node); syncFormFromWidgets(node);
}); });
actionsRow.appendChild(analyzeButton);
actionsRow.appendChild(settingsButton);
form.appendChild(aliasLabel); form.appendChild(aliasLabel);
form.appendChild(aliasInput); form.appendChild(aliasInput);
form.appendChild(descLabel); form.appendChild(descLabel);
form.appendChild(descInput); form.appendChild(descInput);
form.appendChild(actionsRow);
container.appendChild(previewRow); container.appendChild(previewRow);
container.appendChild(meta); container.appendChild(meta);
@@ -456,79 +251,9 @@ function buildCharacterUi(node) {
node._msrMeta = meta; node._msrMeta = meta;
node._msrAliasInput = aliasInput; node._msrAliasInput = aliasInput;
node._msrDescriptionInput = descInput; node._msrDescriptionInput = descInput;
node._msrAnalyzeButton = analyzeButton;
return container; return container;
} }
async function analyzeCharacterNode(node, buttonWidget) {
const descriptionWidget = findWidget(node, "description");
if (!descriptionWidget) return;
const providerWidget = findWidget(node, "analyze_provider");
const baseUrlWidget = findWidget(node, "analyze_base_url");
const modelWidget = findWidget(node, "analyze_model");
const promptWidget = findWidget(node, "analyze_prompt");
buttonWidget.label = "Analyzing...";
if (node._msrAnalyzeButton) {
node._msrAnalyzeButton.textContent = "Analyzing...";
node._msrAnalyzeButton.disabled = true;
}
node.setDirtyCanvas?.(true, true);
try {
const imageResults = await Promise.all([
imageInputToDataUrl(node, "image_1"),
imageInputToDataUrl(node, "image_2"),
]);
const imageDebug = imageResults.map((result) => result?.debug || null).filter(Boolean);
const images = (
imageResults.map((result) => result?.dataUrl || null)
).filter(Boolean);
if (!images.length) {
throw new Error("Connect at least one image input before analyzing.");
}
const response = await api.fetchApi("/ltx_director/analyze_character", {
method: "POST",
body: JSON.stringify({
provider: providerWidget?.value || "ollama",
base_url: baseUrlWidget?.value || "",
model: modelWidget?.value || "",
prompt: promptWidget?.value || "",
image_b64: images,
image_debug: imageDebug,
}),
});
const result = await response.json();
if (result.status !== "success") {
console.warn("[MSRCharacter] analyze debug", {
sentImages: imageDebug,
response: result,
});
throw new Error(result.message || "Unknown analysis error");
}
const description = result.description || "";
setWidgetValue(node, descriptionWidget, description);
if (node._msrDescriptionInput) {
node._msrDescriptionInput.value = description;
}
syncFormFromWidgets(node);
refreshCharacterPreview(node);
} catch (error) {
console.error("[MSRCharacter] analyze failed", error);
alert(`MSR Character analyze failed: ${error.message || error}`);
} finally {
buttonWidget.label = "Analyze with Ollama";
if (node._msrAnalyzeButton) {
node._msrAnalyzeButton.textContent = "Analyze";
node._msrAnalyzeButton.disabled = false;
}
node.setDirtyCanvas?.(true, true);
}
}
function hideNodeWidget(widget) { function hideNodeWidget(widget) {
if (!widget) return; if (!widget) return;
widget.hidden = true; widget.hidden = true;
@@ -547,30 +272,6 @@ app.registerExtension({
nodeType.prototype.onNodeCreated = function () { nodeType.prototype.onNodeCreated = function () {
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments); if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
if (!(this.widgets || []).find((widget) => widget.msrAnalyze)) {
this.addWidget("button", "Analyze with Ollama", null, () => {
analyzeCharacterNode(this, (this.widgets || []).find((widget) => widget.msrAnalyze));
}, { serialize: false });
const buttonWidget = this.widgets[this.widgets.length - 1];
buttonWidget.msrAnalyze = true;
buttonWidget.label = "Analyze with Ollama";
}
if (!findWidget(this, "analyze_provider")) {
this.addWidget("combo", "analyze_provider", "ollama", null, {
values: ["ollama", "lmstudio", "custom", "off"],
});
}
if (!findWidget(this, "analyze_base_url")) {
this.addWidget("text", "analyze_base_url", "");
}
if (!findWidget(this, "analyze_model")) {
this.addWidget("text", "analyze_model", "");
}
if (!findWidget(this, "analyze_prompt")) {
this.addWidget("text", "analyze_prompt", DEFAULT_ANALYZE_PROMPT);
}
if (!this._msrPreviewWidget) { if (!this._msrPreviewWidget) {
const previewContainer = buildCharacterUi(this); const previewContainer = buildCharacterUi(this);
this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, { this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, {
@@ -597,10 +298,9 @@ app.registerExtension({
return result; return result;
}; };
["alias", "description", "analyze_provider", "analyze_base_url", "analyze_model", "analyze_prompt"].forEach((widgetName) => { ["alias", "description"].forEach((widgetName) => {
hideNodeWidget(findWidget(this, widgetName)); hideNodeWidget(findWidget(this, widgetName));
}); });
hideNodeWidget((this.widgets || []).find((widget) => widget.msrAnalyze));
this.size[0] = Math.max(this.size[0] || 0, 330); this.size[0] = Math.max(this.size[0] || 0, 330);
syncFormFromWidgets(this); syncFormFromWidgets(this);
+33 -362
View File
@@ -366,50 +366,32 @@ def _build_character_tag_groups(characters: list[dict]) -> list[tuple[str, ...]]
return groups return groups
def _character_prompt_replacements(characters: list[dict]) -> dict[str, str]: def _build_character_reference_block(characters: list[dict] | None = None) -> str:
replacements: dict[str, str] = {} lines: list[str] = []
for character, tags in zip(characters or [], _build_character_tag_groups(characters)): for idx, character in enumerate(characters or []):
replacement = character.get("description", "") or "" description = (character.get("description", "") or "").strip()
for tag in tags: if not description:
replacements[tag] = replacement continue
return replacements
alias = _normalize_character_alias(character.get("alias", ""))
canonical_tag = f"@char{idx + 1}"
ref_label = canonical_tag if not alias else f"{canonical_tag} / @{alias}"
lines.append(f"{ref_label} - {description}")
if not lines:
return ""
return "Character references:\n" + "\n".join(lines)
def _prepend_character_descriptions(global_prompt: str, characters: list[dict] | None = None) -> str: def _append_character_references(global_prompt: str, characters: list[dict] | None = None) -> str:
descriptions = [ prompt = (global_prompt or "").strip()
(character.get("description", "") or "").strip() reference_block = _build_character_reference_block(characters)
for character in (characters or []) if not reference_block:
if (character.get("description", "") or "").strip() return prompt
] if not prompt:
if not descriptions: return reference_block
return global_prompt or "" return f"{prompt}\n\n{reference_block}"
prefix = ". ".join(descriptions)
if not (global_prompt or "").strip():
return prefix
return f"{prefix}. {global_prompt.strip()}"
def _preprocess_prompts_with_characters(global_prompt, local_prompts, characters: list[dict] | None = None):
"""Invisibly swaps out @characterN/@charN/@alias tags with their character descriptions."""
if "@" not in (global_prompt or "") and "@" not in (local_prompts or ""):
return global_prompt or "", local_prompts or ""
replacements = _character_prompt_replacements(characters or [])
def apply_replacements(text: str) -> str:
updated = text or ""
for tag, replacement in replacements.items():
if tag in updated:
updated = updated.replace(tag, replacement)
return updated
gp = apply_replacements(global_prompt or "")
if not local_prompts:
return gp, ""
processed_locals = [apply_replacements(part.strip()) for part in local_prompts.split("|")]
return gp, " | ".join(processed_locals)
def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None, def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None,
@@ -512,312 +494,6 @@ async def ltx_director_check_file(request):
# --- Provider defaults shared by the analyze + unload endpoints --- # --- Provider defaults shared by the analyze + unload endpoints ---
_PROVIDER_DEFAULTS = {
"ollama": {"url": "http://127.0.0.1:11434", "model": "huihui_ai/qwen3.5-abliterated:2b"},
"lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
"custom": {"url": "", "model": ""},
}
_ANALYZE_SYSTEM_PROMPT = ""
_ANALYZE_PROMPT = (
"Describe the character's physical appearance in two concise sentences. "
"Specify their hair color/style, face details, and their clothing type/color. "
"Keep the entire response very brief."
)
def _resolve_analyze_prompt(data: dict) -> str:
prompt = (data.get("prompt") or "").strip()
return prompt or _ANALYZE_PROMPT
def _extract_ollama_generated_text(resp_json: dict) -> str:
if not isinstance(resp_json, dict):
return ""
candidates = [resp_json.get("response"), resp_json.get("content"), resp_json.get("thinking")]
message = resp_json.get("message")
if isinstance(message, dict):
candidates.extend([
message.get("content"),
message.get("reasoning_content"),
message.get("thinking"),
])
cleaned_candidates = []
for candidate in candidates:
if isinstance(candidate, str):
text = candidate.strip()
if "<think>" in text:
text = text.split("</think>")[-1].strip()
if text:
cleaned_candidates.append(text)
if not cleaned_candidates:
return ""
usable = [text for text in cleaned_candidates if _analysis_text_is_usable(text)]
if usable:
return max(usable, key=len)
return max(cleaned_candidates, key=len)
def _collect_ollama_text_candidates(resp_json: dict) -> list[str]:
if not isinstance(resp_json, dict):
return []
candidates = [resp_json.get("response"), resp_json.get("content"), resp_json.get("thinking")]
message = resp_json.get("message")
if isinstance(message, dict):
candidates.extend([
message.get("content"),
message.get("reasoning_content"),
message.get("thinking"),
])
cleaned = []
for candidate in candidates:
if isinstance(candidate, str):
text = candidate.strip()
if "<think>" in text:
text = text.split("</think>")[-1].strip()
if text:
cleaned.append(text)
return cleaned
def _analysis_text_is_usable(text: str) -> bool:
text = (text or "").strip()
if not text:
return False
if len(text) < 24:
return False
if len(text.split()) < 6:
return False
return True
def _compress_analysis_image_b64(b64_payload: str, max_dim: int = 768, quality: int = 82) -> str:
"""Shrink analysis images so multimodal providers do not burn their full context on pixels."""
try:
raw = base64.b64decode(_normalise_b64_payload(b64_payload))
with Image.open(_io.BytesIO(raw)) as img:
img = img.convert("RGB")
w, h = img.size
if max(w, h) > max_dim:
scale = max_dim / float(max(w, h))
img = img.resize((max(1, int(round(w * scale))), max(1, int(round(h * scale)))), Image.LANCZOS)
out = _io.BytesIO()
img.save(out, format="JPEG", quality=quality, optimize=True)
return base64.b64encode(out.getvalue()).decode("ascii")
except Exception:
return _normalise_b64_payload(b64_payload)
def _prepare_analysis_images(cleaned_b64_list: list[str], max_dim: int = 768, quality: int = 82) -> list[str]:
return [
_compress_analysis_image_b64(b64, max_dim=max_dim, quality=quality)
for b64 in cleaned_b64_list
]
def _resolve_provider(data):
provider = (data.get("provider") or "ollama").lower()
defs = _PROVIDER_DEFAULTS.get(provider, _PROVIDER_DEFAULTS["ollama"])
base_url = (data.get("base_url") or defs["url"]).rstrip("/")
model = data.get("model") or defs["model"]
return provider, base_url, model
# --- Character reference analysis endpoint (Ollama / LM Studio / Custom OpenAI-compatible) ---
@PromptServer.instance.routes.post("/ltx_director/analyze_character")
async def analyze_character_endpoint(request):
try:
import aiohttp
data = await request.json()
image_b64 = data.get("image_b64", "")
image_debug = data.get("image_debug") or []
char_index = int(data.get("char_index", 0))
provider, base_url, model_name = _resolve_provider(data)
analyze_prompt = _resolve_analyze_prompt(data)
if provider == "off":
return web.json_response({"status": "error", "message": "Analyze is set to Off / Manual."})
if not image_b64:
return web.json_response({"status": "error", "message": "No image provided for analysis."})
b64_list = image_b64 if isinstance(image_b64, list) else [image_b64]
cleaned_b64_list = []
for b64 in b64_list:
if "," in b64:
b64 = b64.split(",", 1)[1]
cleaned_b64_list.append(b64)
if not cleaned_b64_list:
return web.json_response({"status": "error", "message": "No valid base64 images decoded."})
if provider in ("lmstudio", "custom") and not model_name:
return web.json_response({
"status": "error",
"message": f"No model name set for {provider}. Open the gear menu and enter your loaded model's name.",
})
log.info("[LTXDirector] Analyzing Character %d via %s (%s, model '%s')...",
char_index + 1, provider, base_url, model_name)
try:
async with aiohttp.ClientSession() as session:
if provider == "ollama":
debug_candidates = []
analysis_images = cleaned_b64_list
payload = {
"model": model_name,
"system": _ANALYZE_SYSTEM_PROMPT,
"prompt": analyze_prompt,
"images": analysis_images,
"stream": False,
"keep_alive": 0,
"options": {
"temperature": 0.2,
"num_predict": 768,
},
}
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as response:
if response.status != 200:
err_txt = await response.text()
if response.status == 400 and "exceeds the available context size" in err_txt:
analysis_images = _prepare_analysis_images(cleaned_b64_list, max_dim=512, quality=70)
payload["images"] = analysis_images
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as retry_response:
if retry_response.status != 200:
retry_err = await retry_response.text()
return web.json_response({"status": "error", "message": f"Ollama HTTP {retry_response.status}: {retry_err}"})
resp_json = await retry_response.json()
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
generated_text = _extract_ollama_generated_text(resp_json)
else:
return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"})
else:
resp_json = await response.json()
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
generated_text = _extract_ollama_generated_text(resp_json)
if not _analysis_text_is_usable(generated_text):
chat_payload = {
"model": model_name,
"messages": [
{
"role": "system",
"content": _ANALYZE_SYSTEM_PROMPT,
},
{
"role": "user",
"content": analyze_prompt,
"images": analysis_images,
},
],
"stream": False,
"keep_alive": 0,
"options": {
"temperature": 0.2,
"num_predict": 768,
},
}
async with session.post(f"{base_url}/api/chat", json=chat_payload, timeout=300) as response:
if response.status == 200:
resp_json = await response.json()
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
chat_text = _extract_ollama_generated_text(resp_json)
if _analysis_text_is_usable(chat_text):
generated_text = chat_text
if not _analysis_text_is_usable(generated_text):
return web.json_response({
"status": "error",
"message": "Ollama returned only a truncated analysis response.",
"debug_candidates": debug_candidates,
"image_debug": image_debug,
"image_lengths": [len(b64) for b64 in cleaned_b64_list],
"description": generated_text,
})
else:
# OpenAI-compatible vision chat (LM Studio / Custom).
content = [{"type": "text", "text": analyze_prompt}]
for b64 in cleaned_b64_list:
content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}})
payload = {
"model": model_name,
"messages": [
{"role": "system", "content": _ANALYZE_SYSTEM_PROMPT},
{"role": "user", "content": content},
],
"max_tokens": 4096, "stream": False,
}
async with session.post(f"{base_url}/v1/chat/completions", json=payload, timeout=120) as response:
if response.status != 200:
err_txt = await response.text()
return web.json_response({"status": "error", "message": f"{provider} HTTP {response.status}: {err_txt}"})
resp_json = await response.json()
try:
msg = resp_json["choices"][0]["message"]
generated_text = (msg.get("content") or "").strip()
# Reasoning/"thinking" models (Gemma, Qwen-thinking, etc.) may leave
# content empty and put their output in reasoning_content instead.
if not generated_text:
generated_text = (msg.get("reasoning_content") or "").strip()
except (KeyError, IndexError, TypeError):
return web.json_response({"status": "error", "message": f"Unexpected response shape from {provider}."})
except aiohttp.ClientConnectorError:
return web.json_response({
"status": "error",
"message": f"Could not connect to {provider} at {base_url}. Make sure the server is running and reachable.",
})
if "<think>" in generated_text:
generated_text = generated_text.split("</think>")[-1].strip()
log.info("[LTXDirector] Analysis complete: %s", generated_text)
return web.json_response({"status": "success", "description": generated_text})
except Exception as e:
log.error(f"[LTXDirector] Failed to analyze character: {e}")
return web.json_response({"status": "error", "message": str(e)}, status=500)
@PromptServer.instance.routes.post("/ltx_director/unload_ollama")
async def unload_ollama_endpoint(request):
"""Evict the analysis model from VRAM right before a generation run, so the VLM doesn't
compete with LTX for VRAM (the cause of intermittent CUDA offload crashes).
Ollama supports a clean instant unload (keep_alive=0). LM Studio / Custom have no reliable
cross-version HTTP unload, so for those this is a graceful no-op — users should set a short
JIT / auto-unload TTL in their server instead. Fully tolerant: never raises into the run.
"""
try:
import aiohttp
try:
data = await request.json()
except Exception:
data = {}
provider, base_url, model_name = _resolve_provider(data)
if provider == "ollama":
payload = {"model": model_name, "keep_alive": 0}
try:
async with aiohttp.ClientSession() as session:
async with session.post(f"{base_url}/api/generate", json=payload, timeout=8) as response:
await response.text()
log.info("[LTXDirector] Asked Ollama to release '%s' from VRAM before the run.", model_name)
except Exception:
pass # Ollama not running / unreachable -> nothing to free.
return web.json_response({"status": "ok", "provider": provider})
# LM Studio / Custom: no reliable HTTP unload across versions -> graceful no-op.
return web.json_response({"status": "ok", "provider": provider, "note": "no-op (set a JIT/TTL unload in your server)"})
except Exception as e:
return web.json_response({"status": "error", "message": str(e)})
def read_wav_peaks(wav_path): def read_wav_peaks(wav_path):
import wave import wave
with wave.open(wav_path, 'rb') as w: with wave.open(wav_path, 'rb') as w:
@@ -1635,6 +1311,8 @@ def _build_reference_mode_outputs(
vae, vae,
global_prompt: str, global_prompt: str,
local_prompts: str, local_prompts: str,
raw_global_prompt: str,
raw_local_prompts: str,
segment_lengths: str, segment_lengths: str,
duration_frames: int, duration_frames: int,
epsilon: float, epsilon: float,
@@ -1667,7 +1345,7 @@ def _build_reference_mode_outputs(
tensor = _resize_image(tensor, latent_w, latent_h, "stretch to fit", divisible_by) tensor = _resize_image(tensor, latent_w, latent_h, "stretch to fit", divisible_by)
return tensor return tensor
prompt_text = (global_prompt or "") + " " + (local_prompts or "") prompt_text = (raw_global_prompt or "") + " " + (raw_local_prompts or "")
tag_groups = _build_character_tag_groups(characters) tag_groups = _build_character_tag_groups(characters)
referenced_slots = [i for i, tags in enumerate(tag_groups) if any(t in prompt_text for t in tags)] referenced_slots = [i for i, tags in enumerate(tag_groups) if any(t in prompt_text for t in tags)]
selected = [] selected = []
@@ -2103,21 +1781,12 @@ class LTXDirector(io.ComfyNode):
image_cache=image_cache, image_cache=image_cache,
input_dir=input_dir, input_dir=input_dir,
) )
global_prompt = _prepend_character_descriptions(global_prompt, characters) raw_global_prompt = global_prompt or ""
raw_local_prompts = local_prompts or ""
global_prompt = _append_character_references(raw_global_prompt, characters)
char_slot_images = [list(character.get("images") or []) for character in characters] char_slot_images = [list(character.get("images") or []) for character in characters]
char_images = [img for slot_images in char_slot_images for img in slot_images] char_images = [img for slot_images in char_slot_images for img in slot_images]
local_prompts = raw_local_prompts
# --- @charN substitution ---
# OFF: swap @char tags for their VLM descriptions in the prompt text.
# Licon MSR: leave the tags raw — there the reference IMAGE drives identity, and the
# tags are used only to pick which character slots feed the slideshow.
if reference_mode == "Licon MSR (Prefix)":
ref_global, ref_local = global_prompt, local_prompts
else:
ref_global, ref_local = _preprocess_prompts_with_characters(
global_prompt, local_prompts, characters
)
global_prompt, local_prompts = ref_global, ref_local
guide_data, derived_w, derived_h = _build_guide_data_from_timeline( guide_data, derived_w, derived_h = _build_guide_data_from_timeline(
tdata=tdata, tdata=tdata,
@@ -2168,6 +1837,8 @@ class LTXDirector(io.ComfyNode):
vae=vae, vae=vae,
global_prompt=global_prompt, global_prompt=global_prompt,
local_prompts=local_prompts, local_prompts=local_prompts,
raw_global_prompt=raw_global_prompt,
raw_local_prompts=raw_local_prompts,
segment_lengths=segment_lengths, segment_lengths=segment_lengths,
duration_frames=duration_frames, duration_frames=duration_frames,
epsilon=epsilon, epsilon=epsilon,