Remove character image analysis flow
This commit is contained in:
@@ -50,13 +50,7 @@ All of my nodes are created with the help of AI, so there may or may not be redu
|
||||
|
||||
This is a Modded LTX director node from "WhatDreamsCost" with some extra options to help you create videos with references sheets
|
||||
|
||||
This node uses Ollama Locally : you will need to install it (very lightweight) and
|
||||
|
||||
- In your Ollama model folder run this command to install qwen 3.5 2b q4 (1.9gb) "ollama run huihui_ai/qwen3.5-abliterated:2B"
|
||||
|
||||
- The model won't eat memory while generating since there is an auto clear VRAM when you hit Run or after 5min.
|
||||
|
||||
- You can still enter your description manually if you don't want to install it but it works very well!
|
||||
- Character descriptions are entered manually in the MSR character fields and are appended to the director prompt as tagged character references.
|
||||
|
||||
- Reference mode included: Licon MSR.
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -11352,28 +11352,6 @@ const APPENDED_WIDGET_DEFAULTS = [
|
||||
|
||||
app.registerExtension({
|
||||
name: "LTXDirectorCS",
|
||||
async setup() {
|
||||
// On Run, ask Ollama to release the analysis model from VRAM so it doesn't
|
||||
// compete with LTX generation. Fully tolerant: failures are swallowed.
|
||||
if (app._ltxDirectorUnloadHookInstalled) return;
|
||||
app._ltxDirectorUnloadHookInstalled = true;
|
||||
const origQueuePrompt = app.queuePrompt;
|
||||
app.queuePrompt = async function (...args) {
|
||||
try {
|
||||
const nodes = app.graph?._nodes || [];
|
||||
const director = nodes.find(n => n && (n.comfyClass === "LTXDirectorCS" || n.type === "LTXDirectorCS"));
|
||||
if (director) {
|
||||
try {
|
||||
await api.fetchApi("/ltx_director/unload_ollama", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ provider: "ollama" }),
|
||||
});
|
||||
} catch (e) {}
|
||||
}
|
||||
} catch (e) {}
|
||||
return origQueuePrompt.apply(this, args);
|
||||
};
|
||||
},
|
||||
async beforeRegisterNodeDef(nodeType, nodeData, app) {
|
||||
if (nodeData.name === "LTXDirectorCS") {
|
||||
|
||||
|
||||
+2
-302
@@ -1,6 +1,5 @@
|
||||
const { app } = window.comfyAPI.app;
|
||||
const { api } = window.comfyAPI.api;
|
||||
const DEFAULT_ANALYZE_PROMPT = "Describe the character's physical appearance in two concise sentences. Specify their hair color/style, face details, and their clothing type/color. Keep the entire response very brief.";
|
||||
|
||||
function findWidget(node, name) {
|
||||
return (node.widgets || []).find((widget) => widget.name === name);
|
||||
@@ -58,139 +57,6 @@ function resolveCandidateImageUrls(originNode) {
|
||||
return urls;
|
||||
}
|
||||
|
||||
async function blobToOptimizedDataUrl(blob, maxDim = 1024, quality = 0.82) {
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
try {
|
||||
const image = await new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => resolve(img);
|
||||
img.onerror = () => reject(new Error("Failed to decode image blob"));
|
||||
img.src = objectUrl;
|
||||
});
|
||||
|
||||
const width = image.naturalWidth || image.width || 0;
|
||||
const height = image.naturalHeight || image.height || 0;
|
||||
const scale = width > 0 && height > 0 ? Math.min(1, maxDim / Math.max(width, height)) : 1;
|
||||
const targetWidth = Math.max(1, Math.round(width * scale)) || width || 1;
|
||||
const targetHeight = Math.max(1, Math.round(height * scale)) || height || 1;
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = targetWidth;
|
||||
canvas.height = targetHeight;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
throw new Error("Could not acquire canvas context");
|
||||
}
|
||||
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
|
||||
|
||||
const optimizedBlob = await new Promise((resolve, reject) => {
|
||||
canvas.toBlob((result) => {
|
||||
if (result) resolve(result);
|
||||
else reject(new Error("Canvas toBlob failed"));
|
||||
}, "image/jpeg", quality);
|
||||
});
|
||||
|
||||
const dataUrl = await new Promise((resolve) => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => resolve(reader.result);
|
||||
reader.readAsDataURL(optimizedBlob);
|
||||
});
|
||||
|
||||
return {
|
||||
dataUrl,
|
||||
originalWidth: width,
|
||||
originalHeight: height,
|
||||
outputWidth: targetWidth,
|
||||
outputHeight: targetHeight,
|
||||
outputBytes: optimizedBlob.size || 0,
|
||||
};
|
||||
} finally {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function imageInputToDataUrl(node, inputName) {
|
||||
const originNode = getOriginNodeForInput(node, inputName);
|
||||
const candidateUrls = resolveCandidateImageUrls(originNode);
|
||||
if (!candidateUrls.length) {
|
||||
return {
|
||||
dataUrl: null,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls: [],
|
||||
selectedUrl: null,
|
||||
mimeType: null,
|
||||
blobBytes: 0,
|
||||
dataUrlLength: 0,
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
outputWidth: 0,
|
||||
outputHeight: 0,
|
||||
outputBytes: 0,
|
||||
error: "No candidate image URLs found",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
for (const imageUrl of candidateUrls) {
|
||||
try {
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
const blob = await response.blob();
|
||||
if (!blob.type.startsWith("image/")) {
|
||||
throw new Error(`Unexpected blob type: ${blob.type || "unknown"}`);
|
||||
}
|
||||
const optimized = await blobToOptimizedDataUrl(blob);
|
||||
const dataUrl = optimized.dataUrl;
|
||||
return {
|
||||
dataUrl,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls,
|
||||
selectedUrl: imageUrl,
|
||||
mimeType: blob.type || null,
|
||||
blobBytes: blob.size || 0,
|
||||
dataUrlLength: typeof dataUrl === "string" ? dataUrl.length : 0,
|
||||
originalWidth: optimized.originalWidth,
|
||||
originalHeight: optimized.originalHeight,
|
||||
outputWidth: optimized.outputWidth,
|
||||
outputHeight: optimized.outputHeight,
|
||||
outputBytes: optimized.outputBytes,
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn("[MSRCharacter] Failed candidate image source", imageUrl, error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dataUrl: null,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls,
|
||||
selectedUrl: null,
|
||||
mimeType: null,
|
||||
blobBytes: 0,
|
||||
dataUrlLength: 0,
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
outputWidth: 0,
|
||||
outputHeight: 0,
|
||||
outputBytes: 0,
|
||||
error: "All candidate image URLs failed",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function setWidgetValue(node, widget, value) {
|
||||
if (!widget) return;
|
||||
const previous = widget.value;
|
||||
@@ -220,9 +86,6 @@ function setWidgetValue(node, widget, value) {
|
||||
function syncFormFromWidgets(node) {
|
||||
const description = findWidget(node, "description")?.value || "";
|
||||
const alias = findWidget(node, "alias")?.value || "";
|
||||
const provider = findWidget(node, "analyze_provider")?.value || "ollama";
|
||||
const model = findWidget(node, "analyze_model")?.value || "";
|
||||
const baseUrl = findWidget(node, "analyze_base_url")?.value || "";
|
||||
|
||||
if (node._msrAliasInput && node._msrAliasInput.value !== alias) {
|
||||
node._msrAliasInput.value = alias;
|
||||
@@ -235,8 +98,7 @@ function syncFormFromWidgets(node) {
|
||||
alias
|
||||
? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>`
|
||||
: "Alias: <span style=\"color:#666\">none</span>",
|
||||
`<div style="margin-top:4px;color:#888">Analyze: ${provider}${model ? ` / ${model}` : ""}${baseUrl ? ` / ${baseUrl}` : ""}</div>`,
|
||||
`<div style="margin-top:4px;color:#666">Description is stored in the field below.</div>`,
|
||||
`<div style="margin-top:4px;color:#666">Description is stored in the field below and appended to the director prompt as a tagged character reference.</div>`,
|
||||
].join("");
|
||||
}
|
||||
}
|
||||
@@ -323,70 +185,6 @@ function buildCharacterUi(node) {
|
||||
gap: "6px",
|
||||
});
|
||||
|
||||
const actionsRow = document.createElement("div");
|
||||
Object.assign(actionsRow.style, {
|
||||
display: "flex",
|
||||
gap: "8px",
|
||||
alignItems: "center",
|
||||
flexWrap: "wrap",
|
||||
});
|
||||
|
||||
const analyzeButton = document.createElement("button");
|
||||
analyzeButton.type = "button";
|
||||
analyzeButton.textContent = "Analyze";
|
||||
Object.assign(analyzeButton.style, {
|
||||
alignSelf: "flex-start",
|
||||
background: "#2b4f38",
|
||||
color: "#f3f3f3",
|
||||
border: "1px solid #496d56",
|
||||
borderRadius: "6px",
|
||||
padding: "6px 10px",
|
||||
fontSize: "11px",
|
||||
cursor: "pointer",
|
||||
});
|
||||
analyzeButton.addEventListener("click", () => {
|
||||
const widgetButton = (node.widgets || []).find((widget) => widget.msrAnalyze);
|
||||
if (!widgetButton) {
|
||||
alert("Analyze button widget is missing on this node.");
|
||||
return;
|
||||
}
|
||||
analyzeCharacterNode(node, widgetButton);
|
||||
});
|
||||
|
||||
const settingsButton = document.createElement("button");
|
||||
settingsButton.type = "button";
|
||||
settingsButton.textContent = "Analyze Settings";
|
||||
Object.assign(settingsButton.style, {
|
||||
alignSelf: "flex-start",
|
||||
background: "#252525",
|
||||
color: "#ddd",
|
||||
border: "1px solid #444",
|
||||
borderRadius: "6px",
|
||||
padding: "6px 10px",
|
||||
fontSize: "11px",
|
||||
cursor: "pointer",
|
||||
});
|
||||
settingsButton.addEventListener("click", () => {
|
||||
const providerWidget = findWidget(node, "analyze_provider");
|
||||
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||
const modelWidget = findWidget(node, "analyze_model");
|
||||
const promptWidget = findWidget(node, "analyze_prompt");
|
||||
const provider = window.prompt("Analyze provider: ollama, lmstudio, custom, off", providerWidget?.value || "ollama");
|
||||
if (provider == null) return;
|
||||
const baseUrl = window.prompt("Analyze base URL (blank = default)", baseUrlWidget?.value || "");
|
||||
if (baseUrl == null) return;
|
||||
const model = window.prompt("Analyze model (blank = provider default)", modelWidget?.value || "");
|
||||
if (model == null) return;
|
||||
const prompt = window.prompt("Analyze prompt (blank = default)", promptWidget?.value || DEFAULT_ANALYZE_PROMPT);
|
||||
if (prompt == null) return;
|
||||
setWidgetValue(node, providerWidget, provider.trim() || "ollama");
|
||||
setWidgetValue(node, baseUrlWidget, baseUrl.trim());
|
||||
setWidgetValue(node, modelWidget, model.trim());
|
||||
setWidgetValue(node, promptWidget, prompt.trim());
|
||||
syncFormFromWidgets(node);
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
});
|
||||
|
||||
const aliasLabel = document.createElement("label");
|
||||
aliasLabel.textContent = "Alias";
|
||||
Object.assign(aliasLabel.style, {
|
||||
@@ -439,13 +237,10 @@ function buildCharacterUi(node) {
|
||||
syncFormFromWidgets(node);
|
||||
});
|
||||
|
||||
actionsRow.appendChild(analyzeButton);
|
||||
actionsRow.appendChild(settingsButton);
|
||||
form.appendChild(aliasLabel);
|
||||
form.appendChild(aliasInput);
|
||||
form.appendChild(descLabel);
|
||||
form.appendChild(descInput);
|
||||
form.appendChild(actionsRow);
|
||||
|
||||
container.appendChild(previewRow);
|
||||
container.appendChild(meta);
|
||||
@@ -456,79 +251,9 @@ function buildCharacterUi(node) {
|
||||
node._msrMeta = meta;
|
||||
node._msrAliasInput = aliasInput;
|
||||
node._msrDescriptionInput = descInput;
|
||||
node._msrAnalyzeButton = analyzeButton;
|
||||
return container;
|
||||
}
|
||||
|
||||
async function analyzeCharacterNode(node, buttonWidget) {
|
||||
const descriptionWidget = findWidget(node, "description");
|
||||
if (!descriptionWidget) return;
|
||||
const providerWidget = findWidget(node, "analyze_provider");
|
||||
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||
const modelWidget = findWidget(node, "analyze_model");
|
||||
const promptWidget = findWidget(node, "analyze_prompt");
|
||||
|
||||
buttonWidget.label = "Analyzing...";
|
||||
if (node._msrAnalyzeButton) {
|
||||
node._msrAnalyzeButton.textContent = "Analyzing...";
|
||||
node._msrAnalyzeButton.disabled = true;
|
||||
}
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
|
||||
try {
|
||||
const imageResults = await Promise.all([
|
||||
imageInputToDataUrl(node, "image_1"),
|
||||
imageInputToDataUrl(node, "image_2"),
|
||||
]);
|
||||
const imageDebug = imageResults.map((result) => result?.debug || null).filter(Boolean);
|
||||
const images = (
|
||||
imageResults.map((result) => result?.dataUrl || null)
|
||||
).filter(Boolean);
|
||||
|
||||
if (!images.length) {
|
||||
throw new Error("Connect at least one image input before analyzing.");
|
||||
}
|
||||
|
||||
const response = await api.fetchApi("/ltx_director/analyze_character", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
provider: providerWidget?.value || "ollama",
|
||||
base_url: baseUrlWidget?.value || "",
|
||||
model: modelWidget?.value || "",
|
||||
prompt: promptWidget?.value || "",
|
||||
image_b64: images,
|
||||
image_debug: imageDebug,
|
||||
}),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (result.status !== "success") {
|
||||
console.warn("[MSRCharacter] analyze debug", {
|
||||
sentImages: imageDebug,
|
||||
response: result,
|
||||
});
|
||||
throw new Error(result.message || "Unknown analysis error");
|
||||
}
|
||||
|
||||
const description = result.description || "";
|
||||
setWidgetValue(node, descriptionWidget, description);
|
||||
if (node._msrDescriptionInput) {
|
||||
node._msrDescriptionInput.value = description;
|
||||
}
|
||||
syncFormFromWidgets(node);
|
||||
refreshCharacterPreview(node);
|
||||
} catch (error) {
|
||||
console.error("[MSRCharacter] analyze failed", error);
|
||||
alert(`MSR Character analyze failed: ${error.message || error}`);
|
||||
} finally {
|
||||
buttonWidget.label = "Analyze with Ollama";
|
||||
if (node._msrAnalyzeButton) {
|
||||
node._msrAnalyzeButton.textContent = "Analyze";
|
||||
node._msrAnalyzeButton.disabled = false;
|
||||
}
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
}
|
||||
}
|
||||
|
||||
function hideNodeWidget(widget) {
|
||||
if (!widget) return;
|
||||
widget.hidden = true;
|
||||
@@ -547,30 +272,6 @@ app.registerExtension({
|
||||
nodeType.prototype.onNodeCreated = function () {
|
||||
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
|
||||
|
||||
if (!(this.widgets || []).find((widget) => widget.msrAnalyze)) {
|
||||
this.addWidget("button", "Analyze with Ollama", null, () => {
|
||||
analyzeCharacterNode(this, (this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||
}, { serialize: false });
|
||||
const buttonWidget = this.widgets[this.widgets.length - 1];
|
||||
buttonWidget.msrAnalyze = true;
|
||||
buttonWidget.label = "Analyze with Ollama";
|
||||
}
|
||||
|
||||
if (!findWidget(this, "analyze_provider")) {
|
||||
this.addWidget("combo", "analyze_provider", "ollama", null, {
|
||||
values: ["ollama", "lmstudio", "custom", "off"],
|
||||
});
|
||||
}
|
||||
if (!findWidget(this, "analyze_base_url")) {
|
||||
this.addWidget("text", "analyze_base_url", "");
|
||||
}
|
||||
if (!findWidget(this, "analyze_model")) {
|
||||
this.addWidget("text", "analyze_model", "");
|
||||
}
|
||||
if (!findWidget(this, "analyze_prompt")) {
|
||||
this.addWidget("text", "analyze_prompt", DEFAULT_ANALYZE_PROMPT);
|
||||
}
|
||||
|
||||
if (!this._msrPreviewWidget) {
|
||||
const previewContainer = buildCharacterUi(this);
|
||||
this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, {
|
||||
@@ -597,10 +298,9 @@ app.registerExtension({
|
||||
return result;
|
||||
};
|
||||
|
||||
["alias", "description", "analyze_provider", "analyze_base_url", "analyze_model", "analyze_prompt"].forEach((widgetName) => {
|
||||
["alias", "description"].forEach((widgetName) => {
|
||||
hideNodeWidget(findWidget(this, widgetName));
|
||||
});
|
||||
hideNodeWidget((this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||
|
||||
this.size[0] = Math.max(this.size[0] || 0, 330);
|
||||
syncFormFromWidgets(this);
|
||||
|
||||
+33
-362
@@ -366,50 +366,32 @@ def _build_character_tag_groups(characters: list[dict]) -> list[tuple[str, ...]]
|
||||
return groups
|
||||
|
||||
|
||||
def _character_prompt_replacements(characters: list[dict]) -> dict[str, str]:
|
||||
replacements: dict[str, str] = {}
|
||||
for character, tags in zip(characters or [], _build_character_tag_groups(characters)):
|
||||
replacement = character.get("description", "") or ""
|
||||
for tag in tags:
|
||||
replacements[tag] = replacement
|
||||
return replacements
|
||||
def _build_character_reference_block(characters: list[dict] | None = None) -> str:
|
||||
lines: list[str] = []
|
||||
for idx, character in enumerate(characters or []):
|
||||
description = (character.get("description", "") or "").strip()
|
||||
if not description:
|
||||
continue
|
||||
|
||||
alias = _normalize_character_alias(character.get("alias", ""))
|
||||
canonical_tag = f"@char{idx + 1}"
|
||||
ref_label = canonical_tag if not alias else f"{canonical_tag} / @{alias}"
|
||||
lines.append(f"{ref_label} - {description}")
|
||||
|
||||
if not lines:
|
||||
return ""
|
||||
|
||||
return "Character references:\n" + "\n".join(lines)
|
||||
|
||||
|
||||
def _prepend_character_descriptions(global_prompt: str, characters: list[dict] | None = None) -> str:
|
||||
descriptions = [
|
||||
(character.get("description", "") or "").strip()
|
||||
for character in (characters or [])
|
||||
if (character.get("description", "") or "").strip()
|
||||
]
|
||||
if not descriptions:
|
||||
return global_prompt or ""
|
||||
|
||||
prefix = ". ".join(descriptions)
|
||||
if not (global_prompt or "").strip():
|
||||
return prefix
|
||||
return f"{prefix}. {global_prompt.strip()}"
|
||||
|
||||
|
||||
def _preprocess_prompts_with_characters(global_prompt, local_prompts, characters: list[dict] | None = None):
|
||||
"""Invisibly swaps out @characterN/@charN/@alias tags with their character descriptions."""
|
||||
if "@" not in (global_prompt or "") and "@" not in (local_prompts or ""):
|
||||
return global_prompt or "", local_prompts or ""
|
||||
|
||||
replacements = _character_prompt_replacements(characters or [])
|
||||
|
||||
def apply_replacements(text: str) -> str:
|
||||
updated = text or ""
|
||||
for tag, replacement in replacements.items():
|
||||
if tag in updated:
|
||||
updated = updated.replace(tag, replacement)
|
||||
return updated
|
||||
|
||||
gp = apply_replacements(global_prompt or "")
|
||||
if not local_prompts:
|
||||
return gp, ""
|
||||
|
||||
processed_locals = [apply_replacements(part.strip()) for part in local_prompts.split("|")]
|
||||
return gp, " | ".join(processed_locals)
|
||||
def _append_character_references(global_prompt: str, characters: list[dict] | None = None) -> str:
|
||||
prompt = (global_prompt or "").strip()
|
||||
reference_block = _build_character_reference_block(characters)
|
||||
if not reference_block:
|
||||
return prompt
|
||||
if not prompt:
|
||||
return reference_block
|
||||
return f"{prompt}\n\n{reference_block}"
|
||||
|
||||
|
||||
def _load_image_source(b64_or_url: str, filename: str = None, cache: dict | None = None,
|
||||
@@ -512,312 +494,6 @@ async def ltx_director_check_file(request):
|
||||
|
||||
|
||||
# --- Provider defaults shared by the analyze + unload endpoints ---
|
||||
_PROVIDER_DEFAULTS = {
|
||||
"ollama": {"url": "http://127.0.0.1:11434", "model": "huihui_ai/qwen3.5-abliterated:2b"},
|
||||
"lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
|
||||
"custom": {"url": "", "model": ""},
|
||||
}
|
||||
|
||||
_ANALYZE_SYSTEM_PROMPT = ""
|
||||
|
||||
_ANALYZE_PROMPT = (
|
||||
"Describe the character's physical appearance in two concise sentences. "
|
||||
"Specify their hair color/style, face details, and their clothing type/color. "
|
||||
"Keep the entire response very brief."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_analyze_prompt(data: dict) -> str:
|
||||
prompt = (data.get("prompt") or "").strip()
|
||||
return prompt or _ANALYZE_PROMPT
|
||||
|
||||
|
||||
def _extract_ollama_generated_text(resp_json: dict) -> str:
|
||||
if not isinstance(resp_json, dict):
|
||||
return ""
|
||||
|
||||
candidates = [resp_json.get("response"), resp_json.get("content"), resp_json.get("thinking")]
|
||||
message = resp_json.get("message")
|
||||
if isinstance(message, dict):
|
||||
candidates.extend([
|
||||
message.get("content"),
|
||||
message.get("reasoning_content"),
|
||||
message.get("thinking"),
|
||||
])
|
||||
|
||||
cleaned_candidates = []
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, str):
|
||||
text = candidate.strip()
|
||||
if "<think>" in text:
|
||||
text = text.split("</think>")[-1].strip()
|
||||
if text:
|
||||
cleaned_candidates.append(text)
|
||||
|
||||
if not cleaned_candidates:
|
||||
return ""
|
||||
|
||||
usable = [text for text in cleaned_candidates if _analysis_text_is_usable(text)]
|
||||
if usable:
|
||||
return max(usable, key=len)
|
||||
|
||||
return max(cleaned_candidates, key=len)
|
||||
|
||||
|
||||
def _collect_ollama_text_candidates(resp_json: dict) -> list[str]:
|
||||
if not isinstance(resp_json, dict):
|
||||
return []
|
||||
|
||||
candidates = [resp_json.get("response"), resp_json.get("content"), resp_json.get("thinking")]
|
||||
message = resp_json.get("message")
|
||||
if isinstance(message, dict):
|
||||
candidates.extend([
|
||||
message.get("content"),
|
||||
message.get("reasoning_content"),
|
||||
message.get("thinking"),
|
||||
])
|
||||
|
||||
cleaned = []
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, str):
|
||||
text = candidate.strip()
|
||||
if "<think>" in text:
|
||||
text = text.split("</think>")[-1].strip()
|
||||
if text:
|
||||
cleaned.append(text)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _analysis_text_is_usable(text: str) -> bool:
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
if len(text) < 24:
|
||||
return False
|
||||
if len(text.split()) < 6:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _compress_analysis_image_b64(b64_payload: str, max_dim: int = 768, quality: int = 82) -> str:
|
||||
"""Shrink analysis images so multimodal providers do not burn their full context on pixels."""
|
||||
try:
|
||||
raw = base64.b64decode(_normalise_b64_payload(b64_payload))
|
||||
with Image.open(_io.BytesIO(raw)) as img:
|
||||
img = img.convert("RGB")
|
||||
w, h = img.size
|
||||
if max(w, h) > max_dim:
|
||||
scale = max_dim / float(max(w, h))
|
||||
img = img.resize((max(1, int(round(w * scale))), max(1, int(round(h * scale)))), Image.LANCZOS)
|
||||
out = _io.BytesIO()
|
||||
img.save(out, format="JPEG", quality=quality, optimize=True)
|
||||
return base64.b64encode(out.getvalue()).decode("ascii")
|
||||
except Exception:
|
||||
return _normalise_b64_payload(b64_payload)
|
||||
|
||||
|
||||
def _prepare_analysis_images(cleaned_b64_list: list[str], max_dim: int = 768, quality: int = 82) -> list[str]:
|
||||
return [
|
||||
_compress_analysis_image_b64(b64, max_dim=max_dim, quality=quality)
|
||||
for b64 in cleaned_b64_list
|
||||
]
|
||||
|
||||
|
||||
def _resolve_provider(data):
|
||||
provider = (data.get("provider") or "ollama").lower()
|
||||
defs = _PROVIDER_DEFAULTS.get(provider, _PROVIDER_DEFAULTS["ollama"])
|
||||
base_url = (data.get("base_url") or defs["url"]).rstrip("/")
|
||||
model = data.get("model") or defs["model"]
|
||||
return provider, base_url, model
|
||||
|
||||
|
||||
# --- Character reference analysis endpoint (Ollama / LM Studio / Custom OpenAI-compatible) ---
|
||||
@PromptServer.instance.routes.post("/ltx_director/analyze_character")
|
||||
async def analyze_character_endpoint(request):
|
||||
try:
|
||||
import aiohttp
|
||||
data = await request.json()
|
||||
image_b64 = data.get("image_b64", "")
|
||||
image_debug = data.get("image_debug") or []
|
||||
char_index = int(data.get("char_index", 0))
|
||||
provider, base_url, model_name = _resolve_provider(data)
|
||||
analyze_prompt = _resolve_analyze_prompt(data)
|
||||
|
||||
if provider == "off":
|
||||
return web.json_response({"status": "error", "message": "Analyze is set to Off / Manual."})
|
||||
if not image_b64:
|
||||
return web.json_response({"status": "error", "message": "No image provided for analysis."})
|
||||
|
||||
b64_list = image_b64 if isinstance(image_b64, list) else [image_b64]
|
||||
cleaned_b64_list = []
|
||||
for b64 in b64_list:
|
||||
if "," in b64:
|
||||
b64 = b64.split(",", 1)[1]
|
||||
cleaned_b64_list.append(b64)
|
||||
if not cleaned_b64_list:
|
||||
return web.json_response({"status": "error", "message": "No valid base64 images decoded."})
|
||||
if provider in ("lmstudio", "custom") and not model_name:
|
||||
return web.json_response({
|
||||
"status": "error",
|
||||
"message": f"No model name set for {provider}. Open the gear menu and enter your loaded model's name.",
|
||||
})
|
||||
|
||||
log.info("[LTXDirector] Analyzing Character %d via %s (%s, model '%s')...",
|
||||
char_index + 1, provider, base_url, model_name)
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
if provider == "ollama":
|
||||
debug_candidates = []
|
||||
analysis_images = cleaned_b64_list
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"system": _ANALYZE_SYSTEM_PROMPT,
|
||||
"prompt": analyze_prompt,
|
||||
"images": analysis_images,
|
||||
"stream": False,
|
||||
"keep_alive": 0,
|
||||
"options": {
|
||||
"temperature": 0.2,
|
||||
"num_predict": 768,
|
||||
},
|
||||
}
|
||||
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as response:
|
||||
if response.status != 200:
|
||||
err_txt = await response.text()
|
||||
if response.status == 400 and "exceeds the available context size" in err_txt:
|
||||
analysis_images = _prepare_analysis_images(cleaned_b64_list, max_dim=512, quality=70)
|
||||
payload["images"] = analysis_images
|
||||
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as retry_response:
|
||||
if retry_response.status != 200:
|
||||
retry_err = await retry_response.text()
|
||||
return web.json_response({"status": "error", "message": f"Ollama HTTP {retry_response.status}: {retry_err}"})
|
||||
resp_json = await retry_response.json()
|
||||
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
|
||||
generated_text = _extract_ollama_generated_text(resp_json)
|
||||
else:
|
||||
return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"})
|
||||
else:
|
||||
resp_json = await response.json()
|
||||
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
|
||||
generated_text = _extract_ollama_generated_text(resp_json)
|
||||
|
||||
if not _analysis_text_is_usable(generated_text):
|
||||
chat_payload = {
|
||||
"model": model_name,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": _ANALYZE_SYSTEM_PROMPT,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": analyze_prompt,
|
||||
"images": analysis_images,
|
||||
},
|
||||
],
|
||||
"stream": False,
|
||||
"keep_alive": 0,
|
||||
"options": {
|
||||
"temperature": 0.2,
|
||||
"num_predict": 768,
|
||||
},
|
||||
}
|
||||
async with session.post(f"{base_url}/api/chat", json=chat_payload, timeout=300) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
debug_candidates.extend(_collect_ollama_text_candidates(resp_json))
|
||||
chat_text = _extract_ollama_generated_text(resp_json)
|
||||
if _analysis_text_is_usable(chat_text):
|
||||
generated_text = chat_text
|
||||
if not _analysis_text_is_usable(generated_text):
|
||||
return web.json_response({
|
||||
"status": "error",
|
||||
"message": "Ollama returned only a truncated analysis response.",
|
||||
"debug_candidates": debug_candidates,
|
||||
"image_debug": image_debug,
|
||||
"image_lengths": [len(b64) for b64 in cleaned_b64_list],
|
||||
"description": generated_text,
|
||||
})
|
||||
else:
|
||||
# OpenAI-compatible vision chat (LM Studio / Custom).
|
||||
content = [{"type": "text", "text": analyze_prompt}]
|
||||
for b64 in cleaned_b64_list:
|
||||
content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}})
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"messages": [
|
||||
{"role": "system", "content": _ANALYZE_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": content},
|
||||
],
|
||||
"max_tokens": 4096, "stream": False,
|
||||
}
|
||||
async with session.post(f"{base_url}/v1/chat/completions", json=payload, timeout=120) as response:
|
||||
if response.status != 200:
|
||||
err_txt = await response.text()
|
||||
return web.json_response({"status": "error", "message": f"{provider} HTTP {response.status}: {err_txt}"})
|
||||
resp_json = await response.json()
|
||||
try:
|
||||
msg = resp_json["choices"][0]["message"]
|
||||
generated_text = (msg.get("content") or "").strip()
|
||||
# Reasoning/"thinking" models (Gemma, Qwen-thinking, etc.) may leave
|
||||
# content empty and put their output in reasoning_content instead.
|
||||
if not generated_text:
|
||||
generated_text = (msg.get("reasoning_content") or "").strip()
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return web.json_response({"status": "error", "message": f"Unexpected response shape from {provider}."})
|
||||
except aiohttp.ClientConnectorError:
|
||||
return web.json_response({
|
||||
"status": "error",
|
||||
"message": f"Could not connect to {provider} at {base_url}. Make sure the server is running and reachable.",
|
||||
})
|
||||
|
||||
if "<think>" in generated_text:
|
||||
generated_text = generated_text.split("</think>")[-1].strip()
|
||||
|
||||
log.info("[LTXDirector] Analysis complete: %s", generated_text)
|
||||
return web.json_response({"status": "success", "description": generated_text})
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"[LTXDirector] Failed to analyze character: {e}")
|
||||
return web.json_response({"status": "error", "message": str(e)}, status=500)
|
||||
|
||||
|
||||
|
||||
@PromptServer.instance.routes.post("/ltx_director/unload_ollama")
|
||||
async def unload_ollama_endpoint(request):
|
||||
"""Evict the analysis model from VRAM right before a generation run, so the VLM doesn't
|
||||
compete with LTX for VRAM (the cause of intermittent CUDA offload crashes).
|
||||
|
||||
Ollama supports a clean instant unload (keep_alive=0). LM Studio / Custom have no reliable
|
||||
cross-version HTTP unload, so for those this is a graceful no-op — users should set a short
|
||||
JIT / auto-unload TTL in their server instead. Fully tolerant: never raises into the run.
|
||||
"""
|
||||
try:
|
||||
import aiohttp
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
data = {}
|
||||
provider, base_url, model_name = _resolve_provider(data)
|
||||
|
||||
if provider == "ollama":
|
||||
payload = {"model": model_name, "keep_alive": 0}
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(f"{base_url}/api/generate", json=payload, timeout=8) as response:
|
||||
await response.text()
|
||||
log.info("[LTXDirector] Asked Ollama to release '%s' from VRAM before the run.", model_name)
|
||||
except Exception:
|
||||
pass # Ollama not running / unreachable -> nothing to free.
|
||||
return web.json_response({"status": "ok", "provider": provider})
|
||||
|
||||
# LM Studio / Custom: no reliable HTTP unload across versions -> graceful no-op.
|
||||
return web.json_response({"status": "ok", "provider": provider, "note": "no-op (set a JIT/TTL unload in your server)"})
|
||||
except Exception as e:
|
||||
return web.json_response({"status": "error", "message": str(e)})
|
||||
|
||||
|
||||
def read_wav_peaks(wav_path):
|
||||
import wave
|
||||
with wave.open(wav_path, 'rb') as w:
|
||||
@@ -1635,6 +1311,8 @@ def _build_reference_mode_outputs(
|
||||
vae,
|
||||
global_prompt: str,
|
||||
local_prompts: str,
|
||||
raw_global_prompt: str,
|
||||
raw_local_prompts: str,
|
||||
segment_lengths: str,
|
||||
duration_frames: int,
|
||||
epsilon: float,
|
||||
@@ -1667,7 +1345,7 @@ def _build_reference_mode_outputs(
|
||||
tensor = _resize_image(tensor, latent_w, latent_h, "stretch to fit", divisible_by)
|
||||
return tensor
|
||||
|
||||
prompt_text = (global_prompt or "") + " " + (local_prompts or "")
|
||||
prompt_text = (raw_global_prompt or "") + " " + (raw_local_prompts or "")
|
||||
tag_groups = _build_character_tag_groups(characters)
|
||||
referenced_slots = [i for i, tags in enumerate(tag_groups) if any(t in prompt_text for t in tags)]
|
||||
selected = []
|
||||
@@ -2103,21 +1781,12 @@ class LTXDirector(io.ComfyNode):
|
||||
image_cache=image_cache,
|
||||
input_dir=input_dir,
|
||||
)
|
||||
global_prompt = _prepend_character_descriptions(global_prompt, characters)
|
||||
raw_global_prompt = global_prompt or ""
|
||||
raw_local_prompts = local_prompts or ""
|
||||
global_prompt = _append_character_references(raw_global_prompt, characters)
|
||||
char_slot_images = [list(character.get("images") or []) for character in characters]
|
||||
char_images = [img for slot_images in char_slot_images for img in slot_images]
|
||||
|
||||
# --- @charN substitution ---
|
||||
# OFF: swap @char tags for their VLM descriptions in the prompt text.
|
||||
# Licon MSR: leave the tags raw — there the reference IMAGE drives identity, and the
|
||||
# tags are used only to pick which character slots feed the slideshow.
|
||||
if reference_mode == "Licon MSR (Prefix)":
|
||||
ref_global, ref_local = global_prompt, local_prompts
|
||||
else:
|
||||
ref_global, ref_local = _preprocess_prompts_with_characters(
|
||||
global_prompt, local_prompts, characters
|
||||
)
|
||||
global_prompt, local_prompts = ref_global, ref_local
|
||||
local_prompts = raw_local_prompts
|
||||
|
||||
guide_data, derived_w, derived_h = _build_guide_data_from_timeline(
|
||||
tdata=tdata,
|
||||
@@ -2168,6 +1837,8 @@ class LTXDirector(io.ComfyNode):
|
||||
vae=vae,
|
||||
global_prompt=global_prompt,
|
||||
local_prompts=local_prompts,
|
||||
raw_global_prompt=raw_global_prompt,
|
||||
raw_local_prompts=raw_local_prompts,
|
||||
segment_lengths=segment_lengths,
|
||||
duration_frames=duration_frames,
|
||||
epsilon=epsilon,
|
||||
|
||||
Reference in New Issue
Block a user