71 Commits

Author SHA1 Message Date
OpenClaw Agent
beb0892793 Normalize MSR slideshow frame sizes 2026-07-22 10:31:10 +00:00
OpenClaw Agent
2f40aa77ad Revert "Only prepend referenced character descriptions"
This reverts commit df4c936aae.
2026-07-22 10:16:48 +00:00
OpenClaw Agent
c2fea55704 Revert "Remove character description global prompt prefixing"
This reverts commit 5347ca3dbc.
2026-07-22 09:42:19 +00:00
OpenClaw Agent
c36f6f3bf6 Revert "Restore legacy MSR timeline image fallback"
This reverts commit 9ebc488072.
2026-07-22 09:42:19 +00:00
OpenClaw Agent
ee3215c48f Revert "Harden MSR against stale text and slot blending"
This reverts commit cfdee30dee.
2026-07-22 09:42:19 +00:00
OpenClaw Agent
cfdee30dee Harden MSR against stale text and slot blending 2026-07-22 09:24:52 +00:00
OpenClaw Agent
9ebc488072 Restore legacy MSR timeline image fallback 2026-07-22 09:03:57 +00:00
OpenClaw Agent
5347ca3dbc Remove character description global prompt prefixing 2026-07-22 06:58:46 +00:00
OpenClaw Agent
df4c936aae Only prepend referenced character descriptions 2026-07-20 15:41:32 +00:00
OpenClaw Agent
fa546cbc8f Log effective director global prompt 2026-07-20 14:55:03 +00:00
OpenClaw Agent
575838aeb0 Allow ten timeline sections 2026-07-20 14:30:32 +00:00
OpenClaw Agent
a1a1184155 Prepend character descriptions to global prompt 2026-07-20 14:20:46 +00:00
OpenClaw Agent
3f24406697 Allow custom character analyze prompts 2026-07-20 14:02:28 +00:00
OpenClaw Agent
a711a9d34b Trim character node meta description 2026-07-20 13:55:13 +00:00
OpenClaw Agent
00d5703137 Restore character analyze prompt and button 2026-07-20 13:32:41 +00:00
OpenClaw Agent
64cbfd2f36 Remove legacy timeline adapter node 2026-07-20 09:27:35 +00:00
OpenClaw Agent
e113472c7d Accept section sets directly in director 2026-07-20 09:24:16 +00:00
OpenClaw Agent
e01ceef71b Derive external timeline duration from sections 2026-07-20 09:17:04 +00:00
OpenClaw Agent
8736d4a463 Remove stale ref_images input from saved nodes 2026-07-17 10:24:04 +00:00
OpenClaw Agent
cc51b298fd Use external timeline runtime images 2026-07-17 09:21:55 +00:00
OpenClaw Agent
1cfe7f1932 Avoid serializing runtime timeline tensors 2026-07-17 09:12:31 +00:00
OpenClaw Agent
c782f3263b Allow director width and height input links 2026-07-17 08:10:29 +00:00
OpenClaw Agent
ab6415caa4 Show width and height widgets on director node 2026-07-17 08:07:25 +00:00
OpenClaw Agent
5c4349d9b9 Remove Ghost Mask reference mode 2026-07-17 07:45:53 +00:00
OpenClaw Agent
7b4e8865d6 feat: add external timeline nodes 2026-07-16 21:05:27 +00:00
OpenClaw Agent
4c0358c11b feat: live sync linked director controls 2026-07-16 16:14:10 +00:00
OpenClaw Agent
4dfcb1e597 Update MSR character analysis prompt 2026-07-10 12:10:05 +00:00
OpenClaw Agent
73918a28b3 Update MSR analysis prompt wording 2026-07-10 11:54:13 +00:00
OpenClaw Agent
48ac1c242f Tighten MSR analysis prompt and drop sanitizer 2026-07-10 11:49:07 +00:00
OpenClaw Agent
6bc0aa1f71 Sanitize MSR analysis prompt echo 2026-07-10 11:45:03 +00:00
OpenClaw Agent
e2c5c38b14 Compress MSR analysis images before upload 2026-07-10 11:42:33 +00:00
OpenClaw Agent
f372a2ed69 Expose MSR image analysis payload debug 2026-07-10 11:40:10 +00:00
OpenClaw Agent
2da1de2f6d Prefer source images for MSR analysis 2026-07-10 11:36:38 +00:00
OpenClaw Agent
b059974544 Expose truncated Ollama analysis candidates 2026-07-10 11:33:26 +00:00
OpenClaw Agent
27f1d6a1e4 Prefer fuller Ollama analysis text 2026-07-10 11:28:09 +00:00
OpenClaw Agent
6ec28d33a4 Retry truncated Ollama character analysis 2026-07-10 11:23:28 +00:00
OpenClaw Agent
2bd748c8a6 Restore MSR character analyze config and parsing 2026-07-10 11:14:49 +00:00
OpenClaw Agent
c20ded3319 Restore legacy Ollama character analysis flow 2026-07-10 11:01:31 +00:00
OpenClaw Agent
64a7b3d371 Auto-select Ollama vision model for analysis 2026-07-10 10:56:00 +00:00
OpenClaw Agent
6551b4f6c3 Add JSON fallback for Ollama character analysis 2026-07-10 10:46:04 +00:00
OpenClaw Agent
33ba18cac1 Sanitize leaked reasoning in character analysis 2026-07-10 10:41:16 +00:00
OpenClaw Agent
f88a1c7da6 Reduce Ollama analysis image context 2026-07-10 10:26:35 +00:00
OpenClaw Agent
c5a4aecdd7 Retry truncated Ollama character analysis 2026-07-10 10:22:47 +00:00
OpenClaw Agent
f0cd0b382d Harden Ollama MSR analysis parsing 2026-07-10 10:17:37 +00:00
OpenClaw Agent
c2f39a2f0e Stabilize MSR character form syncing 2026-07-10 10:10:56 +00:00
OpenClaw Agent
9b62c51a55 Add synced MSR character DOM fields 2026-07-10 10:04:35 +00:00
OpenClaw Agent
9a6f01ed67 Sync MSR character description widget after analyze 2026-07-10 09:58:55 +00:00
OpenClaw Agent
bd56a21b7f Rename visible node labels to DUMAS 2026-07-10 09:49:50 +00:00
OpenClaw Agent
67696a4e59 Remove legacy director MSR character path 2026-07-10 09:46:36 +00:00
OpenClaw Agent
e68357f384 Extract MSR characters into reusable nodes 2026-07-10 09:23:01 +00:00
OpenClaw Agent
119692160a perf: reduce editor redraw sorting overhead 2026-07-09 21:10:06 +00:00
OpenClaw Agent
7bb54f61d6 perf: cache render selection and segment state 2026-07-09 21:05:50 +00:00
OpenClaw Agent
264fdcf7cd perf: trim editor render selection lookups 2026-07-09 21:02:30 +00:00
OpenClaw Agent
e87b5d8d8f perf: reuse guide images and drag selections 2026-07-09 20:44:38 +00:00
OpenClaw Agent
50664026a1 perf: reduce editor drag lookup overhead 2026-07-09 20:34:40 +00:00
OpenClaw Agent
39af2b0150 perf: tighten director tag and segment helpers 2026-07-09 20:03:37 +00:00
OpenClaw Agent
58469ccb4b perf: cache director audio sources and trim timeline scans 2026-07-09 19:58:15 +00:00
OpenClaw Agent
b9c935466f perf: reduce prompt relay and audio ui overhead 2026-07-09 19:48:24 +00:00
OpenClaw Agent
ab7d14bd89 perf: streamline prompt relay and guide ui 2026-07-09 15:51:58 +00:00
OpenClaw Agent
a536ac5a14 perf: refactor director guide execution paths 2026-07-09 15:30:50 +00:00
OpenClaw Agent
7099b3c6f1 refactor: reuse director serialization helpers 2026-07-09 15:23:17 +00:00
OpenClaw Agent
7e569d5deb refactor: simplify director video sync paths 2026-07-09 15:17:24 +00:00
OpenClaw Agent
b9d68aea62 refactor: extract director playback helpers 2026-07-09 15:13:38 +00:00
OpenClaw Agent
268e543210 refactor: reuse director file-key segment lookup 2026-07-09 15:09:50 +00:00
OpenClaw Agent
a1a4be4fc3 refactor: dedupe director audio loading helpers 2026-07-09 14:17:30 +00:00
OpenClaw Agent
5df6ed6669 refactor: extract director branch and audio helpers 2026-07-09 13:59:25 +00:00
OpenClaw Agent
04ff3fc38a refactor: split director execute helpers 2026-07-09 13:53:10 +00:00
OpenClaw Agent
51bd3b4968 perf: cache reused guide and audio work 2026-07-09 13:44:23 +00:00
OpenClaw Agent
b8cedad997 perf: reduce redundant media work in director 2026-07-09 13:29:55 +00:00
CGlide
6695b1a1aa workflow CS 2026-06-28 00:27:01 +02:00
CGlide
b6a1e373d3 Update README.md 2026-06-27 23:54:46 +02:00
13 changed files with 4116 additions and 2644 deletions

View File

@@ -58,13 +58,12 @@ This node uses Ollama Locally : you will need to install it (very lightweight) a
- You can still enter your description manually if you don't want to install it but it works very well!
- Both references mode work (fixed), Licon MSR and Ghost Mask.
- Reference mode included: Licon MSR.
- Use their Lora, that is an important step : https://huggingface.co/LiconStudio/LTX-2.3-Multiple-Subject-Reference/tree/main
- Enjoy!
<img width="596" height="668" alt="Capture d&#39;écran 2026-06-13 234036" src="https://github.com/user-attachments/assets/2dfe7992-a72a-4eda-a103-41b3a8f77714" />
# 🔄 Recent Updates

View File

@@ -1,5 +1,7 @@
from .ltx_director import LTXDirector
from .ltx_director_guide import LTXDirectorGuide, LTXDirectorCropGuides
from .msr_character import MSRCharacter, MSRCharacterSet
from .timeline_nodes import TimelineSection, TimelineSectionSet
from comfy_api.latest import ComfyExtension, io
from typing_extensions import override
from .latent_slice import CleanLatentSlice
@@ -8,6 +10,10 @@ class PromptRelay(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
MSRCharacter,
MSRCharacterSet,
TimelineSection,
TimelineSectionSet,
LTXDirector,
]
@@ -16,16 +22,24 @@ async def comfy_entrypoint() -> PromptRelay:
NODE_CLASS_MAPPINGS = {
"LTXDirectorCS": LTXDirector,
"MSRCharacterCS": MSRCharacter,
"MSRCharacterSetCS": MSRCharacterSet,
"TimelineSectionCS": TimelineSection,
"TimelineSectionSetCS": TimelineSectionSet,
"LTXDirectorGuideCS": LTXDirectorGuide,
"LTXDirectorCropGuidesCS": LTXDirectorCropGuides,
"CleanLatentSliceCS": CleanLatentSlice,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"LTXDirectorCS": "LTX Director CS",
"LTXDirectorGuideCS": "LTX Director Guide CS",
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides CS",
"CleanLatentSliceCS": "Clean Latent Slice CS",
"LTXDirectorCS": "LTX Director DUMAS",
"MSRCharacterCS": "MSR Character DUMAS",
"MSRCharacterSetCS": "MSR Character Set DUMAS",
"TimelineSectionCS": "Timeline Section DUMAS",
"TimelineSectionSetCS": "Timeline Section Set DUMAS",
"LTXDirectorGuideCS": "LTX Director Guide DUMAS",
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides DUMAS",
"CleanLatentSliceCS": "Clean Latent Slice DUMAS",
}
WEB_DIRECTORY = "./js"

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@@ -1,24 +1,30 @@
import { app } from "../../scripts/app.js";
// LTX Director Guide is a pure pass-through processor node.
// All configuration (images, insert frames, strengths) comes from
// the guide_data output of Prompt Relay Encode (Timeline).
app.registerExtension({
name: "Comfy.LTXDirectorGuideCS",
async nodeCreated(node) {
if (node.comfyClass !== "LTXDirectorGuideCS") return;
// Hide retake_mode widget on LiteGraph as it is dynamically auto-detected from the timeline data.
const w = node.widgets?.find(x => x.name === "retake_mode");
if (w) {
w.hidden = true;
if (!w.options) w.options = {};
w.options.hidden = true;
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
w.computeSize = () => [0, -4];
w.draw = () => { };
}
if (w.element) w.element.style.display = "none";
}
},
});
import { app } from "../../scripts/app.js";
// LTX Director Guide is a pure pass-through processor node.
// All configuration (images, insert frames, strengths) comes from
// the guide_data output of Prompt Relay Encode (Timeline).
function hideWidget(node, widgetName) {
const widget = node.widgets?.find((entry) => entry.name === widgetName);
if (!widget) return;
widget.hidden = true;
widget.options = widget.options || {};
widget.options.hidden = true;
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
widget.computeSize = () => [0, -4];
widget.draw = () => {};
}
if (widget.element) widget.element.style.display = "none";
}
app.registerExtension({
name: "Comfy.LTXDirectorGuideCS",
async nodeCreated(node) {
if (node.comfyClass !== "LTXDirectorGuideCS") return;
// Hide retake_mode widget on LiteGraph as it is dynamically auto-detected from the timeline data.
hideWidget(node, "retake_mode");
},
});

619
js/msr_character.js Normal file
View File

@@ -0,0 +1,619 @@
const { app } = window.comfyAPI.app;
const { api } = window.comfyAPI.api;
const DEFAULT_ANALYZE_PROMPT = "Describe the character's physical appearance in two concise sentences. Specify their hair color/style, face details, and their clothing type/color. Keep the entire response very brief.";
function findWidget(node, name) {
return (node.widgets || []).find((widget) => widget.name === name);
}
function findInput(node, name) {
return (node.inputs || []).find((input) => input.name === name);
}
function getOriginNodeForInput(node, inputName) {
const input = findInput(node, inputName);
if (!input || input.link == null) return null;
const link = app.graph.links[input.link];
if (!link) return null;
return app.graph.getNodeById(link.origin_id);
}
function coercePreviewUrl(value) {
if (!value) return "";
if (typeof value === "string") return value;
if (typeof value === "object") {
if (typeof value.src === "string") return value.src;
if (typeof value.url === "string") return value.url;
}
return "";
}
function resolveFileWidgetUrlFromOrigin(originNode) {
if (!originNode) return "";
const widgets = originNode.widgets || [];
const fileWidget = widgets.find((widget) => ["image", "filename", "file"].includes(widget.name));
const subfolderWidget = widgets.find((widget) => widget.name === "subfolder");
const filename = typeof fileWidget?.value === "string" ? fileWidget.value : "";
const subfolder = typeof subfolderWidget?.value === "string" ? subfolderWidget.value : "";
if (!filename) return "";
return api.apiURL(
`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`
);
}
function resolvePreviewUrlFromOrigin(originNode) {
if (!originNode) return "";
const preview = Array.isArray(originNode.imgs) ? coercePreviewUrl(originNode.imgs[0]) : "";
return preview || "";
}
function resolveCandidateImageUrls(originNode) {
const urls = [];
const fileUrl = resolveFileWidgetUrlFromOrigin(originNode);
const previewUrl = resolvePreviewUrlFromOrigin(originNode);
if (fileUrl) urls.push(fileUrl);
if (previewUrl && previewUrl !== fileUrl) urls.push(previewUrl);
return urls;
}
async function blobToOptimizedDataUrl(blob, maxDim = 1024, quality = 0.82) {
const objectUrl = URL.createObjectURL(blob);
try {
const image = await new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = () => reject(new Error("Failed to decode image blob"));
img.src = objectUrl;
});
const width = image.naturalWidth || image.width || 0;
const height = image.naturalHeight || image.height || 0;
const scale = width > 0 && height > 0 ? Math.min(1, maxDim / Math.max(width, height)) : 1;
const targetWidth = Math.max(1, Math.round(width * scale)) || width || 1;
const targetHeight = Math.max(1, Math.round(height * scale)) || height || 1;
const canvas = document.createElement("canvas");
canvas.width = targetWidth;
canvas.height = targetHeight;
const ctx = canvas.getContext("2d");
if (!ctx) {
throw new Error("Could not acquire canvas context");
}
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
const optimizedBlob = await new Promise((resolve, reject) => {
canvas.toBlob((result) => {
if (result) resolve(result);
else reject(new Error("Canvas toBlob failed"));
}, "image/jpeg", quality);
});
const dataUrl = await new Promise((resolve) => {
const reader = new FileReader();
reader.onloadend = () => resolve(reader.result);
reader.readAsDataURL(optimizedBlob);
});
return {
dataUrl,
originalWidth: width,
originalHeight: height,
outputWidth: targetWidth,
outputHeight: targetHeight,
outputBytes: optimizedBlob.size || 0,
};
} finally {
URL.revokeObjectURL(objectUrl);
}
}
async function imageInputToDataUrl(node, inputName) {
const originNode = getOriginNodeForInput(node, inputName);
const candidateUrls = resolveCandidateImageUrls(originNode);
if (!candidateUrls.length) {
return {
dataUrl: null,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls: [],
selectedUrl: null,
mimeType: null,
blobBytes: 0,
dataUrlLength: 0,
originalWidth: 0,
originalHeight: 0,
outputWidth: 0,
outputHeight: 0,
outputBytes: 0,
error: "No candidate image URLs found",
},
};
}
for (const imageUrl of candidateUrls) {
try {
const response = await fetch(imageUrl);
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const blob = await response.blob();
if (!blob.type.startsWith("image/")) {
throw new Error(`Unexpected blob type: ${blob.type || "unknown"}`);
}
const optimized = await blobToOptimizedDataUrl(blob);
const dataUrl = optimized.dataUrl;
return {
dataUrl,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls,
selectedUrl: imageUrl,
mimeType: blob.type || null,
blobBytes: blob.size || 0,
dataUrlLength: typeof dataUrl === "string" ? dataUrl.length : 0,
originalWidth: optimized.originalWidth,
originalHeight: optimized.originalHeight,
outputWidth: optimized.outputWidth,
outputHeight: optimized.outputHeight,
outputBytes: optimized.outputBytes,
error: null,
},
};
} catch (error) {
console.warn("[MSRCharacter] Failed candidate image source", imageUrl, error);
}
}
return {
dataUrl: null,
debug: {
inputName,
originNodeId: originNode?.id ?? null,
originNodeType: originNode?.type ?? null,
candidateUrls,
selectedUrl: null,
mimeType: null,
blobBytes: 0,
dataUrlLength: 0,
originalWidth: 0,
originalHeight: 0,
outputWidth: 0,
outputHeight: 0,
outputBytes: 0,
error: "All candidate image URLs failed",
},
};
}
function setWidgetValue(node, widget, value) {
if (!widget) return;
const previous = widget.value;
widget.value = value;
if (widget.element) {
if (widget.element.type === "checkbox") {
widget.element.checked = !!value;
} else {
widget.element.value = value;
}
}
if (widget.callback) {
try {
widget.callback(value);
} catch (error) {}
}
if (node.onWidgetChanged) {
try {
node.onWidgetChanged(widget.name, value, previous, widget);
} catch (error) {}
}
if (node.properties) {
node.properties[widget.name] = value;
}
}
function syncFormFromWidgets(node) {
const description = findWidget(node, "description")?.value || "";
const alias = findWidget(node, "alias")?.value || "";
const provider = findWidget(node, "analyze_provider")?.value || "ollama";
const model = findWidget(node, "analyze_model")?.value || "";
const baseUrl = findWidget(node, "analyze_base_url")?.value || "";
if (node._msrAliasInput && node._msrAliasInput.value !== alias) {
node._msrAliasInput.value = alias;
}
if (node._msrDescriptionInput && node._msrDescriptionInput.value !== description) {
node._msrDescriptionInput.value = description;
}
if (node._msrMeta) {
node._msrMeta.innerHTML = [
alias
? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>`
: "Alias: <span style=\"color:#666\">none</span>",
`<div style="margin-top:4px;color:#888">Analyze: ${provider}${model ? ` / ${model}` : ""}${baseUrl ? ` / ${baseUrl}` : ""}</div>`,
`<div style="margin-top:4px;color:#666">Description is stored in the field below.</div>`,
].join("");
}
}
function refreshCharacterPreview(node) {
if (!node._msrPreviewRow) return;
const inputNames = ["image_1", "image_2"];
const previewUrls = inputNames
.map((inputName) => {
const originNode = getOriginNodeForInput(node, inputName);
const candidates = resolveCandidateImageUrls(originNode);
return candidates[0] || "";
})
.filter(Boolean);
const previewRow = node._msrPreviewRow;
previewRow.innerHTML = "";
if (previewUrls.length) {
previewUrls.forEach((previewUrl) => {
const image = document.createElement("img");
image.src = previewUrl;
Object.assign(image.style, {
flex: "1",
minWidth: "0",
height: "74px",
objectFit: "cover",
borderRadius: "6px",
border: "1px solid #444",
background: "#111",
});
previewRow.appendChild(image);
});
} else {
const placeholder = document.createElement("div");
placeholder.textContent = "Connect up to 2 image nodes";
Object.assign(placeholder.style, {
width: "100%",
height: "74px",
display: "flex",
alignItems: "center",
justifyContent: "center",
color: "#777",
fontSize: "11px",
borderRadius: "6px",
border: "1px dashed #444",
background: "#181818",
});
previewRow.appendChild(placeholder);
}
syncFormFromWidgets(node);
}
function buildCharacterUi(node) {
const container = document.createElement("div");
Object.assign(container.style, {
boxSizing: "border-box",
width: "100%",
padding: "6px 2px 2px 2px",
});
const previewRow = document.createElement("div");
Object.assign(previewRow.style, {
display: "flex",
gap: "6px",
minHeight: "74px",
});
const meta = document.createElement("div");
Object.assign(meta.style, {
marginTop: "8px",
fontSize: "11px",
color: "#aaa",
lineHeight: "1.4",
});
const form = document.createElement("div");
Object.assign(form.style, {
marginTop: "10px",
display: "flex",
flexDirection: "column",
gap: "6px",
});
const actionsRow = document.createElement("div");
Object.assign(actionsRow.style, {
display: "flex",
gap: "8px",
alignItems: "center",
flexWrap: "wrap",
});
const analyzeButton = document.createElement("button");
analyzeButton.type = "button";
analyzeButton.textContent = "Analyze";
Object.assign(analyzeButton.style, {
alignSelf: "flex-start",
background: "#2b4f38",
color: "#f3f3f3",
border: "1px solid #496d56",
borderRadius: "6px",
padding: "6px 10px",
fontSize: "11px",
cursor: "pointer",
});
analyzeButton.addEventListener("click", () => {
const widgetButton = (node.widgets || []).find((widget) => widget.msrAnalyze);
if (!widgetButton) {
alert("Analyze button widget is missing on this node.");
return;
}
analyzeCharacterNode(node, widgetButton);
});
const settingsButton = document.createElement("button");
settingsButton.type = "button";
settingsButton.textContent = "Analyze Settings";
Object.assign(settingsButton.style, {
alignSelf: "flex-start",
background: "#252525",
color: "#ddd",
border: "1px solid #444",
borderRadius: "6px",
padding: "6px 10px",
fontSize: "11px",
cursor: "pointer",
});
settingsButton.addEventListener("click", () => {
const providerWidget = findWidget(node, "analyze_provider");
const baseUrlWidget = findWidget(node, "analyze_base_url");
const modelWidget = findWidget(node, "analyze_model");
const promptWidget = findWidget(node, "analyze_prompt");
const provider = window.prompt("Analyze provider: ollama, lmstudio, custom, off", providerWidget?.value || "ollama");
if (provider == null) return;
const baseUrl = window.prompt("Analyze base URL (blank = default)", baseUrlWidget?.value || "");
if (baseUrl == null) return;
const model = window.prompt("Analyze model (blank = provider default)", modelWidget?.value || "");
if (model == null) return;
const prompt = window.prompt("Analyze prompt (blank = default)", promptWidget?.value || DEFAULT_ANALYZE_PROMPT);
if (prompt == null) return;
setWidgetValue(node, providerWidget, provider.trim() || "ollama");
setWidgetValue(node, baseUrlWidget, baseUrl.trim());
setWidgetValue(node, modelWidget, model.trim());
setWidgetValue(node, promptWidget, prompt.trim());
syncFormFromWidgets(node);
node.setDirtyCanvas?.(true, true);
});
const aliasLabel = document.createElement("label");
aliasLabel.textContent = "Alias";
Object.assign(aliasLabel.style, {
fontSize: "11px",
color: "#bbb",
});
const aliasInput = document.createElement("input");
aliasInput.type = "text";
aliasInput.placeholder = "optional alias, e.g. bob";
Object.assign(aliasInput.style, {
width: "100%",
boxSizing: "border-box",
background: "#1c1c1c",
color: "#eee",
border: "1px solid #444",
borderRadius: "6px",
padding: "6px 8px",
fontSize: "12px",
});
aliasInput.addEventListener("input", () => {
setWidgetValue(node, findWidget(node, "alias"), aliasInput.value);
syncFormFromWidgets(node);
});
const descLabel = document.createElement("label");
descLabel.textContent = "Description";
Object.assign(descLabel.style, {
fontSize: "11px",
color: "#bbb",
});
const descInput = document.createElement("textarea");
descInput.placeholder = "Character description...";
Object.assign(descInput.style, {
width: "100%",
minHeight: "86px",
boxSizing: "border-box",
resize: "vertical",
background: "#1c1c1c",
color: "#eee",
border: "1px solid #444",
borderRadius: "6px",
padding: "8px",
fontSize: "12px",
lineHeight: "1.35",
});
descInput.addEventListener("input", () => {
setWidgetValue(node, findWidget(node, "description"), descInput.value);
syncFormFromWidgets(node);
});
actionsRow.appendChild(analyzeButton);
actionsRow.appendChild(settingsButton);
form.appendChild(aliasLabel);
form.appendChild(aliasInput);
form.appendChild(descLabel);
form.appendChild(descInput);
form.appendChild(actionsRow);
container.appendChild(previewRow);
container.appendChild(meta);
container.appendChild(form);
node._msrPreviewContainer = container;
node._msrPreviewRow = previewRow;
node._msrMeta = meta;
node._msrAliasInput = aliasInput;
node._msrDescriptionInput = descInput;
node._msrAnalyzeButton = analyzeButton;
return container;
}
async function analyzeCharacterNode(node, buttonWidget) {
const descriptionWidget = findWidget(node, "description");
if (!descriptionWidget) return;
const providerWidget = findWidget(node, "analyze_provider");
const baseUrlWidget = findWidget(node, "analyze_base_url");
const modelWidget = findWidget(node, "analyze_model");
const promptWidget = findWidget(node, "analyze_prompt");
buttonWidget.label = "Analyzing...";
if (node._msrAnalyzeButton) {
node._msrAnalyzeButton.textContent = "Analyzing...";
node._msrAnalyzeButton.disabled = true;
}
node.setDirtyCanvas?.(true, true);
try {
const imageResults = await Promise.all([
imageInputToDataUrl(node, "image_1"),
imageInputToDataUrl(node, "image_2"),
]);
const imageDebug = imageResults.map((result) => result?.debug || null).filter(Boolean);
const images = (
imageResults.map((result) => result?.dataUrl || null)
).filter(Boolean);
if (!images.length) {
throw new Error("Connect at least one image input before analyzing.");
}
const response = await api.fetchApi("/ltx_director/analyze_character", {
method: "POST",
body: JSON.stringify({
provider: providerWidget?.value || "ollama",
base_url: baseUrlWidget?.value || "",
model: modelWidget?.value || "",
prompt: promptWidget?.value || "",
image_b64: images,
image_debug: imageDebug,
}),
});
const result = await response.json();
if (result.status !== "success") {
console.warn("[MSRCharacter] analyze debug", {
sentImages: imageDebug,
response: result,
});
throw new Error(result.message || "Unknown analysis error");
}
const description = result.description || "";
setWidgetValue(node, descriptionWidget, description);
if (node._msrDescriptionInput) {
node._msrDescriptionInput.value = description;
}
syncFormFromWidgets(node);
refreshCharacterPreview(node);
} catch (error) {
console.error("[MSRCharacter] analyze failed", error);
alert(`MSR Character analyze failed: ${error.message || error}`);
} finally {
buttonWidget.label = "Analyze with Ollama";
if (node._msrAnalyzeButton) {
node._msrAnalyzeButton.textContent = "Analyze";
node._msrAnalyzeButton.disabled = false;
}
node.setDirtyCanvas?.(true, true);
}
}
function hideNodeWidget(widget) {
if (!widget) return;
widget.hidden = true;
if (!widget.options) widget.options = {};
widget.options.hidden = true;
if (widget.element) {
widget.element.style.display = "none";
}
}
app.registerExtension({
name: "Comfy.MSRCharacterCS",
async beforeRegisterNodeDef(nodeType, nodeData) {
if (nodeData.name === "MSRCharacterCS") {
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
if (!(this.widgets || []).find((widget) => widget.msrAnalyze)) {
this.addWidget("button", "Analyze with Ollama", null, () => {
analyzeCharacterNode(this, (this.widgets || []).find((widget) => widget.msrAnalyze));
}, { serialize: false });
const buttonWidget = this.widgets[this.widgets.length - 1];
buttonWidget.msrAnalyze = true;
buttonWidget.label = "Analyze with Ollama";
}
if (!findWidget(this, "analyze_provider")) {
this.addWidget("combo", "analyze_provider", "ollama", null, {
values: ["ollama", "lmstudio", "custom", "off"],
});
}
if (!findWidget(this, "analyze_base_url")) {
this.addWidget("text", "analyze_base_url", "");
}
if (!findWidget(this, "analyze_model")) {
this.addWidget("text", "analyze_model", "");
}
if (!findWidget(this, "analyze_prompt")) {
this.addWidget("text", "analyze_prompt", DEFAULT_ANALYZE_PROMPT);
}
if (!this._msrPreviewWidget) {
const previewContainer = buildCharacterUi(this);
this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, {
getValue: () => "",
setValue: () => {},
});
this._msrPreviewWidget.computeSize = () => [Math.max(10, (this.size?.[0] || 320) - 30), 260];
}
const originalConnectionsChange = this.onConnectionsChange;
this.onConnectionsChange = function () {
const result = originalConnectionsChange?.apply(this, arguments);
refreshCharacterPreview(this);
return result;
};
const originalConfigure = this.onConfigure;
this.onConfigure = function () {
const result = originalConfigure?.apply(this, arguments);
setTimeout(() => {
syncFormFromWidgets(this);
refreshCharacterPreview(this);
}, 0);
return result;
};
["alias", "description", "analyze_provider", "analyze_base_url", "analyze_model", "analyze_prompt"].forEach((widgetName) => {
hideNodeWidget(findWidget(this, widgetName));
});
hideNodeWidget((this.widgets || []).find((widget) => widget.msrAnalyze));
this.size[0] = Math.max(this.size[0] || 0, 330);
syncFormFromWidgets(this);
refreshCharacterPreview(this);
};
}
if (nodeData.name === "MSRCharacterSetCS") {
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
this.size[0] = Math.max(this.size[0] || 0, 260);
};
}
},
});

View File

@@ -1,7 +1,15 @@
import folder_paths
import os
import torch
import av
import folder_paths
import numpy as np
import os
import torch
import av
DEFAULT_SAMPLE_RATE = 44100
DEFAULT_SILENCE_SECONDS = 1
def build_silence(sample_rate: int = DEFAULT_SAMPLE_RATE, seconds: int = DEFAULT_SILENCE_SECONDS, channels: int = 2) -> torch.Tensor:
return torch.zeros((channels, int(sample_rate * seconds)), dtype=torch.float32)
def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
"""Convert audio to float 32 bits PCM format."""
@@ -13,30 +21,29 @@ def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
return wav.float() / (2 ** 31)
raise ValueError(f"Unsupported wav dtype: {wav.dtype}")
def load_audio_file(filepath: str) -> tuple[torch.Tensor, int]:
"""Uses the latest ComfyUI av-based decoding for maximum compatibility."""
with av.open(filepath) as af:
if not af.streams.audio:
raise ValueError("No audio stream found in the file.")
def load_audio_file(filepath: str) -> tuple[torch.Tensor, int]:
"""Uses the latest ComfyUI av-based decoding for maximum compatibility."""
with av.open(filepath) as af:
if not af.streams.audio:
raise ValueError("No audio stream found in the file.")
stream = af.streams.audio[0]
sr = stream.codec_context.sample_rate
n_channels = stream.channels
frames = []
for frame in af.decode(streams=stream.index):
buf = torch.from_numpy(frame.to_ndarray())
if buf.shape[0] != n_channels:
buf = buf.view(-1, n_channels).t()
frames.append(buf)
if not frames:
raise ValueError("No audio frames decoded.")
wav = torch.cat(frames, dim=1)
wav = f32_pcm(wav)
return wav, sr
frame_arrays = []
for frame in af.decode(streams=stream.index):
array = frame.to_ndarray()
if array.shape[0] != n_channels:
array = array.reshape(-1, n_channels).T
frame_arrays.append(array)
if not frame_arrays:
raise ValueError("No audio frames decoded.")
wav = torch.from_numpy(np.concatenate(frame_arrays, axis=1))
wav = f32_pcm(wav)
return wav, sr
class LoadAudioUI:
@@ -95,21 +102,20 @@ class LoadAudioUI:
# --- FALLBACK LOGIC ---
# If the file is 'none' or doesn't exist on disk, provide 1 second of silence
if audio == "none" or not audio_path or not os.path.exists(audio_path):
missing_info = audio if audio != "none" else "None selected"
print(f"!!! [LoadAudioUI] Warning: Audio file '{missing_info}' not found. Outputting 1 second of silence.")
sample_rate = 44100
# 1 second of silence (stereo) -> shape [channels, time]
waveform = torch.zeros((2, 44100))
else:
try:
waveform, sample_rate = load_audio_file(audio_path)
except Exception as e:
# If decoding fails for any reason, fallback to silence rather than crashing the workflow
print(f"!!! [LoadAudioUI] Error decoding {audio}: {e}. Falling back to silence.")
sample_rate = 44100
waveform = torch.zeros((2, 44100))
if audio == "none" or not audio_path or not os.path.exists(audio_path):
missing_info = audio if audio != "none" else "None selected"
print(f"!!! [LoadAudioUI] Warning: Audio file '{missing_info}' not found. Outputting 1 second of silence.")
sample_rate = DEFAULT_SAMPLE_RATE
waveform = build_silence(sample_rate)
else:
try:
waveform, sample_rate = load_audio_file(audio_path)
except Exception as e:
# If decoding fails for any reason, fallback to silence rather than crashing the workflow
print(f"!!! [LoadAudioUI] Error decoding {audio}: {e}. Falling back to silence.")
sample_rate = DEFAULT_SAMPLE_RATE
waveform = build_silence(sample_rate)
# Convert seconds to frames
start_frame = int(start_time * sample_rate)
@@ -138,4 +144,4 @@ class LoadAudioUI:
# Calculate the final trimmed duration in seconds as a float
final_duration = float(trimmed_waveform.shape[1] / sample_rate)
return (audio_output, final_duration)
return (audio_output, final_duration)

File diff suppressed because it is too large Load Diff

View File

@@ -9,11 +9,10 @@ import comfy.sd
import comfy.utils
import folder_paths
import node_helpers
from comfy_extras import nodes_lt
from comfy_api.latest import io
from .ltx_director import GuideData, MotionGuideData, _resize_image, _execute_comfy_node, _unpack
from comfy_extras import nodes_lt
from .ltx_director import _resize_image, _execute_comfy_node, _unpack
log = logging.getLogger(__name__)
log = logging.getLogger(__name__)
# --- Helper Functions from Nghtdrp ---
@@ -47,15 +46,27 @@ def _append_guide_attention_entry(
)
return _set_guide_attention_entries(conditioning, entries)
def _clone_noise_mask(latent, latent_image):
def _clone_noise_mask(latent, latent_image):
if "noise_mask" in latent and latent["noise_mask"] is not None:
return latent["noise_mask"].clone()
batch, _, frames, _, _ = latent_image.shape
return torch.ones(
(batch, 1, frames, 1, 1),
dtype=torch.float32,
device=latent_image.device,
)
return torch.ones(
(batch, 1, frames, 1, 1),
dtype=torch.float32,
device=latent_image.device,
)
def _safe_json_loads(raw_value, default=None):
import json
if isinstance(raw_value, dict):
return raw_value
if not raw_value:
return {} if default is None else default
try:
return json.loads(raw_value)
except Exception:
return {} if default is None else default
def _resize_latent_spatial(latent_image, noise_mask, width, height, method):
b, c, f, h, w = latent_image.shape
@@ -74,9 +85,68 @@ def _resize_latent_spatial(latent_image, noise_mask, width, height, method):
return latent_image, noise_mask
def _ceil_to_multiple(value, multiple):
multiple = max(1, int(multiple))
return int(math.ceil(value / multiple) * multiple)
def _ceil_to_multiple(value, multiple):
multiple = max(1, int(multiple))
return int(math.ceil(value / multiple) * multiple)
def _normalize_resize_method_for_encode(resize_method):
if resize_method == "maintain aspect ratio":
return "pad"
return resize_method
def _append_latent_frames(latent_image, noise_mask, extra_frames, mask_fill=1.0):
extra_frames = int(max(0, extra_frames))
if extra_frames == 0:
return latent_image, noise_mask
batch, channels, _, height, width = latent_image.shape
latent_tail = torch.zeros(
(batch, channels, extra_frames, height, width),
dtype=latent_image.dtype,
device=latent_image.device,
)
latent_image = torch.cat([latent_image, latent_tail], dim=2)
mask_batch, mask_channels, _, mask_height, mask_width = noise_mask.shape
mask_tail = torch.full(
(mask_batch, mask_channels, extra_frames, mask_height, mask_width),
float(mask_fill),
dtype=noise_mask.dtype,
device=noise_mask.device,
)
noise_mask = torch.cat([noise_mask, mask_tail], dim=2)
return latent_image, noise_mask
def _encode_resized_video_frames(
vae,
frames,
target_width,
target_height,
resize_method,
time_scale_factor,
use_tiled_encode,
tile_size,
tile_overlap,
):
pixels = _resize_image(
frames,
target_width,
target_height,
_normalize_resize_method_for_encode(resize_method),
divisible_by=1,
)
num_frames_to_keep = ((pixels.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
encode_src = pixels[:num_frames_to_keep, :, :, :3]
if use_tiled_encode:
encoded = vae.encode_tiled(
encode_src,
tile_x=tile_size,
tile_y=tile_size,
overlap=tile_overlap,
)
else:
encoded = vae.encode(encode_src)
return pixels, encoded
def _snap_latent_to_downscale(latent_image, noise_mask, downscale_factor, method):
factor = int(max(1, round(float(downscale_factor))))
@@ -272,7 +342,7 @@ def _load_motion_video_frames(video_file, trim_start_frames, length_frames, dire
# --- Main Class ---
class LTXDirectorGuide:
class LTXDirectorGuide:
@classmethod
def INPUT_TYPES(cls):
loras = folder_paths.get_filename_list("loras")
@@ -304,19 +374,421 @@ class LTXDirectorGuide:
}
RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT", "MODEL", "FLOAT")
RETURN_NAMES = ("positive", "negative", "latent", "model", "latent_downscale_factor")
FUNCTION = "execute"
@classmethod
def execute(cls, positive, negative, vae, latent, guide_data, motion_guide_data=None, model=None, ic_lora_name="None", ic_lora_strength=1.0, scale_by=1.0, upscale_method="bicubic", image_attention_strength=1.0, crop="center", auto_snap_ic_grid=True, use_tiled_encode=False, tile_size=256, tile_overlap=64, retake_mode=False, msr_strength=0.0):
motion_segments = (motion_guide_data or {}).get("segments", []) if motion_guide_data else []
image_guides_count = len(guide_data.get("images", [])) if guide_data else 0
print(f"[LTXDirectorGuide] execute started. motion_segments: {len(motion_segments)}, image_guides: {image_guides_count}, ic_lora_name: {ic_lora_name}, model connected: {model is not None}, retake_mode: {retake_mode}")
active_resize_method = guide_data.get("resize_method") if guide_data else None
if not active_resize_method:
active_resize_method = motion_guide_data.get("resize_method") if motion_guide_data else None
if not active_resize_method:
RETURN_NAMES = ("positive", "negative", "latent", "model", "latent_downscale_factor")
FUNCTION = "execute"
@classmethod
def _maybe_add_attention(cls, positive, negative, is_lora_active, tokens_added, guide_orig_shape, attention_strength):
if not is_lora_active:
return positive, negative
positive = _append_guide_attention_entry(
positive,
tokens_added,
guide_orig_shape,
attention_strength=attention_strength,
)
negative = _append_guide_attention_entry(
negative,
tokens_added,
guide_orig_shape,
attention_strength=attention_strength,
)
return positive, negative
@classmethod
def _apply_crop_count(cls, positive, negative, crop_frames):
crop_frames = max(0, int(crop_frames))
values = {"nghtdrp_guide_crop_latent_frames": crop_frames}
positive = node_helpers.conditioning_set_values(positive, values)
negative = node_helpers.conditioning_set_values(negative, values)
return positive, negative
@classmethod
def _load_cached_motion_video_frames(cls, video_cache, video_file, trim_start_frames, length_frames, director_fps, resample_mode):
cache_key = (
str(video_file),
int(trim_start_frames),
int(length_frames),
float(director_fps),
str(resample_mode),
)
cached = video_cache.get(cache_key)
if cached is None:
cached = _load_motion_video_frames(
video_file,
trim_start_frames=trim_start_frames,
length_frames=length_frames,
director_fps=director_fps,
resample_mode=resample_mode,
)
video_cache[cache_key] = cached
return cached
@classmethod
def _encode_image_guide(cls, vae, latent_width, latent_height, img_tensor, scale_factors, target_pix_w, target_pix_h, upscale_method):
_, height, width, _ = img_tensor.shape
if target_pix_w != width or target_pix_h != height:
img_nchw = img_tensor.permute(0, 3, 1, 2)
img_resized = comfy.utils.common_upscale(
img_nchw,
target_pix_w,
target_pix_h,
upscale_method,
"disabled",
)
img_tensor = img_resized.permute(0, 2, 3, 1)
return nodes_lt.LTXVAddGuide.encode(
vae,
latent_width,
latent_height,
img_tensor,
scale_factors,
)
@classmethod
def _apply_image_guides(
cls,
positive,
negative,
vae,
latent_image,
noise_mask,
images,
insert_frames,
strengths,
latent_width,
latent_height,
latent_length,
scale_factors,
image_attention_strength,
is_lora_active,
upscale_method,
):
target_pix_w = int(latent_width * 32)
target_pix_h = int(latent_height * 32)
for idx, img_tensor in enumerate(images):
frame_value = insert_frames[idx] if idx < len(insert_frames) else 0
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
if strength <= 0.0:
continue
image_pixels, guide_latent = cls._encode_image_guide(
vae,
latent_width,
latent_height,
img_tensor,
scale_factors,
target_pix_w,
target_pix_h,
upscale_method,
)
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
positive,
latent_length,
len(image_pixels),
int(frame_value),
scale_factors,
)
if latent_idx >= latent_length:
continue
max_frames = latent_length - latent_idx
if guide_latent.shape[2] > max_frames:
guide_latent = guide_latent[:, :, :max_frames]
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
guide_orig_shape = list(guide_latent.shape[2:])
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
positive,
negative,
frame_idx,
latent_image,
noise_mask,
guide_latent,
strength,
scale_factors,
)
positive, negative = cls._maybe_add_attention(
positive,
negative,
is_lora_active,
tokens_added,
guide_orig_shape,
image_attention_strength,
)
return positive, negative, latent_image, noise_mask
@classmethod
def _apply_motion_segment_guides(
cls,
positive,
negative,
vae,
latent_image,
noise_mask,
segments,
director_fps,
latent_length,
latent_width,
latent_height,
time_scale_factor,
scale_factors,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
active_resize_method,
is_lora_active,
video_cache,
):
for seg in segments:
try:
video_file = seg.get("videoFile")
if not video_file:
continue
start_frame = int(seg.get("start", 0))
length_frames = int(seg.get("length", 1))
trim_start = int(seg.get("trimStart", 0))
video_strength = float(seg.get("videoStrength", 1.0))
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
if length_frames <= 0 or video_strength <= 0.0:
continue
video_frames = cls._load_cached_motion_video_frames(
video_cache,
video_file,
trim_start,
length_frames,
director_fps,
seg.get("resampleMode", "nearest"),
)
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
video_frames = video_frames[:num_frames_to_keep]
causal_fix = start_frame == 0 or num_frames_to_keep == 1
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
_, guide_latent = _encode_video_iclora_guide(
vae,
latent_width,
latent_height,
encode_frames,
scale_factors,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
resize_method=active_resize_method,
)
if not causal_fix:
guide_latent = guide_latent[:, :, 1:, :, :]
frame_idx = start_frame
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
if latent_idx >= latent_length:
continue
if start_frame > 0 and guide_latent.shape[2] > 1:
guide_latent = guide_latent[:, :, 1:, :, :]
frame_idx += time_scale_factor
latent_idx += 1
if latent_idx >= latent_length:
continue
max_frames = latent_length - latent_idx
if guide_latent.shape[2] > max_frames:
guide_latent = guide_latent[:, :, :max_frames]
guide_orig_shape = list(guide_latent.shape[2:])
batch, _, frames, height, width = guide_latent.shape
guide_mask = torch.ones(
(batch, 1, frames, height, width),
device=guide_latent.device,
dtype=guide_latent.dtype,
)
if start_frame > 0:
ramp_steps = [0.25, 0.65]
for i, step in enumerate(ramp_steps):
if i < frames:
guide_mask[:, :, i, :, :] = 1.0 + video_strength * (1.0 - step)
ldf = int(max(1, round(float(latent_downscale_factor))))
if ldf > 1:
dilated = _dilate_latent(
{"samples": guide_latent, "noise_mask": guide_mask},
horizontal_scale=ldf,
vertical_scale=ldf,
)
guide_latent = dilated["samples"]
guide_mask = dilated["noise_mask"]
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
positive,
negative,
frame_idx,
latent_image,
noise_mask,
guide_latent,
video_strength,
scale_factors,
guide_mask=guide_mask,
latent_downscale_factor=float(latent_downscale_factor),
causal_fix=causal_fix,
)
positive, negative = cls._maybe_add_attention(
positive,
negative,
is_lora_active,
tokens_added,
guide_orig_shape,
video_attention_strength,
)
except Exception as e:
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
return positive, negative, latent_image, noise_mask
@classmethod
def _execute_retake_mode(
cls,
positive,
negative,
vae,
latent_image,
noise_mask,
guide_data,
tdata,
segments,
model,
latent_downscale_factor,
director_fps,
latent_length,
latent_width,
latent_height,
time_scale_factor,
active_resize_method,
use_tiled_encode,
tile_size,
tile_overlap,
is_empty_latent,
initial_latent_length,
video_cache,
):
print(
f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. "
f"is_empty_latent: {is_empty_latent}"
)
target_width = latent_width * 32
target_height = latent_height * 32
retake_start = int(tdata.get("retakeStart", 0))
retake_len = int(tdata.get("retakeLength", 0))
retake_strength = float(tdata.get("retakeStrength", 1.0))
start_frame = int(guide_data.get("start_frame", 0))
relative_start = max(0, retake_start - start_frame)
l_start = min(max(0, relative_start // time_scale_factor), latent_length)
l_end = min(
max(0, int(math.ceil((relative_start + retake_len) / time_scale_factor))),
latent_length,
)
need_base_video = is_empty_latent or l_start > 0 or l_end < latent_length
if not need_base_video:
print(
"[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. "
"Skipping base video loading and VAE encoding."
)
retake_vid_info = tdata.get("retakeVideo") or {}
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
if not video_file and not retake_vid_info and segments:
video_file = segments[0].get("videoFile", "")
if need_base_video and not video_file:
if retake_vid_info and not retake_vid_info.get("imageFile"):
raise ValueError(
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
)
raise ValueError(
"Retake Mode is active, but no base video has been selected on the timeline. "
"Please drag and drop or upload a base video on the timeline first."
)
if need_base_video and video_file:
try:
ltxv_length = (latent_length - 1) * time_scale_factor + 1
print(
f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} "
f"starting at frame {start_frame} for length {ltxv_length} at resolution "
f"{target_width}x{target_height}"
)
video_frames = cls._load_cached_motion_video_frames(
video_cache,
video_file,
start_frame,
ltxv_length,
director_fps,
"nearest",
)
_, base_latent = _encode_resized_video_frames(
vae,
video_frames,
target_width,
target_height,
active_resize_method,
time_scale_factor,
use_tiled_encode,
tile_size,
tile_overlap,
)
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
paste_len = min(base_latent.shape[2], latent_length)
if is_empty_latent:
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
else:
print(
f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved "
f"regions (0-{l_start} and {l_end}-{paste_len})"
)
if l_start > 0:
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
if l_end < paste_len:
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
except Exception as e:
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
noise_mask = torch.zeros_like(noise_mask)
if l_end > l_start:
noise_mask[:, :, l_start:l_end] = retake_strength
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
return (
positive,
negative,
{"samples": latent_image, "noise_mask": noise_mask},
model,
float(latent_downscale_factor),
)
@classmethod
def execute(cls, positive, negative, vae, latent, guide_data, motion_guide_data=None, model=None, ic_lora_name="None", ic_lora_strength=1.0, scale_by=1.0, upscale_method="bicubic", image_attention_strength=1.0, crop="center", auto_snap_ic_grid=True, use_tiled_encode=False, tile_size=256, tile_overlap=64, retake_mode=False, msr_strength=0.0):
motion_segments = (motion_guide_data or {}).get("segments", []) if motion_guide_data else []
image_guides_count = len(guide_data.get("images", [])) if guide_data else 0
print(f"[LTXDirectorGuide] execute started. motion_segments: {len(motion_segments)}, image_guides: {image_guides_count}, ic_lora_name: {ic_lora_name}, model connected: {model is not None}, retake_mode: {retake_mode}")
video_cache = {}
active_resize_method = guide_data.get("resize_method") if guide_data else None
if not active_resize_method:
active_resize_method = motion_guide_data.get("resize_method") if motion_guide_data else None
if not active_resize_method:
active_resize_method = "crop" if crop == "center" else "stretch to fit"
latent_downscale_factor = 1.0
@@ -336,26 +808,8 @@ class LTXDirectorGuide:
if auto_snap_ic_grid and model is not None and ic_lora_name != "None":
latent_image, noise_mask = _snap_latent_to_downscale(latent_image, noise_mask, latent_downscale_factor, upscale_method)
# Ghost Mask: the Director kept the latent clean (so it stays length-matched with the audio
# latent) and asked us to pre-extend it here by one hidden frame per reference. The tail
# sits past the clean region; the reference images (in guide_data["images"], anchored at
# (clean + i)) are written into it by the loop below, then dropped downstream by Clean
# Latent Slice (length = clean_latent_frames).
_ghost_pre_extend = int(guide_data.get("ghost_pre_extend", 0)) if guide_data else 0
if _ghost_pre_extend > 0:
gb, gc, gf, gh, gw = latent_image.shape
latent_image = torch.cat(
[latent_image, torch.zeros((gb, gc, _ghost_pre_extend, gh, gw), dtype=latent_image.dtype, device=latent_image.device)],
dim=2,
)
mb, mc, mf, mh, mw = noise_mask.shape
noise_mask = torch.cat(
[noise_mask, torch.ones((mb, mc, _ghost_pre_extend, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
dim=2,
)
_, _, latent_length, latent_height, latent_width = latent_image.shape
initial_latent_length = int(latent_length)
_, _, latent_length, latent_height, latent_width = latent_image.shape
initial_latent_length = int(latent_length)
# Licon MSR (Prefix) mode: the Director handed us raw references via guide_data["msr"].
# Inject them at THIS node's resolution (so a half-res Stage 1 and full-res Stage 2 each
@@ -364,16 +818,10 @@ class LTXDirectorGuide:
if msr is not None:
return cls._inject_msr(positive, negative, vae, latent_image, noise_mask, msr, model, latent_downscale_factor, msr_strength)
# Parse timeline JSON to see if retake mode is active in UI
import json
timeline_data_str = guide_data.get("timeline_data", "{}") if guide_data else "{}"
try:
tdata = json.loads(timeline_data_str)
except Exception:
tdata = {}
is_retake_active = bool(retake_mode) or tdata.get("retakeMode", False)
is_empty_latent = (latent_image.abs().max().item() < 1e-5)
tdata = _safe_json_loads(guide_data.get("timeline_data", "{}") if guide_data else "{}")
is_retake_active = bool(retake_mode) or tdata.get("retakeMode", False)
is_empty_latent = (latent_image.abs().max().item() < 1e-5)
# Director image guides and motion video segments
images = guide_data.get("images", []) if guide_data else []
@@ -391,112 +839,31 @@ class LTXDirectorGuide:
# Retake Mode Branch:
# Load single base video, encode continuously, apply temporal mask.
# -----------------------------------------------------------------------
if is_retake_active:
print(f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. is_empty_latent: {is_empty_latent}")
target_width = latent_width * 32
target_height = latent_height * 32
# Calculate retake region latent indices first so we know what to copy/paste
retake_start = int(tdata.get("retakeStart", 0))
retake_len = int(tdata.get("retakeLength", 0))
retake_strength = float(tdata.get("retakeStrength", 1.0))
start_frame = int(guide_data.get("start_frame", 0))
relative_start = max(0, retake_start - start_frame)
l_start = relative_start // time_scale_factor
l_end = int(math.ceil((relative_start + retake_len) / time_scale_factor))
l_start = min(l_start, latent_length)
l_end = min(l_end, latent_length)
# Stage 2 optimization: If the retake region covers the entire generation area,
# there are no preserved regions to copy over in Stage 2. We can bypass video loading/encoding.
need_base_video = True
if not is_empty_latent and l_start == 0 and l_end >= latent_length:
need_base_video = False
print("[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. Skipping base video loading and VAE encoding.")
# 1. Try to load and encode base video from timeline data
retake_vid_info = tdata.get("retakeVideo") or {}
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
# Fallback to first segment only if it exists and we have no retake video info at all (old workflows)
if not video_file and not retake_vid_info and len(segments) > 0:
video_file = segments[0].get("videoFile", "")
if need_base_video:
if not video_file:
if retake_vid_info and not retake_vid_info.get("imageFile"):
raise ValueError(
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
)
else:
raise ValueError(
"Retake Mode is active, but no base video has been selected on the timeline. "
"Please drag and drop or upload a base video on the timeline first."
)
if video_file and need_base_video:
try:
print(f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} starting at frame {start_frame} for length {ltxv_length} at resolution {target_width}x{target_height}")
video_frames = _load_motion_video_frames(
video_file, trim_start_frames=start_frame, length_frames=ltxv_length, director_fps=director_fps, resample_mode="nearest"
)
# Retake base video must match the exact target latent shape.
# "maintain aspect ratio" doesn't pad, which causes shape mismatch in VAE encode.
# So we fallback "maintain aspect ratio" to "pad" for retake base video.
retake_resize_method = active_resize_method
if retake_resize_method == "maintain aspect ratio":
retake_resize_method = "pad"
pixels = _resize_image(video_frames, target_width, target_height, retake_resize_method, divisible_by=1)
num_clip_frames = pixels.shape[0]
num_frames_to_keep = ((num_clip_frames - 1) // time_scale_factor) * time_scale_factor + 1
encode_src = pixels[:num_frames_to_keep, :, :, :3]
if use_tiled_encode:
base_latent = vae.encode_tiled(encode_src, tile_x=tile_size, tile_y=tile_size, overlap=tile_overlap)
else:
base_latent = vae.encode(encode_src)
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
# Copy to latent_image
paste_len = min(base_latent.shape[2], latent_length)
if is_empty_latent:
# Stage 1: Overwrite entire latent with base video VAE encode
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
else:
# Stage 2: Overwrite only preserved regions (before l_start and after l_end)
# leaving the generated retake region from Stage 1 untouched
print(f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved regions (0-{l_start} and {l_end}-{paste_len})")
if l_start > 0:
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
if l_end < paste_len:
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
except Exception as e:
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
# 2. Build the temporal noise mask (0.0 = frozen, 1.0 = regenerate)
noise_mask = torch.zeros_like(noise_mask) # Initialize fully frozen
l_start = min(l_start, latent_length)
l_end = min(l_end, latent_length)
if l_end > l_start:
noise_mask[:, :, l_start:l_end] = retake_strength
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
# In retake mode, skip normal mode processing entirely and return immediately!
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
if is_retake_active:
return cls._execute_retake_mode(
positive,
negative,
vae,
latent_image,
noise_mask,
guide_data,
tdata,
segments,
model,
latent_downscale_factor,
director_fps,
latent_length,
latent_width,
latent_height,
time_scale_factor,
active_resize_method,
use_tiled_encode,
tile_size,
tile_overlap,
is_empty_latent,
initial_latent_length,
video_cache,
)
# -----------------------------------------------------------------------
# Standard Timeline Keyframe Guidance:
@@ -504,126 +871,56 @@ class LTXDirectorGuide:
# to the latent stream using standard LTX-Video cross-attention conditioning.
# Registers guide attention entries if IC-LoRA is active.
# -----------------------------------------------------------------------
if len(images) > 0 or len(segments) > 0:
print(f"[LTXDirectorGuide] Using Appended Keyframe Guidance. is_lora_active: {is_lora_active}")
# A. Process Image Guides
for idx, img_tensor in enumerate(images):
f_idx = insert_frames[idx] if idx < len(insert_frames) else 0
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
if strength <= 0.0:
continue
B_img, H_img, W_img, C_img = img_tensor.shape
target_pix_w = int(latent_width * 32)
target_pix_h = int(latent_height * 32)
if target_pix_w != W_img or target_pix_h != H_img:
img_nchw = img_tensor.permute(0, 3, 1, 2)
img_resized = comfy.utils.common_upscale(img_nchw, target_pix_w, target_pix_h, upscale_method, "disabled")
img_tensor = img_resized.permute(0, 2, 3, 1)
image_pixels, guide_latent = nodes_lt.LTXVAddGuide.encode(vae, latent_width, latent_height, img_tensor, scale_factors)
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(positive, latent_length, len(image_pixels), int(f_idx), scale_factors)
if latent_idx >= latent_length:
continue
max_frames = latent_length - latent_idx
if guide_latent.shape[2] > max_frames:
guide_latent = guide_latent[:, :, :max_frames]
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
guide_orig_shape = list(guide_latent.shape[2:])
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, strength, scale_factors
)
if is_lora_active:
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
# B. Process Motion Video Segments
for seg in segments:
try:
video_file = seg.get("videoFile")
if not video_file:
continue
start_frame = int(seg.get("start", 0))
length_frames = int(seg.get("length", 1))
trim_start = int(seg.get("trimStart", 0))
video_strength = float(seg.get("videoStrength", 1.0))
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
if length_frames <= 0 or video_strength <= 0.0:
continue
start_frame_aligned = start_frame
video_frames = _load_motion_video_frames(video_file, trim_start, length_frames, director_fps, seg.get("resampleMode", "nearest"))
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
video_frames = video_frames[:num_frames_to_keep]
causal_fix = int(start_frame_aligned) == 0 or num_frames_to_keep == 1
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
_, guide_latent = _encode_video_iclora_guide(vae, latent_width, latent_height, encode_frames, scale_factors, latent_downscale_factor, crop, use_tiled_encode, tile_size, tile_overlap, resize_method=active_resize_method)
if not causal_fix:
guide_latent = guide_latent[:, :, 1:, :, :]
frame_idx = start_frame_aligned
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
if latent_idx >= latent_length:
continue
if start_frame > 0 and guide_latent.shape[2] > 1:
guide_latent = guide_latent[:, :, 1:, :, :]
frame_idx += time_scale_factor
latent_idx += 1
if latent_idx >= latent_length:
continue
max_frames = latent_length - latent_idx
if guide_latent.shape[2] > max_frames:
guide_latent = guide_latent[:, :, :max_frames]
guide_orig_shape = list(guide_latent.shape[2:])
B_g, C_g, F_g, H_g, W_g = guide_latent.shape
guide_mask = torch.ones((B_g, 1, F_g, H_g, W_g), device=guide_latent.device, dtype=guide_latent.dtype)
if start_frame > 0:
ramp_steps = [0.25, 0.65]
for i, s in enumerate(ramp_steps):
if i < F_g:
guide_mask_val = 1.0 + video_strength * (1.0 - s)
guide_mask[:, :, i, :, :] = guide_mask_val
ldf = int(max(1, round(float(latent_downscale_factor))))
if ldf > 1:
dilated = _dilate_latent({"samples": guide_latent, "noise_mask": guide_mask}, horizontal_scale=ldf, vertical_scale=ldf)
guide_mask = dilated["noise_mask"]
guide_latent = dilated["samples"]
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, video_strength, scale_factors, guide_mask=guide_mask, latent_downscale_factor=float(latent_downscale_factor), causal_fix=causal_fix
)
if is_lora_active:
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
except Exception as e:
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
else:
print("[LTXDirectorGuide] No timeline guides present. Passing through.")
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
if len(images) > 0 or len(segments) > 0:
print(f"[LTXDirectorGuide] Using Appended Keyframe Guidance. is_lora_active: {is_lora_active}")
positive, negative, latent_image, noise_mask = cls._apply_image_guides(
positive,
negative,
vae,
latent_image,
noise_mask,
images,
insert_frames,
strengths,
latent_width,
latent_height,
latent_length,
scale_factors,
image_attention_strength,
is_lora_active,
upscale_method,
)
positive, negative, latent_image, noise_mask = cls._apply_motion_segment_guides(
positive,
negative,
vae,
latent_image,
noise_mask,
segments,
director_fps,
latent_length,
latent_width,
latent_height,
time_scale_factor,
scale_factors,
latent_downscale_factor,
crop,
use_tiled_encode,
tile_size,
tile_overlap,
active_resize_method,
is_lora_active,
video_cache,
)
else:
print("[LTXDirectorGuide] No timeline guides present. Passing through.")
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
@classmethod
def _inject_msr(cls, positive, negative, vae, latent_image, noise_mask, msr, model, latent_downscale_factor, msr_strength=0.0):
@@ -651,20 +948,11 @@ class LTXDirectorGuide:
initial_latent_length = int(latent_image.shape[2])
# Pad the generated region so (pad + keyframes) == prefix_latents, so the downstream crop
# (which trims the slideshow's temporal footprint) lands on the true clean length.
pad_latents = max(0, prefix_latents - len(keyframes))
if pad_latents > 0:
B, C, F, H, W = latent_image.shape
latent_image = torch.cat(
[latent_image, torch.zeros((B, C, pad_latents, H, W), dtype=latent_image.dtype, device=latent_image.device)],
dim=2,
)
mb, mc, mf, mh, mw = noise_mask.shape
noise_mask = torch.cat(
[noise_mask, torch.ones((mb, mc, pad_latents, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
dim=2,
)
# Pad the generated region so (pad + keyframes) == prefix_latents, so the downstream crop
# (which trims the slideshow's temporal footprint) lands on the true clean length.
pad_latents = max(0, prefix_latents - len(keyframes))
if pad_latents > 0:
latent_image, noise_mask = _append_latent_frames(latent_image, noise_mask, pad_latents, mask_fill=1.0)
latent = {"samples": latent_image, "noise_mask": noise_mask}
@@ -764,4 +1052,4 @@ class LTXDirectorCropGuides:
NODE_CLASS_MAPPINGS = {
"LTXDirectorGuideCS": LTXDirectorGuide,
"LTXDirectorCropGuidesCS": LTXDirectorCropGuides,
}
}

126
msr_character.py Normal file
View File

@@ -0,0 +1,126 @@
from __future__ import annotations
from comfy_api.latest import io
MSRCharacterData = io.Custom("MSR_CHARACTER")
MSRCharacterSetData = io.Custom("MSR_CHARACTER_SET")
def _compact_alias(alias: str) -> str:
if not alias:
return ""
alias = alias.strip()
if alias.startswith("@"):
alias = alias[1:]
return alias.strip()
class MSRCharacter(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="MSRCharacterCS",
display_name="MSR Character DUMAS",
category="WhatDreamsCost DUMAS",
description=(
"Reusable MSR character payload. Accepts up to two reference images plus "
"a description and optional alias for @alias prompt replacement."
),
inputs=[
io.Image.Input(
"image_1",
optional=True,
tooltip="Optional first reference image for this character.",
),
io.Image.Input(
"image_2",
optional=True,
tooltip="Optional second reference image for this character.",
),
io.String.Input(
"description",
multiline=True,
default="",
optional=True,
tooltip="Manual or generated appearance description used for prompt replacement.",
),
io.String.Input(
"alias",
default="",
optional=True,
tooltip="Optional alias. Prompts can use @alias alongside @charN/@characterN.",
),
],
outputs=[
MSRCharacterData.Output(display_name="character"),
],
)
@classmethod
def execute(cls, image_1=None, image_2=None, description="", alias="") -> io.NodeOutput:
images = []
if image_1 is not None:
images.append(image_1)
if image_2 is not None:
images.append(image_2)
payload = {
"images": images,
"description": (description or "").strip(),
"alias": _compact_alias(alias or ""),
}
return io.NodeOutput(payload)
class MSRCharacterSet(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="MSRCharacterSetCS",
display_name="MSR Character Set DUMAS",
category="WhatDreamsCost DUMAS",
description="Bundles up to six MSR Character nodes into one character set for the director.",
inputs=[
MSRCharacterData.Input("character_1", optional=True),
MSRCharacterData.Input("character_2", optional=True),
MSRCharacterData.Input("character_3", optional=True),
MSRCharacterData.Input("character_4", optional=True),
MSRCharacterData.Input("character_5", optional=True),
MSRCharacterData.Input("character_6", optional=True),
],
outputs=[
MSRCharacterSetData.Output(display_name="character_set"),
],
)
@classmethod
def execute(
cls,
character_1=None,
character_2=None,
character_3=None,
character_4=None,
character_5=None,
character_6=None,
) -> io.NodeOutput:
characters = []
for item in (
character_1,
character_2,
character_3,
character_4,
character_5,
character_6,
):
if not item:
continue
characters.append(
{
"images": list(item.get("images") or []),
"description": (item.get("description") or "").strip(),
"alias": _compact_alias(item.get("alias") or ""),
}
)
return io.NodeOutput({"characters": characters})

View File

@@ -1,9 +1,12 @@
import logging
import types
import comfy.ldm.modules.attention
log = logging.getLogger(__name__)
import logging
import os
import types
import comfy.ldm.modules.attention
log = logging.getLogger(__name__)
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs):
@@ -72,14 +75,14 @@ def _make_masked_override(prev_override):
return override
def debug_log(msg):
try:
import os
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
with open(log_path, "a", encoding="utf-8") as f:
f.write(msg + "\n")
except Exception:
pass
def debug_log(msg):
if not _DEBUG_PROMPT_RELAY:
return
try:
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
f.write(msg + "\n")
except Exception:
pass
def _make_ltx_mask_wrapper(underlying, mask_fn, attr):

View File

@@ -1,71 +1,87 @@
import logging
import math
import torch
log = logging.getLogger(__name__)
import logging
import math
import os
import torch
log = logging.getLogger(__name__)
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
def _build_segment_cost(query_frames, seg, dtype):
local = seg["local_token_idx"]
distance = (query_frames[:, None] - seg["midpoint"]).abs()
cost = seg["strength"] * (torch.relu(distance - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
return local, cost.to(dtype)
def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame):
"""Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing)."""
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
query_frames = (torch.arange(Lq, device=device, dtype=torch.float32) / float(tokens_per_frame)).floor()
for seg in q_token_idx:
local, cost = _build_segment_cost(query_frames, seg, offset.dtype)
offset[:, local.to(device=device)] = cost
return offset
def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames, is_audio=False):
"""Penalty matrix for queries that don't map to integer frames (e.g. LTXAV audio tokens)."""
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq
for seg in q_token_idx:
if is_audio:
active_seg = {
"local_token_idx": seg["local_token_idx"],
"midpoint": seg["midpoint"],
"window": seg.get("window_audio", seg["window"]),
"sigma": seg.get("sigma_audio", seg["sigma"]),
"strength": seg.get("strength_audio", 1.0),
}
else:
active_seg = seg
local, cost = _build_segment_cost(query_frames, active_seg, offset.dtype)
offset[:, local.to(device=device)] = cost
return offset
def debug_log(msg):
if not _DEBUG_PROMPT_RELAY:
return
try:
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
f.write(msg + "\n")
except Exception:
pass
def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame):
"""Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing)."""
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
query_frames = torch.arange(Lq, device=device, dtype=torch.long) // tokens_per_frame
for seg in q_token_idx:
local = seg["local_token_idx"].to(device=device)
d = (query_frames.float()[:, None] - seg["midpoint"]).abs()
strength = seg.get("strength", 1.0)
cost = strength * (torch.relu(d - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
offset[:, local] = cost.to(offset.dtype)
return offset
def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames, is_audio=False):
"""Penalty matrix for queries that don't map to integer frames (e.g. LTXAV audio tokens)."""
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq
for seg in q_token_idx:
local = seg["local_token_idx"].to(device=device)
d = (query_frames[:, None] - seg["midpoint"]).abs()
if is_audio:
sigma_val = seg.get("sigma_audio", seg["sigma"])
window_val = seg.get("window_audio", seg["window"])
strength_val = seg.get("strength_audio", 1.0)
else:
sigma_val = seg["sigma"]
window_val = seg["window"]
strength_val = seg.get("strength", 1.0)
cost = strength_val * (torch.relu(d - window_val) ** 2) / (2 * sigma_val ** 2)
offset[:, local] = cost.to(offset.dtype)
return offset
def debug_log(msg):
try:
import os
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
with open(log_path, "a", encoding="utf-8") as f:
f.write(msg + "\n")
except Exception:
pass
def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
"""Closure: mask_fn(Lq, Lk, dtype, device, transformer_options) -> additive mask or None.
Takes shapes/dtype/device instead of tensors so callers can compute the mask
without first materializing q/k projections — required so PromptRelay can
wrap an existing cross-attn forward (e.g. KJNodes NAG) instead of replacing it.
"""
cache = {}
max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1
def mask_fn(Lq, Lk, dtype, device, transformer_options):
debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}")
if Lq == Lk:
debug_log("mask_fn: Lq == Lk, returning None")
without first materializing q/k projections — required so PromptRelay can
wrap an existing cross-attn forward (e.g. KJNodes NAG) instead of replacing it.
"""
cache = {}
max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1
latent_frames = max(1, int(latent_frames))
fallback_tokens_per_frame = max(1, int(fallback_tokens_per_frame))
def _get_video_query_layout(Lq, grid_sizes):
if grid_sizes is not None:
return int(grid_sizes[1]) * int(grid_sizes[2])
if Lq % latent_frames == 0:
return Lq // latent_frames
return fallback_tokens_per_frame
def mask_fn(Lq, Lk, dtype, device, transformer_options):
debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}")
if Lq == Lk:
debug_log("mask_fn: Lq == Lk, returning None")
return None
# Only apply on conditional pass — not unconditional (negative prompt)
@@ -78,22 +94,16 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
attn_type = transformer_options.get("promptrelay_attn_type", "attn2")
is_audio = (attn_type == "audio_attn2")
if is_audio:
mode = "scaled"
video_lq = -1
else:
if grid_sizes is not None:
video_tpf = int(grid_sizes[1]) * int(grid_sizes[2])
else:
if Lq % latent_frames == 0:
video_tpf = Lq // latent_frames
else:
video_tpf = fallback_tokens_per_frame
video_lq = latent_frames * video_tpf
# Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq
if Lk == video_lq or Lk < max_token_idx:
debug_log(f"mask_fn: Lk == video_lq ({Lk == video_lq}) or Lk < max_token_idx ({Lk < max_token_idx}), returning None")
if is_audio:
mode = "scaled"
video_lq = -1
else:
video_tpf = _get_video_query_layout(Lq, grid_sizes)
video_lq = latent_frames * video_tpf
# Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq
if Lk == video_lq or Lk < max_token_idx:
debug_log(f"mask_fn: Lk == video_lq ({Lk == video_lq}) or Lk < max_token_idx ({Lk < max_token_idx}), returning None")
return None
mode = "video" if Lq == video_lq else "scaled"
@@ -113,7 +123,7 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
return cache[key].to(dtype)
return mask_fn
return mask_fn
def build_segments(token_ranges, segment_lengths, epsilon=1e-3, relay_options=None):
@@ -173,7 +183,7 @@ def build_segments(token_ranges, segment_lengths, epsilon=1e-3, relay_options=No
return q_token_idx
def get_raw_tokenizer(clip):
def get_raw_tokenizer(clip):
"""Extract the raw SPiece/HF tokenizer from a ComfyUI CLIP object."""
tokenizer_wrapper = clip.tokenizer
for attr_name in dir(tokenizer_wrapper):
@@ -183,56 +193,60 @@ def get_raw_tokenizer(clip):
if inner is not None and hasattr(inner, "tokenizer"):
return inner.tokenizer
raise RuntimeError(
f"Could not find raw tokenizer on CLIP object. "
f"Known attributes: {[a for a in dir(tokenizer_wrapper) if not a.startswith('_')]}"
)
def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
raise RuntimeError(
f"Could not find raw tokenizer on CLIP object. "
f"Known attributes: {[a for a in dir(tokenizer_wrapper) if not a.startswith('_')]}"
)
def _get_tokenizer_input_ids(raw_tokenizer, text):
tokenized = raw_tokenizer(text)
if isinstance(tokenized, dict) and "input_ids" in tokenized:
return tokenized["input_ids"]
if hasattr(tokenized, "input_ids"):
return tokenized.input_ids
if isinstance(tokenized, list):
return tokenized
return []
def _tokenized_length(raw_tokenizer, text, eos_adjustment):
return max(0, len(_get_tokenizer_input_ids(raw_tokenizer, text)) - eos_adjustment)
def _tokenizer_has_eos(raw_tokenizer):
if getattr(raw_tokenizer, "add_eos", False):
return True
try:
ids = _get_tokenizer_input_ids(raw_tokenizer, "test")
if not ids:
return False
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
return (eos_id is not None and ids[-1] == eos_id) or ids[-1] == 1
except Exception:
return False
def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
"""Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges).
Uses incremental tokenization to avoid SentencePiece context-dependency issues.
"""
prefixed_locals = [" " + lp for lp in local_prompts]
full_prompt = global_prompt + "".join(prefixed_locals)
# Detect if the tokenizer appends EOS dynamically
has_eos = getattr(raw_tokenizer, "add_eos", False)
if not has_eos:
try:
test_res = raw_tokenizer("test")
if isinstance(test_res, dict) and "input_ids" in test_res:
ids = test_res["input_ids"]
elif hasattr(test_res, "input_ids"):
ids = test_res.input_ids
elif isinstance(test_res, list):
ids = test_res
else:
ids = []
if ids:
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
if eos_id is not None and ids[-1] == eos_id:
has_eos = True
elif ids[-1] == 1:
has_eos = True
except Exception:
pass
eos_adj = 1 if has_eos else 0
prev_len = len(raw_tokenizer(global_prompt)["input_ids"]) - eos_adj
token_ranges = []
built = global_prompt
for plp in prefixed_locals:
built += plp
cur_len = len(raw_tokenizer(built)["input_ids"]) - eos_adj
if cur_len <= prev_len:
raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'")
token_ranges.append((prev_len, cur_len))
prev_len = cur_len
"""
prefixed_locals = [" " + lp for lp in local_prompts]
full_prompt = global_prompt + "".join(prefixed_locals)
eos_adj = 1 if _tokenizer_has_eos(raw_tokenizer) else 0
prev_len = _tokenized_length(raw_tokenizer, global_prompt, eos_adj)
token_ranges = []
built = global_prompt
for plp in prefixed_locals:
built += plp
cur_len = _tokenized_length(raw_tokenizer, built, eos_adj)
if cur_len <= prev_len:
raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'")
token_ranges.append((prev_len, cur_len))
prev_len = cur_len
return full_prompt, token_ranges

117
timeline_nodes.py Normal file
View File

@@ -0,0 +1,117 @@
from __future__ import annotations
from comfy_api.latest import io
TimelineSectionData = io.Custom("TIMELINE_SECTION")
TimelineSectionSetData = io.Custom("TIMELINE_SECTION_SET")
class TimelineSection(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="TimelineSectionCS",
display_name="Timeline Section DUMAS",
category="WhatDreamsCost DUMAS",
description="Reusable timeline section with required text, optional image, and a duration in seconds.",
inputs=[
io.Image.Input(
"image",
optional=True,
tooltip="Optional image for this timeline section. When omitted, the section becomes a text-only block.",
),
io.String.Input(
"text",
multiline=True,
default="",
tooltip="Required text/prompt for this timeline section.",
),
io.Float.Input(
"duration",
default=1.0,
min=0.1,
max=1000.0,
step=0.01,
tooltip="Section duration in seconds.",
),
],
outputs=[
TimelineSectionData.Output(display_name="section"),
],
)
@classmethod
def execute(cls, image=None, text="", duration=1.0) -> io.NodeOutput:
payload = {
"image": image,
"text": (text or "").strip(),
"duration": max(0.1, float(duration or 0.0)),
}
return io.NodeOutput(payload)
class TimelineSectionSet(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="TimelineSectionSetCS",
display_name="Timeline Section Set DUMAS",
category="WhatDreamsCost DUMAS",
description="Bundles up to ten Timeline Section nodes into one reusable section set.",
inputs=[
TimelineSectionData.Input("section_1", optional=True),
TimelineSectionData.Input("section_2", optional=True),
TimelineSectionData.Input("section_3", optional=True),
TimelineSectionData.Input("section_4", optional=True),
TimelineSectionData.Input("section_5", optional=True),
TimelineSectionData.Input("section_6", optional=True),
TimelineSectionData.Input("section_7", optional=True),
TimelineSectionData.Input("section_8", optional=True),
TimelineSectionData.Input("section_9", optional=True),
TimelineSectionData.Input("section_10", optional=True),
],
outputs=[
TimelineSectionSetData.Output(display_name="section_set"),
],
)
@classmethod
def execute(
cls,
section_1=None,
section_2=None,
section_3=None,
section_4=None,
section_5=None,
section_6=None,
section_7=None,
section_8=None,
section_9=None,
section_10=None,
) -> io.NodeOutput:
sections = []
for item in (
section_1,
section_2,
section_3,
section_4,
section_5,
section_6,
section_7,
section_8,
section_9,
section_10,
):
if not item:
continue
text = (item.get("text") or "").strip()
if not text:
continue
sections.append(
{
"image": item.get("image"),
"text": text,
"duration": max(0.1, float(item.get("duration") or 0.0)),
}
)
return io.NodeOutput({"sections": sections})