Compare commits
69 Commits
main_cs
...
main_dumas
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
beb0892793 | ||
|
|
2f40aa77ad | ||
|
|
c2fea55704 | ||
|
|
c36f6f3bf6 | ||
|
|
ee3215c48f | ||
|
|
cfdee30dee | ||
|
|
9ebc488072 | ||
|
|
5347ca3dbc | ||
|
|
df4c936aae | ||
|
|
fa546cbc8f | ||
|
|
575838aeb0 | ||
|
|
a1a1184155 | ||
|
|
3f24406697 | ||
|
|
a711a9d34b | ||
|
|
00d5703137 | ||
|
|
64cbfd2f36 | ||
|
|
e113472c7d | ||
|
|
e01ceef71b | ||
|
|
8736d4a463 | ||
|
|
cc51b298fd | ||
|
|
1cfe7f1932 | ||
|
|
c782f3263b | ||
|
|
ab6415caa4 | ||
|
|
5c4349d9b9 | ||
|
|
7b4e8865d6 | ||
|
|
4c0358c11b | ||
|
|
4dfcb1e597 | ||
|
|
73918a28b3 | ||
|
|
48ac1c242f | ||
|
|
6bc0aa1f71 | ||
|
|
e2c5c38b14 | ||
|
|
f372a2ed69 | ||
|
|
2da1de2f6d | ||
|
|
b059974544 | ||
|
|
27f1d6a1e4 | ||
|
|
6ec28d33a4 | ||
|
|
2bd748c8a6 | ||
|
|
c20ded3319 | ||
|
|
64a7b3d371 | ||
|
|
6551b4f6c3 | ||
|
|
33ba18cac1 | ||
|
|
f88a1c7da6 | ||
|
|
c5a4aecdd7 | ||
|
|
f0cd0b382d | ||
|
|
c2f39a2f0e | ||
|
|
9b62c51a55 | ||
|
|
9a6f01ed67 | ||
|
|
bd56a21b7f | ||
|
|
67696a4e59 | ||
|
|
e68357f384 | ||
|
|
119692160a | ||
|
|
7bb54f61d6 | ||
|
|
264fdcf7cd | ||
|
|
e87b5d8d8f | ||
|
|
50664026a1 | ||
|
|
39af2b0150 | ||
|
|
58469ccb4b | ||
|
|
b9c935466f | ||
|
|
ab7d14bd89 | ||
|
|
a536ac5a14 | ||
|
|
7099b3c6f1 | ||
|
|
7e569d5deb | ||
|
|
b9d68aea62 | ||
|
|
268e543210 | ||
|
|
a1a4be4fc3 | ||
|
|
5df6ed6669 | ||
|
|
04ff3fc38a | ||
|
|
51bd3b4968 | ||
|
|
b8cedad997 |
@@ -58,7 +58,7 @@ This node uses Ollama Locally : you will need to install it (very lightweight) a
|
||||
|
||||
- You can still enter your description manually if you don't want to install it but it works very well!
|
||||
|
||||
- Both references mode work (fixed), Licon MSR and Ghost Mask.
|
||||
- Reference mode included: Licon MSR.
|
||||
|
||||
- Use their Lora, that is an important step : https://huggingface.co/LiconStudio/LTX-2.3-Multiple-Subject-Reference/tree/main
|
||||
|
||||
|
||||
22
__init__.py
22
__init__.py
@@ -1,5 +1,7 @@
|
||||
from .ltx_director import LTXDirector
|
||||
from .ltx_director_guide import LTXDirectorGuide, LTXDirectorCropGuides
|
||||
from .msr_character import MSRCharacter, MSRCharacterSet
|
||||
from .timeline_nodes import TimelineSection, TimelineSectionSet
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
from typing_extensions import override
|
||||
from .latent_slice import CleanLatentSlice
|
||||
@@ -8,6 +10,10 @@ class PromptRelay(ComfyExtension):
|
||||
@override
|
||||
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
||||
return [
|
||||
MSRCharacter,
|
||||
MSRCharacterSet,
|
||||
TimelineSection,
|
||||
TimelineSectionSet,
|
||||
LTXDirector,
|
||||
]
|
||||
|
||||
@@ -16,16 +22,24 @@ async def comfy_entrypoint() -> PromptRelay:
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"LTXDirectorCS": LTXDirector,
|
||||
"MSRCharacterCS": MSRCharacter,
|
||||
"MSRCharacterSetCS": MSRCharacterSet,
|
||||
"TimelineSectionCS": TimelineSection,
|
||||
"TimelineSectionSetCS": TimelineSectionSet,
|
||||
"LTXDirectorGuideCS": LTXDirectorGuide,
|
||||
"LTXDirectorCropGuidesCS": LTXDirectorCropGuides,
|
||||
"CleanLatentSliceCS": CleanLatentSlice,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"LTXDirectorCS": "LTX Director CS",
|
||||
"LTXDirectorGuideCS": "LTX Director Guide CS",
|
||||
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides CS",
|
||||
"CleanLatentSliceCS": "Clean Latent Slice CS",
|
||||
"LTXDirectorCS": "LTX Director DUMAS",
|
||||
"MSRCharacterCS": "MSR Character DUMAS",
|
||||
"MSRCharacterSetCS": "MSR Character Set DUMAS",
|
||||
"TimelineSectionCS": "Timeline Section DUMAS",
|
||||
"TimelineSectionSetCS": "Timeline Section Set DUMAS",
|
||||
"LTXDirectorGuideCS": "LTX Director Guide DUMAS",
|
||||
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides DUMAS",
|
||||
"CleanLatentSliceCS": "Clean Latent Slice DUMAS",
|
||||
}
|
||||
|
||||
WEB_DIRECTORY = "./js"
|
||||
|
||||
File diff suppressed because one or more lines are too long
1711
js/ltx_director.js
1711
js/ltx_director.js
File diff suppressed because it is too large
Load Diff
@@ -3,22 +3,28 @@ import { app } from "../../scripts/app.js";
|
||||
// LTX Director Guide is a pure pass-through processor node.
|
||||
// All configuration (images, insert frames, strengths) comes from
|
||||
// the guide_data output of Prompt Relay Encode (Timeline).
|
||||
function hideWidget(node, widgetName) {
|
||||
const widget = node.widgets?.find((entry) => entry.name === widgetName);
|
||||
if (!widget) return;
|
||||
|
||||
widget.hidden = true;
|
||||
widget.options = widget.options || {};
|
||||
widget.options.hidden = true;
|
||||
|
||||
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
|
||||
widget.computeSize = () => [0, -4];
|
||||
widget.draw = () => {};
|
||||
}
|
||||
|
||||
if (widget.element) widget.element.style.display = "none";
|
||||
}
|
||||
|
||||
app.registerExtension({
|
||||
name: "Comfy.LTXDirectorGuideCS",
|
||||
async nodeCreated(node) {
|
||||
if (node.comfyClass !== "LTXDirectorGuideCS") return;
|
||||
|
||||
// Hide retake_mode widget on LiteGraph as it is dynamically auto-detected from the timeline data.
|
||||
const w = node.widgets?.find(x => x.name === "retake_mode");
|
||||
if (w) {
|
||||
w.hidden = true;
|
||||
if (!w.options) w.options = {};
|
||||
w.options.hidden = true;
|
||||
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
|
||||
w.computeSize = () => [0, -4];
|
||||
w.draw = () => { };
|
||||
}
|
||||
if (w.element) w.element.style.display = "none";
|
||||
}
|
||||
hideWidget(node, "retake_mode");
|
||||
},
|
||||
});
|
||||
619
js/msr_character.js
Normal file
619
js/msr_character.js
Normal file
@@ -0,0 +1,619 @@
|
||||
const { app } = window.comfyAPI.app;
|
||||
const { api } = window.comfyAPI.api;
|
||||
const DEFAULT_ANALYZE_PROMPT = "Describe the character's physical appearance in two concise sentences. Specify their hair color/style, face details, and their clothing type/color. Keep the entire response very brief.";
|
||||
|
||||
function findWidget(node, name) {
|
||||
return (node.widgets || []).find((widget) => widget.name === name);
|
||||
}
|
||||
|
||||
function findInput(node, name) {
|
||||
return (node.inputs || []).find((input) => input.name === name);
|
||||
}
|
||||
|
||||
function getOriginNodeForInput(node, inputName) {
|
||||
const input = findInput(node, inputName);
|
||||
if (!input || input.link == null) return null;
|
||||
const link = app.graph.links[input.link];
|
||||
if (!link) return null;
|
||||
return app.graph.getNodeById(link.origin_id);
|
||||
}
|
||||
|
||||
function coercePreviewUrl(value) {
|
||||
if (!value) return "";
|
||||
if (typeof value === "string") return value;
|
||||
if (typeof value === "object") {
|
||||
if (typeof value.src === "string") return value.src;
|
||||
if (typeof value.url === "string") return value.url;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function resolveFileWidgetUrlFromOrigin(originNode) {
|
||||
if (!originNode) return "";
|
||||
|
||||
const widgets = originNode.widgets || [];
|
||||
const fileWidget = widgets.find((widget) => ["image", "filename", "file"].includes(widget.name));
|
||||
const subfolderWidget = widgets.find((widget) => widget.name === "subfolder");
|
||||
const filename = typeof fileWidget?.value === "string" ? fileWidget.value : "";
|
||||
const subfolder = typeof subfolderWidget?.value === "string" ? subfolderWidget.value : "";
|
||||
if (!filename) return "";
|
||||
|
||||
return api.apiURL(
|
||||
`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`
|
||||
);
|
||||
}
|
||||
|
||||
function resolvePreviewUrlFromOrigin(originNode) {
|
||||
if (!originNode) return "";
|
||||
const preview = Array.isArray(originNode.imgs) ? coercePreviewUrl(originNode.imgs[0]) : "";
|
||||
return preview || "";
|
||||
}
|
||||
|
||||
function resolveCandidateImageUrls(originNode) {
|
||||
const urls = [];
|
||||
const fileUrl = resolveFileWidgetUrlFromOrigin(originNode);
|
||||
const previewUrl = resolvePreviewUrlFromOrigin(originNode);
|
||||
if (fileUrl) urls.push(fileUrl);
|
||||
if (previewUrl && previewUrl !== fileUrl) urls.push(previewUrl);
|
||||
return urls;
|
||||
}
|
||||
|
||||
async function blobToOptimizedDataUrl(blob, maxDim = 1024, quality = 0.82) {
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
try {
|
||||
const image = await new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => resolve(img);
|
||||
img.onerror = () => reject(new Error("Failed to decode image blob"));
|
||||
img.src = objectUrl;
|
||||
});
|
||||
|
||||
const width = image.naturalWidth || image.width || 0;
|
||||
const height = image.naturalHeight || image.height || 0;
|
||||
const scale = width > 0 && height > 0 ? Math.min(1, maxDim / Math.max(width, height)) : 1;
|
||||
const targetWidth = Math.max(1, Math.round(width * scale)) || width || 1;
|
||||
const targetHeight = Math.max(1, Math.round(height * scale)) || height || 1;
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = targetWidth;
|
||||
canvas.height = targetHeight;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
throw new Error("Could not acquire canvas context");
|
||||
}
|
||||
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
|
||||
|
||||
const optimizedBlob = await new Promise((resolve, reject) => {
|
||||
canvas.toBlob((result) => {
|
||||
if (result) resolve(result);
|
||||
else reject(new Error("Canvas toBlob failed"));
|
||||
}, "image/jpeg", quality);
|
||||
});
|
||||
|
||||
const dataUrl = await new Promise((resolve) => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => resolve(reader.result);
|
||||
reader.readAsDataURL(optimizedBlob);
|
||||
});
|
||||
|
||||
return {
|
||||
dataUrl,
|
||||
originalWidth: width,
|
||||
originalHeight: height,
|
||||
outputWidth: targetWidth,
|
||||
outputHeight: targetHeight,
|
||||
outputBytes: optimizedBlob.size || 0,
|
||||
};
|
||||
} finally {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function imageInputToDataUrl(node, inputName) {
|
||||
const originNode = getOriginNodeForInput(node, inputName);
|
||||
const candidateUrls = resolveCandidateImageUrls(originNode);
|
||||
if (!candidateUrls.length) {
|
||||
return {
|
||||
dataUrl: null,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls: [],
|
||||
selectedUrl: null,
|
||||
mimeType: null,
|
||||
blobBytes: 0,
|
||||
dataUrlLength: 0,
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
outputWidth: 0,
|
||||
outputHeight: 0,
|
||||
outputBytes: 0,
|
||||
error: "No candidate image URLs found",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
for (const imageUrl of candidateUrls) {
|
||||
try {
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
const blob = await response.blob();
|
||||
if (!blob.type.startsWith("image/")) {
|
||||
throw new Error(`Unexpected blob type: ${blob.type || "unknown"}`);
|
||||
}
|
||||
const optimized = await blobToOptimizedDataUrl(blob);
|
||||
const dataUrl = optimized.dataUrl;
|
||||
return {
|
||||
dataUrl,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls,
|
||||
selectedUrl: imageUrl,
|
||||
mimeType: blob.type || null,
|
||||
blobBytes: blob.size || 0,
|
||||
dataUrlLength: typeof dataUrl === "string" ? dataUrl.length : 0,
|
||||
originalWidth: optimized.originalWidth,
|
||||
originalHeight: optimized.originalHeight,
|
||||
outputWidth: optimized.outputWidth,
|
||||
outputHeight: optimized.outputHeight,
|
||||
outputBytes: optimized.outputBytes,
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn("[MSRCharacter] Failed candidate image source", imageUrl, error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dataUrl: null,
|
||||
debug: {
|
||||
inputName,
|
||||
originNodeId: originNode?.id ?? null,
|
||||
originNodeType: originNode?.type ?? null,
|
||||
candidateUrls,
|
||||
selectedUrl: null,
|
||||
mimeType: null,
|
||||
blobBytes: 0,
|
||||
dataUrlLength: 0,
|
||||
originalWidth: 0,
|
||||
originalHeight: 0,
|
||||
outputWidth: 0,
|
||||
outputHeight: 0,
|
||||
outputBytes: 0,
|
||||
error: "All candidate image URLs failed",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function setWidgetValue(node, widget, value) {
|
||||
if (!widget) return;
|
||||
const previous = widget.value;
|
||||
widget.value = value;
|
||||
if (widget.element) {
|
||||
if (widget.element.type === "checkbox") {
|
||||
widget.element.checked = !!value;
|
||||
} else {
|
||||
widget.element.value = value;
|
||||
}
|
||||
}
|
||||
if (widget.callback) {
|
||||
try {
|
||||
widget.callback(value);
|
||||
} catch (error) {}
|
||||
}
|
||||
if (node.onWidgetChanged) {
|
||||
try {
|
||||
node.onWidgetChanged(widget.name, value, previous, widget);
|
||||
} catch (error) {}
|
||||
}
|
||||
if (node.properties) {
|
||||
node.properties[widget.name] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function syncFormFromWidgets(node) {
|
||||
const description = findWidget(node, "description")?.value || "";
|
||||
const alias = findWidget(node, "alias")?.value || "";
|
||||
const provider = findWidget(node, "analyze_provider")?.value || "ollama";
|
||||
const model = findWidget(node, "analyze_model")?.value || "";
|
||||
const baseUrl = findWidget(node, "analyze_base_url")?.value || "";
|
||||
|
||||
if (node._msrAliasInput && node._msrAliasInput.value !== alias) {
|
||||
node._msrAliasInput.value = alias;
|
||||
}
|
||||
if (node._msrDescriptionInput && node._msrDescriptionInput.value !== description) {
|
||||
node._msrDescriptionInput.value = description;
|
||||
}
|
||||
if (node._msrMeta) {
|
||||
node._msrMeta.innerHTML = [
|
||||
alias
|
||||
? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>`
|
||||
: "Alias: <span style=\"color:#666\">none</span>",
|
||||
`<div style="margin-top:4px;color:#888">Analyze: ${provider}${model ? ` / ${model}` : ""}${baseUrl ? ` / ${baseUrl}` : ""}</div>`,
|
||||
`<div style="margin-top:4px;color:#666">Description is stored in the field below.</div>`,
|
||||
].join("");
|
||||
}
|
||||
}
|
||||
|
||||
function refreshCharacterPreview(node) {
|
||||
if (!node._msrPreviewRow) return;
|
||||
|
||||
const inputNames = ["image_1", "image_2"];
|
||||
const previewUrls = inputNames
|
||||
.map((inputName) => {
|
||||
const originNode = getOriginNodeForInput(node, inputName);
|
||||
const candidates = resolveCandidateImageUrls(originNode);
|
||||
return candidates[0] || "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
const previewRow = node._msrPreviewRow;
|
||||
previewRow.innerHTML = "";
|
||||
|
||||
if (previewUrls.length) {
|
||||
previewUrls.forEach((previewUrl) => {
|
||||
const image = document.createElement("img");
|
||||
image.src = previewUrl;
|
||||
Object.assign(image.style, {
|
||||
flex: "1",
|
||||
minWidth: "0",
|
||||
height: "74px",
|
||||
objectFit: "cover",
|
||||
borderRadius: "6px",
|
||||
border: "1px solid #444",
|
||||
background: "#111",
|
||||
});
|
||||
previewRow.appendChild(image);
|
||||
});
|
||||
} else {
|
||||
const placeholder = document.createElement("div");
|
||||
placeholder.textContent = "Connect up to 2 image nodes";
|
||||
Object.assign(placeholder.style, {
|
||||
width: "100%",
|
||||
height: "74px",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
color: "#777",
|
||||
fontSize: "11px",
|
||||
borderRadius: "6px",
|
||||
border: "1px dashed #444",
|
||||
background: "#181818",
|
||||
});
|
||||
previewRow.appendChild(placeholder);
|
||||
}
|
||||
|
||||
syncFormFromWidgets(node);
|
||||
}
|
||||
|
||||
function buildCharacterUi(node) {
|
||||
const container = document.createElement("div");
|
||||
Object.assign(container.style, {
|
||||
boxSizing: "border-box",
|
||||
width: "100%",
|
||||
padding: "6px 2px 2px 2px",
|
||||
});
|
||||
|
||||
const previewRow = document.createElement("div");
|
||||
Object.assign(previewRow.style, {
|
||||
display: "flex",
|
||||
gap: "6px",
|
||||
minHeight: "74px",
|
||||
});
|
||||
|
||||
const meta = document.createElement("div");
|
||||
Object.assign(meta.style, {
|
||||
marginTop: "8px",
|
||||
fontSize: "11px",
|
||||
color: "#aaa",
|
||||
lineHeight: "1.4",
|
||||
});
|
||||
|
||||
const form = document.createElement("div");
|
||||
Object.assign(form.style, {
|
||||
marginTop: "10px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: "6px",
|
||||
});
|
||||
|
||||
const actionsRow = document.createElement("div");
|
||||
Object.assign(actionsRow.style, {
|
||||
display: "flex",
|
||||
gap: "8px",
|
||||
alignItems: "center",
|
||||
flexWrap: "wrap",
|
||||
});
|
||||
|
||||
const analyzeButton = document.createElement("button");
|
||||
analyzeButton.type = "button";
|
||||
analyzeButton.textContent = "Analyze";
|
||||
Object.assign(analyzeButton.style, {
|
||||
alignSelf: "flex-start",
|
||||
background: "#2b4f38",
|
||||
color: "#f3f3f3",
|
||||
border: "1px solid #496d56",
|
||||
borderRadius: "6px",
|
||||
padding: "6px 10px",
|
||||
fontSize: "11px",
|
||||
cursor: "pointer",
|
||||
});
|
||||
analyzeButton.addEventListener("click", () => {
|
||||
const widgetButton = (node.widgets || []).find((widget) => widget.msrAnalyze);
|
||||
if (!widgetButton) {
|
||||
alert("Analyze button widget is missing on this node.");
|
||||
return;
|
||||
}
|
||||
analyzeCharacterNode(node, widgetButton);
|
||||
});
|
||||
|
||||
const settingsButton = document.createElement("button");
|
||||
settingsButton.type = "button";
|
||||
settingsButton.textContent = "Analyze Settings";
|
||||
Object.assign(settingsButton.style, {
|
||||
alignSelf: "flex-start",
|
||||
background: "#252525",
|
||||
color: "#ddd",
|
||||
border: "1px solid #444",
|
||||
borderRadius: "6px",
|
||||
padding: "6px 10px",
|
||||
fontSize: "11px",
|
||||
cursor: "pointer",
|
||||
});
|
||||
settingsButton.addEventListener("click", () => {
|
||||
const providerWidget = findWidget(node, "analyze_provider");
|
||||
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||
const modelWidget = findWidget(node, "analyze_model");
|
||||
const promptWidget = findWidget(node, "analyze_prompt");
|
||||
const provider = window.prompt("Analyze provider: ollama, lmstudio, custom, off", providerWidget?.value || "ollama");
|
||||
if (provider == null) return;
|
||||
const baseUrl = window.prompt("Analyze base URL (blank = default)", baseUrlWidget?.value || "");
|
||||
if (baseUrl == null) return;
|
||||
const model = window.prompt("Analyze model (blank = provider default)", modelWidget?.value || "");
|
||||
if (model == null) return;
|
||||
const prompt = window.prompt("Analyze prompt (blank = default)", promptWidget?.value || DEFAULT_ANALYZE_PROMPT);
|
||||
if (prompt == null) return;
|
||||
setWidgetValue(node, providerWidget, provider.trim() || "ollama");
|
||||
setWidgetValue(node, baseUrlWidget, baseUrl.trim());
|
||||
setWidgetValue(node, modelWidget, model.trim());
|
||||
setWidgetValue(node, promptWidget, prompt.trim());
|
||||
syncFormFromWidgets(node);
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
});
|
||||
|
||||
const aliasLabel = document.createElement("label");
|
||||
aliasLabel.textContent = "Alias";
|
||||
Object.assign(aliasLabel.style, {
|
||||
fontSize: "11px",
|
||||
color: "#bbb",
|
||||
});
|
||||
|
||||
const aliasInput = document.createElement("input");
|
||||
aliasInput.type = "text";
|
||||
aliasInput.placeholder = "optional alias, e.g. bob";
|
||||
Object.assign(aliasInput.style, {
|
||||
width: "100%",
|
||||
boxSizing: "border-box",
|
||||
background: "#1c1c1c",
|
||||
color: "#eee",
|
||||
border: "1px solid #444",
|
||||
borderRadius: "6px",
|
||||
padding: "6px 8px",
|
||||
fontSize: "12px",
|
||||
});
|
||||
aliasInput.addEventListener("input", () => {
|
||||
setWidgetValue(node, findWidget(node, "alias"), aliasInput.value);
|
||||
syncFormFromWidgets(node);
|
||||
});
|
||||
|
||||
const descLabel = document.createElement("label");
|
||||
descLabel.textContent = "Description";
|
||||
Object.assign(descLabel.style, {
|
||||
fontSize: "11px",
|
||||
color: "#bbb",
|
||||
});
|
||||
|
||||
const descInput = document.createElement("textarea");
|
||||
descInput.placeholder = "Character description...";
|
||||
Object.assign(descInput.style, {
|
||||
width: "100%",
|
||||
minHeight: "86px",
|
||||
boxSizing: "border-box",
|
||||
resize: "vertical",
|
||||
background: "#1c1c1c",
|
||||
color: "#eee",
|
||||
border: "1px solid #444",
|
||||
borderRadius: "6px",
|
||||
padding: "8px",
|
||||
fontSize: "12px",
|
||||
lineHeight: "1.35",
|
||||
});
|
||||
descInput.addEventListener("input", () => {
|
||||
setWidgetValue(node, findWidget(node, "description"), descInput.value);
|
||||
syncFormFromWidgets(node);
|
||||
});
|
||||
|
||||
actionsRow.appendChild(analyzeButton);
|
||||
actionsRow.appendChild(settingsButton);
|
||||
form.appendChild(aliasLabel);
|
||||
form.appendChild(aliasInput);
|
||||
form.appendChild(descLabel);
|
||||
form.appendChild(descInput);
|
||||
form.appendChild(actionsRow);
|
||||
|
||||
container.appendChild(previewRow);
|
||||
container.appendChild(meta);
|
||||
container.appendChild(form);
|
||||
|
||||
node._msrPreviewContainer = container;
|
||||
node._msrPreviewRow = previewRow;
|
||||
node._msrMeta = meta;
|
||||
node._msrAliasInput = aliasInput;
|
||||
node._msrDescriptionInput = descInput;
|
||||
node._msrAnalyzeButton = analyzeButton;
|
||||
return container;
|
||||
}
|
||||
|
||||
async function analyzeCharacterNode(node, buttonWidget) {
|
||||
const descriptionWidget = findWidget(node, "description");
|
||||
if (!descriptionWidget) return;
|
||||
const providerWidget = findWidget(node, "analyze_provider");
|
||||
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||
const modelWidget = findWidget(node, "analyze_model");
|
||||
const promptWidget = findWidget(node, "analyze_prompt");
|
||||
|
||||
buttonWidget.label = "Analyzing...";
|
||||
if (node._msrAnalyzeButton) {
|
||||
node._msrAnalyzeButton.textContent = "Analyzing...";
|
||||
node._msrAnalyzeButton.disabled = true;
|
||||
}
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
|
||||
try {
|
||||
const imageResults = await Promise.all([
|
||||
imageInputToDataUrl(node, "image_1"),
|
||||
imageInputToDataUrl(node, "image_2"),
|
||||
]);
|
||||
const imageDebug = imageResults.map((result) => result?.debug || null).filter(Boolean);
|
||||
const images = (
|
||||
imageResults.map((result) => result?.dataUrl || null)
|
||||
).filter(Boolean);
|
||||
|
||||
if (!images.length) {
|
||||
throw new Error("Connect at least one image input before analyzing.");
|
||||
}
|
||||
|
||||
const response = await api.fetchApi("/ltx_director/analyze_character", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
provider: providerWidget?.value || "ollama",
|
||||
base_url: baseUrlWidget?.value || "",
|
||||
model: modelWidget?.value || "",
|
||||
prompt: promptWidget?.value || "",
|
||||
image_b64: images,
|
||||
image_debug: imageDebug,
|
||||
}),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (result.status !== "success") {
|
||||
console.warn("[MSRCharacter] analyze debug", {
|
||||
sentImages: imageDebug,
|
||||
response: result,
|
||||
});
|
||||
throw new Error(result.message || "Unknown analysis error");
|
||||
}
|
||||
|
||||
const description = result.description || "";
|
||||
setWidgetValue(node, descriptionWidget, description);
|
||||
if (node._msrDescriptionInput) {
|
||||
node._msrDescriptionInput.value = description;
|
||||
}
|
||||
syncFormFromWidgets(node);
|
||||
refreshCharacterPreview(node);
|
||||
} catch (error) {
|
||||
console.error("[MSRCharacter] analyze failed", error);
|
||||
alert(`MSR Character analyze failed: ${error.message || error}`);
|
||||
} finally {
|
||||
buttonWidget.label = "Analyze with Ollama";
|
||||
if (node._msrAnalyzeButton) {
|
||||
node._msrAnalyzeButton.textContent = "Analyze";
|
||||
node._msrAnalyzeButton.disabled = false;
|
||||
}
|
||||
node.setDirtyCanvas?.(true, true);
|
||||
}
|
||||
}
|
||||
|
||||
function hideNodeWidget(widget) {
|
||||
if (!widget) return;
|
||||
widget.hidden = true;
|
||||
if (!widget.options) widget.options = {};
|
||||
widget.options.hidden = true;
|
||||
if (widget.element) {
|
||||
widget.element.style.display = "none";
|
||||
}
|
||||
}
|
||||
|
||||
app.registerExtension({
|
||||
name: "Comfy.MSRCharacterCS",
|
||||
async beforeRegisterNodeDef(nodeType, nodeData) {
|
||||
if (nodeData.name === "MSRCharacterCS") {
|
||||
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
|
||||
nodeType.prototype.onNodeCreated = function () {
|
||||
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
|
||||
|
||||
if (!(this.widgets || []).find((widget) => widget.msrAnalyze)) {
|
||||
this.addWidget("button", "Analyze with Ollama", null, () => {
|
||||
analyzeCharacterNode(this, (this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||
}, { serialize: false });
|
||||
const buttonWidget = this.widgets[this.widgets.length - 1];
|
||||
buttonWidget.msrAnalyze = true;
|
||||
buttonWidget.label = "Analyze with Ollama";
|
||||
}
|
||||
|
||||
if (!findWidget(this, "analyze_provider")) {
|
||||
this.addWidget("combo", "analyze_provider", "ollama", null, {
|
||||
values: ["ollama", "lmstudio", "custom", "off"],
|
||||
});
|
||||
}
|
||||
if (!findWidget(this, "analyze_base_url")) {
|
||||
this.addWidget("text", "analyze_base_url", "");
|
||||
}
|
||||
if (!findWidget(this, "analyze_model")) {
|
||||
this.addWidget("text", "analyze_model", "");
|
||||
}
|
||||
if (!findWidget(this, "analyze_prompt")) {
|
||||
this.addWidget("text", "analyze_prompt", DEFAULT_ANALYZE_PROMPT);
|
||||
}
|
||||
|
||||
if (!this._msrPreviewWidget) {
|
||||
const previewContainer = buildCharacterUi(this);
|
||||
this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, {
|
||||
getValue: () => "",
|
||||
setValue: () => {},
|
||||
});
|
||||
this._msrPreviewWidget.computeSize = () => [Math.max(10, (this.size?.[0] || 320) - 30), 260];
|
||||
}
|
||||
|
||||
const originalConnectionsChange = this.onConnectionsChange;
|
||||
this.onConnectionsChange = function () {
|
||||
const result = originalConnectionsChange?.apply(this, arguments);
|
||||
refreshCharacterPreview(this);
|
||||
return result;
|
||||
};
|
||||
|
||||
const originalConfigure = this.onConfigure;
|
||||
this.onConfigure = function () {
|
||||
const result = originalConfigure?.apply(this, arguments);
|
||||
setTimeout(() => {
|
||||
syncFormFromWidgets(this);
|
||||
refreshCharacterPreview(this);
|
||||
}, 0);
|
||||
return result;
|
||||
};
|
||||
|
||||
["alias", "description", "analyze_provider", "analyze_base_url", "analyze_model", "analyze_prompt"].forEach((widgetName) => {
|
||||
hideNodeWidget(findWidget(this, widgetName));
|
||||
});
|
||||
hideNodeWidget((this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||
|
||||
this.size[0] = Math.max(this.size[0] || 0, 330);
|
||||
syncFormFromWidgets(this);
|
||||
refreshCharacterPreview(this);
|
||||
};
|
||||
}
|
||||
|
||||
if (nodeData.name === "MSRCharacterSetCS") {
|
||||
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
|
||||
nodeType.prototype.onNodeCreated = function () {
|
||||
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
|
||||
this.size[0] = Math.max(this.size[0] || 0, 260);
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -1,8 +1,16 @@
|
||||
import folder_paths
|
||||
import numpy as np
|
||||
import os
|
||||
import torch
|
||||
import av
|
||||
|
||||
DEFAULT_SAMPLE_RATE = 44100
|
||||
DEFAULT_SILENCE_SECONDS = 1
|
||||
|
||||
|
||||
def build_silence(sample_rate: int = DEFAULT_SAMPLE_RATE, seconds: int = DEFAULT_SILENCE_SECONDS, channels: int = 2) -> torch.Tensor:
|
||||
return torch.zeros((channels, int(sample_rate * seconds)), dtype=torch.float32)
|
||||
|
||||
def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert audio to float 32 bits PCM format."""
|
||||
if wav.dtype.is_floating_point:
|
||||
@@ -23,18 +31,17 @@ def load_audio_file(filepath: str) -> tuple[torch.Tensor, int]:
|
||||
sr = stream.codec_context.sample_rate
|
||||
n_channels = stream.channels
|
||||
|
||||
frames = []
|
||||
frame_arrays = []
|
||||
for frame in af.decode(streams=stream.index):
|
||||
buf = torch.from_numpy(frame.to_ndarray())
|
||||
if buf.shape[0] != n_channels:
|
||||
buf = buf.view(-1, n_channels).t()
|
||||
array = frame.to_ndarray()
|
||||
if array.shape[0] != n_channels:
|
||||
array = array.reshape(-1, n_channels).T
|
||||
frame_arrays.append(array)
|
||||
|
||||
frames.append(buf)
|
||||
|
||||
if not frames:
|
||||
if not frame_arrays:
|
||||
raise ValueError("No audio frames decoded.")
|
||||
|
||||
wav = torch.cat(frames, dim=1)
|
||||
wav = torch.from_numpy(np.concatenate(frame_arrays, axis=1))
|
||||
wav = f32_pcm(wav)
|
||||
return wav, sr
|
||||
|
||||
@@ -99,17 +106,16 @@ class LoadAudioUI:
|
||||
missing_info = audio if audio != "none" else "None selected"
|
||||
print(f"!!! [LoadAudioUI] Warning: Audio file '{missing_info}' not found. Outputting 1 second of silence.")
|
||||
|
||||
sample_rate = 44100
|
||||
# 1 second of silence (stereo) -> shape [channels, time]
|
||||
waveform = torch.zeros((2, 44100))
|
||||
sample_rate = DEFAULT_SAMPLE_RATE
|
||||
waveform = build_silence(sample_rate)
|
||||
else:
|
||||
try:
|
||||
waveform, sample_rate = load_audio_file(audio_path)
|
||||
except Exception as e:
|
||||
# If decoding fails for any reason, fallback to silence rather than crashing the workflow
|
||||
print(f"!!! [LoadAudioUI] Error decoding {audio}: {e}. Falling back to silence.")
|
||||
sample_rate = 44100
|
||||
waveform = torch.zeros((2, 44100))
|
||||
sample_rate = DEFAULT_SAMPLE_RATE
|
||||
waveform = build_silence(sample_rate)
|
||||
|
||||
# Convert seconds to frames
|
||||
start_frame = int(start_time * sample_rate)
|
||||
|
||||
1831
ltx_director.py
1831
ltx_director.py
File diff suppressed because it is too large
Load Diff
@@ -10,8 +10,7 @@ import comfy.utils
|
||||
import folder_paths
|
||||
import node_helpers
|
||||
from comfy_extras import nodes_lt
|
||||
from comfy_api.latest import io
|
||||
from .ltx_director import GuideData, MotionGuideData, _resize_image, _execute_comfy_node, _unpack
|
||||
from .ltx_director import _resize_image, _execute_comfy_node, _unpack
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -57,6 +56,18 @@ def _clone_noise_mask(latent, latent_image):
|
||||
device=latent_image.device,
|
||||
)
|
||||
|
||||
def _safe_json_loads(raw_value, default=None):
|
||||
import json
|
||||
|
||||
if isinstance(raw_value, dict):
|
||||
return raw_value
|
||||
if not raw_value:
|
||||
return {} if default is None else default
|
||||
try:
|
||||
return json.loads(raw_value)
|
||||
except Exception:
|
||||
return {} if default is None else default
|
||||
|
||||
def _resize_latent_spatial(latent_image, noise_mask, width, height, method):
|
||||
b, c, f, h, w = latent_image.shape
|
||||
if width == w and height == h:
|
||||
@@ -78,6 +89,65 @@ def _ceil_to_multiple(value, multiple):
|
||||
multiple = max(1, int(multiple))
|
||||
return int(math.ceil(value / multiple) * multiple)
|
||||
|
||||
def _normalize_resize_method_for_encode(resize_method):
|
||||
if resize_method == "maintain aspect ratio":
|
||||
return "pad"
|
||||
return resize_method
|
||||
|
||||
def _append_latent_frames(latent_image, noise_mask, extra_frames, mask_fill=1.0):
|
||||
extra_frames = int(max(0, extra_frames))
|
||||
if extra_frames == 0:
|
||||
return latent_image, noise_mask
|
||||
|
||||
batch, channels, _, height, width = latent_image.shape
|
||||
latent_tail = torch.zeros(
|
||||
(batch, channels, extra_frames, height, width),
|
||||
dtype=latent_image.dtype,
|
||||
device=latent_image.device,
|
||||
)
|
||||
latent_image = torch.cat([latent_image, latent_tail], dim=2)
|
||||
|
||||
mask_batch, mask_channels, _, mask_height, mask_width = noise_mask.shape
|
||||
mask_tail = torch.full(
|
||||
(mask_batch, mask_channels, extra_frames, mask_height, mask_width),
|
||||
float(mask_fill),
|
||||
dtype=noise_mask.dtype,
|
||||
device=noise_mask.device,
|
||||
)
|
||||
noise_mask = torch.cat([noise_mask, mask_tail], dim=2)
|
||||
return latent_image, noise_mask
|
||||
|
||||
def _encode_resized_video_frames(
|
||||
vae,
|
||||
frames,
|
||||
target_width,
|
||||
target_height,
|
||||
resize_method,
|
||||
time_scale_factor,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
):
|
||||
pixels = _resize_image(
|
||||
frames,
|
||||
target_width,
|
||||
target_height,
|
||||
_normalize_resize_method_for_encode(resize_method),
|
||||
divisible_by=1,
|
||||
)
|
||||
num_frames_to_keep = ((pixels.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
||||
encode_src = pixels[:num_frames_to_keep, :, :, :3]
|
||||
if use_tiled_encode:
|
||||
encoded = vae.encode_tiled(
|
||||
encode_src,
|
||||
tile_x=tile_size,
|
||||
tile_y=tile_size,
|
||||
overlap=tile_overlap,
|
||||
)
|
||||
else:
|
||||
encoded = vae.encode(encode_src)
|
||||
return pixels, encoded
|
||||
|
||||
def _snap_latent_to_downscale(latent_image, noise_mask, downscale_factor, method):
|
||||
factor = int(max(1, round(float(downscale_factor))))
|
||||
if factor <= 1:
|
||||
@@ -307,11 +377,413 @@ class LTXDirectorGuide:
|
||||
RETURN_NAMES = ("positive", "negative", "latent", "model", "latent_downscale_factor")
|
||||
FUNCTION = "execute"
|
||||
|
||||
@classmethod
|
||||
def _maybe_add_attention(cls, positive, negative, is_lora_active, tokens_added, guide_orig_shape, attention_strength):
|
||||
if not is_lora_active:
|
||||
return positive, negative
|
||||
positive = _append_guide_attention_entry(
|
||||
positive,
|
||||
tokens_added,
|
||||
guide_orig_shape,
|
||||
attention_strength=attention_strength,
|
||||
)
|
||||
negative = _append_guide_attention_entry(
|
||||
negative,
|
||||
tokens_added,
|
||||
guide_orig_shape,
|
||||
attention_strength=attention_strength,
|
||||
)
|
||||
return positive, negative
|
||||
|
||||
@classmethod
|
||||
def _apply_crop_count(cls, positive, negative, crop_frames):
|
||||
crop_frames = max(0, int(crop_frames))
|
||||
values = {"nghtdrp_guide_crop_latent_frames": crop_frames}
|
||||
positive = node_helpers.conditioning_set_values(positive, values)
|
||||
negative = node_helpers.conditioning_set_values(negative, values)
|
||||
return positive, negative
|
||||
|
||||
@classmethod
|
||||
def _load_cached_motion_video_frames(cls, video_cache, video_file, trim_start_frames, length_frames, director_fps, resample_mode):
|
||||
cache_key = (
|
||||
str(video_file),
|
||||
int(trim_start_frames),
|
||||
int(length_frames),
|
||||
float(director_fps),
|
||||
str(resample_mode),
|
||||
)
|
||||
cached = video_cache.get(cache_key)
|
||||
if cached is None:
|
||||
cached = _load_motion_video_frames(
|
||||
video_file,
|
||||
trim_start_frames=trim_start_frames,
|
||||
length_frames=length_frames,
|
||||
director_fps=director_fps,
|
||||
resample_mode=resample_mode,
|
||||
)
|
||||
video_cache[cache_key] = cached
|
||||
return cached
|
||||
|
||||
@classmethod
|
||||
def _encode_image_guide(cls, vae, latent_width, latent_height, img_tensor, scale_factors, target_pix_w, target_pix_h, upscale_method):
|
||||
_, height, width, _ = img_tensor.shape
|
||||
if target_pix_w != width or target_pix_h != height:
|
||||
img_nchw = img_tensor.permute(0, 3, 1, 2)
|
||||
img_resized = comfy.utils.common_upscale(
|
||||
img_nchw,
|
||||
target_pix_w,
|
||||
target_pix_h,
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
img_tensor = img_resized.permute(0, 2, 3, 1)
|
||||
return nodes_lt.LTXVAddGuide.encode(
|
||||
vae,
|
||||
latent_width,
|
||||
latent_height,
|
||||
img_tensor,
|
||||
scale_factors,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _apply_image_guides(
|
||||
cls,
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
images,
|
||||
insert_frames,
|
||||
strengths,
|
||||
latent_width,
|
||||
latent_height,
|
||||
latent_length,
|
||||
scale_factors,
|
||||
image_attention_strength,
|
||||
is_lora_active,
|
||||
upscale_method,
|
||||
):
|
||||
target_pix_w = int(latent_width * 32)
|
||||
target_pix_h = int(latent_height * 32)
|
||||
for idx, img_tensor in enumerate(images):
|
||||
frame_value = insert_frames[idx] if idx < len(insert_frames) else 0
|
||||
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
|
||||
if strength <= 0.0:
|
||||
continue
|
||||
|
||||
image_pixels, guide_latent = cls._encode_image_guide(
|
||||
vae,
|
||||
latent_width,
|
||||
latent_height,
|
||||
img_tensor,
|
||||
scale_factors,
|
||||
target_pix_w,
|
||||
target_pix_h,
|
||||
upscale_method,
|
||||
)
|
||||
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
|
||||
positive,
|
||||
latent_length,
|
||||
len(image_pixels),
|
||||
int(frame_value),
|
||||
scale_factors,
|
||||
)
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
max_frames = latent_length - latent_idx
|
||||
if guide_latent.shape[2] > max_frames:
|
||||
guide_latent = guide_latent[:, :, :max_frames]
|
||||
|
||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||
guide_orig_shape = list(guide_latent.shape[2:])
|
||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||
positive,
|
||||
negative,
|
||||
frame_idx,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
guide_latent,
|
||||
strength,
|
||||
scale_factors,
|
||||
)
|
||||
positive, negative = cls._maybe_add_attention(
|
||||
positive,
|
||||
negative,
|
||||
is_lora_active,
|
||||
tokens_added,
|
||||
guide_orig_shape,
|
||||
image_attention_strength,
|
||||
)
|
||||
|
||||
return positive, negative, latent_image, noise_mask
|
||||
|
||||
@classmethod
|
||||
def _apply_motion_segment_guides(
|
||||
cls,
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
segments,
|
||||
director_fps,
|
||||
latent_length,
|
||||
latent_width,
|
||||
latent_height,
|
||||
time_scale_factor,
|
||||
scale_factors,
|
||||
latent_downscale_factor,
|
||||
crop,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
active_resize_method,
|
||||
is_lora_active,
|
||||
video_cache,
|
||||
):
|
||||
for seg in segments:
|
||||
try:
|
||||
video_file = seg.get("videoFile")
|
||||
if not video_file:
|
||||
continue
|
||||
|
||||
start_frame = int(seg.get("start", 0))
|
||||
length_frames = int(seg.get("length", 1))
|
||||
trim_start = int(seg.get("trimStart", 0))
|
||||
video_strength = float(seg.get("videoStrength", 1.0))
|
||||
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
|
||||
if length_frames <= 0 or video_strength <= 0.0:
|
||||
continue
|
||||
|
||||
video_frames = cls._load_cached_motion_video_frames(
|
||||
video_cache,
|
||||
video_file,
|
||||
trim_start,
|
||||
length_frames,
|
||||
director_fps,
|
||||
seg.get("resampleMode", "nearest"),
|
||||
)
|
||||
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
||||
video_frames = video_frames[:num_frames_to_keep]
|
||||
causal_fix = start_frame == 0 or num_frames_to_keep == 1
|
||||
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
|
||||
|
||||
_, guide_latent = _encode_video_iclora_guide(
|
||||
vae,
|
||||
latent_width,
|
||||
latent_height,
|
||||
encode_frames,
|
||||
scale_factors,
|
||||
latent_downscale_factor,
|
||||
crop,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
resize_method=active_resize_method,
|
||||
)
|
||||
if not causal_fix:
|
||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||
|
||||
frame_idx = start_frame
|
||||
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
if start_frame > 0 and guide_latent.shape[2] > 1:
|
||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||
frame_idx += time_scale_factor
|
||||
latent_idx += 1
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
max_frames = latent_length - latent_idx
|
||||
if guide_latent.shape[2] > max_frames:
|
||||
guide_latent = guide_latent[:, :, :max_frames]
|
||||
|
||||
guide_orig_shape = list(guide_latent.shape[2:])
|
||||
batch, _, frames, height, width = guide_latent.shape
|
||||
guide_mask = torch.ones(
|
||||
(batch, 1, frames, height, width),
|
||||
device=guide_latent.device,
|
||||
dtype=guide_latent.dtype,
|
||||
)
|
||||
if start_frame > 0:
|
||||
ramp_steps = [0.25, 0.65]
|
||||
for i, step in enumerate(ramp_steps):
|
||||
if i < frames:
|
||||
guide_mask[:, :, i, :, :] = 1.0 + video_strength * (1.0 - step)
|
||||
|
||||
ldf = int(max(1, round(float(latent_downscale_factor))))
|
||||
if ldf > 1:
|
||||
dilated = _dilate_latent(
|
||||
{"samples": guide_latent, "noise_mask": guide_mask},
|
||||
horizontal_scale=ldf,
|
||||
vertical_scale=ldf,
|
||||
)
|
||||
guide_latent = dilated["samples"]
|
||||
guide_mask = dilated["noise_mask"]
|
||||
|
||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||
positive,
|
||||
negative,
|
||||
frame_idx,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
guide_latent,
|
||||
video_strength,
|
||||
scale_factors,
|
||||
guide_mask=guide_mask,
|
||||
latent_downscale_factor=float(latent_downscale_factor),
|
||||
causal_fix=causal_fix,
|
||||
)
|
||||
positive, negative = cls._maybe_add_attention(
|
||||
positive,
|
||||
negative,
|
||||
is_lora_active,
|
||||
tokens_added,
|
||||
guide_orig_shape,
|
||||
video_attention_strength,
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
|
||||
|
||||
return positive, negative, latent_image, noise_mask
|
||||
|
||||
@classmethod
|
||||
def _execute_retake_mode(
|
||||
cls,
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
guide_data,
|
||||
tdata,
|
||||
segments,
|
||||
model,
|
||||
latent_downscale_factor,
|
||||
director_fps,
|
||||
latent_length,
|
||||
latent_width,
|
||||
latent_height,
|
||||
time_scale_factor,
|
||||
active_resize_method,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
is_empty_latent,
|
||||
initial_latent_length,
|
||||
video_cache,
|
||||
):
|
||||
print(
|
||||
f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. "
|
||||
f"is_empty_latent: {is_empty_latent}"
|
||||
)
|
||||
target_width = latent_width * 32
|
||||
target_height = latent_height * 32
|
||||
retake_start = int(tdata.get("retakeStart", 0))
|
||||
retake_len = int(tdata.get("retakeLength", 0))
|
||||
retake_strength = float(tdata.get("retakeStrength", 1.0))
|
||||
|
||||
start_frame = int(guide_data.get("start_frame", 0))
|
||||
relative_start = max(0, retake_start - start_frame)
|
||||
l_start = min(max(0, relative_start // time_scale_factor), latent_length)
|
||||
l_end = min(
|
||||
max(0, int(math.ceil((relative_start + retake_len) / time_scale_factor))),
|
||||
latent_length,
|
||||
)
|
||||
|
||||
need_base_video = is_empty_latent or l_start > 0 or l_end < latent_length
|
||||
if not need_base_video:
|
||||
print(
|
||||
"[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. "
|
||||
"Skipping base video loading and VAE encoding."
|
||||
)
|
||||
|
||||
retake_vid_info = tdata.get("retakeVideo") or {}
|
||||
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
|
||||
if not video_file and not retake_vid_info and segments:
|
||||
video_file = segments[0].get("videoFile", "")
|
||||
|
||||
if need_base_video and not video_file:
|
||||
if retake_vid_info and not retake_vid_info.get("imageFile"):
|
||||
raise ValueError(
|
||||
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
|
||||
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
|
||||
)
|
||||
raise ValueError(
|
||||
"Retake Mode is active, but no base video has been selected on the timeline. "
|
||||
"Please drag and drop or upload a base video on the timeline first."
|
||||
)
|
||||
|
||||
if need_base_video and video_file:
|
||||
try:
|
||||
ltxv_length = (latent_length - 1) * time_scale_factor + 1
|
||||
print(
|
||||
f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} "
|
||||
f"starting at frame {start_frame} for length {ltxv_length} at resolution "
|
||||
f"{target_width}x{target_height}"
|
||||
)
|
||||
video_frames = cls._load_cached_motion_video_frames(
|
||||
video_cache,
|
||||
video_file,
|
||||
start_frame,
|
||||
ltxv_length,
|
||||
director_fps,
|
||||
"nearest",
|
||||
)
|
||||
_, base_latent = _encode_resized_video_frames(
|
||||
vae,
|
||||
video_frames,
|
||||
target_width,
|
||||
target_height,
|
||||
active_resize_method,
|
||||
time_scale_factor,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
)
|
||||
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
|
||||
|
||||
paste_len = min(base_latent.shape[2], latent_length)
|
||||
if is_empty_latent:
|
||||
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
|
||||
else:
|
||||
print(
|
||||
f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved "
|
||||
f"regions (0-{l_start} and {l_end}-{paste_len})"
|
||||
)
|
||||
if l_start > 0:
|
||||
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
|
||||
if l_end < paste_len:
|
||||
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
|
||||
except Exception as e:
|
||||
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
|
||||
|
||||
noise_mask = torch.zeros_like(noise_mask)
|
||||
if l_end > l_start:
|
||||
noise_mask[:, :, l_start:l_end] = retake_strength
|
||||
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
|
||||
|
||||
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
||||
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
|
||||
return (
|
||||
positive,
|
||||
negative,
|
||||
{"samples": latent_image, "noise_mask": noise_mask},
|
||||
model,
|
||||
float(latent_downscale_factor),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, positive, negative, vae, latent, guide_data, motion_guide_data=None, model=None, ic_lora_name="None", ic_lora_strength=1.0, scale_by=1.0, upscale_method="bicubic", image_attention_strength=1.0, crop="center", auto_snap_ic_grid=True, use_tiled_encode=False, tile_size=256, tile_overlap=64, retake_mode=False, msr_strength=0.0):
|
||||
motion_segments = (motion_guide_data or {}).get("segments", []) if motion_guide_data else []
|
||||
image_guides_count = len(guide_data.get("images", [])) if guide_data else 0
|
||||
print(f"[LTXDirectorGuide] execute started. motion_segments: {len(motion_segments)}, image_guides: {image_guides_count}, ic_lora_name: {ic_lora_name}, model connected: {model is not None}, retake_mode: {retake_mode}")
|
||||
video_cache = {}
|
||||
|
||||
active_resize_method = guide_data.get("resize_method") if guide_data else None
|
||||
if not active_resize_method:
|
||||
@@ -336,24 +808,6 @@ class LTXDirectorGuide:
|
||||
if auto_snap_ic_grid and model is not None and ic_lora_name != "None":
|
||||
latent_image, noise_mask = _snap_latent_to_downscale(latent_image, noise_mask, latent_downscale_factor, upscale_method)
|
||||
|
||||
# Ghost Mask: the Director kept the latent clean (so it stays length-matched with the audio
|
||||
# latent) and asked us to pre-extend it here by one hidden frame per reference. The tail
|
||||
# sits past the clean region; the reference images (in guide_data["images"], anchored at
|
||||
# (clean + i)) are written into it by the loop below, then dropped downstream by Clean
|
||||
# Latent Slice (length = clean_latent_frames).
|
||||
_ghost_pre_extend = int(guide_data.get("ghost_pre_extend", 0)) if guide_data else 0
|
||||
if _ghost_pre_extend > 0:
|
||||
gb, gc, gf, gh, gw = latent_image.shape
|
||||
latent_image = torch.cat(
|
||||
[latent_image, torch.zeros((gb, gc, _ghost_pre_extend, gh, gw), dtype=latent_image.dtype, device=latent_image.device)],
|
||||
dim=2,
|
||||
)
|
||||
mb, mc, mf, mh, mw = noise_mask.shape
|
||||
noise_mask = torch.cat(
|
||||
[noise_mask, torch.ones((mb, mc, _ghost_pre_extend, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
||||
initial_latent_length = int(latent_length)
|
||||
|
||||
@@ -364,13 +818,7 @@ class LTXDirectorGuide:
|
||||
if msr is not None:
|
||||
return cls._inject_msr(positive, negative, vae, latent_image, noise_mask, msr, model, latent_downscale_factor, msr_strength)
|
||||
|
||||
# Parse timeline JSON to see if retake mode is active in UI
|
||||
import json
|
||||
timeline_data_str = guide_data.get("timeline_data", "{}") if guide_data else "{}"
|
||||
try:
|
||||
tdata = json.loads(timeline_data_str)
|
||||
except Exception:
|
||||
tdata = {}
|
||||
tdata = _safe_json_loads(guide_data.get("timeline_data", "{}") if guide_data else "{}")
|
||||
|
||||
is_retake_active = bool(retake_mode) or tdata.get("retakeMode", False)
|
||||
is_empty_latent = (latent_image.abs().max().item() < 1e-5)
|
||||
@@ -392,111 +840,30 @@ class LTXDirectorGuide:
|
||||
# Load single base video, encode continuously, apply temporal mask.
|
||||
# -----------------------------------------------------------------------
|
||||
if is_retake_active:
|
||||
print(f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. is_empty_latent: {is_empty_latent}")
|
||||
target_width = latent_width * 32
|
||||
target_height = latent_height * 32
|
||||
|
||||
# Calculate retake region latent indices first so we know what to copy/paste
|
||||
retake_start = int(tdata.get("retakeStart", 0))
|
||||
retake_len = int(tdata.get("retakeLength", 0))
|
||||
retake_strength = float(tdata.get("retakeStrength", 1.0))
|
||||
|
||||
start_frame = int(guide_data.get("start_frame", 0))
|
||||
relative_start = max(0, retake_start - start_frame)
|
||||
|
||||
l_start = relative_start // time_scale_factor
|
||||
l_end = int(math.ceil((relative_start + retake_len) / time_scale_factor))
|
||||
|
||||
l_start = min(l_start, latent_length)
|
||||
l_end = min(l_end, latent_length)
|
||||
|
||||
# Stage 2 optimization: If the retake region covers the entire generation area,
|
||||
# there are no preserved regions to copy over in Stage 2. We can bypass video loading/encoding.
|
||||
need_base_video = True
|
||||
if not is_empty_latent and l_start == 0 and l_end >= latent_length:
|
||||
need_base_video = False
|
||||
print("[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. Skipping base video loading and VAE encoding.")
|
||||
|
||||
# 1. Try to load and encode base video from timeline data
|
||||
retake_vid_info = tdata.get("retakeVideo") or {}
|
||||
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
|
||||
|
||||
# Fallback to first segment only if it exists and we have no retake video info at all (old workflows)
|
||||
if not video_file and not retake_vid_info and len(segments) > 0:
|
||||
video_file = segments[0].get("videoFile", "")
|
||||
|
||||
if need_base_video:
|
||||
if not video_file:
|
||||
if retake_vid_info and not retake_vid_info.get("imageFile"):
|
||||
raise ValueError(
|
||||
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
|
||||
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Retake Mode is active, but no base video has been selected on the timeline. "
|
||||
"Please drag and drop or upload a base video on the timeline first."
|
||||
)
|
||||
|
||||
if video_file and need_base_video:
|
||||
try:
|
||||
print(f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} starting at frame {start_frame} for length {ltxv_length} at resolution {target_width}x{target_height}")
|
||||
video_frames = _load_motion_video_frames(
|
||||
video_file, trim_start_frames=start_frame, length_frames=ltxv_length, director_fps=director_fps, resample_mode="nearest"
|
||||
)
|
||||
|
||||
# Retake base video must match the exact target latent shape.
|
||||
# "maintain aspect ratio" doesn't pad, which causes shape mismatch in VAE encode.
|
||||
# So we fallback "maintain aspect ratio" to "pad" for retake base video.
|
||||
retake_resize_method = active_resize_method
|
||||
if retake_resize_method == "maintain aspect ratio":
|
||||
retake_resize_method = "pad"
|
||||
|
||||
pixels = _resize_image(video_frames, target_width, target_height, retake_resize_method, divisible_by=1)
|
||||
|
||||
num_clip_frames = pixels.shape[0]
|
||||
num_frames_to_keep = ((num_clip_frames - 1) // time_scale_factor) * time_scale_factor + 1
|
||||
encode_src = pixels[:num_frames_to_keep, :, :, :3]
|
||||
|
||||
if use_tiled_encode:
|
||||
base_latent = vae.encode_tiled(encode_src, tile_x=tile_size, tile_y=tile_size, overlap=tile_overlap)
|
||||
else:
|
||||
base_latent = vae.encode(encode_src)
|
||||
|
||||
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
|
||||
|
||||
# Copy to latent_image
|
||||
paste_len = min(base_latent.shape[2], latent_length)
|
||||
if is_empty_latent:
|
||||
# Stage 1: Overwrite entire latent with base video VAE encode
|
||||
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
|
||||
else:
|
||||
# Stage 2: Overwrite only preserved regions (before l_start and after l_end)
|
||||
# leaving the generated retake region from Stage 1 untouched
|
||||
print(f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved regions (0-{l_start} and {l_end}-{paste_len})")
|
||||
if l_start > 0:
|
||||
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
|
||||
if l_end < paste_len:
|
||||
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
|
||||
except Exception as e:
|
||||
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
|
||||
|
||||
# 2. Build the temporal noise mask (0.0 = frozen, 1.0 = regenerate)
|
||||
noise_mask = torch.zeros_like(noise_mask) # Initialize fully frozen
|
||||
|
||||
l_start = min(l_start, latent_length)
|
||||
l_end = min(l_end, latent_length)
|
||||
|
||||
if l_end > l_start:
|
||||
noise_mask[:, :, l_start:l_end] = retake_strength
|
||||
|
||||
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
|
||||
|
||||
# In retake mode, skip normal mode processing entirely and return immediately!
|
||||
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
||||
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
||||
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
||||
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
|
||||
return cls._execute_retake_mode(
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
guide_data,
|
||||
tdata,
|
||||
segments,
|
||||
model,
|
||||
latent_downscale_factor,
|
||||
director_fps,
|
||||
latent_length,
|
||||
latent_width,
|
||||
latent_height,
|
||||
time_scale_factor,
|
||||
active_resize_method,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
is_empty_latent,
|
||||
initial_latent_length,
|
||||
video_cache,
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Standard Timeline Keyframe Guidance:
|
||||
@@ -507,121 +874,51 @@ class LTXDirectorGuide:
|
||||
if len(images) > 0 or len(segments) > 0:
|
||||
print(f"[LTXDirectorGuide] Using Appended Keyframe Guidance. is_lora_active: {is_lora_active}")
|
||||
|
||||
# A. Process Image Guides
|
||||
for idx, img_tensor in enumerate(images):
|
||||
f_idx = insert_frames[idx] if idx < len(insert_frames) else 0
|
||||
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
|
||||
if strength <= 0.0:
|
||||
continue
|
||||
|
||||
B_img, H_img, W_img, C_img = img_tensor.shape
|
||||
target_pix_w = int(latent_width * 32)
|
||||
target_pix_h = int(latent_height * 32)
|
||||
if target_pix_w != W_img or target_pix_h != H_img:
|
||||
img_nchw = img_tensor.permute(0, 3, 1, 2)
|
||||
img_resized = comfy.utils.common_upscale(img_nchw, target_pix_w, target_pix_h, upscale_method, "disabled")
|
||||
img_tensor = img_resized.permute(0, 2, 3, 1)
|
||||
|
||||
image_pixels, guide_latent = nodes_lt.LTXVAddGuide.encode(vae, latent_width, latent_height, img_tensor, scale_factors)
|
||||
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(positive, latent_length, len(image_pixels), int(f_idx), scale_factors)
|
||||
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
max_frames = latent_length - latent_idx
|
||||
if guide_latent.shape[2] > max_frames:
|
||||
guide_latent = guide_latent[:, :, :max_frames]
|
||||
|
||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||
guide_orig_shape = list(guide_latent.shape[2:])
|
||||
|
||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, strength, scale_factors
|
||||
)
|
||||
if is_lora_active:
|
||||
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
|
||||
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
|
||||
|
||||
# B. Process Motion Video Segments
|
||||
for seg in segments:
|
||||
try:
|
||||
video_file = seg.get("videoFile")
|
||||
if not video_file:
|
||||
continue
|
||||
|
||||
start_frame = int(seg.get("start", 0))
|
||||
length_frames = int(seg.get("length", 1))
|
||||
trim_start = int(seg.get("trimStart", 0))
|
||||
video_strength = float(seg.get("videoStrength", 1.0))
|
||||
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
|
||||
|
||||
if length_frames <= 0 or video_strength <= 0.0:
|
||||
continue
|
||||
|
||||
start_frame_aligned = start_frame
|
||||
video_frames = _load_motion_video_frames(video_file, trim_start, length_frames, director_fps, seg.get("resampleMode", "nearest"))
|
||||
|
||||
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
||||
video_frames = video_frames[:num_frames_to_keep]
|
||||
causal_fix = int(start_frame_aligned) == 0 or num_frames_to_keep == 1
|
||||
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
|
||||
|
||||
_, guide_latent = _encode_video_iclora_guide(vae, latent_width, latent_height, encode_frames, scale_factors, latent_downscale_factor, crop, use_tiled_encode, tile_size, tile_overlap, resize_method=active_resize_method)
|
||||
|
||||
if not causal_fix:
|
||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||
|
||||
frame_idx = start_frame_aligned
|
||||
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
|
||||
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
if start_frame > 0 and guide_latent.shape[2] > 1:
|
||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||
frame_idx += time_scale_factor
|
||||
latent_idx += 1
|
||||
if latent_idx >= latent_length:
|
||||
continue
|
||||
|
||||
max_frames = latent_length - latent_idx
|
||||
if guide_latent.shape[2] > max_frames:
|
||||
guide_latent = guide_latent[:, :, :max_frames]
|
||||
|
||||
guide_orig_shape = list(guide_latent.shape[2:])
|
||||
|
||||
B_g, C_g, F_g, H_g, W_g = guide_latent.shape
|
||||
guide_mask = torch.ones((B_g, 1, F_g, H_g, W_g), device=guide_latent.device, dtype=guide_latent.dtype)
|
||||
|
||||
if start_frame > 0:
|
||||
ramp_steps = [0.25, 0.65]
|
||||
for i, s in enumerate(ramp_steps):
|
||||
if i < F_g:
|
||||
guide_mask_val = 1.0 + video_strength * (1.0 - s)
|
||||
guide_mask[:, :, i, :, :] = guide_mask_val
|
||||
|
||||
ldf = int(max(1, round(float(latent_downscale_factor))))
|
||||
if ldf > 1:
|
||||
dilated = _dilate_latent({"samples": guide_latent, "noise_mask": guide_mask}, horizontal_scale=ldf, vertical_scale=ldf)
|
||||
guide_mask = dilated["noise_mask"]
|
||||
guide_latent = dilated["samples"]
|
||||
|
||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, video_strength, scale_factors, guide_mask=guide_mask, latent_downscale_factor=float(latent_downscale_factor), causal_fix=causal_fix
|
||||
)
|
||||
if is_lora_active:
|
||||
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
|
||||
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
|
||||
positive, negative, latent_image, noise_mask = cls._apply_image_guides(
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
images,
|
||||
insert_frames,
|
||||
strengths,
|
||||
latent_width,
|
||||
latent_height,
|
||||
latent_length,
|
||||
scale_factors,
|
||||
image_attention_strength,
|
||||
is_lora_active,
|
||||
upscale_method,
|
||||
)
|
||||
positive, negative, latent_image, noise_mask = cls._apply_motion_segment_guides(
|
||||
positive,
|
||||
negative,
|
||||
vae,
|
||||
latent_image,
|
||||
noise_mask,
|
||||
segments,
|
||||
director_fps,
|
||||
latent_length,
|
||||
latent_width,
|
||||
latent_height,
|
||||
time_scale_factor,
|
||||
scale_factors,
|
||||
latent_downscale_factor,
|
||||
crop,
|
||||
use_tiled_encode,
|
||||
tile_size,
|
||||
tile_overlap,
|
||||
active_resize_method,
|
||||
is_lora_active,
|
||||
video_cache,
|
||||
)
|
||||
|
||||
else:
|
||||
print("[LTXDirectorGuide] No timeline guides present. Passing through.")
|
||||
|
||||
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
||||
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
||||
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
||||
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
|
||||
|
||||
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
|
||||
|
||||
@@ -655,16 +952,7 @@ class LTXDirectorGuide:
|
||||
# (which trims the slideshow's temporal footprint) lands on the true clean length.
|
||||
pad_latents = max(0, prefix_latents - len(keyframes))
|
||||
if pad_latents > 0:
|
||||
B, C, F, H, W = latent_image.shape
|
||||
latent_image = torch.cat(
|
||||
[latent_image, torch.zeros((B, C, pad_latents, H, W), dtype=latent_image.dtype, device=latent_image.device)],
|
||||
dim=2,
|
||||
)
|
||||
mb, mc, mf, mh, mw = noise_mask.shape
|
||||
noise_mask = torch.cat(
|
||||
[noise_mask, torch.ones((mb, mc, pad_latents, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
|
||||
dim=2,
|
||||
)
|
||||
latent_image, noise_mask = _append_latent_frames(latent_image, noise_mask, pad_latents, mask_fill=1.0)
|
||||
|
||||
latent = {"samples": latent_image, "noise_mask": noise_mask}
|
||||
|
||||
|
||||
126
msr_character.py
Normal file
126
msr_character.py
Normal file
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from comfy_api.latest import io
|
||||
|
||||
|
||||
MSRCharacterData = io.Custom("MSR_CHARACTER")
|
||||
MSRCharacterSetData = io.Custom("MSR_CHARACTER_SET")
|
||||
|
||||
|
||||
def _compact_alias(alias: str) -> str:
|
||||
if not alias:
|
||||
return ""
|
||||
alias = alias.strip()
|
||||
if alias.startswith("@"):
|
||||
alias = alias[1:]
|
||||
return alias.strip()
|
||||
|
||||
|
||||
class MSRCharacter(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="MSRCharacterCS",
|
||||
display_name="MSR Character DUMAS",
|
||||
category="WhatDreamsCost DUMAS",
|
||||
description=(
|
||||
"Reusable MSR character payload. Accepts up to two reference images plus "
|
||||
"a description and optional alias for @alias prompt replacement."
|
||||
),
|
||||
inputs=[
|
||||
io.Image.Input(
|
||||
"image_1",
|
||||
optional=True,
|
||||
tooltip="Optional first reference image for this character.",
|
||||
),
|
||||
io.Image.Input(
|
||||
"image_2",
|
||||
optional=True,
|
||||
tooltip="Optional second reference image for this character.",
|
||||
),
|
||||
io.String.Input(
|
||||
"description",
|
||||
multiline=True,
|
||||
default="",
|
||||
optional=True,
|
||||
tooltip="Manual or generated appearance description used for prompt replacement.",
|
||||
),
|
||||
io.String.Input(
|
||||
"alias",
|
||||
default="",
|
||||
optional=True,
|
||||
tooltip="Optional alias. Prompts can use @alias alongside @charN/@characterN.",
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
MSRCharacterData.Output(display_name="character"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, image_1=None, image_2=None, description="", alias="") -> io.NodeOutput:
|
||||
images = []
|
||||
if image_1 is not None:
|
||||
images.append(image_1)
|
||||
if image_2 is not None:
|
||||
images.append(image_2)
|
||||
|
||||
payload = {
|
||||
"images": images,
|
||||
"description": (description or "").strip(),
|
||||
"alias": _compact_alias(alias or ""),
|
||||
}
|
||||
return io.NodeOutput(payload)
|
||||
|
||||
|
||||
class MSRCharacterSet(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="MSRCharacterSetCS",
|
||||
display_name="MSR Character Set DUMAS",
|
||||
category="WhatDreamsCost DUMAS",
|
||||
description="Bundles up to six MSR Character nodes into one character set for the director.",
|
||||
inputs=[
|
||||
MSRCharacterData.Input("character_1", optional=True),
|
||||
MSRCharacterData.Input("character_2", optional=True),
|
||||
MSRCharacterData.Input("character_3", optional=True),
|
||||
MSRCharacterData.Input("character_4", optional=True),
|
||||
MSRCharacterData.Input("character_5", optional=True),
|
||||
MSRCharacterData.Input("character_6", optional=True),
|
||||
],
|
||||
outputs=[
|
||||
MSRCharacterSetData.Output(display_name="character_set"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(
|
||||
cls,
|
||||
character_1=None,
|
||||
character_2=None,
|
||||
character_3=None,
|
||||
character_4=None,
|
||||
character_5=None,
|
||||
character_6=None,
|
||||
) -> io.NodeOutput:
|
||||
characters = []
|
||||
for item in (
|
||||
character_1,
|
||||
character_2,
|
||||
character_3,
|
||||
character_4,
|
||||
character_5,
|
||||
character_6,
|
||||
):
|
||||
if not item:
|
||||
continue
|
||||
characters.append(
|
||||
{
|
||||
"images": list(item.get("images") or []),
|
||||
"description": (item.get("description") or "").strip(),
|
||||
"alias": _compact_alias(item.get("alias") or ""),
|
||||
}
|
||||
)
|
||||
|
||||
return io.NodeOutput({"characters": characters})
|
||||
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
import os
|
||||
import types
|
||||
|
||||
import comfy.ldm.modules.attention
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||
|
||||
|
||||
def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs):
|
||||
@@ -73,10 +76,10 @@ def _make_masked_override(prev_override):
|
||||
|
||||
|
||||
def debug_log(msg):
|
||||
if not _DEBUG_PROMPT_RELAY:
|
||||
return
|
||||
try:
|
||||
import os
|
||||
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||
with open(log_path, "a", encoding="utf-8") as f:
|
||||
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
|
||||
f.write(msg + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
120
prompt_relay.py
120
prompt_relay.py
@@ -1,21 +1,28 @@
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import torch
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||
|
||||
|
||||
def _build_segment_cost(query_frames, seg, dtype):
|
||||
local = seg["local_token_idx"]
|
||||
distance = (query_frames[:, None] - seg["midpoint"]).abs()
|
||||
cost = seg["strength"] * (torch.relu(distance - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
|
||||
return local, cost.to(dtype)
|
||||
|
||||
|
||||
def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame):
|
||||
"""Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing)."""
|
||||
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
|
||||
query_frames = torch.arange(Lq, device=device, dtype=torch.long) // tokens_per_frame
|
||||
query_frames = (torch.arange(Lq, device=device, dtype=torch.float32) / float(tokens_per_frame)).floor()
|
||||
|
||||
for seg in q_token_idx:
|
||||
local = seg["local_token_idx"].to(device=device)
|
||||
d = (query_frames.float()[:, None] - seg["midpoint"]).abs()
|
||||
strength = seg.get("strength", 1.0)
|
||||
cost = strength * (torch.relu(d - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
|
||||
offset[:, local] = cost.to(offset.dtype)
|
||||
local, cost = _build_segment_cost(query_frames, seg, offset.dtype)
|
||||
offset[:, local.to(device=device)] = cost
|
||||
|
||||
return offset
|
||||
|
||||
@@ -26,27 +33,27 @@ def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames
|
||||
query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq
|
||||
|
||||
for seg in q_token_idx:
|
||||
local = seg["local_token_idx"].to(device=device)
|
||||
d = (query_frames[:, None] - seg["midpoint"]).abs()
|
||||
if is_audio:
|
||||
sigma_val = seg.get("sigma_audio", seg["sigma"])
|
||||
window_val = seg.get("window_audio", seg["window"])
|
||||
strength_val = seg.get("strength_audio", 1.0)
|
||||
active_seg = {
|
||||
"local_token_idx": seg["local_token_idx"],
|
||||
"midpoint": seg["midpoint"],
|
||||
"window": seg.get("window_audio", seg["window"]),
|
||||
"sigma": seg.get("sigma_audio", seg["sigma"]),
|
||||
"strength": seg.get("strength_audio", 1.0),
|
||||
}
|
||||
else:
|
||||
sigma_val = seg["sigma"]
|
||||
window_val = seg["window"]
|
||||
strength_val = seg.get("strength", 1.0)
|
||||
cost = strength_val * (torch.relu(d - window_val) ** 2) / (2 * sigma_val ** 2)
|
||||
offset[:, local] = cost.to(offset.dtype)
|
||||
active_seg = seg
|
||||
local, cost = _build_segment_cost(query_frames, active_seg, offset.dtype)
|
||||
offset[:, local.to(device=device)] = cost
|
||||
|
||||
return offset
|
||||
|
||||
|
||||
def debug_log(msg):
|
||||
if not _DEBUG_PROMPT_RELAY:
|
||||
return
|
||||
try:
|
||||
import os
|
||||
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||
with open(log_path, "a", encoding="utf-8") as f:
|
||||
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
|
||||
f.write(msg + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
@@ -61,6 +68,15 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
|
||||
"""
|
||||
cache = {}
|
||||
max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1
|
||||
latent_frames = max(1, int(latent_frames))
|
||||
fallback_tokens_per_frame = max(1, int(fallback_tokens_per_frame))
|
||||
|
||||
def _get_video_query_layout(Lq, grid_sizes):
|
||||
if grid_sizes is not None:
|
||||
return int(grid_sizes[1]) * int(grid_sizes[2])
|
||||
if Lq % latent_frames == 0:
|
||||
return Lq // latent_frames
|
||||
return fallback_tokens_per_frame
|
||||
|
||||
def mask_fn(Lq, Lk, dtype, device, transformer_options):
|
||||
debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}")
|
||||
@@ -82,13 +98,7 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
|
||||
mode = "scaled"
|
||||
video_lq = -1
|
||||
else:
|
||||
if grid_sizes is not None:
|
||||
video_tpf = int(grid_sizes[1]) * int(grid_sizes[2])
|
||||
else:
|
||||
if Lq % latent_frames == 0:
|
||||
video_tpf = Lq // latent_frames
|
||||
else:
|
||||
video_tpf = fallback_tokens_per_frame
|
||||
video_tpf = _get_video_query_layout(Lq, grid_sizes)
|
||||
video_lq = latent_frames * video_tpf
|
||||
|
||||
# Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq
|
||||
@@ -189,6 +199,34 @@ def get_raw_tokenizer(clip):
|
||||
)
|
||||
|
||||
|
||||
def _get_tokenizer_input_ids(raw_tokenizer, text):
|
||||
tokenized = raw_tokenizer(text)
|
||||
if isinstance(tokenized, dict) and "input_ids" in tokenized:
|
||||
return tokenized["input_ids"]
|
||||
if hasattr(tokenized, "input_ids"):
|
||||
return tokenized.input_ids
|
||||
if isinstance(tokenized, list):
|
||||
return tokenized
|
||||
return []
|
||||
|
||||
|
||||
def _tokenized_length(raw_tokenizer, text, eos_adjustment):
|
||||
return max(0, len(_get_tokenizer_input_ids(raw_tokenizer, text)) - eos_adjustment)
|
||||
|
||||
|
||||
def _tokenizer_has_eos(raw_tokenizer):
|
||||
if getattr(raw_tokenizer, "add_eos", False):
|
||||
return True
|
||||
try:
|
||||
ids = _get_tokenizer_input_ids(raw_tokenizer, "test")
|
||||
if not ids:
|
||||
return False
|
||||
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
|
||||
return (eos_id is not None and ids[-1] == eos_id) or ids[-1] == 1
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
|
||||
"""Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges).
|
||||
|
||||
@@ -197,38 +235,14 @@ def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
|
||||
prefixed_locals = [" " + lp for lp in local_prompts]
|
||||
full_prompt = global_prompt + "".join(prefixed_locals)
|
||||
|
||||
# Detect if the tokenizer appends EOS dynamically
|
||||
has_eos = getattr(raw_tokenizer, "add_eos", False)
|
||||
if not has_eos:
|
||||
try:
|
||||
test_res = raw_tokenizer("test")
|
||||
if isinstance(test_res, dict) and "input_ids" in test_res:
|
||||
ids = test_res["input_ids"]
|
||||
elif hasattr(test_res, "input_ids"):
|
||||
ids = test_res.input_ids
|
||||
elif isinstance(test_res, list):
|
||||
ids = test_res
|
||||
else:
|
||||
ids = []
|
||||
|
||||
if ids:
|
||||
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
|
||||
if eos_id is not None and ids[-1] == eos_id:
|
||||
has_eos = True
|
||||
elif ids[-1] == 1:
|
||||
has_eos = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
eos_adj = 1 if has_eos else 0
|
||||
|
||||
prev_len = len(raw_tokenizer(global_prompt)["input_ids"]) - eos_adj
|
||||
eos_adj = 1 if _tokenizer_has_eos(raw_tokenizer) else 0
|
||||
prev_len = _tokenized_length(raw_tokenizer, global_prompt, eos_adj)
|
||||
token_ranges = []
|
||||
built = global_prompt
|
||||
|
||||
for plp in prefixed_locals:
|
||||
built += plp
|
||||
cur_len = len(raw_tokenizer(built)["input_ids"]) - eos_adj
|
||||
cur_len = _tokenized_length(raw_tokenizer, built, eos_adj)
|
||||
if cur_len <= prev_len:
|
||||
raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'")
|
||||
token_ranges.append((prev_len, cur_len))
|
||||
|
||||
117
timeline_nodes.py
Normal file
117
timeline_nodes.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from comfy_api.latest import io
|
||||
|
||||
|
||||
TimelineSectionData = io.Custom("TIMELINE_SECTION")
|
||||
TimelineSectionSetData = io.Custom("TIMELINE_SECTION_SET")
|
||||
class TimelineSection(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="TimelineSectionCS",
|
||||
display_name="Timeline Section DUMAS",
|
||||
category="WhatDreamsCost DUMAS",
|
||||
description="Reusable timeline section with required text, optional image, and a duration in seconds.",
|
||||
inputs=[
|
||||
io.Image.Input(
|
||||
"image",
|
||||
optional=True,
|
||||
tooltip="Optional image for this timeline section. When omitted, the section becomes a text-only block.",
|
||||
),
|
||||
io.String.Input(
|
||||
"text",
|
||||
multiline=True,
|
||||
default="",
|
||||
tooltip="Required text/prompt for this timeline section.",
|
||||
),
|
||||
io.Float.Input(
|
||||
"duration",
|
||||
default=1.0,
|
||||
min=0.1,
|
||||
max=1000.0,
|
||||
step=0.01,
|
||||
tooltip="Section duration in seconds.",
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
TimelineSectionData.Output(display_name="section"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, image=None, text="", duration=1.0) -> io.NodeOutput:
|
||||
payload = {
|
||||
"image": image,
|
||||
"text": (text or "").strip(),
|
||||
"duration": max(0.1, float(duration or 0.0)),
|
||||
}
|
||||
return io.NodeOutput(payload)
|
||||
|
||||
|
||||
class TimelineSectionSet(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="TimelineSectionSetCS",
|
||||
display_name="Timeline Section Set DUMAS",
|
||||
category="WhatDreamsCost DUMAS",
|
||||
description="Bundles up to ten Timeline Section nodes into one reusable section set.",
|
||||
inputs=[
|
||||
TimelineSectionData.Input("section_1", optional=True),
|
||||
TimelineSectionData.Input("section_2", optional=True),
|
||||
TimelineSectionData.Input("section_3", optional=True),
|
||||
TimelineSectionData.Input("section_4", optional=True),
|
||||
TimelineSectionData.Input("section_5", optional=True),
|
||||
TimelineSectionData.Input("section_6", optional=True),
|
||||
TimelineSectionData.Input("section_7", optional=True),
|
||||
TimelineSectionData.Input("section_8", optional=True),
|
||||
TimelineSectionData.Input("section_9", optional=True),
|
||||
TimelineSectionData.Input("section_10", optional=True),
|
||||
],
|
||||
outputs=[
|
||||
TimelineSectionSetData.Output(display_name="section_set"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(
|
||||
cls,
|
||||
section_1=None,
|
||||
section_2=None,
|
||||
section_3=None,
|
||||
section_4=None,
|
||||
section_5=None,
|
||||
section_6=None,
|
||||
section_7=None,
|
||||
section_8=None,
|
||||
section_9=None,
|
||||
section_10=None,
|
||||
) -> io.NodeOutput:
|
||||
sections = []
|
||||
for item in (
|
||||
section_1,
|
||||
section_2,
|
||||
section_3,
|
||||
section_4,
|
||||
section_5,
|
||||
section_6,
|
||||
section_7,
|
||||
section_8,
|
||||
section_9,
|
||||
section_10,
|
||||
):
|
||||
if not item:
|
||||
continue
|
||||
text = (item.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
sections.append(
|
||||
{
|
||||
"image": item.get("image"),
|
||||
"text": text,
|
||||
"duration": max(0.1, float(item.get("duration") or 0.0)),
|
||||
}
|
||||
)
|
||||
|
||||
return io.NodeOutput({"sections": sections})
|
||||
Reference in New Issue
Block a user