Compare commits
71 Commits
0.20
...
main_dumas
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
beb0892793 | ||
|
|
2f40aa77ad | ||
|
|
c2fea55704 | ||
|
|
c36f6f3bf6 | ||
|
|
ee3215c48f | ||
|
|
cfdee30dee | ||
|
|
9ebc488072 | ||
|
|
5347ca3dbc | ||
|
|
df4c936aae | ||
|
|
fa546cbc8f | ||
|
|
575838aeb0 | ||
|
|
a1a1184155 | ||
|
|
3f24406697 | ||
|
|
a711a9d34b | ||
|
|
00d5703137 | ||
|
|
64cbfd2f36 | ||
|
|
e113472c7d | ||
|
|
e01ceef71b | ||
|
|
8736d4a463 | ||
|
|
cc51b298fd | ||
|
|
1cfe7f1932 | ||
|
|
c782f3263b | ||
|
|
ab6415caa4 | ||
|
|
5c4349d9b9 | ||
|
|
7b4e8865d6 | ||
|
|
4c0358c11b | ||
|
|
4dfcb1e597 | ||
|
|
73918a28b3 | ||
|
|
48ac1c242f | ||
|
|
6bc0aa1f71 | ||
|
|
e2c5c38b14 | ||
|
|
f372a2ed69 | ||
|
|
2da1de2f6d | ||
|
|
b059974544 | ||
|
|
27f1d6a1e4 | ||
|
|
6ec28d33a4 | ||
|
|
2bd748c8a6 | ||
|
|
c20ded3319 | ||
|
|
64a7b3d371 | ||
|
|
6551b4f6c3 | ||
|
|
33ba18cac1 | ||
|
|
f88a1c7da6 | ||
|
|
c5a4aecdd7 | ||
|
|
f0cd0b382d | ||
|
|
c2f39a2f0e | ||
|
|
9b62c51a55 | ||
|
|
9a6f01ed67 | ||
|
|
bd56a21b7f | ||
|
|
67696a4e59 | ||
|
|
e68357f384 | ||
|
|
119692160a | ||
|
|
7bb54f61d6 | ||
|
|
264fdcf7cd | ||
|
|
e87b5d8d8f | ||
|
|
50664026a1 | ||
|
|
39af2b0150 | ||
|
|
58469ccb4b | ||
|
|
b9c935466f | ||
|
|
ab7d14bd89 | ||
|
|
a536ac5a14 | ||
|
|
7099b3c6f1 | ||
|
|
7e569d5deb | ||
|
|
b9d68aea62 | ||
|
|
268e543210 | ||
|
|
a1a4be4fc3 | ||
|
|
5df6ed6669 | ||
|
|
04ff3fc38a | ||
|
|
51bd3b4968 | ||
|
|
b8cedad997 | ||
|
|
6695b1a1aa | ||
|
|
b6a1e373d3 |
@@ -58,13 +58,12 @@ This node uses Ollama Locally : you will need to install it (very lightweight) a
|
|||||||
|
|
||||||
- You can still enter your description manually if you don't want to install it but it works very well!
|
- You can still enter your description manually if you don't want to install it but it works very well!
|
||||||
|
|
||||||
- Both references mode work (fixed), Licon MSR and Ghost Mask.
|
- Reference mode included: Licon MSR.
|
||||||
|
|
||||||
- Use their Lora, that is an important step : https://huggingface.co/LiconStudio/LTX-2.3-Multiple-Subject-Reference/tree/main
|
- Use their Lora, that is an important step : https://huggingface.co/LiconStudio/LTX-2.3-Multiple-Subject-Reference/tree/main
|
||||||
|
|
||||||
- Enjoy!
|
- Enjoy!
|
||||||
|
|
||||||
<img width="596" height="668" alt="Capture d'écran 2026-06-13 234036" src="https://github.com/user-attachments/assets/2dfe7992-a72a-4eda-a103-41b3a8f77714" />
|
|
||||||
|
|
||||||
|
|
||||||
# 🔄 Recent Updates
|
# 🔄 Recent Updates
|
||||||
|
|||||||
22
__init__.py
22
__init__.py
@@ -1,5 +1,7 @@
|
|||||||
from .ltx_director import LTXDirector
|
from .ltx_director import LTXDirector
|
||||||
from .ltx_director_guide import LTXDirectorGuide, LTXDirectorCropGuides
|
from .ltx_director_guide import LTXDirectorGuide, LTXDirectorCropGuides
|
||||||
|
from .msr_character import MSRCharacter, MSRCharacterSet
|
||||||
|
from .timeline_nodes import TimelineSection, TimelineSectionSet
|
||||||
from comfy_api.latest import ComfyExtension, io
|
from comfy_api.latest import ComfyExtension, io
|
||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
from .latent_slice import CleanLatentSlice
|
from .latent_slice import CleanLatentSlice
|
||||||
@@ -8,6 +10,10 @@ class PromptRelay(ComfyExtension):
|
|||||||
@override
|
@override
|
||||||
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
||||||
return [
|
return [
|
||||||
|
MSRCharacter,
|
||||||
|
MSRCharacterSet,
|
||||||
|
TimelineSection,
|
||||||
|
TimelineSectionSet,
|
||||||
LTXDirector,
|
LTXDirector,
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -16,16 +22,24 @@ async def comfy_entrypoint() -> PromptRelay:
|
|||||||
|
|
||||||
NODE_CLASS_MAPPINGS = {
|
NODE_CLASS_MAPPINGS = {
|
||||||
"LTXDirectorCS": LTXDirector,
|
"LTXDirectorCS": LTXDirector,
|
||||||
|
"MSRCharacterCS": MSRCharacter,
|
||||||
|
"MSRCharacterSetCS": MSRCharacterSet,
|
||||||
|
"TimelineSectionCS": TimelineSection,
|
||||||
|
"TimelineSectionSetCS": TimelineSectionSet,
|
||||||
"LTXDirectorGuideCS": LTXDirectorGuide,
|
"LTXDirectorGuideCS": LTXDirectorGuide,
|
||||||
"LTXDirectorCropGuidesCS": LTXDirectorCropGuides,
|
"LTXDirectorCropGuidesCS": LTXDirectorCropGuides,
|
||||||
"CleanLatentSliceCS": CleanLatentSlice,
|
"CleanLatentSliceCS": CleanLatentSlice,
|
||||||
}
|
}
|
||||||
|
|
||||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||||
"LTXDirectorCS": "LTX Director CS",
|
"LTXDirectorCS": "LTX Director DUMAS",
|
||||||
"LTXDirectorGuideCS": "LTX Director Guide CS",
|
"MSRCharacterCS": "MSR Character DUMAS",
|
||||||
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides CS",
|
"MSRCharacterSetCS": "MSR Character Set DUMAS",
|
||||||
"CleanLatentSliceCS": "Clean Latent Slice CS",
|
"TimelineSectionCS": "Timeline Section DUMAS",
|
||||||
|
"TimelineSectionSetCS": "Timeline Section Set DUMAS",
|
||||||
|
"LTXDirectorGuideCS": "LTX Director Guide DUMAS",
|
||||||
|
"LTXDirectorCropGuidesCS": "LTX Director Crop Guides DUMAS",
|
||||||
|
"CleanLatentSliceCS": "Clean Latent Slice DUMAS",
|
||||||
}
|
}
|
||||||
|
|
||||||
WEB_DIRECTORY = "./js"
|
WEB_DIRECTORY = "./js"
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
1711
js/ltx_director.js
1711
js/ltx_director.js
File diff suppressed because it is too large
Load Diff
@@ -3,22 +3,28 @@ import { app } from "../../scripts/app.js";
|
|||||||
// LTX Director Guide is a pure pass-through processor node.
|
// LTX Director Guide is a pure pass-through processor node.
|
||||||
// All configuration (images, insert frames, strengths) comes from
|
// All configuration (images, insert frames, strengths) comes from
|
||||||
// the guide_data output of Prompt Relay Encode (Timeline).
|
// the guide_data output of Prompt Relay Encode (Timeline).
|
||||||
|
function hideWidget(node, widgetName) {
|
||||||
|
const widget = node.widgets?.find((entry) => entry.name === widgetName);
|
||||||
|
if (!widget) return;
|
||||||
|
|
||||||
|
widget.hidden = true;
|
||||||
|
widget.options = widget.options || {};
|
||||||
|
widget.options.hidden = true;
|
||||||
|
|
||||||
|
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
|
||||||
|
widget.computeSize = () => [0, -4];
|
||||||
|
widget.draw = () => {};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (widget.element) widget.element.style.display = "none";
|
||||||
|
}
|
||||||
|
|
||||||
app.registerExtension({
|
app.registerExtension({
|
||||||
name: "Comfy.LTXDirectorGuideCS",
|
name: "Comfy.LTXDirectorGuideCS",
|
||||||
async nodeCreated(node) {
|
async nodeCreated(node) {
|
||||||
if (node.comfyClass !== "LTXDirectorGuideCS") return;
|
if (node.comfyClass !== "LTXDirectorGuideCS") return;
|
||||||
|
|
||||||
// Hide retake_mode widget on LiteGraph as it is dynamically auto-detected from the timeline data.
|
// Hide retake_mode widget on LiteGraph as it is dynamically auto-detected from the timeline data.
|
||||||
const w = node.widgets?.find(x => x.name === "retake_mode");
|
hideWidget(node, "retake_mode");
|
||||||
if (w) {
|
|
||||||
w.hidden = true;
|
|
||||||
if (!w.options) w.options = {};
|
|
||||||
w.options.hidden = true;
|
|
||||||
if (!window.LiteGraph || !window.LiteGraph.vueNodesMode) {
|
|
||||||
w.computeSize = () => [0, -4];
|
|
||||||
w.draw = () => { };
|
|
||||||
}
|
|
||||||
if (w.element) w.element.style.display = "none";
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
619
js/msr_character.js
Normal file
619
js/msr_character.js
Normal file
@@ -0,0 +1,619 @@
|
|||||||
|
const { app } = window.comfyAPI.app;
|
||||||
|
const { api } = window.comfyAPI.api;
|
||||||
|
const DEFAULT_ANALYZE_PROMPT = "Describe the character's physical appearance in two concise sentences. Specify their hair color/style, face details, and their clothing type/color. Keep the entire response very brief.";
|
||||||
|
|
||||||
|
function findWidget(node, name) {
|
||||||
|
return (node.widgets || []).find((widget) => widget.name === name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function findInput(node, name) {
|
||||||
|
return (node.inputs || []).find((input) => input.name === name);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOriginNodeForInput(node, inputName) {
|
||||||
|
const input = findInput(node, inputName);
|
||||||
|
if (!input || input.link == null) return null;
|
||||||
|
const link = app.graph.links[input.link];
|
||||||
|
if (!link) return null;
|
||||||
|
return app.graph.getNodeById(link.origin_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
function coercePreviewUrl(value) {
|
||||||
|
if (!value) return "";
|
||||||
|
if (typeof value === "string") return value;
|
||||||
|
if (typeof value === "object") {
|
||||||
|
if (typeof value.src === "string") return value.src;
|
||||||
|
if (typeof value.url === "string") return value.url;
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFileWidgetUrlFromOrigin(originNode) {
|
||||||
|
if (!originNode) return "";
|
||||||
|
|
||||||
|
const widgets = originNode.widgets || [];
|
||||||
|
const fileWidget = widgets.find((widget) => ["image", "filename", "file"].includes(widget.name));
|
||||||
|
const subfolderWidget = widgets.find((widget) => widget.name === "subfolder");
|
||||||
|
const filename = typeof fileWidget?.value === "string" ? fileWidget.value : "";
|
||||||
|
const subfolder = typeof subfolderWidget?.value === "string" ? subfolderWidget.value : "";
|
||||||
|
if (!filename) return "";
|
||||||
|
|
||||||
|
return api.apiURL(
|
||||||
|
`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvePreviewUrlFromOrigin(originNode) {
|
||||||
|
if (!originNode) return "";
|
||||||
|
const preview = Array.isArray(originNode.imgs) ? coercePreviewUrl(originNode.imgs[0]) : "";
|
||||||
|
return preview || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveCandidateImageUrls(originNode) {
|
||||||
|
const urls = [];
|
||||||
|
const fileUrl = resolveFileWidgetUrlFromOrigin(originNode);
|
||||||
|
const previewUrl = resolvePreviewUrlFromOrigin(originNode);
|
||||||
|
if (fileUrl) urls.push(fileUrl);
|
||||||
|
if (previewUrl && previewUrl !== fileUrl) urls.push(previewUrl);
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function blobToOptimizedDataUrl(blob, maxDim = 1024, quality = 0.82) {
|
||||||
|
const objectUrl = URL.createObjectURL(blob);
|
||||||
|
try {
|
||||||
|
const image = await new Promise((resolve, reject) => {
|
||||||
|
const img = new Image();
|
||||||
|
img.onload = () => resolve(img);
|
||||||
|
img.onerror = () => reject(new Error("Failed to decode image blob"));
|
||||||
|
img.src = objectUrl;
|
||||||
|
});
|
||||||
|
|
||||||
|
const width = image.naturalWidth || image.width || 0;
|
||||||
|
const height = image.naturalHeight || image.height || 0;
|
||||||
|
const scale = width > 0 && height > 0 ? Math.min(1, maxDim / Math.max(width, height)) : 1;
|
||||||
|
const targetWidth = Math.max(1, Math.round(width * scale)) || width || 1;
|
||||||
|
const targetHeight = Math.max(1, Math.round(height * scale)) || height || 1;
|
||||||
|
|
||||||
|
const canvas = document.createElement("canvas");
|
||||||
|
canvas.width = targetWidth;
|
||||||
|
canvas.height = targetHeight;
|
||||||
|
const ctx = canvas.getContext("2d");
|
||||||
|
if (!ctx) {
|
||||||
|
throw new Error("Could not acquire canvas context");
|
||||||
|
}
|
||||||
|
ctx.drawImage(image, 0, 0, targetWidth, targetHeight);
|
||||||
|
|
||||||
|
const optimizedBlob = await new Promise((resolve, reject) => {
|
||||||
|
canvas.toBlob((result) => {
|
||||||
|
if (result) resolve(result);
|
||||||
|
else reject(new Error("Canvas toBlob failed"));
|
||||||
|
}, "image/jpeg", quality);
|
||||||
|
});
|
||||||
|
|
||||||
|
const dataUrl = await new Promise((resolve) => {
|
||||||
|
const reader = new FileReader();
|
||||||
|
reader.onloadend = () => resolve(reader.result);
|
||||||
|
reader.readAsDataURL(optimizedBlob);
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
dataUrl,
|
||||||
|
originalWidth: width,
|
||||||
|
originalHeight: height,
|
||||||
|
outputWidth: targetWidth,
|
||||||
|
outputHeight: targetHeight,
|
||||||
|
outputBytes: optimizedBlob.size || 0,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
URL.revokeObjectURL(objectUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function imageInputToDataUrl(node, inputName) {
|
||||||
|
const originNode = getOriginNodeForInput(node, inputName);
|
||||||
|
const candidateUrls = resolveCandidateImageUrls(originNode);
|
||||||
|
if (!candidateUrls.length) {
|
||||||
|
return {
|
||||||
|
dataUrl: null,
|
||||||
|
debug: {
|
||||||
|
inputName,
|
||||||
|
originNodeId: originNode?.id ?? null,
|
||||||
|
originNodeType: originNode?.type ?? null,
|
||||||
|
candidateUrls: [],
|
||||||
|
selectedUrl: null,
|
||||||
|
mimeType: null,
|
||||||
|
blobBytes: 0,
|
||||||
|
dataUrlLength: 0,
|
||||||
|
originalWidth: 0,
|
||||||
|
originalHeight: 0,
|
||||||
|
outputWidth: 0,
|
||||||
|
outputHeight: 0,
|
||||||
|
outputBytes: 0,
|
||||||
|
error: "No candidate image URLs found",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const imageUrl of candidateUrls) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(imageUrl);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
const blob = await response.blob();
|
||||||
|
if (!blob.type.startsWith("image/")) {
|
||||||
|
throw new Error(`Unexpected blob type: ${blob.type || "unknown"}`);
|
||||||
|
}
|
||||||
|
const optimized = await blobToOptimizedDataUrl(blob);
|
||||||
|
const dataUrl = optimized.dataUrl;
|
||||||
|
return {
|
||||||
|
dataUrl,
|
||||||
|
debug: {
|
||||||
|
inputName,
|
||||||
|
originNodeId: originNode?.id ?? null,
|
||||||
|
originNodeType: originNode?.type ?? null,
|
||||||
|
candidateUrls,
|
||||||
|
selectedUrl: imageUrl,
|
||||||
|
mimeType: blob.type || null,
|
||||||
|
blobBytes: blob.size || 0,
|
||||||
|
dataUrlLength: typeof dataUrl === "string" ? dataUrl.length : 0,
|
||||||
|
originalWidth: optimized.originalWidth,
|
||||||
|
originalHeight: optimized.originalHeight,
|
||||||
|
outputWidth: optimized.outputWidth,
|
||||||
|
outputHeight: optimized.outputHeight,
|
||||||
|
outputBytes: optimized.outputBytes,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[MSRCharacter] Failed candidate image source", imageUrl, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
dataUrl: null,
|
||||||
|
debug: {
|
||||||
|
inputName,
|
||||||
|
originNodeId: originNode?.id ?? null,
|
||||||
|
originNodeType: originNode?.type ?? null,
|
||||||
|
candidateUrls,
|
||||||
|
selectedUrl: null,
|
||||||
|
mimeType: null,
|
||||||
|
blobBytes: 0,
|
||||||
|
dataUrlLength: 0,
|
||||||
|
originalWidth: 0,
|
||||||
|
originalHeight: 0,
|
||||||
|
outputWidth: 0,
|
||||||
|
outputHeight: 0,
|
||||||
|
outputBytes: 0,
|
||||||
|
error: "All candidate image URLs failed",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function setWidgetValue(node, widget, value) {
|
||||||
|
if (!widget) return;
|
||||||
|
const previous = widget.value;
|
||||||
|
widget.value = value;
|
||||||
|
if (widget.element) {
|
||||||
|
if (widget.element.type === "checkbox") {
|
||||||
|
widget.element.checked = !!value;
|
||||||
|
} else {
|
||||||
|
widget.element.value = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (widget.callback) {
|
||||||
|
try {
|
||||||
|
widget.callback(value);
|
||||||
|
} catch (error) {}
|
||||||
|
}
|
||||||
|
if (node.onWidgetChanged) {
|
||||||
|
try {
|
||||||
|
node.onWidgetChanged(widget.name, value, previous, widget);
|
||||||
|
} catch (error) {}
|
||||||
|
}
|
||||||
|
if (node.properties) {
|
||||||
|
node.properties[widget.name] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function syncFormFromWidgets(node) {
|
||||||
|
const description = findWidget(node, "description")?.value || "";
|
||||||
|
const alias = findWidget(node, "alias")?.value || "";
|
||||||
|
const provider = findWidget(node, "analyze_provider")?.value || "ollama";
|
||||||
|
const model = findWidget(node, "analyze_model")?.value || "";
|
||||||
|
const baseUrl = findWidget(node, "analyze_base_url")?.value || "";
|
||||||
|
|
||||||
|
if (node._msrAliasInput && node._msrAliasInput.value !== alias) {
|
||||||
|
node._msrAliasInput.value = alias;
|
||||||
|
}
|
||||||
|
if (node._msrDescriptionInput && node._msrDescriptionInput.value !== description) {
|
||||||
|
node._msrDescriptionInput.value = description;
|
||||||
|
}
|
||||||
|
if (node._msrMeta) {
|
||||||
|
node._msrMeta.innerHTML = [
|
||||||
|
alias
|
||||||
|
? `Alias: <span style="color:#e8e8e8">@${String(alias).replace(/^@/, "")}</span>`
|
||||||
|
: "Alias: <span style=\"color:#666\">none</span>",
|
||||||
|
`<div style="margin-top:4px;color:#888">Analyze: ${provider}${model ? ` / ${model}` : ""}${baseUrl ? ` / ${baseUrl}` : ""}</div>`,
|
||||||
|
`<div style="margin-top:4px;color:#666">Description is stored in the field below.</div>`,
|
||||||
|
].join("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function refreshCharacterPreview(node) {
|
||||||
|
if (!node._msrPreviewRow) return;
|
||||||
|
|
||||||
|
const inputNames = ["image_1", "image_2"];
|
||||||
|
const previewUrls = inputNames
|
||||||
|
.map((inputName) => {
|
||||||
|
const originNode = getOriginNodeForInput(node, inputName);
|
||||||
|
const candidates = resolveCandidateImageUrls(originNode);
|
||||||
|
return candidates[0] || "";
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
const previewRow = node._msrPreviewRow;
|
||||||
|
previewRow.innerHTML = "";
|
||||||
|
|
||||||
|
if (previewUrls.length) {
|
||||||
|
previewUrls.forEach((previewUrl) => {
|
||||||
|
const image = document.createElement("img");
|
||||||
|
image.src = previewUrl;
|
||||||
|
Object.assign(image.style, {
|
||||||
|
flex: "1",
|
||||||
|
minWidth: "0",
|
||||||
|
height: "74px",
|
||||||
|
objectFit: "cover",
|
||||||
|
borderRadius: "6px",
|
||||||
|
border: "1px solid #444",
|
||||||
|
background: "#111",
|
||||||
|
});
|
||||||
|
previewRow.appendChild(image);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
const placeholder = document.createElement("div");
|
||||||
|
placeholder.textContent = "Connect up to 2 image nodes";
|
||||||
|
Object.assign(placeholder.style, {
|
||||||
|
width: "100%",
|
||||||
|
height: "74px",
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
color: "#777",
|
||||||
|
fontSize: "11px",
|
||||||
|
borderRadius: "6px",
|
||||||
|
border: "1px dashed #444",
|
||||||
|
background: "#181818",
|
||||||
|
});
|
||||||
|
previewRow.appendChild(placeholder);
|
||||||
|
}
|
||||||
|
|
||||||
|
syncFormFromWidgets(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCharacterUi(node) {
|
||||||
|
const container = document.createElement("div");
|
||||||
|
Object.assign(container.style, {
|
||||||
|
boxSizing: "border-box",
|
||||||
|
width: "100%",
|
||||||
|
padding: "6px 2px 2px 2px",
|
||||||
|
});
|
||||||
|
|
||||||
|
const previewRow = document.createElement("div");
|
||||||
|
Object.assign(previewRow.style, {
|
||||||
|
display: "flex",
|
||||||
|
gap: "6px",
|
||||||
|
minHeight: "74px",
|
||||||
|
});
|
||||||
|
|
||||||
|
const meta = document.createElement("div");
|
||||||
|
Object.assign(meta.style, {
|
||||||
|
marginTop: "8px",
|
||||||
|
fontSize: "11px",
|
||||||
|
color: "#aaa",
|
||||||
|
lineHeight: "1.4",
|
||||||
|
});
|
||||||
|
|
||||||
|
const form = document.createElement("div");
|
||||||
|
Object.assign(form.style, {
|
||||||
|
marginTop: "10px",
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
gap: "6px",
|
||||||
|
});
|
||||||
|
|
||||||
|
const actionsRow = document.createElement("div");
|
||||||
|
Object.assign(actionsRow.style, {
|
||||||
|
display: "flex",
|
||||||
|
gap: "8px",
|
||||||
|
alignItems: "center",
|
||||||
|
flexWrap: "wrap",
|
||||||
|
});
|
||||||
|
|
||||||
|
const analyzeButton = document.createElement("button");
|
||||||
|
analyzeButton.type = "button";
|
||||||
|
analyzeButton.textContent = "Analyze";
|
||||||
|
Object.assign(analyzeButton.style, {
|
||||||
|
alignSelf: "flex-start",
|
||||||
|
background: "#2b4f38",
|
||||||
|
color: "#f3f3f3",
|
||||||
|
border: "1px solid #496d56",
|
||||||
|
borderRadius: "6px",
|
||||||
|
padding: "6px 10px",
|
||||||
|
fontSize: "11px",
|
||||||
|
cursor: "pointer",
|
||||||
|
});
|
||||||
|
analyzeButton.addEventListener("click", () => {
|
||||||
|
const widgetButton = (node.widgets || []).find((widget) => widget.msrAnalyze);
|
||||||
|
if (!widgetButton) {
|
||||||
|
alert("Analyze button widget is missing on this node.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
analyzeCharacterNode(node, widgetButton);
|
||||||
|
});
|
||||||
|
|
||||||
|
const settingsButton = document.createElement("button");
|
||||||
|
settingsButton.type = "button";
|
||||||
|
settingsButton.textContent = "Analyze Settings";
|
||||||
|
Object.assign(settingsButton.style, {
|
||||||
|
alignSelf: "flex-start",
|
||||||
|
background: "#252525",
|
||||||
|
color: "#ddd",
|
||||||
|
border: "1px solid #444",
|
||||||
|
borderRadius: "6px",
|
||||||
|
padding: "6px 10px",
|
||||||
|
fontSize: "11px",
|
||||||
|
cursor: "pointer",
|
||||||
|
});
|
||||||
|
settingsButton.addEventListener("click", () => {
|
||||||
|
const providerWidget = findWidget(node, "analyze_provider");
|
||||||
|
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||||
|
const modelWidget = findWidget(node, "analyze_model");
|
||||||
|
const promptWidget = findWidget(node, "analyze_prompt");
|
||||||
|
const provider = window.prompt("Analyze provider: ollama, lmstudio, custom, off", providerWidget?.value || "ollama");
|
||||||
|
if (provider == null) return;
|
||||||
|
const baseUrl = window.prompt("Analyze base URL (blank = default)", baseUrlWidget?.value || "");
|
||||||
|
if (baseUrl == null) return;
|
||||||
|
const model = window.prompt("Analyze model (blank = provider default)", modelWidget?.value || "");
|
||||||
|
if (model == null) return;
|
||||||
|
const prompt = window.prompt("Analyze prompt (blank = default)", promptWidget?.value || DEFAULT_ANALYZE_PROMPT);
|
||||||
|
if (prompt == null) return;
|
||||||
|
setWidgetValue(node, providerWidget, provider.trim() || "ollama");
|
||||||
|
setWidgetValue(node, baseUrlWidget, baseUrl.trim());
|
||||||
|
setWidgetValue(node, modelWidget, model.trim());
|
||||||
|
setWidgetValue(node, promptWidget, prompt.trim());
|
||||||
|
syncFormFromWidgets(node);
|
||||||
|
node.setDirtyCanvas?.(true, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
const aliasLabel = document.createElement("label");
|
||||||
|
aliasLabel.textContent = "Alias";
|
||||||
|
Object.assign(aliasLabel.style, {
|
||||||
|
fontSize: "11px",
|
||||||
|
color: "#bbb",
|
||||||
|
});
|
||||||
|
|
||||||
|
const aliasInput = document.createElement("input");
|
||||||
|
aliasInput.type = "text";
|
||||||
|
aliasInput.placeholder = "optional alias, e.g. bob";
|
||||||
|
Object.assign(aliasInput.style, {
|
||||||
|
width: "100%",
|
||||||
|
boxSizing: "border-box",
|
||||||
|
background: "#1c1c1c",
|
||||||
|
color: "#eee",
|
||||||
|
border: "1px solid #444",
|
||||||
|
borderRadius: "6px",
|
||||||
|
padding: "6px 8px",
|
||||||
|
fontSize: "12px",
|
||||||
|
});
|
||||||
|
aliasInput.addEventListener("input", () => {
|
||||||
|
setWidgetValue(node, findWidget(node, "alias"), aliasInput.value);
|
||||||
|
syncFormFromWidgets(node);
|
||||||
|
});
|
||||||
|
|
||||||
|
const descLabel = document.createElement("label");
|
||||||
|
descLabel.textContent = "Description";
|
||||||
|
Object.assign(descLabel.style, {
|
||||||
|
fontSize: "11px",
|
||||||
|
color: "#bbb",
|
||||||
|
});
|
||||||
|
|
||||||
|
const descInput = document.createElement("textarea");
|
||||||
|
descInput.placeholder = "Character description...";
|
||||||
|
Object.assign(descInput.style, {
|
||||||
|
width: "100%",
|
||||||
|
minHeight: "86px",
|
||||||
|
boxSizing: "border-box",
|
||||||
|
resize: "vertical",
|
||||||
|
background: "#1c1c1c",
|
||||||
|
color: "#eee",
|
||||||
|
border: "1px solid #444",
|
||||||
|
borderRadius: "6px",
|
||||||
|
padding: "8px",
|
||||||
|
fontSize: "12px",
|
||||||
|
lineHeight: "1.35",
|
||||||
|
});
|
||||||
|
descInput.addEventListener("input", () => {
|
||||||
|
setWidgetValue(node, findWidget(node, "description"), descInput.value);
|
||||||
|
syncFormFromWidgets(node);
|
||||||
|
});
|
||||||
|
|
||||||
|
actionsRow.appendChild(analyzeButton);
|
||||||
|
actionsRow.appendChild(settingsButton);
|
||||||
|
form.appendChild(aliasLabel);
|
||||||
|
form.appendChild(aliasInput);
|
||||||
|
form.appendChild(descLabel);
|
||||||
|
form.appendChild(descInput);
|
||||||
|
form.appendChild(actionsRow);
|
||||||
|
|
||||||
|
container.appendChild(previewRow);
|
||||||
|
container.appendChild(meta);
|
||||||
|
container.appendChild(form);
|
||||||
|
|
||||||
|
node._msrPreviewContainer = container;
|
||||||
|
node._msrPreviewRow = previewRow;
|
||||||
|
node._msrMeta = meta;
|
||||||
|
node._msrAliasInput = aliasInput;
|
||||||
|
node._msrDescriptionInput = descInput;
|
||||||
|
node._msrAnalyzeButton = analyzeButton;
|
||||||
|
return container;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function analyzeCharacterNode(node, buttonWidget) {
|
||||||
|
const descriptionWidget = findWidget(node, "description");
|
||||||
|
if (!descriptionWidget) return;
|
||||||
|
const providerWidget = findWidget(node, "analyze_provider");
|
||||||
|
const baseUrlWidget = findWidget(node, "analyze_base_url");
|
||||||
|
const modelWidget = findWidget(node, "analyze_model");
|
||||||
|
const promptWidget = findWidget(node, "analyze_prompt");
|
||||||
|
|
||||||
|
buttonWidget.label = "Analyzing...";
|
||||||
|
if (node._msrAnalyzeButton) {
|
||||||
|
node._msrAnalyzeButton.textContent = "Analyzing...";
|
||||||
|
node._msrAnalyzeButton.disabled = true;
|
||||||
|
}
|
||||||
|
node.setDirtyCanvas?.(true, true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const imageResults = await Promise.all([
|
||||||
|
imageInputToDataUrl(node, "image_1"),
|
||||||
|
imageInputToDataUrl(node, "image_2"),
|
||||||
|
]);
|
||||||
|
const imageDebug = imageResults.map((result) => result?.debug || null).filter(Boolean);
|
||||||
|
const images = (
|
||||||
|
imageResults.map((result) => result?.dataUrl || null)
|
||||||
|
).filter(Boolean);
|
||||||
|
|
||||||
|
if (!images.length) {
|
||||||
|
throw new Error("Connect at least one image input before analyzing.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await api.fetchApi("/ltx_director/analyze_character", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
provider: providerWidget?.value || "ollama",
|
||||||
|
base_url: baseUrlWidget?.value || "",
|
||||||
|
model: modelWidget?.value || "",
|
||||||
|
prompt: promptWidget?.value || "",
|
||||||
|
image_b64: images,
|
||||||
|
image_debug: imageDebug,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const result = await response.json();
|
||||||
|
if (result.status !== "success") {
|
||||||
|
console.warn("[MSRCharacter] analyze debug", {
|
||||||
|
sentImages: imageDebug,
|
||||||
|
response: result,
|
||||||
|
});
|
||||||
|
throw new Error(result.message || "Unknown analysis error");
|
||||||
|
}
|
||||||
|
|
||||||
|
const description = result.description || "";
|
||||||
|
setWidgetValue(node, descriptionWidget, description);
|
||||||
|
if (node._msrDescriptionInput) {
|
||||||
|
node._msrDescriptionInput.value = description;
|
||||||
|
}
|
||||||
|
syncFormFromWidgets(node);
|
||||||
|
refreshCharacterPreview(node);
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[MSRCharacter] analyze failed", error);
|
||||||
|
alert(`MSR Character analyze failed: ${error.message || error}`);
|
||||||
|
} finally {
|
||||||
|
buttonWidget.label = "Analyze with Ollama";
|
||||||
|
if (node._msrAnalyzeButton) {
|
||||||
|
node._msrAnalyzeButton.textContent = "Analyze";
|
||||||
|
node._msrAnalyzeButton.disabled = false;
|
||||||
|
}
|
||||||
|
node.setDirtyCanvas?.(true, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function hideNodeWidget(widget) {
|
||||||
|
if (!widget) return;
|
||||||
|
widget.hidden = true;
|
||||||
|
if (!widget.options) widget.options = {};
|
||||||
|
widget.options.hidden = true;
|
||||||
|
if (widget.element) {
|
||||||
|
widget.element.style.display = "none";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
app.registerExtension({
|
||||||
|
name: "Comfy.MSRCharacterCS",
|
||||||
|
async beforeRegisterNodeDef(nodeType, nodeData) {
|
||||||
|
if (nodeData.name === "MSRCharacterCS") {
|
||||||
|
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
|
||||||
|
nodeType.prototype.onNodeCreated = function () {
|
||||||
|
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
|
||||||
|
|
||||||
|
if (!(this.widgets || []).find((widget) => widget.msrAnalyze)) {
|
||||||
|
this.addWidget("button", "Analyze with Ollama", null, () => {
|
||||||
|
analyzeCharacterNode(this, (this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||||
|
}, { serialize: false });
|
||||||
|
const buttonWidget = this.widgets[this.widgets.length - 1];
|
||||||
|
buttonWidget.msrAnalyze = true;
|
||||||
|
buttonWidget.label = "Analyze with Ollama";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!findWidget(this, "analyze_provider")) {
|
||||||
|
this.addWidget("combo", "analyze_provider", "ollama", null, {
|
||||||
|
values: ["ollama", "lmstudio", "custom", "off"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!findWidget(this, "analyze_base_url")) {
|
||||||
|
this.addWidget("text", "analyze_base_url", "");
|
||||||
|
}
|
||||||
|
if (!findWidget(this, "analyze_model")) {
|
||||||
|
this.addWidget("text", "analyze_model", "");
|
||||||
|
}
|
||||||
|
if (!findWidget(this, "analyze_prompt")) {
|
||||||
|
this.addWidget("text", "analyze_prompt", DEFAULT_ANALYZE_PROMPT);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this._msrPreviewWidget) {
|
||||||
|
const previewContainer = buildCharacterUi(this);
|
||||||
|
this._msrPreviewWidget = this.addDOMWidget("msr_character_ui", "msr_character_ui", previewContainer, {
|
||||||
|
getValue: () => "",
|
||||||
|
setValue: () => {},
|
||||||
|
});
|
||||||
|
this._msrPreviewWidget.computeSize = () => [Math.max(10, (this.size?.[0] || 320) - 30), 260];
|
||||||
|
}
|
||||||
|
|
||||||
|
const originalConnectionsChange = this.onConnectionsChange;
|
||||||
|
this.onConnectionsChange = function () {
|
||||||
|
const result = originalConnectionsChange?.apply(this, arguments);
|
||||||
|
refreshCharacterPreview(this);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const originalConfigure = this.onConfigure;
|
||||||
|
this.onConfigure = function () {
|
||||||
|
const result = originalConfigure?.apply(this, arguments);
|
||||||
|
setTimeout(() => {
|
||||||
|
syncFormFromWidgets(this);
|
||||||
|
refreshCharacterPreview(this);
|
||||||
|
}, 0);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
["alias", "description", "analyze_provider", "analyze_base_url", "analyze_model", "analyze_prompt"].forEach((widgetName) => {
|
||||||
|
hideNodeWidget(findWidget(this, widgetName));
|
||||||
|
});
|
||||||
|
hideNodeWidget((this.widgets || []).find((widget) => widget.msrAnalyze));
|
||||||
|
|
||||||
|
this.size[0] = Math.max(this.size[0] || 0, 330);
|
||||||
|
syncFormFromWidgets(this);
|
||||||
|
refreshCharacterPreview(this);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nodeData.name === "MSRCharacterSetCS") {
|
||||||
|
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
|
||||||
|
nodeType.prototype.onNodeCreated = function () {
|
||||||
|
if (originalOnNodeCreated) originalOnNodeCreated.apply(this, arguments);
|
||||||
|
this.size[0] = Math.max(this.size[0] || 0, 260);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -1,8 +1,16 @@
|
|||||||
import folder_paths
|
import folder_paths
|
||||||
|
import numpy as np
|
||||||
import os
|
import os
|
||||||
import torch
|
import torch
|
||||||
import av
|
import av
|
||||||
|
|
||||||
|
DEFAULT_SAMPLE_RATE = 44100
|
||||||
|
DEFAULT_SILENCE_SECONDS = 1
|
||||||
|
|
||||||
|
|
||||||
|
def build_silence(sample_rate: int = DEFAULT_SAMPLE_RATE, seconds: int = DEFAULT_SILENCE_SECONDS, channels: int = 2) -> torch.Tensor:
|
||||||
|
return torch.zeros((channels, int(sample_rate * seconds)), dtype=torch.float32)
|
||||||
|
|
||||||
def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
|
def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
|
||||||
"""Convert audio to float 32 bits PCM format."""
|
"""Convert audio to float 32 bits PCM format."""
|
||||||
if wav.dtype.is_floating_point:
|
if wav.dtype.is_floating_point:
|
||||||
@@ -23,18 +31,17 @@ def load_audio_file(filepath: str) -> tuple[torch.Tensor, int]:
|
|||||||
sr = stream.codec_context.sample_rate
|
sr = stream.codec_context.sample_rate
|
||||||
n_channels = stream.channels
|
n_channels = stream.channels
|
||||||
|
|
||||||
frames = []
|
frame_arrays = []
|
||||||
for frame in af.decode(streams=stream.index):
|
for frame in af.decode(streams=stream.index):
|
||||||
buf = torch.from_numpy(frame.to_ndarray())
|
array = frame.to_ndarray()
|
||||||
if buf.shape[0] != n_channels:
|
if array.shape[0] != n_channels:
|
||||||
buf = buf.view(-1, n_channels).t()
|
array = array.reshape(-1, n_channels).T
|
||||||
|
frame_arrays.append(array)
|
||||||
|
|
||||||
frames.append(buf)
|
if not frame_arrays:
|
||||||
|
|
||||||
if not frames:
|
|
||||||
raise ValueError("No audio frames decoded.")
|
raise ValueError("No audio frames decoded.")
|
||||||
|
|
||||||
wav = torch.cat(frames, dim=1)
|
wav = torch.from_numpy(np.concatenate(frame_arrays, axis=1))
|
||||||
wav = f32_pcm(wav)
|
wav = f32_pcm(wav)
|
||||||
return wav, sr
|
return wav, sr
|
||||||
|
|
||||||
@@ -99,17 +106,16 @@ class LoadAudioUI:
|
|||||||
missing_info = audio if audio != "none" else "None selected"
|
missing_info = audio if audio != "none" else "None selected"
|
||||||
print(f"!!! [LoadAudioUI] Warning: Audio file '{missing_info}' not found. Outputting 1 second of silence.")
|
print(f"!!! [LoadAudioUI] Warning: Audio file '{missing_info}' not found. Outputting 1 second of silence.")
|
||||||
|
|
||||||
sample_rate = 44100
|
sample_rate = DEFAULT_SAMPLE_RATE
|
||||||
# 1 second of silence (stereo) -> shape [channels, time]
|
waveform = build_silence(sample_rate)
|
||||||
waveform = torch.zeros((2, 44100))
|
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
waveform, sample_rate = load_audio_file(audio_path)
|
waveform, sample_rate = load_audio_file(audio_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# If decoding fails for any reason, fallback to silence rather than crashing the workflow
|
# If decoding fails for any reason, fallback to silence rather than crashing the workflow
|
||||||
print(f"!!! [LoadAudioUI] Error decoding {audio}: {e}. Falling back to silence.")
|
print(f"!!! [LoadAudioUI] Error decoding {audio}: {e}. Falling back to silence.")
|
||||||
sample_rate = 44100
|
sample_rate = DEFAULT_SAMPLE_RATE
|
||||||
waveform = torch.zeros((2, 44100))
|
waveform = build_silence(sample_rate)
|
||||||
|
|
||||||
# Convert seconds to frames
|
# Convert seconds to frames
|
||||||
start_frame = int(start_time * sample_rate)
|
start_frame = int(start_time * sample_rate)
|
||||||
|
|||||||
1831
ltx_director.py
1831
ltx_director.py
File diff suppressed because it is too large
Load Diff
@@ -10,8 +10,7 @@ import comfy.utils
|
|||||||
import folder_paths
|
import folder_paths
|
||||||
import node_helpers
|
import node_helpers
|
||||||
from comfy_extras import nodes_lt
|
from comfy_extras import nodes_lt
|
||||||
from comfy_api.latest import io
|
from .ltx_director import _resize_image, _execute_comfy_node, _unpack
|
||||||
from .ltx_director import GuideData, MotionGuideData, _resize_image, _execute_comfy_node, _unpack
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -57,6 +56,18 @@ def _clone_noise_mask(latent, latent_image):
|
|||||||
device=latent_image.device,
|
device=latent_image.device,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _safe_json_loads(raw_value, default=None):
|
||||||
|
import json
|
||||||
|
|
||||||
|
if isinstance(raw_value, dict):
|
||||||
|
return raw_value
|
||||||
|
if not raw_value:
|
||||||
|
return {} if default is None else default
|
||||||
|
try:
|
||||||
|
return json.loads(raw_value)
|
||||||
|
except Exception:
|
||||||
|
return {} if default is None else default
|
||||||
|
|
||||||
def _resize_latent_spatial(latent_image, noise_mask, width, height, method):
|
def _resize_latent_spatial(latent_image, noise_mask, width, height, method):
|
||||||
b, c, f, h, w = latent_image.shape
|
b, c, f, h, w = latent_image.shape
|
||||||
if width == w and height == h:
|
if width == w and height == h:
|
||||||
@@ -78,6 +89,65 @@ def _ceil_to_multiple(value, multiple):
|
|||||||
multiple = max(1, int(multiple))
|
multiple = max(1, int(multiple))
|
||||||
return int(math.ceil(value / multiple) * multiple)
|
return int(math.ceil(value / multiple) * multiple)
|
||||||
|
|
||||||
|
def _normalize_resize_method_for_encode(resize_method):
|
||||||
|
if resize_method == "maintain aspect ratio":
|
||||||
|
return "pad"
|
||||||
|
return resize_method
|
||||||
|
|
||||||
|
def _append_latent_frames(latent_image, noise_mask, extra_frames, mask_fill=1.0):
|
||||||
|
extra_frames = int(max(0, extra_frames))
|
||||||
|
if extra_frames == 0:
|
||||||
|
return latent_image, noise_mask
|
||||||
|
|
||||||
|
batch, channels, _, height, width = latent_image.shape
|
||||||
|
latent_tail = torch.zeros(
|
||||||
|
(batch, channels, extra_frames, height, width),
|
||||||
|
dtype=latent_image.dtype,
|
||||||
|
device=latent_image.device,
|
||||||
|
)
|
||||||
|
latent_image = torch.cat([latent_image, latent_tail], dim=2)
|
||||||
|
|
||||||
|
mask_batch, mask_channels, _, mask_height, mask_width = noise_mask.shape
|
||||||
|
mask_tail = torch.full(
|
||||||
|
(mask_batch, mask_channels, extra_frames, mask_height, mask_width),
|
||||||
|
float(mask_fill),
|
||||||
|
dtype=noise_mask.dtype,
|
||||||
|
device=noise_mask.device,
|
||||||
|
)
|
||||||
|
noise_mask = torch.cat([noise_mask, mask_tail], dim=2)
|
||||||
|
return latent_image, noise_mask
|
||||||
|
|
||||||
|
def _encode_resized_video_frames(
|
||||||
|
vae,
|
||||||
|
frames,
|
||||||
|
target_width,
|
||||||
|
target_height,
|
||||||
|
resize_method,
|
||||||
|
time_scale_factor,
|
||||||
|
use_tiled_encode,
|
||||||
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
|
):
|
||||||
|
pixels = _resize_image(
|
||||||
|
frames,
|
||||||
|
target_width,
|
||||||
|
target_height,
|
||||||
|
_normalize_resize_method_for_encode(resize_method),
|
||||||
|
divisible_by=1,
|
||||||
|
)
|
||||||
|
num_frames_to_keep = ((pixels.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
||||||
|
encode_src = pixels[:num_frames_to_keep, :, :, :3]
|
||||||
|
if use_tiled_encode:
|
||||||
|
encoded = vae.encode_tiled(
|
||||||
|
encode_src,
|
||||||
|
tile_x=tile_size,
|
||||||
|
tile_y=tile_size,
|
||||||
|
overlap=tile_overlap,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
encoded = vae.encode(encode_src)
|
||||||
|
return pixels, encoded
|
||||||
|
|
||||||
def _snap_latent_to_downscale(latent_image, noise_mask, downscale_factor, method):
|
def _snap_latent_to_downscale(latent_image, noise_mask, downscale_factor, method):
|
||||||
factor = int(max(1, round(float(downscale_factor))))
|
factor = int(max(1, round(float(downscale_factor))))
|
||||||
if factor <= 1:
|
if factor <= 1:
|
||||||
@@ -307,11 +377,413 @@ class LTXDirectorGuide:
|
|||||||
RETURN_NAMES = ("positive", "negative", "latent", "model", "latent_downscale_factor")
|
RETURN_NAMES = ("positive", "negative", "latent", "model", "latent_downscale_factor")
|
||||||
FUNCTION = "execute"
|
FUNCTION = "execute"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _maybe_add_attention(cls, positive, negative, is_lora_active, tokens_added, guide_orig_shape, attention_strength):
|
||||||
|
if not is_lora_active:
|
||||||
|
return positive, negative
|
||||||
|
positive = _append_guide_attention_entry(
|
||||||
|
positive,
|
||||||
|
tokens_added,
|
||||||
|
guide_orig_shape,
|
||||||
|
attention_strength=attention_strength,
|
||||||
|
)
|
||||||
|
negative = _append_guide_attention_entry(
|
||||||
|
negative,
|
||||||
|
tokens_added,
|
||||||
|
guide_orig_shape,
|
||||||
|
attention_strength=attention_strength,
|
||||||
|
)
|
||||||
|
return positive, negative
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _apply_crop_count(cls, positive, negative, crop_frames):
|
||||||
|
crop_frames = max(0, int(crop_frames))
|
||||||
|
values = {"nghtdrp_guide_crop_latent_frames": crop_frames}
|
||||||
|
positive = node_helpers.conditioning_set_values(positive, values)
|
||||||
|
negative = node_helpers.conditioning_set_values(negative, values)
|
||||||
|
return positive, negative
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _load_cached_motion_video_frames(cls, video_cache, video_file, trim_start_frames, length_frames, director_fps, resample_mode):
|
||||||
|
cache_key = (
|
||||||
|
str(video_file),
|
||||||
|
int(trim_start_frames),
|
||||||
|
int(length_frames),
|
||||||
|
float(director_fps),
|
||||||
|
str(resample_mode),
|
||||||
|
)
|
||||||
|
cached = video_cache.get(cache_key)
|
||||||
|
if cached is None:
|
||||||
|
cached = _load_motion_video_frames(
|
||||||
|
video_file,
|
||||||
|
trim_start_frames=trim_start_frames,
|
||||||
|
length_frames=length_frames,
|
||||||
|
director_fps=director_fps,
|
||||||
|
resample_mode=resample_mode,
|
||||||
|
)
|
||||||
|
video_cache[cache_key] = cached
|
||||||
|
return cached
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _encode_image_guide(cls, vae, latent_width, latent_height, img_tensor, scale_factors, target_pix_w, target_pix_h, upscale_method):
|
||||||
|
_, height, width, _ = img_tensor.shape
|
||||||
|
if target_pix_w != width or target_pix_h != height:
|
||||||
|
img_nchw = img_tensor.permute(0, 3, 1, 2)
|
||||||
|
img_resized = comfy.utils.common_upscale(
|
||||||
|
img_nchw,
|
||||||
|
target_pix_w,
|
||||||
|
target_pix_h,
|
||||||
|
upscale_method,
|
||||||
|
"disabled",
|
||||||
|
)
|
||||||
|
img_tensor = img_resized.permute(0, 2, 3, 1)
|
||||||
|
return nodes_lt.LTXVAddGuide.encode(
|
||||||
|
vae,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
img_tensor,
|
||||||
|
scale_factors,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _apply_image_guides(
|
||||||
|
cls,
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
vae,
|
||||||
|
latent_image,
|
||||||
|
noise_mask,
|
||||||
|
images,
|
||||||
|
insert_frames,
|
||||||
|
strengths,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
latent_length,
|
||||||
|
scale_factors,
|
||||||
|
image_attention_strength,
|
||||||
|
is_lora_active,
|
||||||
|
upscale_method,
|
||||||
|
):
|
||||||
|
target_pix_w = int(latent_width * 32)
|
||||||
|
target_pix_h = int(latent_height * 32)
|
||||||
|
for idx, img_tensor in enumerate(images):
|
||||||
|
frame_value = insert_frames[idx] if idx < len(insert_frames) else 0
|
||||||
|
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
|
||||||
|
if strength <= 0.0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
image_pixels, guide_latent = cls._encode_image_guide(
|
||||||
|
vae,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
img_tensor,
|
||||||
|
scale_factors,
|
||||||
|
target_pix_w,
|
||||||
|
target_pix_h,
|
||||||
|
upscale_method,
|
||||||
|
)
|
||||||
|
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(
|
||||||
|
positive,
|
||||||
|
latent_length,
|
||||||
|
len(image_pixels),
|
||||||
|
int(frame_value),
|
||||||
|
scale_factors,
|
||||||
|
)
|
||||||
|
if latent_idx >= latent_length:
|
||||||
|
continue
|
||||||
|
|
||||||
|
max_frames = latent_length - latent_idx
|
||||||
|
if guide_latent.shape[2] > max_frames:
|
||||||
|
guide_latent = guide_latent[:, :, :max_frames]
|
||||||
|
|
||||||
|
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||||
|
guide_orig_shape = list(guide_latent.shape[2:])
|
||||||
|
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
frame_idx,
|
||||||
|
latent_image,
|
||||||
|
noise_mask,
|
||||||
|
guide_latent,
|
||||||
|
strength,
|
||||||
|
scale_factors,
|
||||||
|
)
|
||||||
|
positive, negative = cls._maybe_add_attention(
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
is_lora_active,
|
||||||
|
tokens_added,
|
||||||
|
guide_orig_shape,
|
||||||
|
image_attention_strength,
|
||||||
|
)
|
||||||
|
|
||||||
|
return positive, negative, latent_image, noise_mask
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _apply_motion_segment_guides(
|
||||||
|
cls,
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
vae,
|
||||||
|
latent_image,
|
||||||
|
noise_mask,
|
||||||
|
segments,
|
||||||
|
director_fps,
|
||||||
|
latent_length,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
time_scale_factor,
|
||||||
|
scale_factors,
|
||||||
|
latent_downscale_factor,
|
||||||
|
crop,
|
||||||
|
use_tiled_encode,
|
||||||
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
|
active_resize_method,
|
||||||
|
is_lora_active,
|
||||||
|
video_cache,
|
||||||
|
):
|
||||||
|
for seg in segments:
|
||||||
|
try:
|
||||||
|
video_file = seg.get("videoFile")
|
||||||
|
if not video_file:
|
||||||
|
continue
|
||||||
|
|
||||||
|
start_frame = int(seg.get("start", 0))
|
||||||
|
length_frames = int(seg.get("length", 1))
|
||||||
|
trim_start = int(seg.get("trimStart", 0))
|
||||||
|
video_strength = float(seg.get("videoStrength", 1.0))
|
||||||
|
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
|
||||||
|
if length_frames <= 0 or video_strength <= 0.0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
video_frames = cls._load_cached_motion_video_frames(
|
||||||
|
video_cache,
|
||||||
|
video_file,
|
||||||
|
trim_start,
|
||||||
|
length_frames,
|
||||||
|
director_fps,
|
||||||
|
seg.get("resampleMode", "nearest"),
|
||||||
|
)
|
||||||
|
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
||||||
|
video_frames = video_frames[:num_frames_to_keep]
|
||||||
|
causal_fix = start_frame == 0 or num_frames_to_keep == 1
|
||||||
|
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
|
||||||
|
|
||||||
|
_, guide_latent = _encode_video_iclora_guide(
|
||||||
|
vae,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
encode_frames,
|
||||||
|
scale_factors,
|
||||||
|
latent_downscale_factor,
|
||||||
|
crop,
|
||||||
|
use_tiled_encode,
|
||||||
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
|
resize_method=active_resize_method,
|
||||||
|
)
|
||||||
|
if not causal_fix:
|
||||||
|
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||||
|
|
||||||
|
frame_idx = start_frame
|
||||||
|
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
|
||||||
|
if latent_idx >= latent_length:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if start_frame > 0 and guide_latent.shape[2] > 1:
|
||||||
|
guide_latent = guide_latent[:, :, 1:, :, :]
|
||||||
|
frame_idx += time_scale_factor
|
||||||
|
latent_idx += 1
|
||||||
|
if latent_idx >= latent_length:
|
||||||
|
continue
|
||||||
|
|
||||||
|
max_frames = latent_length - latent_idx
|
||||||
|
if guide_latent.shape[2] > max_frames:
|
||||||
|
guide_latent = guide_latent[:, :, :max_frames]
|
||||||
|
|
||||||
|
guide_orig_shape = list(guide_latent.shape[2:])
|
||||||
|
batch, _, frames, height, width = guide_latent.shape
|
||||||
|
guide_mask = torch.ones(
|
||||||
|
(batch, 1, frames, height, width),
|
||||||
|
device=guide_latent.device,
|
||||||
|
dtype=guide_latent.dtype,
|
||||||
|
)
|
||||||
|
if start_frame > 0:
|
||||||
|
ramp_steps = [0.25, 0.65]
|
||||||
|
for i, step in enumerate(ramp_steps):
|
||||||
|
if i < frames:
|
||||||
|
guide_mask[:, :, i, :, :] = 1.0 + video_strength * (1.0 - step)
|
||||||
|
|
||||||
|
ldf = int(max(1, round(float(latent_downscale_factor))))
|
||||||
|
if ldf > 1:
|
||||||
|
dilated = _dilate_latent(
|
||||||
|
{"samples": guide_latent, "noise_mask": guide_mask},
|
||||||
|
horizontal_scale=ldf,
|
||||||
|
vertical_scale=ldf,
|
||||||
|
)
|
||||||
|
guide_latent = dilated["samples"]
|
||||||
|
guide_mask = dilated["noise_mask"]
|
||||||
|
|
||||||
|
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
||||||
|
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
frame_idx,
|
||||||
|
latent_image,
|
||||||
|
noise_mask,
|
||||||
|
guide_latent,
|
||||||
|
video_strength,
|
||||||
|
scale_factors,
|
||||||
|
guide_mask=guide_mask,
|
||||||
|
latent_downscale_factor=float(latent_downscale_factor),
|
||||||
|
causal_fix=causal_fix,
|
||||||
|
)
|
||||||
|
positive, negative = cls._maybe_add_attention(
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
is_lora_active,
|
||||||
|
tokens_added,
|
||||||
|
guide_orig_shape,
|
||||||
|
video_attention_strength,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
|
||||||
|
|
||||||
|
return positive, negative, latent_image, noise_mask
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _execute_retake_mode(
|
||||||
|
cls,
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
vae,
|
||||||
|
latent_image,
|
||||||
|
noise_mask,
|
||||||
|
guide_data,
|
||||||
|
tdata,
|
||||||
|
segments,
|
||||||
|
model,
|
||||||
|
latent_downscale_factor,
|
||||||
|
director_fps,
|
||||||
|
latent_length,
|
||||||
|
latent_width,
|
||||||
|
latent_height,
|
||||||
|
time_scale_factor,
|
||||||
|
active_resize_method,
|
||||||
|
use_tiled_encode,
|
||||||
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
|
is_empty_latent,
|
||||||
|
initial_latent_length,
|
||||||
|
video_cache,
|
||||||
|
):
|
||||||
|
print(
|
||||||
|
f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. "
|
||||||
|
f"is_empty_latent: {is_empty_latent}"
|
||||||
|
)
|
||||||
|
target_width = latent_width * 32
|
||||||
|
target_height = latent_height * 32
|
||||||
|
retake_start = int(tdata.get("retakeStart", 0))
|
||||||
|
retake_len = int(tdata.get("retakeLength", 0))
|
||||||
|
retake_strength = float(tdata.get("retakeStrength", 1.0))
|
||||||
|
|
||||||
|
start_frame = int(guide_data.get("start_frame", 0))
|
||||||
|
relative_start = max(0, retake_start - start_frame)
|
||||||
|
l_start = min(max(0, relative_start // time_scale_factor), latent_length)
|
||||||
|
l_end = min(
|
||||||
|
max(0, int(math.ceil((relative_start + retake_len) / time_scale_factor))),
|
||||||
|
latent_length,
|
||||||
|
)
|
||||||
|
|
||||||
|
need_base_video = is_empty_latent or l_start > 0 or l_end < latent_length
|
||||||
|
if not need_base_video:
|
||||||
|
print(
|
||||||
|
"[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. "
|
||||||
|
"Skipping base video loading and VAE encoding."
|
||||||
|
)
|
||||||
|
|
||||||
|
retake_vid_info = tdata.get("retakeVideo") or {}
|
||||||
|
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
|
||||||
|
if not video_file and not retake_vid_info and segments:
|
||||||
|
video_file = segments[0].get("videoFile", "")
|
||||||
|
|
||||||
|
if need_base_video and not video_file:
|
||||||
|
if retake_vid_info and not retake_vid_info.get("imageFile"):
|
||||||
|
raise ValueError(
|
||||||
|
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
|
||||||
|
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
"Retake Mode is active, but no base video has been selected on the timeline. "
|
||||||
|
"Please drag and drop or upload a base video on the timeline first."
|
||||||
|
)
|
||||||
|
|
||||||
|
if need_base_video and video_file:
|
||||||
|
try:
|
||||||
|
ltxv_length = (latent_length - 1) * time_scale_factor + 1
|
||||||
|
print(
|
||||||
|
f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} "
|
||||||
|
f"starting at frame {start_frame} for length {ltxv_length} at resolution "
|
||||||
|
f"{target_width}x{target_height}"
|
||||||
|
)
|
||||||
|
video_frames = cls._load_cached_motion_video_frames(
|
||||||
|
video_cache,
|
||||||
|
video_file,
|
||||||
|
start_frame,
|
||||||
|
ltxv_length,
|
||||||
|
director_fps,
|
||||||
|
"nearest",
|
||||||
|
)
|
||||||
|
_, base_latent = _encode_resized_video_frames(
|
||||||
|
vae,
|
||||||
|
video_frames,
|
||||||
|
target_width,
|
||||||
|
target_height,
|
||||||
|
active_resize_method,
|
||||||
|
time_scale_factor,
|
||||||
|
use_tiled_encode,
|
||||||
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
|
)
|
||||||
|
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
|
||||||
|
|
||||||
|
paste_len = min(base_latent.shape[2], latent_length)
|
||||||
|
if is_empty_latent:
|
||||||
|
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved "
|
||||||
|
f"regions (0-{l_start} and {l_end}-{paste_len})"
|
||||||
|
)
|
||||||
|
if l_start > 0:
|
||||||
|
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
|
||||||
|
if l_end < paste_len:
|
||||||
|
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
|
||||||
|
|
||||||
|
noise_mask = torch.zeros_like(noise_mask)
|
||||||
|
if l_end > l_start:
|
||||||
|
noise_mask[:, :, l_start:l_end] = retake_strength
|
||||||
|
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
|
||||||
|
|
||||||
|
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
||||||
|
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
|
||||||
|
return (
|
||||||
|
positive,
|
||||||
|
negative,
|
||||||
|
{"samples": latent_image, "noise_mask": noise_mask},
|
||||||
|
model,
|
||||||
|
float(latent_downscale_factor),
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def execute(cls, positive, negative, vae, latent, guide_data, motion_guide_data=None, model=None, ic_lora_name="None", ic_lora_strength=1.0, scale_by=1.0, upscale_method="bicubic", image_attention_strength=1.0, crop="center", auto_snap_ic_grid=True, use_tiled_encode=False, tile_size=256, tile_overlap=64, retake_mode=False, msr_strength=0.0):
|
def execute(cls, positive, negative, vae, latent, guide_data, motion_guide_data=None, model=None, ic_lora_name="None", ic_lora_strength=1.0, scale_by=1.0, upscale_method="bicubic", image_attention_strength=1.0, crop="center", auto_snap_ic_grid=True, use_tiled_encode=False, tile_size=256, tile_overlap=64, retake_mode=False, msr_strength=0.0):
|
||||||
motion_segments = (motion_guide_data or {}).get("segments", []) if motion_guide_data else []
|
motion_segments = (motion_guide_data or {}).get("segments", []) if motion_guide_data else []
|
||||||
image_guides_count = len(guide_data.get("images", [])) if guide_data else 0
|
image_guides_count = len(guide_data.get("images", [])) if guide_data else 0
|
||||||
print(f"[LTXDirectorGuide] execute started. motion_segments: {len(motion_segments)}, image_guides: {image_guides_count}, ic_lora_name: {ic_lora_name}, model connected: {model is not None}, retake_mode: {retake_mode}")
|
print(f"[LTXDirectorGuide] execute started. motion_segments: {len(motion_segments)}, image_guides: {image_guides_count}, ic_lora_name: {ic_lora_name}, model connected: {model is not None}, retake_mode: {retake_mode}")
|
||||||
|
video_cache = {}
|
||||||
|
|
||||||
active_resize_method = guide_data.get("resize_method") if guide_data else None
|
active_resize_method = guide_data.get("resize_method") if guide_data else None
|
||||||
if not active_resize_method:
|
if not active_resize_method:
|
||||||
@@ -336,24 +808,6 @@ class LTXDirectorGuide:
|
|||||||
if auto_snap_ic_grid and model is not None and ic_lora_name != "None":
|
if auto_snap_ic_grid and model is not None and ic_lora_name != "None":
|
||||||
latent_image, noise_mask = _snap_latent_to_downscale(latent_image, noise_mask, latent_downscale_factor, upscale_method)
|
latent_image, noise_mask = _snap_latent_to_downscale(latent_image, noise_mask, latent_downscale_factor, upscale_method)
|
||||||
|
|
||||||
# Ghost Mask: the Director kept the latent clean (so it stays length-matched with the audio
|
|
||||||
# latent) and asked us to pre-extend it here by one hidden frame per reference. The tail
|
|
||||||
# sits past the clean region; the reference images (in guide_data["images"], anchored at
|
|
||||||
# (clean + i)) are written into it by the loop below, then dropped downstream by Clean
|
|
||||||
# Latent Slice (length = clean_latent_frames).
|
|
||||||
_ghost_pre_extend = int(guide_data.get("ghost_pre_extend", 0)) if guide_data else 0
|
|
||||||
if _ghost_pre_extend > 0:
|
|
||||||
gb, gc, gf, gh, gw = latent_image.shape
|
|
||||||
latent_image = torch.cat(
|
|
||||||
[latent_image, torch.zeros((gb, gc, _ghost_pre_extend, gh, gw), dtype=latent_image.dtype, device=latent_image.device)],
|
|
||||||
dim=2,
|
|
||||||
)
|
|
||||||
mb, mc, mf, mh, mw = noise_mask.shape
|
|
||||||
noise_mask = torch.cat(
|
|
||||||
[noise_mask, torch.ones((mb, mc, _ghost_pre_extend, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
|
|
||||||
dim=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
_, _, latent_length, latent_height, latent_width = latent_image.shape
|
||||||
initial_latent_length = int(latent_length)
|
initial_latent_length = int(latent_length)
|
||||||
|
|
||||||
@@ -364,13 +818,7 @@ class LTXDirectorGuide:
|
|||||||
if msr is not None:
|
if msr is not None:
|
||||||
return cls._inject_msr(positive, negative, vae, latent_image, noise_mask, msr, model, latent_downscale_factor, msr_strength)
|
return cls._inject_msr(positive, negative, vae, latent_image, noise_mask, msr, model, latent_downscale_factor, msr_strength)
|
||||||
|
|
||||||
# Parse timeline JSON to see if retake mode is active in UI
|
tdata = _safe_json_loads(guide_data.get("timeline_data", "{}") if guide_data else "{}")
|
||||||
import json
|
|
||||||
timeline_data_str = guide_data.get("timeline_data", "{}") if guide_data else "{}"
|
|
||||||
try:
|
|
||||||
tdata = json.loads(timeline_data_str)
|
|
||||||
except Exception:
|
|
||||||
tdata = {}
|
|
||||||
|
|
||||||
is_retake_active = bool(retake_mode) or tdata.get("retakeMode", False)
|
is_retake_active = bool(retake_mode) or tdata.get("retakeMode", False)
|
||||||
is_empty_latent = (latent_image.abs().max().item() < 1e-5)
|
is_empty_latent = (latent_image.abs().max().item() < 1e-5)
|
||||||
@@ -392,111 +840,30 @@ class LTXDirectorGuide:
|
|||||||
# Load single base video, encode continuously, apply temporal mask.
|
# Load single base video, encode continuously, apply temporal mask.
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
if is_retake_active:
|
if is_retake_active:
|
||||||
print(f"[LTXDirectorGuide] Retake Mode active. Preserving base latent, masking selected regions. is_empty_latent: {is_empty_latent}")
|
return cls._execute_retake_mode(
|
||||||
target_width = latent_width * 32
|
positive,
|
||||||
target_height = latent_height * 32
|
negative,
|
||||||
|
vae,
|
||||||
# Calculate retake region latent indices first so we know what to copy/paste
|
latent_image,
|
||||||
retake_start = int(tdata.get("retakeStart", 0))
|
noise_mask,
|
||||||
retake_len = int(tdata.get("retakeLength", 0))
|
guide_data,
|
||||||
retake_strength = float(tdata.get("retakeStrength", 1.0))
|
tdata,
|
||||||
|
segments,
|
||||||
start_frame = int(guide_data.get("start_frame", 0))
|
model,
|
||||||
relative_start = max(0, retake_start - start_frame)
|
latent_downscale_factor,
|
||||||
|
director_fps,
|
||||||
l_start = relative_start // time_scale_factor
|
latent_length,
|
||||||
l_end = int(math.ceil((relative_start + retake_len) / time_scale_factor))
|
latent_width,
|
||||||
|
latent_height,
|
||||||
l_start = min(l_start, latent_length)
|
time_scale_factor,
|
||||||
l_end = min(l_end, latent_length)
|
active_resize_method,
|
||||||
|
use_tiled_encode,
|
||||||
# Stage 2 optimization: If the retake region covers the entire generation area,
|
tile_size,
|
||||||
# there are no preserved regions to copy over in Stage 2. We can bypass video loading/encoding.
|
tile_overlap,
|
||||||
need_base_video = True
|
is_empty_latent,
|
||||||
if not is_empty_latent and l_start == 0 and l_end >= latent_length:
|
initial_latent_length,
|
||||||
need_base_video = False
|
video_cache,
|
||||||
print("[LTXDirectorGuide] Stage 2: Retake region covers the entire generated range. Skipping base video loading and VAE encoding.")
|
)
|
||||||
|
|
||||||
# 1. Try to load and encode base video from timeline data
|
|
||||||
retake_vid_info = tdata.get("retakeVideo") or {}
|
|
||||||
video_file = retake_vid_info.get("imageFile", "") if isinstance(retake_vid_info, dict) else ""
|
|
||||||
|
|
||||||
# Fallback to first segment only if it exists and we have no retake video info at all (old workflows)
|
|
||||||
if not video_file and not retake_vid_info and len(segments) > 0:
|
|
||||||
video_file = segments[0].get("videoFile", "")
|
|
||||||
|
|
||||||
if need_base_video:
|
|
||||||
if not video_file:
|
|
||||||
if retake_vid_info and not retake_vid_info.get("imageFile"):
|
|
||||||
raise ValueError(
|
|
||||||
"Retake Mode is active, but the base video file upload is still in progress (or failed). "
|
|
||||||
"Please wait for the 'Uploading base video...' overlay on the timeline to disappear before queuing the prompt."
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
"Retake Mode is active, but no base video has been selected on the timeline. "
|
|
||||||
"Please drag and drop or upload a base video on the timeline first."
|
|
||||||
)
|
|
||||||
|
|
||||||
if video_file and need_base_video:
|
|
||||||
try:
|
|
||||||
print(f"[LTXDirectorGuide] Loading and encoding base video file: {video_file} starting at frame {start_frame} for length {ltxv_length} at resolution {target_width}x{target_height}")
|
|
||||||
video_frames = _load_motion_video_frames(
|
|
||||||
video_file, trim_start_frames=start_frame, length_frames=ltxv_length, director_fps=director_fps, resample_mode="nearest"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Retake base video must match the exact target latent shape.
|
|
||||||
# "maintain aspect ratio" doesn't pad, which causes shape mismatch in VAE encode.
|
|
||||||
# So we fallback "maintain aspect ratio" to "pad" for retake base video.
|
|
||||||
retake_resize_method = active_resize_method
|
|
||||||
if retake_resize_method == "maintain aspect ratio":
|
|
||||||
retake_resize_method = "pad"
|
|
||||||
|
|
||||||
pixels = _resize_image(video_frames, target_width, target_height, retake_resize_method, divisible_by=1)
|
|
||||||
|
|
||||||
num_clip_frames = pixels.shape[0]
|
|
||||||
num_frames_to_keep = ((num_clip_frames - 1) // time_scale_factor) * time_scale_factor + 1
|
|
||||||
encode_src = pixels[:num_frames_to_keep, :, :, :3]
|
|
||||||
|
|
||||||
if use_tiled_encode:
|
|
||||||
base_latent = vae.encode_tiled(encode_src, tile_x=tile_size, tile_y=tile_size, overlap=tile_overlap)
|
|
||||||
else:
|
|
||||||
base_latent = vae.encode(encode_src)
|
|
||||||
|
|
||||||
base_latent = base_latent.to(device=latent_image.device, dtype=latent_image.dtype)
|
|
||||||
|
|
||||||
# Copy to latent_image
|
|
||||||
paste_len = min(base_latent.shape[2], latent_length)
|
|
||||||
if is_empty_latent:
|
|
||||||
# Stage 1: Overwrite entire latent with base video VAE encode
|
|
||||||
latent_image[:, :, :paste_len] = base_latent[:, :, :paste_len]
|
|
||||||
else:
|
|
||||||
# Stage 2: Overwrite only preserved regions (before l_start and after l_end)
|
|
||||||
# leaving the generated retake region from Stage 1 untouched
|
|
||||||
print(f"[LTXDirectorGuide] Stage 2: Copying high-resolution base latent for preserved regions (0-{l_start} and {l_end}-{paste_len})")
|
|
||||||
if l_start > 0:
|
|
||||||
latent_image[:, :, :l_start] = base_latent[:, :, :l_start]
|
|
||||||
if l_end < paste_len:
|
|
||||||
latent_image[:, :, l_end:paste_len] = base_latent[:, :, l_end:paste_len]
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[LTXDirectorGuide] Failed to load/encode base video: {e}. Falling back to input latent.")
|
|
||||||
|
|
||||||
# 2. Build the temporal noise mask (0.0 = frozen, 1.0 = regenerate)
|
|
||||||
noise_mask = torch.zeros_like(noise_mask) # Initialize fully frozen
|
|
||||||
|
|
||||||
l_start = min(l_start, latent_length)
|
|
||||||
l_end = min(l_end, latent_length)
|
|
||||||
|
|
||||||
if l_end > l_start:
|
|
||||||
noise_mask[:, :, l_start:l_end] = retake_strength
|
|
||||||
|
|
||||||
print(f"[LTXDirectorGuide] noise_mask slice: {noise_mask[0, 0, :, 0, 0].tolist()}")
|
|
||||||
|
|
||||||
# In retake mode, skip normal mode processing entirely and return immediately!
|
|
||||||
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
|
||||||
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
|
||||||
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
|
||||||
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
|
|
||||||
|
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
# Standard Timeline Keyframe Guidance:
|
# Standard Timeline Keyframe Guidance:
|
||||||
@@ -507,121 +874,51 @@ class LTXDirectorGuide:
|
|||||||
if len(images) > 0 or len(segments) > 0:
|
if len(images) > 0 or len(segments) > 0:
|
||||||
print(f"[LTXDirectorGuide] Using Appended Keyframe Guidance. is_lora_active: {is_lora_active}")
|
print(f"[LTXDirectorGuide] Using Appended Keyframe Guidance. is_lora_active: {is_lora_active}")
|
||||||
|
|
||||||
# A. Process Image Guides
|
positive, negative, latent_image, noise_mask = cls._apply_image_guides(
|
||||||
for idx, img_tensor in enumerate(images):
|
positive,
|
||||||
f_idx = insert_frames[idx] if idx < len(insert_frames) else 0
|
negative,
|
||||||
strength = float(strengths[idx] if idx < len(strengths) else 1.0)
|
vae,
|
||||||
if strength <= 0.0:
|
latent_image,
|
||||||
continue
|
noise_mask,
|
||||||
|
images,
|
||||||
B_img, H_img, W_img, C_img = img_tensor.shape
|
insert_frames,
|
||||||
target_pix_w = int(latent_width * 32)
|
strengths,
|
||||||
target_pix_h = int(latent_height * 32)
|
latent_width,
|
||||||
if target_pix_w != W_img or target_pix_h != H_img:
|
latent_height,
|
||||||
img_nchw = img_tensor.permute(0, 3, 1, 2)
|
latent_length,
|
||||||
img_resized = comfy.utils.common_upscale(img_nchw, target_pix_w, target_pix_h, upscale_method, "disabled")
|
scale_factors,
|
||||||
img_tensor = img_resized.permute(0, 2, 3, 1)
|
image_attention_strength,
|
||||||
|
is_lora_active,
|
||||||
image_pixels, guide_latent = nodes_lt.LTXVAddGuide.encode(vae, latent_width, latent_height, img_tensor, scale_factors)
|
upscale_method,
|
||||||
frame_idx, latent_idx = nodes_lt.LTXVAddGuide.get_latent_index(positive, latent_length, len(image_pixels), int(f_idx), scale_factors)
|
)
|
||||||
|
positive, negative, latent_image, noise_mask = cls._apply_motion_segment_guides(
|
||||||
if latent_idx >= latent_length:
|
positive,
|
||||||
continue
|
negative,
|
||||||
|
vae,
|
||||||
max_frames = latent_length - latent_idx
|
latent_image,
|
||||||
if guide_latent.shape[2] > max_frames:
|
noise_mask,
|
||||||
guide_latent = guide_latent[:, :, :max_frames]
|
segments,
|
||||||
|
director_fps,
|
||||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
latent_length,
|
||||||
guide_orig_shape = list(guide_latent.shape[2:])
|
latent_width,
|
||||||
|
latent_height,
|
||||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
time_scale_factor,
|
||||||
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, strength, scale_factors
|
scale_factors,
|
||||||
)
|
latent_downscale_factor,
|
||||||
if is_lora_active:
|
crop,
|
||||||
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
|
use_tiled_encode,
|
||||||
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=image_attention_strength)
|
tile_size,
|
||||||
|
tile_overlap,
|
||||||
# B. Process Motion Video Segments
|
active_resize_method,
|
||||||
for seg in segments:
|
is_lora_active,
|
||||||
try:
|
video_cache,
|
||||||
video_file = seg.get("videoFile")
|
)
|
||||||
if not video_file:
|
|
||||||
continue
|
|
||||||
|
|
||||||
start_frame = int(seg.get("start", 0))
|
|
||||||
length_frames = int(seg.get("length", 1))
|
|
||||||
trim_start = int(seg.get("trimStart", 0))
|
|
||||||
video_strength = float(seg.get("videoStrength", 1.0))
|
|
||||||
video_attention_strength = float(seg.get("videoAttentionStrength", 0.65))
|
|
||||||
|
|
||||||
if length_frames <= 0 or video_strength <= 0.0:
|
|
||||||
continue
|
|
||||||
|
|
||||||
start_frame_aligned = start_frame
|
|
||||||
video_frames = _load_motion_video_frames(video_file, trim_start, length_frames, director_fps, seg.get("resampleMode", "nearest"))
|
|
||||||
|
|
||||||
num_frames_to_keep = ((video_frames.shape[0] - 1) // time_scale_factor) * time_scale_factor + 1
|
|
||||||
video_frames = video_frames[:num_frames_to_keep]
|
|
||||||
causal_fix = int(start_frame_aligned) == 0 or num_frames_to_keep == 1
|
|
||||||
encode_frames = video_frames if causal_fix else torch.cat([video_frames[:1], video_frames], dim=0)
|
|
||||||
|
|
||||||
_, guide_latent = _encode_video_iclora_guide(vae, latent_width, latent_height, encode_frames, scale_factors, latent_downscale_factor, crop, use_tiled_encode, tile_size, tile_overlap, resize_method=active_resize_method)
|
|
||||||
|
|
||||||
if not causal_fix:
|
|
||||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
|
||||||
|
|
||||||
frame_idx = start_frame_aligned
|
|
||||||
latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor if frame_idx > 0 else 0
|
|
||||||
|
|
||||||
if latent_idx >= latent_length:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if start_frame > 0 and guide_latent.shape[2] > 1:
|
|
||||||
guide_latent = guide_latent[:, :, 1:, :, :]
|
|
||||||
frame_idx += time_scale_factor
|
|
||||||
latent_idx += 1
|
|
||||||
if latent_idx >= latent_length:
|
|
||||||
continue
|
|
||||||
|
|
||||||
max_frames = latent_length - latent_idx
|
|
||||||
if guide_latent.shape[2] > max_frames:
|
|
||||||
guide_latent = guide_latent[:, :, :max_frames]
|
|
||||||
|
|
||||||
guide_orig_shape = list(guide_latent.shape[2:])
|
|
||||||
|
|
||||||
B_g, C_g, F_g, H_g, W_g = guide_latent.shape
|
|
||||||
guide_mask = torch.ones((B_g, 1, F_g, H_g, W_g), device=guide_latent.device, dtype=guide_latent.dtype)
|
|
||||||
|
|
||||||
if start_frame > 0:
|
|
||||||
ramp_steps = [0.25, 0.65]
|
|
||||||
for i, s in enumerate(ramp_steps):
|
|
||||||
if i < F_g:
|
|
||||||
guide_mask_val = 1.0 + video_strength * (1.0 - s)
|
|
||||||
guide_mask[:, :, i, :, :] = guide_mask_val
|
|
||||||
|
|
||||||
ldf = int(max(1, round(float(latent_downscale_factor))))
|
|
||||||
if ldf > 1:
|
|
||||||
dilated = _dilate_latent({"samples": guide_latent, "noise_mask": guide_mask}, horizontal_scale=ldf, vertical_scale=ldf)
|
|
||||||
guide_mask = dilated["noise_mask"]
|
|
||||||
guide_latent = dilated["samples"]
|
|
||||||
|
|
||||||
tokens_added = guide_latent.shape[2] * guide_latent.shape[3] * guide_latent.shape[4]
|
|
||||||
positive, negative, latent_image, noise_mask = nodes_lt.LTXVAddGuide.append_keyframe(
|
|
||||||
positive, negative, frame_idx, latent_image, noise_mask, guide_latent, video_strength, scale_factors, guide_mask=guide_mask, latent_downscale_factor=float(latent_downscale_factor), causal_fix=causal_fix
|
|
||||||
)
|
|
||||||
if is_lora_active:
|
|
||||||
positive = _append_guide_attention_entry(positive, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
|
|
||||||
negative = _append_guide_attention_entry(negative, tokens_added, guide_orig_shape, attention_strength=video_attention_strength)
|
|
||||||
except Exception as e:
|
|
||||||
raise RuntimeError(f"LTX Director Guide motion segment failed for {seg}: {e}") from e
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
print("[LTXDirectorGuide] No timeline guides present. Passing through.")
|
print("[LTXDirectorGuide] No timeline guides present. Passing through.")
|
||||||
|
|
||||||
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
exact_crop_frames = max(0, int(latent_image.shape[2]) - initial_latent_length)
|
||||||
positive = node_helpers.conditioning_set_values(positive, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
positive, negative = cls._apply_crop_count(positive, negative, exact_crop_frames)
|
||||||
negative = node_helpers.conditioning_set_values(negative, {"nghtdrp_guide_crop_latent_frames": exact_crop_frames})
|
|
||||||
|
|
||||||
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
|
return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask}, model, float(latent_downscale_factor))
|
||||||
|
|
||||||
@@ -655,16 +952,7 @@ class LTXDirectorGuide:
|
|||||||
# (which trims the slideshow's temporal footprint) lands on the true clean length.
|
# (which trims the slideshow's temporal footprint) lands on the true clean length.
|
||||||
pad_latents = max(0, prefix_latents - len(keyframes))
|
pad_latents = max(0, prefix_latents - len(keyframes))
|
||||||
if pad_latents > 0:
|
if pad_latents > 0:
|
||||||
B, C, F, H, W = latent_image.shape
|
latent_image, noise_mask = _append_latent_frames(latent_image, noise_mask, pad_latents, mask_fill=1.0)
|
||||||
latent_image = torch.cat(
|
|
||||||
[latent_image, torch.zeros((B, C, pad_latents, H, W), dtype=latent_image.dtype, device=latent_image.device)],
|
|
||||||
dim=2,
|
|
||||||
)
|
|
||||||
mb, mc, mf, mh, mw = noise_mask.shape
|
|
||||||
noise_mask = torch.cat(
|
|
||||||
[noise_mask, torch.ones((mb, mc, pad_latents, mh, mw), dtype=noise_mask.dtype, device=noise_mask.device)],
|
|
||||||
dim=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
latent = {"samples": latent_image, "noise_mask": noise_mask}
|
latent = {"samples": latent_image, "noise_mask": noise_mask}
|
||||||
|
|
||||||
|
|||||||
126
msr_character.py
Normal file
126
msr_character.py
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from comfy_api.latest import io
|
||||||
|
|
||||||
|
|
||||||
|
MSRCharacterData = io.Custom("MSR_CHARACTER")
|
||||||
|
MSRCharacterSetData = io.Custom("MSR_CHARACTER_SET")
|
||||||
|
|
||||||
|
|
||||||
|
def _compact_alias(alias: str) -> str:
|
||||||
|
if not alias:
|
||||||
|
return ""
|
||||||
|
alias = alias.strip()
|
||||||
|
if alias.startswith("@"):
|
||||||
|
alias = alias[1:]
|
||||||
|
return alias.strip()
|
||||||
|
|
||||||
|
|
||||||
|
class MSRCharacter(io.ComfyNode):
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return io.Schema(
|
||||||
|
node_id="MSRCharacterCS",
|
||||||
|
display_name="MSR Character DUMAS",
|
||||||
|
category="WhatDreamsCost DUMAS",
|
||||||
|
description=(
|
||||||
|
"Reusable MSR character payload. Accepts up to two reference images plus "
|
||||||
|
"a description and optional alias for @alias prompt replacement."
|
||||||
|
),
|
||||||
|
inputs=[
|
||||||
|
io.Image.Input(
|
||||||
|
"image_1",
|
||||||
|
optional=True,
|
||||||
|
tooltip="Optional first reference image for this character.",
|
||||||
|
),
|
||||||
|
io.Image.Input(
|
||||||
|
"image_2",
|
||||||
|
optional=True,
|
||||||
|
tooltip="Optional second reference image for this character.",
|
||||||
|
),
|
||||||
|
io.String.Input(
|
||||||
|
"description",
|
||||||
|
multiline=True,
|
||||||
|
default="",
|
||||||
|
optional=True,
|
||||||
|
tooltip="Manual or generated appearance description used for prompt replacement.",
|
||||||
|
),
|
||||||
|
io.String.Input(
|
||||||
|
"alias",
|
||||||
|
default="",
|
||||||
|
optional=True,
|
||||||
|
tooltip="Optional alias. Prompts can use @alias alongside @charN/@characterN.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
MSRCharacterData.Output(display_name="character"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def execute(cls, image_1=None, image_2=None, description="", alias="") -> io.NodeOutput:
|
||||||
|
images = []
|
||||||
|
if image_1 is not None:
|
||||||
|
images.append(image_1)
|
||||||
|
if image_2 is not None:
|
||||||
|
images.append(image_2)
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"images": images,
|
||||||
|
"description": (description or "").strip(),
|
||||||
|
"alias": _compact_alias(alias or ""),
|
||||||
|
}
|
||||||
|
return io.NodeOutput(payload)
|
||||||
|
|
||||||
|
|
||||||
|
class MSRCharacterSet(io.ComfyNode):
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return io.Schema(
|
||||||
|
node_id="MSRCharacterSetCS",
|
||||||
|
display_name="MSR Character Set DUMAS",
|
||||||
|
category="WhatDreamsCost DUMAS",
|
||||||
|
description="Bundles up to six MSR Character nodes into one character set for the director.",
|
||||||
|
inputs=[
|
||||||
|
MSRCharacterData.Input("character_1", optional=True),
|
||||||
|
MSRCharacterData.Input("character_2", optional=True),
|
||||||
|
MSRCharacterData.Input("character_3", optional=True),
|
||||||
|
MSRCharacterData.Input("character_4", optional=True),
|
||||||
|
MSRCharacterData.Input("character_5", optional=True),
|
||||||
|
MSRCharacterData.Input("character_6", optional=True),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
MSRCharacterSetData.Output(display_name="character_set"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def execute(
|
||||||
|
cls,
|
||||||
|
character_1=None,
|
||||||
|
character_2=None,
|
||||||
|
character_3=None,
|
||||||
|
character_4=None,
|
||||||
|
character_5=None,
|
||||||
|
character_6=None,
|
||||||
|
) -> io.NodeOutput:
|
||||||
|
characters = []
|
||||||
|
for item in (
|
||||||
|
character_1,
|
||||||
|
character_2,
|
||||||
|
character_3,
|
||||||
|
character_4,
|
||||||
|
character_5,
|
||||||
|
character_6,
|
||||||
|
):
|
||||||
|
if not item:
|
||||||
|
continue
|
||||||
|
characters.append(
|
||||||
|
{
|
||||||
|
"images": list(item.get("images") or []),
|
||||||
|
"description": (item.get("description") or "").strip(),
|
||||||
|
"alias": _compact_alias(item.get("alias") or ""),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return io.NodeOutput({"characters": characters})
|
||||||
@@ -1,9 +1,12 @@
|
|||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import types
|
import types
|
||||||
|
|
||||||
import comfy.ldm.modules.attention
|
import comfy.ldm.modules.attention
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||||
|
|
||||||
|
|
||||||
def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs):
|
def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs):
|
||||||
@@ -73,10 +76,10 @@ def _make_masked_override(prev_override):
|
|||||||
|
|
||||||
|
|
||||||
def debug_log(msg):
|
def debug_log(msg):
|
||||||
|
if not _DEBUG_PROMPT_RELAY:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
import os
|
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
|
||||||
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
|
||||||
with open(log_path, "a", encoding="utf-8") as f:
|
|
||||||
f.write(msg + "\n")
|
f.write(msg + "\n")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|||||||
120
prompt_relay.py
120
prompt_relay.py
@@ -1,21 +1,28 @@
|
|||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
|
import os
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
_DEBUG_PROMPT_RELAY = os.environ.get("PROMPT_RELAY_DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||||
|
_DEBUG_LOG_PATH = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
||||||
|
|
||||||
|
|
||||||
|
def _build_segment_cost(query_frames, seg, dtype):
|
||||||
|
local = seg["local_token_idx"]
|
||||||
|
distance = (query_frames[:, None] - seg["midpoint"]).abs()
|
||||||
|
cost = seg["strength"] * (torch.relu(distance - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
|
||||||
|
return local, cost.to(dtype)
|
||||||
|
|
||||||
|
|
||||||
def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame):
|
def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame):
|
||||||
"""Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing)."""
|
"""Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing)."""
|
||||||
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
|
offset = torch.zeros(Lq, Lk, device=device, dtype=dtype)
|
||||||
query_frames = torch.arange(Lq, device=device, dtype=torch.long) // tokens_per_frame
|
query_frames = (torch.arange(Lq, device=device, dtype=torch.float32) / float(tokens_per_frame)).floor()
|
||||||
|
|
||||||
for seg in q_token_idx:
|
for seg in q_token_idx:
|
||||||
local = seg["local_token_idx"].to(device=device)
|
local, cost = _build_segment_cost(query_frames, seg, offset.dtype)
|
||||||
d = (query_frames.float()[:, None] - seg["midpoint"]).abs()
|
offset[:, local.to(device=device)] = cost
|
||||||
strength = seg.get("strength", 1.0)
|
|
||||||
cost = strength * (torch.relu(d - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2)
|
|
||||||
offset[:, local] = cost.to(offset.dtype)
|
|
||||||
|
|
||||||
return offset
|
return offset
|
||||||
|
|
||||||
@@ -26,27 +33,27 @@ def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames
|
|||||||
query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq
|
query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq
|
||||||
|
|
||||||
for seg in q_token_idx:
|
for seg in q_token_idx:
|
||||||
local = seg["local_token_idx"].to(device=device)
|
|
||||||
d = (query_frames[:, None] - seg["midpoint"]).abs()
|
|
||||||
if is_audio:
|
if is_audio:
|
||||||
sigma_val = seg.get("sigma_audio", seg["sigma"])
|
active_seg = {
|
||||||
window_val = seg.get("window_audio", seg["window"])
|
"local_token_idx": seg["local_token_idx"],
|
||||||
strength_val = seg.get("strength_audio", 1.0)
|
"midpoint": seg["midpoint"],
|
||||||
|
"window": seg.get("window_audio", seg["window"]),
|
||||||
|
"sigma": seg.get("sigma_audio", seg["sigma"]),
|
||||||
|
"strength": seg.get("strength_audio", 1.0),
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
sigma_val = seg["sigma"]
|
active_seg = seg
|
||||||
window_val = seg["window"]
|
local, cost = _build_segment_cost(query_frames, active_seg, offset.dtype)
|
||||||
strength_val = seg.get("strength", 1.0)
|
offset[:, local.to(device=device)] = cost
|
||||||
cost = strength_val * (torch.relu(d - window_val) ** 2) / (2 * sigma_val ** 2)
|
|
||||||
offset[:, local] = cost.to(offset.dtype)
|
|
||||||
|
|
||||||
return offset
|
return offset
|
||||||
|
|
||||||
|
|
||||||
def debug_log(msg):
|
def debug_log(msg):
|
||||||
|
if not _DEBUG_PROMPT_RELAY:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
import os
|
with open(_DEBUG_LOG_PATH, "a", encoding="utf-8") as f:
|
||||||
log_path = os.path.join(os.path.dirname(__file__), "debug_prompt_relay.log")
|
|
||||||
with open(log_path, "a", encoding="utf-8") as f:
|
|
||||||
f.write(msg + "\n")
|
f.write(msg + "\n")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -61,6 +68,15 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
|
|||||||
"""
|
"""
|
||||||
cache = {}
|
cache = {}
|
||||||
max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1
|
max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1
|
||||||
|
latent_frames = max(1, int(latent_frames))
|
||||||
|
fallback_tokens_per_frame = max(1, int(fallback_tokens_per_frame))
|
||||||
|
|
||||||
|
def _get_video_query_layout(Lq, grid_sizes):
|
||||||
|
if grid_sizes is not None:
|
||||||
|
return int(grid_sizes[1]) * int(grid_sizes[2])
|
||||||
|
if Lq % latent_frames == 0:
|
||||||
|
return Lq // latent_frames
|
||||||
|
return fallback_tokens_per_frame
|
||||||
|
|
||||||
def mask_fn(Lq, Lk, dtype, device, transformer_options):
|
def mask_fn(Lq, Lk, dtype, device, transformer_options):
|
||||||
debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}")
|
debug_log(f"mask_fn check: Lq={Lq} Lk={Lk} max_token_idx={max_token_idx} cond_or_uncond={transformer_options.get('cond_or_uncond', [])}")
|
||||||
@@ -82,13 +98,7 @@ def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames):
|
|||||||
mode = "scaled"
|
mode = "scaled"
|
||||||
video_lq = -1
|
video_lq = -1
|
||||||
else:
|
else:
|
||||||
if grid_sizes is not None:
|
video_tpf = _get_video_query_layout(Lq, grid_sizes)
|
||||||
video_tpf = int(grid_sizes[1]) * int(grid_sizes[2])
|
|
||||||
else:
|
|
||||||
if Lq % latent_frames == 0:
|
|
||||||
video_tpf = Lq // latent_frames
|
|
||||||
else:
|
|
||||||
video_tpf = fallback_tokens_per_frame
|
|
||||||
video_lq = latent_frames * video_tpf
|
video_lq = latent_frames * video_tpf
|
||||||
|
|
||||||
# Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq
|
# Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq
|
||||||
@@ -189,6 +199,34 @@ def get_raw_tokenizer(clip):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_tokenizer_input_ids(raw_tokenizer, text):
|
||||||
|
tokenized = raw_tokenizer(text)
|
||||||
|
if isinstance(tokenized, dict) and "input_ids" in tokenized:
|
||||||
|
return tokenized["input_ids"]
|
||||||
|
if hasattr(tokenized, "input_ids"):
|
||||||
|
return tokenized.input_ids
|
||||||
|
if isinstance(tokenized, list):
|
||||||
|
return tokenized
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenized_length(raw_tokenizer, text, eos_adjustment):
|
||||||
|
return max(0, len(_get_tokenizer_input_ids(raw_tokenizer, text)) - eos_adjustment)
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenizer_has_eos(raw_tokenizer):
|
||||||
|
if getattr(raw_tokenizer, "add_eos", False):
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
ids = _get_tokenizer_input_ids(raw_tokenizer, "test")
|
||||||
|
if not ids:
|
||||||
|
return False
|
||||||
|
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
|
||||||
|
return (eos_id is not None and ids[-1] == eos_id) or ids[-1] == 1
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
|
def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
|
||||||
"""Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges).
|
"""Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges).
|
||||||
|
|
||||||
@@ -197,38 +235,14 @@ def map_token_indices(raw_tokenizer, global_prompt, local_prompts):
|
|||||||
prefixed_locals = [" " + lp for lp in local_prompts]
|
prefixed_locals = [" " + lp for lp in local_prompts]
|
||||||
full_prompt = global_prompt + "".join(prefixed_locals)
|
full_prompt = global_prompt + "".join(prefixed_locals)
|
||||||
|
|
||||||
# Detect if the tokenizer appends EOS dynamically
|
eos_adj = 1 if _tokenizer_has_eos(raw_tokenizer) else 0
|
||||||
has_eos = getattr(raw_tokenizer, "add_eos", False)
|
prev_len = _tokenized_length(raw_tokenizer, global_prompt, eos_adj)
|
||||||
if not has_eos:
|
|
||||||
try:
|
|
||||||
test_res = raw_tokenizer("test")
|
|
||||||
if isinstance(test_res, dict) and "input_ids" in test_res:
|
|
||||||
ids = test_res["input_ids"]
|
|
||||||
elif hasattr(test_res, "input_ids"):
|
|
||||||
ids = test_res.input_ids
|
|
||||||
elif isinstance(test_res, list):
|
|
||||||
ids = test_res
|
|
||||||
else:
|
|
||||||
ids = []
|
|
||||||
|
|
||||||
if ids:
|
|
||||||
eos_id = getattr(raw_tokenizer, "eos_token_id", None)
|
|
||||||
if eos_id is not None and ids[-1] == eos_id:
|
|
||||||
has_eos = True
|
|
||||||
elif ids[-1] == 1:
|
|
||||||
has_eos = True
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
eos_adj = 1 if has_eos else 0
|
|
||||||
|
|
||||||
prev_len = len(raw_tokenizer(global_prompt)["input_ids"]) - eos_adj
|
|
||||||
token_ranges = []
|
token_ranges = []
|
||||||
built = global_prompt
|
built = global_prompt
|
||||||
|
|
||||||
for plp in prefixed_locals:
|
for plp in prefixed_locals:
|
||||||
built += plp
|
built += plp
|
||||||
cur_len = len(raw_tokenizer(built)["input_ids"]) - eos_adj
|
cur_len = _tokenized_length(raw_tokenizer, built, eos_adj)
|
||||||
if cur_len <= prev_len:
|
if cur_len <= prev_len:
|
||||||
raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'")
|
raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'")
|
||||||
token_ranges.append((prev_len, cur_len))
|
token_ranges.append((prev_len, cur_len))
|
||||||
|
|||||||
117
timeline_nodes.py
Normal file
117
timeline_nodes.py
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from comfy_api.latest import io
|
||||||
|
|
||||||
|
|
||||||
|
TimelineSectionData = io.Custom("TIMELINE_SECTION")
|
||||||
|
TimelineSectionSetData = io.Custom("TIMELINE_SECTION_SET")
|
||||||
|
class TimelineSection(io.ComfyNode):
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return io.Schema(
|
||||||
|
node_id="TimelineSectionCS",
|
||||||
|
display_name="Timeline Section DUMAS",
|
||||||
|
category="WhatDreamsCost DUMAS",
|
||||||
|
description="Reusable timeline section with required text, optional image, and a duration in seconds.",
|
||||||
|
inputs=[
|
||||||
|
io.Image.Input(
|
||||||
|
"image",
|
||||||
|
optional=True,
|
||||||
|
tooltip="Optional image for this timeline section. When omitted, the section becomes a text-only block.",
|
||||||
|
),
|
||||||
|
io.String.Input(
|
||||||
|
"text",
|
||||||
|
multiline=True,
|
||||||
|
default="",
|
||||||
|
tooltip="Required text/prompt for this timeline section.",
|
||||||
|
),
|
||||||
|
io.Float.Input(
|
||||||
|
"duration",
|
||||||
|
default=1.0,
|
||||||
|
min=0.1,
|
||||||
|
max=1000.0,
|
||||||
|
step=0.01,
|
||||||
|
tooltip="Section duration in seconds.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
TimelineSectionData.Output(display_name="section"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def execute(cls, image=None, text="", duration=1.0) -> io.NodeOutput:
|
||||||
|
payload = {
|
||||||
|
"image": image,
|
||||||
|
"text": (text or "").strip(),
|
||||||
|
"duration": max(0.1, float(duration or 0.0)),
|
||||||
|
}
|
||||||
|
return io.NodeOutput(payload)
|
||||||
|
|
||||||
|
|
||||||
|
class TimelineSectionSet(io.ComfyNode):
|
||||||
|
@classmethod
|
||||||
|
def define_schema(cls):
|
||||||
|
return io.Schema(
|
||||||
|
node_id="TimelineSectionSetCS",
|
||||||
|
display_name="Timeline Section Set DUMAS",
|
||||||
|
category="WhatDreamsCost DUMAS",
|
||||||
|
description="Bundles up to ten Timeline Section nodes into one reusable section set.",
|
||||||
|
inputs=[
|
||||||
|
TimelineSectionData.Input("section_1", optional=True),
|
||||||
|
TimelineSectionData.Input("section_2", optional=True),
|
||||||
|
TimelineSectionData.Input("section_3", optional=True),
|
||||||
|
TimelineSectionData.Input("section_4", optional=True),
|
||||||
|
TimelineSectionData.Input("section_5", optional=True),
|
||||||
|
TimelineSectionData.Input("section_6", optional=True),
|
||||||
|
TimelineSectionData.Input("section_7", optional=True),
|
||||||
|
TimelineSectionData.Input("section_8", optional=True),
|
||||||
|
TimelineSectionData.Input("section_9", optional=True),
|
||||||
|
TimelineSectionData.Input("section_10", optional=True),
|
||||||
|
],
|
||||||
|
outputs=[
|
||||||
|
TimelineSectionSetData.Output(display_name="section_set"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def execute(
|
||||||
|
cls,
|
||||||
|
section_1=None,
|
||||||
|
section_2=None,
|
||||||
|
section_3=None,
|
||||||
|
section_4=None,
|
||||||
|
section_5=None,
|
||||||
|
section_6=None,
|
||||||
|
section_7=None,
|
||||||
|
section_8=None,
|
||||||
|
section_9=None,
|
||||||
|
section_10=None,
|
||||||
|
) -> io.NodeOutput:
|
||||||
|
sections = []
|
||||||
|
for item in (
|
||||||
|
section_1,
|
||||||
|
section_2,
|
||||||
|
section_3,
|
||||||
|
section_4,
|
||||||
|
section_5,
|
||||||
|
section_6,
|
||||||
|
section_7,
|
||||||
|
section_8,
|
||||||
|
section_9,
|
||||||
|
section_10,
|
||||||
|
):
|
||||||
|
if not item:
|
||||||
|
continue
|
||||||
|
text = (item.get("text") or "").strip()
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
sections.append(
|
||||||
|
{
|
||||||
|
"image": item.get("image"),
|
||||||
|
"text": text,
|
||||||
|
"duration": max(0.1, float(item.get("duration") or 0.0)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return io.NodeOutput({"sections": sections})
|
||||||
Reference in New Issue
Block a user