From 7b4e8865d616a9912318ad9343dbb5d9f6eb5130 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Thu, 16 Jul 2026 21:05:27 +0000 Subject: [PATCH] feat: add external timeline nodes --- __init__.py | 10 +++ js/ltx_director.js | 166 +++++++++++++++++++++++++++++++++++++++++++++ ltx_director.py | 71 ++++++++++++++++++- timeline_nodes.py | 136 +++++++++++++++++++++++++++++++++++++ 4 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 timeline_nodes.py diff --git a/__init__.py b/__init__.py index a8cf190..295f208 100644 --- a/__init__.py +++ b/__init__.py @@ -1,6 +1,7 @@ from .ltx_director import LTXDirector from .ltx_director_guide import LTXDirectorGuide, LTXDirectorCropGuides from .msr_character import MSRCharacter, MSRCharacterSet +from .timeline_nodes import TimelineSection, TimelineSectionSet, Timeline from comfy_api.latest import ComfyExtension, io from typing_extensions import override from .latent_slice import CleanLatentSlice @@ -11,6 +12,9 @@ class PromptRelay(ComfyExtension): return [ MSRCharacter, MSRCharacterSet, + TimelineSection, + TimelineSectionSet, + Timeline, LTXDirector, ] @@ -21,6 +25,9 @@ NODE_CLASS_MAPPINGS = { "LTXDirectorCS": LTXDirector, "MSRCharacterCS": MSRCharacter, "MSRCharacterSetCS": MSRCharacterSet, + "TimelineSectionCS": TimelineSection, + "TimelineSectionSetCS": TimelineSectionSet, + "TimelineCS": Timeline, "LTXDirectorGuideCS": LTXDirectorGuide, "LTXDirectorCropGuidesCS": LTXDirectorCropGuides, "CleanLatentSliceCS": CleanLatentSlice, @@ -30,6 +37,9 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LTXDirectorCS": "LTX Director DUMAS", "MSRCharacterCS": "MSR Character DUMAS", "MSRCharacterSetCS": "MSR Character Set DUMAS", + "TimelineSectionCS": "Timeline Section DUMAS", + "TimelineSectionSetCS": "Timeline Section Set DUMAS", + "TimelineCS": "Timeline DUMAS", "LTXDirectorGuideCS": "LTX Director Guide DUMAS", "LTXDirectorCropGuidesCS": "LTX Director Crop Guides DUMAS", "CleanLatentSliceCS": "Clean Latent Slice DUMAS", diff --git a/js/ltx_director.js b/js/ltx_director.js index 4eef2ef..d64fa7d 100644 --- a/js/ltx_director.js +++ b/js/ltx_director.js @@ -11503,6 +11503,170 @@ app.registerExtension({ const decimals = Number.isInteger(options.decimals) ? options.decimals : 3; return parseFloat(value.toFixed(decimals)); }; + this._findLinkedOriginNode = function (targetNode, inputName) { + const graph = app.graph; + const visited = new Set(); + const walkLink = (linkId) => { + if (!graph || linkId == null) return null; + if (visited.has(linkId)) return null; + visited.add(linkId); + + const link = graph.links?.[linkId]; + if (!link) return null; + + const originNode = graph.getNodeById?.(link.origin_id); + if (!originNode) return null; + + const comfyClass = originNode.comfyClass || originNode.type || ""; + if (/reroute/i.test(comfyClass)) { + const upstreamLink = originNode.inputs?.[0]?.link; + if (upstreamLink != null) { + return walkLink(upstreamLink); + } + } + + return originNode; + }; + + const input = targetNode?.inputs?.find(i => i.name === inputName); + return input?.link != null ? walkLink(input.link) : null; + }; + this._readNodeWidgetValue = function (node, widgetName, fallbackIndex = 0) { + if (!Array.isArray(node?.widgets)) return null; + const namedWidget = node.widgets.find(w => w?.name === widgetName); + if (namedWidget?.value != null) return namedWidget.value; + const fallbackWidget = node.widgets[fallbackIndex]; + return fallbackWidget?.value ?? null; + }; + this._resolveNodeLinkedValue = function (targetNode, inputName, options = {}) { + const originNode = self._findLinkedOriginNode(targetNode, inputName); + if (!originNode) return null; + const value = self._readNodeWidgetValue(originNode, inputName, 0); + if (value == null || !Number.isFinite(Number(value))) return null; + if (options.integer) return Math.round(Number(value)); + const decimals = Number.isInteger(options.decimals) ? options.decimals : 3; + return parseFloat(Number(value).toFixed(decimals)); + }; + this._extractLinkedImageSource = function (targetNode, inputName) { + const originNode = self._findLinkedOriginNode(targetNode, inputName); + if (!originNode) return null; + + const filenameValue = + self._readNodeWidgetValue(originNode, "image") ?? + self._readNodeWidgetValue(originNode, "filename") ?? + self._readNodeWidgetValue(originNode, "file") ?? + self._readNodeWidgetValue(originNode, "", 0); + if (typeof filenameValue !== "string" || !filenameValue.trim()) { + return null; + } + + const subfolderValue = + self._readNodeWidgetValue(originNode, "subfolder") ?? + self._readNodeWidgetValue(originNode, "folder"); + const filename = filenameValue.trim(); + const subfolder = typeof subfolderValue === "string" ? subfolderValue.trim() : ""; + const imageFile = subfolder ? `${subfolder}/${filename}` : filename; + const imageUrl = api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + return { imageFile, imageB64: imageUrl }; + }; + this._buildExternalTimelinePayload = function () { + const timelineNode = self._findLinkedOriginNode(self, "timeline"); + const comfyClass = timelineNode?.comfyClass || timelineNode?.type || ""; + if (!timelineNode || comfyClass !== "TimelineCS") { + return null; + } + + const rawDuration = + self._resolveNodeLinkedValue(timelineNode, "duration", { decimals: 3 }) ?? + parseFloat(self._readNodeWidgetValue(timelineNode, "duration") || 0); + const duration = Math.max(0.1, Number.isFinite(rawDuration) ? rawDuration : 5.0); + + const sectionSetNode = self._findLinkedOriginNode(timelineNode, "section_set"); + const setClass = sectionSetNode?.comfyClass || sectionSetNode?.type || ""; + const sections = []; + if (sectionSetNode && setClass === "TimelineSectionSetCS") { + for (let i = 1; i <= 5; i++) { + const sectionNode = self._findLinkedOriginNode(sectionSetNode, `section_${i}`); + const sectionClass = sectionNode?.comfyClass || sectionNode?.type || ""; + if (!sectionNode || sectionClass !== "TimelineSectionCS") continue; + + const text = String(self._readNodeWidgetValue(sectionNode, "text") || "").trim(); + if (!text) continue; + + const rawSectionDuration = + self._resolveNodeLinkedValue(sectionNode, "duration", { decimals: 3 }) ?? + parseFloat(self._readNodeWidgetValue(sectionNode, "duration") || 0); + const sectionDuration = Math.max(0.1, Number.isFinite(rawSectionDuration) ? rawSectionDuration : 1.0); + const imageSource = self._extractLinkedImageSource(sectionNode, "image"); + sections.push({ + text, + duration: sectionDuration, + imageFile: imageSource?.imageFile || "", + imageB64: imageSource?.imageB64 || "", + }); + } + } + + return { duration, sections }; + }; + this._syncTimelineFromLink = function () { + const payload = self._buildExternalTimelinePayload(); + const snapshot = JSON.stringify(payload || null); + if (snapshot === self._lastLinkedTimelineSnapshot) { + return; + } + self._lastLinkedTimelineSnapshot = snapshot; + if (!payload || !self._timelineEditor) { + return; + } + + const editor = self._timelineEditor; + const frameRate = editor.getFrameRate ? editor.getFrameRate() : (parseFloat(self.widgets?.find(w => w.name === "frame_rate")?.value) || 24); + let currentStart = 0; + const builtSegments = []; + + for (let i = 0; i < payload.sections.length; i++) { + const section = payload.sections[i]; + const segLength = Math.max(1, Math.round(section.duration * frameRate)); + const seg = { + id: `linked_section_${i}`, + start: currentStart, + length: segLength, + prompt: section.text, + type: section.imageFile ? "image" : "text", + }; + if (section.imageFile) { + seg.imageFile = section.imageFile; + seg.imageB64 = section.imageB64; + } + builtSegments.push(seg); + currentStart += segLength; + } + + const effectiveDuration = Math.max(payload.duration, currentStart / Math.max(frameRate, 1)); + const durationWidget = self.widgets?.find(w => w.name === "duration_seconds"); + if (durationWidget && durationWidget.value !== effectiveDuration) { + durationWidget.value = effectiveDuration; + if (durationWidget.callback) { + try { durationWidget.callback(effectiveDuration); } catch (e) {} + } + } + + editor.timeline.segments = builtSegments; + editor.selectionType = "image"; + editor.selectedIndex = builtSegments.length ? 0 : -1; + editor.loadMedia(); + editor.updateUIFromSelection(); + editor.commitChanges(); + editor.render(); + + if (self._ltxSettingsRefresh) { + try { self._ltxSettingsRefresh(); } catch (e) {} + } + if (self.setDirtyCanvas) { + self.setDirtyCanvas(true, true); + } + }; this._syncGlobalPromptFromLink = function () { const globalInput = self.inputs?.find(i => i.name === "global_prompt"); if (globalInput && globalInput.link !== null && globalInput.link !== undefined) { @@ -11611,6 +11775,7 @@ app.registerExtension({ self._syncGlobalPromptFromLink(); self._syncTimingFromLinks(); self._syncResolutionFromLinks?.(); + self._syncTimelineFromLink?.(); }; const origOnDrawForeground = this.onDrawForeground; @@ -11621,6 +11786,7 @@ app.registerExtension({ self._syncGlobalPromptFromLink(); self._syncTimingFromLinks(); self._syncResolutionFromLinks?.(); + self._syncTimelineFromLink?.(); }; // --- LTX Director settings panel (Stage 1: Resolution | Timing/Reference) --- diff --git a/ltx_director.py b/ltx_director.py index 4c3891d..a979e46 100644 --- a/ltx_director.py +++ b/ltx_director.py @@ -33,6 +33,7 @@ from .prompt_relay import ( from .patches import detect_model_type, apply_patches from .msr_character import MSRCharacterSetData +from .timeline_nodes import TimelineData log = logging.getLogger(__name__) @@ -166,6 +167,47 @@ def _load_image_reference(b64_or_url: str = "", filename: str = None, cache: dic return _empty_image_tensor() +def _build_runtime_timeline_payload(external_timeline, frame_rate: float) -> tuple[dict, str, str, float, int]: + tdata = { + "segments": [], + "motionSegments": [], + "audioSegments": [], + } + if not isinstance(external_timeline, dict): + return tdata, "", "", 0.0, 0 + + sections = external_timeline.get("sections") or [] + duration_seconds = max(0.1, float(external_timeline.get("duration") or 0.0)) + duration_frames = max(1, int(round(duration_seconds * frame_rate))) + current_start = 0 + local_prompts = [] + segment_lengths = [] + + for index, section in enumerate(sections): + if not isinstance(section, dict): + continue + prompt = (section.get("text") or "").strip() + if not prompt: + continue + section_seconds = max(0.1, float(section.get("duration") or 0.0)) + section_frames = max(1, int(round(section_seconds * frame_rate))) + seg = { + "id": f"external_{index}", + "start": current_start, + "length": section_frames, + "prompt": prompt, + "type": "image" if section.get("image") is not None else "text", + } + if section.get("image") is not None: + seg["runtime_image"] = section.get("image") + tdata["segments"].append(seg) + local_prompts.append(prompt) + segment_lengths.append(str(section_frames)) + current_start += section_frames + + return tdata, " | ".join(local_prompts), ",".join(segment_lengths), duration_seconds, duration_frames + + def _compute_signal_peaks(samples: np.ndarray, num_peaks: int = PEAK_BUCKETS) -> list[float]: if samples.size == 0: return [0.0] * num_peaks @@ -1048,6 +1090,8 @@ async def ltx_director_upload_chunk(request): def _load_image_tensor(seg: dict, cache: dict | None = None, input_dir: str = None) -> torch.Tensor: """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" + if seg.get("runtime_image") is not None: + return seg.get("runtime_image") return _load_image_reference( b64_or_url=seg.get("imageB64", ""), filename=seg.get("imageFile"), @@ -2023,6 +2067,11 @@ class LTXDirector(io.ComfyNode): optional=True, tooltip="Optional external MSR Character Set. Replaces the legacy inline character panel in the director UI.", ), + TimelineData.Input( + "timeline", + optional=True, + tooltip="Optional external Timeline DUMAS payload. When connected, it populates the main timeline track and duration.", + ), io.Float.Input( "start", force_input=True, optional=True, default=0.0, tooltip="Automation (connection-only). Start time in SECONDS. Overrides the panel Start when connected.", @@ -2058,7 +2107,7 @@ class LTXDirector(io.ComfyNode): custom_width=768, custom_height=512, resize_method="maintain aspect ratio", divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None, use_custom_audio=False, inpaint_audio=True, use_custom_motion=True, override_audio=False, - vae=None, reference_strength=1.0, ref_images=None, character_set=None, + vae=None, reference_strength=1.0, ref_images=None, character_set=None, timeline=None, start=None, end=None, duration=None) -> io.NodeOutput: input_dir = folder_paths.get_input_directory() image_cache = {} @@ -2075,6 +2124,26 @@ class LTXDirector(io.ComfyNode): processed_image_cache[cache_key] = processed return processed + if timeline is not None: + external_tdata, external_local_prompts, external_segment_lengths, external_duration_seconds, external_duration_frames = _build_runtime_timeline_payload( + timeline, + float(frame_rate) if frame_rate else 24.0, + ) + try: + existing_tdata = _safe_json_loads(timeline_data) + except Exception: + existing_tdata = {} + merged_tdata = dict(existing_tdata or {}) + merged_tdata["segments"] = external_tdata["segments"] + merged_tdata.setdefault("motionSegments", existing_tdata.get("motionSegments", [])) + merged_tdata.setdefault("audioSegments", existing_tdata.get("audioSegments", [])) + timeline_data = json.dumps(merged_tdata) + local_prompts = external_local_prompts + segment_lengths = external_segment_lengths + if duration is None: + duration_seconds = external_duration_seconds + duration_frames = external_duration_frames + # Parse timeline data try: tdata = _safe_json_loads(timeline_data) diff --git a/timeline_nodes.py b/timeline_nodes.py new file mode 100644 index 0000000..22ed227 --- /dev/null +++ b/timeline_nodes.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from comfy_api.latest import io + + +TimelineSectionData = io.Custom("TIMELINE_SECTION") +TimelineSectionSetData = io.Custom("TIMELINE_SECTION_SET") +TimelineData = io.Custom("TIMELINE") + + +class TimelineSection(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TimelineSectionCS", + display_name="Timeline Section DUMAS", + category="WhatDreamsCost DUMAS", + description="Reusable timeline section with required text, optional image, and a duration in seconds.", + inputs=[ + io.Image.Input( + "image", + optional=True, + tooltip="Optional image for this timeline section. When omitted, the section becomes a text-only block.", + ), + io.String.Input( + "text", + multiline=True, + default="", + tooltip="Required text/prompt for this timeline section.", + ), + io.Float.Input( + "duration", + default=1.0, + min=0.1, + max=1000.0, + step=0.01, + tooltip="Section duration in seconds.", + ), + ], + outputs=[ + TimelineSectionData.Output(display_name="section"), + ], + ) + + @classmethod + def execute(cls, image=None, text="", duration=1.0) -> io.NodeOutput: + payload = { + "image": image, + "text": (text or "").strip(), + "duration": max(0.1, float(duration or 0.0)), + } + return io.NodeOutput(payload) + + +class TimelineSectionSet(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TimelineSectionSetCS", + display_name="Timeline Section Set DUMAS", + category="WhatDreamsCost DUMAS", + description="Bundles up to five Timeline Section nodes into one reusable section set.", + inputs=[ + TimelineSectionData.Input("section_1", optional=True), + TimelineSectionData.Input("section_2", optional=True), + TimelineSectionData.Input("section_3", optional=True), + TimelineSectionData.Input("section_4", optional=True), + TimelineSectionData.Input("section_5", optional=True), + ], + outputs=[ + TimelineSectionSetData.Output(display_name="section_set"), + ], + ) + + @classmethod + def execute( + cls, + section_1=None, + section_2=None, + section_3=None, + section_4=None, + section_5=None, + ) -> io.NodeOutput: + sections = [] + for item in (section_1, section_2, section_3, section_4, section_5): + if not item: + continue + text = (item.get("text") or "").strip() + if not text: + continue + sections.append( + { + "image": item.get("image"), + "text": text, + "duration": max(0.1, float(item.get("duration") or 0.0)), + } + ) + + return io.NodeOutput({"sections": sections}) + + +class Timeline(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TimelineCS", + display_name="Timeline DUMAS", + category="WhatDreamsCost DUMAS", + description="Builds an external timeline payload for LTX Director from a section set and total duration.", + inputs=[ + TimelineSectionSetData.Input( + "section_set", + optional=True, + tooltip="Optional section set. When omitted, the timeline payload contains no sections.", + ), + io.Float.Input( + "duration", + default=5.0, + min=0.1, + max=1000.0, + step=0.01, + tooltip="Total timeline duration in seconds.", + ), + ], + outputs=[ + TimelineData.Output(display_name="timeline"), + ], + ) + + @classmethod + def execute(cls, section_set=None, duration=5.0) -> io.NodeOutput: + payload = { + "duration": max(0.1, float(duration or 0.0)), + "sections": list((section_set or {}).get("sections") or []), + } + return io.NodeOutput(payload)