diff --git a/__init__.py b/__init__.py index a9472f1..24fd466 100644 --- a/__init__.py +++ b/__init__.py @@ -3,26 +3,43 @@ from .multi_image_loader import MultiImageLoader from .ltx_sequencer import LTXSequencer from .speech_length_calculator import SpeechLengthCalculator from .load_audio_ui import LoadAudioUI -from .load_video_ui import LoadVideoUI # Imported new node +from .load_video_ui import LoadVideoUI +from .ltx_director import LTXDirector +from .ltx_director_guide import LTXDirectorGuide +from comfy_api.latest import ComfyExtension, io +from typing_extensions import override -# Register the node classes +class PromptRelay(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + LTXDirector, + LTXDirectorGuide + ] + +async def comfy_entrypoint() -> PromptRelay: + return PromptRelay() + NODE_CLASS_MAPPINGS = { "LTXKeyframer": LTXKeyframer, "MultiImageLoader": MultiImageLoader, "LTXSequencer": LTXSequencer, "SpeechLengthCalculator": SpeechLengthCalculator, "LoadAudioUI": LoadAudioUI, - "LoadVideoUI": LoadVideoUI # Registered new node + "LoadVideoUI": LoadVideoUI, + "LTXDirector": LTXDirector, + "LTXDirectorGuide": LTXDirectorGuide, } -# Provide clean display names for the ComfyUI interface NODE_DISPLAY_NAME_MAPPINGS = { "LTXKeyframer": "LTX Keyframer", "MultiImageLoader": "Multi Image Loader", "LTXSequencer": "LTX Sequencer", "SpeechLengthCalculator": "Speech Length Calculator", "LoadAudioUI": "Load Audio UI", - "LoadVideoUI": "Load Video UI" # Registered display name + "LoadVideoUI": "Load Video UI", + "LTXDirector": "LTX Director", + "LTXDirectorGuide": "LTX Director Guide", } WEB_DIRECTORY = "./js" diff --git a/example_workflows/LTX Director Example Workflow Compa Subgraph.json b/example_workflows/LTX Director Example Workflow Compa Subgraph.json new file mode 100644 index 0000000..79b5367 --- /dev/null +++ b/example_workflows/LTX Director Example Workflow Compa Subgraph.json @@ -0,0 +1,3638 @@ +{ + "id": "c92620ea-60b4-4f41-a78f-621383a62c2a", + "revision": 0, + "last_node_id": 96, + "last_link_id": 249, + "nodes": [ + { + "id": 46, + "type": "LTXDirector", + "pos": [ + 1860, + 3970 + ], + "size": [ + 1120, + 777 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 212 + }, + { + "name": "clip", + "type": "CLIP", + "link": 213 + }, + { + "name": "audio_vae", + "shape": 7, + "type": "VAE", + "link": 215 + }, + { + "name": "optional_latent", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 220, + 239 + ] + }, + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 225 + ] + }, + { + "name": "video_latent", + "type": "LATENT", + "links": [ + 223 + ] + }, + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 221 + ] + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "links": [ + 224, + 238 + ] + }, + { + "name": "frame_rate", + "type": "FLOAT", + "links": [ + 226, + 248 + ] + }, + { + "name": "combined_audio", + "type": "AUDIO", + "links": [] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirector", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + "", + 120, + 5, + "{\"segments\":[],\"audioSegments\":[]}", + "", + "", + 0.001, + "", + false, + 24, + "seconds", + 0, + 0, + "maintain aspect ratio", + 32, + 18, + "" + ] + }, + { + "id": 30, + "type": "SaveVideo", + "pos": [ + 3230, + 3970 + ], + "size": [ + 690, + 530 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 249 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/LTX_Director", + "auto", + "auto" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 92, + "type": "MarkdownNote", + "pos": [ + 1430, + 4220 + ], + "size": [ + 400, + 530 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "FAQ", + "properties": {}, + "widgets_values": [ + "# FAQ\n\n**How do I use this??**\n\nI'll be posting tutorials on my YouTube channel soon!\n\n\n**Where do I download these models??**\n\nThere is a good list of models and download links here: \n\nAlso the latest v1.1 spatial upscaler can be found here: \n\nThe tiny vae (taeltx2_3.safetensors) is found here:\n\n\n**I'm getting errors when running this workflow!**\n\nDouble check that you have all of the correct models loaded. This is using the newest version of 2x upscaler (v1.1). You can change it to the older one under the Model Loader.\n\n**How can I support you??**\n\nHelp me get a job. I'm unemployed and literally broke. (seriously, i'll even scrub toilets for minimum wage)\n\nOr follow me on my socials :)\n\n\n\n\n\n\n" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 94, + "type": "4b017e47-bf16-441b-ad03-edce978b3be3", + "pos": [ + 3000, + 3970 + ], + "size": [ + 210, + 260 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 220 + }, + { + "name": "audio_latent", + "type": "LATENT", + "link": 221 + }, + { + "name": "vae", + "type": "VAE", + "link": 222 + }, + { + "label": "video_latent", + "name": "latent", + "type": "LATENT", + "link": 223 + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 224 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 225 + }, + { + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 226 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 234 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 235 + ] + }, + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 240 + ] + }, + { + "name": "video_latent", + "type": "LATENT", + "links": [ + 236 + ] + }, + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 241 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "5", + "frame_rate" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.3" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 95, + "type": "7cf6430b-4784-4af1-8cde-9268e9465c62", + "pos": [ + 3000, + 4290 + ], + "size": [ + 210, + 270 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 234 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 235 + }, + { + "label": "video_latent", + "name": "latent", + "type": "LATENT", + "link": 236 + }, + { + "label": "video_vae", + "name": "vae", + "type": "VAE", + "link": 237 + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 238 + }, + { + "name": "model", + "type": "MODEL", + "link": 239 + }, + { + "name": "noise", + "type": "NOISE", + "link": 240 + }, + { + "name": "audio_latent", + "type": "LATENT", + "link": 241 + } + ], + "outputs": [ + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 244 + ] + }, + { + "name": "latent", + "type": "LATENT", + "links": [ + 246 + ] + } + ], + "properties": { + "proxyWidgets": [], + "cnr_id": "comfy-core", + "ver": "0.19.3" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 96, + "type": "739520a4-718f-490f-87d1-34ffbca98e3f", + "pos": [ + 3000, + 4620 + ], + "size": [ + 210, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "label": "audio_latent", + "name": "samples", + "type": "LATENT", + "link": 244 + }, + { + "label": "audio_vae", + "name": "audio_vae", + "type": "VAE", + "link": 245 + }, + { + "label": "video_latent", + "name": "samples_1", + "type": "LATENT", + "link": 246 + }, + { + "label": "video_vae", + "name": "vae", + "type": "VAE", + "link": 247 + }, + { + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 248 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 249 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "17", + "fps" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.3" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 93, + "type": "dc399e64-01cd-40b2-9337-972428cc66de", + "pos": [ + 1430, + 3970 + ], + "size": [ + 400, + 190 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "model", + "name": "MODEL", + "type": "MODEL", + "links": [ + 212 + ] + }, + { + "label": "clip", + "name": "CLIP", + "type": "CLIP", + "links": [ + 213 + ] + }, + { + "label": "audio_vae", + "name": "VAE", + "type": "VAE", + "links": [ + 215, + 245 + ] + }, + { + "label": "video_vae", + "name": "VAE_1", + "type": "VAE", + "links": [ + 222, + 237, + 247 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "77", + "ckpt_name" + ], + [ + "84", + "clip_name1" + ], + [ + "4", + "vae_name" + ], + [ + "3", + "vae_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.3" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + } + ], + "links": [ + [ + 212, + 93, + 0, + 46, + 0, + "MODEL" + ], + [ + 213, + 93, + 1, + 46, + 1, + "CLIP" + ], + [ + 215, + 93, + 2, + 46, + 2, + "VAE" + ], + [ + 220, + 46, + 0, + 94, + 0, + "MODEL" + ], + [ + 221, + 46, + 3, + 94, + 1, + "LATENT" + ], + [ + 222, + 93, + 3, + 94, + 2, + "VAE" + ], + [ + 223, + 46, + 2, + 94, + 3, + "LATENT" + ], + [ + 224, + 46, + 4, + 94, + 4, + "GUIDE_DATA" + ], + [ + 225, + 46, + 1, + 94, + 5, + "CONDITIONING" + ], + [ + 226, + 46, + 5, + 94, + 6, + "FLOAT" + ], + [ + 234, + 94, + 0, + 95, + 0, + "CONDITIONING" + ], + [ + 235, + 94, + 1, + 95, + 1, + "CONDITIONING" + ], + [ + 236, + 94, + 3, + 95, + 2, + "LATENT" + ], + [ + 237, + 93, + 3, + 95, + 3, + "VAE" + ], + [ + 238, + 46, + 4, + 95, + 4, + "GUIDE_DATA" + ], + [ + 239, + 46, + 0, + 95, + 5, + "MODEL" + ], + [ + 240, + 94, + 2, + 95, + 6, + "NOISE" + ], + [ + 241, + 94, + 4, + 95, + 7, + "LATENT" + ], + [ + 244, + 95, + 0, + 96, + 0, + "LATENT" + ], + [ + 245, + 93, + 2, + 96, + 1, + "VAE" + ], + [ + 246, + 95, + 1, + 96, + 2, + "LATENT" + ], + [ + 247, + 93, + 3, + 96, + 3, + "VAE" + ], + [ + 248, + 46, + 5, + 96, + 4, + "FLOAT" + ], + [ + 249, + 96, + 0, + 30, + 0, + "VIDEO" + ] + ], + "groups": [], + "definitions": { + "subgraphs": [ + { + "id": "dc399e64-01cd-40b2-9337-972428cc66de", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 96, + "lastLinkId": 249, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Model Loader", + "inputNode": { + "id": -10, + "bounding": [ + 1060, + 4265, + 120, + 40 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1870, + 4225, + 120, + 120 + ] + }, + "inputs": [], + "outputs": [ + { + "id": "68f91346-0127-4451-a92f-5cfb56fc7f02", + "name": "MODEL", + "type": "MODEL", + "linkIds": [ + 199 + ], + "localized_name": "MODEL", + "label": "model", + "pos": [ + 1890, + 4245 + ] + }, + { + "id": "d50bc485-29c2-44a9-9653-fbb738989bdc", + "name": "CLIP", + "type": "CLIP", + "linkIds": [ + 211 + ], + "localized_name": "CLIP", + "label": "clip", + "pos": [ + 1890, + 4265 + ] + }, + { + "id": "cfbbcf84-8477-40b1-a7ba-318c8d3b03a0", + "name": "VAE", + "type": "VAE", + "linkIds": [ + 58, + 58 + ], + "localized_name": "VAE", + "label": "audio_vae", + "pos": [ + 1890, + 4285 + ] + }, + { + "id": "815c9c85-34bc-49fc-8161-aca22b170346", + "name": "VAE_1", + "type": "VAE", + "linkIds": [ + 16, + 16, + 16, + 16 + ], + "localized_name": "VAE_1", + "label": "video_vae", + "pos": [ + 1890, + 4305 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 77, + "type": "CheckpointLoaderSimple", + "pos": [ + 1240, + 4020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "label": "model", + "localized_name": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 210 + ] + }, + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": null + }, + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.17.2", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "ltx-2.3-22b-dev-fp8.safetensors" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 80, + "type": "LoraLoaderModelOnly", + "pos": [ + 1530, + 4160 + ], + "size": [ + 280, + 90 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 197 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 199 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.75", + "Node name for S&R": "LoraLoaderModelOnly", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "models": [ + { + "name": "ltx-2.3-22b-distilled-lora-384.safetensors", + "url": "https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-22b-distilled-lora-384.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "ltx2\\ltx-2.3-22b-distilled-lora-dynamic_fro09_avg_rank_105_bf16.safetensors", + 0.5 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 79, + "type": "LTX2SamplingPreviewOverride", + "pos": [ + 1530, + 4020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 210 + }, + { + "localized_name": "latent_upscale_model", + "name": "latent_upscale_model", + "shape": 7, + "type": "LATENT_UPSCALE_MODEL", + "link": null + }, + { + "localized_name": "vae", + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 196 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 197 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.3.4", + "Node name for S&R": "LTX2SamplingPreviewOverride" + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 78, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 3870 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 196 + ] + } + ], + "title": "Tiny VAELoader KJ", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.2.5", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "taeltx2_3.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 84, + "type": "DualCLIPLoader", + "pos": [ + 1530, + 4290 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "label": "text_encoder", + "localized_name": "clip_name1", + "name": "clip_name1", + "type": "COMBO", + "widget": { + "name": "clip_name1" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 211 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "DualCLIPLoader" + }, + "widgets_values": [ + "gemma_3_12B_it_fp4_mixed.safetensors", + "ltx-2.3_text_projection_bf16.safetensors", + "sdxl", + "default" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 4, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 4470 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "label": "audio_vae", + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 58 + ] + } + ], + "title": "Audio VAELoader", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "487c7d86a0230aae3d8c0a37d517159b73834f85", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "LTX23_audio_vae_bf16.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 3, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 4620 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "label": "video_vae", + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 16 + ] + } + ], + "title": "Video VAELoader", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.2.5", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "LTX23_video_vae_bf16.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + } + ], + "groups": [], + "links": [ + { + "id": 197, + "origin_id": 79, + "origin_slot": 0, + "target_id": 80, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 210, + "origin_id": 77, + "origin_slot": 0, + "target_id": 79, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 196, + "origin_id": 78, + "origin_slot": 0, + "target_id": 79, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 199, + "origin_id": 80, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 211, + "origin_id": 84, + "origin_slot": 0, + "target_id": -20, + "target_slot": 1, + "type": "CLIP" + }, + { + "id": 58, + "origin_id": 4, + "origin_slot": 0, + "target_id": -20, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 16, + "origin_id": 3, + "origin_slot": 0, + "target_id": -20, + "target_slot": 3, + "type": "VAE" + } + ], + "extra": {} + }, + { + "id": "4b017e47-bf16-441b-ad03-edce978b3be3", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 96, + "lastLinkId": 249, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Stage #1", + "inputNode": { + "id": -10, + "bounding": [ + 2860, + 4380, + 120, + 180 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 4450, + 4360, + 120, + 140 + ] + }, + "inputs": [ + { + "id": "3c716dfb-4465-4519-8705-d5e7b88f1e35", + "name": "model", + "type": "MODEL", + "linkIds": [ + 115, + 114 + ], + "localized_name": "model", + "pos": [ + 2960, + 4400 + ] + }, + { + "id": "dcb44156-9e87-4ef1-a7fd-6b8cb4de0beb", + "name": "audio_latent", + "type": "LATENT", + "linkIds": [ + 119 + ], + "localized_name": "audio_latent", + "pos": [ + 2960, + 4420 + ] + }, + { + "id": "0c7b8f30-c29e-4147-96e9-db7f6ca77e06", + "name": "vae", + "type": "VAE", + "linkIds": [ + 216 + ], + "localized_name": "vae", + "pos": [ + 2960, + 4440 + ] + }, + { + "id": "4286d415-0cbb-4310-94d2-0efdccfb429f", + "name": "latent", + "type": "LATENT", + "linkIds": [ + 118 + ], + "localized_name": "latent", + "label": "video_latent", + "pos": [ + 2960, + 4460 + ] + }, + { + "id": "24eebedb-6830-4afb-9337-bd06b1300d44", + "name": "guide_data", + "type": "GUIDE_DATA", + "linkIds": [ + 122 + ], + "localized_name": "guide_data", + "pos": [ + 2960, + 4480 + ] + }, + { + "id": "1dc63a0e-c430-406b-86a8-6eb45cbc7885", + "name": "positive", + "type": "CONDITIONING", + "linkIds": [ + 116, + 117 + ], + "localized_name": "positive", + "pos": [ + 2960, + 4500 + ] + }, + { + "id": "02c2f048-017d-4fec-8440-19aa412bf8c0", + "name": "frame_rate", + "type": "FLOAT", + "linkIds": [ + 120 + ], + "localized_name": "frame_rate", + "pos": [ + 2960, + 4520 + ] + } + ], + "outputs": [ + { + "id": "84f8d65a-1c3c-4fdc-8cab-7abaf67c1875", + "name": "positive", + "type": "CONDITIONING", + "linkIds": [ + 54, + 54 + ], + "localized_name": "positive", + "pos": [ + 4470, + 4380 + ] + }, + { + "id": "6275837a-94b0-4318-bbce-15969c26e42b", + "name": "negative", + "type": "CONDITIONING", + "linkIds": [ + 55, + 55 + ], + "localized_name": "negative", + "pos": [ + 4470, + 4400 + ] + }, + { + "id": "4c1b1c7e-3c92-4c13-8495-539df653574e", + "name": "NOISE", + "type": "NOISE", + "linkIds": [ + 157 + ], + "localized_name": "NOISE", + "pos": [ + 4470, + 4420 + ] + }, + { + "id": "f0f1c347-9ebf-496e-aea8-ca0a0ff7601e", + "name": "video_latent", + "type": "LATENT", + "linkIds": [ + 142 + ], + "localized_name": "video_latent", + "pos": [ + 4470, + 4440 + ] + }, + { + "id": "bf7e617a-cf3a-4ef6-a485-d3ab3e84742a", + "name": "audio_latent", + "type": "LATENT", + "linkIds": [ + 143 + ], + "localized_name": "audio_latent", + "pos": [ + 4470, + 4460 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 11, + "type": "BasicScheduler", + "pos": [ + 3740, + 4620 + ], + "size": [ + 210, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 115 + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 52 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "linear_quadratic", + 8, + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 10, + "type": "SamplerCustomAdvanced", + "pos": [ + 3960, + 4390 + ], + "size": [ + 220, + 210 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "noise", + "name": "noise", + "type": "NOISE", + "link": 51 + }, + { + "localized_name": "guider", + "name": "guider", + "type": "GUIDER", + "link": 12 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 53 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 52 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 156 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.60", + "Node name for S&R": "SamplerCustomAdvanced", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 29, + "type": "KSamplerSelect", + "pos": [ + 3740, + 4520 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 53 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "KSamplerSelect", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 7, + "type": "LTXVConcatAVLatent", + "pos": [ + 3520, + 4530 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "video_latent", + "name": "video_latent", + "type": "LATENT", + "link": 108 + }, + { + "localized_name": "audio_latent", + "name": "audio_latent", + "type": "LATENT", + "link": 119 + } + ], + "outputs": [ + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 14 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVConcatAVLatent" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 8, + "type": "LTXDirectorGuide", + "pos": [ + 3290, + 4400 + ], + "size": [ + 210, + 180 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 6 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 7 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 216 + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "link": 118 + }, + { + "localized_name": "guide_data", + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 122 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 10, + 54 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 11, + 55 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 108 + ] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirectorGuide", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + 0.5, + "bicubic" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 9, + "type": "CFGGuider", + "pos": [ + 3520, + 4390 + ], + "size": [ + 210, + 100 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 114 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 10 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 11 + } + ], + "outputs": [ + { + "localized_name": "GUIDER", + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 12 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.64", + "Node name for S&R": "CFGGuider", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 28, + "type": "RandomNoise", + "pos": [ + 3740, + 4390 + ], + "size": [ + 210, + 90 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "noise_seed", + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "NOISE", + "name": "NOISE", + "type": "NOISE", + "links": [ + 51, + 157 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 12, + "fixed" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 5, + "type": "LTXVConditioning", + "pos": [ + 3040, + 4400 + ], + "size": [ + 240, + 80 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 116 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 50 + }, + { + "localized_name": "frame_rate", + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 120 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 6 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 7 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "LTXVConditioning", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 26, + "type": "ConditioningZeroOut", + "pos": [ + 3040, + 4530 + ], + "size": [ + 240, + 30 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 117 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 50 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "ConditioningZeroOut" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 13, + "type": "LTXVSeparateAVLatent", + "pos": [ + 4190, + 4390 + ], + "size": [ + 200, + 50 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "av_latent", + "name": "av_latent", + "type": "LATENT", + "link": 156 + } + ], + "outputs": [ + { + "localized_name": "video_latent", + "name": "video_latent", + "type": "LATENT", + "links": [ + 142 + ] + }, + { + "localized_name": "audio_latent", + "name": "audio_latent", + "type": "LATENT", + "links": [ + 143 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVSeparateAVLatent" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + } + ], + "groups": [], + "links": [ + { + "id": 51, + "origin_id": 28, + "origin_slot": 0, + "target_id": 10, + "target_slot": 0, + "type": "NOISE" + }, + { + "id": 12, + "origin_id": 9, + "origin_slot": 0, + "target_id": 10, + "target_slot": 1, + "type": "GUIDER" + }, + { + "id": 53, + "origin_id": 29, + "origin_slot": 0, + "target_id": 10, + "target_slot": 2, + "type": "SAMPLER" + }, + { + "id": 52, + "origin_id": 11, + "origin_slot": 0, + "target_id": 10, + "target_slot": 3, + "type": "SIGMAS" + }, + { + "id": 14, + "origin_id": 7, + "origin_slot": 0, + "target_id": 10, + "target_slot": 4, + "type": "LATENT" + }, + { + "id": 108, + "origin_id": 8, + "origin_slot": 2, + "target_id": 7, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 6, + "origin_id": 5, + "origin_slot": 0, + "target_id": 8, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 7, + "origin_id": 5, + "origin_slot": 1, + "target_id": 8, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 10, + "origin_id": 8, + "origin_slot": 0, + "target_id": 9, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 11, + "origin_id": 8, + "origin_slot": 1, + "target_id": 9, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 50, + "origin_id": 26, + "origin_slot": 0, + "target_id": 5, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 156, + "origin_id": 10, + "origin_slot": 0, + "target_id": 13, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 115, + "origin_id": -10, + "origin_slot": 0, + "target_id": 11, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 114, + "origin_id": -10, + "origin_slot": 0, + "target_id": 9, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 119, + "origin_id": -10, + "origin_slot": 1, + "target_id": 7, + "target_slot": 1, + "type": "LATENT" + }, + { + "id": 216, + "origin_id": -10, + "origin_slot": 2, + "target_id": 8, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 118, + "origin_id": -10, + "origin_slot": 3, + "target_id": 8, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 122, + "origin_id": -10, + "origin_slot": 4, + "target_id": 8, + "target_slot": 4, + "type": "GUIDE_DATA" + }, + { + "id": 116, + "origin_id": -10, + "origin_slot": 5, + "target_id": 5, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 117, + "origin_id": -10, + "origin_slot": 5, + "target_id": 26, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 120, + "origin_id": -10, + "origin_slot": 6, + "target_id": 5, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 54, + "origin_id": 8, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 55, + "origin_id": 8, + "origin_slot": 1, + "target_id": -20, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 157, + "origin_id": 28, + "origin_slot": 0, + "target_id": -20, + "target_slot": 2, + "type": "NOISE" + }, + { + "id": 142, + "origin_id": 13, + "origin_slot": 0, + "target_id": -20, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 143, + "origin_id": 13, + "origin_slot": 1, + "target_id": -20, + "target_slot": 4, + "type": "LATENT" + } + ], + "extra": {} + }, + { + "id": "7cf6430b-4784-4af1-8cde-9268e9465c62", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 96, + "lastLinkId": 249, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Stage #2", + "inputNode": { + "id": -10, + "bounding": [ + 2870, + 3980, + 120, + 200 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 4430, + 3900, + 120, + 80 + ] + }, + "inputs": [ + { + "id": "da8ac1d4-baaa-49af-a44f-51de815b8adc", + "name": "positive", + "type": "CONDITIONING", + "linkIds": [ + 228, + 227 + ], + "localized_name": "positive", + "pos": [ + 2970, + 4000 + ] + }, + { + "id": "2a2ae901-93e9-4551-97e5-bf33db9a7761", + "name": "negative", + "type": "CONDITIONING", + "linkIds": [ + 230, + 229 + ], + "localized_name": "negative", + "pos": [ + 2970, + 4020 + ] + }, + { + "id": "f6f33172-a93d-4505-a3c8-4e4e61488f94", + "name": "latent", + "type": "LATENT", + "linkIds": [ + 232 + ], + "localized_name": "latent", + "label": "video_latent", + "pos": [ + 2970, + 4040 + ] + }, + { + "id": "9d67923c-e4e8-4eb5-a91b-b37dbfb5882f", + "name": "vae", + "type": "VAE", + "linkIds": [ + 219, + 218 + ], + "localized_name": "vae", + "label": "video_vae", + "pos": [ + 2970, + 4060 + ] + }, + { + "id": "c5f8e7de-87ec-4509-892c-8cc4e1f14575", + "name": "guide_data", + "type": "GUIDE_DATA", + "linkIds": [ + 151 + ], + "localized_name": "guide_data", + "pos": [ + 2970, + 4080 + ] + }, + { + "id": "c9c467d0-0426-4288-a31e-b41be1891bdc", + "name": "model", + "type": "MODEL", + "linkIds": [ + 158, + 159 + ], + "localized_name": "model", + "pos": [ + 2970, + 4100 + ] + }, + { + "id": "1c4bd34a-4e48-455a-bd66-323524e5c2d8", + "name": "noise", + "type": "NOISE", + "linkIds": [ + 231 + ], + "localized_name": "noise", + "pos": [ + 2970, + 4120 + ] + }, + { + "id": "ef0cc4fb-4a9f-404b-a62c-92e04fd74171", + "name": "audio_latent", + "type": "LATENT", + "linkIds": [ + 233 + ], + "localized_name": "audio_latent", + "pos": [ + 2970, + 4140 + ] + } + ], + "outputs": [ + { + "id": "402aed7a-321d-48a8-a407-34f05a5ab1e2", + "name": "audio_latent", + "type": "LATENT", + "linkIds": [ + 202 + ], + "localized_name": "audio_latent", + "pos": [ + 4450, + 3920 + ] + }, + { + "id": "8fe53957-38d9-459e-a590-3ad4d96d3616", + "name": "latent", + "type": "LATENT", + "linkIds": [ + 208 + ], + "localized_name": "latent", + "pos": [ + 4450, + 3940 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 55, + "type": "LTXVCropGuides", + "pos": [ + 3050, + 3930 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 228 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 230 + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "link": 232 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 146 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 147 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 135 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "LTXVCropGuides" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 58, + "type": "LTXDirectorGuide", + "pos": [ + 3310, + 3930 + ], + "size": [ + 210, + 190 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 146 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 147 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 219 + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "link": 150 + }, + { + "localized_name": "guide_data", + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 151 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 152 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 153 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 154 + ] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirectorGuide", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + 1, + "bicubic" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 49, + "type": "CFGGuider", + "pos": [ + 3540, + 3920 + ], + "size": [ + 210, + 100 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 158 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 152 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 153 + } + ], + "outputs": [ + { + "localized_name": "GUIDER", + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 124 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.64", + "Node name for S&R": "CFGGuider", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 53, + "type": "KSamplerSelect", + "pos": [ + 3540, + 4060 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 125 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "KSamplerSelect", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 47, + "type": "SamplerCustomAdvanced", + "pos": [ + 3760, + 3920 + ], + "size": [ + 230, + 220 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "noise", + "name": "noise", + "type": "NOISE", + "link": 231 + }, + { + "localized_name": "guider", + "name": "guider", + "type": "GUIDER", + "link": 124 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 125 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 126 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 127 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 180 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.60", + "Node name for S&R": "SamplerCustomAdvanced", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 57, + "type": "LatentUpscaleModelLoader", + "pos": [ + 3050, + 4040 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "localized_name": "LATENT_UPSCALE_MODEL", + "name": "LATENT_UPSCALE_MODEL", + "type": "LATENT_UPSCALE_MODEL", + "links": [ + 144 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "LatentUpscaleModelLoader" + }, + "widgets_values": [ + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 48, + "type": "LTXVSeparateAVLatent", + "pos": [ + 4000, + 3920 + ], + "size": [ + 200, + 50 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "av_latent", + "name": "av_latent", + "type": "LATENT", + "link": 180 + } + ], + "outputs": [ + { + "localized_name": "video_latent", + "name": "video_latent", + "type": "LATENT", + "links": [ + 136 + ] + }, + { + "localized_name": "audio_latent", + "name": "audio_latent", + "type": "LATENT", + "links": [ + 202 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.5.1", + "Node name for S&R": "LTXVSeparateAVLatent", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 52, + "type": "LTXVLatentUpsampler", + "pos": [ + 3050, + 4150 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 135 + }, + { + "localized_name": "upscale_model", + "name": "upscale_model", + "type": "LATENT_UPSCALE_MODEL", + "link": 144 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 218 + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 150 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.14.1", + "Node name for S&R": "LTXVLatentUpsampler", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 54, + "type": "BasicScheduler", + "pos": [ + 3540, + 4160 + ], + "size": [ + 210, + 110 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 159 + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 126 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "linear_quadratic", + 4, + 0.42 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 50, + "type": "LTXVConcatAVLatent", + "pos": [ + 3310, + 4160 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video_latent", + "name": "video_latent", + "type": "LATENT", + "link": 154 + }, + { + "localized_name": "audio_latent", + "name": "audio_latent", + "type": "LATENT", + "link": 233 + } + ], + "outputs": [ + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 127 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.7.0", + "Node name for S&R": "LTXVConcatAVLatent", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 14, + "type": "LTXVCropGuides", + "pos": [ + 4210, + 3920 + ], + "size": [ + 180, + 70 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 227 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 229 + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "link": 136 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": null + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": null + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 208 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "LTXVCropGuides" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + } + ], + "groups": [], + "links": [ + { + "id": 146, + "origin_id": 55, + "origin_slot": 0, + "target_id": 58, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 147, + "origin_id": 55, + "origin_slot": 1, + "target_id": 58, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 150, + "origin_id": 52, + "origin_slot": 0, + "target_id": 58, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 152, + "origin_id": 58, + "origin_slot": 0, + "target_id": 49, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 153, + "origin_id": 58, + "origin_slot": 1, + "target_id": 49, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 124, + "origin_id": 49, + "origin_slot": 0, + "target_id": 47, + "target_slot": 1, + "type": "GUIDER" + }, + { + "id": 125, + "origin_id": 53, + "origin_slot": 0, + "target_id": 47, + "target_slot": 2, + "type": "SAMPLER" + }, + { + "id": 126, + "origin_id": 54, + "origin_slot": 0, + "target_id": 47, + "target_slot": 3, + "type": "SIGMAS" + }, + { + "id": 127, + "origin_id": 50, + "origin_slot": 0, + "target_id": 47, + "target_slot": 4, + "type": "LATENT" + }, + { + "id": 180, + "origin_id": 47, + "origin_slot": 0, + "target_id": 48, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 135, + "origin_id": 55, + "origin_slot": 2, + "target_id": 52, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 144, + "origin_id": 57, + "origin_slot": 0, + "target_id": 52, + "target_slot": 1, + "type": "LATENT_UPSCALE_MODEL" + }, + { + "id": 154, + "origin_id": 58, + "origin_slot": 2, + "target_id": 50, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 136, + "origin_id": 48, + "origin_slot": 0, + "target_id": 14, + "target_slot": 2, + "type": "LATENT" + }, + { + "id": 228, + "origin_id": -10, + "origin_slot": 0, + "target_id": 55, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 227, + "origin_id": -10, + "origin_slot": 0, + "target_id": 14, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 230, + "origin_id": -10, + "origin_slot": 1, + "target_id": 55, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 229, + "origin_id": -10, + "origin_slot": 1, + "target_id": 14, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 232, + "origin_id": -10, + "origin_slot": 2, + "target_id": 55, + "target_slot": 2, + "type": "LATENT" + }, + { + "id": 219, + "origin_id": -10, + "origin_slot": 3, + "target_id": 58, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 218, + "origin_id": -10, + "origin_slot": 3, + "target_id": 52, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 151, + "origin_id": -10, + "origin_slot": 4, + "target_id": 58, + "target_slot": 4, + "type": "GUIDE_DATA" + }, + { + "id": 158, + "origin_id": -10, + "origin_slot": 5, + "target_id": 49, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 159, + "origin_id": -10, + "origin_slot": 5, + "target_id": 54, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 231, + "origin_id": -10, + "origin_slot": 6, + "target_id": 47, + "target_slot": 0, + "type": "NOISE" + }, + { + "id": 233, + "origin_id": -10, + "origin_slot": 7, + "target_id": 50, + "target_slot": 1, + "type": "LATENT" + }, + { + "id": 202, + "origin_id": 48, + "origin_slot": 1, + "target_id": -20, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 208, + "origin_id": 14, + "origin_slot": 2, + "target_id": -20, + "target_slot": 1, + "type": "LATENT" + } + ], + "extra": {} + }, + { + "id": "739520a4-718f-490f-87d1-34ffbca98e3f", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 96, + "lastLinkId": 249, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Decode", + "inputNode": { + "id": -10, + "bounding": [ + 3330, + 4105, + 120, + 140 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 3780, + 4145, + 120, + 60 + ] + }, + "inputs": [ + { + "id": "35c40a03-3855-445e-a691-ae0ada359280", + "name": "samples", + "type": "LATENT", + "linkIds": [ + 242 + ], + "localized_name": "samples", + "label": "audio_latent", + "pos": [ + 3430, + 4125 + ] + }, + { + "id": "ddfb8ed1-1fd2-4086-ad94-d816dab3b6bb", + "name": "audio_vae", + "type": "VAE", + "linkIds": [ + 214 + ], + "localized_name": "audio_vae", + "label": "audio_vae", + "pos": [ + 3430, + 4145 + ] + }, + { + "id": "458157ba-efe8-4aa2-847e-be97d78a6873", + "name": "samples_1", + "type": "LATENT", + "linkIds": [ + 243 + ], + "localized_name": "samples_1", + "label": "video_latent", + "pos": [ + 3430, + 4165 + ] + }, + { + "id": "b02f1f71-c594-437f-8dda-4bb0f19c2e3e", + "name": "vae", + "type": "VAE", + "linkIds": [ + 217 + ], + "localized_name": "vae", + "label": "video_vae", + "pos": [ + 3430, + 4185 + ] + }, + { + "id": "83aa76da-c7d7-41b5-a3d6-4942595be90c", + "name": "fps", + "type": "FLOAT", + "linkIds": [ + 121 + ], + "localized_name": "fps", + "pos": [ + 3430, + 4205 + ] + } + ], + "outputs": [ + { + "id": "7d63e3c3-78d0-4613-9631-0fa830d1bf9a", + "name": "VIDEO", + "type": "VIDEO", + "linkIds": [ + 56 + ], + "localized_name": "VIDEO", + "pos": [ + 3800, + 4165 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 16, + "type": "LTXVAudioVAEDecode", + "pos": [ + 3510, + 4060 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 242 + }, + { + "label": "Audio VAE", + "localized_name": "audio_vae", + "name": "audio_vae", + "type": "VAE", + "link": 214 + } + ], + "outputs": [ + { + "localized_name": "Audio", + "name": "Audio", + "type": "AUDIO", + "links": [ + 25 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVAudioVAEDecode" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "VAEDecode", + "pos": [ + 3510, + 4150 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 243 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 217 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 24 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.9.1", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 17, + "type": "CreateVideo", + "pos": [ + 3510, + 4240 + ], + "size": [ + 210, + 80 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 24 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 25 + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 121 + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 56 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + } + ], + "groups": [], + "links": [ + { + "id": 24, + "origin_id": 15, + "origin_slot": 0, + "target_id": 17, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 25, + "origin_id": 16, + "origin_slot": 0, + "target_id": 17, + "target_slot": 1, + "type": "AUDIO" + }, + { + "id": 242, + "origin_id": -10, + "origin_slot": 0, + "target_id": 16, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 214, + "origin_id": -10, + "origin_slot": 1, + "target_id": 16, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 243, + "origin_id": -10, + "origin_slot": 2, + "target_id": 15, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 217, + "origin_id": -10, + "origin_slot": 3, + "target_id": 15, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 121, + "origin_id": -10, + "origin_slot": 4, + "target_id": 17, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 56, + "origin_id": 17, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "VIDEO" + } + ], + "extra": {} + } + ] + }, + "config": {}, + "extra": { + "frontendVersion": "1.42.14", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "ds": { + "scale": 0.8964143292718676, + "offset": [ + -1001.6666666666667, + -3562.019444444444 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/LTX Director Example Workflow Compact Subgraph.png b/example_workflows/LTX Director Example Workflow Compact Subgraph.png new file mode 100644 index 0000000..9d08318 Binary files /dev/null and b/example_workflows/LTX Director Example Workflow Compact Subgraph.png differ diff --git a/example_workflows/LTX Director Example Workflow.json b/example_workflows/LTX Director Example Workflow.json new file mode 100644 index 0000000..8c221a6 --- /dev/null +++ b/example_workflows/LTX Director Example Workflow.json @@ -0,0 +1,2337 @@ +{ + "id": "5976fef9-a511-4781-82c4-0b78c14a37d3", + "revision": 0, + "last_node_id": 92, + "last_link_id": 211, + "nodes": [ + { + "id": 80, + "type": "LoraLoaderModelOnly", + "pos": [ + 1530, + 4160 + ], + "size": [ + 280, + 90 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 197 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 199 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.75", + "Node name for S&R": "LoraLoaderModelOnly", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "models": [ + { + "name": "ltx-2.3-22b-distilled-lora-384.safetensors", + "url": "https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-22b-distilled-lora-384.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "ltx2\\ltx-2.3-22b-distilled-lora-dynamic_fro09_avg_rank_105_bf16.safetensors", + 0.5 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 4, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 4470 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 58, + 113 + ] + } + ], + "title": "Audio VAELoader", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "487c7d86a0230aae3d8c0a37d517159b73834f85", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "LTX23_audio_vae_bf16.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 3, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 4620 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 16, + 59, + 145, + 149 + ] + } + ], + "title": "Video VAELoader", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.2.5", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "LTX23_video_vae_bf16.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 13, + "type": "LTXVSeparateAVLatent", + "pos": [ + 4190, + 4390 + ], + "size": [ + 200, + 50 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "av_latent", + "type": "LATENT", + "link": 156 + } + ], + "outputs": [ + { + "name": "video_latent", + "type": "LATENT", + "links": [ + 142 + ] + }, + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 143 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVSeparateAVLatent" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 54, + "type": "BasicScheduler", + "pos": [ + 3530, + 4170 + ], + "size": [ + 210, + 110 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 159 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 126 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "linear_quadratic", + 4, + 0.42 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 53, + "type": "KSamplerSelect", + "pos": [ + 3530, + 4070 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 125 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "KSamplerSelect", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 47, + "type": "SamplerCustomAdvanced", + "pos": [ + 3750, + 3930 + ], + "size": [ + 230, + 220 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 157 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 124 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 125 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 126 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 127 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 180 + ] + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.60", + "Node name for S&R": "SamplerCustomAdvanced", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 52, + "type": "LTXVLatentUpsampler", + "pos": [ + 3040, + 4160 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 135 + }, + { + "name": "upscale_model", + "type": "LATENT_UPSCALE_MODEL", + "link": 144 + }, + { + "name": "vae", + "type": "VAE", + "link": 145 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 150 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.14.1", + "Node name for S&R": "LTXVLatentUpsampler", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 50, + "type": "LTXVConcatAVLatent", + "pos": [ + 3300, + 4170 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "video_latent", + "type": "LATENT", + "link": 154 + }, + { + "name": "audio_latent", + "type": "LATENT", + "link": 143 + } + ], + "outputs": [ + { + "name": "latent", + "type": "LATENT", + "links": [ + 127 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.7.0", + "Node name for S&R": "LTXVConcatAVLatent", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 79, + "type": "LTX2SamplingPreviewOverride", + "pos": [ + 1530, + 4020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 210 + }, + { + "name": "latent_upscale_model", + "shape": 7, + "type": "LATENT_UPSCALE_MODEL", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 196 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 197 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.3.4", + "Node name for S&R": "LTX2SamplingPreviewOverride" + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 78, + "type": "VAELoaderKJ", + "pos": [ + 1530, + 3870 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 196 + ] + } + ], + "title": "Tiny VAELoader KJ", + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.2.5", + "Node name for S&R": "VAELoaderKJ" + }, + "widgets_values": [ + "taeltx2_3.safetensors", + "main_device", + "bf16" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 10, + "type": "SamplerCustomAdvanced", + "pos": [ + 3960, + 4390 + ], + "size": [ + 220, + 210 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 51 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 12 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 53 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 52 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 14 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 156 + ] + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.60", + "Node name for S&R": "SamplerCustomAdvanced", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 28, + "type": "RandomNoise", + "pos": [ + 3740, + 4390 + ], + "size": [ + 210, + 90 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 51, + 157 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 12, + "fixed" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 29, + "type": "KSamplerSelect", + "pos": [ + 3740, + 4520 + ], + "size": [ + 210, + 60 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 53 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "KSamplerSelect", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 11, + "type": "BasicScheduler", + "pos": [ + 3740, + 4620 + ], + "size": [ + 210, + 110 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 115 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 52 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "linear_quadratic", + 8, + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 9, + "type": "CFGGuider", + "pos": [ + 3520, + 4390 + ], + "size": [ + 210, + 100 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 114 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 10 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 11 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 12 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.64", + "Node name for S&R": "CFGGuider", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 7, + "type": "LTXVConcatAVLatent", + "pos": [ + 3520, + 4530 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "video_latent", + "type": "LATENT", + "link": 108 + }, + { + "name": "audio_latent", + "type": "LATENT", + "link": 119 + } + ], + "outputs": [ + { + "name": "latent", + "type": "LATENT", + "links": [ + 14 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVConcatAVLatent" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 48, + "type": "LTXVSeparateAVLatent", + "pos": [ + 3990, + 3930 + ], + "size": [ + 200, + 50 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "av_latent", + "type": "LATENT", + "link": 180 + } + ], + "outputs": [ + { + "name": "video_latent", + "type": "LATENT", + "links": [ + 136 + ] + }, + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 202 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.5.1", + "Node name for S&R": "LTXVSeparateAVLatent", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 30, + "type": "SaveVideo", + "pos": [ + 4670, + 4260 + ], + "size": [ + 680, + 470 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 56 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/LTX_Director", + "auto", + "auto" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 57, + "type": "LatentUpscaleModelLoader", + "pos": [ + 3040, + 4050 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT_UPSCALE_MODEL", + "type": "LATENT_UPSCALE_MODEL", + "links": [ + 144 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "LatentUpscaleModelLoader" + }, + "widgets_values": [ + "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 8, + "type": "LTXDirectorGuide", + "pos": [ + 3290, + 4400 + ], + "size": [ + 210, + 180 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 16 + }, + { + "name": "latent", + "type": "LATENT", + "link": 118 + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 122 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 10, + 54, + 188 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 11, + 55, + 189 + ] + }, + { + "name": "latent", + "type": "LATENT", + "links": [ + 108 + ] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirectorGuide", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + 0.5, + "bicubic" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 49, + "type": "CFGGuider", + "pos": [ + 3530, + 3930 + ], + "size": [ + 210, + 100 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 158 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 152 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 153 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 124 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.64", + "Node name for S&R": "CFGGuider", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 55, + "type": "LTXVCropGuides", + "pos": [ + 3040, + 3940 + ], + "size": [ + 250, + 70 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 188 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 189 + }, + { + "name": "latent", + "type": "LATENT", + "link": 142 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 146 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 147 + ] + }, + { + "name": "latent", + "type": "LATENT", + "links": [ + 135 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.4", + "Node name for S&R": "LTXVCropGuides" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 58, + "type": "LTXDirectorGuide", + "pos": [ + 3300, + 3940 + ], + "size": [ + 210, + 190 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 146 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 147 + }, + { + "name": "vae", + "type": "VAE", + "link": 149 + }, + { + "name": "latent", + "type": "LATENT", + "link": 150 + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "link": 151 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 152 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 153 + ] + }, + { + "name": "latent", + "type": "LATENT", + "links": [ + 154 + ] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirectorGuide", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + 1, + "bicubic" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 14, + "type": "LTXVCropGuides", + "pos": [ + 4200, + 3930 + ], + "size": [ + 180, + 70 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 54 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 55 + }, + { + "name": "latent", + "type": "LATENT", + "link": 136 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": null + }, + { + "name": "latent", + "type": "LATENT", + "links": [ + 208 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "LTXVCropGuides" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 16, + "type": "LTXVAudioVAEDecode", + "pos": [ + 4440, + 4260 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 202 + }, + { + "label": "Audio VAE", + "name": "audio_vae", + "type": "VAE", + "link": 58 + } + ], + "outputs": [ + { + "name": "Audio", + "type": "AUDIO", + "links": [ + 25 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.16.0", + "Node name for S&R": "LTXVAudioVAEDecode" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 46, + "type": "LTXDirector", + "pos": [ + 1860, + 3970 + ], + "size": [ + 1120, + 777 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 199 + }, + { + "name": "clip", + "type": "CLIP", + "link": 211 + }, + { + "name": "audio_vae", + "shape": 7, + "type": "VAE", + "link": 113 + }, + { + "name": "optional_latent", + "shape": 7, + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 114, + 115, + 158, + 159 + ] + }, + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 116, + 117 + ] + }, + { + "name": "video_latent", + "type": "LATENT", + "links": [ + 118 + ] + }, + { + "name": "audio_latent", + "type": "LATENT", + "links": [ + 119 + ] + }, + { + "name": "guide_data", + "type": "GUIDE_DATA", + "links": [ + 122, + 151 + ] + }, + { + "name": "frame_rate", + "type": "FLOAT", + "links": [ + 120, + 121 + ] + }, + { + "name": "combined_audio", + "type": "AUDIO", + "links": [] + } + ], + "properties": { + "cnr_id": "whatdreamscost-comfyui", + "ver": "5cae3c00186ff2eb79771070f05a2c4e02f329bb", + "Node name for S&R": "LTXDirector", + "aux_id": "kijai/ComfyUI-PromptRelay" + }, + "widgets_values": [ + "", + 120, + 5, + "{\"segments\":[],\"audioSegments\":[]}", + "", + "", + 0.001, + "", + false, + 24, + "seconds", + 0, + 0, + "maintain aspect ratio", + 32, + 18, + "" + ] + }, + { + "id": 84, + "type": "DualCLIPLoader", + "pos": [ + 1530, + 4290 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 211 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "DualCLIPLoader" + }, + "widgets_values": [ + "gemma_3_12B_it_fp4_mixed.safetensors", + "ltx-2.3_text_projection_bf16.safetensors", + "sdxl", + "default" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 77, + "type": "CheckpointLoaderSimple", + "pos": [ + 1240, + 4020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 210 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": null + }, + { + "name": "VAE", + "type": "VAE", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.17.2", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "ltx-2.3-22b-dev-fp8.safetensors" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "VAEDecode", + "pos": [ + 4440, + 4350 + ], + "size": [ + 210, + 50 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 208 + }, + { + "name": "vae", + "type": "VAE", + "link": 59 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 24 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.9.1", + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 17, + "type": "CreateVideo", + "pos": [ + 4440, + 4440 + ], + "size": [ + 210, + 80 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 24 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 25 + }, + { + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 121 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 56 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 5, + "type": "LTXVConditioning", + "pos": [ + 3040, + 4400 + ], + "size": [ + 240, + 80 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 116 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 50 + }, + { + "name": "frame_rate", + "type": "FLOAT", + "widget": { + "name": "frame_rate" + }, + "link": 120 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 6 + ] + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [ + 7 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.56", + "Node name for S&R": "LTXVConditioning", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 24 + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 26, + "type": "ConditioningZeroOut", + "pos": [ + 3040, + 4530 + ], + "size": [ + 240, + 30 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 117 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 50 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.19.3", + "Node name for S&R": "ConditioningZeroOut" + }, + "widgets_values": [], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 92, + "type": "MarkdownNote", + "pos": [ + 1240, + 4170 + ], + "size": [ + 270, + 560 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "FAQ", + "properties": {}, + "widgets_values": [ + "# FAQ\n\n**How do I use this??**\n\nI'll be posting tutorials on my YouTube channel soon!\n\n\n**Where do I download these models??**\n\nThere is a good list of models and download links here: \n\nAlso the latest v1.1 spatial upscaler can be found here: \n\nThe tiny vae (taeltx2_3.safetensors) is found here:\n\n\n**I'm getting errors when running this workflow!**\n\nDouble check that you have all of the correct models loaded. This is using the newest version of 2x upscaler (v1.1). You can change it to the older one under the Model Loader.\n\n**How can I support you?**\n\nHelp me get a job. I'm unemployed and literally broke. (seriously, i'll even scrub toilets for minimum wage)\n\nOr follow me on my socials :)\n\n\n\n\n\n\n" + ], + "color": "#322", + "bgcolor": "#533" + } + ], + "links": [ + [ + 6, + 5, + 0, + 8, + 0, + "CONDITIONING" + ], + [ + 7, + 5, + 1, + 8, + 1, + "CONDITIONING" + ], + [ + 10, + 8, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 11, + 8, + 1, + 9, + 2, + "CONDITIONING" + ], + [ + 12, + 9, + 0, + 10, + 1, + "GUIDER" + ], + [ + 14, + 7, + 0, + 10, + 4, + "LATENT" + ], + [ + 16, + 3, + 0, + 8, + 2, + "VAE" + ], + [ + 24, + 15, + 0, + 17, + 0, + "IMAGE" + ], + [ + 25, + 16, + 0, + 17, + 1, + "AUDIO" + ], + [ + 50, + 26, + 0, + 5, + 1, + "CONDITIONING" + ], + [ + 51, + 28, + 0, + 10, + 0, + "NOISE" + ], + [ + 52, + 11, + 0, + 10, + 3, + "SIGMAS" + ], + [ + 53, + 29, + 0, + 10, + 2, + "SAMPLER" + ], + [ + 54, + 8, + 0, + 14, + 0, + "CONDITIONING" + ], + [ + 55, + 8, + 1, + 14, + 1, + "CONDITIONING" + ], + [ + 56, + 17, + 0, + 30, + 0, + "VIDEO" + ], + [ + 58, + 4, + 0, + 16, + 1, + "VAE" + ], + [ + 59, + 3, + 0, + 15, + 1, + "VAE" + ], + [ + 108, + 8, + 2, + 7, + 0, + "LATENT" + ], + [ + 113, + 4, + 0, + 46, + 2, + "VAE" + ], + [ + 114, + 46, + 0, + 9, + 0, + "MODEL" + ], + [ + 115, + 46, + 0, + 11, + 0, + "MODEL" + ], + [ + 116, + 46, + 1, + 5, + 0, + "CONDITIONING" + ], + [ + 117, + 46, + 1, + 26, + 0, + "CONDITIONING" + ], + [ + 118, + 46, + 2, + 8, + 3, + "LATENT" + ], + [ + 119, + 46, + 3, + 7, + 1, + "LATENT" + ], + [ + 120, + 46, + 5, + 5, + 2, + "FLOAT" + ], + [ + 121, + 46, + 5, + 17, + 2, + "FLOAT" + ], + [ + 122, + 46, + 4, + 8, + 4, + "GUIDE_DATA" + ], + [ + 124, + 49, + 0, + 47, + 1, + "GUIDER" + ], + [ + 125, + 53, + 0, + 47, + 2, + "SAMPLER" + ], + [ + 126, + 54, + 0, + 47, + 3, + "SIGMAS" + ], + [ + 127, + 50, + 0, + 47, + 4, + "LATENT" + ], + [ + 135, + 55, + 2, + 52, + 0, + "LATENT" + ], + [ + 136, + 48, + 0, + 14, + 2, + "LATENT" + ], + [ + 142, + 13, + 0, + 55, + 2, + "LATENT" + ], + [ + 143, + 13, + 1, + 50, + 1, + "LATENT" + ], + [ + 144, + 57, + 0, + 52, + 1, + "LATENT_UPSCALE_MODEL" + ], + [ + 145, + 3, + 0, + 52, + 2, + "VAE" + ], + [ + 146, + 55, + 0, + 58, + 0, + "CONDITIONING" + ], + [ + 147, + 55, + 1, + 58, + 1, + "CONDITIONING" + ], + [ + 149, + 3, + 0, + 58, + 2, + "VAE" + ], + [ + 150, + 52, + 0, + 58, + 3, + "LATENT" + ], + [ + 151, + 46, + 4, + 58, + 4, + "GUIDE_DATA" + ], + [ + 152, + 58, + 0, + 49, + 1, + "CONDITIONING" + ], + [ + 153, + 58, + 1, + 49, + 2, + "CONDITIONING" + ], + [ + 154, + 58, + 2, + 50, + 0, + "LATENT" + ], + [ + 156, + 10, + 0, + 13, + 0, + "LATENT" + ], + [ + 157, + 28, + 0, + 47, + 0, + "NOISE" + ], + [ + 158, + 46, + 0, + 49, + 0, + "MODEL" + ], + [ + 159, + 46, + 0, + 54, + 0, + "MODEL" + ], + [ + 180, + 47, + 0, + 48, + 0, + "LATENT" + ], + [ + 188, + 8, + 0, + 55, + 0, + "CONDITIONING" + ], + [ + 189, + 8, + 1, + 55, + 1, + "CONDITIONING" + ], + [ + 196, + 78, + 0, + 79, + 2, + "VAE" + ], + [ + 197, + 79, + 0, + 80, + 0, + "MODEL" + ], + [ + 199, + 80, + 0, + 46, + 0, + "MODEL" + ], + [ + 202, + 48, + 1, + 16, + 0, + "LATENT" + ], + [ + 208, + 14, + 2, + 15, + 0, + "LATENT" + ], + [ + 210, + 77, + 0, + 79, + 0, + "MODEL" + ], + [ + 211, + 84, + 0, + 46, + 1, + "CLIP" + ] + ], + "groups": [ + { + "id": 3, + "title": "Models", + "bounding": [ + 1230, + 3800, + 590, + 943.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Stage #1", + "bounding": [ + 3020, + 4320, + 1380, + 423.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Stage #2 Upscale", + "bounding": [ + 3020, + 3860, + 1380, + 430 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 6, + "title": "Process Video", + "bounding": [ + 4430, + 4180, + 930, + 563.6 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1.116781577942553, + "offset": [ + -3185.8854794552367, + -3676.7424382820964 + ] + }, + "frontendVersion": "1.42.14", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/LTX Director Example Workflow.png b/example_workflows/LTX Director Example Workflow.png new file mode 100644 index 0000000..7af9ff5 Binary files /dev/null and b/example_workflows/LTX Director Example Workflow.png differ diff --git a/js/ltx_director.js b/js/ltx_director.js new file mode 100644 index 0000000..c62faf3 --- /dev/null +++ b/js/ltx_director.js @@ -0,0 +1,3870 @@ +const { app } = window.comfyAPI.app; +const { api } = window.comfyAPI.api; + +// --- UI Constants & Configuration --- +const RULER_HEIGHT = 24; +const BLOCK_HEIGHT = 160; // Increased to make the image timeline area much taller +const AUDIO_TRACK_HEIGHT = 80; +const CANVAS_HEIGHT = RULER_HEIGHT + BLOCK_HEIGHT + AUDIO_TRACK_HEIGHT; +const HANDLE_HIT_PX = 14; +const MIN_SEGMENT_LENGTH = 6; +const MAX_THUMBNAIL_DIM = 512; // Increased to maintain quality for taller images + +const HIDDEN_WIDGET_NAMES = ["timeline_data", "local_prompts", "segment_lengths", "guide_strength", "audio_data", "use_custom_audio"]; + +function hideWidget(w) { + if (!w) return; + if (!w._origType && w.type !== "hidden") w._origType = w.type; + w.type = "hidden"; + w.hidden = true; + if (!w.options) w.options = {}; + w.options.hidden = true; + if (w.element) w.element.style.display = "none"; +} + +function clamp(v, min, max) { return Math.max(min, Math.min(max, v)); } + +// --- Modern Dark/Grey UI CSS (ComfyUI Match) --- +const STYLES = ` + .pr-wrapper { + font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + display: flex; + flex-direction: column; + gap: 8px; + width: 100%; + height: 100%; + box-sizing: border-box; + padding-bottom: 4px; + } + .pr-wrapper.drag-active { + outline: 2px dashed #888; + background: rgba(255, 255, 255, 0.05); + border-radius: 6px; + } + .pr-toolbar { + display: flex; + justify-content: space-between; + align-items: center; + padding: 2px 0px; + flex-wrap: wrap; + gap: 6px; + } + .pr-actions { + display: flex; + gap: 6px; + flex-wrap: wrap; + } + .pr-btn { + background: #222; + color: #e0e0e0; + border: 1px solid #111; + border-radius: 4px; + padding: 6px 12px; + font-size: 11px; + font-weight: 500; + cursor: pointer; + display: flex; + align-items: center; + gap: 6px; + transition: all 0.2s ease; + } + .pr-btn:hover { + background: #333; + border-color: #555; + } + .pr-btn-danger:hover { + background: #4a1515; + border-color: #cc4444; + color: #ffaaaa; + } + .pr-canvas { + border-radius: 6px; + border: 1px solid #111; + background: #2a2a2a; + cursor: pointer; + width: 100%; + outline: none; + display: block; /* Ensure no inline baseline gaps */ + } + .pr-prop-container { + display: flex; + flex-direction: column; + width: 100%; + flex-grow: 1; /* Automatically scales to fill node height */ + min-height: 40px; + } + .pr-prompt-area { + width: 100%; + height: 100%; + background: #222; + color: #e0e0e0; + border: 1px solid #111; + border-radius: 6px; + padding: 8px; + resize: none; /* Removed the manual resize corner handle */ + font-size: 12px; + line-height: 1.4; + box-sizing: border-box; + outline: none; + transition: border-color 0.2s ease; + } + .pr-prompt-area:focus { + border-color: #888; + } + .pr-audio-info { + width: 100%; + height: 100%; + background: #181818; + color: #aaa; + border: 1px solid #111; + border-radius: 6px; + padding: 10px; + font-size: 12px; + line-height: 1.6; + box-sizing: border-box; + display: none; + } + .pr-audio-info span { color: #fff; font-weight: 500; } + .pr-controls-group { + background: #1e1e1e; + border: 1px solid #333; + border-radius: 6px; + padding: 6px 10px; + display: flex; + flex-direction: column; + gap: 4px; + margin-bottom: 4px; + box-sizing: border-box; + width: 100%; + } + .pr-strength-row { + display: flex; + align-items: center; + gap: 12px; + width: 100%; + box-sizing: border-box; + } + .pr-height-resizer { + height: 6px; + background: #2a2a2a; + cursor: ns-resize; + border-radius: 3px; + margin: 2px 0; + transition: background 0.15s; + border: 1px solid #1e1e1e; + } + .pr-height-resizer:hover { + background: #444; + border-color: #555; + } + .pr-strength-label { + font-size: 11px; + font-weight: 600; + color: #fff; + white-space: nowrap; + margin-left: auto; + } + .pr-strength-slider { + -webkit-appearance: none; + appearance: none; + width: 80px; + height: 4px; + background: #444; + border-radius: 2px; + outline: none; + cursor: pointer; + border: 1px solid #222; + } + .pr-strength-slider::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: #aaa; + cursor: pointer; + } + .pr-strength-slider:disabled { + opacity: 0.3; + cursor: not-allowed; + } + .pr-strength-input { + font-size: 12px; + color: #fff; + background: #222; + border: 1px solid #444; + border-radius: 4px; + width: 52px; + text-align: center; + padding: 3px; + } + .pr-strength-input::-webkit-outer-spin-button, + .pr-strength-input::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; + } + .pr-strength-input[type=number] { + -moz-appearance: textfield; + } + .pr-strength-input:disabled { + opacity: 0.35; + cursor: not-allowed; + } + .pr-gap-menu { + position: fixed; + background: #1e1e1e; + border: 1px solid #444; + border-radius: 6px; + padding: 4px; + display: flex; + flex-direction: column; + gap: 4px; + z-index: 9999; + box-shadow: 0 4px 16px rgba(0,0,0,0.6); + } + .pr-gap-menu-btn { + background: #2a2a2a; + color: #e0e0e0; + border: 1px solid #333; + border-radius: 4px; + padding: 6px 14px; + font-size: 11px; + font-family: inherit; + cursor: pointer; + text-align: left; + white-space: nowrap; + display: flex; + align-items: center; + gap: 6px; + transition: background 0.15s ease; + } + .pr-gap-menu-btn:hover { + background: #3a3a3a; + border-color: #666; + } + .pr-player-controls { + display: flex; + justify-content: center; + align-items: center; + gap: 12px; + padding: 2px 0; + flex-wrap: wrap; + width: 100%; + } + .pr-icon-btn { + background: #2a2a2a; + border: 1px solid #444; + color: #eee; + cursor: pointer; + padding: 6px 12px; + border-radius: 4px; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s; + } + .pr-icon-btn * { + pointer-events: none; + } + .pr-icon-btn:hover { + color: #fff; + background: #3a3a3a; + border-color: #666; + } + .pr-icon-btn.active { + color: #4fff8f; + border-color: #4fff8f; + background: #1a3a2a; + } + .pr-seek-bar { + -webkit-appearance: none; + appearance: none; + height: 6px; + background: #444; + border-radius: 3px; + outline: none; + cursor: pointer; + border: 1px solid #222; + } + .pr-seek-bar::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 14px; + height: 14px; + border-radius: 50%; + background: #ff4444; + cursor: pointer; + border: 2px solid #222; + } + .pr-timeline-viewport { + width: 100%; + overflow-x: auto; + overflow-y: hidden; + } + .pr-timeline-viewport::-webkit-scrollbar { + height: 10px; + } + .pr-timeline-viewport::-webkit-scrollbar-track { + background: #151515; + border-radius: 5px; + } + .pr-timeline-viewport::-webkit-scrollbar-thumb { + background: #444 + border-radius: 5px; + border: 1px solid #000; + } + .pr-timeline-viewport::-webkit-scrollbar-thumb:hover { + background: #666 + border-color: #000; + } + .pr-zoom-controls { + display: flex; + align-items: center; + gap: 4px; + margin-left: 12px; + } + .pr-zoom-slider { + width: 80px; + -webkit-appearance: none; + appearance: none; + height: 4px; + background: #444; + border-radius: 2px; + outline: none; + cursor: pointer; + } + .pr-zoom-slider::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: #aaa; + cursor: pointer; + } + .pr-right-group { + display: flex; + align-items: center; + gap: 12px; + } + .pr-segment-bounds { + font-size: 12px; + color: #aaa; + font-family: monospace; + } + .pr-timecode { + font-size: 14px; + font-weight: bold; + color: #e0e0e0; + font-family: monospace; + } + .pr-settings-menu { + position: fixed; + background: #1e1e1e; + border: 1px solid #444; + border-radius: 6px; + padding: 10px; + display: flex; + flex-direction: column; + gap: 8px; + z-index: 9999; + box-shadow: 0 4px 20px rgba(0,0,0,0.7); + min-width: 220px; + } + .pr-settings-title { + font-size: 11px; + font-weight: 600; + color: #888; + text-transform: uppercase; + letter-spacing: 0.06em; + padding-bottom: 4px; + border-bottom: 1px solid #333; + margin-bottom: 2px; + } + .pr-settings-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + } + .pr-settings-label { + font-size: 12px; + color: #bbb; + flex: 1; + white-space: nowrap; + } + .pr-number-control { + display: flex; + align-items: center; + border: 1px solid #444; + border-radius: 4px; + background: #2a2a2a; + overflow: hidden; + } + .pr-number-btn { + background: #333; + color: #aaa; + border: none; + width: 20px; + height: 22px; + cursor: pointer; + font-size: 12px; + display: flex; + align-items: center; + justify-content: center; + transition: background 0.15s; + user-select: none; + } + .pr-number-btn:hover { + background: #444; + color: #fff; + } + .pr-settings-input { + background: transparent; + color: #e0e0e0; + border: none; + padding: 0 4px; + font-size: 12px; + width: 50px; + height: 22px; + text-align: center; + font-family: monospace; + outline: none; + -moz-appearance: textfield; + } + .pr-settings-input::-webkit-outer-spin-button, + .pr-settings-input::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; + } + .pr-settings-select { + background: #2a2a2a; + color: #e0e0e0; + border: 1px solid #444; + border-radius: 4px; + padding: 3px 4px; + font-size: 12px; + width: 98px; + cursor: pointer; + } + .pr-settings-divider { + border: none; + border-top: 1px solid #2a2a2a; + margin: 2px 0; + } + .pr-settings-toggle-btn { + width: 100%; + background: #252525; + color: #aaa; + border: 1px solid #333; + border-radius: 4px; + padding: 5px 8px; + font-size: 11px; + cursor: pointer; + text-align: center; + transition: all 0.15s; + } + .pr-settings-toggle-btn:hover { + background: #2e2e2e; + color: #ccc; + border-color: #555; + } + .pr-settings-close-btn { + background: transparent; + color: #888; + border: none; + cursor: pointer; + padding: 2px; + display: flex; + align-items: center; + justify-content: center; + border-radius: 4px; + transition: all 0.15s; + } + .pr-settings-close-btn:hover { + color: #fff; + background: rgba(255,255,255,0.1); + } + .pr-segmented-control { + display: flex; + background: #1e1e1e; + border: 1px solid #333; + border-radius: 6px; + padding: 2px; + width: 110px; + height: 22px; + align-items: center; + box-sizing: border-box; + } + .pr-segment { + flex: 1; + text-align: center; + font-size: 10px; + font-weight: 500; + line-height: 18px; + cursor: pointer; + border-radius: 4px; + color: #888; + transition: all 0.15s ease; + } + .pr-segment.active { + background: #333; + color: #fff; + } + .pr-segment:hover:not(.active) { + color: #ccc; + } +`; + +if (!document.getElementById("prompt-relay-styles")) { + const styleEl = document.createElement("style"); + styleEl.id = "prompt-relay-styles"; + styleEl.textContent = STYLES; + document.head.appendChild(styleEl); +} + +// --- Icons --- +const ICONS = { + upload: ``, + audio: ``, + trash: ``, + text: ``, + play: ``, + pause: ``, + loop: ``, + minus: ``, + plus: ``, + fit: ``, + gear: ``, + close: `` +}; + +// --- Data Models --- +function parseInitial(jsonStr) { + let parsed = { segments: [], audioSegments: [] }; + try { + if (jsonStr) { + const p = JSON.parse(jsonStr); + if (Array.isArray(p.segments)) parsed.segments = p.segments; + if (Array.isArray(p.audioSegments)) parsed.audioSegments = p.audioSegments; + } + } catch (e) { } + + let currentStart = 0; + for (let seg of parsed.segments) { + if (seg.start === undefined) { + seg.start = currentStart; + currentStart += seg.length; + } + // Guarantee ID assignment to prevent node loading drag breaks + if (!seg.id) { + seg.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); + } + } + + for (let seg of parsed.audioSegments) { + if (!seg.id) { + seg.id = Date.now().toString() + Math.random().toString(36).substr(2, 5); + } + if (seg.trimStart === undefined) seg.trimStart = 0; + } + + return parsed; +} + +class TimelineEditor { + constructor(node, container, domWidget) { + this.node = node; + this.container = container; + this.domWidget = domWidget; + + // Track heights (dynamic) + this.rulerHeight = RULER_HEIGHT; + this.blockHeight = BLOCK_HEIGHT; + this.audioTrackHeight = AUDIO_TRACK_HEIGHT; + this.canvasHeight = CANVAS_HEIGHT; + + // Core data + this.timeline = { segments: [], audioSegments: [] }; + this.selectionType = "image"; // "image" or "audio" + this.selectedIndex = -1; + + // Interactions + this._isDragging = false; + this._dragType = null; + this._dragStartX = 0; + this._dragInitialTimeline = null; + this.zoomLevel = 1.0; + this._lastZoom = 1.0; + this._lastScale = 1.0; + this._dragTargetId = null; + this._dragTargetIdRight = null; + this._previewSegments = null; + this._lastWidth = 0; + this._hoveredGapIdx = -1; + this._isHovering = false; + + // Playback state + this.currentFrame = 0; + this.isPlaying = false; + this.isLooping = false; + this.audioContext = null; + this.activeAudioNodes = []; + this.playbackStartTime = 0; + this.playbackStartFrame = 0; + this._playLoopId = null; + + // --- Ghost dragging state --- + this._ghostSegmentId = null; + this._ghostTrack = null; + this._ghostInitialTimeline = null; + + // Attach to Python widgets + this._gapMenu = null; // Active gap popup menu element + this._gapMenuDismisser = null; + + // Attach to Python widgets + this.durationFramesWidget = this.node.widgets.find(w => w.name === "duration_frames"); + this.durationSecondsWidget = this.node.widgets.find(w => w.name === "duration_seconds"); + this.frameRateWidget = this.node.widgets.find(w => w.name === "frame_rate"); + this.timelineDataWidget = this.node.widgets.find(w => w.name === "timeline_data"); + this.localPromptsWidget = this.node.widgets.find(w => w.name === "local_prompts"); + this.segmentLengthsWidget = this.node.widgets.find(w => w.name === "segment_lengths"); + this.guideStrengthWidget = this.node.widgets.find(w => w.name === "guide_strength"); + this.displayModeWidget = this.node.widgets.find(w => w.name === "display_mode"); + + this.timeline = parseInitial(this.timelineDataWidget?.value); + this.loadImages(); + + this.createDOM(); + if (this.timeline.segments.length > 0) { + this.selectedIndex = 0; + } + this.updateUIFromSelection(); + this.commitChanges(true); + // Hide settings widgets by default to reduce node clutter. + // Deferred so all widget types are finalized before we touch them. + setTimeout(() => this.hideSettingsWidgets(), 0); + + let isSyncing = false; + + const origDurationFramesCallback = this.durationFramesWidget?.callback; + if (this.durationFramesWidget) { + this.durationFramesWidget.callback = (...args) => { + if (origDurationFramesCallback) origDurationFramesCallback.apply(this.durationFramesWidget, args); + + if (!isSyncing && this.durationSecondsWidget) { + isSyncing = true; + this.durationSecondsWidget.value = parseFloat((this.getDurationFrames() / this.getFrameRate()).toFixed(3)); + isSyncing = false; + } + + this.commitChanges(); + }; + } + + const origDurationSecondsCallback = this.durationSecondsWidget?.callback; + if (this.durationSecondsWidget) { + this.durationSecondsWidget.callback = (...args) => { + if (origDurationSecondsCallback) origDurationSecondsCallback.apply(this.durationSecondsWidget, args); + + if (!isSyncing && this.durationFramesWidget) { + isSyncing = true; + const newFrames = Math.max(1, Math.round(this.durationSecondsWidget.value * this.getFrameRate())); + this.durationFramesWidget.value = newFrames; + if (this.durationFramesWidget.callback) this.durationFramesWidget.callback(newFrames); + isSyncing = false; + } + }; + } + + const origFrameRateCallback = this.frameRateWidget?.callback; + if (this.frameRateWidget) { + this.frameRateWidget.callback = (...args) => { + if (origFrameRateCallback) origFrameRateCallback.apply(this.frameRateWidget, args); + if (!isSyncing && this.durationSecondsWidget) { + isSyncing = true; + this.durationSecondsWidget.value = parseFloat((this.getDurationFrames() / this.getFrameRate()).toFixed(3)); + isSyncing = false; + } + }; + } + + const origDisplayModeCallback = this.displayModeWidget?.callback; + if (this.displayModeWidget) { + this.displayModeWidget.callback = (...args) => { + if (origDisplayModeCallback) origDisplayModeCallback.apply(this.displayModeWidget, args); + this.updateWidgetVisibility(); + this.updateUIFromSelection(); + this.render(); + }; + this.updateWidgetVisibility(); // Initial trigger + } + + // Polling is much more reliable in Comfy than ResizeObserver due to scale transforms + this._renderLoop = requestAnimationFrame(this.checkResize.bind(this)); + } + + destroy() { + cancelAnimationFrame(this._renderLoop); + this.pauseAudio(); + window.removeEventListener("keydown", this.handleKeyDown, true); + window.removeEventListener("paste", this.handlePaste, true); + } + + getDurationFrames() { + return parseInt((this.durationFramesWidget && this.durationFramesWidget.value > 0) ? this.durationFramesWidget.value : 24, 10); + } + + getFrameRate() { + return parseInt((this.frameRateWidget && this.frameRateWidget.value > 0) ? this.frameRateWidget.value : 24, 10); + } + + // Grow the timeline duration to fit `requiredFrames` if it is currently shorter. + // The timeline only ever grows — never shrinks — through this method. + growTimelineIfNeeded(requiredFrames) { + const current = this.getDurationFrames(); + if (requiredFrames <= current) return; // already big enough + + const newFrames = Math.ceil(requiredFrames); + if (this.durationFramesWidget) { + this.durationFramesWidget.value = newFrames; + } + if (this.durationSecondsWidget) { + this.durationSecondsWidget.value = parseFloat((newFrames / this.getFrameRate()).toFixed(3)); + } + // Notify ComfyUI that the widget value changed so it serialises correctly. + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + } + + // Returns the maximum allowed zoom level, computed so that at max zoom + // the viewport shows exactly 4 seconds of the visual timeline. + getMaxZoom() { + const visualDurationSecs = this.getVisualDurationFrames() / this.getFrameRate(); + const baseMaxZoom = Math.max(1, visualDurationSecs / 4); + + // Limit max zoom to prevent canvas width from exceeding browser limits (causing crash) + const viewportWidth = this.viewport ? this.viewport.clientWidth : 1000; + const MAX_CANVAS_WIDTH = 32768; // Extended limit for modern browsers + const limitMaxZoom = MAX_CANVAS_WIDTH / Math.max(1, viewportWidth); + + return Math.max(1, Math.min(baseMaxZoom, limitMaxZoom)); + } + + // Returns the visual timeline length in frames: + // the furthest segment end (across both tracks) × 1.30, with a floor of getDurationFrames(). + // This is used for all rendering/positioning — the actual output duration is getDurationFrames(). + getVisualDurationFrames() { + let furthest = 0; + for (const seg of this.timeline.segments) { + furthest = Math.max(furthest, seg.start + seg.length); + } + for (const seg of this.timeline.audioSegments) { + furthest = Math.max(furthest, seg.start + seg.length); + } + const outputDuration = this.getDurationFrames(); + if (furthest <= 0) return outputDuration; + return Math.max(outputDuration, Math.ceil(furthest * 1.30)); + } + + // Sync the zoom slider's max attribute to the current getMaxZoom() value, + // clamping zoomLevel if it now exceeds the new max. + updateZoomSliderMax() { + if (!this.zoomSlider) return; + const maxZoom = this.getMaxZoom(); + this.zoomSlider.max = maxZoom.toFixed(2); + if (this.zoomLevel > maxZoom) { + this.zoomLevel = maxZoom; + this.zoomSlider.value = maxZoom; + // Resize the canvas to match the clamped zoom + const viewportWidth = this.viewport ? this.viewport.clientWidth : 0; + if (viewportWidth > 0) { + const newCanvasWidth = Math.max(viewportWidth, viewportWidth * this.zoomLevel); + this.canvas.style.width = newCanvasWidth + "px"; + this.resizeCanvas(newCanvasWidth); + } + } + } + + loadImages() { + for (const seg of this.timeline.segments) { + if (seg.imageB64 && !seg.imgObj) { + seg.imgObj = new Image(); + seg.imgObj.onload = () => this.render(); + seg.imgObj.src = seg.imageB64; + } + } + } + + createDOM() { + this.wrapper = document.createElement("div"); + this.wrapper.className = "pr-wrapper"; + + this.wrapper.addEventListener("mouseenter", () => { this._isHovering = true; }); + this.wrapper.addEventListener("mouseleave", () => { this._isHovering = false; }); + + this.handleKeyDown = (e) => { + const activeTag = document.activeElement ? document.activeElement.tagName : ""; + if (activeTag === "INPUT" || activeTag === "TEXTAREA") return; + + if ((e.key === "Delete" || e.key === "Backspace") && this.selectedIndex !== -1 && this._isHovering) { + this.deleteSelectedSegment(); + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } else if ((e.key === " " || e.code === "Space") && this._isHovering) { + this.togglePlay(); + e.stopPropagation(); + e.stopImmediatePropagation(); + e.preventDefault(); + } + }; + window.addEventListener("keydown", this.handleKeyDown, true); + + this.handlePaste = (e) => { + if (this._isHovering) { + const activeTag = document.activeElement ? document.activeElement.tagName : ""; + if (activeTag === "INPUT" || activeTag === "TEXTAREA") return; + + if (e.clipboardData && e.clipboardData.files && e.clipboardData.files.length > 0) { + const imageFiles = Array.from(e.clipboardData.files).filter(f => f.type.startsWith("image/")); + if (imageFiles.length > 0) { + this.handleImageUpload(imageFiles, this.currentFrame); + e.preventDefault(); + e.stopPropagation(); + } + } + } + }; + window.addEventListener("paste", this.handlePaste, true); + + // --- Toolbar --- + const toolbar = document.createElement("div"); + toolbar.className = "pr-toolbar"; + + const actionGroup = document.createElement("div"); + actionGroup.className = "pr-actions"; + + this.fileInput = document.createElement("input"); + this.fileInput.type = "file"; + this.fileInput.accept = "image/*"; + this.fileInput.multiple = true; + this.fileInput.style.display = "none"; + this.fileInput.addEventListener("change", (e) => this.handleImageUpload(e.target.files)); + + this.audioFileInput = document.createElement("input"); + this.audioFileInput.type = "file"; + this.audioFileInput.accept = "audio/*"; + this.audioFileInput.multiple = true; + this.audioFileInput.style.display = "none"; + this.audioFileInput.addEventListener("change", (e) => this.handleAudioUpload(e.target.files)); + + const uploadBtn = document.createElement("button"); + uploadBtn.className = "pr-btn"; + uploadBtn.innerHTML = `${ICONS.upload} Add Image`; + uploadBtn.addEventListener("click", () => this.fileInput.click()); + + const uploadAudioBtn = document.createElement("button"); + uploadAudioBtn.className = "pr-btn"; + uploadAudioBtn.innerHTML = `${ICONS.audio} Add Audio`; + uploadAudioBtn.addEventListener("click", () => this.audioFileInput.click()); + + const addTextBtn = document.createElement("button"); + addTextBtn.className = "pr-btn"; + addTextBtn.innerHTML = `${ICONS.text} Add Text`; + addTextBtn.addEventListener("click", () => this.addTextSegmentFreeSpace()); + + const deleteBtn = document.createElement("button"); + deleteBtn.className = "pr-btn pr-btn-danger"; + deleteBtn.innerHTML = `${ICONS.trash} Delete`; + deleteBtn.addEventListener("click", () => this.deleteSelectedSegment()); + + actionGroup.appendChild(this.fileInput); + actionGroup.appendChild(this.audioFileInput); + actionGroup.appendChild(uploadBtn); + actionGroup.appendChild(addTextBtn); + actionGroup.appendChild(uploadAudioBtn); + actionGroup.appendChild(deleteBtn); + toolbar.appendChild(actionGroup); + + const rightGroup = document.createElement("div"); + rightGroup.className = "pr-right-group"; + + this.segmentBoundsDisplay = document.createElement("div"); + this.segmentBoundsDisplay.className = "pr-segment-bounds"; + this.segmentBoundsDisplay.textContent = "Start: - | End: -"; + + this.timeCodeDisplay = document.createElement("div"); + this.timeCodeDisplay.className = "pr-timecode"; + this.timeCodeDisplay.textContent = this.formatTime(0); + + const settingsBtn = document.createElement("button"); + settingsBtn.className = "pr-btn"; + settingsBtn.style.padding = "6px"; + settingsBtn.style.justifyContent = "center"; + settingsBtn.style.width = "28px"; + settingsBtn.style.height = "28px"; + settingsBtn.style.boxSizing = "border-box"; + settingsBtn.innerHTML = ICONS.gear; + settingsBtn.title = "Settings"; + settingsBtn.addEventListener("click", (e) => { + e.stopPropagation(); + if (this._settingsMenu) { + this.dismissSettingsMenu(); + } else { + this.showSettingsMenu(settingsBtn); + } + }); + + const toggleBtn = document.createElement("button"); + toggleBtn.className = "pr-btn"; + toggleBtn.style.padding = "6px 8px"; + toggleBtn.style.fontSize = "11px"; + toggleBtn.style.marginRight = "0px"; + toggleBtn.textContent = "Custom Audio: OFF"; + toggleBtn.title = "Toggle Custom Audio Output"; + + const updateToggleStyle = (isOn) => { + toggleBtn.textContent = isOn ? "Custom Audio: ON" : "Custom Audio: OFF"; + if (isOn) { + toggleBtn.style.background = "#1c222d"; + toggleBtn.style.borderColor = "#283142"; + toggleBtn.style.color = "#e0e0e0"; + } else { + toggleBtn.style.background = "#222"; + toggleBtn.style.borderColor = "#111"; + toggleBtn.style.color = "#e0e0e0"; + } + }; + + toggleBtn.addEventListener("click", (e) => { + e.stopPropagation(); + const widget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (widget) { + widget.value = !widget.value; + updateToggleStyle(widget.value); + this.node.setDirtyCanvas(true, true); + } + }); + + // Initial state check (widgets might not be ready immediately) + setTimeout(() => { + const widget = this.node.widgets?.find(w => w.name === "use_custom_audio"); + if (widget) { + updateToggleStyle(widget.value); + } + }, 100); + + const helpBtn = document.createElement("button"); + helpBtn.className = "pr-btn"; + helpBtn.style.padding = "6px"; + helpBtn.style.justifyContent = "center"; + helpBtn.style.width = "28px"; + helpBtn.style.height = "28px"; + helpBtn.style.boxSizing = "border-box"; + helpBtn.innerHTML = "?"; + helpBtn.title = "Help / Documentation"; + helpBtn.addEventListener("click", (e) => { + e.stopPropagation(); + window.open("https://github.com/WhatDreamsCost/WhatDreamsCost-ComfyUI", "_blank"); + }); + + const btnGroup = document.createElement("div"); + btnGroup.style.display = "flex"; + btnGroup.style.gap = "6px"; + btnGroup.style.alignItems = "center"; + btnGroup.appendChild(toggleBtn); + btnGroup.appendChild(helpBtn); + btnGroup.appendChild(settingsBtn); + rightGroup.appendChild(btnGroup); + + toolbar.appendChild(rightGroup); + + // --- Canvas & Viewport --- + this.viewport = document.createElement("div"); + this.viewport.className = "pr-timeline-viewport"; + + this.viewport.addEventListener("wheel", (e) => { + if (e.ctrlKey || e.metaKey) { + e.preventDefault(); + e.stopPropagation(); + + let zoomDelta = e.deltaY > 0 ? -0.5 : 0.5; + this.zoomLevel = Math.max(1, Math.min(this.getMaxZoom(), this.zoomLevel + zoomDelta)); + if (this.zoomSlider) this.zoomSlider.value = this.zoomLevel; + + const oldWidth = this.canvas.offsetWidth; + const newWidth = this.viewport.clientWidth * this.zoomLevel; + const mouseX = e.clientX - this.viewport.getBoundingClientRect().left; + const scrollRatio = (this.viewport.scrollLeft + mouseX) / oldWidth; + + this.canvas.style.width = newWidth + "px"; + this.viewport.scrollLeft = scrollRatio * newWidth - mouseX; + } + }, { passive: false, capture: true }); + + this.canvas = document.createElement("canvas"); + this.canvas.className = "pr-canvas"; + this.ctx = this.canvas.getContext("2d"); + this.canvas.style.width = "100%"; + + this.viewport.appendChild(this.canvas); + + this.canvas.addEventListener("mousedown", this.onMouseDown.bind(this)); + this.canvas.addEventListener("contextmenu", this.onContextMenu.bind(this)); + this.canvas.style.height = `${CANVAS_HEIGHT}px`; + + // --- Content Area Container --- + const propContainer = document.createElement("div"); + propContainer.className = "pr-prop-container"; + + // --- Text Area (Image/Text) --- + this.promptInput = document.createElement("textarea"); + this.promptInput.className = "pr-prompt-area"; + this.promptInput.placeholder = "Enter prompt for selected segment..."; + this.promptInput.addEventListener("input", () => { + if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { + this.timeline.segments[this.selectedIndex].prompt = this.promptInput.value; + this.commitChanges(); + } + }); + + // --- Audio Info Area --- + this.audioInfoArea = document.createElement("div"); + this.audioInfoArea.className = "pr-audio-info"; + + propContainer.appendChild(this.promptInput); + propContainer.appendChild(this.audioInfoArea); + + this.wrapper.addEventListener("dragover", (e) => { + e.preventDefault(); + this.wrapper.classList.add("drag-active"); + + const { x, y } = this.getMousePos(e); + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + if (!logicalWidth || totalFrames <= 0) return; + + const isAudioTrack = y > RULER_HEIGHT + this.blockHeight; + const trackType = isAudioTrack ? "audio" : "image"; + const arrToModify = isAudioTrack ? this.timeline.audioSegments : this.timeline.segments; + + if (!this._ghostSegmentId || this._ghostTrack !== trackType) { + this._ghostSegmentId = "GHOST_" + Date.now(); + this._ghostTrack = trackType; + this._ghostInitialTimeline = JSON.parse(JSON.stringify(arrToModify)); + + const frameRate = this.getFrameRate(); + const newLength = Math.max(1, frameRate * 1); + + let mouseFrameX = x * (totalFrames / logicalWidth); + let startFrame = clamp(Math.round(mouseFrameX - newLength / 2), 0, totalFrames - newLength); + + this._ghostInitialTimeline.push({ + id: this._ghostSegmentId, + start: startFrame, + length: newLength, + type: "ghost" + }); + } + + let mouseFrameX = x * (totalFrames / logicalWidth); + const ghost = this._ghostInitialTimeline.find(s => s.id === this._ghostSegmentId); + let D_mouse_start = mouseFrameX - ghost.length / 2; + + this._previewSegments = this._applyCenterDragPhysics( + this._ghostInitialTimeline, + this._ghostSegmentId, + D_mouse_start, + mouseFrameX, + totalFrames, + totalFrames, + logicalWidth + ); + this.render(); + }); + + this.wrapper.addEventListener("dragleave", (e) => { + const rect = this.wrapper.getBoundingClientRect(); + if (e.clientX < rect.left || e.clientX >= rect.right || + e.clientY < rect.top || e.clientY >= rect.bottom) { + this.wrapper.classList.remove("drag-active"); + this._ghostSegmentId = null; + this._ghostTrack = null; + this._ghostInitialTimeline = null; + this._previewSegments = null; + this.render(); + } + }); + + this.wrapper.addEventListener("drop", (e) => { + e.preventDefault(); + e.stopPropagation(); + this.wrapper.classList.remove("drag-active"); + + let targetFrameStart = null; + let targetTrack = this._ghostTrack || "image"; + + if (this._ghostSegmentId && this._previewSegments) { + const ghost = this._previewSegments.find(s => s.id === this._ghostSegmentId); + if (ghost) { + targetFrameStart = ghost.resolvedStart !== undefined ? ghost.resolvedStart : ghost.start; + } + } + this._ghostSegmentId = null; + this._ghostTrack = null; + this._ghostInitialTimeline = null; + this._previewSegments = null; + this.render(); + + if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { + const imageFiles = []; + const audioFiles = []; + for (let file of e.dataTransfer.files) { + if (file.type.startsWith("audio/")) audioFiles.push(file); + if (file.type.startsWith("image/")) imageFiles.push(file); + } + + // Let implicit intent handle mixing drops: use the track we hovered over + // for the first type we process, or fallback. + if (audioFiles.length > 0 && (targetTrack === "audio" || imageFiles.length === 0)) { + this.handleAudioUpload(audioFiles, targetFrameStart); + } else if (imageFiles.length > 0) { + this.handleImageUpload(imageFiles, targetFrameStart); + } + } + }); + + window.addEventListener("mousemove", this.onMouseMove.bind(this)); + window.addEventListener("mouseup", this.onMouseUp.bind(this)); + + // --- Player Controls --- + const playerControls = document.createElement("div"); + playerControls.className = "pr-player-controls"; + + this.playBtn = document.createElement("button"); + this.playBtn.className = "pr-icon-btn"; + this.playBtn.style.padding = "4px"; + this.playBtn.innerHTML = ICONS.play; + this.playBtn.title = "Play/Pause Audio"; + this.playBtn.addEventListener("click", () => this.togglePlay()); + + this.loopBtn = document.createElement("button"); + this.loopBtn.className = "pr-icon-btn"; + this.loopBtn.style.padding = "4px"; + this.loopBtn.innerHTML = ICONS.loop; + this.loopBtn.title = "Toggle Loop"; + this.loopBtn.addEventListener("click", () => this.toggleLoop()); + + this.seekBar = document.createElement("input"); + this.seekBar.type = "range"; + this.seekBar.className = "pr-seek-bar"; + this.seekBar.min = "0"; + this.seekBar.value = "0"; + this.seekBar.style.flex = "1"; // take up remaining space + this.seekBar.addEventListener("input", (e) => { + this.currentFrame = parseInt(e.target.value, 10); + this.render(); + if (this.isPlaying) { + this.playAudio(); + } + }); + + // --- Zoom Controls --- + const zoomControls = document.createElement("div"); + zoomControls.className = "pr-zoom-controls"; + + const zoomOutBtn = document.createElement("button"); + zoomOutBtn.className = "pr-icon-btn"; + zoomOutBtn.style.padding = "4px"; + zoomOutBtn.innerHTML = ICONS.minus; + zoomOutBtn.title = "Zoom Out"; + zoomOutBtn.addEventListener("click", () => { + const currentZoom = parseFloat(this.zoomSlider.value); + this.zoomSlider.value = Math.max(1, currentZoom - 0.5); + this.zoomSlider.dispatchEvent(new Event("input")); + }); + + this.zoomSlider = document.createElement("input"); + this.zoomSlider.type = "range"; + this.zoomSlider.className = "pr-zoom-slider"; + this.zoomSlider.min = "1"; + this.zoomSlider.max = "1"; // Updated dynamically via updateZoomSliderMax() + this.zoomSlider.step = "0.1"; + this.zoomSlider.value = "1"; + this.zoomSlider.title = "Zoom Level"; + this.zoomSlider.addEventListener("input", (e) => { + this.zoomLevel = parseFloat(e.target.value); + + const viewportWidth = this.viewport.clientWidth; + const newCanvasWidth = Math.max(viewportWidth, viewportWidth * this.zoomLevel); + + this.canvas.style.width = newCanvasWidth + "px"; + this.resizeCanvas(newCanvasWidth); + this._lastWidth = viewportWidth; + this._lastZoom = this.zoomLevel; + + // Keep playhead centered + const totalFrames = this.getVisualDurationFrames(); + const playheadRatio = this.currentFrame / totalFrames; + const newPlayheadX = playheadRatio * newCanvasWidth; + this.viewport.scrollLeft = newPlayheadX - (viewportWidth / 2); + }); + + const zoomInBtn = document.createElement("button"); + zoomInBtn.className = "pr-icon-btn"; + zoomInBtn.style.padding = "4px"; + zoomInBtn.innerHTML = ICONS.plus; + zoomInBtn.title = "Zoom In"; + zoomInBtn.addEventListener("click", () => { + const currentZoom = parseFloat(this.zoomSlider.value); + this.zoomSlider.value = Math.min(this.getMaxZoom(), currentZoom + 0.5); + this.zoomSlider.dispatchEvent(new Event("input")); + }); + + const zoomFitBtn = document.createElement("button"); + zoomFitBtn.className = "pr-icon-btn"; + zoomFitBtn.style.padding = "4px"; + zoomFitBtn.style.marginLeft = "4px"; + zoomFitBtn.innerHTML = ICONS.fit; + zoomFitBtn.title = "Zoom to Fit (show full timeline)"; + zoomFitBtn.addEventListener("click", () => { + this.zoomLevel = 1; + this.zoomSlider.value = 1; + const viewportWidth = this.viewport.clientWidth; + this.canvas.style.width = viewportWidth + "px"; + this.resizeCanvas(viewportWidth); + this._lastWidth = viewportWidth; + this._lastZoom = 1; + this.viewport.scrollLeft = 0; + }); + + zoomControls.appendChild(zoomOutBtn); + zoomControls.appendChild(this.zoomSlider); + zoomControls.appendChild(zoomInBtn); + zoomControls.appendChild(zoomFitBtn); + + playerControls.appendChild(this.playBtn); + playerControls.appendChild(this.loopBtn); + playerControls.appendChild(this.seekBar); + playerControls.appendChild(zoomControls); + + + + // --- Guide Strength Slider --- + this.strengthRow = document.createElement("div"); + this.strengthRow.className = "pr-strength-row"; + + const strengthLabel = document.createElement("span"); + strengthLabel.className = "pr-strength-label"; + strengthLabel.textContent = "Guide Strength:"; + + this.strengthValue = document.createElement("input"); + this.strengthValue.type = "text"; + this.strengthValue.className = "pr-strength-input"; + this.strengthValue.value = "1.00"; + this.strengthValue.disabled = true; + this.strengthValue.style.cursor = "ew-resize"; + + // Dragging logic for guide strength + let isDragging = false; + let startX = 0; + let startVal = 0; + let hasMoved = false; + + this.strengthValue.addEventListener("mousedown", (e) => { + if (this.strengthValue.disabled) return; + startX = e.clientX; + startVal = parseFloat(this.strengthValue.value) || 1.0; + hasMoved = false; + + const onMouseMove = (moveEvent) => { + const deltaX = moveEvent.clientX - startX; + if (Math.abs(deltaX) > 3) { + hasMoved = true; + isDragging = true; + } + + if (isDragging) { + moveEvent.preventDefault(); + const sensitivity = 0.002; + let newVal = startVal + deltaX * sensitivity; + + if (newVal < 0) newVal = 0; + if (newVal > 1) newVal = 1; + + this.strengthValue.value = newVal.toFixed(2); + + if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { + const seg = this.timeline.segments[this.selectedIndex]; + if (seg.type !== "text") { + seg.guideStrength = newVal; + this.commitChanges(); + } + } + } + }; + + const onMouseUp = () => { + document.removeEventListener("mousemove", onMouseMove); + document.removeEventListener("mouseup", onMouseUp); + + if (!hasMoved) { + this.strengthValue.focus(); + this.strengthValue.select(); + } + isDragging = false; + }; + + document.addEventListener("mousemove", onMouseMove); + document.addEventListener("mouseup", onMouseUp); + }); + + this.strengthValue.addEventListener("change", (e) => { + let val = parseFloat(e.target.value); + if (isNaN(val)) val = 1; + val = Math.max(0, Math.min(1, val)); + this.strengthValue.value = val.toFixed(2); + if (this.selectionType === "image" && this.timeline.segments[this.selectedIndex]) { + const seg = this.timeline.segments[this.selectedIndex]; + if (seg.type !== "text") { + seg.guideStrength = val; + this.commitChanges(); + } + } + }); + + this.strengthRow.appendChild(this.timeCodeDisplay); + this.strengthRow.appendChild(this.segmentBoundsDisplay); + this.strengthRow.appendChild(strengthLabel); + this.strengthRow.appendChild(this.strengthValue); + + + this.wrapper.appendChild(toolbar); + this.wrapper.appendChild(this.viewport); + + const controlsGroup = document.createElement("div"); + controlsGroup.className = "pr-controls-group"; + controlsGroup.appendChild(this.strengthRow); + controlsGroup.appendChild(playerControls); + this.wrapper.appendChild(controlsGroup); + this.wrapper.appendChild(propContainer); + + this.container.appendChild(this.wrapper); + } + + checkResize() { + const viewportWidth = this.viewport.clientWidth; + const currentScale = this.getRenderScale(); + + if (viewportWidth > 0 && (this._lastWidth !== viewportWidth || this._lastZoom !== this.zoomLevel || this._lastScale !== currentScale)) { + this._lastWidth = viewportWidth; + this._lastZoom = this.zoomLevel; + this._lastScale = currentScale; + + const newCanvasWidth = Math.max(viewportWidth, viewportWidth * this.zoomLevel); + this.canvas.style.width = newCanvasWidth + "px"; + this.resizeCanvas(newCanvasWidth); + } + this._renderLoop = requestAnimationFrame(this.checkResize.bind(this)); + } + + getRenderScale() { + const dpr = window.devicePixelRatio || 1; + let graphScale = 1; + try { + if (window.app && window.app.canvas && window.app.canvas.ds && window.app.canvas.ds.scale) { + graphScale = window.app.canvas.ds.scale; + } + } catch (e) { } + // Scale up if zoomed in, but don't drop below 1x DPR if zoomed out + return dpr * Math.max(1, graphScale); + } + + resizeCanvas(widthPx) { + const scale = this.getRenderScale(); + const targetWidth = Math.round(widthPx * scale); + const targetHeight = Math.round(this.canvasHeight * scale); + + this.canvas.width = targetWidth; + this.canvas.height = targetHeight; + this.ctx.setTransform(scale, 0, 0, scale, 0, 0); + this.render(); + } + + // Helper to map mouse events accurately regardless of canvas scaling + getMousePos(e) { + const rect = this.canvas.getBoundingClientRect(); + + const scaleX = this.canvas.offsetWidth / rect.width; + const scaleY = this.canvas.offsetHeight / rect.height; + + const x = (e.clientX - rect.left) * scaleX; + const y = (e.clientY - rect.top) * scaleY; + return { x, y }; + } + + // --- Async Image Upload Logic (Handles multiple images simultaneously) --- + async handleImageUpload(files, targetFrameStart = null, explicitLength = null) { + const frameRate = this.getFrameRate(); + const durationFrames = this.getDurationFrames(); + const newLength = explicitLength !== null ? explicitLength : frameRate * 1; // Default to 1 second long + + for (let file of files) { + if (!file.type.startsWith("image/")) continue; + + await new Promise(async (resolve) => { + try { + const body = new FormData(); + body.append("image", file); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.status !== 200) { resolve(); return; } + + const data = await resp.json(); + const filename = data.name; + const subfolder = data.subfolder || ""; + const imageFile = subfolder ? subfolder + "/" + filename : filename; + const imgUrl = api.apiURL(`/view?filename=${encodeURIComponent(filename)}&type=input&subfolder=${encodeURIComponent(subfolder)}`); + + const img = new Image(); + img.onload = () => { + + let newStart = targetFrameStart; + if (newStart === null) { + // Fallback: find the first free slot, or append past the end + newStart = 0; + this.timeline.segments.sort((a, b) => a.start - b.start); + for (let i = 0; i < this.timeline.segments.length; i++) { + let seg = this.timeline.segments[i]; + if (newStart + newLength <= seg.start) break; + newStart = Math.max(newStart, seg.start + seg.length); + } + } + + // Use the visual timeline as the physics bound so segments can + // land anywhere in the padded visual area without touching duration_frames. + const currentDuration = this.getVisualDurationFrames(); + + if (targetFrameStart !== null) { + // Resolve physics to push existing segments + let tempId = "TEMP_" + Date.now(); + this.timeline.segments.push({ id: tempId, start: newStart, length: newLength, type: "temp" }); + let result = this._applyCenterDragPhysics(this.timeline.segments, tempId, newStart, newStart + newLength / 2, currentDuration, currentDuration, 1); + + // Update original segments with resolved physics to preserve imgObj + for (let shiftedSeg of result) { + let original = this.timeline.segments.find(s => s.id === shiftedSeg.id); + if (original) { + original.start = shiftedSeg.resolvedStart !== undefined ? shiftedSeg.resolvedStart : shiftedSeg.start; + } + } + + let tempSeg = this.timeline.segments.find(s => s.id === tempId); + newStart = tempSeg.start; + this.timeline.segments = this.timeline.segments.filter(s => s.id !== tempId); + targetFrameStart = newStart + newLength; // For the next file in batch + } + + // Use the full intended length — the timeline has already been grown to fit. + let constrainedLength = newLength; + + const seg = { + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: newStart, + length: constrainedLength, + prompt: "", + type: "image", + imageFile: imageFile, + imageB64: imgUrl + }; + + const displayImg = new Image(); + displayImg.onload = () => { + seg.imgObj = displayImg; + this.render(); + resolve(); // Resolve promise letting next image process + }; + displayImg.src = imgUrl; + + this.timeline.segments.push(seg); + this.timeline.segments.sort((a, b) => a.start - b.start); + this.selectionType = "image"; + this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); + + this.updateUIFromSelection(); + this.commitChanges(true); + }; + img.src = imgUrl; + } catch (err) { + console.error("[PromptRelay] Image upload failed", err); + resolve(); + } + }); + } + this.fileInput.value = ""; + } + + // --- Async Audio Upload Logic --- + async handleAudioUpload(files, targetFrameStart = null) { + const frameRate = this.getFrameRate(); + const durationFrames = this.getDurationFrames(); + + for (let file of files) { + if (!file.type.startsWith("audio/")) continue; + + await new Promise(async (resolve) => { + try { + const body = new FormData(); + body.append("image", file); + const resp = await api.fetchApi("/upload/image", { method: "POST", body }); + if (resp.status !== 200) { resolve(); return; } + + const data = await resp.json(); + const filename = data.name; + const subfolder = data.subfolder || ""; + const audioFile = subfolder ? subfolder + "/" + filename : filename; + + const arrayBuffer = await file.arrayBuffer(); + const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer); + const clipDurationSecs = audioBuffer.duration; + const clipFrames = Math.max(1, Math.ceil(clipDurationSecs * frameRate)); + + const channelData = audioBuffer.getChannelData(0); + const peaks = []; + const numPeaks = 200; + const step = Math.floor(channelData.length / numPeaks); + for (let i = 0; i < numPeaks; i++) { + let max = 0; + for (let j = 0; j < step; j++) { + const val = Math.abs(channelData[i * step + j]); + if (val > max) max = val; + } + peaks.push(max); + } + + let newLength = clipFrames; + let newStart = targetFrameStart; + + if (newStart === null) { + // Find the first free slot, or place past the end of all existing audio + newStart = 0; + this.timeline.audioSegments.sort((a, b) => a.start - b.start); + for (let i = 0; i < this.timeline.audioSegments.length; i++) { + let seg = this.timeline.audioSegments[i]; + if (newStart + newLength <= seg.start) break; + newStart = Math.max(newStart, seg.start + seg.length); + } + } + + // Use the visual timeline as the physics bound so segments can + // land anywhere in the padded visual area without touching duration_frames. + const currentDuration = this.getVisualDurationFrames(); + + if (targetFrameStart !== null) { + let tempId = "TEMP_" + Date.now(); + this.timeline.audioSegments.push({ id: tempId, start: newStart, length: newLength, type: "temp" }); + let result = this._applyCenterDragPhysics(this.timeline.audioSegments, tempId, newStart, newStart + newLength / 2, currentDuration, currentDuration, 1); + + for (let shiftedSeg of result) { + let original = this.timeline.audioSegments.find(s => s.id === shiftedSeg.id); + if (original) original.start = shiftedSeg.resolvedStart !== undefined ? shiftedSeg.resolvedStart : shiftedSeg.start; + } + + let tempSeg = this.timeline.audioSegments.find(s => s.id === tempId); + newStart = tempSeg.start; + this.timeline.audioSegments = this.timeline.audioSegments.filter(s => s.id !== tempId); + targetFrameStart = newStart + newLength; + } + + // Use the full clip length — timeline has already grown to fit. + let constrainedLength = newLength; + + const seg = { + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + type: "audio", + start: newStart, + length: constrainedLength, + trimStart: 0, + audioDurationFrames: clipFrames, + audioFile: audioFile, + fileName: file.name, + waveformPeaks: peaks + }; + + this.timeline.audioSegments.push(seg); + this.timeline.audioSegments.sort((a, b) => a.start - b.start); + this.selectionType = "audio"; + this.selectedIndex = this.timeline.audioSegments.findIndex(s => s.id === seg.id); + + this.updateUIFromSelection(); + this.commitChanges(true); + this.render(); + resolve(); + } catch (err) { + console.error("[PromptRelay] Audio processing failed", err); + resolve(); + } + }); + } + this.audioFileInput.value = ""; + } + + deleteSelectedSegment() { + if (this.selectionType === "audio") { + if (this.timeline.audioSegments.length === 0 || this.selectedIndex === -1) return; + this.timeline.audioSegments.splice(this.selectedIndex, 1); + this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + } else { + if (this.timeline.segments.length === 0 || this.selectedIndex === -1) return; + this.timeline.segments.splice(this.selectedIndex, 1); + this.selectedIndex = Math.max(-1, this.selectedIndex - 1); + } + this.updateUIFromSelection(); + this.commitChanges(); + this.render(); + } + + formatTime(frames, dropSuffix = false) { + const mode = this.displayModeWidget ? this.displayModeWidget.value : "seconds"; + if (mode === "seconds") { + const secs = frames / this.getFrameRate(); + return dropSuffix ? secs.toFixed(2) : secs.toFixed(2) + "s"; + } + return dropSuffix ? Math.round(frames).toString() : Math.round(frames) + " frames"; + } + + updateWidgetVisibility() { + const mode = this.displayModeWidget ? this.displayModeWidget.value : "seconds"; + + if (this.durationFramesWidget) { + const isVisible = mode === "frames"; + this.durationFramesWidget.type = isVisible ? "INT" : "hidden"; + if (!this.durationFramesWidget.options) this.durationFramesWidget.options = {}; + this.durationFramesWidget.options.hidden = !isVisible; + this.durationFramesWidget.hidden = !isVisible; + + if (isVisible) { + delete this.durationFramesWidget.computeSize; + } else { + this.durationFramesWidget.computeSize = () => [0, 0]; + } + } + if (this.durationSecondsWidget) { + const isVisible = mode === "seconds"; + this.durationSecondsWidget.type = isVisible ? "FLOAT" : "hidden"; + if (!this.durationSecondsWidget.options) this.durationSecondsWidget.options = {}; + this.durationSecondsWidget.options.hidden = !isVisible; + this.durationSecondsWidget.hidden = !isVisible; + + if (isVisible) { + delete this.durationSecondsWidget.computeSize; + } else { + this.durationSecondsWidget.computeSize = () => [0, 0]; + } + } + + // Force node resize and redraw deferred to next tick + setTimeout(() => { + if (this.node && this.node.computeSize) { + const sz = this.node.computeSize(); + this.node.size[1] = sz[1]; + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + } + }, 0); + } + + updateUIFromSelection() { + let seg = null; + if (this.selectedIndex >= 0) { + if (this.selectionType === "audio") { + const origSeg = this.timeline.audioSegments[this.selectedIndex]; + if (origSeg) { + const previewIsAudio = this._ghostTrack === 'audio' || (this._previewSegments && this._ghostTrack === null && this.selectionType === 'audio'); + const arr = (this._previewSegments && previewIsAudio) ? this._previewSegments : this.timeline.audioSegments; + seg = arr.find(s => s.id === origSeg.id) || origSeg; + } + } else { + const origSeg = this.timeline.segments[this.selectedIndex]; + if (origSeg) { + const previewIsImage = this._ghostTrack === 'image' || (this._previewSegments && this._ghostTrack === null && this.selectionType === 'image'); + const arr = (this._previewSegments && previewIsImage) ? this._previewSegments : this.timeline.segments; + seg = arr.find(s => s.id === origSeg.id) || origSeg; + } + } + } + + if (this.selectionType === "audio" && seg) { + this.promptInput.style.display = "none"; + this.strengthRow.style.display = "flex"; + this.audioInfoArea.style.display = "block"; + this.audioInfoArea.innerHTML = ` + File: ${seg.fileName || "Unknown"}
+ Length: ${this.formatTime(seg.audioDurationFrames)} Output Length: ${this.formatTime(seg.length)}
+ Trim-in: ${this.formatTime(Math.round(seg.trimStart))} Trim-Out: ${this.formatTime(Math.round(seg.audioDurationFrames - (seg.trimStart + seg.length)))} + `; + this.strengthValue.value = "1.00"; + this.strengthValue.disabled = true; + } else { + this.audioInfoArea.style.display = "none"; + this.promptInput.style.display = "block"; + this.strengthRow.style.display = "flex"; + + if (seg) { + this.promptInput.value = seg.prompt || ""; + this.promptInput.disabled = false; + + const isImage = seg.type !== "text"; + const strength = isImage ? (seg.guideStrength ?? 1.0) : 1.0; + this.strengthValue.value = strength.toFixed(2); + this.strengthValue.disabled = !isImage; + } else { + this.promptInput.value = ""; + this.promptInput.disabled = true; + this.strengthValue.value = "1.00"; + this.strengthValue.disabled = true; + } + } + + if (this.segmentBoundsDisplay) { + if (seg) { + const startStr = this.formatTime(seg.start, true); + const endStr = this.formatTime(seg.start + seg.length, true); + this.segmentBoundsDisplay.textContent = `Start: ${startStr} | End: ${endStr}`; + } else { + this.segmentBoundsDisplay.textContent = "Start: - | End: -"; + } + } + } + + // --- Rendering logic --- + render() { + const width = this.canvas.offsetWidth || this._lastWidth; + const height = this.canvasHeight; + const totalFrames = this.getVisualDurationFrames(); + + if (!width || width <= 0) return; + + this.ctx.clearRect(0, 0, width, height); + + + + // Render Track Backgrounds + this.ctx.fillStyle = "#111"; // Image track bg + this.ctx.fillRect(0, RULER_HEIGHT, width, this.blockHeight); + this.ctx.fillStyle = "#111"; // Audio track bg + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight, width, this.audioTrackHeight); + + + + // Determine which track the preview belongs to. + // _ghostTrack is set during HTML file drag-and-drop. + // During canvas mouse drags, _ghostTrack is null, so fall back to selectionType. + const previewIsAudio = this._ghostTrack === 'audio' || + (this._previewSegments && this._ghostTrack === null && this.selectionType === 'audio'); + + let renderSegments = (this._previewSegments && !previewIsAudio) + ? this._previewSegments : this.timeline.segments; + + let renderAudioSegments = (this._previewSegments && previewIsAudio) + ? this._previewSegments : this.timeline.audioSegments; + + + + const activeSegId = this.timeline.segments[this.selectedIndex]?.id; + const activeAudioSegId = this.timeline.audioSegments[this.selectedIndex]?.id; + + // Sort segments so that the selected one is drawn last (on top) + const isImageSelection = this.selectionType === "image"; + const sortedSegments = [...renderSegments].sort((a, b) => { + const aSel = isImageSelection && a.id === activeSegId; + const bSel = isImageSelection && b.id === activeSegId; + return aSel - bSel; + }); + + const isAudioSelection = this.selectionType === "audio"; + const sortedAudioSegments = [...renderAudioSegments].sort((a, b) => { + const aSel = isAudioSelection && a.id === activeAudioSegId; + const bSel = isAudioSelection && b.id === activeAudioSegId; + return aSel - bSel; + }); + + // --- Draw Image/Text Segments --- + for (let i = 0; i < sortedSegments.length; i++) { + const seg = sortedSegments[i]; + const startX = (seg.start / totalFrames) * width; + const pxWidth = (seg.length / totalFrames) * width; + const isSelected = (this.selectionType === "image" && seg.id === activeSegId); + + const originalSeg = this.timeline.segments.find(s => s.id === seg.id); + const imgObj = originalSeg ? originalSeg.imgObj : seg.imgObj; + + if ((this._isDragging && this.selectionType === "image" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { + this.ctx.globalAlpha = 0.65; + } else { + this.ctx.globalAlpha = 1.0; + } + + if (seg.type === "ghost") { + this.ctx.fillStyle = "#2a2a2a"; + this.ctx.fillRect(startX, RULER_HEIGHT, pxWidth, this.blockHeight); + + this.ctx.strokeStyle = "#777"; + this.ctx.lineWidth = 2; + this.ctx.setLineDash([5, 5]); + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + this.ctx.setLineDash([]); + + this.ctx.fillStyle = "#aaa"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "bold 12px sans-serif"; + this.ctx.fillText("Drop to Place", startX + pxWidth / 2, RULER_HEIGHT + this.blockHeight / 2); + } else { + this.ctx.fillStyle = seg.type === "text" ? "#000b12" : "#000"; + this.ctx.fillRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + } + + if (imgObj && imgObj.complete && imgObj.naturalWidth > 0 && seg.type !== "ghost") { + const imgRatio = imgObj.naturalWidth / imgObj.naturalHeight; + const boxRatio = pxWidth / this.blockHeight; + let drawW, drawH, drawX, drawY; + if (imgRatio > boxRatio) { + drawW = pxWidth; drawH = pxWidth / imgRatio; + drawX = startX; drawY = RULER_HEIGHT + (this.blockHeight - drawH) / 2; + } else { + drawH = this.blockHeight; drawW = this.blockHeight * imgRatio; + drawY = RULER_HEIGHT; drawX = startX + (pxWidth - drawW) / 2; + } + + // Clip to segment bounds so tiled images don't bleed into adjacent segments + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + this.ctx.clip(); + + if (imgRatio > boxRatio) { + // Fits width, vertical letterboxing (black bars top/bottom) — keep as is + this.ctx.drawImage(imgObj, drawX, drawY, drawW, drawH); + } else { + // Fits height, horizontal letterboxing (black bars left/right) — tile horizontally + this.ctx.drawImage(imgObj, drawX, drawY, drawW, drawH); + + // Tile left + let leftX = drawX - drawW; + while (leftX + drawW > startX) { + this.ctx.drawImage(imgObj, leftX, drawY, drawW, drawH); + leftX -= drawW; + } + + // Tile right + let rightX = drawX + drawW; + while (rightX < startX + pxWidth) { + this.ctx.drawImage(imgObj, rightX, drawY, drawW, drawH); + rightX += drawW; + } + } + this.ctx.restore(); + + // --- Prompt subtitle overlay --- + if (seg.prompt && seg.type !== "ghost" && pxWidth > 24) { + const overlayH = Math.round(this.blockHeight * 0.20); + const overlayY = RULER_HEIGHT + this.blockHeight - overlayH; + + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX, overlayY, pxWidth, overlayH); + this.ctx.clip(); + + // Translucent background + this.ctx.fillStyle = "rgba(0, 0, 0, 0.70)"; + this.ctx.fillRect(startX, overlayY, pxWidth, overlayH); + + // Text + const fontSize = Math.min(11, overlayH * 0.58); + this.ctx.font = `${fontSize}px sans-serif`; + this.ctx.fillStyle = "#e0e3ed"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + + // Measure and truncate to single line + const maxTextW = pxWidth - 10; + let label = seg.prompt; + if (this.ctx.measureText(label).width > maxTextW) { + while (label.length > 0 && this.ctx.measureText(label + "…").width > maxTextW) { + label = label.slice(0, -1); + } + label += "…"; + } + + this.ctx.fillText(label, startX + pxWidth / 2, overlayY + overlayH / 2); + this.ctx.restore(); + } + } else if (seg.type === "text") { + const pad = 8; + const boxW = pxWidth - pad * 2; + if (boxW > 12) { + this.ctx.save(); + this.ctx.beginPath(); + this.ctx.rect(startX + pad, RULER_HEIGHT + pad, boxW, this.blockHeight - pad * 2); + this.ctx.clip(); + this.ctx.fillStyle = "#e0e3ed"; + this.ctx.font = "11px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "top"; + const label = seg.prompt || "(no prompt)"; + const words = label.split(" "); + const lineH = 15; + let line = ""; + let lines = []; + for (const word of words) { + const test = line ? line + " " + word : word; + if (this.ctx.measureText(test).width > boxW && line) { + lines.push(line); + line = word; + } else { + line = test; + } + } + if (line) lines.push(line); + + const maxLines = Math.max(1, Math.floor((this.blockHeight - pad * 2) / lineH)); + if (lines.length > maxLines) { + lines = lines.slice(0, maxLines); + lines[lines.length - 1] += "…"; + } + + const totalTextHeight = lines.length * lineH; + let ty = RULER_HEIGHT + (this.blockHeight - totalTextHeight) / 2 + 2; + + for (const l of lines) { + this.ctx.fillText(l, startX + pxWidth / 2, ty); + ty += lineH; + } + this.ctx.restore(); + } + } + + if (isSelected) { + this.ctx.strokeStyle = "#fff"; + this.ctx.lineWidth = 2; + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + this.ctx.fillStyle = "#fff"; + this.ctx.beginPath(); + this.ctx.roundRect(startX, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + this.ctx.beginPath(); + this.ctx.roundRect(startX + pxWidth - 4, RULER_HEIGHT + this.blockHeight / 2 - 12, 4, 24, 2); + this.ctx.fill(); + } else { + this.ctx.strokeStyle = "#000"; + this.ctx.lineWidth = 1.5; + this.ctx.strokeRect(startX, RULER_HEIGHT + 1, pxWidth, this.blockHeight - 2); + } + this.ctx.globalAlpha = 1.0; + } + + // --- Draw Audio Segments --- + for (let i = 0; i < sortedAudioSegments.length; i++) { + const seg = sortedAudioSegments[i]; + const startX = (seg.start / totalFrames) * width; + const pxWidth = (seg.length / totalFrames) * width; + const isSelected = (this.selectionType === "audio" && seg.id === activeAudioSegId); + const trackY = RULER_HEIGHT + this.blockHeight; + + if ((this._isDragging && this.selectionType === "audio" && seg.id === this._dragTargetId) || (this._ghostSegmentId && seg.id === this._ghostSegmentId)) { + this.ctx.globalAlpha = 0.65; + } else { + this.ctx.globalAlpha = 1.0; + } + + if (seg.type === "ghost") { + this.ctx.fillStyle = "#1a1a1a"; + this.ctx.fillRect(startX, trackY, pxWidth, this.audioTrackHeight); + this.ctx.strokeStyle = "#555"; + this.ctx.lineWidth = 2; + this.ctx.setLineDash([5, 5]); + this.ctx.strokeRect(startX, trackY, pxWidth, this.audioTrackHeight); + this.ctx.setLineDash([]); + this.ctx.fillStyle = "#888"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "bold 12px sans-serif"; + this.ctx.fillText("Drop Audio", startX + pxWidth / 2, trackY + this.audioTrackHeight / 2); + } else { + this.drawAudioSegmentVisuals(this.ctx, seg, isSelected, trackY, this.audioTrackHeight, startX, pxWidth); + } + this.ctx.globalAlpha = 1.0; + } + + // --- Draw Ruler & Divider AFTER segments to prevent overlap --- + // Ruler Background + this.ctx.fillStyle = "#1e1e1e"; + this.ctx.fillRect(0, 0, width, RULER_HEIGHT); + + // Crisp Ruler Text + this.ctx.fillStyle = "#aaa"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.font = "10px sans-serif"; + + const frameRate = this.getFrameRate(); + const mode = this.displayModeWidget ? this.displayModeWidget.value : "seconds"; + + // Define logical steps for both modes + let steps; + if (mode === "seconds") { + steps = [0.1, 0.2, 0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600]; + } else { + steps = [1, 2, 5, 10, 24, 48, 120, 240, 480, 960, 1920]; + } + + const minSpacingPx = 60; + let majorStep = steps[steps.length - 1]; + for (let i = 0; i < steps.length; i++) { + const stepFrames = mode === "seconds" ? steps[i] * frameRate : steps[i]; + const spacingPx = (stepFrames / totalFrames) * width; + if (spacingPx >= minSpacingPx) { + majorStep = steps[i]; + break; + } + } + + const majorStepFrames = mode === "seconds" ? majorStep * frameRate : majorStep; + + let minorStep; + if (mode === "seconds") { + if (majorStep <= 0.2) minorStep = majorStep / 2; + else if (majorStep <= 1) minorStep = majorStep / 5; + else if (majorStep <= 5) minorStep = 1; + else if (majorStep <= 15) minorStep = 5; + else if (majorStep <= 30) minorStep = 10; + else if (majorStep <= 60) minorStep = 10; + else minorStep = majorStep / 5; + } else { + if (majorStep <= 5) minorStep = 1; + else if (majorStep <= 10) minorStep = 2; + else if (majorStep <= 24) minorStep = 6; + else if (majorStep <= 48) minorStep = 12; + else minorStep = majorStep / 5; + } + const minorStepFrames = mode === "seconds" ? minorStep * frameRate : minorStep; + + this.ctx.fillStyle = "#444"; + const totalMinorTicks = Math.floor(totalFrames / minorStepFrames); + for (let i = 0; i <= totalMinorTicks; i++) { + const frameVal = i * minorStepFrames; + if (Math.abs(frameVal % majorStepFrames) < 0.1) continue; + + const x = (frameVal / totalFrames) * width; + this.ctx.fillRect(Math.floor(x), RULER_HEIGHT - 3, 1, 3); + } + + this.ctx.fillStyle = "#aaa"; + const totalMajorTicks = Math.floor(totalFrames / majorStepFrames); + for (let i = 0; i <= totalMajorTicks; i++) { + const frameVal = i * majorStepFrames; + const x = (frameVal / totalFrames) * width; + + this.ctx.fillStyle = "#aaa"; + this.ctx.fillRect(Math.floor(x), RULER_HEIGHT - 6, 1, 6); + + if (frameVal > 0 && frameVal < totalFrames) { + this.ctx.textAlign = "center"; + this.ctx.fillText(this.formatTime(frameVal, true), x, RULER_HEIGHT / 2); + } + } + + this.ctx.textAlign = "left"; + const zeroLabel = mode === "seconds" ? "0" : this.formatTime(0, true); + this.ctx.fillText(zeroLabel, 4, RULER_HEIGHT / 2); + + // Divider + this.ctx.fillStyle = "#333"; + this.ctx.fillRect(0, RULER_HEIGHT + this.blockHeight, width, 1); + + // Draw gap "+" buttons + if (!this._isDragging) { + const BTN_R = 12; + const gapRegions = this.getGapRegions(); + for (let i = 0; i < gapRegions.length; i++) { + const gap = gapRegions[i]; + if (gap.widthPx < BTN_R * 2 + 8) continue; + const hov = this._hoveredGapIdx === i; + const BTN_W = 18; + const BTN_H = 18; + this.ctx.beginPath(); + this.ctx.roundRect(gap.centerX - BTN_W / 2, gap.centerY - BTN_H / 2, BTN_W, BTN_H, 4); + this.ctx.fillStyle = hov ? "rgba(255,255,255,0.15)" : "rgba(255,255,255,0.05)"; + this.ctx.fill(); + this.ctx.fillStyle = hov ? "#fff" : "#888"; + this.ctx.font = "14px sans-serif"; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + this.ctx.fillText("+", gap.centerX, gap.centerY + 1); + } + } + + // --- Out-of-duration shadow overlay --- + // Draw a translucent black mask over the region beyond the actual output duration + // so the user can clearly see which content will be included in the render. + const outputFrames = this.getDurationFrames(); + if (outputFrames < totalFrames) { + const cutoffX = (outputFrames / totalFrames) * width; + // Semi-transparent black overlay on both tracks + this.ctx.fillStyle = "rgba(0, 0, 0, 0.45)"; + this.ctx.fillRect(cutoffX, RULER_HEIGHT, width - cutoffX, this.blockHeight + this.audioTrackHeight); + // Subtle tinted ruler overlay + this.ctx.fillStyle = "rgba(0, 0, 0, 0.25)"; + this.ctx.fillRect(cutoffX, 0, width - cutoffX, RULER_HEIGHT); + /* + // Dashed boundary line at the output duration cutoff + this.ctx.save(); + this.ctx.strokeStyle = "rgba(255, 80, 80, 0.7)"; + this.ctx.lineWidth = 1.5; + this.ctx.setLineDash([5, 4]); + this.ctx.beginPath(); + this.ctx.moveTo(cutoffX, 0); + this.ctx.lineTo(cutoffX, CANVAS_HEIGHT); + this.ctx.stroke(); + this.ctx.setLineDash([]); + this.ctx.restore(); + */ + } + + // --- Draw Playhead --- + const playheadX = (this.currentFrame / totalFrames) * width; + + // Playhead Line + this.ctx.beginPath(); + this.ctx.moveTo(playheadX, 14); + this.ctx.lineTo(playheadX, this.canvasHeight); + this.ctx.strokeStyle = "#ff4444"; + this.ctx.lineWidth = 1.5; + this.ctx.stroke(); + + // Playhead Handle (Polygon above numbers) + this.ctx.fillStyle = "#ff4444"; + this.ctx.beginPath(); + this.ctx.moveTo(playheadX - 6, 0); + this.ctx.lineTo(playheadX + 6, 0); + this.ctx.lineTo(playheadX + 6, 8); + this.ctx.lineTo(playheadX, 14); + this.ctx.lineTo(playheadX - 6, 8); + this.ctx.fill(); + + // Draw vertical grab bar on the right edge of viewport for resizing width + const grabBarW = 4; + const grabBarH = 50; + const grabBarX = this.viewport.scrollLeft + this.viewport.clientWidth - grabBarW - 3; + const grabBarY = RULER_HEIGHT + (this.blockHeight + this.audioTrackHeight - grabBarH) / 2; + + this.ctx.fillStyle = "rgba(40, 40, 40, 0.6)"; + this.ctx.beginPath(); + this.ctx.roundRect(grabBarX, grabBarY, grabBarW, grabBarH, 2); + this.ctx.fill(); + + // Draw horizontal grab bar at the bottom of viewport for resizing height + const hBarW = 50; + const hBarH = 4; + const hBarX = this.viewport.scrollLeft + (this.viewport.clientWidth - hBarW) / 2; + const hBarY = this.canvasHeight - hBarH - 3; // 3px from the bottom edge + + this.ctx.fillStyle = "rgba(20, 20, 20, 0.8)"; + this.ctx.beginPath(); + this.ctx.roundRect(hBarX, hBarY, hBarW, hBarH, 2); + this.ctx.fill(); + + this.updatePlayerUI(); + } + + drawAudioSegmentVisuals(ctx, seg, isSelected, yOffset, trackHeight, startX, pxWidth) { + ctx.fillStyle = isSelected ? "#2a4a3a" : "#1a2a1a"; + ctx.fillRect(startX, yOffset + 2, pxWidth, trackHeight - 3); + + if (seg.waveformPeaks && pxWidth > 0) { + ctx.fillStyle = isSelected ? "rgba(100, 255, 100, 0.6)" : "rgba(100, 255, 100, 0.3)"; + const startRatio = seg.trimStart / seg.audioDurationFrames; + const endRatio = (seg.trimStart + seg.length) / seg.audioDurationFrames; + const peakCount = seg.waveformPeaks.length; + const centerY = yOffset + trackHeight / 2; + + ctx.beginPath(); + for (let i = 0; i < pxWidth; i++) { + const pixelRatio = i / pxWidth; + const globalRatio = startRatio + pixelRatio * (endRatio - startRatio); + const peakIdx = Math.floor(globalRatio * peakCount); + + if (peakIdx >= 0 && peakIdx < peakCount) { + const val = seg.waveformPeaks[peakIdx]; + const amp = (val * (trackHeight - 12) / 2) * 0.9; + ctx.fillRect(startX + i, centerY - amp, 1, amp * 2); + } + } + } + + ctx.strokeStyle = isSelected ? "#4fff8f" : "#000"; + ctx.lineWidth = 1.5; + ctx.strokeRect(startX, yOffset + 2, pxWidth, trackHeight - 3); + + if (isSelected) { + ctx.fillStyle = "#4fff8f"; + ctx.beginPath(); + ctx.roundRect(startX, yOffset + trackHeight / 2 - 12, 4, 24, 2); + ctx.fill(); + ctx.beginPath(); + ctx.roundRect(startX + pxWidth - 4, yOffset + trackHeight / 2 - 12, 4, 24, 2); + ctx.fill(); + } + + ctx.fillStyle = "#ccc"; + ctx.font = "11px sans-serif"; + ctx.textBaseline = "top"; + ctx.textAlign = "left"; + ctx.save(); + ctx.beginPath(); + ctx.rect(startX, yOffset + 2, pxWidth, trackHeight - 3); + ctx.clip(); + + let text = seg.fileName || "Audio Track"; + const maxWidth = pxWidth - 12; + if (ctx.measureText(text).width > maxWidth && maxWidth > 0) { + while (text.length > 0 && ctx.measureText(text + "...").width > maxWidth) { + text = text.slice(0, -1); + } + text = text + "..."; + } + + ctx.fillText(text, startX + 6, yOffset + 8); + ctx.restore(); + } + + + // --- Interaction Logic --- + getHitTest(mouseX, mouseY) { + const width = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + + // Check Playhead Handle first + const playheadX = (this.currentFrame / totalFrames) * width; + if (mouseY <= 24 && Math.abs(mouseX - playheadX) <= 12) { + return { type: "playhead" }; + } + + if (mouseY <= RULER_HEIGHT) { + return { type: "ruler" }; + } + + if (mouseY < RULER_HEIGHT || mouseY > this.canvasHeight) return null; + + const isAudioTrack = mouseY > RULER_HEIGHT + this.blockHeight; + const trackSegments = isAudioTrack ? this.timeline.audioSegments : this.timeline.segments; + const trackType = isAudioTrack ? "audio" : "image"; + + if (trackSegments.length === 0) return null; + + // The variables width and totalFrames are already declared above. + + let sortedSegments = [...trackSegments] + .map((s, i) => ({ ...s, originalIndex: i })) + .sort((a, b) => a.start - b.start); + + const HANDLE_CORE = 4; + + for (let i = 0; i < sortedSegments.length; i++) { + const seg = sortedSegments[i]; + const startX = (seg.start / totalFrames) * width; + const pxWidth = (seg.length / totalFrames) * width; + const endX = startX + pxWidth; + + const prevSeg = sortedSegments[i - 1]; + const nextSeg = sortedSegments[i + 1]; + + const isLeftJoint = prevSeg && prevSeg.start + prevSeg.length === seg.start; + if (!isLeftJoint) { + if (Math.abs(mouseX - startX) <= HANDLE_HIT_PX) { + return { type: "edge", index: seg.originalIndex, dir: "left", track: trackType }; + } + } + + const isRightJoint = nextSeg && nextSeg.start === seg.start + seg.length; + if (isRightJoint) { + const dx = mouseX - endX; + if (Math.abs(dx) <= HANDLE_HIT_PX) { + if (dx < -HANDLE_CORE) { + return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + } else if (dx > HANDLE_CORE) { + return { type: "edge", index: nextSeg.originalIndex, dir: "left", track: trackType }; + } else { + return { type: "joint", leftIndex: seg.originalIndex, rightIndex: nextSeg.originalIndex, track: trackType }; + } + } + } else { + if (Math.abs(mouseX - endX) <= HANDLE_HIT_PX) { + return { type: "edge", index: seg.originalIndex, dir: "right", track: trackType }; + } + } + } + + for (let i = 0; i < sortedSegments.length; i++) { + const seg = sortedSegments[i]; + const startX = (seg.start / totalFrames) * width; + const pxWidth = (seg.length / totalFrames) * width; + const endX = startX + pxWidth; + + if (mouseX >= startX && mouseX < endX) { + return { type: "center", index: seg.originalIndex, track: trackType }; + } + } + + return null; + } + + onMouseDown(e) { + if (e.button !== 0) return; + const { x, y } = this.getMousePos(e); + + const isOverDivider = Math.abs(y - (RULER_HEIGHT + this.blockHeight)) <= 4; + if (isOverDivider) { + this._isDragging = true; + this._dragType = "divider"; + this._startBlockHeight = this.blockHeight; + this._startAudioTrackHeight = this.audioTrackHeight; + this._startY = y; + return; + } + + const isAtBottom = Math.abs(y - this.canvasHeight) <= 15; + if (isAtBottom) { + this._isDragging = true; + this._dragType = "height_resize"; + this._startBlockHeight = this.blockHeight; + this._startY = y; + document.body.style.userSelect = "none"; + return; + } + + const viewRect = this.viewport.getBoundingClientRect(); + const isAtRightEdge = Math.abs(e.clientX - viewRect.right) <= 20; + if (isAtRightEdge) { + this._isDragging = true; + this._dragType = "width_resize"; + this._startNodeWidth = this.node.size[0]; + this._startX = e.clientX; + document.body.style.userSelect = "none"; + return; + } + + if (y >= RULER_HEIGHT && y <= this.canvasHeight) { + const BTN_R = 12; + const gapRegions = this.getGapRegions(); + for (let i = 0; i < gapRegions.length; i++) { + const gap = gapRegions[i]; + if (gap.widthPx < BTN_R * 2 + 8) continue; + const dx = x - gap.centerX, dy2 = y - gap.centerY; + if (dx * dx + dy2 * dy2 <= BTN_R * BTN_R) { + if (gap.track === "audio") { + // Direct to audio upload + this.promptAddAudioInGap(gap.frameStart, gap.frameEnd); + } else { + this.showGapMenu(e.clientX, e.clientY, gap); + } + return; + } + } + } + + const hit = this.getHitTest(x, y); + if (!hit) { + // Only deselect if they clicked the same track but hit empty space + const clickedTrack = y > RULER_HEIGHT + this.blockHeight ? "audio" : "image"; + if (this.selectionType === clickedTrack) { + this.selectedIndex = -1; + this.updateUIFromSelection(); + } + this.render(); + return; + } + + if (hit.type === "playhead" || hit.type === "ruler") { + this._isDragging = true; + this._dragType = "playhead"; + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + let mouseFrameX = x * (totalFrames / logicalWidth); + this.currentFrame = clamp(mouseFrameX, 0, totalFrames); + this.render(); + if (this.isPlaying) { + this.playAudio(); + } + return; + } + + this.selectionType = hit.track; + const targetArray = hit.track === "audio" ? this.timeline.audioSegments : this.timeline.segments; + + if (hit.type === "joint") { + this.selectedIndex = hit.leftIndex; + this.updateUIFromSelection(); + this._dragType = "joint"; + this._dragTargetId = targetArray[hit.leftIndex].id; + this._dragTargetIdRight = targetArray[hit.rightIndex].id; + } else if (hit.type === "center") { + this.selectedIndex = hit.index; + this.updateUIFromSelection(); + this._dragType = "center"; + } else { + if (this.selectedIndex !== hit.index) { + this.selectedIndex = hit.index; + this.updateUIFromSelection(); + } + this._dragType = hit.dir; + } + + this._isDragging = true; + this._previewSegments = null; + this._dragStartX = x; + this._dragInitialTimeline = JSON.parse(JSON.stringify(targetArray)); + + if (hit.type !== "joint") { + this._dragTargetId = targetArray[hit.index].id; + } + this.render(); + } + + onMouseMove(e) { + const { x: mouseX, y: mouseY } = this.getMousePos(e); + + if (!this._isDragging) { + let newHoveredGapIdx = -1; + const BTN_R = 12; + const gapRegions = this.getGapRegions(); + for (let i = 0; i < gapRegions.length; i++) { + const gap = gapRegions[i]; + if (gap.widthPx < BTN_R * 2 + 8) continue; + const dx = mouseX - gap.centerX, dy2 = mouseY - gap.centerY; + if (dx * dx + dy2 * dy2 <= BTN_R * BTN_R) { newHoveredGapIdx = i; break; } + } + if (this._hoveredGapIdx !== newHoveredGapIdx) { + this._hoveredGapIdx = newHoveredGapIdx; + this.render(); + } + + const isOverDivider = Math.abs(mouseY - (RULER_HEIGHT + this.blockHeight)) <= 4; + const isAtBottom = Math.abs(mouseY - this.canvasHeight) <= 15; + const viewRect = this.viewport.getBoundingClientRect(); + const isAtRightEdge = Math.abs(e.clientX - viewRect.right) <= 20; + const hit = this.getHitTest(mouseX, mouseY); + if (isOverDivider || isAtBottom) { + this.canvas.style.cursor = "ns-resize"; + } else if (isAtRightEdge) { + this.canvas.style.cursor = "ew-resize"; + } else if (newHoveredGapIdx >= 0) { + this.canvas.style.cursor = "pointer"; + } else if (hit?.type === "edge") { + this.canvas.style.cursor = "ew-resize"; + } else if (hit?.type === "joint") { + this.canvas.style.cursor = "col-resize"; + } else if (hit?.type === "center") { + this.canvas.style.cursor = "grab"; + } else if (hit?.type === "playhead") { + this.canvas.style.cursor = "ew-resize"; + } else { + this.canvas.style.cursor = "default"; + } + return; + } + + if (this._dragType === "divider") { + this.canvas.style.cursor = "ns-resize"; + const deltaY = mouseY - this._startY; + + const minBlockH = 50; + const minAudioH = 50; + + let newBlockHeight = this._startBlockHeight + deltaY; + let newAudioTrackHeight = this._startAudioTrackHeight - deltaY; + + if (newBlockHeight < minBlockH) { + newBlockHeight = minBlockH; + newAudioTrackHeight = this._startBlockHeight + this._startAudioTrackHeight - minBlockH; + } + if (newAudioTrackHeight < minAudioH) { + newAudioTrackHeight = minAudioH; + newBlockHeight = this._startBlockHeight + this._startAudioTrackHeight - minAudioH; + } + + this.blockHeight = newBlockHeight; + this.audioTrackHeight = newAudioTrackHeight; + + this.render(); + return; + } + + if (this._dragType === "height_resize") { + this.canvas.style.cursor = "ns-resize"; + const deltaY = mouseY - this._startY; + + this.blockHeight = Math.max(100, this._startBlockHeight + deltaY); + this.canvasHeight = this.rulerHeight + this.blockHeight + this.audioTrackHeight; + + this.canvas.style.height = `${this.canvasHeight}px`; + + this.resizeCanvas(this.canvas.offsetWidth); + this.render(); + + if (this.node && this.node.computeSize) { + const sz = this.node.computeSize(); + this.node.size[1] = sz[1]; + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + } + return; + } + + if (this._dragType === "width_resize") { + this.canvas.style.cursor = "ew-resize"; + const deltaX = e.clientX - this._startX; + + this.node.size[0] = Math.max(300, this._startNodeWidth + deltaX); + + if (window.app && window.app.graph) { + window.app.graph.setDirtyCanvas(true, true); + } + return; + } + + if (this._dragType === "playhead") { + this.canvas.style.cursor = "ew-resize"; + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + let mouseFrameX = mouseX * (totalFrames / logicalWidth); + this.currentFrame = clamp(mouseFrameX, 0, totalFrames); + this.render(); + if (this.isPlaying) { + this.playAudio(); // Scrub (restart from new position) + } + return; + } + + this.canvas.style.cursor = this._dragType === "center" ? "grabbing" : + this._dragType === "joint" ? "col-resize" : "ew-resize"; + + const logicalWidth = this.canvas.offsetWidth; + const totalFrames = this.getVisualDurationFrames(); + const durationFrames = totalFrames; + const dragDelta = Math.round((mouseX - this._dragStartX) * (totalFrames / logicalWidth)); + + let t = JSON.parse(JSON.stringify(this._dragInitialTimeline)); + + // --- Rolling Edit (Slide Edit) --- + if (this._dragType === "joint") { + let leftIdx = t.findIndex(s => s.id === this._dragTargetId); + let rightIdx = t.findIndex(s => s.id === this._dragTargetIdRight); + + if (leftIdx >= 0 && rightIdx >= 0) { + let origLeft = this._dragInitialTimeline.find(s => s.id === this._dragTargetId); + let origRight = this._dragInitialTimeline.find(s => s.id === this._dragTargetIdRight); + + let maxDeltaRight = origRight.length - MIN_SEGMENT_LENGTH; + let maxDeltaLeft = origLeft.length - MIN_SEGMENT_LENGTH; + + if (this.selectionType === "audio") { + // Drag LEFT: right clip extends left by un-trimming its head. + // Can only un-trim as much as the right clip has been trimmed (trimStart >= 0). + maxDeltaLeft = Math.min(maxDeltaLeft, origRight.trimStart || 0); + // Drag RIGHT: left clip extends right by consuming its remaining tail audio. + // Can only extend as far as the left clip's unplayed tail allows. + let availLeftTail = (origLeft.audioDurationFrames || origLeft.length) - ((origLeft.trimStart || 0) + origLeft.length); + maxDeltaRight = Math.min(maxDeltaRight, availLeftTail); + } + + let safeDelta = clamp(dragDelta, -maxDeltaLeft, maxDeltaRight); + + t[leftIdx].length = origLeft.length + safeDelta; + t[rightIdx].start = origRight.start + safeDelta; + t[rightIdx].length = origRight.length - safeDelta; + + if (this.selectionType === "audio") { + t[rightIdx].trimStart = origRight.trimStart + safeDelta; + } + } + } + // --- Edge & Center Drags --- + else { + const targetIdx = t.findIndex((s) => s.id === this._dragTargetId); + if (targetIdx < 0) return; + + if (this._dragType === "right") { + let newLen = t[targetIdx].length + dragDelta; + let maxPossibleLength = totalFrames - t[targetIdx].start; + let nextSeg = t.find(s => s.start >= t[targetIdx].start + t[targetIdx].length && s.id !== t[targetIdx].id); + if (nextSeg) { + maxPossibleLength = nextSeg.start - t[targetIdx].start; + } + + if (this.selectionType === "audio") { + maxPossibleLength = Math.min(maxPossibleLength, (t[targetIdx].audioDurationFrames || t[targetIdx].length) - (t[targetIdx].trimStart || 0)); + } + + t[targetIdx].length = Math.max(MIN_SEGMENT_LENGTH, Math.min(newLen, maxPossibleLength)); + + } else if (this._dragType === "left") { + let newStart = t[targetIdx].start + dragDelta; + let minPossibleStart = 0; + let prevSeg = t.slice().reverse().find(s => s.start + s.length <= t[targetIdx].start && s.id !== t[targetIdx].id); + if (prevSeg) { + minPossibleStart = prevSeg.start + prevSeg.length; + } + + if (this.selectionType === "audio") { + minPossibleStart = Math.max(minPossibleStart, t[targetIdx].start - (t[targetIdx].trimStart || 0)); + } + + let maxStart = t[targetIdx].start + t[targetIdx].length - MIN_SEGMENT_LENGTH; + newStart = Math.max(minPossibleStart, Math.min(newStart, maxStart)); + + let diff = newStart - t[targetIdx].start; + t[targetIdx].start = newStart; + t[targetIdx].length -= diff; + if (this.selectionType === "audio") { + t[targetIdx].trimStart += diff; + } + + } else if (this._dragType === "center") { + let initT = this._dragInitialTimeline; + let dIdx = initT.findIndex(s => s.id === this._dragTargetId); + if (dIdx < 0) return; + let D = JSON.parse(JSON.stringify(initT[dIdx])); + + let D_mouse_start = D.start + dragDelta; + let mouseFrameX = mouseX * (totalFrames / logicalWidth); + + t = this._applyCenterDragPhysics(initT, D.id, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth); + } + } + + this._previewSegments = t; + this.updateUIFromSelection(); // Live update of trim values + this.render(); + } + + _applyCenterDragPhysics(initT, D_id, D_mouse_start, mouseFrameX, durationFrames, totalFrames, logicalWidth) { + let t_copy = JSON.parse(JSON.stringify(initT)); + let dIdx = t_copy.findIndex(s => s.id === D_id); + if (dIdx < 0) return t_copy; + + let D = t_copy[dIdx]; + let D_clamped_start = clamp(D_mouse_start, 0, durationFrames - D.length); + + let baseSegments = t_copy.filter(s => s.id !== D.id); + + let insertIdx = baseSegments.length; + for (let i = 0; i < baseSegments.length; i++) { + let centerBase = baseSegments[i].start + baseSegments[i].length / 2; + if (mouseFrameX < centerBase) { + insertIdx = i; + break; + } + } + + let leftBound = insertIdx > 0 ? baseSegments[insertIdx - 1].start + baseSegments[insertIdx - 1].length : 0; + let rightBound = insertIdx < baseSegments.length ? baseSegments[insertIdx].start : durationFrames; + + if (rightBound - leftBound >= D.length) { + D_clamped_start = clamp(D_clamped_start, leftBound, rightBound - D.length); + } else { + let gapCenter = (leftBound + rightBound) / 2; + D_clamped_start = gapCenter - D.length / 2; + } + + let t_test = []; + for (let i = 0; i < insertIdx; i++) { + t_test.push({ ...baseSegments[i], original_start: baseSegments[i].start }); + } + t_test.push({ ...D, start: D_clamped_start, original_start: D_clamped_start }); + let D_index = insertIdx; + + for (let i = insertIdx; i < baseSegments.length; i++) { + t_test.push({ ...baseSegments[i], original_start: baseSegments[i].start }); + } + + for (let i = D_index + 1; i < t_test.length; i++) { + let prev = t_test[i - 1]; + t_test[i].start = Math.max(t_test[i].original_start, prev.start + prev.length); + } + + for (let i = D_index - 1; i >= 0; i--) { + let next = t_test[i + 1]; + t_test[i].start = Math.min(t_test[i].original_start, next.start - t_test[i].length); + } + + let rightCursor = durationFrames; + for (let i = t_test.length - 1; i >= 0; i--) { + if (t_test[i].start + t_test[i].length > rightCursor) { + t_test[i].start = rightCursor - t_test[i].length; + } + rightCursor = t_test[i].start; + } + let leftCursor = 0; + for (let i = 0; i < t_test.length; i++) { + if (t_test[i].start < leftCursor) { + t_test[i].start = leftCursor; + } + leftCursor = t_test[i].start + t_test[i].length; + } + + let result = t_test.map(s => { + let clean = { ...s }; + delete clean.original_start; + return clean; + }); + + let draggedPreview = result.find(s => s.id === D.id); + if (draggedPreview) { + draggedPreview.resolvedStart = draggedPreview.start; + } + + return result; + } + + onMouseUp(e) { + document.body.style.userSelect = ""; + if (this._isDragging) { + if (this._previewSegments) { + const targetArray = this.selectionType === "audio" ? this.timeline.audioSegments : this.timeline.segments; + + const mappedArray = this._previewSegments.map(ps => { + const orig = targetArray.find(s => s.id === ps.id); + let finalStart = ps.resolvedStart !== undefined ? ps.resolvedStart : ps.start; + let newPs = { ...ps, start: finalStart }; + if (orig && orig.imgObj) newPs.imgObj = orig.imgObj; + delete newPs.resolvedStart; + return newPs; + }); + + if (this.selectionType === "audio") { + this.timeline.audioSegments = mappedArray; + if (this._dragTargetId) this.selectedIndex = this.timeline.audioSegments.findIndex(s => s.id === this._dragTargetId); + } else { + this.timeline.segments = mappedArray; + if (this._dragTargetId) this.selectedIndex = this.timeline.segments.findIndex(s => s.id === this._dragTargetId); + } + } + + this._isDragging = false; + this._previewSegments = null; + this._ghostTrack = null; + this.canvas.style.cursor = "default"; + this.commitChanges(); + } + } + + // --- Backend Data Sync --- + commitChanges(skipRender = false) { + let sortedSegments = [...this.timeline.segments].sort((a, b) => a.start - b.start); + let contiguousLengths = []; + let contiguousPrompts = []; + let currentCursor = 0; + const durationFrames = this.getDurationFrames(); + + // Build segment lengths clipped at the duration cutoff. + // - Gaps before the first segment, or between segments, are absorbed into the adjacent + // segment's length (same as before), but are also clipped at durationFrames. + // - Segments that start at or past the cutoff are excluded entirely. + // - Segments that cross the cutoff are trimmed so their end = durationFrames exactly. + let pendingGap = 0; + for (let seg of sortedSegments) { + // Skip segments entirely outside the duration. + if (seg.start >= durationFrames) break; + + if (seg.start > currentCursor) { + // Gap between the cursor and this segment — clip it at the cutoff too. + const gapLength = Math.min(seg.start, durationFrames) - currentCursor; + if (contiguousLengths.length > 0) { + contiguousLengths[contiguousLengths.length - 1] += gapLength; + } else { + pendingGap += gapLength; + } + } + + // Clip segment end at the duration cutoff. + const clippedEnd = Math.min(seg.start + seg.length, durationFrames); + const clippedLength = clippedEnd - seg.start; + + contiguousLengths.push(clippedLength + pendingGap); + contiguousPrompts.push(seg.prompt || ""); + pendingGap = 0; + currentCursor = seg.start + seg.length; // advance by the real (unclipped) end for gap detection + } + + // If segments don't fill to the end of the duration, pad the last segment to reach it. + const clampedCursor = Math.min(currentCursor, durationFrames); + if (contiguousLengths.length > 0 && clampedCursor < durationFrames) { + contiguousLengths[contiguousLengths.length - 1] += durationFrames - clampedCursor; + } + + const toSave = { + segments: sortedSegments.map(s => { + const { imgObj, ...rest } = s; + return rest; + }), + audioSegments: (this.timeline.audioSegments || []).map(s => ({ ...s })) + }; + + const jsonStr = JSON.stringify(toSave); + if (this.timelineDataWidget) this.timelineDataWidget.value = jsonStr; + + if (this.localPromptsWidget) { + this.localPromptsWidget.value = contiguousPrompts.join(" | "); + } + if (this.segmentLengthsWidget) { + this.segmentLengthsWidget.value = contiguousLengths.join(","); + } + + if (this.guideStrengthWidget) { + const imgStrengths = sortedSegments + .filter(s => s.type !== "text") + .map(s => (s.guideStrength !== undefined ? s.guideStrength : 1.0).toFixed(2)); + this.guideStrengthWidget.value = imgStrengths.join(","); + } + + // Keep zoom slider max in sync with the current timeline duration. + this.updateZoomSliderMax(); + + setTimeout(() => { + if (this.node && this.node.computeSize) { + const sz = this.node.computeSize(); + this.node.size[1] = sz[1]; + if (app.graph) app.graph.setDirtyCanvas(true, true); + } + }, 0); + + if (!skipRender) this.render(); + } + + // --- Gap Region Calculation --- + getGapRegions() { + const totalFrames = this.getVisualDurationFrames(); + const outputFrames = this.getDurationFrames(); + const width = this.canvas.offsetWidth || this._lastWidth || 0; + const gaps = []; + if (!width) return gaps; + + // Image gaps + let cursor = 0; + const sortedImg = [...this.timeline.segments].sort((a, b) => a.start - b.start); + for (const seg of sortedImg) { + if (seg.start > cursor) { + const x0 = (cursor / totalFrames) * width; + const x1 = (seg.start / totalFrames) * width; + gaps.push({ track: 'image', frameStart: cursor, frameEnd: seg.start, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight / 2, widthPx: x1 - x0 }); + } + cursor = seg.start + seg.length; + } + if (cursor < outputFrames) { + const x0 = (cursor / totalFrames) * width; + const x1 = (outputFrames / totalFrames) * width; + gaps.push({ track: 'image', frameStart: cursor, frameEnd: outputFrames, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight / 2, widthPx: x1 - x0 }); + } + + // Audio gaps + cursor = 0; + const sortedAud = [...this.timeline.audioSegments].sort((a, b) => a.start - b.start); + for (const seg of sortedAud) { + if (seg.start > cursor) { + const x0 = (cursor / totalFrames) * width; + const x1 = (seg.start / totalFrames) * width; + gaps.push({ track: 'audio', frameStart: cursor, frameEnd: seg.start, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight + this.audioTrackHeight / 2, widthPx: x1 - x0 }); + } + cursor = seg.start + seg.length; + } + if (cursor < outputFrames) { + const x0 = (cursor / totalFrames) * width; + const x1 = (outputFrames / totalFrames) * width; + gaps.push({ track: 'audio', frameStart: cursor, frameEnd: outputFrames, centerX: (x0 + x1) / 2, centerY: RULER_HEIGHT + this.blockHeight + this.audioTrackHeight / 2, widthPx: x1 - x0 }); + } + + return gaps; + } + + promptAddAudioInGap(frameStart, frameEnd) { + const fi = document.createElement("input"); + fi.type = "file"; + fi.accept = "audio/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) this.handleAudioUpload([ev.target.files[0]], frameStart); + }); + fi.click(); + } + + // --- Context Menu --- + onContextMenu(e) { + e.preventDefault(); + const { x: mouseX, y: mouseY } = this.getMousePos(e); + + const trackHeight = this.blockHeight; + const isAudioTrack = mouseY >= RULER_HEIGHT + trackHeight && mouseY <= RULER_HEIGHT + trackHeight + this.audioTrackHeight; + const isImageTrack = mouseY >= RULER_HEIGHT && mouseY <= RULER_HEIGHT + trackHeight; + + const logicalWidth = this.canvas.offsetWidth || 1; + const totalFrames = this.getVisualDurationFrames(); + const cursor = mouseX * (totalFrames / logicalWidth); + + let clickedSeg = null; + let trackType = ""; + + if (isAudioTrack) { + clickedSeg = this.timeline.audioSegments.find(s => cursor >= s.start && cursor <= s.start + s.length); + trackType = "audio"; + } else if (isImageTrack) { + clickedSeg = this.timeline.segments.find(s => cursor >= s.start && cursor <= s.start + s.length); + trackType = clickedSeg ? clickedSeg.type : ""; + } + + if (clickedSeg) { + this.showContextMenu(e.clientX, e.clientY, clickedSeg, trackType); + } else if (isAudioTrack || isImageTrack) { + const gapRegions = this.getGapRegions(); + const currentTrack = isAudioTrack ? "audio" : "image"; + let gap = gapRegions.find(g => cursor >= g.frameStart && cursor <= g.frameEnd && g.track === currentTrack); + + if (!gap) { + const startFrame = Math.round(cursor); + gap = { + track: currentTrack, + frameStart: startFrame, + frameEnd: startFrame + Math.max(1, this.getFrameRate()) + }; + } + gap.clickedFrame = cursor; + + this.showGapContextMenu(e.clientX, e.clientY, gap); + } + } + + showContextMenu(clientX, clientY, seg, trackType) { + this.dismissContextMenu(); + const menu = document.createElement("div"); + menu.className = "pr-gap-menu"; + menu.style.left = `${clientX + 6}px`; + menu.style.top = `${clientY - 10}px`; + + const isImage = trackType !== "audio" && trackType !== "text" && seg.imageB64; + + if (isImage) { + const copyBtn = document.createElement("button"); + copyBtn.className = "pr-gap-menu-btn"; + copyBtn.innerHTML = `Copy Image`; + copyBtn.onclick = async () => { + try { + const res = await fetch(seg.imageB64); + const blob = await res.blob(); + await navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]); + } catch (err) { + console.error("Failed to copy image", err); + } + this.dismissContextMenu(); + }; + menu.appendChild(copyBtn); + + const saveBtn = document.createElement("button"); + saveBtn.className = "pr-gap-menu-btn"; + saveBtn.innerHTML = `Save Image`; + saveBtn.onclick = () => { + const a = document.createElement("a"); + a.href = seg.imageB64; + a.download = "timeline_image.jpg"; + a.click(); + this.dismissContextMenu(); + }; + menu.appendChild(saveBtn); + + const openBtn = document.createElement("button"); + openBtn.className = "pr-gap-menu-btn"; + openBtn.innerHTML = `Open Image in New Tab`; + openBtn.onclick = () => { + const win = window.open(); + if (win) { + win.document.write(``); + win.document.close(); + } + this.dismissContextMenu(); + }; + menu.appendChild(openBtn); + } + + if (trackType !== "audio") { + const copyPromptBtn = document.createElement("button"); + copyPromptBtn.className = "pr-gap-menu-btn"; + copyPromptBtn.innerHTML = `Copy Prompt`; + copyPromptBtn.onclick = async () => { + try { + await navigator.clipboard.writeText(seg.prompt || ""); + } catch (err) { + console.error("Failed to copy prompt", err); + } + this.dismissContextMenu(); + }; + menu.appendChild(copyPromptBtn); + } + + const copySegBtn = document.createElement("button"); + copySegBtn.className = "pr-gap-menu-btn"; + copySegBtn.innerHTML = `Copy Segment`; + copySegBtn.onclick = () => { + this._copiedSegment = { ...seg, id: Date.now().toString() + Math.random().toString(36).substr(2, 5) }; + this._copiedSegmentTrack = trackType === "audio" ? "audio" : "image"; + this.dismissContextMenu(); + }; + menu.appendChild(copySegBtn); + + const currentTrack = trackType === "audio" ? "audio" : "image"; + if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { + const pasteReplaceBtn = document.createElement("button"); + pasteReplaceBtn.className = "pr-gap-menu-btn"; + pasteReplaceBtn.innerHTML = `Paste & Replace`; + pasteReplaceBtn.onclick = () => { + const newSeg = { + ...this._copiedSegment, + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: seg.start, + length: this._copiedSegment.length + }; + const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; + const idx = targetArray.findIndex(s => s.id === seg.id); + if (idx >= 0) targetArray[idx] = newSeg; + this.commitChanges(); + this.dismissContextMenu(); + }; + menu.appendChild(pasteReplaceBtn); + } + + const delBtn = document.createElement("button"); + delBtn.className = "pr-gap-menu-btn"; + delBtn.innerHTML = `Delete`; + delBtn.style.color = "#ff4444"; + delBtn.onclick = () => { + this.selectionType = trackType === "audio" ? "audio" : "image"; + const list = trackType === "audio" ? this.timeline.audioSegments : this.timeline.segments; + this.selectedIndex = list.findIndex(s => s.id === seg.id); + this.deleteSelectedSegment(); + this.dismissContextMenu(); + }; + menu.appendChild(delBtn); + + document.body.appendChild(menu); + this._contextMenu = menu; + + setTimeout(() => { + this._contextMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissContextMenu(); }; + document.addEventListener("pointerdown", this._contextMenuDismisser, true); + }, 0); + } + + showGapContextMenu(clientX, clientY, gap) { + this.dismissContextMenu(); + const menu = document.createElement("div"); + menu.className = "pr-gap-menu"; + menu.style.left = `${clientX + 6}px`; + menu.style.top = `${clientY - 10}px`; + + const currentTrack = gap.track === "audio" ? "audio" : "image"; + + if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { + const pasteBtn = document.createElement("button"); + pasteBtn.className = "pr-gap-menu-btn"; + pasteBtn.innerHTML = `Paste Segment`; + pasteBtn.onclick = () => { + const startFrame = Math.round(gap.clickedFrame !== undefined ? gap.clickedFrame : gap.frameStart); + const gapLength = gap.frameEnd - startFrame; + + const newSeg = { + ...this._copiedSegment, + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: startFrame, + length: Math.min(this._copiedSegment.length, gapLength) + }; + const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; + targetArray.push(newSeg); + targetArray.sort((a, b) => a.start - b.start); + this.commitChanges(); + this.dismissContextMenu(); + }; + menu.appendChild(pasteBtn); + } + + if (currentTrack === "image") { + const textBtn = document.createElement("button"); + textBtn.className = "pr-gap-menu-btn"; + textBtn.innerHTML = `${ICONS.text} Text Segment`; + textBtn.onclick = () => { + this.addSegmentInGap(gap.frameStart, gap.frameEnd, "text"); + this.dismissContextMenu(); + }; + menu.appendChild(textBtn); + + const imgBtn = document.createElement("button"); + imgBtn.className = "pr-gap-menu-btn"; + imgBtn.innerHTML = `${ICONS.upload} Image Segment`; + imgBtn.onclick = () => { + this.dismissContextMenu(); + const fi = document.createElement("input"); + fi.type = "file"; fi.accept = "image/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) { + const gapLength = gap.frameEnd - gap.frameStart; + this.handleImageUpload([ev.target.files[0]], gap.frameStart, gapLength); + } + }); + fi.click(); + }; + menu.appendChild(imgBtn); + } + + document.body.appendChild(menu); + this._contextMenu = menu; + setTimeout(() => { + this._contextMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissContextMenu(); }; + document.addEventListener("pointerdown", this._contextMenuDismisser, true); + }, 0); + } + dismissContextMenu() { + if (this._contextMenu) { this._contextMenu.remove(); this._contextMenu = null; } + if (this._contextMenuDismisser) { document.removeEventListener("pointerdown", this._contextMenuDismisser, true); this._contextMenuDismisser = null; } + } + + // --- Gap Popup Menu --- + showGapMenu(clientX, clientY, gap) { + this.dismissGapMenu(); + const menu = document.createElement("div"); + menu.className = "pr-gap-menu"; + menu.style.left = `${clientX + 6}px`; + menu.style.top = `${clientY - 10}px`; + + const textBtn = document.createElement("button"); + textBtn.className = "pr-gap-menu-btn"; + textBtn.innerHTML = `${ICONS.text} Text Segment`; + textBtn.addEventListener("click", () => { + this.addSegmentInGap(gap.frameStart, gap.frameEnd, "text"); + this.dismissGapMenu(); + }); + + const imgBtn = document.createElement("button"); + imgBtn.className = "pr-gap-menu-btn"; + imgBtn.innerHTML = `${ICONS.upload} Image Segment`; + imgBtn.addEventListener("click", () => { + this.dismissGapMenu(); + const fi = document.createElement("input"); + fi.type = "file"; fi.accept = "image/*"; + fi.addEventListener("change", (ev) => { + if (ev.target.files?.[0]) { + const gapLength = gap.frameEnd - gap.frameStart; + this.handleImageUpload([ev.target.files[0]], gap.frameStart, gapLength); + } + }); + fi.click(); + }); + + menu.appendChild(textBtn); + menu.appendChild(imgBtn); + const currentTrack = gap.track === "audio" ? "audio" : "image"; + if (this._copiedSegment && this._copiedSegmentTrack === currentTrack) { + const pasteBtn = document.createElement("button"); + pasteBtn.className = "pr-gap-menu-btn"; + pasteBtn.innerHTML = `Paste Segment`; + pasteBtn.onclick = () => { + const gapLength = gap.frameEnd - gap.frameStart; + + let finalLength = Math.min(this._copiedSegment.length, gapLength); + if (currentTrack === "image") { + finalLength = gapLength; + } + + const newSeg = { + ...this._copiedSegment, + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: gap.frameStart, + length: finalLength + }; + const targetArray = currentTrack === "audio" ? this.timeline.audioSegments : this.timeline.segments; + targetArray.push(newSeg); + targetArray.sort((a, b) => a.start - b.start); + this.commitChanges(); + this.dismissGapMenu(); + }; + menu.appendChild(pasteBtn); + } + + document.body.appendChild(menu); + this._gapMenu = menu; + setTimeout(() => { + this._gapMenuDismisser = (ev) => { if (!menu.contains(ev.target)) this.dismissGapMenu(); }; + document.addEventListener("pointerdown", this._gapMenuDismisser, true); + }, 0); + } + + dismissGapMenu() { + if (this._gapMenu) { this._gapMenu.remove(); this._gapMenu = null; } + if (this._gapMenuDismisser) { document.removeEventListener("pointerdown", this._gapMenuDismisser, true); this._gapMenuDismisser = null; } + } + + // --- Settings Menu --- + // Widgets that are managed by the settings menu (hidden from node by default). + get _settingsWidgetNames() { + return ["display_mode", "epsilon", "divisible_by", "img_compression"]; + } + + // Hide all settings widgets on the node (called on init). + hideSettingsWidgets() { + for (const name of this._settingsWidgetNames) { + const w = this.node.widgets?.find(w => w.name === name); + if (w) hideWidget(w); + } + this.updateWidgetVisibility(); + + // Workaround: toggle display mode to force ComfyUI to refresh the node + if (this.displayModeWidget) { + const origVal = this.displayModeWidget.value; + const otherVal = origVal === "frames" ? "seconds" : "frames"; + + this.displayModeWidget.value = otherVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(otherVal); + + this.displayModeWidget.value = origVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(origVal); + } + } + + // Restore all settings widgets on the node. + showSettingsWidgets() { + for (const name of this._settingsWidgetNames) { + const w = this.node.widgets?.find(w => w.name === name); + if (!w) continue; + // Restore original type from widget's origType if available, otherwise guess. + const typeMap = { + display_mode: "combo", epsilon: "number", divisible_by: "number", + img_compression: "number", + }; + w.type = typeMap[name] || "number"; + w.hidden = false; + if (w.options) w.options.hidden = false; + w.computeSize = null; + } + this.updateWidgetVisibility(); + + // Workaround: toggle display mode to force ComfyUI to refresh the node + if (this.displayModeWidget) { + const origVal = this.displayModeWidget.value; + const otherVal = origVal === "frames" ? "seconds" : "frames"; + + this.displayModeWidget.value = otherVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(otherVal); + + this.displayModeWidget.value = origVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(origVal); + } + } + + _makeSettingRow(label, inputEl) { + const row = document.createElement("div"); + row.className = "pr-settings-row"; + const lbl = document.createElement("span"); + lbl.className = "pr-settings-label"; + lbl.textContent = label; + row.appendChild(lbl); + row.appendChild(inputEl); + return row; + } + + showSettingsMenu(anchorEl) { + this.dismissSettingsMenu(); + const menu = document.createElement("div"); + menu.className = "pr-settings-menu"; + + // Title & Close Button Container + const titleContainer = document.createElement("div"); + titleContainer.className = "pr-settings-title"; + titleContainer.style.display = "flex"; + titleContainer.style.justifyContent = "space-between"; + titleContainer.style.alignItems = "center"; + + const titleText = document.createElement("span"); + titleText.textContent = "Timeline Settings"; + titleContainer.appendChild(titleText); + + const closeBtn = document.createElement("button"); + closeBtn.className = "pr-settings-close-btn"; + closeBtn.innerHTML = ICONS.close; + closeBtn.title = "Close Settings"; + closeBtn.addEventListener("click", () => this.dismissSettingsMenu()); + titleContainer.appendChild(closeBtn); + + menu.appendChild(titleContainer); + + // Helper: fire a widget's callback safely + const fireCallback = (w, val) => { + w.value = val; + if (w.callback) { + try { w.callback(val, app.canvas, this.node, null, null); } catch (e) { } + } + if (window.app && window.app.graph) window.app.graph.setDirtyCanvas(true, true); + }; + + // --- Display Mode --- + const dmWidget = this.node.widgets?.find(w => w.name === "display_mode"); + if (dmWidget) { + const ctrl = document.createElement("div"); + ctrl.className = "pr-segmented-control"; + + const framesSeg = document.createElement("div"); + framesSeg.className = "pr-segment"; + framesSeg.textContent = "Frames"; + + const secondsSeg = document.createElement("div"); + secondsSeg.className = "pr-segment"; + secondsSeg.textContent = "Seconds"; + + const updateActive = (val) => { + if (val === "frames") { + framesSeg.classList.add("active"); + secondsSeg.classList.remove("active"); + } else { + secondsSeg.classList.add("active"); + framesSeg.classList.remove("active"); + } + }; + + updateActive(dmWidget.value); + + const onSegClick = (val) => { + fireCallback(dmWidget, val); + updateActive(val); + // Update ruler/timecode immediately + if (this.updateWidgetVisibility) this.updateWidgetVisibility(); + if (this.updateUIFromSelection) this.updateUIFromSelection(); + this.render(); + }; + + framesSeg.addEventListener("click", () => onSegClick("frames")); + secondsSeg.addEventListener("click", () => onSegClick("seconds")); + + ctrl.appendChild(secondsSeg); + ctrl.appendChild(framesSeg); + + menu.appendChild(this._makeSettingRow("Display Mode", ctrl)); + } + + const divider1 = document.createElement("hr"); + divider1.className = "pr-settings-divider"; + menu.appendChild(divider1); + + // Helper to create scrubbable number control with horizontal buttons + const createScrubbableNumberControl = (w, step, min, max, isFloat = false) => { + const container = document.createElement("div"); + container.className = "pr-number-control"; + + const decBtn = document.createElement("button"); + decBtn.className = "pr-number-btn"; + decBtn.textContent = "-"; + + const inp = document.createElement("input"); + inp.type = "number"; + inp.className = "pr-settings-input"; + inp.value = w.value; + inp.step = step.toString(); + inp.min = min.toString(); + inp.max = max.toString(); + + const incBtn = document.createElement("button"); + incBtn.className = "pr-number-btn"; + incBtn.textContent = "+"; + + decBtn.addEventListener("click", () => { + let val = parseFloat(inp.value) - step; + if (val < min) val = min; + inp.value = isFloat ? val.toFixed(4) : Math.round(val); + fireCallback(w, parseFloat(inp.value)); + }); + + incBtn.addEventListener("click", () => { + let val = parseFloat(inp.value) + step; + if (val > max) val = max; + inp.value = isFloat ? val.toFixed(4) : Math.round(val); + fireCallback(w, parseFloat(inp.value)); + }); + + inp.addEventListener("change", () => { + let val = parseFloat(inp.value); + if (isNaN(val)) val = w.value; + if (val < min) val = min; + if (val > max) val = max; + inp.value = isFloat ? val.toFixed(4) : Math.round(val); + fireCallback(w, parseFloat(inp.value)); + }); + + // Dragging logic + let isDragging = false; + let startX = 0; + let startVal = 0; + let hasMoved = false; + + inp.style.cursor = "ew-resize"; + + inp.addEventListener("mousedown", (e) => { + startX = e.clientX; + startVal = parseFloat(inp.value); + hasMoved = false; + + const onMouseMove = (moveEvent) => { + const deltaX = moveEvent.clientX - startX; + if (Math.abs(deltaX) > 3) { + hasMoved = true; + isDragging = true; + } + + if (isDragging) { + moveEvent.preventDefault(); + const sensitivity = isFloat ? 0.001 : 0.5; + let newVal = startVal + deltaX * sensitivity; + + if (newVal < min) newVal = min; + if (newVal > max) newVal = max; + + inp.value = isFloat ? newVal.toFixed(4) : Math.round(newVal); + fireCallback(w, parseFloat(inp.value)); + } + }; + + const onMouseUp = () => { + document.removeEventListener("mousemove", onMouseMove); + document.removeEventListener("mouseup", onMouseUp); + + if (!hasMoved) { + inp.focus(); + inp.select(); + } + isDragging = false; + }; + + document.addEventListener("mousemove", onMouseMove); + document.addEventListener("mouseup", onMouseUp); + }); + + container.appendChild(decBtn); + container.appendChild(inp); + container.appendChild(incBtn); + + return container; + }; + + // --- Epsilon --- + const epsWidget = this.node.widgets?.find(w => w.name === "epsilon"); + if (epsWidget) { + menu.appendChild(this._makeSettingRow("Epsilon", createScrubbableNumberControl(epsWidget, 0.0001, 0.0001, 0.99, true))); + } + + // --- Divisible By --- + const divByWidget = this.node.widgets?.find(w => w.name === "divisible_by"); + if (divByWidget) { + menu.appendChild(this._makeSettingRow("Divisible By", createScrubbableNumberControl(divByWidget, 1, 1, 256, false))); + } + + // --- Img Compression --- + const compWidget = this.node.widgets?.find(w => w.name === "img_compression"); + if (compWidget) { + menu.appendChild(this._makeSettingRow("Img Compression", createScrubbableNumberControl(compWidget, 1, 0, 100, false))); + } + + // --- Global Prompt Toggle --- + const globalPromptWidget = this.node.widgets?.find(w => w.name === "global_prompt"); + if (globalPromptWidget) { + const cb = document.createElement("input"); + cb.type = "checkbox"; + cb.checked = !(globalPromptWidget.options && globalPromptWidget.options.hidden); + cb.style.cursor = "pointer"; + cb.addEventListener("change", () => { + const isVisible = cb.checked; + if (!globalPromptWidget.options) globalPromptWidget.options = {}; + globalPromptWidget.options.hidden = !isVisible; + + if (isVisible) { + delete globalPromptWidget.computeSize; + globalPromptWidget.hidden = false; + if (globalPromptWidget.element) globalPromptWidget.element.style.display = ""; + } else { + globalPromptWidget.computeSize = () => [0, 0]; + globalPromptWidget.hidden = true; + if (globalPromptWidget.element) globalPromptWidget.element.style.display = "none"; + } + + // Force refresh via display mode double-toggle trick + if (this.displayModeWidget) { + const origVal = this.displayModeWidget.value; + const otherVal = origVal === "frames" ? "seconds" : "frames"; + this.displayModeWidget.value = otherVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(otherVal); + this.displayModeWidget.value = origVal; + if (this.displayModeWidget.callback) this.displayModeWidget.callback(origVal); + } + }); + menu.appendChild(this._makeSettingRow("Use Global Prompt", cb)); + } + + + // --- Show/Hide on Node Toggle --- + const toggleBtn = document.createElement("button"); + toggleBtn.className = "pr-settings-toggle-btn"; + const widgetsVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); + toggleBtn.textContent = widgetsVisible ? "Hide Widgets on Node" : "Show Widgets on Node"; + toggleBtn.addEventListener("click", () => { + const nowVisible = !!(this.node.widgets?.find(w => w.name === "display_mode" && !(w.options && w.options.hidden))); + if (nowVisible) { + this.hideSettingsWidgets(); + toggleBtn.textContent = "Show Widgets on Node"; + } else { + this.showSettingsWidgets(); + toggleBtn.textContent = "Hide Widgets on Node"; + } + }); + menu.appendChild(toggleBtn); + + // Position the menu below the anchor button (pop down) + document.body.appendChild(menu); + const rect = anchorEl.getBoundingClientRect(); + const menuW = menu.offsetWidth || 230; + const menuH = menu.offsetHeight || 350; + let left = rect.right - menuW; + let top = rect.bottom + 6; + if (left < 4) left = 4; + // Fallback to top if it overflows the bottom of the screen + if (top + menuH > window.innerHeight - 4) { + top = rect.top - menuH - 6; + } + menu.style.left = `${left}px`; + menu.style.top = `${top}px`; + + this._settingsMenu = menu; + setTimeout(() => { + this._settingsDismisser = (ev) => { + if (!menu.contains(ev.target) && !anchorEl.contains(ev.target)) this.dismissSettingsMenu(); + }; + document.addEventListener("mousedown", this._settingsDismisser); + }, 0); + } + + dismissSettingsMenu() { + if (this._settingsMenu) { this._settingsMenu.remove(); this._settingsMenu = null; } + if (this._settingsDismisser) { document.removeEventListener("mousedown", this._settingsDismisser); this._settingsDismisser = null; } + } + + + addSegmentInGap(frameStart, frameEnd, type = "text") { + const seg = { + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: frameStart, length: frameEnd - frameStart, + prompt: "", type, + }; + this.timeline.segments.push(seg); + this.timeline.segments.sort((a, b) => a.start - b.start); + this.selectionType = "image"; + this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); + this.updateUIFromSelection(); + this.commitChanges(); + } + + addTextSegmentFreeSpace() { + const frameRate = this.getFrameRate(); + const newLength = Math.max(1, frameRate); // 1 second default + const sorted = [...this.timeline.segments].sort((a, b) => a.start - b.start); + let newStart = 0; + for (const seg of sorted) { + if (newStart + newLength <= seg.start) break; + newStart = Math.max(newStart, seg.start + seg.length); + } + // Place the segment at the first free slot in the visual timeline (no output duration change). + const durationFrames = this.getVisualDurationFrames(); + const seg = { + id: Date.now().toString() + Math.random().toString(36).substr(2, 5), + start: newStart, length: Math.min(newLength, Math.max(newLength, durationFrames - newStart)), + prompt: "", type: "text", + }; + this.timeline.segments.push(seg); + this.timeline.segments.sort((a, b) => a.start - b.start); + this.selectionType = "image"; + this.selectedIndex = this.timeline.segments.findIndex(s => s.id === seg.id); + this.updateUIFromSelection(); + this.commitChanges(); + } + + // --- Audio Player Engine --- + updatePlayerUI() { + if (!this.playBtn || !this.loopBtn) return; + this.playBtn.innerHTML = this.isPlaying ? ICONS.pause : ICONS.play; + if (this.isLooping) { + this.loopBtn.classList.add("active"); + } else { + this.loopBtn.classList.remove("active"); + } + if (this.seekBar) { + this.seekBar.max = this.getVisualDurationFrames(); + this.seekBar.value = this.currentFrame; + } + if (this.timeCodeDisplay) { + this.timeCodeDisplay.textContent = this.formatTime(this.currentFrame); + } + } + + togglePlay() { + if (this.isPlaying) { + this.pauseAudio(); + } else { + if (this.currentFrame >= this.getVisualDurationFrames()) { + this.currentFrame = 0; + } + this.playAudio(); + } + } + + toggleLoop() { + this.isLooping = !this.isLooping; + this.updatePlayerUI(); + } + + async playAudio() { + this.pauseAudio(true); // clear any existing playback, but don't suspend context if scrubbing + + this._playCounter = (this._playCounter || 0) + 1; + const playId = this._playCounter; + this._currentPlayId = playId; + this.isPlaying = true; + + if (!this.audioContext) { + this.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + } + if (this.audioContext.state !== 'running') { + try { await this.audioContext.resume(); } catch (e) { } + } + if (this._currentPlayId !== playId || !this.isPlaying) return; + + this.updatePlayerUI(); + + const frameRate = this.getFrameRate(); + this.playbackStartFrame = this.currentFrame; + this.playbackStartTime = this.audioContext.currentTime; + + // Decode and schedule all audio segments that happen AT or AFTER currentFrame + for (let seg of this.timeline.audioSegments) { + const segStartFrame = seg.start; + const segEndFrame = seg.start + seg.length; + + if (segEndFrame <= this.currentFrame) continue; + + try { + // Build audio buffer: fetch from server URL if audioFile is set, otherwise fall back to audioB64 + let audioBuffer; + if (seg.audioFile) { + const audioUrl = api.apiURL(`/view?filename=${encodeURIComponent(seg.audioFile.split("/").pop())}&type=input&subfolder=${encodeURIComponent(seg.audioFile.includes("/") ? seg.audioFile.split("/").slice(0, -1).join("/") : "")}`); + const resp = await fetch(audioUrl); + const arrayBuffer = await resp.arrayBuffer(); + audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer); + } else if (seg.audioB64) { + const binaryString = window.atob(seg.audioB64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + audioBuffer = await this.audioContext.decodeAudioData(bytes.buffer); + } else { + continue; + } + if (this._currentPlayId !== playId || !this.isPlaying) return; + + const framesToSkipInSegment = Math.max(0, this.currentFrame - segStartFrame); + const waitFrames = Math.max(0, segStartFrame - this.currentFrame); + const waitTimeSec = waitFrames / frameRate; + + const fileOffsetFrames = seg.trimStart + framesToSkipInSegment; + const fileOffsetSec = fileOffsetFrames / frameRate; + + const playDurationFrames = seg.length - framesToSkipInSegment; + const playDurationSec = playDurationFrames / frameRate; + + if (playDurationSec <= 0) continue; + + const source = this.audioContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.audioContext.destination); + + const startTime = this.audioContext.currentTime + waitTimeSec; + source.start(startTime, fileOffsetSec, playDurationSec); + + this.activeAudioNodes.push(source); + } catch (err) { + console.error("Playback decode error for segment:", err); + } + } + + if (this._currentPlayId !== playId || !this.isPlaying) return; + + const loop = () => { + if (!this.isPlaying || this._currentPlayId !== playId) return; + + const elapsedSec = this.audioContext.currentTime - this.playbackStartTime; + const elapsedFrames = elapsedSec * frameRate; + + this.currentFrame = this.playbackStartFrame + elapsedFrames; + + const visualDurationFrames = this.getVisualDurationFrames(); + const durationFrames = this.getDurationFrames(); + + if (this.isLooping) { + const loopBound = (this.playbackStartFrame >= durationFrames) ? visualDurationFrames : durationFrames; + if (this.currentFrame >= loopBound) { + this.currentFrame = 0; + this.playAudio(); // Restart playback + return; + } + } else { + if (this.currentFrame >= visualDurationFrames) { + this.currentFrame = visualDurationFrames; + this.pauseAudio(); + this.render(); + return; + } + } + + this.render(); + this._playLoopId = requestAnimationFrame(loop); + }; + + this._playLoopId = requestAnimationFrame(loop); + } + + pauseAudio(isScrubbing = false) { + this.isPlaying = false; + this._currentPlayId = null; + + if (!isScrubbing && this.audioContext && this.audioContext.state === 'running') { + try { this.audioContext.suspend(); } catch (e) { } + } + + for (let node of this.activeAudioNodes) { + try { node.stop(); } catch (e) { } + try { node.disconnect(); } catch (e) { } + } + this.activeAudioNodes = []; + + if (this._playLoopId) { + cancelAnimationFrame(this._playLoopId); + this._playLoopId = null; + } + this.updatePlayerUI(); + } +} + +// --- Node Registration Hooks --- +const APPENDED_WIDGET_DEFAULTS = [ + ["timeline_data", "{}"], + ["local_prompts", ""], + ["segment_lengths", ""], +]; + +app.registerExtension({ + name: "LTXDirector", + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData.name === "LTXDirector") { + + const onNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + if (onNodeCreated) onNodeCreated.apply(this, arguments); + + for (const [name, def] of APPENDED_WIDGET_DEFAULTS) { + if (!this.widgets?.find(w => w.name === name)) { + this.addWidget("string", name, def, () => { }); + } + } + for (const w of this.widgets) { + if (HIDDEN_WIDGET_NAMES.includes(w.name)) hideWidget(w); + } + + // Set default width to be wider on creation (approx 2.5x default ~220px) + this.size[0] = 1000; + + // Force default for img_compression if not set (ComfyUI sometimes skips optional defaults) + const compWidget = this.widgets?.find(w => w.name === "img_compression"); + if (compWidget && (compWidget.value === undefined || compWidget.value === null || compWidget.value === 0)) { + compWidget.value = 18; + } + + // Hide global prompt by default on creation without destroying its DOM element + const globalPromptWidget = this.widgets?.find(w => w.name === "global_prompt"); + if (globalPromptWidget) { + if (!globalPromptWidget.options) globalPromptWidget.options = {}; + globalPromptWidget.options.hidden = true; + globalPromptWidget.hidden = true; + globalPromptWidget.computeSize = () => [0, 0]; + setTimeout(() => { + if (globalPromptWidget.element) globalPromptWidget.element.style.display = "none"; + }, 0); + } + + const container = document.createElement("div"); + const widget = this.addDOMWidget("timeline_ui", "timeline_ui", container, { + getValue: () => "", + setValue: () => { }, + }); + + widget.computeSize = function (width) { + const canvasH = self._timelineEditor ? self._timelineEditor.canvasHeight : CANVAS_HEIGHT; + return [width, canvasH + 235]; + }; + + const self = this; + setTimeout(() => { + try { + self._timelineEditor = new TimelineEditor(self, container, widget); + } catch (err) { + console.error("[PromptRelay] timeline editor init failed:", err); + } + }, 0); + }; + + const onRemoved = nodeType.prototype.onRemoved; + nodeType.prototype.onRemoved = function () { + this._timelineEditor?.destroy(); + return onRemoved?.apply(this, arguments); + }; + + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function (info) { + const out = onConfigure?.apply(this, arguments); + for (const [name, def] of APPENDED_WIDGET_DEFAULTS) { + const w = this.widgets.find(x => x.name === name); + if (w && (w.value == null || w.value === "")) w.value = def; + } + + setTimeout(() => { + if (this._timelineEditor) { + this._timelineEditor.timeline = parseInitial(this._timelineEditor.timelineDataWidget?.value); + this._timelineEditor.loadImages(); + this._timelineEditor.selectionType = "image"; + this._timelineEditor.selectedIndex = clamp( + this._timelineEditor.selectedIndex, -1, + Math.max(-1, this._timelineEditor.timeline.segments.length - 1) + ); + this._timelineEditor.updateUIFromSelection(); + this._timelineEditor.render(); + } + }, 0); + return out; + }; + } + }, +}); diff --git a/js/ltx_director_guide.js b/js/ltx_director_guide.js new file mode 100644 index 0000000..9edc2a1 --- /dev/null +++ b/js/ltx_director_guide.js @@ -0,0 +1,17 @@ +import { app } from "../../scripts/app.js"; + +// LTX Director Guide is a pure pass-through processor node. +// All configuration (images, insert frames, strengths) comes from +// the guide_data output of Prompt Relay Encode (Timeline). +// No dynamic widgets or sync logic needed. +app.registerExtension({ + name: "Comfy.LTXDirectorGuide", + async nodeCreated(node) { + if (node.comfyClass !== "LTXDirectorGuide") return; + // Nothing to initialize — the node has no configurable widgets. + }, +}); + + // Nothing to initialize — the node has no configurable widgets. + }, +}); \ No newline at end of file diff --git a/ltx_director.py b/ltx_director.py new file mode 100644 index 0000000..9a96e93 --- /dev/null +++ b/ltx_director.py @@ -0,0 +1,643 @@ +import logging +import json +import base64 +import io as _io +import math + +import numpy as np +import torch +import av +from PIL import Image + +import os +import folder_paths +import comfy.model_management + +from comfy_api.latest import io + +from .prompt_relay import ( + get_raw_tokenizer, + map_token_indices, + build_segments, + create_mask_fn, + distribute_segment_lengths, +) + +from .patches import detect_model_type, apply_patches + +log = logging.getLogger(__name__) + +# Custom socket type shared with LTXSequencer +GuideData = io.Custom("GUIDE_DATA") + + +def _load_image_tensor(seg: dict) -> torch.Tensor: + """Decode an image from the ComfyUI input folder (if imageFile provided) or fallback to base64 + to a ComfyUI-style image tensor of shape [1, H, W, 3], float32 in [0, 1].""" + if seg.get("imageFile"): + file_path = os.path.join(folder_paths.get_input_directory(), seg["imageFile"]) + if os.path.exists(file_path): + img = Image.open(file_path).convert("RGB") + arr = np.array(img, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + b64_str = seg.get("imageB64", "") + if not b64_str or b64_str.startswith("/view?"): + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + + if "," in b64_str: + b64_str = b64_str.split(",", 1)[1] + + try: + img_bytes = base64.b64decode(b64_str) + img = Image.open(_io.BytesIO(img_bytes)).convert("RGB") + arr = np.array(img, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + except: + return torch.zeros((1, 512, 512, 3), dtype=torch.float32) + + +def _resize_image(tensor: torch.Tensor, target_w: int, target_h: int, method: str, divisible_by: int) -> torch.Tensor: + """Resize a [1, H, W, 3] float32 tensor to target dimensions using the given method, + then snap the final dimensions to be divisible by `divisible_by`.""" + from PIL import Image as _PilImage + import torchvision.transforms.functional as TF + + def snap(val, div): + return max(div, (val // div) * div) + + tw = snap(target_w, divisible_by) + th = snap(target_h, divisible_by) + + img_np = (tensor[0].cpu().numpy() * 255.0).clip(0, 255).astype(np.uint8) + pil = _PilImage.fromarray(img_np) + src_w, src_h = pil.size + + if method == "stretch to fit": + resized = pil.resize((tw, th), _PilImage.LANCZOS) + + elif method == "maintain aspect ratio": + ratio = min(tw / src_w, th / src_h) + new_w = int(src_w * ratio) + new_h = int(src_h * ratio) + new_w = snap(new_w, divisible_by) + new_h = snap(new_h, divisible_by) + resized = pil.resize((new_w, new_h), _PilImage.LANCZOS) + + elif method == "pad": + ratio = min(tw / src_w, th / src_h) + new_w = snap(int(src_w * ratio), divisible_by) + new_h = snap(int(src_h * ratio), divisible_by) + inner = pil.resize((new_w, new_h), _PilImage.LANCZOS) + resized = _PilImage.new("RGB", (tw, th), (0, 0, 0)) + resized.paste(inner, ((tw - new_w) // 2, (th - new_h) // 2)) + + elif method == "crop": + ratio = max(tw / src_w, th / src_h) + new_w = int(src_w * ratio) + new_h = int(src_h * ratio) + inner = pil.resize((new_w, new_h), _PilImage.LANCZOS) + left = (new_w - tw) // 2 + top = (new_h - th) // 2 + resized = inner.crop((left, top, left + tw, top + th)) + + else: + resized = pil.resize((tw, th), _PilImage.LANCZOS) + + arr = np.array(resized, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def _compress_image(tensor: torch.Tensor, crf: int) -> torch.Tensor: + """Apply H.264 compression artefacts to a [1, H, W, 3] float32 tensor (ComfyUI image format). + crf=0 means no compression. Uses PyAV to encode/decode a single frame in-memory.""" + if crf == 0: + return tensor + img = tensor[0] # [H, W, 3] + # Dimensions must be even for H.264 + h = (img.shape[0] // 2) * 2 + w = (img.shape[1] // 2) * 2 + img_np = (img[:h, :w] * 255.0).byte().cpu().numpy() # uint8 [H, W, 3] + + try: + buf = _io.BytesIO() + container = av.open(buf, mode="w", format="mp4") + stream = container.add_stream("libx264", rate=1) + stream.width = w + stream.height = h + stream.pix_fmt = "yuv420p" + stream.options = {"crf": str(crf), "preset": "ultrafast"} + frame = av.VideoFrame.from_ndarray(img_np, format="rgb24") + for pkt in stream.encode(frame): + container.mux(pkt) + for pkt in stream.encode(None): + container.mux(pkt) + container.close() + + buf.seek(0) + container_r = av.open(buf, mode="r") + decoded = None + for frame_r in container_r.decode(video=0): + decoded = frame_r.to_ndarray(format="rgb24") # [H, W, 3] + break + container_r.close() + + if decoded is None: + return tensor + arr = torch.from_numpy(decoded.astype(np.float32) / 255.0).to(tensor.device, tensor.dtype) + # Re-embed into original tensor shape (may have been cropped by even-rounding) + out = tensor.clone() + out[0, :h, :w] = arr + return out + except Exception as e: + log.warning("[PromptRelay] img_compression encode/decode failed: %s", e) + return tensor + + +def _build_combined_audio(timeline_data_str: str, duration_frames: int, frame_rate: float) -> dict: + """Parses timeline JSON, loads/trims audio directly from memory using PyAV, + and aligns to a global timeline yielding ComfyUI's format. + Output length explicitly mimics the timeline's duration_frames length.""" + target_sr = 44100 + total_samples = max(1, int(math.ceil(duration_frames / frame_rate * target_sr))) + empty_audio = {"waveform": torch.zeros((1, 2, total_samples), dtype=torch.float32), "sample_rate": target_sr} + + if not timeline_data_str: + return empty_audio + + try: + data = json.loads(timeline_data_str) + audio_segs = data.get("audioSegments", []) + except Exception: + return empty_audio + + if not audio_segs: + return empty_audio + + out_waveform = torch.zeros((2, total_samples), dtype=torch.float32) + + for seg in audio_segs: + buffer = None + if seg.get("audioFile"): + file_path = os.path.join(folder_paths.get_input_directory(), seg["audioFile"]) + if os.path.exists(file_path): + with open(file_path, "rb") as f: + buffer = _io.BytesIO(f.read()) + + if not buffer and seg.get("audioB64"): + b64 = seg.get("audioB64") + if "," in b64: + b64 = b64.split(",", 1)[1] + try: + audio_bytes = base64.b64decode(b64) + buffer = _io.BytesIO(audio_bytes) + except: + pass + + if not buffer: + continue + + try: + clip_frames = [] + + # Use PyAV to decode directly from memory buffer + with av.open(buffer) as container: + stream = container.streams.audio[0] + + # Setup resampler to ensure output is 44.1kHz, Stereo, Float32 Planar + resampler = av.AudioResampler( + format='fltp', + layout='stereo', + rate=target_sr, + ) + + for frame in container.decode(stream): + for resampled_frame in resampler.resample(frame): + # to_ndarray() on fltp gives shape (channels, samples) + arr = resampled_frame.to_ndarray() + clip_frames.append(torch.from_numpy(arr)) + + # Flush the resampler to get any remaining samples + for resampled_frame in resampler.resample(None): + arr = resampled_frame.to_ndarray() + clip_frames.append(torch.from_numpy(arr)) + + if not clip_frames: + continue + + # Concatenate all frame blocks along the samples dimension (dim 1) + waveform = torch.cat(clip_frames, dim=1) # Shape: [2, total_clip_samples] + + # Calculate interactive trim boundaries + trim_start_frames = float(seg.get("trimStart", 0)) + length_frames = float(seg.get("length", 1)) + start_frames = float(seg.get("start", 0)) + + start_sample_src = int(trim_start_frames / frame_rate * target_sr) + length_samples = int(length_frames / frame_rate * target_sr) + end_sample_src = start_sample_src + length_samples + + if start_sample_src < 0: start_sample_src = 0 + if end_sample_src > waveform.shape[1]: + end_sample_src = waveform.shape[1] + + actual_length = end_sample_src - start_sample_src + if actual_length <= 0: continue + + # Extract the correct segment of the audio + clip_waveform = waveform[:, start_sample_src:end_sample_src] + + # Position onto the timeline + start_sample_dst = int(start_frames / frame_rate * target_sr) + + if start_sample_dst >= out_waveform.shape[1]: + continue + + end_sample_dst = start_sample_dst + actual_length + + # Clip any trailing overflow so we don't index past the timeline bounds + if end_sample_dst > out_waveform.shape[1]: + actual_length = out_waveform.shape[1] - start_sample_dst + clip_waveform = clip_waveform[:, :actual_length] + end_sample_dst = start_sample_dst + actual_length + + if actual_length <= 0: + continue + + # Additive composite (allows clips overlapping to sum together naturally) + out_waveform[:, start_sample_dst:end_sample_dst] += clip_waveform + + except Exception as e: + log.warning("[PromptRelay] Audio process error for segment %s: %s", seg.get("fileName"), e) + continue + + return {"waveform": out_waveform.unsqueeze(0), "sample_rate": target_sr} + + +def _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames): + """Convert pixel-space segment lengths to integer latent-space lengths using the + largest-remainder method. Targets the full `latent_frames` when the pixel sum looks + like full coverage (within one stride of latent_frames * stride). Otherwise targets + round(total_pixel / temporal_stride) so partial-coverage timelines stay partial. + """ + if not pixel_lengths: + return [] + total_pixel = sum(pixel_lengths) + if total_pixel <= 0: + return [1] * len(pixel_lengths) + + naive_total = max(1, round(total_pixel / temporal_stride)) + target_total = min(latent_frames, naive_total) + # Within one frame of full → user clearly intended full coverage; pin to latent_frames. + if target_total >= latent_frames - 1: + target_total = latent_frames + + exact = [p * target_total / total_pixel for p in pixel_lengths] + result = [int(e) for e in exact] + diff = target_total - sum(result) + if diff > 0: + order = sorted(range(len(exact)), key=lambda i: -(exact[i] - int(exact[i]))) + for k in range(diff): + result[order[k % len(order)]] += 1 + + # Ensure every segment has ≥ 1 latent frame (steal from the largest if needed). + for i in range(len(result)): + if result[i] < 1: + max_idx = max(range(len(result)), key=lambda j: result[j]) + if result[max_idx] > 1: + result[max_idx] -= 1 + result[i] = 1 + + return result + + +def _encode_relay(model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon): + for name, val in (("global_prompt", global_prompt), + ("local_prompts", local_prompts), + ("segment_lengths", segment_lengths)): + if val is None: + raise ValueError( + f"PromptRelay: '{name}' arrived as None. " + "Likely causes: a stale workflow JSON saved with null, the timeline " + "editor's web extension failing to load, or an upstream node returning None. " + "Set the field to an empty string or fix the upstream connection." + ) + + # Split prompts but do NOT filter out empty ones yet, so we can detect them + locals_list = [p.strip() for p in local_prompts.split("|")] + + # Check if any specific segment is empty + for p in locals_list: + if not p: + raise ValueError("There is a segment on the timeline missing a prompt!") + + if not locals_list or (len(locals_list) == 1 and not locals_list[0]): + raise ValueError("At least one local prompt is required.") + + arch, patch_size, temporal_stride = detect_model_type(model) + + samples = latent["samples"] + latent_frames = samples.shape[2] + tokens_per_frame = (samples.shape[3] // patch_size[1]) * (samples.shape[4] // patch_size[2]) + + parsed_lengths = None + if segment_lengths.strip(): + pixel_lengths = [int(float(x.strip())) for x in segment_lengths.split(",") if x.strip()] + parsed_lengths = _convert_to_latent_lengths(pixel_lengths, temporal_stride, latent_frames) + + raw_tokenizer = get_raw_tokenizer(clip) + full_prompt, token_ranges = map_token_indices(raw_tokenizer, global_prompt, locals_list) + + log.info("[PromptRelay] Global: tokens [0:%d] (%d tokens)", token_ranges[0][0], token_ranges[0][0]) + for i, (s, e) in enumerate(token_ranges): + log.info("[PromptRelay] Segment %d: tokens [%d:%d] (%d tokens)", i, s, e, e - s) + + conditioning = clip.encode_from_tokens_scheduled(clip.tokenize(full_prompt)) + + effective_lengths = distribute_segment_lengths(len(locals_list), latent_frames, parsed_lengths) + + log.info( + "[PromptRelay] Latent: %d frames, %d tokens/frame, segments: %s", + latent_frames, tokens_per_frame, effective_lengths, + ) + + q_token_idx = build_segments(token_ranges, effective_lengths, epsilon, None) + mask_fn = create_mask_fn(q_token_idx, tokens_per_frame, latent_frames) + + patched = model.clone() + apply_patches(patched, arch, mask_fn) + + return patched, conditioning + + +class LTXDirector(io.ComfyNode): + """WYSIWYG timeline variant — segments and lengths come from a visual editor in the node UI.""" + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXDirector", + display_name="LTX Director", + category="conditioning/prompt_relay", + description=( + "Same as Prompt Relay Encode, but local prompts and segment lengths are edited " + "visually as draggable blocks on a timeline. The duration_frames input only sets the " + "timeline scale (pixel space) — actual frame count is still read from the latent." + ), + inputs=[ + io.Model.Input("model"), + io.Clip.Input("clip"), + io.Vae.Input("audio_vae", optional=True, tooltip="Optional. Connect an Audio VAE to generate audio latents."), + io.Latent.Input("optional_latent", optional=True, tooltip="Optional. Connect a latent to override the auto-generated one."), + io.String.Input( + "global_prompt", multiline=True, default="", + tooltip="Conditions the entire video. Anchors persistent characters, objects, and scene context.", + ), + io.Int.Input( + "duration_frames", default=120, min=1, max=10000, step=1, + tooltip="Total timeline length in pixel-space frames. Used by the editor for visual scale only.", + ), + io.Float.Input( + "duration_seconds", default=5, min=0.1, max=1000.0, step=0.01, + tooltip="Total timeline duration in seconds (computed/synced from frames).", + ), + io.String.Input( + "timeline_data", default="", + tooltip="JSON state of the timeline editor (auto-managed; do not edit by hand).", + ), + io.Boolean.Input( + "use_custom_audio", default=False, optional=True, + tooltip="Toggle between using timeline audio (ON) and generating audio from scratch (OFF).", + ), + io.String.Input( + "local_prompts", multiline=True, default="", + tooltip="Auto-populated from the timeline editor.", + ), + io.String.Input( + "segment_lengths", default="", + tooltip="Auto-populated from the timeline editor (pixel-space frame counts).", + ), + io.Float.Input( + "epsilon", default=0.001, min=0.0001, max=0.99, step=0.0001, + tooltip="Penalty decay parameter. Values below ~0.1 all produce sharp boundaries (paper default 0.001). For softer transitions, try 0.5 or higher.", + ), + io.Float.Input( + "frame_rate", default=24, min=1, max=240, step=1, optional=True, + tooltip="Frames per second — only affects how time is displayed in the timeline editor when time_units is set to 'seconds'.", + ), + io.Combo.Input( + "display_mode", options=["frames", "seconds"], default="seconds", optional=True, + tooltip="Display the ruler, segment ranges, length input, and total in frames or seconds. Internal storage is always pixel-space frames.", + ), + io.String.Input( + "guide_strength", default="", + tooltip="Auto-populated from the timeline editor (comma-separated guide strengths for image segments).", + ), + io.Int.Input( + "custom_width", default=0, min=0, max=8192, step=1, optional=True, + tooltip="Target output width for all image segments. Set to 0 to use the original image width.", + ), + io.Int.Input( + "custom_height", default=0, min=0, max=8192, step=1, optional=True, + tooltip="Target output height for all image segments. Set to 0 to use the original image height.", + ), + io.Combo.Input( + "resize_method", + options=["maintain aspect ratio", "stretch to fit", "pad", "crop"], + default="maintain aspect ratio", + optional=True, + tooltip="How to resize image segments to fit the target dimensions.", + ), + io.Int.Input( + "divisible_by", default=32, min=1, max=256, step=1, optional=True, + tooltip="Snap the final output image dimensions to be divisible by this number (e.g. 32 for LTX).", + ), + io.Int.Input( + "img_compression", default=18, min=0, max=100, step=1, optional=True, + tooltip="H.264 CRF compression to apply to each guide image. 0 = no compression, higher = more artefacts.", + ), + ], + outputs=[ + io.Model.Output(display_name="model"), + io.Conditioning.Output(display_name="positive"), + io.Latent.Output(display_name="video_latent", tooltip="Auto-generated LTXV empty latent (only populated when no latent is connected)."), + io.Latent.Output(display_name="audio_latent", tooltip="Auto-generated audio latent (uses custom audio if enabled)."), + GuideData.Output(display_name="guide_data"), + io.Float.Output(display_name="frame_rate", tooltip="The frame rate used for the timeline."), + io.Audio.Output(display_name="combined_audio", tooltip="Combined timeline audio layout."), + ], + ) + + @classmethod + def execute(cls, model, clip, global_prompt, duration_frames, duration_seconds, + timeline_data, local_prompts, segment_lengths, guide_strength="", epsilon=1e-3, + frame_rate=24, display_mode="seconds", + custom_width=768, custom_height=512, resize_method="maintain aspect ratio", + divisible_by=32, img_compression=0, audio_vae=None, optional_latent=None, + use_custom_audio=False) -> io.NodeOutput: + + # --- Build guide_data from image segments FIRST (to derive output dimensions) --- + guide_data = {"images": [], "insert_frames": [], "strengths": [], "frame_rate": frame_rate} + derived_w, derived_h = custom_width, custom_height + try: + tdata = json.loads(timeline_data) if timeline_data else {} + img_segs = [ + s for s in tdata.get("segments", []) + if s.get("type", "image") == "image" + and (s.get("imageFile") or s.get("imageB64")) + and int(s.get("start", 0)) < duration_frames # exclude segments fully outside duration + ] + img_segs.sort(key=lambda s: s["start"]) + + strengths = [] + if guide_strength.strip(): + strengths = [float(x.strip()) for x in guide_strength.split(",") if x.strip()] + + for idx, seg in enumerate(img_segs): + tensor = _load_image_tensor(seg) + + # Apply resize + src_h, src_w = tensor.shape[1], tensor.shape[2] + + def snap(val, div): + return max(div, (val // div) * div) + + if custom_width > 0 and custom_height > 0: + # Both dimensions set — apply selected resize_method (pad, crop, stretch, maintain AR) + tensor = _resize_image(tensor, custom_width, custom_height, resize_method, divisible_by) + elif custom_width > 0: + # Width only — scale height from AR, snap both, then resize to exact dimensions + tgt_w = snap(custom_width, divisible_by) + tgt_h = snap(int(src_h * tgt_w / src_w), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + elif custom_height > 0: + # Height only — scale width from AR, snap both, then resize to exact dimensions + tgt_h = snap(custom_height, divisible_by) + tgt_w = snap(int(src_w * tgt_h / src_h), divisible_by) + tensor = _resize_image(tensor, tgt_w, tgt_h, "stretch to fit", divisible_by) + else: + # Both zero — keep original dimensions, just snap to divisible_by + tensor = _resize_image(tensor, src_w, src_h, "maintain aspect ratio", divisible_by) + + + # Apply compression + if img_compression > 0: + tensor = _compress_image(tensor, img_compression) + + # Record dimensions of the first processed image for latent generation + if idx == 0: + derived_h = tensor.shape[1] + derived_w = tensor.shape[2] + + strength = strengths[idx] if idx < len(strengths) else 1.0 + guide_data["images"].append(tensor) + guide_data["insert_frames"].append(int(seg["start"])) + guide_data["strengths"].append(float(strength)) + + # If no images were loaded from the timeline, create a dummy image at strength 0 + # to prevent artifacts in text-to-video mode. + if not guide_data["images"]: + w = derived_w if derived_w > 0 else 768 + h = derived_h if derived_h > 0 else 512 + w = (w // 32) * 32 + h = (h // 32) * 32 + + dummy_image = torch.zeros((1, h, w, 3), dtype=torch.float32) + guide_data["images"].append(dummy_image) + guide_data["insert_frames"].append(0) + guide_data["strengths"].append(0.0) + + derived_w = w + derived_h = h + except Exception as e: + log.warning("[PromptRelay] Could not build guide_data: %s", e) + + # --- Auto-generate LTXV latent if none was provided --- + ltxv_length = duration_frames + 1 + if optional_latent is None: + latent_w = max(32, (derived_w // 32) * 32) + latent_h = max(32, (derived_h // 32) * 32) + # LTXV temporal: ((length - 1) // 8) + 1 latent frames; invert to get pixel frames -> length + latent_t = ((ltxv_length - 1) // 8) + 1 + samples = torch.zeros( + [1, 128, latent_t, latent_h // 32, latent_w // 32], + device=comfy.model_management.intermediate_device(), + ) + latent = {"samples": samples} + log.info( + "[PromptRelay] Auto-generated LTXV latent: %dx%d, %d pixel frames (%d latent frames)", + latent_w, latent_h, ltxv_length, latent_t, + ) + else: + latent = optional_latent + + patched, conditioning = _encode_relay( + model, clip, latent, global_prompt, local_prompts, segment_lengths, epsilon, + ) + + # --- Build Audio Output --- + audio_out = _build_combined_audio(timeline_data, ltxv_length, float(frame_rate)) + + # --- Audio Latent Generation --- + audio_latent = {} + + if audio_vae is not None: + # Helper to generate empty latent + def get_empty_latent(): + z_channels = audio_vae.latent_channels + audio_freq = audio_vae.first_stage_model.latent_frequency_bins + num_audio_latents = audio_vae.first_stage_model.num_of_latents_from_frames(ltxv_length, float(frame_rate)) + audio_latents = torch.zeros( + (1, z_channels, num_audio_latents, audio_freq), + device=comfy.model_management.intermediate_device(), + ) + return {"samples": audio_latents, "type": "audio"} + + if use_custom_audio: + try: + if audio_out is not None: + # 1. Encode audio waveform into latent space + # VAE expects shape (batch, samples, channels), so we move dim 1 (channels) to the end + latent_samples = audio_vae.encode(audio_out["waveform"].movedim(1, -1)) + + if latent_samples.numel() == 0: + raise ValueError("Encoded audio latent is empty (0 elements).") + + # 2. Create solid mask with value 0.0 (0 means keep/use conditioning, 1 means generate noise) + mask = torch.full( + (1, latent_samples.shape[-2], latent_samples.shape[-1]), + 0.0, + dtype=torch.float32, + device=comfy.model_management.intermediate_device() + ) + + # 3. Set Latent Noise Mask + audio_latent = { + "samples": latent_samples, + "type": "audio", + "noise_mask": mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])) + } + log.info("[PromptRelay] Generated custom audio latent with noise mask (value=0.0).") + else: + raise ValueError("No audio waveform to encode.") + except Exception as e: + log.error("[PromptRelay] Failed to generate custom audio latent: %s", e) + raise e + else: + # Generate empty latent + try: + audio_latent = get_empty_latent() + log.info("[PromptRelay] Auto-generated empty audio latent.") + except Exception as e: + log.warning("[PromptRelay] Could not generate empty audio latent: %s", e) + + return io.NodeOutput(patched, conditioning, latent, audio_latent, guide_data, float(frame_rate), audio_out) + + +NODE_CLASS_MAPPINGS = { + "LTXDirector": LTXDirector, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "PromptRelayEncodeTimeline": "Prompt Relay Encode (Timeline)", +} \ No newline at end of file diff --git a/ltx_director_guide.py b/ltx_director_guide.py new file mode 100644 index 0000000..6ede4c9 --- /dev/null +++ b/ltx_director_guide.py @@ -0,0 +1,90 @@ +from comfy_extras.nodes_lt import LTXVAddGuide +import torch +import comfy.utils +from comfy_api.latest import io +from .ltx_director import GuideData + + +class LTXDirectorGuide(LTXVAddGuide): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="LTXDirectorGuide", + display_name="LTX Director Guide", + category="LTXVCustom", + description=( + "Applies guide images from a Prompt Relay Timeline node at the frame positions " + "and strengths defined on the timeline. Connect guide_data from the timeline node." + ), + inputs=[ + io.Conditioning.Input("positive", tooltip="Positive conditioning to add guide keyframe info to."), + io.Conditioning.Input("negative", tooltip="Negative conditioning to add guide keyframe info to."), + io.Vae.Input("vae", tooltip="Video VAE used to encode the guide images."), + io.Latent.Input("latent", tooltip="Video latent — guides are inserted into this latent."), + GuideData.Input("guide_data", tooltip="Guide data produced by Prompt Relay Encode (Timeline)."), + io.Float.Input("scale_by", default=1.0, min=0.01, max=8.0, step=0.01, tooltip="Scale the latent by this factor."), + io.Combo.Input("upscale_method", options=["nearest-exact", "bilinear", "area", "bicubic", "bislerp"], default="bicubic", tooltip="Method used to upscale/downscale the latent."), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + io.Latent.Output(display_name="latent", tooltip="Video latent with guide frames applied."), + ], + ) + + @classmethod + def execute(cls, positive, negative, vae, latent, guide_data, scale_by=1.0, upscale_method="bicubic") -> io.NodeOutput: + scale_factors = vae.downscale_index_formula + + # Clone latents to avoid mutating upstream nodes + latent_image = latent["samples"].clone() + + if "noise_mask" in latent: + noise_mask = latent["noise_mask"].clone() + else: + batch, _, latent_frames, latent_height, latent_width = latent_image.shape + noise_mask = torch.ones( + (batch, 1, latent_frames, 1, 1), + dtype=torch.float32, + device=latent_image.device, + ) + + # Apply scale factor if not 1.0 + if scale_by != 1.0: + B, C, F, H, W = latent_image.shape + width = round(W * scale_by) + height = round(H * scale_by) + + # Reshape to 4D for common_upscale + latent_4d = latent_image.permute(0, 2, 1, 3, 4).reshape(B * F, C, H, W) + latent_resized_4d = comfy.utils.common_upscale(latent_4d, width, height, upscale_method, "disabled") + latent_image = latent_resized_4d.reshape(B, F, C, height, width).permute(0, 2, 1, 3, 4) + + # Also resize noise mask if it's not a broadcasted mask + if noise_mask.shape[-1] > 1 or noise_mask.shape[-2] > 1: + mask_4d = noise_mask.permute(0, 2, 1, 3, 4).reshape(B * F, 1, H, W) + mask_resized_4d = comfy.utils.common_upscale(mask_4d, width, height, upscale_method, "disabled") + noise_mask = mask_resized_4d.reshape(B, F, 1, height, width).permute(0, 2, 1, 3, 4) + + _, _, latent_length, latent_height, latent_width = latent_image.shape + + images = guide_data.get("images", []) + insert_frames = guide_data.get("insert_frames", []) + strengths = guide_data.get("strengths", []) + + for idx, img_tensor in enumerate(images): + f_idx = insert_frames[idx] if idx < len(insert_frames) else 0 + strength = strengths[idx] if idx < len(strengths) else 1.0 + + image_1, t = cls.encode(vae, latent_width, latent_height, img_tensor, scale_factors) + frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image_1), f_idx, scale_factors) + + assert latent_idx + t.shape[2] <= latent_length, ( + f"Guide image {idx + 1}: conditioning frames exceed the length of the latent sequence." + ) + + positive, negative, latent_image, noise_mask = cls.append_keyframe( + positive, negative, frame_idx, latent_image, noise_mask, t, strength, scale_factors, + ) + + return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask}) \ No newline at end of file diff --git a/patches.py b/patches.py new file mode 100644 index 0000000..1404146 --- /dev/null +++ b/patches.py @@ -0,0 +1,167 @@ +import types +import torch +import comfy.ldm.modules.attention + + +def _masked_attention(q, k, v, heads, mask, transformer_options={}, **kwargs): + # Bypass wrap_attn (sage/etc may ignore masks) by calling attention_pytorch directly. + return comfy.ldm.modules.attention.attention_pytorch( + q, k, v, heads, mask=mask, + _inside_attn_wrapper=True, + transformer_options=transformer_options, + **kwargs, + ) + + +def _wan_t2v_forward(self, mask_fn, x, context, transformer_options={}, **kwargs): + q = self.norm_q(self.q(x)) + k = self.norm_k(self.k(context)) + v = self.v(context) + + mask = mask_fn(q, k, transformer_options) + if mask is not None: + x = _masked_attention(q, k, v, heads=self.num_heads, mask=mask, + transformer_options=transformer_options) + else: + x = comfy.ldm.modules.attention.optimized_attention( + q, k, v, heads=self.num_heads, transformer_options=transformer_options, + ) + return self.o(x) + + +def _wan_i2v_forward(self, mask_fn, x, context, context_img_len, transformer_options={}, **kwargs): + context_img = context[:, :context_img_len] + context_text = context[:, context_img_len:] + + q = self.norm_q(self.q(x)) + + k_img = self.norm_k_img(self.k_img(context_img)) + v_img = self.v_img(context_img) + img_x = comfy.ldm.modules.attention.optimized_attention( + q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, + ) + + k = self.norm_k(self.k(context_text)) + v = self.v(context_text) + + mask = mask_fn(q, k, transformer_options) + if mask is not None: + x = _masked_attention(q, k, v, heads=self.num_heads, mask=mask, + transformer_options=transformer_options) + else: + x = comfy.ldm.modules.attention.optimized_attention( + q, k, v, heads=self.num_heads, transformer_options=transformer_options, + ) + + return self.o(x + img_x) + + +def _ltx_forward(self, mask_fn, x, context=None, mask=None, pe=None, k_pe=None, transformer_options={}): + from comfy.ldm.lightricks.model import apply_rotary_emb + + is_self_attn = context is None + context = x if is_self_attn else context + + q = self.q_norm(self.to_q(x)) + k = self.k_norm(self.to_k(context)) + v = self.to_v(context) + + if pe is not None: + q = apply_rotary_emb(q, pe) + k = apply_rotary_emb(k, pe if k_pe is None else k_pe) + + if not is_self_attn: + temporal_mask = mask_fn(q, k, transformer_options) + if temporal_mask is not None: + mask = temporal_mask if mask is None else mask + temporal_mask + + if mask is None: + out = comfy.ldm.modules.attention.optimized_attention( + q, k, v, self.heads, attn_precision=self.attn_precision, + transformer_options=transformer_options, + ) + else: + out = _masked_attention(q, k, v, self.heads, mask=mask, + attn_precision=self.attn_precision, + transformer_options=transformer_options) + + if self.to_gate_logits is not None: + gate_logits = self.to_gate_logits(x) + b, t, _ = out.shape + out = out.view(b, t, self.heads, self.dim_head) + out = out * (2.0 * torch.sigmoid(gate_logits)).unsqueeze(-1) + out = out.view(b, t, self.heads * self.dim_head) + + return self.to_out(out) + + +class _CrossAttnPatch: + """Descriptor that binds (impl, mask_fn) as a method onto a cross-attn module.""" + + def __init__(self, impl, mask_fn): + self.impl = impl + self.mask_fn = mask_fn + + def __get__(self, obj, objtype=None): + impl, mask_fn = self.impl, self.mask_fn + + def wrapped(self_module, *args, **kwargs): + return impl(self_module, mask_fn, *args, **kwargs) + + return types.MethodType(wrapped, obj) + + +def detect_model_type(model): + """Return (arch, patch_size, temporal_stride) for latent geometry. + + temporal_stride is the VAE's pixel→latent temporal compression factor, + used to convert user-facing pixel frame counts to latent frames. + """ + diff_model = model.model.diffusion_model + + if hasattr(diff_model, "patch_size") and not hasattr(diff_model, "patchifier"): + return "wan", tuple(diff_model.patch_size), 4 + + if hasattr(diff_model, "patchifier"): + return "ltx", (1, 1, 1), int(diff_model.vae_scale_factors[0]) + + raise ValueError( + f"Unsupported model type: {type(diff_model).__name__}. " + f"Currently supports Wan and LTX models." + ) + + +def _check_unpatched(model_clone, key): + if key in getattr(model_clone, "object_patches", {}): + raise RuntimeError( + f"PromptRelay: cross-attention forward at '{key}' is already patched by " + "another node (e.g. KJNodes NAG). Stacking is not supported — remove the " + "conflicting node." + ) + + +def apply_patches(model_clone, arch, mask_fn): + diffusion_model = model_clone.get_model_object("diffusion_model") + + if arch == "wan": + from comfy.ldm.wan.model import WanI2VCrossAttention + for idx, block in enumerate(diffusion_model.blocks): + key = f"diffusion_model.blocks.{idx}.cross_attn.forward" + _check_unpatched(model_clone, key) + cross_attn = block.cross_attn + impl = _wan_i2v_forward if isinstance(cross_attn, WanI2VCrossAttention) else _wan_t2v_forward + model_clone.add_object_patch(key, _CrossAttnPatch(impl, mask_fn).__get__(cross_attn, cross_attn.__class__)) + return + + if arch == "ltx": + for idx, block in enumerate(diffusion_model.transformer_blocks): + for attr in ("attn2", "audio_attn2"): + module = getattr(block, attr, None) + if module is None: + continue + key = f"diffusion_model.transformer_blocks.{idx}.{attr}.forward" + _check_unpatched(model_clone, key) + model_clone.add_object_patch(key, _CrossAttnPatch(_ltx_forward, mask_fn).__get__(module, module.__class__)) + return + + raise ValueError(f"Unknown model arch: {arch}") diff --git a/prompt_relay.py b/prompt_relay.py new file mode 100644 index 0000000..adfffe0 --- /dev/null +++ b/prompt_relay.py @@ -0,0 +1,201 @@ +import logging +import math +import torch + +log = logging.getLogger(__name__) + + +def build_temporal_cost(q_token_idx, Lq, Lk, device, dtype, tokens_per_frame): + """Gaussian penalty matrix [Lq, Lk] for video cross-attention (integer frame indexing).""" + offset = torch.zeros(Lq, Lk, device=device, dtype=dtype) + query_frames = torch.arange(Lq, device=device, dtype=torch.long) // tokens_per_frame + + for seg in q_token_idx: + local = seg["local_token_idx"].to(device=device) + d = (query_frames.float()[:, None] - seg["midpoint"]).abs() + strength = seg.get("strength", 1.0) + cost = strength * (torch.relu(d - seg["window"]) ** 2) / (2 * seg["sigma"] ** 2) + offset[:, local] = cost.to(offset.dtype) + + return offset + + +def build_temporal_cost_scaled(q_token_idx, Lq, Lk, device, dtype, latent_frames): + """Penalty matrix for queries that don't map to integer frames (e.g. LTXAV audio tokens).""" + offset = torch.zeros(Lq, Lk, device=device, dtype=dtype) + query_frames = torch.arange(Lq, device=device, dtype=torch.float32) * latent_frames / Lq + + for seg in q_token_idx: + local = seg["local_token_idx"].to(device=device) + d = (query_frames[:, None] - seg["midpoint"]).abs() + sigma_a = seg.get("sigma_audio", seg["sigma"]) + window_a = seg.get("window_audio", seg["window"]) + strength_a = seg.get("strength_audio", 1.0) + cost = strength_a * (torch.relu(d - window_a) ** 2) / (2 * sigma_a ** 2) + offset[:, local] = cost.to(offset.dtype) + + return offset + + +def create_mask_fn(q_token_idx, fallback_tokens_per_frame, latent_frames): + """Closure: mask_fn(q, k, transformer_options) -> additive mask or None.""" + cache = {} + max_token_idx = max(int(seg["local_token_idx"].max().item()) for seg in q_token_idx) + 1 + + def mask_fn(q, k, transformer_options): + Lq, Lk = q.shape[1], k.shape[1] + + if Lq == Lk: + return None + + # Only apply on conditional pass — not unconditional (negative prompt) + cond_or_uncond = transformer_options.get("cond_or_uncond", []) + if 1 in cond_or_uncond and 0 not in cond_or_uncond: + return None + + grid_sizes = transformer_options.get("grid_sizes", None) + video_tpf = int(grid_sizes[1]) * int(grid_sizes[2]) if grid_sizes is not None else fallback_tokens_per_frame + video_lq = latent_frames * video_tpf + + # Skip cross-modal attention — text keys are padded to a fixed length ≥ max_token_idx and != video_lq + if Lk == video_lq or Lk < max_token_idx: + return None + + mode = "video" if Lq == video_lq else "scaled" + + key = (Lq, Lk, mode, q.device) + if key not in cache: + if mode == "video": + cost = build_temporal_cost(q_token_idx, Lq, Lk, q.device, q.dtype, video_tpf) + else: + cost = build_temporal_cost_scaled(q_token_idx, Lq, Lk, q.device, q.dtype, latent_frames) + log.info( + "[PromptRelay] Built penalty matrix (%s): Lq=%d, Lk=%d, nonzero=%d/%d", + mode, Lq, Lk, (cost > 0).sum().item(), cost.numel(), + ) + cache[key] = -cost + + return cache[key].to(q.dtype) + + return mask_fn + + +def build_segments(token_ranges, segment_lengths, epsilon=1e-3, relay_options=None): + """Per-segment metadata for the temporal penalty. + + relay_options (optional dict) overrides per-stream knobs: + video_strength, video_window_scale, + audio_epsilon, audio_strength, audio_window_scale + Audio knobs only affect architectures whose cross-attention takes the scaled + (non-integer-frame) path — currently LTX audio_attn2. + """ + # Paper uses constant sigma = 1/ln(1/epsilon) regardless of segment length + sigma = 1.0 / math.log(1.0 / epsilon) if 0 < epsilon < 1 else 0.1448 + + opts = relay_options or {} + v_strength = opts.get("video_strength", 1.0) + v_window_scale = opts.get("video_window_scale", 1.0) + a_epsilon = opts.get("audio_epsilon") + a_strength = opts.get("audio_strength", 1.0) + a_window_scale = opts.get("audio_window_scale", 1.0) + + if a_epsilon is not None and 0 < a_epsilon < 1: + sigma_audio = 1.0 / math.log(1.0 / a_epsilon) + else: + sigma_audio = sigma + + if relay_options: + log.info( + "[PromptRelay] Advanced options active — video: strength=%.3f window_scale=%.3f | " + "audio: epsilon=%s strength=%.3f window_scale=%.3f", + v_strength, v_window_scale, + f"{a_epsilon:.4f}" if a_epsilon is not None else "inherit", + a_strength, a_window_scale, + ) + + q_token_idx = [] + frame_cursor = 0 + + for (tok_start, tok_end), L in zip(token_ranges, segment_lengths): + if L <= 0: + frame_cursor += L + continue + midpoint = (2 * frame_cursor + L) // 2 + base_window = max(L // 2 - 2, 0) + q_token_idx.append({ + "local_token_idx": torch.arange(tok_start, tok_end), + "midpoint": midpoint, + "window": max(base_window * v_window_scale, 0.0), + "sigma": sigma, + "strength": v_strength, + "window_audio": max(base_window * a_window_scale, 0.0), + "sigma_audio": sigma_audio, + "strength_audio": a_strength, + }) + frame_cursor += L + + return q_token_idx + + +def get_raw_tokenizer(clip): + """Extract the raw SPiece/HF tokenizer from a ComfyUI CLIP object.""" + tokenizer_wrapper = clip.tokenizer + for attr_name in dir(tokenizer_wrapper): + if attr_name.startswith("_"): + continue + inner = getattr(tokenizer_wrapper, attr_name, None) + if inner is not None and hasattr(inner, "tokenizer"): + return inner.tokenizer + + raise RuntimeError( + f"Could not find raw tokenizer on CLIP object. " + f"Known attributes: {[a for a in dir(tokenizer_wrapper) if not a.startswith('_')]}" + ) + + +def map_token_indices(raw_tokenizer, global_prompt, local_prompts): + """Tokenize global + space-prefixed locals; return (full_prompt, per-local token ranges). + + Uses incremental tokenization to avoid SentencePiece context-dependency issues. + """ + prefixed_locals = [" " + lp for lp in local_prompts] + full_prompt = global_prompt + "".join(prefixed_locals) + has_eos = getattr(raw_tokenizer, "add_eos", False) + eos_adj = 1 if has_eos else 0 + + prev_len = len(raw_tokenizer(global_prompt)["input_ids"]) - eos_adj + token_ranges = [] + built = global_prompt + + for plp in prefixed_locals: + built += plp + cur_len = len(raw_tokenizer(built)["input_ids"]) - eos_adj + if cur_len <= prev_len: + raise ValueError(f"Local prompt produced no tokens: '{plp.strip()}'") + token_ranges.append((prev_len, cur_len)) + prev_len = cur_len + + return full_prompt, token_ranges + + +def distribute_segment_lengths(num_segments, latent_frames, specified_lengths=None): + """Validate or auto-distribute segment frame counts, capped to fit within latent_frames.""" + if specified_lengths: + if len(specified_lengths) != num_segments: + raise ValueError( + f"Number of segment_lengths ({len(specified_lengths)}) " + f"must match number of local prompts ({num_segments})" + ) + lengths = specified_lengths + else: + # ceil division — matches reference implementation + step = -(-latent_frames // num_segments) + lengths = [step] * num_segments + + effective = [] + cursor = 0 + for L in lengths: + end = min(cursor + L, latent_frames) + effective.append(max(end - cursor, 0)) + cursor = end + return effective diff --git a/pyproject.toml b/pyproject.toml index 28226d2..1d68418 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "WhatDreamsCost-ComfyUI" description = "A variety of custom ComfyUI nodes and workflows for creatives." -version = "1.2.9" +version = "1.3.0" license = {file = "LICENSE"} # classifiers = [ # # For OS-independent nodes (works on all operating systems)