diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 69a5597..c84c29f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,9 +16,6 @@ jobs: with: python-version: "3.11" cache: pip - - uses: actions/setup-node@v4 - with: - node-version: "20" - name: Install test dependencies run: | python -m pip install --upgrade pip @@ -28,7 +25,3 @@ jobs: run: python -m compileall -q . - name: Run tests run: python -m pytest -q -p no:cacheprovider - - name: Check frontend JavaScript - run: | - node --check web/h3_continuum_v2.js - node --check web/h3_continuum_v214.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a4f7e0b..075f2fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,62 +1,22 @@ # Changelog -## 2.1.5 +## 2.1.7 -- Restore compact Individual Clip Overrides and Advanced accordions without removing or reordering backend slots. -- Add `Seam Correction: Auto / Off / Basic` and make guarded Auto the new-node default. -- Consolidate the Sampler V2 frontend behavior into one extension wrapper. -- Preserve backend widget type, serialization behavior, and relative widget order. -- Keep all backend input and output slots registered instead of removing and re-adding them. -- Collapse advanced widgets only through computeSize and DOM display state. -- Add static widget-order and workflow-value mapping regressions for UI serialization stability. +- Added the compact static `H3 Continuum Sampler` facade over the unchanged V2 execution core. +- Added Clip Overrides, Advanced, and Result pack helper nodes. +- Preserved `H3ContinuumSamplerV2` as a Legacy/Core workflow compatibility node. +- Removed the frontend display controller and all dynamic socket, proxy-widget, DOM, resize, and serialization workarounds. +- Kept State, Session, Prompt Plan, Sampling, Native Continuity, and Spectrum Interop contracts unchanged. +## 2.1.6 -## 2.1.4 - -- Rename the public package and repository branding to ComfyUI-H3-Continuum. -- Synchronize release metadata, validation records, repository URLs, and the SHA256 manifest. -- Add CPU GitHub Actions checks for Python, pytest, and frontend JavaScript syntax. -- Make the integrated V2 sampler the only primary Continuum workflow node. -- Mark V1 building blocks and auxiliary plan/state nodes as deprecated Legacy nodes while preserving saved-workflow identifiers. -- Collapse continuation, session, prompt-plan, diagnostics, and advanced generation controls by default. -- Automatically expose Advanced settings when an existing workflow has an advanced connection. -- Preserve output ordering, State/Session schemas, sampling, and accelerator behavior. - -## 2.1.3 - -- Added one stable external `Sequence Prompt` STRING socket for the complete sequence. -- Added `Auto / Fixed / List / Timeline` Prompt Format selection without changing socket connections. -- Added deterministic Auto detection for `---` lists and `[0-5s]` / `[Chunk 1]` timeline sections. -- Kept individual `Clip N Prompt` inputs as optional overrides, collapsed by default while preserving connected legacy sockets. -- Preserved connected Prompt Plan and saved prompt widget fallbacks without changing Prompt Plan schema. - -## 2.1.2 - -- Added fail-closed Native Continuity preflight for raw AV tails, native video-cycle phase, signed 40/24 audio-grid placement, finite Context tensors, and packed Context/Target row separation. -- Added branch-local immutable PackedLayout signatures on native Actual calls while allowing Spectrum Forecast calls to bypass native layout execution. -- Added per-chunk MODEL copy-on-write isolation so Spectrum runtime history, archives, and disable state cannot leak between chunks. -- Added read-only Spectrum Interop API v1 hints only when a valid previous Context exists; initial chunks emit no hint. -- Added Spectrum-compatible Actual Prefix 2 requests with strict metadata types, invalid/unknown fail-open behavior, total-step clamping, and no prefix during offline replay. -- Preserved State, Session, Prompt Plan, Seam Correction, node identifiers, and the raw-latent-only continuation path. - -## 2.1.0 - -- Added opt-in `Seam Correction: Off / Basic` while preserving the complete V2.0.2 decode path for `Off`. -- Added normalized CPU-only video seam scoring and conservative adaptive cuts from 0 to 3 frames before the legacy boundary. -- Added bounded luminance gain, chroma bias, and motion-aware 0–2 frame cosine blending with score-based rollback. -- Added boundary-local stereo-safe audio correlation alignment, level/DC guards, and 10–60ms equal-power crossfades without shifting the following clip. -- Added legacy, cut, and corrected seam scores plus video/audio decisions to Basic and Full diagnostics. -- Added boundary-level fallback without changing Sampling, Context, State, Session, Prompt Plan, V1 nodes, or accelerator composition. - -## 2.0.2 - -- Added external `Clip N Prompt` STRING inputs for 1–16 chunks, intended for ComfyUI's `Text (Multiline)` node. -- Made the visible prompt inputs follow the `chunks` widget while preserving the internal Fixed/List/Timeline values as a legacy fallback. -- Added deterministic per-clip prompt overrides with regenerated prompt hashes and unchanged Prompt Plan schema. -- Added `Settings > H3 Continuum > Preview > Show latent preview`, enabled by default. -- Kept latent-preview control out of the sampler UI and passed the setting to the existing sampling callback without changing final AV output. -- Added regression coverage for selective external prompt overrides, hashes, and the 16 connectable prompt inputs. -- Preserved all V1/V2 node identifiers and State, Session, and Prompt Plan schemas. +- Rebuilt the Sampler V2 frontend as one Vue-aware display controller. +- Removed the Individual Clip Overrides accordion; only active Clip Prompt sockets are shown. +- Kept connected out-of-range Clip Prompt sockets visible when chunks is reduced. +- Added one Advanced Settings control; connected advanced sockets stay visible while collapsed. +- Removed fixed node-height handling and all input/output add/remove UI folding. +- Kept backend socket/widget order, types, serialization, State/Session/Prompt Plan schemas, sampling, and Spectrum Interop unchanged. +- Exposed show_preview as a per-node basic control and removed the duplicate global preview setting. ## 2.0.1 diff --git a/INSTALL_JA.txt b/INSTALL_JA.txt index d9d0d74..d4726ce 100644 --- a/INSTALL_JA.txt +++ b/INSTALL_JA.txt @@ -1,49 +1,21 @@ -ComfyUI-H3-Continuum 2.1.5 — インストール +ComfyUI-H3-Continuum 2.1.7 — インストール 1. ComfyUIを終了します。 2. ZIPを展開します。 -3. フォルダーを次へ配置します。 +3. `ComfyUI-H3-Continuum` フォルダーを ComfyUI/custom_nodes/ に配置します。 +4. 旧 `ComfyUI-H3-Continuum-Join` が残っている場合は退避します。 +5. ComfyUIを再起動します。 +6. 旧V2.1.6 UIが残る場合はブラウザをハードリロードします。 - D:\StabilityMatrix\Data\Packages\ComfyUI_W\custom_nodes\ComfyUI-H3-Continuum\ - -4. ComfyUIを再起動します。 -5. ノード検索で「H3 Continuum Sampler V2」を確認します。 +StabilityMatrix標準例: +D:\StabilityMatrix\Data\Packages\ComfyUI_W\custom_nodes\ComfyUI-H3-Continuum\ 自動インストール: +install_windows.bat - install_windows.bat - -または: - - powershell -ExecutionPolicy Bypass -File .\install_windows.ps1 - -V2標準配線: - -H3 UNET → SageAttention → Sol-Attn → LoRA → Spectrum - ↓ -CLIP / Video VAE / Audio VAE / Sampler / Sigmas - ↓ - H3 Continuum Sampler V2 - ↓ - IMAGE + AUDIO → CreateVideo - -標準値: -- chunks: 3 +標準検証: +- chunks: 2 または 3 - chunk_seconds: 5.0 - continuity: Balanced — 22 frames -- audio_continuity: true -- exact_total_duration: true -- diagnostics: Basic -- strict_compatibility: true -- seam_correction: Auto(初期値)/ Off(2.0.2互換)/ Basic(常時V2.1境界補正) - -Prompt入力: -- Sequence PromptへComfyUI標準のText (Multiline)を1本接続します。 -- Prompt FormatはAuto / Fixed / List / Timelineから選択します。 -- Clip N Prompt Overrideは個別上書き用です。内部slot順を維持し、展開時はchunks数まで表示します。 - -生成中サムネイル: -- Settings > H3 Continuum > Preview > Show latent preview -- 初期値はONです。OFFにしても最終生成結果は変更されません。 - -V1ワークフローも読み込めますが、V1ノードはDeprecated Legacy扱いです。 +- Seam Correction: Off(Native Continuity比較時)/ Auto(通常利用) +- Sequence Prompt: Text (Multiline)を接続 diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index 3b4d36c..75653b8 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -1,77 +1,76 @@ -10da5e25dc84ddc7086d181b47466dc2a197c98958d60e4352605a2a3211ee1c .github/workflows/tests.yml -10da5e25dc84ddc7086d181b47466dc2a197c98958d60e4352605a2a3211ee1c .github/workflows/tests.yml -85eada29c12706d55b78e4d8100899d1dadde4d9867406b8bac4eec22b81c6b6 .gitignore -dca65fb1393e67f59c6f81f9411e3e745eb8f1097e9a2adf7ffc6f4e3901ca9f __init__.py 0005be4e1e56a88d5b2168dc3fe203d5f87944922aa400d87e8c840805d2a4e1 AGENTS.md -59acf85415443af8ba38370128a3ff8af1740049a3ba35d6898ad12e998c417c CHANGELOG.md -7dbbc8ce1018d6ac07c53eaa40f2d8d144babf9c6fe6bd94b7ec1d08adffa215 compatibility.py -a88d873af14b69988963b52bb57681445ee45bd4e60ab47073909a46ac5891bd constants.py -28e9f211d53668b929e80b9b65ade03879467db0804d95eafd73f4b3929a510b continuation.py -c93a3e52e4deabe5ef0db7f34a7605856a543089574ba17227e697e9a2c0139a docs/ARCHITECTURE.md -7660570840229b6fcbe62f0fbb52966d0099d64d7e6f01a739fbf9899293c8e4 docs/FUTURE_ROADMAP.md -90acddd795d84c5a24f79598670aec1655eaf58d45fd6d362dc758e2b880a071 docs/V2_1_4_UI_SIMPLIFICATION.md -ef3af02d3c8fb5fd41f33a94287112962d4c0024ebdd7a8becb041fc18d9dd2e docs/V2_1_5_UI_SERIALIZATION_STABILITY.md -f0806bb2b999794ddda072e98424dbd1b613df1cd8a096b12d979f124e066d8a docs/VALIDATION_JA.md -6771c9905d35666121c4892d2ec26202505f96d244ce814ad9a87a02878952cd examples/H3_CONTINUUM_V1_WIRING.md -f85ba3c69a473f0ac4bb96895c6d6532e8270121da65f858e364b19d31f05ef5 examples/H3_Continuum_V2_3x5s.json -13e2fb33d663201b893b1804a4b2020e7197c7004b42d99c86b039af7893cea8 examples/H3_CONTINUUM_V2_WIRING_JA.md -68e1a724d3b99458b86b8f643dcb916183e7e8dd12d7b4b3850991feb8077694 examples/recommended_settings_v1.json -5a26408c770381cdb259cb710de7dab12d7ddecca262bdbf35ba1566056050f5 examples/recommended_settings_v2.json -b8275c8cecde12aee0418cea456d1e7f4b875cbb1bce32f1f3d9e06a44dd0be1 examples/v2_prompt_list_example.txt -0e59ea821081cb93165aa3f88c6949736dd225837fe1cdcf1173aa5cd54c6d29 examples/v2_prompt_timeline_example.txt -ffabbfbb34d24085bf65a94d0b879654cff8cb8ea3d141c10e1b8f9af0ac5660 INSTALL_JA.txt -10c26a052ae8ee8f6fa9168971b92e27da9f0b838cfaf64170e228fbb8e131fa install_windows.bat -c0f1ccebc07cbee0159a2e0fd04683f5725d54e30edd386759e6964b46d84308 install_windows.ps1 -600634455576c58a79e4919603af0fdeb80478f64e3a3de5969a95383957c4b5 layout_adapter.py -3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986 LICENSE -d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py -f305dbc4a96221f2c5bc7102b43e5fbe4ac4b5622634ffc0deb444b1297bc4fa metadata.ini -43fc9e66a96adb9f5c6b653118e48cd5f5c3366dd3e7fc970d2500023bd5cc55 model_patch.py -a905ef1f536543af0f7b0de98da1c4daa618063364bc49ecb9bd4e1aafaf2b22 nodes.py -64b8826837053fc5767a71ccb9551a53fd711abdfa23a1b4232ccb6d29621943 PACKAGE_INFO_JA.txt -dee53deb806384e1c9933347f78bc1652aaf21da0c8336095048ada0948969cc PACKAGE_VALIDATION.txt -b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py -092e2959098b74ef38eb5d6f53262c669d067e0f554a0de43edf4d1d52e2ddab pyproject.toml -94fdc2d545b2d4077a4d70229fd22ba3ccac21d49d1097d2873f76d04e9a3c7a pytest.ini -a9e0b523ecbd503f6f75e9f3f79d28d66564ef0930548c1ad9d70b89772a240f README.md -568d480e5abd494e1364477c67b5c3b6af8c2a3662d015a50557e23b34e48020 README_JA.md -467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt -f5f4eb24800ef702e62bee6306c3f77bcf7a2c94d3d98ebad4674cdded3f3ec7 state.py -6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py -b5540af2a13af411630f8fc46ac4f484ac135988af417b3a902bcf3b94f6d7b7 temporal.py -3eeed61b7ee519631825e5f89e9a441644c66e552fce4207927cb8536867c8b1 tests/conftest.py 041289556cc621ea1857cbb8931ae41d817f69e084b2e8a7622e184a3197b61b tests/test_compatibility.py -e6fecb17e373333b315f0ce89e8dda3bda9dfdd34dac2656ae8785d26256c422 tests/test_layout_adapter.py -5ea3d7be506fa9ce04471f7306028e9feaf2faa41ba569bee3c99aa90c1f2a9a tests/test_model_patch.py -45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py -a9a4c10770a4a1b50db421d79d58f7aca8efa46232030ea9bd9495199f6e0cf8 tests/test_public_api.py -98a33c75b33ff0c10a68e04bd856e4e43b3e70b6ae1ea458a2b21b4e4d74f46d tests/test_state.py -d4c53d2c260b635d70fa211029d608f63776e0a56c20253475fa47c58363f8a7 tests/test_state_io.py -58c06925ccf06c4feee19b927b3515c540c0536aa5f3f8d29be35b8e919915a7 tests/test_temporal.py -912661d91c7f11f5fcaf0bd14d7d3ecf5c74c33b1507272f0ae72af6e5bffa6f tests/test_v2_decoder.py -07dff078ab3826da44e3147f22642d616eb82f472d558badcaaf20137eb7e47c tests/test_v2_prompts.py -3561cf0174317de812f6a0ef2dfe69cb4247ae12fa4efc2b5f412ab5282b261a tests/test_v2_registration.py -687fed96f73d848ad776d21139bfcce20abb359b7567d7b2c8d8af4ef8159174 tests/test_v2_seam.py -74e623120a963cb96b68651b44f47a836ea8449d30ce89d749c71ffad4766d4e tests/test_v2_seeds_motion.py -c55aa93ae8edb23441fe80b0673d045ac3538c9a8698cddbd970b1fec8452b68 tests/test_v2_sequence_orchestration.py -a6e200c21036521383a838070f2fd6e822e1928d06c0fab52b52cc01f64951bb tests/test_v2_session.py -58438c3ed814adf40cefde098a37cae00ddcb703b7e5548bbbb295e54bed5a6d tests/test_v2_widget_migration.py -9286c14fa84d581184ad65c42801180179a7c7efb8e6ef9ff6f536f9cb526af0 tests/test_v215_ui_serialization.py +06ae7327e491a324c7eec4d26b2ae053c96cfa489dc018c07e33eb63facde30c tests/test_temporal.py 0dbe8a5f71c92f3523942343b40787104c8b96781461ba3ba82934a6bdbe5834 THIRD_PARTY_NOTICES.md -dbf44a536c550e0cdffc01f8c3725b1b9af60dbeb16f45ff735f5d682276d389 tools/verify_runtime.py -ffd61f3f31c9b6430e1169007ee906c8f5000eba63ee44e2976ad4f5f1112671 v2/__init__.py -52d9a5a469b3419ed3ed4fc1eb13448e2969252732023650935bdefa8446c777 v2/decoder.py -c1da9e58e12cafea6ad380f7045abdc61a106657e2e191d3afe03f2844c670b3 v2/diagnostics.py -5543ec570f18f1d97327ffcf07f9eeaf078e7c7d3b3d722559d65a19d11dcb84 v2/h3_builder.py -aea01a48a8c03a25ed97ea05e5b137f89e388832d1be7d38f2da2be3d1b9830b v2/motion.py -5e6b694291d544b6a2f057a04cc29c534c9df6e8f6347d7b918c9b2c7592e946 v2/nodes.py -66058170e37eb3ac03f2428120f3b69ce7cff4789cd3fc4d00f991c61f6786d5 v2/prompts.py -4080f1875b958b316e8e76852c4be5527a6de5c71d604e7642d3098478eb4a36 v2/sampling.py -9261525777716ea71cc72ae06c1174acb9f7004016ee6f14fa3b89969f108e1b v2/seam_guard.py -d2b9020491e2a1e113cac72f8d129eacfc24c51507b53e9a0bf9f66da965bdad v2/seam_types.py +0e59ea821081cb93165aa3f88c6949736dd225837fe1cdcf1173aa5cd54c6d29 examples/v2_prompt_timeline_example.txt +11877692d4aadea79ff35d4d5035457c26ac2ace7443889ed3fd7ee51e99a103 continuation.py 12cfcc0c71ca087fe39a5ce2922959a62b2bb311bfe664f05c4480affce03708 v2/seeds.py -2f45074ab563f22289b65145a87fca66ffb25bfa4b88d19adad97260aea04e9f v2/sequence.py +13213e7709fd02bb810af3c18e4cd7d010f8cdfb121a4b99866a9f2333331089 v2/session_io.py +179fa62fcdac08439ae7fe0cd954a46b9bf03fd45418f503c3cb7c80ad1b9352 README.md +1d9ca37c408eaa73acc84e6c23fbae36723f57ad7ffcf0eea6d4150537c55e9a PACKAGE_INFO_JA.txt +23357af6f53e2a7283726d16c12043d8c01066167ecdbdbe74374942886b9816 constants.py +23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat +2854ff803998d70e08121813b225d8187cf693855d75d0af84ba8a52eaa5fce9 CHANGELOG.md +2c13aefd137db92a3dbf5576e9442d58012b1f0de1240c07d039ceeb702e7c1f docs/V2_1_4_UI_SIMPLIFICATION.md +34bb5b023c504e6ebcaaa52f37a5dd3e7ab592696848f38b44322095bff0cb84 tests/test_v2_registration.py +396fc6f02d6bc57758557ba0c84eb35ea067f8962ba633628c79dff975adad67 tests/test_model_patch.py +3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986 LICENSE +3a072882839c690b25006a318c99bedb2819ff75f7637c504541d7c110488acc nodes.py +3a359a0467725cc292ccb4c86cf0c3d30bb6a114d01f5345792b09ea3d45441d state.py +3d3f7cbad1070f14ad9b80f086eee5dfec4cf1e6b427287b457ae4793ca3b792 tests/test_v2_seam.py +3eeed61b7ee519631825e5f89e9a441644c66e552fce4207927cb8536867c8b1 tests/conftest.py +4080f1875b958b316e8e76852c4be5527a6de5c71d604e7642d3098478eb4a36 v2/sampling.py +43ec4023dc06c941e5d7a04db4b8a89fb58d744ceb986f6eb9bf534d4fdcb87b temporal.py +45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py +467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt +5543ec570f18f1d97327ffcf07f9eeaf078e7c7d3b3d722559d65a19d11dcb84 v2/h3_builder.py +58438c3ed814adf40cefde098a37cae00ddcb703b7e5548bbbb295e54bed5a6d tests/test_v2_widget_migration.py +5a26408c770381cdb259cb710de7dab12d7ddecca262bdbf35ba1566056050f5 examples/recommended_settings_v2.json +5fd282429f4c69f0678c9b7d64c26ebc06a2b1c64ef0bc7b3c6826420d4177ad PACKAGE_VALIDATION.txt +6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py +64a74d2e3d6e97cf31ad01bd73393e1f2ce90f3c81c26bda4ee86df6ecbd07a5 version.py +6771c9905d35666121c4892d2ec26202505f96d244ce814ad9a87a02878952cd examples/H3_CONTINUUM_V1_WIRING.md +68e1a724d3b99458b86b8f643dcb916183e7e8dd12d7b4b3850991feb8077694 examples/recommended_settings_v1.json +6b43294ebf00fde03acc3f2ac52f841f3c4323085c21643aa075a347cf4a432c layout_adapter.py +74e623120a963cb96b68651b44f47a836ea8449d30ce89d749c71ffad4766d4e tests/test_v2_seeds_motion.py +7660570840229b6fcbe62f0fbb52966d0099d64d7e6f01a739fbf9899293c8e4 docs/FUTURE_ROADMAP.md +7cbf23aee03b82f9081a81c77d66c25bcb1b4b562aebb2a9e7b9ccecfb51addc v2/seam_types.py +7dbbc8ce1018d6ac07c53eaa40f2d8d144babf9c6fe6bd94b7ec1d08adffa215 compatibility.py +7e27310301b13d3b8265b5e54359595ec0081ad55198435ae13932fdd2ad8164 v2/seam_guard.py +7f2dcb47992b5b3cfe32c9cf48b9e123daab505c04830656d7e8f70a0f54bc12 INSTALL_JA.txt +8127fdb840040ec85bb531226d3aa09f42b2b9cd44e93e761cc665d06f92b5af v2/prompts.py +85eada29c12706d55b78e4d8100899d1dadde4d9867406b8bac4eec22b81c6b6 .gitignore +873b2d42bef37e7d8c90616bef4cab2c29c0eb69393ba0bfd88d22e6aed4d382 pyproject.toml +9078617c14b1a7af17f6fa278cf3423c7b6f35373611600a3f52f268fa0cccae metadata.ini +912661d91c7f11f5fcaf0bd14d7d3ecf5c74c33b1507272f0ae72af6e5bffa6f tests/test_v2_decoder.py +94fda0afbc974125fa227757f0fdd2b22948c79761b271f481c404c2386afb5d .github/workflows/tests.yml +94fdc2d545b2d4077a4d70229fd22ba3ccac21d49d1097d2873f76d04e9a3c7a pytest.ini +98a33c75b33ff0c10a68e04bd856e4e43b3e70b6ae1ea458a2b21b4e4d74f46d tests/test_state.py +9946efb1ae119b877098526d5ed1d769e92561fd98e806dc36470d4315aead41 tests/test_v217_stable_facade.py +9bcbf75309c340e29e7e46a99f4084c329f40677953e8f2264070bd98948a927 v2/nodes.py +a032738b0bc5acb136e92d8bf7694643a5f518b9c7fc0fc878d66d81ba8b1b04 docs/V2_1_6_UNIFIED_UI.md +a6e200c21036521383a838070f2fd6e822e1928d06c0fab52b52cc01f64951bb tests/test_v2_session.py +a9a4c10770a4a1b50db421d79d58f7aca8efa46232030ea9bd9495199f6e0cf8 tests/test_public_api.py +aea01a48a8c03a25ed97ea05e5b137f89e388832d1be7d38f2da2be3d1b9830b v2/motion.py af89216e549f0a549340645a401b5dccc6aefb73e9f4cbc5e02285117ae46b3a v2/session.py -f3a94dfd2adbc97e2747941d5e50ab07b8b572e9f2684b9e3e827d78dfed86fa v2/session_io.py -f0a3910918986ddb01c816848027c5e462f57eb032ad618e64876b83f586f4fc version.py -9a0e87debfc8892fcc9a890d68877626d82f2debcfe89975c3f7233ffeab32c1 web/h3_continuum_v2.js +b08022df068576c759c8bf80ac6b6519693f3c65c917f1822aeb06ea359aa5b1 docs/V2_1_5_UI_SERIALIZATION_STABILITY.md +b3c1d1107706af437f8bca2e5053f54b6d357adc71417eeb6bb0074747e5e454 model_patch.py +b3de69a70bf0ccd291269651addca775d0295c30d53158b9eedd9818976d6ffe v2/sequence.py +b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py +b8275c8cecde12aee0418cea456d1e7f4b875cbb1bce32f1f3d9e06a44dd0be1 examples/v2_prompt_list_example.txt +c3cefcdf50df9d6609f583a008453b587c011a0fb41e290d5f1196ce62f58877 examples/H3_CONTINUUM_V2_WIRING_JA.md +c93a3e52e4deabe5ef0db7f34a7605856a543089574ba17227e697e9a2c0139a docs/ARCHITECTURE.md +cb473f21310c12c33007b218ad36eaccc0bfb2f1269f5d1036affc3e3f3027b5 tests/test_v2_sequence_orchestration.py +cde27db19b677e318cc4dfb4235be7a11429c19945f4083052a91fe117e74c82 v2/decoder.py +d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py +d406a9ce3446fae4f30810e8fea09d5b70c3bd027afbeecd67f007e8863998be README_JA.md +d4c53d2c260b635d70fa211029d608f63776e0a56c20253475fa47c58363f8a7 tests/test_state_io.py +dbf44a536c550e0cdffc01f8c3725b1b9af60dbeb16f45ff735f5d682276d389 tools/verify_runtime.py +dca65fb1393e67f59c6f81f9411e3e745eb8f1097e9a2adf7ffc6f4e3901ca9f __init__.py +e0c3d3b7b5d8ec5cd82b89e8a1e8ec9dd3364cabe0913abd8ac330e18352170e install_windows.ps1 +e30eaca61510ad5b224dacf2ee0797eabe5cc58594a0724dc2a6eaced8b34694 v2/diagnostics.py +e6fecb17e373333b315f0ce89e8dda3bda9dfdd34dac2656ae8785d26256c422 tests/test_layout_adapter.py +f0806bb2b999794ddda072e98424dbd1b613df1cd8a096b12d979f124e066d8a docs/VALIDATION_JA.md +f85ba3c69a473f0ac4bb96895c6d6532e8270121da65f858e364b19d31f05ef5 examples/H3_Continuum_V2_3x5s.json +fcf068c7a7e606b084fa8483b9b50506d05fa370b747114ac958240fd1a73757 tests/test_v2_prompts.py +ffd61f3f31c9b6430e1169007ee906c8f5000eba63ee44e2976ad4f5f1112671 v2/__init__.py diff --git a/PACKAGE_INFO_JA.txt b/PACKAGE_INFO_JA.txt index f833dae..f18c99f 100644 --- a/PACKAGE_INFO_JA.txt +++ b/PACKAGE_INFO_JA.txt @@ -1,33 +1,21 @@ 名称: ComfyUI-H3-Continuum -バージョン: 2.1.5 +バージョン: 2.1.7 対象: ComfyUI native MiniMax H3 -主ノード: H3 Continuum Sampler V2 +主ノード: H3 Continuum Sampler ライセンス: GPL-3.0-or-later -追加pip依存: なし(ComfyUI同梱のPyTorch / safetensorsを使用) +追加pip依存: なし -通常利用するV2ノード: -- H3 Continuum Sampler V2 +V2.1.7: +- 標準ComfyUI描画だけを使用する静的Facade UI +- Clip Overrides / Advanced / Resultをpack補助ノードへ分離 +- 動的socket、DOM非表示、proxy widget、手動serialization処理を廃止 +- 旧H3ContinuumSamplerV2をLegacy/Core互換ノードとして維持 +- State/Session/Prompt Plan schemaとSampling処理は変更なし -Legacy / 高度補助ノード(Deprecated): -- H3 Continuum Prompt Plan -- H3 Continuum Save Session -- H3 Continuum Load Session -- H3 Continuum Session Info -- H3 Continuum Join -- H3 Continuum Finish -- H3 Continuum Assemble -- H3 Continuum Save State -- H3 Continuum Load State - -設計: -- V1のnative video/audio latent continuation coreを再利用 -- Nチャンクを1ノードで統合管理 -- Sequence PromptでAuto / Fixed / List / Timelineを選択 -- 各チャンクでcall-local MODEL cloneを作成 -- Prompt/Identity conditioningをSampling前にキャッシュ -- H3チャンク間のVAE Decodeなし -- CPU Session、再開、指定チャンク以降のリロール -- 累積フレーム基準の音声sample境界 -- SageAttention / Sol-Attn / Spectrumは外部ノードとして併用 -- Spectrumのprivate APIには依存しない -- グローバルH3 Monkey Patchなし +Core: +- native H3 raw video/audio latent continuation +- 5/22/39f + Auto context +- Spectrum Interop API v1 Actual Prefix 2 request +- SageAttention / Sol-Attn / Spectrum外部併用 +- Session resume / reroll +- Seam Correction Auto / Off / Basic diff --git a/PACKAGE_VALIDATION.txt b/PACKAGE_VALIDATION.txt index f56baf1..e567c82 100644 --- a/PACKAGE_VALIDATION.txt +++ b/PACKAGE_VALIDATION.txt @@ -1,51 +1,17 @@ -ComfyUI-H3-Continuum 2.1.5 — Package Validation +ComfyUI-H3-Continuum 2.1.7 — Package Validation Date: 2026-08-10 -Release scope: -- Integrated H3 Continuum Sampler V2 for 1–16 native MiniMax H3 chunks. -- V1 Join / Finish / Assemble / State nodes preserved with unchanged node IDs. -- External SageAttention, Sol-Attn, LoRA, and Spectrum composition retained. +Build basis: +- GitHub repository: ukr8b3g-cmyk/ComfyUI-H3-Continuum +- Repository source starts from V2.1.5 HEAD 28b5b751a665fe47a026eb1c8222ab83ce859360 +- V2.1.7 adds a static Facade over the unchanged V2 execution core. -Completed checks: -- Python bytecode syntax compilation: PASS -- Python grammar compilation: PASS -- Unit and orchestration tests: PASS (71 tests) -- Package import and node registration: PASS (10 nodes) -- V1 node-ID compatibility: PASS -- V1 State schema compatibility: PASS -- Native H3 17k+5 temporal-grid tests: PASS -- 5 / 22 / 39-frame latent-context mapping: PASS -- Signed 40 Hz audio-grid offset tests: PASS -- PackedLayout in-place MM-RoPE patch tests: PASS -- Sol-Attn position_ids identity preservation: PASS -- Sol-Attn-style *args/**kwargs PackedLayout wrapper compatibility: PASS -- Genuine native-H3 API removal still fails closed: PASS -- Spectrum/public-wrapper marker detection: PASS -- V2 Fixed / List / Timeline prompt parsing: PASS -- V2 Sequence Prompt Auto detection and override priority: PASS -- Native Continuity call-local MODEL isolation and Spectrum Interop hint emission: PASS -- Basic Seam Correction and complete Off bypass: PASS -- V2.1.5 frontend serialization stability and Legacy-node registration checks: PASS -- Deterministic per-chunk seed and reroll derivation: PASS -- Conservative Auto-context selection: PASS -- Session validation, branch-safe reuse, and V1 State extraction: PASS -- Atomic safetensors + JSON State/Session round-trip: PASS -- Cumulative 24 fps / PCM sample-boundary alignment: PASS -- Exact-duration AV correction and final-anchor preservation tests: PASS -- Decode-after-all-sampling orchestration test: PASS -- Shared MODEL / shared conditioning-cache orchestration test: PASS -- V2 2.0.0 malformed widget-serialization migration regression: PASS -- Example V2 3x5s workflow structural validation: PASS (17 nodes, 20 links) -- Repository manifest regenerated after all V2.1.5 files: PASS +Automated validation for this V2.1.7 change: +- Not executed yet. +- Required commands: python -m compileall -q .; python -m pytest -q -Runtime verifier included: -- tools/verify_runtime.py checks the installed ComfyUI native H3 API. -- It constructs and patches a real native PackedLayout without loading the H3 checkpoint. -- It verifies all registered node IDs and the V2 prompt-plan core. - -Not performed in this build container: -- Full MiniMax H3 checkpoint generation on an RTX GPU. -- Perceptual A/B validation of Sage, Sol-Attn, and Spectrum combinations. -- Timing comparison against the equivalent V1 3x5-second workflow on the user's RTX 5060 Ti. - -The first-machine GPU validation matrix is documented in docs/VALIDATION_JA.md. +Still requires the user's ComfyUI browser/GPU runtime validation: +- New Facade and three pack helper nodes appear without Import Error +- Legacy H3ContinuumSamplerV2 workflows still load +- Workflow save/reload retains Facade and pack values/connections +- Full MiniMax H3 generation and Spectrum Prefix 2 log verification diff --git a/README.md b/README.md index d87d954..dc9c6f3 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,53 @@ -# ComfyUI-H3-Continuum 2.1.4 +# ComfyUI-H3-Continuum 2.1.7 -Integrated long-form continuation for ComfyUI's native MiniMax H3 audio/video model. +Long-form native MiniMax H3 audio/video continuation for ComfyUI. -**H3 Continuum Sampler V2** runs N native H3 chunks through the proven V1 continuation core, keeps text/image preparation outside the H3-only sampling loop, captures AV latent state before decoding, and decodes/assembles the sequence only after all sampling is complete. All V1 node identifiers remain available. +**H3 Continuum Sampler** generates 1–16 H3 chunks in one compact facade, carries the previous raw video/audio latent directly into the next chunk, stores accepted chunks on CPU, and defers VAE decode until sampling is complete. -## V2 highlights +## V2.1.7 Stable Facade UI -- One integrated N-chunk sampler node -- Native video/audio latent continuation with 5, 22, 39-frame and conservative Auto profiles -- One externally patched MODEL input with an isolated call-local clone per chunk -- Unique prompt conditioning cached before sampling -- One stable `Sequence Prompt` socket with Auto, Fixed, List, and Timeline parsing -- Optional `Clip N Prompt` overrides follow `chunks` from 1–16 and are collapsed by default -- Local latent-preview toggle under `Settings > H3 Continuum > Preview` -- Optional V2.1 Basic seam correction after deferred decode; Off keeps the V2.0.2 path -- CPU-only adaptive cut, bounded color/luma guard, short motion-aware blend, and boundary-local audio crossfade -- No Video/Audio VAE decode between H3 chunks -- Full chunk latents stored on CPU; VRAM does not grow with accepted chunks -- Session resume, branch-safe reroll, atomic safetensors + JSON persistence -- Cumulative 24fps audio boundaries, preventing per-chunk rounding drift -- Single-allocation final image assembly and preflight RAM safety check -- External SageAttention, Sol-Attn, and Spectrum composition -- No global ComfyUI H3 monkey patch and no accelerator private-API dependency -- No additional pip dependencies +V2.1.7 replaces the dynamic V2.1.6 display controller with a small static facade. It uses standard ComfyUI rendering and delegates generation to the fully compatible `H3ContinuumSamplerV2` execution core. -See [README_JA.md](README_JA.md) for the complete guide. +- `H3 Continuum Sampler` contains only the normal sequence controls and three optional pack inputs. +- `H3 Continuum Clip Overrides` supplies per-clip prompts through one static pack socket. +- `H3 Continuum Advanced` contains resume/session inputs and infrequent settings. +- `H3 Continuum Result` expands `last_state`, `session`, and `report` only when needed. +- The old `H3ContinuumSamplerV2` identifier remains registered as a Legacy/Core compatibility node. +- No dynamic sockets, proxy widgets, DOM hiding, MutationObserver, widget-size overrides, or `widgets_values` reconstruction. +- State, Session, Prompt Plan, Sampling, Native Continuity, and Spectrum Interop contracts are unchanged. + +## Core features + +- Native raw H3 video/audio latent continuation; no decode/re-encode between chunks. +- 5 / 22 / 39-frame context profiles plus conservative Auto. +- Native H3 17k+5 temporal grid and signed 40/24 audio-grid alignment. +- Call-local MODEL clone per chunk; Spectrum runtime/history does not leak across chunks. +- Spectrum Interop API v1 requests Actual Prefix 2 only when valid continuation context exists. +- `Sequence Prompt` with Auto / Fixed / List / Timeline parsing and per-clip overrides. +- Seam Correction: `Auto / Off / Basic`. +- Session resume and branch-safe reroll. +- Exact cumulative audio boundaries and exact total-duration correction. +- No additional pip dependencies beyond the normal ComfyUI environment. ## Recommended graph ```text -H3 UNET → SageAttention → Sol-Attn → LoRA → Spectrum - ↓ -CLIP / Video VAE / Audio VAE / Sampler / Sigmas - ↓ - H3 Continuum Sampler V2 - ↓ - IMAGE + AUDIO → CreateVideo +H3 UNET -> SageAttention -> Sol-Attn -> LoRA -> Spectrum + | +CLIP / Video VAE / Audio VAE / Sampler / Sigmas | + v + H3 Continuum Sampler + | | + IMAGE AUDIO ``` -## V2 nodes +Recommended first validation: `chunks=2`, `chunk_seconds=5`, `Balanced — 22 frames`, `Seam Correction=Off` for native-continuity A/B, then test Auto separately. -- H3 Continuum Sampler V2 -- H3 Continuum Prompt Plan -- H3 Continuum Save Session -- H3 Continuum Load Session -- H3 Continuum Session Info +## Install -V1 compatibility nodes remain registered: Join, Finish, Assemble, Save State, and Load State. +Copy the extracted `ComfyUI-H3-Continuum` folder into `ComfyUI/custom_nodes/` and restart ComfyUI, or run `install_windows.bat`. + +If replacing the older `ComfyUI-H3-Continuum-Join` folder, remove or rename the old folder so both packages are not loaded at once. The included installer backs up both old and new destination names. ## Development checks @@ -57,21 +58,4 @@ pytest -q ## License -GPL-3.0-or-later. This package was independently implemented against public ComfyUI H3 runtime contracts. Optional accelerator source code is not bundled. -## V2.1.4 Feature Summary - -- One integrated `H3 Continuum Sampler V2` node generates 1 to 16 sequential chunks. -- `Sequence Prompt` accepts one external Text (Multiline) input. -- Prompt formats: `Auto`, `Fixed`, `List`, and `Timeline`. -- Optional per-clip prompt overrides remain collapsed until needed. -- Native video and audio latents pass directly between chunks without VAE decode/re-encode. -- Continuity presets: `Fast - 5 frames`, `Balanced - 22 frames` (default), and `Strong - 39 frames`. -- Seam Correction provides `Off` and `Basic`; `Basic` is the default safe post-decode correction path. -- Exact total duration and audio continuity are enabled by default. -- SageAttention, Sol-Attn, LoRA, and Spectrum remain external and composable. -- Spectrum Interop API v1 requests a two-step Actual prefix only when continuation context exists. -- Advanced continuation, State, Session, Prompt Plan, reroll, and diagnostic controls are collapsed by default. -- Existing advanced connections automatically reopen the Advanced section. -- V1 and auxiliary nodes remain load-compatible but are marked deprecated and moved to the Legacy category. -- Existing State and Session schemas, output ordering, and saved workflow node identifiers are preserved. -- Repository: https://github.com/ukr8b3g-cmyk/ComfyUI-H3-Continuum +GPL-3.0-or-later. External accelerator source code is not bundled. diff --git a/README_JA.md b/README_JA.md index c733875..b7371f4 100644 --- a/README_JA.md +++ b/README_JA.md @@ -1,368 +1,76 @@ -# H3 Continuum Join 2.1 +# ComfyUI-H3-Continuum 2.1.7 -MiniMax H3を4〜15秒のチャンク単位で連続生成し、前チャンク末尾の**映像latentと音声latent**を次チャンクへ直接継承するComfyUIカスタムノードです。 +MiniMax H3を複数チャンクで連続生成し、直前チャンク末尾の**映像latent / 音声latentを直接**次チャンクへ継承するComfyUIカスタムノードです。チャンク間でVideo/Audio VAEのDecode→Encodeは行いません。 -V2の主要ノード **H3 Continuum Sampler V2** は、V1で実機確認されたContinuation処理を変更せず、Nチャンクの準備・Sampling・State管理・Decode・結合を1ノードへ統合します。V1の5ノードもそのまま残るため、既存ワークフローは継続利用できます。 +## V2.1.7 Stable Facade UI -## V2の主な特徴 +V2.1.7では、V2.1.6の動的表示制御を廃止し、標準ComfyUI描画だけを使用する静的Facadeへ移行しました。生成処理は互換用`H3ContinuumSamplerV2`を再利用し、State、Session、Prompt Plan、Spectrum Interop、保存Schemaは変更していません。 -- `chunks`を指定するだけでNチャンクを順番に生成 -- V1と同じnative H3 video/audio latent continuation coreを使用 -- 同一MODEL、Sampler、Sigmasを全チャンクで共有 -- CLIP conditioningと入力画像latentをSampling前にまとめて準備 -- H3チャンク間でVideo/Audio VAE Decodeを行わず、全Sampling終了後にDecode -- 全チャンクのraw latentをCPU RAMへ退避し、VRAMに蓄積しない -- Spectrumは1ノードのみ。各チャンクでSpectrum自身のrunを開始・終了 -- SageAttention、Sol-Attn、Spectrumのprivate APIを内蔵・複製しない -- 固定`Sequence Prompt`端子とAuto / Fixed / List / Timeline解析 -- `chunks`に連動する1〜16個の`Clip N Prompt`は折りたたみ式の任意Override -- `Settings > H3 Continuum > Preview`で生成中latentサムネイルをON/OFF -- `Seam Correction: Off / Basic`。Offは2.0.2経路、BasicはDecode後だけの境界補正 -- Basicでは最大3fのAdaptive Cut、制限付き色・輝度補正、最大2f Blend、境界限定Audio補正 -- 同じPromptのQwen conditioningを1回だけ作成して再利用 -- 5 / 22 / 39フレーム、および保守的なAuto Context -- 64bitの決定的なチャンク別Seed -- Session保存、再開、指定チャンク以降の安全なリロール -- Decode時に最終IMAGEを1回だけ確保し、反復`torch.cat`を回避 -- 音声境界を累積フレーム数から計算し、チャンクごとの丸め誤差を蓄積しない -- `chunks × chunk_seconds × 24fps`への最終尺補正 -- 大きすぎる出力はSampling前のRAM見積りで安全停止 -- 追加pip依存なし - -## 対応環境 - -- ComfyUI v0.31.0以降のnative MiniMax H3 -- MiniMax H3 FL2VA / Image-to-Videoモデル -- Python 3.10以降 -- batch size 1 -- Windows / Linux - -本パッケージはComfyUI本体、Spectrum、Sol-Attn、SageAttentionのソースコードを同梱しません。各アクセラレータは別途導入してください。 - -## インストール - -1. ComfyUIを終了します。 -2. ZIPを展開します。 -3. フォルダー全体を次へ配置します。 +通常表示は次を中心にします。 ```text -D:\StabilityMatrix\Data\Packages\ComfyUI_W\custom_nodes\ComfyUI-H3-Continuum-Join\ +model / clip / video_vae / audio_vae / sampler / sigmas +Sequence Prompt +first_frame +prompt_overrides(任意pack) +advanced(任意pack) + +Prompt Format +chunks +chunk_seconds +width / height +continuity +base_seed +control after generate +Seam Correction +Seam Correction + +outputs: images / audio / result ``` -または展開後に`install_windows.bat`を実行します。インストーラーは既存フォルダーを日時付きバックアップへ移し、native H3のAPI、PackedLayout、ノード登録を検査します。 +### 補助ノード -```powershell -powershell -ExecutionPolicy Bypass -File .\install_windows.ps1 -``` +- `H3 Continuum Clip Overrides`: Clip別Promptを1本のpackへまとめます。 +- `H3 Continuum Advanced`: resume/session入力と低頻度設定をまとめます。 +- `H3 Continuum Result`: resultから`last_state / session / report`を展開します。 +- 補助ノードを使わない通常生成では、Sampler本体だけで動作します。 -導入後、ComfyUIを再起動してください。 +### Legacy互換 -## V2の最小ワークフロー - -```text -H3 UNET Loader - → SageAttention - → Sol-Attn - → 通常LoRA(必要時) - → Spectrum Apply MiniMax H3 - → H3 Continuum Sampler V2 - -CLIP Loader ───────────────→ H3 Continuum Sampler V2 -Video VAE Loader ──────────→ H3 Continuum Sampler V2 -Audio VAE Loader ──────────→ H3 Continuum Sampler V2 -KSampler Select ───────────→ H3 Continuum Sampler V2 -Basic Scheduler ───────────→ H3 Continuum Sampler V2 -Load Image / scaled IMAGE ─→ first_frame - -H3 Continuum Sampler V2 - ├─ images ─→ CreateVideo - ├─ audio ─→ CreateVideo - ├─ session ─→ Save Session(任意) - └─ report -``` - -MODELへの推奨順は次です。 - -```text -UNET → Sage → Sol → LoRA → Spectrum → H3 Continuum Sampler V2 -``` - -ContinuumはSampler内部でMODELを1回だけ追加cloneし、MODEL単位の`APPLY_MODEL` wrapperを付加します。グローバルなH3クラス変更は行いません。 - -## H3 Continuum Sampler V2 - -### 必須入力 - -- `model`:Sage / Sol / LoRA / Spectrumを適用済みのH3 MODEL -- `clip`:MiniMax用Qwen3-VL CLIP -- `video_vae`:H3 Video VAE -- `audio_vae`:H3 Audio VAE -- `sampler`:標準は`res_multistep` -- `sigmas`:標準は`simple / 20 steps / denoise 1.0` -- `Sequence Prompt`:標準`Text (Multiline)`を1個接続する固定STRING入力 -- `Prompt Format`:Auto / Fixed / List / Timeline。形式変更時も接続を維持 -- `Clip N Prompt`:折りたたみ内の任意Override。接続済み旧端子は折りたたんでも維持 -- `prompt_script`:画面上では非表示。旧Workflow用の内部フォールバックとして保持 -- `chunks`:1〜16 -- `chunk_seconds`:4.0〜15.0。標準5.0 -- `width` / `height`:32の倍数 -- `continuity` -- `base_seed` -- `audio_continuity` -- `exact_total_duration` -- `diagnostics` -- `reroll_from_chunk` / `reroll_nonce` -- `strict_compatibility` -- `seam_correction`:`Off`は2.0.2互換、`Basic`はV2.1の非破壊境界補正 - -### 任意入力 - -- `first_frame`:全チャンクの長期Identity cue。画像ロードは1回だけでよい -- `last_frame`:最終チャンク末尾のFL2VA anchor -- `session`:保存済みV2 Sessionの再開・リロール -- `initial_state`:V1の`H3_CONTINUUM_STATE`からV2を開始 -- `prompt_plan`:`H3 Continuum Prompt Plan`出力を接続可能 - -`session`と`initial_state`は同時接続できません。 - -Promptの優先順位は次のとおりです。 - -```text -外部 Clip N Prompt Override -→ Sequence Prompt -→ prompt_plan -→ 保存済み prompt_mode / prompt_script -``` - -Clip別Overrideがないチャンクは、`Sequence Prompt`、接続済み`prompt_plan`、内部フォールバックの順に対応Promptを維持します。生成中サムネイルは`Settings > H3 Continuum > Preview > Show latent preview`で切り替えます。初期値はONで、OFFにしても最終IMAGE / AUDIOには影響しません。 - -## Promptモード - -### Fixed — one prompt - -同一Promptを全チャンクで使用します。Qwen conditioningは1回だけ作成されます。 - -```text -The same woman continues singing in one uninterrupted shot... -``` - -### List — split with --- - -```text -Chunk 1 prompt... ---- -Chunk 2 prompt... ---- -Chunk 3 prompt... -``` - -Prompt数が不足すると最後のPromptを繰り返し、余分なPromptは無視してReportへ記録します。JSON文字列配列も使用できます。 - -### Timeline — [0-5s] sections - -```text -[0-5s] -First section prompt... - -[5-10s] -Second section prompt... - -[10-15s] -Third section prompt... -``` - -`[Chunk 1]` / `[Clip 2]`形式も使えます。時間範囲がチャンクをまたぐ場合は、重なり時間が最大のセクションを採用します。 +旧`H3ContinuumSamplerV2`は削除せずLegacy/Coreノードとして登録を維持します。既存WorkflowのノードID、入力順、出力順は変更していません。 ## Continuity -- `Auto — conservative`:latent motion量を測定。境界付近は22fへ戻す -- `Balanced — 22 frames`:V1標準。最初の実機検証に推奨 -- `Fast — 5 frames`:静かな動き、速度優先 -- `Strong — 39 frames (Experimental)`:激しい動き・カメラ、計算量増加 +- `Auto — conservative` +- `Balanced — 22 frames`(標準) +- `Fast — 5 frames` +- `Strong — 39 frames (Experimental)` -Autoは前Stateのvideo latent時間差を正規化して判定します。判定不能、State容量不足、しきい値付近では安全側の22fを選びます。39fは実験的です。 +5/22/39フレームはH3の時間latent周期`1,4,4,4,4`に対して、それぞれ2/7/12 temporal latent stepsです。 -## IdentityとMotionの分離 +## Spectrum -V2は次を別に扱います。 +Continuumは有効な前チャンクContextがある場合だけ`h3_continuum` Interop API v1をMODEL optionsへ付与し、対応Spectrumへ`min_actual_prefix_steps=2`を要求します。初回チャンクにはInterop keyを付与しません。未知・未対応Spectrumでは通常Spectrumとしてfail-openします。 + +## 推奨配線 ```text -Long-term Identity - = first_frameをQwen visual tokenとして全チャンクで共有 - -Short-term Motion / Audio - = 直前チャンク末尾の5 / 22 / 39f video latent - + 対応するaudio latent +UNET -> SageAttention -> Sol-Attn -> LoRA -> Spectrum -> H3 Continuum Sampler ``` -継続チャンクでは旧first-frameの空間keyframeを外しますが、Qwen visual tokenは人物・衣装・背景のIdentity cueとして残します。前チャンクの姿勢、移動方向、口の動き、カメラ運動、音声はnative latent Contextから継続します。 +まず`chunks=2 / chunk_seconds=5 / Balanced 22f / Seam Correction=Off`でNative Continuityを確認し、その後`Seam Correction=Auto`を比較してください。 -## Sessionとリロール +## インストール -V2出力の`session`には、各チャンクのraw AV latent、Prompt hash、Seed、Plan、Context情報をCPUで保持します。 +ZIPを展開して、`ComfyUI-H3-Continuum`フォルダーを`ComfyUI/custom_nodes/`へ置き、ComfyUIを再起動します。 + +旧`ComfyUI-H3-Continuum-Join`が残っている場合は同時ロードを避けるため削除または退避してください。同梱`install_windows.bat`は旧名・新名の既存フォルダーを日時付きでバックアップします。 + +## 検査 ```text -H3 Continuum Sampler V2.session - → H3 Continuum Save Session -``` - -保存形式: - -```text -h3_continuum_session_slot0001.safetensors -h3_continuum_session_slot0001.json -``` - -### 再開 - -保存済みSessionをLoadし、Samplerの`session`へ接続します。Prompt、解像度、first_frame fingerprintが一致する採用済みチャンクは再生成しません。追加したチャンクだけ生成します。 - -### リロール - -```text -reroll_from_chunk = 3 -reroll_nonce = 1 -``` - -この場合、Chunk 1〜2を保持し、Chunk 3以降を新しい決定的Seedで生成します。途中チャンクを変更すると、それ以降は旧Continuationと一致しないため自動的に新しいSession branchになります。元Sessionは上書きされません。 - -`reroll_from_chunk=0`は、接続Sessionの一致する採用済みチャンクをすべて再利用します。 - -## SamplingとDecodeの実行順 - -```text -Phase 1 - first/last image resize・VAE encode - unique PromptのQwen encode - -Phase 2 - H3 Chunk 1 Sampling → CPU latent / State - H3 Chunk 2 Sampling → CPU latent / State - H3 Chunk 3 Sampling → CPU latent / State - ※この間Video/Audio Decodeなし - -Phase 3 - Video/Audio VAE Decode - overlap Trim - 累積sample境界で音声を配置 - final IMAGE/AUDIOを組立 -``` - -ComfyUIの動的VRAM管理は引き続き有効です。RTX 5060 Ti 16GBでH3全重みを完全VRAM常駐させるという意味ではありません。V2は**同じMODEL objectを使い、H3チャンク間にCLIP/VAE処理を挟まない**ことで、不要なモデル切替を抑えます。 - -## Spectrum互換 - -SpectrumはV2の外部MODELノードとして1回だけ適用します。V2はSpectrumのprivate APIを呼びません。 - -各チャンクの`guider.sample()`がSpectrumの通常wrapperを通るため、Spectrumはチャンクごとに独立したrunを開始・終了します。前チャンクのSpectrum forecast historyは次チャンクへ流用されません。継承するのはnative video/audio Contextだけです。 - -推奨初期値: - -```text -blend_weight: 0.50 -degree: 1 -warmup_steps: 1 -offline_smoothing_replay: true -audio_blend_weight: 0.00 -``` - -`offline_smoothing_replay=true`ではSpectrumが1チャンクにつきcapture/replayの2 passを行います。V2はこの動作を変更しません。 - -## SageAttention / Sol-Attn互換 - -- SageAttention:Attention kernelを変更せず、そのまま使用 -- Sol-Attn:Continuumは`PackedLayout.position_ids` Tensorを置換せずin-place補正 -- SolからSageへのfallback chainを維持 -- Contextはnative H3 `video` / `video_audio` reference blockとして渡す - -本ノードからSolのtau、sink、Morton設定は変更しません。GPU、解像度、内容ごとにA/B比較してください。 - -## Audio / exact duration - -V2は各セグメントを個別に丸めて連結しません。累積フレーム境界から、次を直接計算します。 - -```text -sample_start = round(cumulative_frames / 24 × sample_rate) -sample_stop = round(next_cumulative_frames / 24 × sample_rate) -``` - -そのため119フレーム区間を多数つないでも、1sample単位の丸め誤差が累積しません。 - -`exact_total_duration=true`では、最終結果を次へ合わせます。 - -```text -target_frames = round(chunks × chunk_seconds × 24) -``` - -余分なフレームは通常末尾からTrimします。`last_frame`が接続されている場合は最終アンカーを失わないよう、余剰を最終フレーム直前から除去し、音声側も短い補間でクリックを抑えながら同じ長さへ合わせます。不足が小さい場合のみ最終フレームと最終音声sampleを最小限反復します。 - -## RAM安全設計 - -長尺の最終`IMAGE`はCPU float tensorになるため、解像度と長さに比例してRAMを消費します。V2はSampling開始前に、 - -- 最終IMAGE -- 1チャンク分のraw Decode -- 基本headroom - -を見積もります。現在利用可能RAMの安全範囲を超える場合は、すべてのH3 Samplingを終えた後で失敗しないよう事前停止します。 - -Decode時は最終IMAGE tensorを1回だけ確保し、各チャンクを所定範囲へcopyします。反復`torch.cat`による全長コピーは行いません。 - -## Diagnostics - -- `Off`:診断を最小化 -- `Basic`:標準。Seed、Context、motion score、アクセラレータ検出、尺、RAM見積り -- `Full`:Basicに加え、接続部のvideo MAE/PSNR、audio correlation、boundary jump - -FullはDecode後の測定であり、外部の顔認識モデルやoptical-flowモデルには依存しません。 - -## V1互換ノード - -次のV1ノードは2.0でも同じIDで残ります。 - -- H3 Continuum Join -- H3 Continuum Finish -- H3 Continuum Assemble -- H3 Continuum Save State -- H3 Continuum Load State - -V1の`H3_CONTINUUM_STATE`はV2の`initial_state`へ直接接続できます。V2 Sessionからは`H3 Continuum Session Info`で最終V1 Stateを取得できます。 - -## 制限 - -- SVI専用学習済みモデルではなく、native H3 conditioningを利用した独自Continuationです -- H3モデル自体の再生成による長期Identity・音色のドリフトを完全には排除できません -- 39fとAutoは内容ごとの実機比較が必要です -- 解像度をまたぐState/Session継続は安全停止します -- Spectrumは近似高速化です。同一SeedでON/OFF比較してください -- 本配布環境ではH3 checkpointを使ったRTX実生成は実施できません。実機検証手順は`docs/VALIDATION_JA.md`を参照してください - -## 同梱ノード - -### V2 - -- H3 Continuum Sampler V2 -- H3 Continuum Prompt Plan -- H3 Continuum Save Session -- H3 Continuum Load Session -- H3 Continuum Session Info - -### V1互換 - -- H3 Continuum Join -- H3 Continuum Finish -- H3 Continuum Assemble -- H3 Continuum Save State -- H3 Continuum Load State - -## 開発・検査 - -```bash python -m compileall -q . pytest -q -python tools/verify_runtime.py --comfy-root D:\StabilityMatrix\Data\Packages\ComfyUI_W ``` -## ライセンス - GPL-3.0-or-later。 diff --git a/constants.py b/constants.py index a0881e8..907bb08 100644 --- a/constants.py +++ b/constants.py @@ -44,7 +44,6 @@ CONTINUITY_FRAMES = { } DEFAULT_STATE_CAPACITY_FRAMES = 39 - V2_CONTINUITY_AUTO = "Auto — conservative" V2_CONTINUITY_OPTIONS = (V2_CONTINUITY_AUTO,) + CONTINUITY_OPTIONS diff --git a/continuation.py b/continuation.py index a4718d4..9eebcee 100644 --- a/continuation.py +++ b/continuation.py @@ -27,7 +27,6 @@ FIRST_FRAME_POLICIES = (POLICY_REPLACE, POLICY_ERROR, POLICY_KEEP) def clone_conditioning(conditioning): """Clone the outer conditioning list and metadata dictionaries.""" - output = [] for index, item in enumerate(conditioning): if not isinstance(item, (tuple, list)) or len(item) != 2: @@ -42,7 +41,6 @@ def clone_conditioning(conditioning): def conditioning_diagnostics(conditioning) -> dict[str, int]: """Return non-authoritative cues used in the Join report.""" - first_keyframes = 0 visual_tokens = 0 for item in conditioning: @@ -69,13 +67,7 @@ def prepare_conditioning( first_frame_policy: str, preserve_last_frame: bool, ): - """Add one native H3 video/video-audio reference as timeline context. - - A single multi-slot reference keeps the previous motion latent intact and - avoids relying on ComfyUI's first/last-only keyframe constructor. The - per-model wrapper moves the reference's MM-RoPE rows onto target time zero. - """ - + """Add one native H3 video/video-audio reference as timeline context.""" if first_frame_policy not in FIRST_FRAME_POLICIES: raise ValueError(f"unknown first-frame policy: {first_frame_policy!r}") if not torch.is_tensor(video_context) or video_context.ndim != 5: @@ -111,7 +103,6 @@ def prepare_conditioning( "audio_latent": audio_context.contiguous(), MARK_AUDIO_CONTEXT: True, MARK_AUDIO_END_FRAME: float(context_frames), - # Schema-v1 name retained; value is a signed grid offset. MARK_AUDIO_OVERHANG: float(audio_grid_offset), } ) @@ -131,10 +122,7 @@ def prepare_conditioning( ) if first_frame_policy == POLICY_KEEP: kept.append(keyframe) - # POLICY_REPLACE drops the spatial keyframe. Qwen visual tokens, - # if present, remain in conditioning as a global identity cue. continue - is_last = old_frame_count is not None and frame_index == int(old_frame_count) - 1 if is_last and preserve_last_frame: keyframe["resolved_frame_index"] = int(new_frame_count) - 1 @@ -145,13 +133,11 @@ def prepare_conditioning( raise ValueError( f"unsupported existing H3 keyframe index {frame_index}; only first/last are safe" ) - if kept: metadata["minimax_keyframes"] = kept else: metadata.pop("minimax_keyframes", None) metadata["minimax_frame_count"] = int(new_frame_count) - refs = [ dict(item) for item in (metadata.get("minimax_refs") or []) @@ -164,11 +150,10 @@ def prepare_conditioning( def new_h3_latent(template: dict[str, Any], *, video_t: int, audio_t: int): """Create an empty native H3 AV latent matching the template geometry/dtype.""" - try: import comfy.model_management import comfy.nested_tensor - except Exception as exc: # pragma: no cover - requires ComfyUI runtime + except Exception as exc: # pragma: no cover raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc video, audio = extract_av_streams(template) device = comfy.model_management.intermediate_device() @@ -177,7 +162,5 @@ def new_h3_latent(template: dict[str, Any], *, video_t: int, audio_t: int): device=device, dtype=video.dtype, ) - new_audio = torch.zeros( - (1, 32, 2, int(audio_t)), device=device, dtype=audio.dtype - ) + new_audio = torch.zeros((1, 32, 2, int(audio_t)), device=device, dtype=audio.dtype) return {"samples": comfy.nested_tensor.NestedTensor((new_video, new_audio))} diff --git a/docs/V2_1_6_UNIFIED_UI.md b/docs/V2_1_6_UNIFIED_UI.md new file mode 100644 index 0000000..c145a28 --- /dev/null +++ b/docs/V2_1_6_UNIFIED_UI.md @@ -0,0 +1,37 @@ +# V2.1.6 Unified UI + +V2.1.6 replaces the layered V2.1.4/2.1.5 display patches with one frontend +controller in `web/h3_continuum_v2.js`. + +## Contract + +- Sampling, Native Continuity, State, Session, Prompt Plan and Spectrum Interop + semantics are unchanged. +- Backend inputs, outputs and serialized widget order remain registered in the + same order for saved-workflow compatibility. +- The frontend never calls `removeInput`, `addInput`, `removeOutput`, + `addOutput`, `node.widgets.splice`, or fixed-height `node.setSize`. +- Clip Prompt sockets are selected by socket name. For `chunks=N`, unconnected + Clip 1..N sockets are visible; unconnected sockets above N are hidden. +- A connected Clip Prompt remains visible after reducing `chunks` so hidden + links are not created. +- `Individual Clip Overrides` is removed; the visible sockets themselves are + the override UI. +- Advanced inputs/outputs remain registered. They are shown while Advanced is + open, and connected Advanced sockets remain visible while it is closed. +- Advanced scalar widgets are display-only folded; their values and serialized + positions are not rewritten. +- Node height is recalculated from current content; no fixed height is imposed. +- Core visibility works without DOM selectors in both classic LiteGraph canvas and Vue nodes. +- Vue DOM handling is only a fallback for advanced output rows. + +## Browser regression + +Test at minimum: + +1. Set `chunks=3`, then 5, then 2. +2. Connect a prompt to Clip 5 before reducing to 2; Clip 5 must remain visible. +3. Open/close Advanced with both unconnected and connected Advanced sockets. +4. Switch workflow tabs. +5. Save, reload ComfyUI, and reload the workflow. +6. Confirm widget values, socket indices, and links remain unchanged. diff --git a/examples/H3_CONTINUUM_V2_WIRING_JA.md b/examples/H3_CONTINUUM_V2_WIRING_JA.md index f263431..70cfc4e 100644 --- a/examples/H3_CONTINUUM_V2_WIRING_JA.md +++ b/examples/H3_CONTINUUM_V2_WIRING_JA.md @@ -36,8 +36,7 @@ Sampler V2.session ─→ Save Session(任意) ## 3×5秒の初期値 ```text -Prompt Format: Auto -Sequence Prompt: Text (Multiline)を1個接続 +prompt_mode: Fixed — one prompt chunks: 3 chunk_seconds: 5.0 continuity: Balanced — 22 frames diff --git a/install_windows.bat b/install_windows.bat index ae5c461..82a65a9 100644 --- a/install_windows.bat +++ b/install_windows.bat @@ -2,9 +2,8 @@ setlocal set "ROOT=%~1" if "%ROOT%"=="" set "ROOT=D:\StabilityMatrix\Data\Packages\ComfyUI_W" -powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0install_windows.ps1" -ComfyUIRoot "%ROOT%" +powershell -ExecutionPolicy Bypass -File "%~dp0install_windows.ps1" -ComfyUIRoot "%ROOT%" if errorlevel 1 ( - echo. echo Installation failed. pause exit /b 1 diff --git a/install_windows.ps1 b/install_windows.ps1 index 704c7bd..ac47ae1 100644 --- a/install_windows.ps1 +++ b/install_windows.ps1 @@ -5,7 +5,8 @@ param( $ErrorActionPreference = "Stop" $source = (Resolve-Path $PSScriptRoot).Path $customNodes = Join-Path $ComfyUIRoot "custom_nodes" -$destination = Join-Path $customNodes "ComfyUI-H3-Continuum-Join" +$destination = Join-Path $customNodes "ComfyUI-H3-Continuum" +$legacyDestination = Join-Path $customNodes "ComfyUI-H3-Continuum-Join" if (-not (Test-Path $ComfyUIRoot -PathType Container)) { throw "ComfyUI root not found: $ComfyUIRoot" @@ -20,10 +21,13 @@ if ([System.StringComparer]::OrdinalIgnoreCase.Equals($source, $destination)) { exit 0 } -if (Test-Path $destination) { - $backup = "$destination.backup-$(Get-Date -Format 'yyyyMMdd-HHmmss')" - Move-Item -LiteralPath $destination -Destination $backup - Write-Host "Existing installation moved to: $backup" +$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +foreach ($old in @($destination, $legacyDestination)) { + if ((Test-Path $old) -and -not [System.StringComparer]::OrdinalIgnoreCase.Equals($source, $old)) { + $backup = "$old.backup-$stamp" + Move-Item -LiteralPath $old -Destination $backup + Write-Host "Existing installation moved to: $backup" + } } New-Item -ItemType Directory -Force -Path $destination | Out-Null @@ -42,5 +46,5 @@ if (Test-Path $python) { Write-Warning "venv Python was not found; installation was copied but runtime verification was skipped." } -Write-Host "Installed H3 Continuum Join 2.0 to: $destination" -Write-Host "Restart ComfyUI." +Write-Host "Installed H3 Continuum 2.1.6 to: $destination" +Write-Host "Restart ComfyUI. If the browser cached the old frontend, use a hard reload." diff --git a/layout_adapter.py b/layout_adapter.py index e6f76dc..6f5a600 100644 --- a/layout_adapter.py +++ b/layout_adapter.py @@ -9,558 +9,218 @@ The preferred Continuum representation is one native H3 ``video`` or of the target timeline. The older marked-keyframe form remains supported as an internal compatibility path, but the UI does not emit it. """ - from __future__ import annotations - import logging from typing import Any - import torch - from .constants import ( - CONTINUUM_INTEROP_API, - CONTINUUM_REFERENCE_METADATA_KEY, - FRAME_RESCALE, - LAYOUT_DEVICE_CACHE_ATTR, - LAYOUT_ORIGINAL_TIME_ATTR, - LAYOUT_SIGNATURE_ATTR, - MARK_AUDIO_CONTEXT, - MARK_AUDIO_END_FRAME, - MARK_AUDIO_OVERHANG, - MARK_CONTEXT_FRAMES, - MARK_VIDEO_CONTEXT, - MARK_VIDEO_SLOT, + CONTINUUM_INTEROP_API, CONTINUUM_REFERENCE_METADATA_KEY, FRAME_RESCALE, + LAYOUT_DEVICE_CACHE_ATTR, LAYOUT_ORIGINAL_TIME_ATTR, LAYOUT_SIGNATURE_ATTR, + MARK_AUDIO_CONTEXT, MARK_AUDIO_END_FRAME, MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, MARK_VIDEO_CONTEXT, MARK_VIDEO_SLOT, ) - LOG = logging.getLogger("h3_continuum_join") +class LayoutCompatibilityError(RuntimeError): pass - -class LayoutCompatibilityError(RuntimeError): - pass - - -def _reference_metadata(ref: dict[str, Any]) -> dict[str, Any]: - value = ref.get(CONTINUUM_REFERENCE_METADATA_KEY) - return value if isinstance(value, dict) else {} - - -def _is_video_context(ref: dict[str, Any]) -> bool: - return bool( - ref.get(MARK_VIDEO_CONTEXT) - or _reference_metadata(ref).get("role") == "video_context" - ) - - -def _is_audio_context(ref: dict[str, Any]) -> bool: - metadata = _reference_metadata(ref) - return bool( - ref.get(MARK_AUDIO_CONTEXT) - or metadata.get("role") == "audio_context" - or metadata.get("audio_role") == "audio_context" - ) - - -def payload_has_continuum(payload: Any) -> bool: - if not isinstance(payload, dict): - return False - keyframes = payload.get("keyframes") or () - refs = payload.get("refs") or () - return any(MARK_VIDEO_SLOT in item for item in keyframes) or any( - _is_video_context(item) or _is_audio_context(item) for item in refs - ) - - -def normalize_condition_latents(payload: dict[str, Any]) -> None: - """Keep keyframe rows first and reference rows second, matching PackedLayout.""" - keyframes = list(payload.get("keyframes") or ()) - refs = list(payload.get("refs") or ()) - video_latents = [item["latent"] for item in keyframes if item.get("latent") is not None] +def _reference_metadata(ref:dict[str,Any])->dict[str,Any]: + value=ref.get(CONTINUUM_REFERENCE_METADATA_KEY); return value if isinstance(value,dict) else {} +def _is_video_context(ref): + return bool(ref.get(MARK_VIDEO_CONTEXT) or _reference_metadata(ref).get("role")=="video_context") +def _is_audio_context(ref): + metadata=_reference_metadata(ref) + return bool(ref.get(MARK_AUDIO_CONTEXT) or metadata.get("role")=="audio_context" or metadata.get("audio_role")=="audio_context") +def payload_has_continuum(payload): + if not isinstance(payload,dict): return False + return any(MARK_VIDEO_SLOT in item for item in (payload.get("keyframes") or ())) or any(_is_video_context(item) or _is_audio_context(item) for item in (payload.get("refs") or ())) +def normalize_condition_latents(payload): + keyframes=list(payload.get("keyframes") or ()); refs=list(payload.get("refs") or ()) + video_latents=[item["latent"] for item in keyframes if item.get("latent") is not None] video_latents.extend(item["latent"] for item in refs if item.get("latent") is not None) - audio_latents = [ - item["audio_latent"] for item in refs if item.get("audio_latent") is not None - ] - payload["cond_video_latents"] = video_latents - payload["cond_audio_latents"] = audio_latents - - -def _input_device(x: Any) -> torch.device | None: - device = getattr(x, "device", None) - if isinstance(device, torch.device): - return device - if hasattr(x, "unbind"): + payload["cond_video_latents"]=video_latents + payload["cond_audio_latents"]=[item["audio_latent"] for item in refs if item.get("audio_latent") is not None] +def _input_device(x): + device=getattr(x,"device",None) + if isinstance(device,torch.device): return device + if hasattr(x,"unbind"): try: - parts = x.unbind() + parts=x.unbind() if parts: - part_device = getattr(parts[0], "device", None) - if isinstance(part_device, torch.device): - return part_device - except Exception: - return None + d=getattr(parts[0],"device",None) + if isinstance(d,torch.device): return d + except Exception: return None return None - - -def materialize_continuum_latents( - payload: dict[str, Any], x: Any, *, debug: bool = False, -) -> None: - """Move only Continuum context tensors to the active device once per run. - - Native H3 intentionally keeps payload dictionaries outside BaseModel's - generic dtype/device conversion. Without this cache, a CPU continuation - state would be transferred again on every solver step. Existing user refs - are left untouched to avoid unexpectedly pinning large Ref2VA inputs in - VRAM. - """ - layout = payload.get("layout") - device = _input_device(x) - if layout is None or device is None or device.type == "cpu": - return - - keyframes = [dict(item) for item in (payload.get("keyframes") or ())] - refs = [dict(item) for item in (payload.get("refs") or ())] - marked_tensors: list[tuple[str, int, torch.Tensor]] = [] - for index, item in enumerate(keyframes): - if MARK_VIDEO_SLOT in item and torch.is_tensor(item.get("latent")): - marked_tensors.append(("keyframe", index, item["latent"])) - for index, item in enumerate(refs): - if _is_video_context(item) and torch.is_tensor(item.get("latent")): - marked_tensors.append(("ref_video", index, item["latent"])) - if _is_audio_context(item) and torch.is_tensor(item.get("audio_latent")): - marked_tensors.append(("ref_audio", index, item["audio_latent"])) - if not marked_tensors: - return - - cache_key = ( - str(device), - tuple( - (kind, index, id(tensor), tuple(tensor.shape), str(tensor.dtype)) - for kind, index, tensor in marked_tensors - ), - ) - cached = getattr(layout, LAYOUT_DEVICE_CACHE_ATTR, None) - if not isinstance(cached, dict) or cached.get("key") != cache_key: - values: dict[tuple[str, int], torch.Tensor] = {} - for kind, index, tensor in marked_tensors: - values[(kind, index)] = tensor.to( - device=device, dtype=torch.float32, non_blocking=True - ).contiguous() - cached = {"key": cache_key, "values": values} - setattr(layout, LAYOUT_DEVICE_CACHE_ATTR, cached) +def materialize_continuum_latents(payload,x,*,debug=False): + layout=payload.get("layout"); device=_input_device(x) + if layout is None or device is None or device.type=="cpu": return + keyframes=[dict(item) for item in (payload.get("keyframes") or ())]; refs=[dict(item) for item in (payload.get("refs") or ())] + marked=[] + for i,item in enumerate(keyframes): + if MARK_VIDEO_SLOT in item and torch.is_tensor(item.get("latent")): marked.append(("keyframe",i,item["latent"])) + for i,item in enumerate(refs): + if _is_video_context(item) and torch.is_tensor(item.get("latent")): marked.append(("ref_video",i,item["latent"])) + if _is_audio_context(item) and torch.is_tensor(item.get("audio_latent")): marked.append(("ref_audio",i,item["audio_latent"])) + if not marked: return + cache_key=(str(device),tuple((kind,i,id(t),tuple(t.shape),str(t.dtype)) for kind,i,t in marked)) + cached=getattr(layout,LAYOUT_DEVICE_CACHE_ATTR,None) + if not isinstance(cached,dict) or cached.get("key")!=cache_key: + values={(kind,i):t.to(device=device,dtype=torch.float32,non_blocking=True).contiguous() for kind,i,t in marked} + cached={"key":cache_key,"values":values}; setattr(layout,LAYOUT_DEVICE_CACHE_ATTR,cached) if debug: - total_bytes = sum(t.numel() * t.element_size() for t in values.values()) - LOG.info( - "Continuum context cached on %s: %d tensors, %.2f MiB", - device, - len(values), - total_bytes / (1024 * 1024), - ) + total_bytes=sum(t.numel()*t.element_size() for t in values.values()) + LOG.info("Continuum context cached on %s: %d tensors, %.2f MiB",device,len(values),total_bytes/(1024*1024)) + values=cached["values"] + for i,item in enumerate(keyframes): + replacement=values.get(("keyframe",i)) + if replacement is not None: item["latent"]=replacement + for i,item in enumerate(refs): + replacement=values.get(("ref_video",i)) + if replacement is not None: item["latent"]=replacement + replacement=values.get(("ref_audio",i)) + if replacement is not None: item["audio_latent"]=replacement + payload["keyframes"]=keyframes; payload["refs"]=refs - values = cached["values"] - for index, item in enumerate(keyframes): - replacement = values.get(("keyframe", index)) - if replacement is not None: - item["latent"] = replacement - for index, item in enumerate(refs): - replacement = values.get(("ref_video", index)) - if replacement is not None: - item["latent"] = replacement - replacement = values.get(("ref_audio", index)) - if replacement is not None: - item["audio_latent"] = replacement - payload["keyframes"] = keyframes - payload["refs"] = refs - - -def _single_segment(layout: Any, kind: str) -> tuple[int, int]: - matches = [(int(a), int(b)) for a, b, k in layout.segments if k == kind] - if len(matches) != 1: - raise LayoutCompatibilityError(f"expected one H3 '{kind}' segment, found {len(matches)}") +def _single_segment(layout,kind): + matches=[(int(a),int(b)) for a,b,k in layout.segments if k==kind] + if len(matches)!=1: raise LayoutCompatibilityError(f"expected one H3 '{kind}' segment, found {len(matches)}") return matches[0] - - -def _map_refs_to_segments( - layout: Any, refs: list[dict[str, Any]], -) -> list[dict[str, tuple[int, int] | None]]: - available = [ - (int(a), int(b), str(kind)) - for a, b, kind in layout.segments - if kind in ("ref_img", "ref_audio") - ] - cursor = 0 - result: list[dict[str, tuple[int, int] | None]] = [] - - def consume(expected: str) -> tuple[int, int]: +def _map_refs_to_segments(layout,refs): + available=[(int(a),int(b),str(kind)) for a,b,kind in layout.segments if kind in ("ref_img","ref_audio")] + cursor=0; result=[] + def consume(expected): nonlocal cursor - if cursor >= len(available): - raise LayoutCompatibilityError(f"layout ended while mapping '{expected}'") - a, b, kind = available[cursor] - cursor += 1 - if kind != expected: - raise LayoutCompatibilityError( - f"reference layout mismatch: expected '{expected}', found '{kind}'" - ) - return a, b - + if cursor>=len(available): raise LayoutCompatibilityError(f"layout ended while mapping '{expected}'") + a,b,kind=available[cursor]; cursor+=1 + if kind!=expected: raise LayoutCompatibilityError(f"reference layout mismatch: expected '{expected}', found '{kind}'") + return a,b for ref in refs: - kind = ref.get("kind") - mapping: dict[str, tuple[int, int] | None] = {"audio": None, "video": None} - if kind == "image": - mapping["video"] = consume("ref_img") - elif kind == "audio": - if int(ref.get("ref_audio_t", 0)) > 0: - mapping["audio"] = consume("ref_audio") - elif kind in ("video", "video_audio"): - if int(ref.get("ref_audio_t", 0)) > 0: - mapping["audio"] = consume("ref_audio") - mapping["video"] = consume("ref_img") - else: - raise LayoutCompatibilityError(f"unsupported H3 reference kind: {kind!r}") + kind=ref.get("kind"); mapping={"audio":None,"video":None} + if kind=="image": mapping["video"]=consume("ref_img") + elif kind=="audio": + if int(ref.get("ref_audio_t",0))>0: mapping["audio"]=consume("ref_audio") + elif kind in ("video","video_audio"): + if int(ref.get("ref_audio_t",0))>0: mapping["audio"]=consume("ref_audio") + mapping["video"]=consume("ref_img") + else: raise LayoutCompatibilityError(f"unsupported H3 reference kind: {kind!r}") result.append(mapping) - if cursor != len(available): - raise LayoutCompatibilityError( - f"layout contains {len(available)-cursor} unmapped reference segments" - ) + if cursor!=len(available): raise LayoutCompatibilityError(f"layout contains {len(available)-cursor} unmapped reference segments") return result - - -def _patch_signature(payload: dict[str, Any], layout: Any) -> tuple[Any, ...]: +def _patch_signature(payload,layout): return ( - tuple( - ( - int(kf.get("resolved_frame_index", -1)), - int(kf.get(MARK_VIDEO_SLOT, -1)), - ) - for kf in (payload.get("keyframes") or ()) - ), - tuple( - ( - bool(ref.get(MARK_VIDEO_CONTEXT)), - tuple(sorted(_reference_metadata(ref).items())), - int(ref.get("latent_t", 0)), - int(ref.get(MARK_CONTEXT_FRAMES, 0)), - bool(ref.get(MARK_AUDIO_CONTEXT)), - int(ref.get("ref_audio_t", 0)), - float(ref.get(MARK_AUDIO_END_FRAME, 0.0)), - float(ref.get(MARK_AUDIO_OVERHANG, 0.0)), - ) - for ref in (payload.get("refs") or ()) - ), - tuple(getattr(layout, "signature", ())), - tuple((int(a), int(b), str(k)) for a, b, k in layout.segments), + tuple((int(kf.get("resolved_frame_index",-1)),int(kf.get(MARK_VIDEO_SLOT,-1))) for kf in (payload.get("keyframes") or ())), + tuple((bool(ref.get(MARK_VIDEO_CONTEXT)),tuple(sorted(_reference_metadata(ref).items())),int(ref.get("latent_t",0)),int(ref.get(MARK_CONTEXT_FRAMES,0)),bool(ref.get(MARK_AUDIO_CONTEXT)),int(ref.get("ref_audio_t",0)),float(ref.get(MARK_AUDIO_END_FRAME,0.0)),float(ref.get(MARK_AUDIO_OVERHANG,0.0))) for ref in (payload.get("refs") or ())), + tuple(getattr(layout,"signature",())), tuple((int(a),int(b),str(k)) for a,b,k in layout.segments), ) - - -def _branch_key(transformer_options: dict[str, Any]) -> tuple[Any, ...]: - conds = transformer_options.get("cond_or_uncond") - uuids = transformer_options.get("uuids") - if conds is None or uuids is None: - return ("default",) - try: - cond_values = tuple(int(value) for value in conds) - uuid_values = tuple(str(value) for value in uuids) - except (TypeError, ValueError): - return ("default",) - if not cond_values or len(cond_values) != len(uuid_values): - return ("default",) - return tuple( - (cond_values[index], uuid_values[index]) - for index in range(len(cond_values)) - ) - - -def _sampled_tensor_signature(tensor: torch.Tensor | None) -> tuple[Any, ...] | None: - if tensor is None: - return None - value = tensor.detach().to(dtype=torch.float32) - flat = value.reshape(-1) - if flat.numel() <= 64: - sample = flat +def _branch_key(transformer_options): + conds=transformer_options.get("cond_or_uncond"); uuids=transformer_options.get("uuids") + if conds is None or uuids is None: return ("default",) + try: cond_values=tuple(int(v) for v in conds); uuid_values=tuple(str(v) for v in uuids) + except (TypeError,ValueError): return ("default",) + if not cond_values or len(cond_values)!=len(uuid_values): return ("default",) + return tuple((cond_values[i],uuid_values[i]) for i in range(len(cond_values))) +def _sampled_tensor_signature(tensor): + if tensor is None: return None + value=tensor.detach().to(dtype=torch.float32); flat=value.reshape(-1) + if flat.numel()<=64: sample=flat else: - indices = torch.linspace( - 0, - flat.numel() - 1, - 64, - dtype=torch.long, - device=flat.device, - ) - sample = flat.index_select(0, indices) - sample = sample.to("cpu") - finite = bool(torch.isfinite(sample).all().item()) - if not finite: - raise LayoutCompatibilityError("Continuum layout/context contains NaN or Inf") - weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32) - return ( - tuple(int(part) for part in tensor.shape), - str(tensor.dtype), - finite, - float(sample.mean().item()), - float(sample.std(unbiased=False).item()), - float((sample * weights).sum().item()), - ) + indices=torch.linspace(0,flat.numel()-1,64,dtype=torch.long,device=flat.device); sample=flat.index_select(0,indices) + sample=sample.to("cpu"); finite=bool(torch.isfinite(sample).all().item()) + if not finite: raise LayoutCompatibilityError("Continuum layout/context contains NaN or Inf") + weights=torch.arange(1,sample.numel()+1,dtype=torch.float32) + return (tuple(int(part) for part in tensor.shape),str(tensor.dtype),finite,float(sample.mean().item()),float(sample.std(unbiased=False).item()),float((sample*weights).sum().item())) +def _ranges_overlap(left,right): return max(left[0],right[0]) bool: - return max(left[0], right[0]) < min(left[1], right[1]) - - -def validate_native_continuity_layout( - payload: dict[str, Any], - *, - transformer_options: dict[str, Any] | None, - branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]], -) -> dict[str, Any]: - """Validate immutable native-H3 Context/Target topology per logical branch.""" - - layout = payload.get("layout") - if layout is None: - raise LayoutCompatibilityError("Native Continuity payload has no PackedLayout") - position_ids = getattr(layout, "position_ids", None) - segments = tuple( - (int(start), int(stop), str(kind)) - for start, stop, kind in getattr(layout, "segments", ()) - ) - if not torch.is_tensor(position_ids) or position_ids.ndim != 2: - raise LayoutCompatibilityError("Native Continuity position_ids are unavailable") - seq_len = int(getattr(layout, "seq_len", position_ids.shape[0])) - if seq_len != int(position_ids.shape[0]): - raise LayoutCompatibilityError("PackedLayout seq_len does not match position_ids") - - previous_stop = 0 - for start, stop, kind in segments: - if start < previous_stop or stop <= start or stop > seq_len: - raise LayoutCompatibilityError( - f"PackedLayout segment {kind!r} has an overlapping or invalid row range" - ) - previous_stop = stop - - target_audio = [(a, b) for a, b, kind in segments if kind == "audio"] - target_video = [(a, b) for a, b, kind in segments if kind == "video"] - if len(target_audio) != 1 or len(target_video) != 1: - raise LayoutCompatibilityError( - "Native Continuity requires one target audio and one target video segment" - ) - audio_range, video_range = target_audio[0], target_video[0] - if audio_range[1] != video_range[0] or video_range[1] != seq_len: - raise LayoutCompatibilityError( - "target segments must be the contiguous packed tail [audio | video]" - ) - - refs = list(payload.get("refs") or ()) - mappings = _map_refs_to_segments(layout, refs) - context_ranges: list[tuple[int, int, str]] = [] - context_descriptors: list[tuple[Any, ...]] = [] - for ref, mapping in zip(refs, mappings): - metadata = _reference_metadata(ref) +def validate_native_continuity_layout(payload,*,transformer_options,branch_baselines): + layout=payload.get("layout") + if layout is None: raise LayoutCompatibilityError("Native Continuity payload has no PackedLayout") + position_ids=getattr(layout,"position_ids",None); segments=tuple((int(start),int(stop),str(kind)) for start,stop,kind in getattr(layout,"segments",())) + if not torch.is_tensor(position_ids) or position_ids.ndim!=2: raise LayoutCompatibilityError("Native Continuity position_ids are unavailable") + seq_len=int(getattr(layout,"seq_len",position_ids.shape[0])) + if seq_len!=int(position_ids.shape[0]): raise LayoutCompatibilityError("PackedLayout seq_len does not match position_ids") + previous_stop=0 + for start,stop,kind in segments: + if startseq_len: raise LayoutCompatibilityError(f"PackedLayout segment {kind!r} has an overlapping or invalid row range") + previous_stop=stop + target_audio=[(a,b) for a,b,kind in segments if kind=="audio"]; target_video=[(a,b) for a,b,kind in segments if kind=="video"] + if len(target_audio)!=1 or len(target_video)!=1: raise LayoutCompatibilityError("Native Continuity requires one target audio and one target video segment") + audio_range,video_range=target_audio[0],target_video[0] + if audio_range[1]!=video_range[0] or video_range[1]!=seq_len: raise LayoutCompatibilityError("target segments must be the contiguous packed tail [audio | video]") + refs=list(payload.get("refs") or ()); mappings=_map_refs_to_segments(layout,refs); context_ranges=[]; context_descriptors=[] + for ref,mapping in zip(refs,mappings): + metadata=_reference_metadata(ref) if metadata: - if ( - type(metadata.get("api")) is not int - or int(metadata["api"]) != CONTINUUM_INTEROP_API - ): - raise LayoutCompatibilityError("unsupported Continuum reference metadata") - if _is_video_context(ref) and metadata.get("role") != "video_context": - raise LayoutCompatibilityError("Continuum video reference role is invalid") - if _is_audio_context(ref) and metadata.get("audio_role") not in ( - None, - "audio_context", - ): - raise LayoutCompatibilityError("Continuum audio reference role is invalid") - + if type(metadata.get("api")) is not int or int(metadata["api"])!=CONTINUUM_INTEROP_API: raise LayoutCompatibilityError("unsupported Continuum reference metadata") + if _is_video_context(ref) and metadata.get("role")!="video_context": raise LayoutCompatibilityError("Continuum video reference role is invalid") + if _is_audio_context(ref) and metadata.get("audio_role") not in (None,"audio_context"): raise LayoutCompatibilityError("Continuum audio reference role is invalid") if _is_video_context(ref): - row_range = mapping.get("video") - if row_range is None: - raise LayoutCompatibilityError("Continuum video context has no reference rows") - context_ranges.append((*row_range, "video_context")) + row_range=mapping.get("video") + if row_range is None: raise LayoutCompatibilityError("Continuum video context has no reference rows") + context_ranges.append((*row_range,"video_context")) if _is_audio_context(ref): - row_range = mapping.get("audio") - if row_range is None: - raise LayoutCompatibilityError("Continuum audio context has no reference rows") - context_ranges.append((*row_range, "audio_context")) + row_range=mapping.get("audio") + if row_range is None: raise LayoutCompatibilityError("Continuum audio context has no reference rows") + context_ranges.append((*row_range,"audio_context")) if _is_video_context(ref) or _is_audio_context(ref): - context_descriptors.append( - ( - ref.get("kind"), - int(ref.get("latent_t", 0)), - int(ref.get("latent_h", 0)), - int(ref.get("latent_w", 0)), - int(ref.get("ref_audio_t", 0)), - tuple(sorted(metadata.items())), - _sampled_tensor_signature(ref.get("latent")), - _sampled_tensor_signature(ref.get("audio_latent")), - ) - ) + context_descriptors.append((ref.get("kind"),int(ref.get("latent_t",0)),int(ref.get("latent_h",0)),int(ref.get("latent_w",0)),int(ref.get("ref_audio_t",0)),tuple(sorted(metadata.items())),_sampled_tensor_signature(ref.get("latent")),_sampled_tensor_signature(ref.get("audio_latent")))) + if not context_ranges: raise LayoutCompatibilityError("Native Continuity payload has no context rows") + for start,stop,role in context_ranges: + if _ranges_overlap((start,stop),audio_range) or _ranges_overlap((start,stop),video_range): raise LayoutCompatibilityError(f"{role} rows overlap target rows") + topology=(tuple(getattr(layout,"signature",())),segments,audio_range,video_range,tuple(context_ranges),tuple(context_descriptors),int(payload.get("frame_count",0) or 0),_sampled_tensor_signature(position_ids)) + options=transformer_options if isinstance(transformer_options,dict) else {} + if options.get("spectrum_h3_actual") is False: return {"status":"forecast_validated","branch":_branch_key(options)} + branch=_branch_key(options); baseline=branch_baselines.get(branch) + if baseline is None: branch_baselines[branch]=topology; return {"status":"baseline_created","branch":branch} + if baseline!=topology: raise LayoutCompatibilityError(f"Native Continuity topology changed during sampling for branch {branch!r}") + return {"status":"baseline_matched","branch":branch} - if not context_ranges: - raise LayoutCompatibilityError("Native Continuity payload has no context rows") - for start, stop, role in context_ranges: - row_range = (start, stop) - if _ranges_overlap(row_range, audio_range) or _ranges_overlap(row_range, video_range): - raise LayoutCompatibilityError(f"{role} rows overlap target rows") - - topology = ( - tuple(getattr(layout, "signature", ())), - segments, - audio_range, - video_range, - tuple(context_ranges), - tuple(context_descriptors), - int(payload.get("frame_count", 0) or 0), - _sampled_tensor_signature(position_ids), - ) - options = transformer_options if isinstance(transformer_options, dict) else {} - if options.get("spectrum_h3_actual") is False: - return {"status": "forecast_validated", "branch": _branch_key(options)} - - branch = _branch_key(options) - baseline = branch_baselines.get(branch) - if baseline is None: - branch_baselines[branch] = topology - return {"status": "baseline_created", "branch": branch} - if baseline != topology: - raise LayoutCompatibilityError( - f"Native Continuity topology changed during sampling for branch {branch!r}" - ) - return {"status": "baseline_matched", "branch": branch} - - -def patch_layout_in_place( - payload: dict[str, Any], *, strict: bool = True, debug: bool = False, -) -> dict[str, Any]: - layout = payload.get("layout") - if layout is None: - raise LayoutCompatibilityError( - "H3 payload has no prebuilt PackedLayout; use ComfyUI v0.31.0 or newer" - ) - missing = [name for name in ("position_ids", "segments", "signature") if not hasattr(layout, name)] - if missing: - raise LayoutCompatibilityError("PackedLayout is missing: " + ", ".join(missing)) - position_ids = layout.position_ids - if not torch.is_tensor(position_ids) or position_ids.ndim != 2 or position_ids.shape[1] < 3: - raise LayoutCompatibilityError("position_ids must be an [S,3] Tensor") - - signature = _patch_signature(payload, layout) - if getattr(layout, LAYOUT_SIGNATURE_ATTR, None) == signature: - return {"status": "already_patched", "position_ids_id": id(position_ids)} - - if not hasattr(layout, LAYOUT_ORIGINAL_TIME_ATTR): - setattr(layout, LAYOUT_ORIGINAL_TIME_ATTR, position_ids[:, 0].clone()) - original_time = getattr(layout, LAYOUT_ORIGINAL_TIME_ATTR) - if not torch.is_tensor(original_time) or original_time.shape != position_ids[:, 0].shape: - raise LayoutCompatibilityError("stored layout baseline is incompatible") - position_ids[:, 0].copy_(original_time) - - keyframes = list(payload.get("keyframes") or ()) - refs = list(payload.get("refs") or ()) - cond_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "cond"] - if len(cond_segments) != len(keyframes): - raise LayoutCompatibilityError( - f"layout has {len(cond_segments)} cond segments for {len(keyframes)} keyframes" - ) - - video_start, video_stop = _single_segment(layout, "video") - _single_segment(layout, "audio") - text_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "text"] - if len(text_segments) != 1 or text_segments[0][0] != 0: - raise LayoutCompatibilityError("unexpected H3 text segment") - text_len = text_segments[0][1] - - latent_t = int(layout.signature[1]) - video_rows = video_stop - video_start - if latent_t <= 0 or video_rows % latent_t: - raise LayoutCompatibilityError( - f"video rows {video_rows} are incompatible with latent T={latent_t}" - ) - frame_rows = video_rows // latent_t - target_origin = float(position_ids[video_start, 0]) - reference_shift = target_origin - float(text_len) - - # Existing first/last-frame keyframes are text-relative in stock H3. Once - # refs are present, move them by the target-origin shift. The legacy marked - # slot path is retained for compatibility with early development states. - patched_video_slots = 0 - for (start, stop), keyframe in zip(cond_segments, keyframes): - if stop - start != frame_rows: - raise LayoutCompatibilityError("keyframe rows no longer equal one target slot") +def patch_layout_in_place(payload,*,strict=True,debug=False): + layout=payload.get("layout") + if layout is None: raise LayoutCompatibilityError("H3 payload has no prebuilt PackedLayout; use ComfyUI v0.31.0 or newer") + missing=[name for name in ("position_ids","segments","signature") if not hasattr(layout,name)] + if missing: raise LayoutCompatibilityError("PackedLayout is missing: "+", ".join(missing)) + position_ids=layout.position_ids + if not torch.is_tensor(position_ids) or position_ids.ndim!=2 or position_ids.shape[1]<3: raise LayoutCompatibilityError("position_ids must be an [S,3] Tensor") + signature=_patch_signature(payload,layout) + if getattr(layout,LAYOUT_SIGNATURE_ATTR,None)==signature: return {"status":"already_patched","position_ids_id":id(position_ids)} + if not hasattr(layout,LAYOUT_ORIGINAL_TIME_ATTR): setattr(layout,LAYOUT_ORIGINAL_TIME_ATTR,position_ids[:,0].clone()) + original_time=getattr(layout,LAYOUT_ORIGINAL_TIME_ATTR) + if not torch.is_tensor(original_time) or original_time.shape!=position_ids[:,0].shape: raise LayoutCompatibilityError("stored layout baseline is incompatible") + position_ids[:,0].copy_(original_time) + keyframes=list(payload.get("keyframes") or ()); refs=list(payload.get("refs") or ()); cond_segments=[(int(a),int(b)) for a,b,kind in layout.segments if kind=="cond"] + if len(cond_segments)!=len(keyframes): raise LayoutCompatibilityError(f"layout has {len(cond_segments)} cond segments for {len(keyframes)} keyframes") + video_start,video_stop=_single_segment(layout,"video"); _single_segment(layout,"audio") + text_segments=[(int(a),int(b)) for a,b,kind in layout.segments if kind=="text"] + if len(text_segments)!=1 or text_segments[0][0]!=0: raise LayoutCompatibilityError("unexpected H3 text segment") + text_len=text_segments[0][1]; latent_t=int(layout.signature[1]); video_rows=video_stop-video_start + if latent_t<=0 or video_rows%latent_t: raise LayoutCompatibilityError(f"video rows {video_rows} are incompatible with latent T={latent_t}") + frame_rows=video_rows//latent_t; target_origin=float(position_ids[video_start,0]); reference_shift=target_origin-float(text_len) + patched_video_slots=0 + for (start,stop),keyframe in zip(cond_segments,keyframes): + if stop-start!=frame_rows: raise LayoutCompatibilityError("keyframe rows no longer equal one target slot") if MARK_VIDEO_SLOT in keyframe: - slot = int(keyframe[MARK_VIDEO_SLOT]) - if not (0 <= slot < latent_t): - raise LayoutCompatibilityError( - f"context slot {slot} is outside target latent T={latent_t}" - ) - target_row = video_start + slot * frame_rows - position_ids[start:stop].copy_( - position_ids[target_row : target_row + frame_rows] - ) - patched_video_slots += 1 - elif reference_shift: - position_ids[start:stop, 0].add_(reference_shift) - - ref_mappings = _map_refs_to_segments(layout, refs) - patched_video_refs = 0 - patched_audio = 0 - for ref, mapping in zip(refs, ref_mappings): + slot=int(keyframe[MARK_VIDEO_SLOT]) + if not (0<=slot latent_t: - raise LayoutCompatibilityError( - f"video context T={context_t} exceeds target latent T={latent_t}" - ) - target_stop = video_start + expected_rows - # Copy all (t,h,w) coordinates. Resolution equality is already - # enforced by state validation; this also fails safely if row geometry changes. - position_ids[start:stop].copy_(position_ids[video_start:target_stop]) - patched_video_refs += 1 - + segment=mapping.get("video") + if segment is None: raise LayoutCompatibilityError("Continuum video ref has no video segment") + start,stop=segment; context_t=int(ref.get("latent_t",0)); expected_rows=context_t*frame_rows + if context_t<=0 or stop-start!=expected_rows: raise LayoutCompatibilityError(f"video-context rows {stop-start} do not match latent T={context_t}") + if context_t>latent_t: raise LayoutCompatibilityError(f"video context T={context_t} exceeds target latent T={latent_t}") + position_ids[start:stop].copy_(position_ids[video_start:video_start+expected_rows]); patched_video_refs+=1 if _is_audio_context(ref): - segment = mapping.get("audio") - if segment is None: - raise LayoutCompatibilityError("Continuum audio ref has no audio segment") - start, stop = segment - rt = int(ref.get("ref_audio_t", 0)) - if rt <= 0 or stop - start != rt * 2: - raise LayoutCompatibilityError( - f"audio rows {stop-start} do not match {rt} stereo steps" - ) - end_frame = float(ref.get(MARK_AUDIO_END_FRAME, 0.0)) - grid_offset = float(ref.get(MARK_AUDIO_OVERHANG, 0.0)) - desired_end = target_origin + FRAME_RESCALE * end_frame + grid_offset - desired_start = desired_end - float(rt) - old_start = float(position_ids[start, 0]) - position_ids[start:stop, 0].add_(desired_start - old_start) - patched_audio += 1 - - patched_video = patched_video_slots + patched_video_refs - if patched_video == 0: - message = "no marked Continuum video context was found" - if strict: - raise LayoutCompatibilityError(message) + segment=mapping.get("audio") + if segment is None: raise LayoutCompatibilityError("Continuum audio ref has no audio segment") + start,stop=segment; rt=int(ref.get("ref_audio_t",0)) + if rt<=0 or stop-start!=rt*2: raise LayoutCompatibilityError(f"audio rows {stop-start} do not match {rt} stereo steps") + end_frame=float(ref.get(MARK_AUDIO_END_FRAME,0.0)); grid_offset=float(ref.get(MARK_AUDIO_OVERHANG,0.0)); desired_end=target_origin+FRAME_RESCALE*end_frame+grid_offset; desired_start=desired_end-float(rt); old_start=float(position_ids[start,0]); position_ids[start:stop,0].add_(desired_start-old_start); patched_audio+=1 + patched_video=patched_video_slots+patched_video_refs + if patched_video==0: + message="no marked Continuum video context was found" + if strict: raise LayoutCompatibilityError(message) LOG.warning(message) - - setattr(layout, LAYOUT_SIGNATURE_ATTR, signature) - if debug: - LOG.info( - "Continuum layout: video_refs=%d legacy_slots=%d audio_windows=%d " - "origin=%.6f rows=%d pos_id=%s", - patched_video_refs, - patched_video_slots, - patched_audio, - target_origin, - position_ids.shape[0], - id(position_ids), - ) - return { - "status": "patched", - "video_contexts": patched_video, - "video_refs": patched_video_refs, - "legacy_video_slots": patched_video_slots, - "audio_windows": patched_audio, - "target_origin": target_origin, - "position_ids_id": id(position_ids), - } + setattr(layout,LAYOUT_SIGNATURE_ATTR,signature) + if debug: LOG.info("Continuum layout: video_refs=%d legacy_slots=%d audio_windows=%d origin=%.6f rows=%d pos_id=%s",patched_video_refs,patched_video_slots,patched_audio,target_origin,position_ids.shape[0],id(position_ids)) + return {"status":"patched","video_contexts":patched_video,"video_refs":patched_video_refs,"legacy_video_slots":patched_video_slots,"audio_windows":patched_audio,"target_origin":target_origin,"position_ids_id":id(position_ids)} diff --git a/metadata.ini b/metadata.ini index e349bac..5d97fb1 100644 --- a/metadata.ini +++ b/metadata.ini @@ -1,4 +1,4 @@ [project] name = H3 Continuum -version = 2.1.5 -description = Integrated native MiniMax H3 long-video continuation with V1 compatibility and external SageAttention, Sol-Attn and Spectrum composition. +version = 2.1.7 +description = Integrated native MiniMax H3 long-video continuation with stable Vue UI, V1 compatibility and external SageAttention, Sol-Attn and Spectrum composition. diff --git a/model_patch.py b/model_patch.py index ca9d3af..b220618 100644 --- a/model_patch.py +++ b/model_patch.py @@ -21,7 +21,6 @@ from .layout_adapter import ( ) - def _wrapper_factory(*, strict: bool, debug: bool) -> Callable[..., Any]: branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]] = {} @@ -32,16 +31,12 @@ def _wrapper_factory(*, strict: bool, debug: bool) -> Callable[..., Any]: try: ensure_native_h3_base_model(executor.class_obj) patched_payload = dict(payload) - # Patch coordinates before any accelerator sees the layout, then - # cache only the small Continuum state tensors on the active device. patch_layout_in_place(patched_payload, strict=True, debug=debug) transformer_options = kwargs.get("transformer_options") validate_native_continuity_layout( patched_payload, transformer_options=( - transformer_options - if isinstance(transformer_options, dict) - else {} + transformer_options if isinstance(transformer_options, dict) else {} ), branch_baselines=branch_baselines, ) @@ -50,11 +45,9 @@ def _wrapper_factory(*, strict: bool, debug: bool) -> Callable[..., Any]: normalize_condition_latents(patched_payload) kwargs["minimax_payload"] = patched_payload except (CompatibilityError, LayoutCompatibilityError, KeyError, TypeError, ValueError) as exc: - # A stock-time reference would render a plausible but incorrect join. - # Never silently bypass a marked Continuum payload, even when the - # proactive compatibility toggle is disabled. raise RuntimeError(f"H3 Continuum Join compatibility failure: {exc}") from exc return executor(*args, **kwargs) + return apply_model_wrapper @@ -67,8 +60,7 @@ def patch_model(model: Any, *, strict: bool, debug: bool): missing_api = [name for name in required_api if not hasattr(model, name)] if missing_api: raise RuntimeError( - "MODEL lacks the wrapper API required by H3 Continuum Join: " - + ", ".join(missing_api) + "MODEL lacks the wrapper API required by H3 Continuum Join: " + ", ".join(missing_api) ) base_model = getattr(model, "model", None) if base_model is None: @@ -89,11 +81,7 @@ def patch_model(model: Any, *, strict: bool, debug: bool): return patched -def continuum_interop_request( - *, - chunk_index: int, - context_frames: int, -) -> dict[str, Any]: +def continuum_interop_request(*, chunk_index: int, context_frames: int) -> dict[str, Any]: return { "api": CONTINUUM_INTEROP_API, "active": True, @@ -112,7 +100,6 @@ def clone_model_for_chunk( context_frames: int | None, ): """Create a call-local MODEL and attach an optional read-only Spectrum hint.""" - chunk_model = patch_model(model, strict=bool(strict), debug=bool(debug)) model_options = dict(getattr(chunk_model, "model_options", None) or {}) transformer_options = dict(model_options.get("transformer_options") or {}) @@ -120,8 +107,7 @@ def clone_model_for_chunk( transformer_options.pop(CONTINUUM_INTEROP_KEY, None) else: transformer_options[CONTINUUM_INTEROP_KEY] = continuum_interop_request( - chunk_index=int(chunk_index), - context_frames=int(context_frames), + chunk_index=int(chunk_index), context_frames=int(context_frames) ) model_options["transformer_options"] = transformer_options chunk_model.model_options = model_options diff --git a/nodes.py b/nodes.py index 42e0e7c..60a605a 100644 --- a/nodes.py +++ b/nodes.py @@ -363,11 +363,17 @@ from .v2.nodes import ( NODE_CLASS_MAPPINGS.update(V2_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(V2_NODE_DISPLAY_NAME_MAPPINGS) -# V2's integrated sampler is the only primary workflow node. Keep every older -# identifier registered so saved workflows still load, while ComfyUI's native -# deprecated-node filter keeps legacy building blocks out of the normal menu. +# The stable facade and its pack helpers are the public workflow surface. Keep +# every older identifier registered so saved workflows still load, while +# ComfyUI's native deprecated-node filter hides legacy building blocks. +_primary_node_ids = { + "H3ContinuumSampler", + "H3ContinuumClipOverrides", + "H3ContinuumAdvanced", + "H3ContinuumResult", +} for _node_id, _node_class in NODE_CLASS_MAPPINGS.items(): - if _node_id == "H3ContinuumSamplerV2": + if _node_id in _primary_node_ids: continue _node_class.DEPRECATED = True _node_class.CATEGORY = f"{CATEGORY}/Legacy" diff --git a/pyproject.toml b/pyproject.toml index 08cc2f0..50d8017 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "comfyui-h3-continuum" -version = "2.1.5" +version = "2.1.7" description = "Integrated N-chunk continuation for ComfyUI native MiniMax H3 video and audio" readme = "README.md" requires-python = ">=3.10" diff --git a/state.py b/state.py index 55bf1b6..45034c3 100644 --- a/state.py +++ b/state.py @@ -1,286 +1,125 @@ """State and plan containers for H3 Continuum Join.""" from __future__ import annotations - from typing import Any - import torch - from .constants import DEFAULT_STATE_CAPACITY_FRAMES, PLAN_MAGIC, STATE_MAGIC from .temporal import ( - audio_latent_t_from_grid_offset, - audio_grid_offset, - audio_latent_t, - context_slots, - is_valid_frame_count, - make_av_context_window, - pixel_frames_for_latent_t, - video_latent_t, + audio_latent_t_from_grid_offset, audio_grid_offset, audio_latent_t, context_slots, + is_valid_frame_count, make_av_context_window, pixel_frames_for_latent_t, video_latent_t, ) from .version import PLAN_SCHEMA_VERSION, STATE_SCHEMA_VERSION - class StateValidationError(ValueError): pass - def extract_av_streams(latent: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]: if not isinstance(latent, dict) or "samples" not in latent: raise StateValidationError("expected LATENT dictionary containing 'samples'") - samples = latent["samples"] - if hasattr(samples, "unbind"): - parts = list(samples.unbind()) - elif isinstance(samples, (tuple, list)): - parts = list(samples) - else: - raise StateValidationError( - "expected MiniMax H3 NestedTensor samples; got " + type(samples).__name__ - ) - if len(parts) < 2: - raise StateValidationError("MiniMax H3 latent must contain video and audio") - video, audio = parts[0], parts[1] - if video.ndim == 4: - video = video.unsqueeze(0) - if audio.ndim == 3: - audio = audio.unsqueeze(0) - if video.ndim != 5 or audio.ndim != 4: - raise StateValidationError( - f"unexpected H3 latent shapes: video={tuple(video.shape)}, audio={tuple(audio.shape)}" - ) - if video.shape[0] != 1 or audio.shape[0] != 1: - raise StateValidationError("H3 Continuum Join supports batch size 1") - if video.shape[1] != 24: - raise StateValidationError(f"expected 24 video channels, got {video.shape[1]}") - if audio.shape[1] != 32 or audio.shape[2] != 2: + samples=latent["samples"] + if hasattr(samples,"unbind"): parts=list(samples.unbind()) + elif isinstance(samples,(tuple,list)): parts=list(samples) + else: raise StateValidationError("expected MiniMax H3 NestedTensor samples; got "+type(samples).__name__) + if len(parts)<2: raise StateValidationError("MiniMax H3 latent must contain video and audio") + video,audio=parts[0],parts[1] + if video.ndim==4: video=video.unsqueeze(0) + if audio.ndim==3: audio=audio.unsqueeze(0) + if video.ndim!=5 or audio.ndim!=4: + raise StateValidationError(f"unexpected H3 latent shapes: video={tuple(video.shape)}, audio={tuple(audio.shape)}") + if video.shape[0]!=1 or audio.shape[0]!=1: raise StateValidationError("H3 Continuum Join supports batch size 1") + if video.shape[1]!=24: raise StateValidationError(f"expected 24 video channels, got {video.shape[1]}") + if audio.shape[1]!=32 or audio.shape[2]!=2: raise StateValidationError(f"expected stereo [1,32,2,T] audio, got {tuple(audio.shape)}") - return video, audio + return video,audio +def make_plan(*, continuation:bool, clip_index:int, total_frames:int, trim_frames:int, width:int, height:int, + context_frames:int, state_capacity_frames:int, requested_extend_seconds:float, debug:bool)->dict[str,Any]: + return {"magic":PLAN_MAGIC,"schema_version":PLAN_SCHEMA_VERSION,"continuation":bool(continuation), + "clip_index":int(clip_index),"total_frames":int(total_frames),"trim_frames":int(trim_frames), + "net_frames":int(total_frames)-int(trim_frames),"width":int(width),"height":int(height), + "context_frames":int(context_frames),"state_capacity_frames":int(state_capacity_frames), + "requested_extend_seconds":float(requested_extend_seconds),"debug":bool(debug)} -def make_plan( - *, continuation: bool, clip_index: int, total_frames: int, trim_frames: int, - width: int, height: int, context_frames: int, state_capacity_frames: int, - requested_extend_seconds: float, debug: bool, -) -> dict[str, Any]: - return { - "magic": PLAN_MAGIC, - "schema_version": PLAN_SCHEMA_VERSION, - "continuation": bool(continuation), - "clip_index": int(clip_index), - "total_frames": int(total_frames), - "trim_frames": int(trim_frames), - "net_frames": int(total_frames) - int(trim_frames), - "width": int(width), - "height": int(height), - "context_frames": int(context_frames), - "state_capacity_frames": int(state_capacity_frames), - "requested_extend_seconds": float(requested_extend_seconds), - "debug": bool(debug), - } - - -def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: - if not isinstance(plan, dict) or plan.get("magic") != PLAN_MAGIC: - raise StateValidationError("invalid H3 Continuum plan") - if int(plan.get("schema_version", -1)) != PLAN_SCHEMA_VERSION: - raise StateValidationError( - f"unsupported plan schema {plan.get('schema_version')}; expected {PLAN_SCHEMA_VERSION}" - ) - for key in ( - "total_frames", - "trim_frames", - "net_frames", - "width", - "height", - "clip_index", - "state_capacity_frames", - ): - if not isinstance(plan.get(key), int): - raise StateValidationError(f"plan field '{key}' is missing or invalid") - if plan["total_frames"] < 5 or plan["width"] <= 0 or plan["height"] <= 0: - raise StateValidationError("plan dimensions or frame count are invalid") - if plan["trim_frames"] < 0 or plan["trim_frames"] >= plan["total_frames"]: - raise StateValidationError("plan trim range is invalid") - if plan["net_frames"] != plan["total_frames"] - plan["trim_frames"]: - raise StateValidationError("plan net_frames does not match total_frames - trim_frames") - if plan["clip_index"] < 1: - raise StateValidationError("plan clip_index must be at least 1") - if plan["state_capacity_frames"] not in (5, 22, 39): - raise StateValidationError("plan state_capacity_frames must be 5, 22, or 39") - if plan["state_capacity_frames"] > plan["net_frames"]: - raise StateValidationError("plan state capacity exceeds retained output frames") - continuation = bool(plan.get("continuation", False)) - if continuation and plan["trim_frames"] <= 0: - raise StateValidationError("continuation plan must trim a positive overlap") - if not continuation and plan["trim_frames"] != 0: - raise StateValidationError("initial plan cannot trim an overlap") +def validate_plan(plan:dict[str,Any])->dict[str,Any]: + if not isinstance(plan,dict) or plan.get("magic")!=PLAN_MAGIC: raise StateValidationError("invalid H3 Continuum plan") + if int(plan.get("schema_version",-1))!=PLAN_SCHEMA_VERSION: + raise StateValidationError(f"unsupported plan schema {plan.get('schema_version')}; expected {PLAN_SCHEMA_VERSION}") + for key in ("total_frames","trim_frames","net_frames","width","height","clip_index","state_capacity_frames"): + if not isinstance(plan.get(key),int): raise StateValidationError(f"plan field '{key}' is missing or invalid") + if plan["total_frames"]<5 or plan["width"]<=0 or plan["height"]<=0: raise StateValidationError("plan dimensions or frame count are invalid") + if plan["trim_frames"]<0 or plan["trim_frames"]>=plan["total_frames"]: raise StateValidationError("plan trim range is invalid") + if plan["net_frames"]!=plan["total_frames"]-plan["trim_frames"]: raise StateValidationError("plan net_frames does not match total_frames - trim_frames") + if plan["clip_index"]<1: raise StateValidationError("plan clip_index must be at least 1") + if plan["state_capacity_frames"] not in (5,22,39): raise StateValidationError("plan state_capacity_frames must be 5, 22, or 39") + if plan["state_capacity_frames"]>plan["net_frames"]: raise StateValidationError("plan state capacity exceeds retained output frames") + continuation=bool(plan.get("continuation",False)) + if continuation and plan["trim_frames"]<=0: raise StateValidationError("continuation plan must trim a positive overlap") + if not continuation and plan["trim_frames"]!=0: raise StateValidationError("initial plan cannot trim an overlap") return plan +def capture_state(latent:dict[str,Any],*,source_frame_count:int,clip_index:int,capacity_frames:int=DEFAULT_STATE_CAPACITY_FRAMES)->dict[str,Any]: + video,audio=extract_av_streams(latent); actual_video_t=int(video.shape[2]); actual_frames=pixel_frames_for_latent_t(actual_video_t) + if actual_frames!=int(source_frame_count): raise StateValidationError(f"plan says {source_frame_count} frames but latent represents {actual_frames}") + if not is_valid_frame_count(actual_frames) or video_latent_t(actual_frames)!=actual_video_t: + raise StateValidationError(f"sampler video latent T={actual_video_t} is not on the native H3 temporal grid") + try: window=make_av_context_window(source_frame_count,int(audio.shape[-1]),capacity_frames) + except ValueError as exc: raise StateValidationError(str(exc)) from exc + slots=context_slots(capacity_frames); audio_steps=window.audio_steps + if video.shape[2] dict[str, Any]: - video, audio = extract_av_streams(latent) - actual_video_t = int(video.shape[2]) - actual_frames = pixel_frames_for_latent_t(actual_video_t) - if actual_frames != int(source_frame_count): - raise StateValidationError( - f"plan says {source_frame_count} frames but latent represents {actual_frames}" - ) - if not is_valid_frame_count(actual_frames) or video_latent_t(actual_frames) != actual_video_t: - raise StateValidationError( - f"sampler video latent T={actual_video_t} is not on the native H3 temporal grid" - ) - try: - window = make_av_context_window( - source_frame_count, - int(audio.shape[-1]), - capacity_frames, - ) - except ValueError as exc: - raise StateValidationError(str(exc)) from exc - slots = context_slots(capacity_frames) - audio_steps = window.audio_steps - if video.shape[2] < slots or audio.shape[-1] < audio_steps: - raise StateValidationError("sampler output is too short for requested state capacity") - grid_offset = audio_grid_offset(source_frame_count, int(audio.shape[-1])) - if not (-0.500001 <= grid_offset <= 0.500001): - raise StateValidationError(f"unexpected signed audio-grid offset {grid_offset:.6f}") - video_tail = video[:, :, -slots:].detach().to("cpu").contiguous().clone() - audio_tail = audio[..., -audio_steps:].detach().to("cpu").contiguous().clone() - if not bool(torch.isfinite(video_tail.float()).all().item()): - raise StateValidationError("video continuation state contains NaN or Inf") - if not bool(torch.isfinite(audio_tail.float()).all().item()): - raise StateValidationError("audio continuation state contains NaN or Inf") - return { - "magic": STATE_MAGIC, - "schema_version": STATE_SCHEMA_VERSION, - "clip_index": int(clip_index), - "source_frame_count": int(source_frame_count), - "capacity_frames": int(capacity_frames), - "width": int(video.shape[-1]) * 16, - "height": int(video.shape[-2]) * 16, - "video_tail": video_tail, - "audio_tail": audio_tail, - # Schema-v1 field name retained. The value is signed and represents - # audio-grid end offset, not only positive overhang. - "audio_overhang": float(grid_offset), - "source_mode": "latent_direct", - } - - -def validate_state(state: dict[str, Any]) -> dict[str, Any]: - if not isinstance(state, dict) or state.get("magic") != STATE_MAGIC: - raise StateValidationError("invalid H3 Continuum state") - if int(state.get("schema_version", -1)) != STATE_SCHEMA_VERSION: - raise StateValidationError( - f"unsupported state schema {state.get('schema_version')}; expected {STATE_SCHEMA_VERSION}" - ) - video, audio = state.get("video_tail"), state.get("audio_tail") - if not torch.is_tensor(video) or video.ndim != 5 or tuple(video.shape[:2]) != (1, 24): - raise StateValidationError("state video_tail must be [1,24,T,H,W]") - if not torch.is_tensor(audio) or audio.ndim != 4 or tuple(audio.shape[:3]) != (1, 32, 2): - raise StateValidationError("state audio_tail must be [1,32,2,T]") - if int(state.get("width", 0)) != int(video.shape[-1]) * 16: - raise StateValidationError("state width does not match video_tail") - if int(state.get("height", 0)) != int(video.shape[-2]) * 16: - raise StateValidationError("state height does not match video_tail") - capacity = int(state.get("capacity_frames", 0)) - if capacity not in (5, 22, 39): - raise StateValidationError("state capacity must be 5, 22, or 39 frames") - expected_video_t = context_slots(capacity) - expected_audio_t = audio_latent_t(capacity) - if int(video.shape[2]) != expected_video_t or int(audio.shape[-1]) != expected_audio_t: - raise StateValidationError( - "state tensor lengths do not match declared capacity: " - f"video T={video.shape[2]} (expected {expected_video_t}), " - f"audio T={audio.shape[-1]} (expected {expected_audio_t})" - ) - source_frames = int(state.get("source_frame_count", 0)) - if not is_valid_frame_count(source_frames): - raise StateValidationError("state source_frame_count is not on the H3 17k+5 grid") - if int(state.get("clip_index", 0)) < 1: - raise StateValidationError("state clip_index must be at least 1") - grid_offset = float(state.get("audio_overhang", 0.0)) - if not (-0.500001 <= grid_offset <= 0.500001): - raise StateValidationError("state signed audio-grid offset is outside [-0.5,0.5]") - if not bool(torch.isfinite(video.float()).all().item()): - raise StateValidationError("state video_tail contains NaN or Inf") - if not bool(torch.isfinite(audio.float()).all().item()): - raise StateValidationError("state audio_tail contains NaN or Inf") +def validate_state(state:dict[str,Any])->dict[str,Any]: + if not isinstance(state,dict) or state.get("magic")!=STATE_MAGIC: raise StateValidationError("invalid H3 Continuum state") + if int(state.get("schema_version",-1))!=STATE_SCHEMA_VERSION: raise StateValidationError(f"unsupported state schema {state.get('schema_version')}; expected {STATE_SCHEMA_VERSION}") + video,audio=state.get("video_tail"),state.get("audio_tail") + if not torch.is_tensor(video) or video.ndim!=5 or tuple(video.shape[:2])!=(1,24): raise StateValidationError("state video_tail must be [1,24,T,H,W]") + if not torch.is_tensor(audio) or audio.ndim!=4 or tuple(audio.shape[:3])!=(1,32,2): raise StateValidationError("state audio_tail must be [1,32,2,T]") + if int(state.get("width",0))!=int(video.shape[-1])*16: raise StateValidationError("state width does not match video_tail") + if int(state.get("height",0))!=int(video.shape[-2])*16: raise StateValidationError("state height does not match video_tail") + capacity=int(state.get("capacity_frames",0)) + if capacity not in (5,22,39): raise StateValidationError("state capacity must be 5, 22, or 39 frames") + expected_video_t=context_slots(capacity); expected_audio_t=audio_latent_t(capacity) + if int(video.shape[2])!=expected_video_t or int(audio.shape[-1])!=expected_audio_t: + raise StateValidationError(f"state tensor lengths do not match declared capacity: video T={video.shape[2]} (expected {expected_video_t}), audio T={audio.shape[-1]} (expected {expected_audio_t})") + source_frames=int(state.get("source_frame_count",0)) + if not is_valid_frame_count(source_frames): raise StateValidationError("state source_frame_count is not on the H3 17k+5 grid") + if int(state.get("clip_index",0))<1: raise StateValidationError("state clip_index must be at least 1") + grid_offset=float(state.get("audio_overhang",0.0)) + if not (-0.500001<=grid_offset<=0.500001): raise StateValidationError("state signed audio-grid offset is outside [-0.5,0.5]") + if not bool(torch.isfinite(video.float()).all().item()): raise StateValidationError("state video_tail contains NaN or Inf") + if not bool(torch.isfinite(audio.float()).all().item()): raise StateValidationError("state audio_tail contains NaN or Inf") return state - -def select_context( - state: dict[str, Any], context_frames: int, *, include_audio: bool, -) -> tuple[torch.Tensor, torch.Tensor | None, float]: - state = validate_state(state) - if int(context_frames) > int(state["capacity_frames"]): - raise StateValidationError( - f"state retains {state['capacity_frames']} frames; {context_frames} requested" - ) - source_frames = int(state["source_frame_count"]) - grid_offset = float(state["audio_overhang"]) +def select_context(state:dict[str,Any],context_frames:int,*,include_audio:bool)->tuple[torch.Tensor,torch.Tensor|None,float]: + state=validate_state(state) + if int(context_frames)>int(state["capacity_frames"]): raise StateValidationError(f"state retains {state['capacity_frames']} frames; {context_frames} requested") + source_frames=int(state["source_frame_count"]); grid_offset=float(state["audio_overhang"]) try: - source_audio_t = audio_latent_t_from_grid_offset(source_frames, grid_offset) - window = make_av_context_window( - source_frames, - source_audio_t, - int(context_frames), - ) - except ValueError as exc: - raise StateValidationError( - "Native Continuity unavailable for this state: " + str(exc) - ) from exc - slots = context_slots(int(context_frames)) - video = state["video_tail"][:, :, -slots:].contiguous() - audio = None + source_audio_t=audio_latent_t_from_grid_offset(source_frames,grid_offset) + window=make_av_context_window(source_frames,source_audio_t,int(context_frames)) + except ValueError as exc: raise StateValidationError("Native Continuity unavailable for this state: "+str(exc)) from exc + slots=context_slots(int(context_frames)); video=state["video_tail"][:,:,-slots:].contiguous(); audio=None if include_audio: - if int(state["audio_tail"].shape[-1]) < window.audio_steps: - raise StateValidationError( - "Native Continuity state does not retain the required source audio-grid tail" - ) - audio = state["audio_tail"][..., -window.audio_steps:].contiguous() - return video, audio, grid_offset + if int(state["audio_tail"].shape[-1]) tuple[Any, ...] | None: - if tensor is None: - return None - value = tensor.detach().to(device="cpu", dtype=torch.float32) - finite = bool(torch.isfinite(value).all().item()) - if not finite: - raise StateValidationError("continuation context contains NaN or Inf") - flat = value.reshape(-1) - if flat.numel() <= 64: - sample = flat - else: - indices = torch.linspace(0, flat.numel() - 1, 64, dtype=torch.long) - sample = flat.index_select(0, indices) - weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32) - return ( - tuple(int(part) for part in tensor.shape), - str(tensor.dtype), - finite, - float(value.mean().item()), - float(value.std(unbiased=False).item()), - float((sample * weights).sum().item()), - ) - - -def context_fingerprint( - video: torch.Tensor, - audio: torch.Tensor | None, -) -> tuple[Any, ...]: - return (_tensor_fingerprint(video), _tensor_fingerprint(audio)) - - -def assert_context_unchanged( - video: torch.Tensor, - audio: torch.Tensor | None, - expected: tuple[Any, ...], -) -> None: - if context_fingerprint(video, audio) != expected: - raise StateValidationError( - "Native Continuity context tensor changed during sampling" - ) +def _tensor_fingerprint(tensor:torch.Tensor|None)->tuple[Any,...]|None: + if tensor is None: return None + value=tensor.detach().to(device="cpu",dtype=torch.float32); finite=bool(torch.isfinite(value).all().item()) + if not finite: raise StateValidationError("continuation context contains NaN or Inf") + flat=value.reshape(-1) + sample=flat if flat.numel()<=64 else flat.index_select(0,torch.linspace(0,flat.numel()-1,64,dtype=torch.long)) + weights=torch.arange(1,sample.numel()+1,dtype=torch.float32) + return (tuple(int(part) for part in tensor.shape),str(tensor.dtype),finite,float(value.mean().item()),float(value.std(unbiased=False).item()),float((sample*weights).sum().item())) +def context_fingerprint(video:torch.Tensor,audio:torch.Tensor|None)->tuple[Any,...]: return (_tensor_fingerprint(video),_tensor_fingerprint(audio)) +def assert_context_unchanged(video:torch.Tensor,audio:torch.Tensor|None,expected:tuple[Any,...])->None: + if context_fingerprint(video,audio)!=expected: raise StateValidationError("Native Continuity context tensor changed during sampling") diff --git a/temporal.py b/temporal.py index ab5697a..35ffbb9 100644 --- a/temporal.py +++ b/temporal.py @@ -7,10 +7,8 @@ from fractions import Fraction from .constants import AUDIO_LATENT_FPS, FPS, FRAME_PER_TOKEN - _AUDIO_TICKS_PER_FRAME = Fraction(int(AUDIO_LATENT_FPS), int(FPS)) - @dataclass(frozen=True, slots=True) class ExtensionShape: context_frames: int @@ -19,12 +17,10 @@ class ExtensionShape: net_new_frames: int video_latent_t: int audio_latent_t: int - @property def actual_extend_seconds(self) -> float: return self.net_new_frames / FPS - @dataclass(frozen=True, slots=True) class AVContextWindow: source_frame_count: int @@ -34,22 +30,18 @@ class AVContextWindow: audio_start_tick: int audio_stop_tick: int audio_grid_offset: float - @property def audio_steps(self) -> int: return self.audio_stop_tick - self.audio_start_tick - def is_valid_frame_count(frame_count: int) -> bool: return frame_count >= 5 and frame_count % 17 == 5 - def align_frame_count_up(frame_count: int) -> int: n = max(5, int(frame_count)) remainder = (n - 5) % 17 return n if remainder == 0 else n + (17 - remainder) - def align_frame_count_nearest(frame_count: int, *, minimum: int = 5) -> int: target = max(int(frame_count), int(minimum), 5) min_valid = align_frame_count_up(minimum) @@ -58,168 +50,91 @@ def align_frame_count_nearest(frame_count: int, *, minimum: int = 5) -> int: high = max(low + 17, min_valid) return low if abs(target - low) < abs(high - target) else high - def video_latent_t(frame_count: int) -> int: if not is_valid_frame_count(frame_count): raise ValueError(f"H3 frame count must satisfy 17k+5; got {frame_count}") return 2 if frame_count <= 5 else ((frame_count - 5) // 17) * 5 + 2 - def pixel_frames_for_latent_t(latent_t: int) -> int: if int(latent_t) < 1: raise ValueError("video latent T must be positive") return sum(FRAME_PER_TOKEN[index % 5] for index in range(int(latent_t))) - def latent_slot_offsets(latent_t: int) -> tuple[int, ...]: if int(latent_t) < 1: raise ValueError("video latent T must be positive") - out: list[int] = [] - cursor = 0 + out=[]; cursor=0 for index in range(int(latent_t)): - out.append(cursor) - cursor += FRAME_PER_TOKEN[index % 5] + out.append(cursor); cursor += FRAME_PER_TOKEN[index % 5] return tuple(out) - def context_slots(context_frames: int) -> int: - target = int(context_frames) - for slots in range(1, 256): - frames = pixel_frames_for_latent_t(slots) - if frames == target: - return slots - if frames > target: - break - raise ValueError( - f"context_frames={context_frames} is not representable; use 5, 22, or 39" - ) - + target=int(context_frames) + for slots in range(1,256): + frames=pixel_frames_for_latent_t(slots) + if frames==target: return slots + if frames>target: break + raise ValueError(f"context_frames={context_frames} is not representable; use 5, 22, or 39") def audio_latent_t(frame_count: int) -> int: return int(round(Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME)) - def audio_grid_offset(frame_count: int, actual_audio_t: int) -> float: - """Signed audio-grid end offset in 40 Hz latent steps. - - Native H3 uses ``round(duration * 40)``. Depending on the valid 17k+5 - frame count, the audio grid can end up to half a latent step before or - after the exact video end. Keeping the sign is required for sample-accurate - continuation placement. - """ - value = Fraction(int(actual_audio_t)) - Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME - value = float(value) - return 0.0 if abs(value) < 1e-9 else value + value=float(value) + return 0.0 if abs(value)<1e-9 else value - -# Backward-compatible public name used by state schema v1. def audio_overhang(frame_count: int, actual_audio_t: int) -> float: return audio_grid_offset(frame_count, actual_audio_t) - def audio_latent_t_from_grid_offset(frame_count: int, grid_offset: float) -> int: - """Recover the actual source audio-grid end without guessing its phase.""" - - exact_end = float(Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME) - candidate = int(round(exact_end + float(grid_offset))) - recovered = audio_grid_offset(int(frame_count), candidate) - if abs(recovered - float(grid_offset)) > 1e-6: - raise ValueError( - "source audio-grid phase cannot be recovered from frame count and signed offset" - ) + exact_end=float(Fraction(int(frame_count))*_AUDIO_TICKS_PER_FRAME) + candidate=int(round(exact_end+float(grid_offset))) + recovered=audio_grid_offset(int(frame_count),candidate) + if abs(recovered-float(grid_offset))>1e-6: + raise ValueError("source audio-grid phase cannot be recovered from frame count and signed offset") return candidate - -def make_av_context_window( - source_frame_count: int, - actual_audio_t: int, - context_frames: int, -) -> AVContextWindow: - """Resolve a raw AV tail on the native 5-step video and 40/24 audio grids.""" - - source_frames = int(source_frame_count) - context = int(context_frames) +def make_av_context_window(source_frame_count:int, actual_audio_t:int, context_frames:int)->AVContextWindow: + source_frames=int(source_frame_count); context=int(context_frames) if not is_valid_frame_count(source_frames): - raise ValueError( - f"source_frame_count={source_frames} is not on the native H3 17k+5 grid" - ) - if context not in (5, 22, 39) or context > source_frames: + raise ValueError(f"source_frame_count={source_frames} is not on the native H3 17k+5 grid") + if context not in (5,22,39) or context>source_frames: raise ValueError(f"unsupported {context}-frame context for {source_frames} source frames") - - source_video_t = video_latent_t(source_frames) - context_video_t = context_slots(context) - video_start = source_video_t - context_video_t - if video_start < 0 or video_start % len(FRAME_PER_TOKEN): - raise ValueError( - "video context tail does not begin at a uniquely recoverable H3 latent-cycle phase" - ) - - audio_stop = int(actual_audio_t) - grid_offset = audio_grid_offset(source_frames, audio_stop) - if not (-0.500001 <= grid_offset <= 0.500001): + source_video_t=video_latent_t(source_frames); context_video_t=context_slots(context) + video_start=source_video_t-context_video_t + if video_start<0 or video_start % len(FRAME_PER_TOKEN): + raise ValueError("video context tail does not begin at a uniquely recoverable H3 latent-cycle phase") + audio_stop=int(actual_audio_t); grid_offset=audio_grid_offset(source_frames,audio_stop) + if not (-0.500001<=grid_offset<=0.500001): raise ValueError(f"source audio-grid offset {grid_offset:.6f} is outside [-0.5,0.5]") - audio_steps = audio_latent_t(context) - audio_start = audio_stop - audio_steps - if audio_start < 0: - raise ValueError("source audio latent is too short for the requested context") + audio_steps=audio_latent_t(context); audio_start=audio_stop-audio_steps + if audio_start<0: raise ValueError("source audio latent is too short for the requested context") + return AVContextWindow(source_frames,context,video_start,source_video_t,audio_start,audio_stop,grid_offset) - return AVContextWindow( - source_frame_count=source_frames, - context_frames=context, - video_start_slot=video_start, - video_stop_slot=source_video_t, - audio_start_tick=audio_start, - audio_stop_tick=audio_stop, - audio_grid_offset=grid_offset, - ) +def make_extension_shape(context_frames:int, extend_seconds:float)->ExtensionShape: + context_frames=int(context_frames); context_slots(context_frames) + requested_new=max(1,int(round(float(extend_seconds)*FPS))) + total=align_frame_count_nearest(context_frames+requested_new,minimum=context_frames+1) + net=total-context_frames + if net<1: raise RuntimeError("computed continuation contains no new frames") + return ExtensionShape(context_frames,requested_new,total,net,video_latent_t(total),audio_latent_t(total)) +def largest_context_capacity(frame_count:int)->int: + available=int(frame_count) + for capacity in (39,22,5): + if available>=capacity: return capacity + raise ValueError(f"at least 5 retained frames are required for a continuation state; got {available}") -def make_extension_shape(context_frames: int, extend_seconds: float) -> ExtensionShape: - context_frames = int(context_frames) - context_slots(context_frames) - requested_new = max(1, int(round(float(extend_seconds) * FPS))) - total = align_frame_count_nearest( - context_frames + requested_new, minimum=context_frames + 1 - ) - net = total - context_frames - if net < 1: - raise RuntimeError("computed continuation contains no new frames") - return ExtensionShape( - context_frames=context_frames, - requested_new_frames=requested_new, - total_frames=total, - net_new_frames=net, - video_latent_t=video_latent_t(total), - audio_latent_t=audio_latent_t(total), - ) - - -def largest_context_capacity(frame_count: int) -> int: - """Largest supported continuation profile contained in ``frame_count``.""" - - available = int(frame_count) - for capacity in (39, 22, 5): - if available >= capacity: - return capacity - raise ValueError( - f"at least 5 retained frames are required for a continuation state; got {available}" - ) - - -def run_temporal_self_test() -> None: - expected = { - 5: (2, (0, 1)), - 22: (7, (0, 1, 5, 9, 13, 17, 18)), - 39: (12, (0, 1, 5, 9, 13, 17, 18, 22, 26, 30, 34, 35)), - } - for frames, (slots, offsets) in expected.items(): - assert context_slots(frames) == slots - assert latent_slot_offsets(slots) == offsets - assert pixel_frames_for_latent_t(slots) == frames - shape = make_extension_shape(22, 5.0) - assert (shape.total_frames, shape.net_new_frames) == (141, 119) - assert (shape.video_latent_t, shape.audio_latent_t) == (42, 235) - window = make_av_context_window(141, 235, 22) - assert (window.video_start_slot, window.video_stop_slot) == (35, 42) - assert (window.audio_start_tick, window.audio_stop_tick) == (198, 235) +def run_temporal_self_test()->None: + expected={5:(2,(0,1)),22:(7,(0,1,5,9,13,17,18)),39:(12,(0,1,5,9,13,17,18,22,26,30,34,35))} + for frames,(slots,offsets) in expected.items(): + assert context_slots(frames)==slots + assert latent_slot_offsets(slots)==offsets + assert pixel_frames_for_latent_t(slots)==frames + shape=make_extension_shape(22,5.0) + assert (shape.total_frames,shape.net_new_frames)==(141,119) + assert (shape.video_latent_t,shape.audio_latent_t)==(42,235) + window=make_av_context_window(141,235,22) + assert (window.video_start_slot,window.video_stop_slot)==(35,42) + assert (window.audio_start_tick,window.audio_stop_tick)==(198,235) diff --git a/tests/test_model_patch.py b/tests/test_model_patch.py index a344275..d477b62 100644 --- a/tests/test_model_patch.py +++ b/tests/test_model_patch.py @@ -1,21 +1,15 @@ from types import SimpleNamespace -import pytest import torch from ComfyUI_H3_Continuum_Join.constants import ( - CONTINUUM_INTEROP_KEY, MARK_AUDIO_CONTEXT, MARK_AUDIO_END_FRAME, MARK_AUDIO_OVERHANG, MARK_CONTEXT_FRAMES, MARK_VIDEO_CONTEXT, ) -from ComfyUI_H3_Continuum_Join.model_patch import ( - _wrapper_factory, - clone_model_for_chunk, - patch_model, -) +from ComfyUI_H3_Continuum_Join.model_patch import _wrapper_factory, patch_model class Executor: @@ -81,46 +75,6 @@ def test_apply_model_wrapper_repairs_payload_before_executor(): assert torch.equal(result["layout"].position_ids[3:4], result["layout"].position_ids[6:7]) -def test_apply_model_wrapper_fails_when_actual_topology_changes(): - video = torch.zeros(1, 24, 1, 1, 1) - audio = torch.zeros(1, 32, 2, 1) - ref = { - "kind": "video_audio", - "latent_t": 1, - "latent_h": 1, - "latent_w": 1, - "ref_audio_t": 1, - "latent": video, - "audio_latent": audio, - MARK_VIDEO_CONTEXT: True, - MARK_CONTEXT_FRAMES: 1, - MARK_AUDIO_CONTEXT: True, - MARK_AUDIO_END_FRAME: 1.0, - MARK_AUDIO_OVERHANG: 0.0, - } - payload = {"layout": _layout(), "keyframes": [], "refs": [ref]} - wrapper = _wrapper_factory(strict=True, debug=False) - options = { - "spectrum_h3_actual": True, - "cond_or_uncond": [0], - "uuids": ["positive"], - } - wrapper( - Executor(), - torch.zeros(1), - transformer_options=options, - minimax_payload=payload, - ) - payload["layout"].position_ids[-1, 1] = 1.0 - with pytest.raises(RuntimeError, match="topology changed"): - wrapper( - Executor(), - torch.zeros(1), - transformer_options=options, - minimax_payload=payload, - ) - - class FakeModelPatcher: def __init__(self): inner_type = type( @@ -168,52 +122,3 @@ def test_patch_model_validates_and_installs_per_model_wrapper(monkeypatch): assert patched.removed[0][1] == "h3_continuum_join.apply_model.v1" assert patched.added[0][1] == "h3_continuum_join.apply_model.v1" assert patched.model_options["h3_continuum_join"]["api_version"] == 1 - - -def test_chunk_model_copy_on_write_and_context_scoped_hint(monkeypatch): - import sys - from types import ModuleType - - extension = ModuleType("comfy.patcher_extension") - extension.WrappersMP = SimpleNamespace(APPLY_MODEL="apply_model") - comfy = sys.modules.get("comfy") or ModuleType("comfy") - monkeypatch.setitem(sys.modules, "comfy", comfy) - monkeypatch.setitem(sys.modules, "comfy.patcher_extension", extension) - - source = FakeModelPatcher() - source.model_options = {"transformer_options": {"existing": "kept"}} - base = patch_model(source, strict=True, debug=False) - first = clone_model_for_chunk( - base, - strict=True, - debug=False, - chunk_index=1, - context_frames=None, - ) - second = clone_model_for_chunk( - base, - strict=True, - debug=False, - chunk_index=2, - context_frames=22, - ) - third = clone_model_for_chunk( - base, - strict=True, - debug=False, - chunk_index=3, - context_frames=22, - ) - - assert source.model_options == {"transformer_options": {"existing": "kept"}} - assert CONTINUUM_INTEROP_KEY not in first.model_options["transformer_options"] - request = second.model_options["transformer_options"][CONTINUUM_INTEROP_KEY] - assert request == { - "api": 1, - "active": True, - "min_actual_prefix_steps": 2, - "chunk_index": 2, - "context_frames": 22, - } - assert request is not third.model_options["transformer_options"][CONTINUUM_INTEROP_KEY] - assert CONTINUUM_INTEROP_KEY not in base.model_options["transformer_options"] diff --git a/tests/test_temporal.py b/tests/test_temporal.py index 54c7832..7b11da2 100644 --- a/tests/test_temporal.py +++ b/tests/test_temporal.py @@ -1,5 +1,3 @@ -import pytest - from ComfyUI_H3_Continuum_Join.temporal import ( align_frame_count_nearest, audio_grid_offset, @@ -7,7 +5,6 @@ from ComfyUI_H3_Continuum_Join.temporal import ( context_slots, latent_slot_offsets, largest_context_capacity, - make_av_context_window, make_extension_shape, pixel_frames_for_latent_t, video_latent_t, @@ -50,23 +47,6 @@ def test_audio_grid_offset_is_signed_and_bounded_on_valid_h3_lengths(): assert any(value > 0 for value in offsets) -def test_native_context_window_uses_source_av_grid_and_cycle_phase(): - balanced = make_av_context_window(141, audio_latent_t(141), 22) - assert (balanced.video_start_slot, balanced.video_stop_slot) == (35, 42) - assert (balanced.audio_start_tick, balanced.audio_stop_tick) == (198, 235) - assert balanced.audio_steps == 37 - - shifted = make_av_context_window(158, audio_latent_t(158), 22) - assert (shifted.video_start_slot, shifted.video_stop_slot) == (40, 47) - assert (shifted.audio_start_tick, shifted.audio_stop_tick) == (226, 263) - assert shifted.audio_grid_offset < 0 - - -def test_native_context_window_rejects_unknown_audio_grid_phase(): - with pytest.raises(ValueError, match="audio-grid offset"): - make_av_context_window(141, audio_latent_t(141) + 2, 22) - - def test_dynamic_state_capacity_for_short_extensions(): assert largest_context_capacity(119) == 39 assert largest_context_capacity(34) == 22 diff --git a/tests/test_v215_ui_serialization.py b/tests/test_v215_ui_serialization.py deleted file mode 100644 index 1af6974..0000000 --- a/tests/test_v215_ui_serialization.py +++ /dev/null @@ -1,133 +0,0 @@ -import json -from pathlib import Path - -from ComfyUI_H3_Continuum_Join.v2.nodes import H3ContinuumSamplerV2 - - -ROOT = Path(__file__).resolve().parents[1] - -SERIALIZED_WIDGET_ORDER = [ - "prompt_mode", - "prompt_script", - "chunks", - "chunk_seconds", - "width", - "height", - "continuity", - "base_seed", - "control_after_generate", - "audio_continuity", - "exact_total_duration", - "diagnostics", - "reroll_from_chunk", - "reroll_nonce", - "strict_compatibility", - "debug", - "seam_correction", - "show_preview", -] - - -def test_v215_frontend_preserves_backend_serialization_structure(): - source = (ROOT / "web" / "h3_continuum_v2.js").read_text(encoding="utf-8") - assert not (ROOT / "web" / "h3_continuum_v214.js").exists() - for forbidden in ( - 'widget.type = "converted-widget"', - 'widget.type = "hidden"', - "node.widgets.splice(", - "node.removeInput(", - "node.addInput(", - "node.removeOutput(", - "node.addOutput(", - ): - assert forbidden not in source - assert "setWidgetCollapsed" in source - assert "placeTransientAfter" in source - assert "serializedBefore.every" in source - assert "slotVisibilityRules" in source - assert "node.removeInput(" not in source - assert "node.addInput(" not in source - assert "node.removeOutput(" not in source - assert "node.addOutput(" not in source - assert "node.widgets.splice(" not in source - assert "ensureSeamCorrectionProxy" in source - assert "Individual Clip Overrides" in source - assert "button.serialize = false" in source - assert 'name: "H3Continuum.V215StableInterface"' in source - assert source.count("nodeType.prototype.onNodeCreated = function") == 1 - assert source.count("nodeType.prototype.onConfigure = function") == 1 - - -def test_v215_backend_widget_order_and_workflow_values_remain_aligned(): - required = H3ContinuumSamplerV2.INPUT_TYPES()["required"] - assert list(required)[6:] == [ - "prompt_mode", - "prompt_script", - "chunks", - "chunk_seconds", - "width", - "height", - "continuity", - "base_seed", - "audio_continuity", - "exact_total_duration", - "diagnostics", - "reroll_from_chunk", - "reroll_nonce", - "strict_compatibility", - "debug", - "seam_correction", - ] - values = [ - "Auto", - "Preserved fallback prompt", - 2, - 5.0, - 1344, - 768, - "Balanced — 22 frames", - 123456789, - "fixed", - True, - True, - "Basic", - 0, - 0, - True, - False, - "Auto", - True, - ] - mapped = dict(zip(SERIALIZED_WIDGET_ORDER, values, strict=True)) - assert mapped["chunks"] == 2 - assert mapped["chunk_seconds"] == 5.0 - assert mapped["width"] == 1344 - assert mapped["height"] == 768 - assert mapped["continuity"] == "Balanced — 22 frames" - assert mapped["base_seed"] == 123456789 - assert mapped["seam_correction"] == "Auto" - - data = json.loads((ROOT / "examples" / "H3_Continuum_V2_3x5s.json").read_text(encoding="utf-8")) - node = next(item for item in data["nodes"] if item["type"] == "H3ContinuumSamplerV2") - legacy_values = node["widgets_values"] - legacy = dict(zip(SERIALIZED_WIDGET_ORDER[:16], legacy_values, strict=True)) - assert legacy["chunks"] == 3 - assert legacy["chunk_seconds"] == 5.0 - assert legacy["width"] == 1344 - assert legacy["height"] == 768 - assert legacy["continuity"] == "Balanced — 22 frames" - assert legacy["base_seed"] == 123456789 - - -def test_all_non_primary_nodes_use_native_comfyui_deprecation(): - source = (ROOT / "nodes.py").read_text(encoding="utf-8") - assert 'if _node_id == "H3ContinuumSamplerV2"' in source - assert "_node_class.DEPRECATED = True" in source - assert 'f"{CATEGORY}/Legacy"' in source - assert 'f"[Legacy] {_display_name}"' in source - - -def test_v215_version_metadata_matches(): - assert 'PACKAGE_VERSION = "2.1.5"' in (ROOT / "version.py").read_text(encoding="utf-8") - assert "version = 2.1.5" in (ROOT / "metadata.ini").read_text(encoding="utf-8") - assert 'version = "2.1.5"' in (ROOT / "pyproject.toml").read_text(encoding="utf-8") diff --git a/tests/test_v217_stable_facade.py b/tests/test_v217_stable_facade.py new file mode 100644 index 0000000..4b73c61 --- /dev/null +++ b/tests/test_v217_stable_facade.py @@ -0,0 +1,100 @@ +from pathlib import Path + +from ComfyUI_H3_Continuum_Join.constants import PROMPT_FORMAT_OPTIONS +from ComfyUI_H3_Continuum_Join.v2.nodes import ( + H3ContinuumAdvanced, + H3ContinuumClipOverrides, + H3ContinuumResult, + H3ContinuumSampler, + H3ContinuumSamplerV2, + NODE_CLASS_MAPPINGS, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_v217_registers_static_facade_and_preserves_legacy_sampler(): + assert NODE_CLASS_MAPPINGS["H3ContinuumSampler"] is H3ContinuumSampler + assert NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"] is H3ContinuumSamplerV2 + assert NODE_CLASS_MAPPINGS["H3ContinuumClipOverrides"] is H3ContinuumClipOverrides + assert NODE_CLASS_MAPPINGS["H3ContinuumAdvanced"] is H3ContinuumAdvanced + assert NODE_CLASS_MAPPINGS["H3ContinuumResult"] is H3ContinuumResult + + +def test_v217_facade_schema_is_static_and_compact(): + schema = H3ContinuumSampler.INPUT_TYPES() + required = schema["required"] + optional = schema["optional"] + + assert "sequence_prompt" in required + assert required["sequence_prompt"][1]["forceInput"] is True + assert "prompt_script" not in required + assert "prompt_overrides" in optional + assert "advanced" in optional + assert not any(name.startswith("clip_") for name in required | optional) + assert set(H3ContinuumSampler.RETURN_NAMES) == {"images", "audio", "result"} + + +def test_v217_pack_nodes_have_no_dynamic_clip_sockets(): + clip_schema = H3ContinuumClipOverrides.INPUT_TYPES()["required"] + assert set(clip_schema) == {"prompt_mode", "override_script"} + assert clip_schema["override_script"][1]["forceInput"] is True + + advanced = H3ContinuumAdvanced.INPUT_TYPES() + assert set(advanced["optional"]) == { + "last_frame", + "session", + "initial_state", + "prompt_plan", + } + + +def test_v217_facade_delegates_to_legacy_core(monkeypatch): + captured = {} + + def fake_run(_self, **kwargs): + captured.update(kwargs) + return "images", "audio", "state", "session", "report" + + monkeypatch.setattr(H3ContinuumSamplerV2, "run", fake_run) + facade = H3ContinuumSampler() + result = facade.run( + model="model", + clip="clip", + video_vae="video_vae", + audio_vae="audio_vae", + sampler="sampler", + sigmas="sigmas", + sequence_prompt="base prompt", + prompt_mode=PROMPT_FORMAT_OPTIONS[0], + chunks=3, + chunk_seconds=5.0, + width=1344, + height=768, + continuity="Balanced - 22 frames", + base_seed=123, + seam_correction="Auto", + ) + + assert result == ( + "images", + "audio", + {"last_state": "state", "session": "session", "report": "report"}, + ) + assert captured["prompt_script"] == "base prompt" + assert captured["sequence_prompt"] == "base prompt" + assert captured["audio_continuity"] is True + assert captured["show_preview"] is True + + +def test_v217_result_pack_expands_legacy_outputs(): + assert H3ContinuumResult().unpack( + {"last_state": "state", "session": "session", "report": "report"} + ) == ("state", "session", "report") + + +def test_v217_has_no_frontend_display_controller(): + assert not (ROOT / "web" / "h3_continuum_v2.js").exists() + version_source = (ROOT / "version.py").read_text(encoding="utf-8") + assert 'PACKAGE_VERSION = "2.1.7"' in version_source diff --git a/tests/test_v2_prompts.py b/tests/test_v2_prompts.py index 0254b67..c0a765c 100644 --- a/tests/test_v2_prompts.py +++ b/tests/test_v2_prompts.py @@ -1,17 +1,12 @@ import pytest from ComfyUI_H3_Continuum_Join.constants import ( - PROMPT_FORMAT_AUTO, - PROMPT_FORMAT_FIXED, PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE, ) from ComfyUI_H3_Continuum_Join.v2.prompts import ( PromptPlanError, - apply_prompt_overrides, - build_sampler_prompt_plan, - detect_prompt_mode, make_prompt_plan, validate_prompt_plan, ) @@ -24,41 +19,6 @@ def test_fixed_prompt_repeats_without_mutation(): assert validate_prompt_plan(plan) is plan -@pytest.mark.parametrize( - ("script", "expected"), - [ - ("one continuous prompt", PROMPT_MODE_FIXED), - ("first\n---\nsecond", PROMPT_MODE_LIST), - ("[0-5s]\nfirst", PROMPT_MODE_TIMELINE), - ("[Chunk 1]\nfirst", PROMPT_MODE_TIMELINE), - ], -) -def test_auto_detects_prompt_format(script, expected): - assert detect_prompt_mode(script) == expected - - -def test_auto_plan_records_detected_format_without_changing_schema_mode(): - plan = make_prompt_plan( - mode=PROMPT_FORMAT_AUTO, - script="first\n---\nsecond", - chunks=2, - chunk_seconds=5, - ) - assert plan["mode"] == PROMPT_MODE_LIST - assert plan["prompts"] == ["first", "second"] - assert plan["notes"][0] == "Auto detected List" - - -def test_simple_fixed_format_maps_to_stable_fixed_plan_mode(): - plan = make_prompt_plan( - mode=PROMPT_FORMAT_FIXED, - script="same", - chunks=2, - chunk_seconds=5, - ) - assert plan["mode"] == PROMPT_MODE_FIXED - - def test_list_prompt_uses_separator_and_repeats_last(): plan = make_prompt_plan( mode=PROMPT_MODE_LIST, @@ -93,77 +53,3 @@ def test_timeline_requires_coverage(): def test_native_h3_chunk_duration_rejects_sub_four_seconds(): with pytest.raises(PromptPlanError): make_prompt_plan(mode=PROMPT_MODE_FIXED, script="x", chunks=1, chunk_seconds=3.9) - - -def test_external_prompt_overrides_only_the_connected_clip_and_rehashes_it(): - plan = make_prompt_plan( - mode=PROMPT_MODE_LIST, - script="first\n---\nsecond\n---\nthird", - chunks=3, - chunk_seconds=5, - ) - updated = apply_prompt_overrides(plan, [None, "external second", None]) - - assert updated["prompts"] == ["first", "external second", "third"] - assert updated["hashes"][0] == plan["hashes"][0] - assert updated["hashes"][1] != plan["hashes"][1] - assert updated["hashes"][2] == plan["hashes"][2] - assert updated["notes"][-1] == "external Clip Prompt input(s): 2" - - -def test_external_prompt_with_same_text_keeps_the_same_hash(): - plan = make_prompt_plan( - mode=PROMPT_MODE_FIXED, - script="same", - chunks=2, - chunk_seconds=5, - ) - updated = apply_prompt_overrides(plan, [None, "same"]) - assert updated["hashes"] == plan["hashes"] - - -def test_external_prompt_rejects_empty_connected_text(): - plan = make_prompt_plan( - mode=PROMPT_MODE_FIXED, - script="fallback", - chunks=1, - chunk_seconds=5, - ) - with pytest.raises(PromptPlanError): - apply_prompt_overrides(plan, [" "]) - - -def test_sequence_prompt_precedes_connected_plan_and_legacy_script(): - connected = make_prompt_plan( - mode=PROMPT_MODE_FIXED, - script="connected plan", - chunks=2, - chunk_seconds=5, - ) - resolved = build_sampler_prompt_plan( - prompt_mode=PROMPT_FORMAT_AUTO, - prompt_script="legacy", - sequence_prompt="first\n---\nsecond", - prompt_plan=connected, - chunks=2, - chunk_seconds=5, - ) - assert resolved["prompts"] == ["first", "second"] - - -def test_connected_plan_precedes_legacy_script_without_sequence_input(): - connected = make_prompt_plan( - mode=PROMPT_MODE_FIXED, - script="connected plan", - chunks=2, - chunk_seconds=5, - ) - resolved = build_sampler_prompt_plan( - prompt_mode=PROMPT_FORMAT_AUTO, - prompt_script="legacy", - sequence_prompt=None, - prompt_plan=connected, - chunks=2, - chunk_seconds=5, - ) - assert resolved is connected diff --git a/tests/test_v2_registration.py b/tests/test_v2_registration.py index 140dd93..fd24306 100644 --- a/tests/test_v2_registration.py +++ b/tests/test_v2_registration.py @@ -1,10 +1,4 @@ -from pathlib import Path - from ComfyUI_H3_Continuum_Join.nodes import NODE_CLASS_MAPPINGS -from ComfyUI_H3_Continuum_Join.constants import ( - PROMPT_FORMAT_OPTIONS, - SEAM_CORRECTION_AUTO, -) def test_v2_nodes_are_registered_without_removing_v1(): @@ -26,32 +20,3 @@ def test_v2_nodes_are_registered_without_removing_v1(): def test_v2_sampler_accepts_connectable_prompt_plan(): schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() assert "prompt_plan" in schema["optional"] - - -def test_v2_sampler_registers_preview_setting_bridge_and_sixteen_prompt_inputs(): - schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() - optional = schema["optional"] - - assert schema["required"]["prompt_mode"][0] == PROMPT_FORMAT_OPTIONS - assert schema["required"]["prompt_mode"][1]["default"] == "Auto" - assert optional["sequence_prompt"][0] == "STRING" - assert optional["sequence_prompt"][1]["forceInput"] is True - assert optional["show_preview"][1]["default"] is True - for index in range(1, 17): - prompt_input = optional[f"clip_{index}_prompt"] - assert prompt_input[0] == "STRING" - assert prompt_input[1]["forceInput"] is True - - -def test_v2_sampler_defaults_to_auto_seam_correction(): - schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() - assert schema["required"]["seam_correction"][1]["default"] == SEAM_CORRECTION_AUTO - - -def test_v2_interface_keeps_sequence_socket_and_stable_override_slots(): - root = Path(__file__).resolve().parents[1] - source = (root / "web" / "h3_continuum_v2.js").read_text(encoding="utf-8") - assert 'const SEQUENCE_PROMPT_INPUT = "sequence_prompt"' in source - assert "syncPromptInputs(node)" in source - assert "node.removeInput(" not in source - assert "node.addInput(" not in source diff --git a/tests/test_v2_sequence_orchestration.py b/tests/test_v2_sequence_orchestration.py index 4e37467..82500fd 100644 --- a/tests/test_v2_sequence_orchestration.py +++ b/tests/test_v2_sequence_orchestration.py @@ -1,159 +1,53 @@ from types import SimpleNamespace - import torch - -from ComfyUI_H3_Continuum_Join.constants import ( - CONTINUUM_INTEROP_KEY, - DIAGNOSTICS_BASIC, - PROMPT_MODE_FIXED, - V2_CONTINUITY_OPTIONS, -) +from ComfyUI_H3_Continuum_Join.constants import CONTINUUM_INTEROP_KEY, DIAGNOSTICS_BASIC, PROMPT_MODE_FIXED, V2_CONTINUITY_OPTIONS from ComfyUI_H3_Continuum_Join.temporal import audio_latent_t, video_latent_t from ComfyUI_H3_Continuum_Join.v2.h3_builder import IdentityAssets from ComfyUI_H3_Continuum_Join.v2.prompts import make_prompt_plan from ComfyUI_H3_Continuum_Join.v2 import sequence - class Nested: - def __init__(self, parts): - self.parts = list(parts) - def unbind(self): - return self.parts - - -def _latent(width, height, frames): - return { - "samples": Nested( - ( - torch.zeros(1, 24, video_latent_t(frames), height // 16, width // 16), - torch.zeros(1, 32, 2, audio_latent_t(frames)), - ) - ) - } - + def __init__(self,parts): self.parts=list(parts) + def unbind(self): return self.parts +def _latent(width,height,frames): + return {"samples":Nested((torch.zeros(1,24,video_latent_t(frames),height//16,width//16),torch.zeros(1,32,2,audio_latent_t(frames))))} class FakeClip: - def __init__(self, events): - self.events = events - def tokenize(self, prompt, images): - self.events.append(("tokenize", prompt, len(images))) - return prompt - def encode_from_tokens_scheduled(self, tokens): - self.events.append(("encode", tokens)) - return [[torch.zeros(1, 1, 2), {}]] - + def __init__(self,events): self.events=events + def tokenize(self,prompt,images): self.events.append(("tokenize",prompt,len(images))); return prompt + def encode_from_tokens_scheduled(self,tokens): self.events.append(("encode",tokens)); return [[torch.zeros(1,1,2),{}]] def test_v2_samples_every_chunk_before_any_decode(monkeypatch): - events = [] - width, height = 96, 64 - fake_model = SimpleNamespace( - model=SimpleNamespace(diffusion_model=SimpleNamespace()), - model_options={}, - wrappers={}, - model_dtype=lambda: torch.bfloat16, - model_size=lambda: 0, - ) - assets = IdentityAssets(None, None, None, None, "none") - - monkeypatch.setattr(sequence, "check_comfy_h3_runtime", lambda: []) - monkeypatch.setattr(sequence, "patch_model", lambda model, strict, debug: model) - def clone_model_for_chunk( - model, - *, - strict, - debug, - chunk_index, - context_frames, - ): - options = dict(model.model_options) - transformer_options = dict(options.get("transformer_options") or {}) - if context_frames is not None: - transformer_options[CONTINUUM_INTEROP_KEY] = { - "api": 1, - "active": True, - "min_actual_prefix_steps": 2, - } - options["transformer_options"] = transformer_options - events.append(("clone", chunk_index, context_frames)) - return SimpleNamespace( - model=model.model, - model_options=options, - wrappers=model.wrappers, - model_dtype=model.model_dtype, - model_size=model.model_size, - ) - - monkeypatch.setattr(sequence, "clone_model_for_chunk", clone_model_for_chunk) - monkeypatch.setattr(sequence, "prepare_identity_assets", lambda *a, **k: assets) - monkeypatch.setattr(sequence, "encode_identity_latents", lambda *a, **k: assets) - monkeypatch.setattr(sequence, "empty_h3_latent", _latent) - monkeypatch.setattr(sequence, "accelerator_summary", lambda model: "accelerators") - + events=[]; width,height=96,64 + fake_model=SimpleNamespace(model=SimpleNamespace(diffusion_model=SimpleNamespace()),model_options={},wrappers={},model_dtype=lambda:torch.bfloat16,model_size=lambda:0) + assets=IdentityAssets(None,None,None,None,"none") + monkeypatch.setattr(sequence,"check_comfy_h3_runtime",lambda:[]) + monkeypatch.setattr(sequence,"patch_model",lambda model,strict,debug:model) + def clone_model_for_chunk(model,*,strict,debug,chunk_index,context_frames): + options=dict(model.model_options); transformer_options=dict(options.get("transformer_options") or {}) + if context_frames is not None: transformer_options[CONTINUUM_INTEROP_KEY]={"api":1,"active":True,"min_actual_prefix_steps":2} + options["transformer_options"]=transformer_options; events.append(("clone",chunk_index,context_frames)) + return SimpleNamespace(model=model.model,model_options=options,wrappers=model.wrappers,model_dtype=model.model_dtype,model_size=model.model_size) + monkeypatch.setattr(sequence,"clone_model_for_chunk",clone_model_for_chunk) + monkeypatch.setattr(sequence,"prepare_identity_assets",lambda *a,**k:assets) + monkeypatch.setattr(sequence,"encode_identity_latents",lambda *a,**k:assets) + monkeypatch.setattr(sequence,"empty_h3_latent",_latent) + monkeypatch.setattr(sequence,"accelerator_summary",lambda model:"accelerators") def sample_chunk(**kwargs): - frames = kwargs["latent"]["samples"].unbind()[0].shape[2] - hint = kwargs["model"].model_options["transformer_options"].get( - CONTINUUM_INTEROP_KEY - ) - events.append(("sample", int(frames), kwargs["seed"], hint)) - return kwargs["latent"] - + frames=kwargs["latent"]["samples"].unbind()[0].shape[2]; hint=kwargs["model"].model_options["transformer_options"].get(CONTINUUM_INTEROP_KEY); events.append(("sample",int(frames),kwargs["seed"],hint)); return kwargs["latent"] def decode_sequence(**kwargs): - events.append(("decode", len(kwargs["entries"]))) - images = torch.zeros(362, 2, 2, 3) - audio = {"waveform": torch.zeros(1, 2, round(362 / 24 * 32000)), "sample_rate": 32000} - return images, audio, [] - - monkeypatch.setattr(sequence, "sample_chunk", sample_chunk) - monkeypatch.setattr(sequence, "decode_sequence", decode_sequence) - - def unexpected_seam_decode(**kwargs): - raise AssertionError("Off must not enter the V2.1 seam decoder") - - monkeypatch.setattr(sequence, "decode_sequence_with_seam", unexpected_seam_decode) - - plan = make_prompt_plan( - mode=PROMPT_MODE_FIXED, - script="continuous shot", - chunks=3, - chunk_seconds=5.0, - ) - images, audio, last_state, session, report = sequence.run_sequence( - model=fake_model, - clip=FakeClip(events), - video_vae=object(), - audio_vae=object(), - sampler=object(), - sigmas=torch.tensor([1.0, 0.0]), - first_frame=None, - last_frame=None, - prompt_plan=plan, - width=width, - height=height, - continuity=V2_CONTINUITY_OPTIONS[1], - base_seed=42, - audio_continuity=True, - exact_total_duration=True, - diagnostics_mode=DIAGNOSTICS_BASIC, - reroll_from_chunk=0, - reroll_nonce=0, - strict_compatibility=True, - debug=False, - ) - - sample_positions = [i for i, event in enumerate(events) if event[0] == "sample"] - decode_position = next(i for i, event in enumerate(events) if event[0] == "decode") - assert len(sample_positions) == 3 - assert max(sample_positions) < decode_position - sample_events = [event for event in events if event[0] == "sample"] + events.append(("decode",len(kwargs["entries"]))); return torch.zeros(362,2,2,3),{"waveform":torch.zeros(1,2,round(362/24*32000)),"sample_rate":32000},[] + monkeypatch.setattr(sequence,"sample_chunk",sample_chunk); monkeypatch.setattr(sequence,"decode_sequence",decode_sequence) + monkeypatch.setattr(sequence,"decode_sequence_with_seam",lambda **kwargs: (_ for _ in ()).throw(AssertionError("Off must not enter the V2.1 seam decoder"))) + plan=make_prompt_plan(mode=PROMPT_MODE_FIXED,script="continuous shot",chunks=3,chunk_seconds=5.0) + images,audio,last_state,session,report=sequence.run_sequence(model=fake_model,clip=FakeClip(events),video_vae=object(),audio_vae=object(),sampler=object(),sigmas=torch.tensor([1.0,0.0]),first_frame=None,last_frame=None,prompt_plan=plan,width=width,height=height,continuity=V2_CONTINUITY_OPTIONS[1],base_seed=42,audio_continuity=True,exact_total_duration=True,diagnostics_mode=DIAGNOSTICS_BASIC,reroll_from_chunk=0,reroll_nonce=0,strict_compatibility=True,debug=False) + sample_positions=[i for i,e in enumerate(events) if e[0]=="sample"]; decode_position=next(i for i,e in enumerate(events) if e[0]=="decode") + assert len(sample_positions)==3 and max(sample_positions) torch.Tensor: - samples = latent["samples"] - if hasattr(samples, "unbind"): - video = list(samples.unbind())[0] - else: - video = samples[0] - images = video_vae.decode(video) - if images.ndim == 5: - images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1]) - if images.ndim != 4: - raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}") - # Long sequences must not leave decoded frames resident on the GPU. ComfyUI - # IMAGE tensors are conventionally CPU float tensors, so transfer each chunk - # immediately and release its raw decode before moving to the next chunk. +def decode_video(video_vae:Any,latent:dict[str,Any])->torch.Tensor: + samples=latent["samples"]; video=list(samples.unbind())[0] if hasattr(samples,"unbind") else samples[0]; images=video_vae.decode(video) + if images.ndim==5: images=images.reshape(-1,images.shape[-3],images.shape[-2],images.shape[-1]) + if images.ndim!=4: raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}") return images.detach().to("cpu").contiguous() - - -def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]: - samples = latent["samples"] - if hasattr(samples, "unbind"): - audio_latent = list(samples.unbind())[-1] - else: - audio_latent = samples[-1] - audio = audio_vae.decode(audio_latent).movedim(-1, 1) - std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 - std[std < 1.0] = 1.0 - audio = (audio / std).detach().to("cpu").contiguous() - sample_rate = getattr( - audio_vae, - "audio_sample_rate_output", - getattr(audio_vae, "audio_sample_rate", 32000), - ) - return {"waveform": audio, "sample_rate": int(sample_rate)} - - -def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor: - plan = validate_plan(plan) - total_frames = int(plan["total_frames"]) - trim_frames = int(plan["trim_frames"]) - if images.shape[0] < total_frames: - raise ValueError( - f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}" - ) +def decode_audio(audio_vae:Any,latent:dict[str,Any])->dict[str,Any]: + samples=latent["samples"]; audio_latent=list(samples.unbind())[-1] if hasattr(samples,"unbind") else samples[-1]; audio=audio_vae.decode(audio_latent).movedim(-1,1); std=torch.std(audio,dim=[1,2],keepdim=True)*5.0; std[std<1.0]=1.0; audio=(audio/std).detach().to("cpu").contiguous(); sample_rate=getattr(audio_vae,"audio_sample_rate_output",getattr(audio_vae,"audio_sample_rate",32000)); return {"waveform":audio,"sample_rate":int(sample_rate)} +def _trim_images(images,plan): + plan=validate_plan(plan); total_frames=int(plan["total_frames"]); trim_frames=int(plan["trim_frames"]) + if images.shape[0] torch.Tensor: - """Return one segment at its exact cumulative 24fps sample interval. - - Segment-local rounding drifts by one sample on common 119-frame Continuum - chunks. Deriving every boundary from cumulative frame counts guarantees that - the assembled audio length is always ``round(total_frames / 24 * rate)``. - """ - - waveform, sample_rate = validate_audio(audio) - trim_samples = int(round(float(trim_frames) / FPS * sample_rate)) - target_start = int(round(float(frame_start) / FPS * sample_rate)) - target_stop = int(round(float(frame_stop) / FPS * sample_rate)) - wanted = target_stop - target_start - if wanted < 0: - raise ValueError("audio timeline interval is negative") - result = waveform[..., trim_samples : trim_samples + wanted] - if result.shape[-1] < wanted: - missing = wanted - int(result.shape[-1]) - if result.shape[-1] > 0: - tail = result[..., -1:].expand(*result.shape[:-1], missing).clone() - result = torch.cat((result, tail), dim=-1) - else: - result = F.pad(result, (0, missing)) +def _slice_audio_for_timeline(audio,*,trim_frames,frame_start,frame_stop): + waveform,sample_rate=validate_audio(audio); trim_samples=int(round(float(trim_frames)/FPS*sample_rate)); target_start=int(round(float(frame_start)/FPS*sample_rate)); target_stop=int(round(float(frame_stop)/FPS*sample_rate)); wanted=target_stop-target_start + if wanted<0: raise ValueError("audio timeline interval is negative") + result=waveform[...,trim_samples:trim_samples+wanted] + if result.shape[-1]0 else F.pad(result,(0,missing)) return result.contiguous() - - -def decode_sequence( - *, - entries: list[dict[str, Any]], - video_vae: Any, - audio_vae: Any, - diagnostics_full: bool, -) -> tuple[torch.Tensor, dict[str, Any], list[str]]: - """Decode and assemble all chunks with bounded transient memory. - - The final IMAGE tensor is preallocated once. Each raw VAE decode is copied - into its destination range and released before the next chunk, avoiding the - repeated full-sequence copies produced by progressive ``torch.cat``. - """ - - if not entries: - raise ValueError("cannot decode an empty Continuum sequence") - checked = [validate_chunk_entry(entry) for entry in entries] - plans = [validate_plan(entry["plan"]) for entry in checked] - total_retained_frames = sum(int(plan["net_frames"]) for plan in plans) - if total_retained_frames < 1: - raise ValueError("Continuum sequence contains no retained frames") - - image_buffer: torch.Tensor | None = None - audio_buffer: torch.Tensor | None = None - audio_rate: int | None = None - frame_cursor = 0 - previous_segment_images: torch.Tensor | None = None - previous_segment_audio: dict[str, Any] | None = None - reports: list[str] = [] - - for index, (entry, plan) in enumerate(zip(checked, plans), start=1): - latent = entry_to_latent(entry) - raw_images = decode_video(video_vae, latent) - raw_audio = decode_audio(audio_vae, latent) - - if diagnostics_full and previous_segment_images is not None: - metrics = overlap_diagnostics( - previous_images=previous_segment_images, - previous_audio=previous_segment_audio, - next_raw_images=raw_images, - next_raw_audio=raw_audio, - context_frames=int(plan["trim_frames"]), - ) - reports.append(format_join_metrics(index, metrics)) - - images = _trim_images(raw_images, plan) - segment_frames = int(images.shape[0]) - if segment_frames != int(plan["net_frames"]): - raise ValueError( - f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}" - ) - frame_stop = frame_cursor + segment_frames - - waveform, sample_rate = validate_audio(raw_audio) - if audio_rate is None: - audio_rate = sample_rate - elif sample_rate != audio_rate: - raise ValueError( - f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}" - ) - segment_waveform = _slice_audio_for_timeline( - raw_audio, - trim_frames=int(plan["trim_frames"]), - frame_start=frame_cursor, - frame_stop=frame_stop, - ) - - if image_buffer is None: - image_buffer = torch.empty( - (total_retained_frames, *images.shape[1:]), - dtype=images.dtype, - device="cpu", - ) - elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]): - raise ValueError( - f"decoded image geometry changed at chunk {index}: " - f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}" - ) +def decode_sequence(*,entries,video_vae,audio_vae,diagnostics_full): + if not entries: raise ValueError("cannot decode an empty Continuum sequence") + checked=[validate_chunk_entry(e) for e in entries]; plans=[validate_plan(e["plan"]) for e in checked]; total_retained_frames=sum(int(p["net_frames"]) for p in plans) + if total_retained_frames<1: raise ValueError("Continuum sequence contains no retained frames") + image_buffer=None; audio_buffer=None; audio_rate=None; frame_cursor=0; previous_segment_images=None; previous_segment_audio=None; reports=[] + for index,(entry,plan) in enumerate(zip(checked,plans),start=1): + latent=entry_to_latent(entry); raw_images=decode_video(video_vae,latent); raw_audio=decode_audio(audio_vae,latent) + if diagnostics_full and previous_segment_images is not None: reports.append(format_join_metrics(index,overlap_diagnostics(previous_images=previous_segment_images,previous_audio=previous_segment_audio,next_raw_images=raw_images,next_raw_audio=raw_audio,context_frames=int(plan["trim_frames"])))) + images=_trim_images(raw_images,plan); segment_frames=int(images.shape[0]) + if segment_frames!=int(plan["net_frames"]): raise ValueError(f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}") + frame_stop=frame_cursor+segment_frames; waveform,sample_rate=validate_audio(raw_audio) + if audio_rate is None: audio_rate=sample_rate + elif sample_rate!=audio_rate: raise ValueError(f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}") + segment_waveform=_slice_audio_for_timeline(raw_audio,trim_frames=int(plan["trim_frames"]),frame_start=frame_cursor,frame_stop=frame_stop) + if image_buffer is None: image_buffer=torch.empty((total_retained_frames,*images.shape[1:]),dtype=images.dtype,device="cpu") + elif tuple(images.shape[1:])!=tuple(image_buffer.shape[1:]): raise ValueError(f"decoded image geometry changed at chunk {index}: {tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}") image_buffer[frame_cursor:frame_stop].copy_(images) - - if audio_buffer is None: - total_samples = int(round(total_retained_frames / FPS * sample_rate)) - audio_buffer = torch.empty( - (*waveform.shape[:-1], total_samples), - dtype=waveform.dtype, - device="cpu", - ) - elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]): - raise ValueError("decoded audio batch/channel structure changed between chunks") - sample_start = int(round(frame_cursor / FPS * sample_rate)) - sample_stop = int(round(frame_stop / FPS * sample_rate)) - audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform) - - # Retain only what Full diagnostics needs for the next seam. Audio is - # small, while video is bounded to the next overlap instead of a full clip. - next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0 - if next_context > 0: - previous_segment_images = images[-next_context:].contiguous() - previous_segment_audio = { - "waveform": segment_waveform, - "sample_rate": sample_rate, - } - else: - previous_segment_images = None - previous_segment_audio = None - - reports.append( - f"decoded chunk {index}: {segment_frames} retained frames, " - f"cumulative {frame_stop}/{total_retained_frames}" - ) - frame_cursor = frame_stop - del latent, raw_images, raw_audio, images, segment_waveform - - if image_buffer is None or audio_buffer is None or audio_rate is None: - raise RuntimeError("Continuum decoder failed to allocate output buffers") - if frame_cursor != total_retained_frames: - raise RuntimeError( - f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}" - ) - audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate} - reports.append( - f"assembled {len(entries)} chunks into {total_retained_frames} frames with " - "cumulative sample-boundary alignment" - ) - return image_buffer.contiguous(), audio, reports - - -def decode_sequence_with_seam( - *, - entries: list[dict[str, Any]], - video_vae: Any, - audio_vae: Any, - diagnostics_mode: str, - automatic: bool = False, -) -> tuple[torch.Tensor, dict[str, Any], list[str]]: - """Decode with guarded V2.1 correction while preserving legacy output length.""" - - if not entries: - raise ValueError("cannot decode an empty Continuum sequence") - checked = [validate_chunk_entry(entry) for entry in entries] - plans = [validate_plan(entry["plan"]) for entry in checked] - total_retained_frames = sum(int(plan["net_frames"]) for plan in plans) - if total_retained_frames < 1: - raise ValueError("Continuum sequence contains no retained frames") - - image_buffer: torch.Tensor | None = None - audio_buffer: torch.Tensor | None = None - audio_rate: int | None = None - frame_cursor = 0 - reports: list[str] = [] - - for index, (entry, plan) in enumerate(zip(checked, plans), start=1): - latent = entry_to_latent(entry) - raw_images = decode_video(video_vae, latent) - raw_audio = decode_audio(audio_vae, latent) - waveform, sample_rate = validate_audio(raw_audio) - if audio_rate is None: - audio_rate = sample_rate - elif sample_rate != audio_rate: - raise ValueError( - f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}" - ) - - legacy_frames = int(plan["net_frames"]) - legacy_frame_stop = frame_cursor + legacy_frames - frame_start = frame_cursor - trim_frames = int(plan["trim_frames"]) - decision: SeamDecision | None = None - video_patch: torch.Tensor | None = None - audio_patch: torch.Tensor | None = None - - if index == 1: - images = _trim_images(raw_images, plan) + if audio_buffer is None: audio_buffer=torch.empty((*waveform.shape[:-1],int(round(total_retained_frames/FPS*sample_rate))),dtype=waveform.dtype,device="cpu") + elif tuple(segment_waveform.shape[:-1])!=tuple(audio_buffer.shape[:-1]): raise ValueError("decoded audio batch/channel structure changed between chunks") + sample_start=int(round(frame_cursor/FPS*sample_rate)); sample_stop=int(round(frame_stop/FPS*sample_rate)); audio_buffer[...,sample_start:sample_stop].copy_(segment_waveform) + next_context=int(plans[index]["trim_frames"]) if index0: previous_segment_images=images[-next_context:].contiguous(); previous_segment_audio={"waveform":segment_waveform,"sample_rate":sample_rate} + else: previous_segment_images=None; previous_segment_audio=None + reports.append(f"decoded chunk {index}: {segment_frames} retained frames, cumulative {frame_stop}/{total_retained_frames}"); frame_cursor=frame_stop + del latent,raw_images,raw_audio,images,segment_waveform + if image_buffer is None or audio_buffer is None or audio_rate is None: raise RuntimeError("Continuum decoder failed to allocate output buffers") + if frame_cursor!=total_retained_frames: raise RuntimeError(f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}") + reports.append(f"assembled {len(entries)} chunks into {total_retained_frames} frames with cumulative sample-boundary alignment") + return image_buffer.contiguous(),{"waveform":audio_buffer.contiguous(),"sample_rate":audio_rate},reports +def decode_sequence_with_seam(*,entries,video_vae,audio_vae,diagnostics_mode,automatic=False): + if not entries: raise ValueError("cannot decode an empty Continuum sequence") + checked=[validate_chunk_entry(e) for e in entries]; plans=[validate_plan(e["plan"]) for e in checked]; total_retained_frames=sum(int(p["net_frames"]) for p in plans) + if total_retained_frames<1: raise ValueError("Continuum sequence contains no retained frames") + image_buffer=None; audio_buffer=None; audio_rate=None; frame_cursor=0; reports=[] + for index,(entry,plan) in enumerate(zip(checked,plans),start=1): + latent=entry_to_latent(entry); raw_images=decode_video(video_vae,latent); raw_audio=decode_audio(audio_vae,latent); waveform,sample_rate=validate_audio(raw_audio) + if audio_rate is None: audio_rate=sample_rate + elif sample_rate!=audio_rate: raise ValueError(f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}") + legacy_frames=int(plan["net_frames"]); legacy_frame_stop=frame_cursor+legacy_frames; frame_start=frame_cursor; trim_frames=int(plan["trim_frames"]); decision=None; video_patch=None; audio_patch=None + if index==1: images=_trim_images(raw_images,plan) else: try: - if image_buffer is None or audio_buffer is None: - raise RuntimeError("seam buffers are unavailable") - limited_raw_images = raw_images[: int(plan["total_frames"])] - images, decision, video_patch = correct_seam( - image_buffer[:frame_cursor], - limited_raw_images, - context_frames=trim_frames, - automatic=automatic, - ) - frame_start = frame_cursor - int(decision.cut_rewind_frames) - trim_frames -= int(decision.cut_rewind_frames) - if frame_start < 0 or trim_frames < 0: - raise ValueError("adaptive cut produced a negative boundary") - if frame_start + int(images.shape[0]) != legacy_frame_stop: - raise ValueError("adaptive cut changed the cumulative frame boundary") - - sample_start = int(round(frame_start / FPS * sample_rate)) - cut_sample = int(round(trim_frames / FPS * sample_rate)) - ( - audio_patch, - audio_metrics, - crossfade_samples, - level_gain, - dc_bias, - ) = correct_audio_seam( - audio_buffer[..., :sample_start], - waveform, - sample_rate=sample_rate, - cut_sample=cut_sample, - ) - decision.audio_metrics = audio_metrics - decision.audio_crossfade_samples = crossfade_samples - decision.audio_level_gain = level_gain - decision.audio_dc_bias = dc_bias + if image_buffer is None or audio_buffer is None: raise RuntimeError("seam buffers are unavailable") + images,decision,video_patch=correct_seam(image_buffer[:frame_cursor],raw_images[:int(plan["total_frames"])],context_frames=trim_frames,automatic=automatic); frame_start=frame_cursor-int(decision.cut_rewind_frames); trim_frames-=int(decision.cut_rewind_frames) + if frame_start<0 or trim_frames<0: raise ValueError("adaptive cut produced a negative boundary") + if frame_start+int(images.shape[0])!=legacy_frame_stop: raise ValueError("adaptive cut changed the cumulative frame boundary") + sample_start=int(round(frame_start/FPS*sample_rate)); cut_sample=int(round(trim_frames/FPS*sample_rate)); audio_patch,audio_metrics,crossfade_samples,level_gain,dc_bias=correct_audio_seam(audio_buffer[...,:sample_start],waveform,sample_rate=sample_rate,cut_sample=cut_sample); decision.audio_metrics=audio_metrics; decision.audio_crossfade_samples=crossfade_samples; decision.audio_level_gain=level_gain; decision.audio_dc_bias=dc_bias except Exception as exc: - images = _trim_images(raw_images, plan) - frame_start = frame_cursor - trim_frames = int(plan["trim_frames"]) - video_patch = None - audio_patch = None - decision = SeamDecision( - fallback_reason=f"{type(exc).__name__}: {exc}", - ) - - segment_frames = int(images.shape[0]) - frame_stop = frame_start + segment_frames - if frame_stop != legacy_frame_stop: - raise ValueError( - f"chunk {index} corrected boundary ends at {frame_stop}; " - f"expected {legacy_frame_stop}" - ) - segment_waveform = _slice_audio_for_timeline( - raw_audio, - trim_frames=trim_frames, - frame_start=frame_start, - frame_stop=frame_stop, - ) - - if image_buffer is None: - image_buffer = torch.empty( - (total_retained_frames, *images.shape[1:]), - dtype=images.dtype, - device="cpu", - ) - elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]): - raise ValueError( - f"decoded image geometry changed at chunk {index}: " - f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}" - ) - if video_patch is not None and video_patch.shape[0] > 0: - patch_start = frame_start - int(video_patch.shape[0]) - if patch_start < 0: - raise ValueError("video seam patch starts before the output") + images=_trim_images(raw_images,plan); frame_start=frame_cursor; trim_frames=int(plan["trim_frames"]); video_patch=None; audio_patch=None; decision=SeamDecision(fallback_reason=f"{type(exc).__name__}: {exc}") + segment_frames=int(images.shape[0]); frame_stop=frame_start+segment_frames + if frame_stop!=legacy_frame_stop: raise ValueError(f"chunk {index} corrected boundary ends at {frame_stop}; expected {legacy_frame_stop}") + segment_waveform=_slice_audio_for_timeline(raw_audio,trim_frames=trim_frames,frame_start=frame_start,frame_stop=frame_stop) + if image_buffer is None: image_buffer=torch.empty((total_retained_frames,*images.shape[1:]),dtype=images.dtype,device="cpu") + elif tuple(images.shape[1:])!=tuple(image_buffer.shape[1:]): raise ValueError(f"decoded image geometry changed at chunk {index}: {tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}") + if video_patch is not None and video_patch.shape[0]>0: + patch_start=frame_start-int(video_patch.shape[0]) + if patch_start<0: raise ValueError("video seam patch starts before the output") image_buffer[patch_start:frame_start].copy_(video_patch) image_buffer[frame_start:frame_stop].copy_(images) - - if audio_buffer is None: - total_samples = int(round(total_retained_frames / FPS * sample_rate)) - audio_buffer = torch.empty( - (*waveform.shape[:-1], total_samples), - dtype=waveform.dtype, - device="cpu", - ) - elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]): - raise ValueError("decoded audio batch/channel structure changed between chunks") - sample_start = int(round(frame_start / FPS * sample_rate)) - sample_stop = int(round(frame_stop / FPS * sample_rate)) - if audio_patch is not None and audio_patch.shape[-1] > 0: - patch_start = sample_start - int(audio_patch.shape[-1]) - if patch_start < 0: - raise ValueError("audio seam patch starts before the output") - audio_buffer[..., patch_start:sample_start].copy_(audio_patch) - audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform) - - if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF: - reports.append( - format_seam_decision( - index, - decision, - full=diagnostics_mode == DIAGNOSTICS_FULL, - ) - ) - reports.append( - f"decoded chunk {index}: {segment_frames} retained/corrected frames, " - f"cumulative {frame_stop}/{total_retained_frames}" - ) - frame_cursor = frame_stop - del latent, raw_images, raw_audio, images, segment_waveform - - if image_buffer is None or audio_buffer is None or audio_rate is None: - raise RuntimeError("Continuum seam decoder failed to allocate output buffers") - if frame_cursor != total_retained_frames: - raise RuntimeError( - f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}" - ) - audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate} - reports.append( - f"assembled {len(entries)} chunks into {total_retained_frames} frames with " - f"V2.1 {'Auto' if automatic else 'Basic'} seam correction and " - "cumulative sample-boundary alignment" - ) - return image_buffer.contiguous(), audio, reports - - -def enforce_total_frames( - images: torch.Tensor, - audio: dict[str, Any], - *, - target_frames: int, - preserve_final_frame: bool = False, -) -> tuple[torch.Tensor, dict[str, Any], str]: - target_frames = int(target_frames) - if target_frames < 1: - raise ValueError("target_frames must be positive") - current = int(images.shape[0]) - adjustment = target_frames - current - anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2) - if adjustment < 0: - if anchor_preserved: - # H3's valid temporal grid rarely lands on an exact whole-second total. - # When a user supplied a final FL2VA frame, trimming only from the end - # would discard the very anchor they requested. Remove the small excess - # immediately before the final decoded frame instead. - images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous() - else: - images = images[:target_frames].contiguous() - elif adjustment > 0: - if current == 0: - raise ValueError("cannot pad an empty image sequence") - pad = images[-1:].expand(adjustment, *images.shape[1:]).clone() - images = torch.cat((images, pad), dim=0).contiguous() - - waveform = audio["waveform"] - sample_rate = int(audio["sample_rate"]) - original_samples = int(waveform.shape[-1]) - target_samples = int(round(target_frames / FPS * sample_rate)) - delta_samples = target_samples - original_samples - if delta_samples < 0: - if anchor_preserved and target_samples > 1: - # Preserve the final video-frame interval of audio as well. A short - # 5 ms interpolation at the deletion boundary suppresses a hard click - # without changing the exact output length. - tail_samples = min( - max(1, int(round(sample_rate / FPS))), - target_samples, - original_samples, - ) - head_samples = target_samples - tail_samples - head = waveform[..., :head_samples] - tail = waveform[..., -tail_samples:].clone() - fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples) - if fade > 0: - start = head[..., -1:].to(dtype=tail.dtype) - alpha = torch.linspace( - 0.0, 1.0, fade, dtype=tail.dtype, device=tail.device - ).reshape(*([1] * (tail.ndim - 1)), fade) - tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha - waveform = torch.cat((head, tail), dim=-1).contiguous() - else: - waveform = waveform[..., :target_samples].contiguous() - elif delta_samples > 0: - # Repeat the final sample instead of introducing a zero click. - if waveform.shape[-1] == 0: - waveform = F.pad(waveform, (0, delta_samples)) - else: - tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone() - waveform = torch.cat((waveform, tail), dim=-1).contiguous() - audio = {**audio, "waveform": waveform, "sample_rate": sample_rate} - mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad" - report = ( - f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), " - f"audio samples correction {delta_samples:+d}; {mode}." - ) - return images, audio, report + if audio_buffer is None: audio_buffer=torch.empty((*waveform.shape[:-1],int(round(total_retained_frames/FPS*sample_rate))),dtype=waveform.dtype,device="cpu") + elif tuple(segment_waveform.shape[:-1])!=tuple(audio_buffer.shape[:-1]): raise ValueError("decoded audio batch/channel structure changed between chunks") + sample_start=int(round(frame_start/FPS*sample_rate)); sample_stop=int(round(frame_stop/FPS*sample_rate)) + if audio_patch is not None and audio_patch.shape[-1]>0: + patch_start=sample_start-int(audio_patch.shape[-1]) + if patch_start<0: raise ValueError("audio seam patch starts before the output") + audio_buffer[...,patch_start:sample_start].copy_(audio_patch) + audio_buffer[...,sample_start:sample_stop].copy_(segment_waveform) + if decision is not None and diagnostics_mode!=DIAGNOSTICS_OFF: reports.append(format_seam_decision(index,decision,full=diagnostics_mode==DIAGNOSTICS_FULL)) + reports.append(f"decoded chunk {index}: {segment_frames} retained/corrected frames, cumulative {frame_stop}/{total_retained_frames}"); frame_cursor=frame_stop + del latent,raw_images,raw_audio,images,segment_waveform + if image_buffer is None or audio_buffer is None or audio_rate is None: raise RuntimeError("Continuum seam decoder failed to allocate output buffers") + if frame_cursor!=total_retained_frames: raise RuntimeError(f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}") + reports.append(f"assembled {len(entries)} chunks into {total_retained_frames} frames with V2.1 {'Auto' if automatic else 'Basic'} seam correction and cumulative sample-boundary alignment") + return image_buffer.contiguous(),{"waveform":audio_buffer.contiguous(),"sample_rate":audio_rate},reports +def enforce_total_frames(images,audio,*,target_frames,preserve_final_frame=False): + target_frames=int(target_frames) + if target_frames<1: raise ValueError("target_frames must be positive") + current=int(images.shape[0]); adjustment=target_frames-current; anchor_preserved=bool(preserve_final_frame and adjustment<0 and target_frames>=2) + if adjustment<0: images=torch.cat((images[:target_frames-1],images[-1:]),dim=0).contiguous() if anchor_preserved else images[:target_frames].contiguous() + elif adjustment>0: + if current==0: raise ValueError("cannot pad an empty image sequence") + images=torch.cat((images,images[-1:].expand(adjustment,*images.shape[1:]).clone()),dim=0).contiguous() + waveform=audio["waveform"]; sample_rate=int(audio["sample_rate"]); original_samples=int(waveform.shape[-1]); target_samples=int(round(target_frames/FPS*sample_rate)); delta_samples=target_samples-original_samples + if delta_samples<0: + if anchor_preserved and target_samples>1: + tail_samples=min(max(1,int(round(sample_rate/FPS))),target_samples,original_samples); head_samples=target_samples-tail_samples; head=waveform[...,:head_samples]; tail=waveform[...,-tail_samples:].clone(); fade=min(int(round(sample_rate*0.005)),head_samples,tail_samples) + if fade>0: + start=head[...,-1:].to(dtype=tail.dtype); alpha=torch.linspace(0.0,1.0,fade,dtype=tail.dtype,device=tail.device).reshape(*([1]*(tail.ndim-1)),fade); tail[...,:fade]=start*(1.0-alpha)+tail[...,:fade]*alpha + waveform=torch.cat((head,tail),dim=-1).contiguous() + else: waveform=waveform[...,:target_samples].contiguous() + elif delta_samples>0: + waveform=F.pad(waveform,(0,delta_samples)) if waveform.shape[-1]==0 else torch.cat((waveform,waveform[...,-1:].expand(*waveform.shape[:-1],delta_samples).clone()),dim=-1).contiguous() + audio={**audio,"waveform":waveform,"sample_rate":sample_rate}; mode="final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad"; return images,audio,f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}." diff --git a/v2/diagnostics.py b/v2/diagnostics.py index 2aee5bf..e5b2502 100644 --- a/v2/diagnostics.py +++ b/v2/diagnostics.py @@ -1,131 +1,31 @@ """Lightweight join diagnostics without external vision/audio models.""" - from __future__ import annotations - import math from typing import Any - import torch - from ..constants import FPS from .seam_types import SeamDecision - - -def _audio_waveform(audio: dict[str, Any]) -> tuple[torch.Tensor, int]: - waveform = audio.get("waveform") - sample_rate = int(audio.get("sample_rate", 0)) - if not torch.is_tensor(waveform) or waveform.ndim != 3: - raise ValueError("audio waveform must be [B,C,S]") - if sample_rate <= 0: - raise ValueError("audio sample_rate is invalid") - return waveform.to(dtype=torch.float32), sample_rate - - -def _normalized_correlation(a: torch.Tensor, b: torch.Tensor) -> float: - length = min(int(a.numel()), int(b.numel())) - if length < 8: - return 0.0 - a = a.reshape(-1)[:length].float() - b = b.reshape(-1)[:length].float() - a = a - a.mean() - b = b - b.mean() - denom = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12) - value = float((torch.sum(a * b) / denom).item()) - return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0 - - -def overlap_diagnostics( - *, - previous_images: torch.Tensor, - previous_audio: dict[str, Any], - next_raw_images: torch.Tensor, - next_raw_audio: dict[str, Any], - context_frames: int, -) -> dict[str, float]: - context_frames = int(context_frames) - frame_count = min( - context_frames, - int(previous_images.shape[0]), - int(next_raw_images.shape[0]), - ) - if frame_count <= 0: - return { - "video_overlap_mae": 0.0, - "video_overlap_psnr": 99.0, - "audio_overlap_correlation": 0.0, - "audio_boundary_jump": 0.0, - } - - prev = previous_images[-frame_count:].detach().to(dtype=torch.float32) - nxt = next_raw_images[:frame_count].detach().to(dtype=torch.float32) - mse = float(torch.mean((prev - nxt).square()).item()) - mae = float(torch.mean(torch.abs(prev - nxt)).item()) - psnr = 99.0 if mse <= 1e-12 else max(0.0, -10.0 * math.log10(mse)) - - prev_wave, prev_sr = _audio_waveform(previous_audio) - next_wave, next_sr = _audio_waveform(next_raw_audio) - if prev_sr != next_sr: - raise ValueError("diagnostic audio sample rates do not match") - samples = int(round(frame_count / FPS * prev_sr)) - samples = min(samples, int(prev_wave.shape[-1]), int(next_wave.shape[-1])) - if samples > 0: - correlation = _normalized_correlation(prev_wave[..., -samples:], next_wave[..., :samples]) - else: - correlation = 0.0 - if prev_wave.shape[-1] and next_wave.shape[-1]: - jump = float(torch.mean(torch.abs(prev_wave[..., -1] - next_wave[..., 0])).item()) - else: - jump = 0.0 - return { - "video_overlap_mae": mae, - "video_overlap_psnr": psnr, - "audio_overlap_correlation": correlation, - "audio_boundary_jump": jump, - } - - -def av_duration_error_samples(images: torch.Tensor, audio: dict[str, Any]) -> int: - waveform, sample_rate = _audio_waveform(audio) - expected = int(round(float(images.shape[0]) / FPS * sample_rate)) - return int(waveform.shape[-1]) - expected - - -def format_join_metrics(index: int, metrics: dict[str, float]) -> str: - return ( - f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, " - f"PSNR={metrics['video_overlap_psnr']:.2f}dB, " - f"audio corr={metrics['audio_overlap_correlation']:.4f}, " - f"boundary jump={metrics['audio_boundary_jump']:.6f}" - ) - - -def format_seam_decision(index: int, decision: SeamDecision, *, full: bool) -> str: - fallback = decision.fallback_reason or "none" - if not full: - return ( - f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, " - f"score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, " - f"video blend={decision.video_blend_frames}f, " - f"audio offset={decision.audio_metrics.offset_samples:+d} samples, " - f"fallback={fallback}" - ) - return ( - f"Seam {index-1}->{index}\n" - f" Legacy score : {decision.legacy_score:.6f}\n" - f" Selected cut : -{decision.cut_rewind_frames} frames\n" - f" Cut score : {decision.cut_score:.6f}\n" - f" Corrected score : {decision.corrected_score:.6f}\n" - f" Improvement : {decision.improvement * 100.0:.2f}%\n" - f" Luminance gain : {decision.luma_gain:.4f}\n" - f" Chroma bias : {decision.chroma_bias}\n" - f" Video blend : {decision.video_blend_frames} frames\n" - f" Audio correlation : " - f"{decision.audio_metrics.correlation_before:.4f}" - f"->{decision.audio_metrics.correlation_after:.4f}\n" - f" Audio jump : " - f"{decision.audio_metrics.boundary_jump_before:.6f}" - f"->{decision.audio_metrics.boundary_jump_after:.6f}\n" - f" Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n" - f" Audio crossfade : {decision.audio_crossfade_samples} samples\n" - f" Fallback : {fallback}" - ) +def _audio_waveform(audio:dict[str,Any]): + waveform=audio.get("waveform"); sample_rate=int(audio.get("sample_rate",0)) + if not torch.is_tensor(waveform) or waveform.ndim!=3: raise ValueError("audio waveform must be [B,C,S]") + if sample_rate<=0: raise ValueError("audio sample_rate is invalid") + return waveform.to(dtype=torch.float32),sample_rate +def _normalized_correlation(a,b): + length=min(int(a.numel()),int(b.numel())) + if length<8:return 0.0 + a=a.reshape(-1)[:length].float(); b=b.reshape(-1)[:length].float(); a=a-a.mean(); b=b-b.mean(); denom=torch.sqrt(torch.sum(a.square())*torch.sum(b.square())).clamp_min(1e-12); value=float((torch.sum(a*b)/denom).item()); return max(-1.0,min(1.0,value)) if math.isfinite(value) else 0.0 +def overlap_diagnostics(*,previous_images,previous_audio,next_raw_images,next_raw_audio,context_frames): + frame_count=min(int(context_frames),int(previous_images.shape[0]),int(next_raw_images.shape[0])) + if frame_count<=0:return {"video_overlap_mae":0.0,"video_overlap_psnr":99.0,"audio_overlap_correlation":0.0,"audio_boundary_jump":0.0} + prev=previous_images[-frame_count:].detach().to(dtype=torch.float32); nxt=next_raw_images[:frame_count].detach().to(dtype=torch.float32); mse=float(torch.mean((prev-nxt).square()).item()); mae=float(torch.mean(torch.abs(prev-nxt)).item()); psnr=99.0 if mse<=1e-12 else max(0.0,-10.0*math.log10(mse)) + prev_wave,prev_sr=_audio_waveform(previous_audio); next_wave,next_sr=_audio_waveform(next_raw_audio) + if prev_sr!=next_sr: raise ValueError("diagnostic audio sample rates do not match") + samples=min(int(round(frame_count/FPS*prev_sr)),int(prev_wave.shape[-1]),int(next_wave.shape[-1])); correlation=_normalized_correlation(prev_wave[...,-samples:],next_wave[...,:samples]) if samples>0 else 0.0; jump=float(torch.mean(torch.abs(prev_wave[...,-1]-next_wave[...,0])).item()) if prev_wave.shape[-1] and next_wave.shape[-1] else 0.0 + return {"video_overlap_mae":mae,"video_overlap_psnr":psnr,"audio_overlap_correlation":correlation,"audio_boundary_jump":jump} +def av_duration_error_samples(images,audio): + waveform,sample_rate=_audio_waveform(audio); return int(waveform.shape[-1])-int(round(float(images.shape[0])/FPS*sample_rate)) +def format_join_metrics(index,metrics): return f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, PSNR={metrics['video_overlap_psnr']:.2f}dB, audio corr={metrics['audio_overlap_correlation']:.4f}, boundary jump={metrics['audio_boundary_jump']:.6f}" +def format_seam_decision(index,decision:SeamDecision,*,full:bool): + fallback=decision.fallback_reason or "none" + if not full:return f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, video blend={decision.video_blend_frames}f, audio offset={decision.audio_metrics.offset_samples:+d} samples, fallback={fallback}" + return (f"Seam {index-1}->{index}\n Legacy score : {decision.legacy_score:.6f}\n Selected cut : -{decision.cut_rewind_frames} frames\n Cut score : {decision.cut_score:.6f}\n Corrected score : {decision.corrected_score:.6f}\n Improvement : {decision.improvement*100.0:.2f}%\n Luminance gain : {decision.luma_gain:.4f}\n Chroma bias : {decision.chroma_bias}\n Video blend : {decision.video_blend_frames} frames\n Audio correlation : {decision.audio_metrics.correlation_before:.4f}->{decision.audio_metrics.correlation_after:.4f}\n Audio jump : {decision.audio_metrics.boundary_jump_before:.6f}->{decision.audio_metrics.boundary_jump_after:.6f}\n Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n Audio crossfade : {decision.audio_crossfade_samples} samples\n Fallback : {fallback}") diff --git a/v2/nodes.py b/v2/nodes.py index 28f8af7..0545ea1 100644 --- a/v2/nodes.py +++ b/v2/nodes.py @@ -86,8 +86,8 @@ def _repair_v200_example_widget_shift( class H3ContinuumSamplerV2: DESCRIPTION = ( "Integrated N-chunk MiniMax H3 continuation sampler. Precomputes prompt/image " - "conditioning, isolates accelerator runtime state per chunk, captures latent " - "state before decoding, then decodes and assembles the final AV stream." + "conditioning, isolates accelerator runtime state per chunk, captures " + "latent state before decoding, then decodes and assembles the final AV stream." ) SEARCH_ALIASES = [ "H3 long video sampler", @@ -228,7 +228,7 @@ class H3ContinuumSamplerV2: "BOOLEAN", { "default": True, - "tooltip": "Controlled by Settings > H3 Continuum > Show latent preview.", + "tooltip": "Show the in-progress latent thumbnail for this sampler node.", }, ), **{ @@ -236,7 +236,7 @@ class H3ContinuumSamplerV2: "STRING", { "forceInput": True, - "tooltip": f"Connect Text (Multiline) for Clip {index}.", + "tooltip": f"Optional prompt override for Clip {index}.", }, ) for index in range(1, 17) @@ -360,7 +360,16 @@ class H3ContinuumPromptPlanPreview: def INPUT_TYPES(cls): return { "required": { - "prompt_mode": (PROMPT_MODE_OPTIONS, {"default": PROMPT_MODE_OPTIONS[0]}), + "prompt_mode": ( + PROMPT_FORMAT_OPTIONS, + { + "default": PROMPT_FORMAT_OPTIONS[0], + "tooltip": ( + "Auto detects Fixed text, --- separated List text, or " + "[0-5s] / [Chunk 1] Timeline sections." + ), + }, + ), "prompt_script": ("STRING", {"default": "Prompt", "multiline": True}), "chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}), "chunk_seconds": ( @@ -458,7 +467,311 @@ class H3ContinuumSessionInfo: return session, state, report +class H3ContinuumClipOverrides: + """Build a static prompt-override pack without per-clip sockets.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt_mode": ( + PROMPT_FORMAT_OPTIONS, + { + "default": PROMPT_FORMAT_OPTIONS[0], + "display_name": "Prompt Format", + }, + ), + "override_script": ( + "STRING", + { + "forceInput": True, + "display_name": "Override Script", + "tooltip": ( + "Connect one Text (Multiline). Use ---, [Chunk N], or " + "timeline sections according to Prompt Format." + ), + }, + ), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_CLIP_OVERRIDES",) + RETURN_NAMES = ("prompt_overrides",) + FUNCTION = "build" + CATEGORY = CATEGORY + + def build(self, prompt_mode, override_script): + return ( + { + "prompt_mode": str(prompt_mode), + "prompt_script": str(override_script), + }, + ) + + +class H3ContinuumAdvanced: + """Package optional continuation inputs and infrequent sampler settings.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "audio_continuity": ("BOOLEAN", {"default": True}), + "exact_total_duration": ("BOOLEAN", {"default": True}), + "diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}), + "reroll_from_chunk": ( + "INT", + {"default": 0, "min": 0, "max": 16, "step": 1}, + ), + "reroll_nonce": ( + "INT", + {"default": 0, "min": 0, "max": 0xFFFFFFFF, "step": 1}, + ), + "strict_compatibility": ("BOOLEAN", {"default": True}), + "debug": ("BOOLEAN", {"default": False}), + "show_preview": ("BOOLEAN", {"default": True}), + }, + "optional": { + "last_frame": ("IMAGE",), + "session": ("H3_CONTINUUM_SESSION",), + "initial_state": ("H3_CONTINUUM_STATE",), + "prompt_plan": ("H3_CONTINUUM_PROMPT_PLAN",), + }, + } + + RETURN_TYPES = ("H3_CONTINUUM_ADVANCED",) + RETURN_NAMES = ("advanced",) + FUNCTION = "build" + CATEGORY = CATEGORY + + def build( + self, + audio_continuity, + exact_total_duration, + diagnostics, + reroll_from_chunk, + reroll_nonce, + strict_compatibility, + debug, + show_preview, + last_frame=None, + session=None, + initial_state=None, + prompt_plan=None, + ): + return ( + { + "audio_continuity": bool(audio_continuity), + "exact_total_duration": bool(exact_total_duration), + "diagnostics": diagnostics, + "reroll_from_chunk": int(reroll_from_chunk), + "reroll_nonce": int(reroll_nonce), + "strict_compatibility": bool(strict_compatibility), + "debug": bool(debug), + "show_preview": bool(show_preview), + "last_frame": last_frame, + "session": session, + "initial_state": initial_state, + "prompt_plan": prompt_plan, + }, + ) + + +class H3ContinuumSampler: + """Stable, compact facade over the fully compatible V2 sampler core.""" + + DESCRIPTION = ( + "Compact H3 Continuum sampler facade. Uses the legacy V2 execution core " + "without dynamic sockets or frontend display patches." + ) + SEARCH_ALIASES = ["H3 Continuum", "MiniMax H3 long video"] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL",), + "clip": ("CLIP",), + "video_vae": ("VAE",), + "audio_vae": ("VAE",), + "sampler": ("SAMPLER",), + "sigmas": ("SIGMAS",), + "sequence_prompt": ( + "STRING", + { + "forceInput": True, + "display_name": "Sequence Prompt", + "tooltip": "Connect one Text (Multiline) for the complete sequence.", + }, + ), + "prompt_mode": ( + PROMPT_FORMAT_OPTIONS, + { + "default": PROMPT_FORMAT_OPTIONS[0], + "display_name": "Prompt Format", + }, + ), + "chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}), + "chunk_seconds": ( + "FLOAT", + {"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1}, + ), + "width": ("INT", {"default": 1344, "min": 32, "max": 16384, "step": 32}), + "height": ("INT", {"default": 768, "min": 32, "max": 16384, "step": 32}), + "continuity": ( + V2_CONTINUITY_OPTIONS, + {"default": V2_CONTINUITY_OPTIONS[1]}, + ), + "base_seed": ( + "INT", + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + }, + ), + "seam_correction": ( + SEAM_CORRECTION_OPTIONS, + {"default": SEAM_CORRECTION_AUTO}, + ), + }, + "optional": { + "first_frame": ("IMAGE",), + "prompt_overrides": ("H3_CONTINUUM_CLIP_OVERRIDES",), + "advanced": ("H3_CONTINUUM_ADVANCED",), + }, + } + + RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_RESULT") + RETURN_NAMES = ("images", "audio", "result") + FUNCTION = "run" + CATEGORY = CATEGORY + + def run( + self, + model, + clip, + video_vae, + audio_vae, + sampler, + sigmas, + sequence_prompt, + prompt_mode, + chunks, + chunk_seconds, + width, + height, + continuity, + base_seed, + seam_correction, + first_frame=None, + prompt_overrides=None, + advanced=None, + ): + if prompt_overrides is not None and not isinstance(prompt_overrides, dict): + raise TypeError("prompt_overrides must be an H3_CONTINUUM_CLIP_OVERRIDES pack") + if advanced is not None and not isinstance(advanced, dict): + raise TypeError("advanced must be an H3_CONTINUUM_ADVANCED pack") + + advanced_values = { + "audio_continuity": True, + "exact_total_duration": True, + "diagnostics": DIAGNOSTICS_OPTIONS[0], + "reroll_from_chunk": 0, + "reroll_nonce": 0, + "strict_compatibility": True, + "debug": False, + "show_preview": True, + "last_frame": None, + "session": None, + "initial_state": None, + "prompt_plan": None, + } + if advanced: + advanced_values.update(advanced) + + clip_prompt_inputs = {} + if prompt_overrides: + override_plan = make_prompt_plan( + mode=prompt_overrides.get("prompt_mode", PROMPT_FORMAT_OPTIONS[0]), + script=prompt_overrides.get("prompt_script", ""), + chunks=int(chunks), + chunk_seconds=float(chunk_seconds), + ) + for index, prompt in enumerate(override_plan.get("prompts", []), start=1): + if isinstance(prompt, str) and prompt.strip(): + clip_prompt_inputs[f"clip_{index}_prompt"] = prompt + + images, audio, last_state, session, report = H3ContinuumSamplerV2().run( + model=model, + clip=clip, + video_vae=video_vae, + audio_vae=audio_vae, + sampler=sampler, + sigmas=sigmas, + prompt_mode=prompt_mode, + prompt_script=sequence_prompt, + chunks=chunks, + chunk_seconds=chunk_seconds, + width=width, + height=height, + continuity=continuity, + base_seed=base_seed, + audio_continuity=advanced_values["audio_continuity"], + exact_total_duration=advanced_values["exact_total_duration"], + diagnostics=advanced_values["diagnostics"], + reroll_from_chunk=advanced_values["reroll_from_chunk"], + reroll_nonce=advanced_values["reroll_nonce"], + strict_compatibility=advanced_values["strict_compatibility"], + debug=advanced_values["debug"], + seam_correction=seam_correction, + first_frame=first_frame, + last_frame=advanced_values["last_frame"], + session=advanced_values["session"], + initial_state=advanced_values["initial_state"], + prompt_plan=advanced_values["prompt_plan"], + sequence_prompt=sequence_prompt, + show_preview=advanced_values["show_preview"], + **clip_prompt_inputs, + ) + return ( + images, + audio, + { + "last_state": last_state, + "session": session, + "report": report, + }, + ) + + +class H3ContinuumResult: + """Expand the compact facade result pack only when advanced outputs are needed.""" + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"result": ("H3_CONTINUUM_RESULT",)}} + + RETURN_TYPES = ("H3_CONTINUUM_STATE", "H3_CONTINUUM_SESSION", "STRING") + RETURN_NAMES = ("last_state", "session", "report") + FUNCTION = "unpack" + CATEGORY = CATEGORY + + def unpack(self, result): + if not isinstance(result, dict): + raise TypeError("result must be an H3_CONTINUUM_RESULT pack") + missing = {"last_state", "session", "report"} - set(result) + if missing: + raise ValueError("H3_CONTINUUM_RESULT is missing: " + ", ".join(sorted(missing))) + return result["last_state"], result["session"], str(result["report"]) + + NODE_CLASS_MAPPINGS = { + "H3ContinuumSampler": H3ContinuumSampler, + "H3ContinuumClipOverrides": H3ContinuumClipOverrides, + "H3ContinuumAdvanced": H3ContinuumAdvanced, + "H3ContinuumResult": H3ContinuumResult, "H3ContinuumSamplerV2": H3ContinuumSamplerV2, "H3ContinuumPromptPlanPreview": H3ContinuumPromptPlanPreview, "H3ContinuumSaveSession": H3ContinuumSaveSession, @@ -467,6 +780,10 @@ NODE_CLASS_MAPPINGS = { } NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumSampler": "H3 Continuum Sampler", + "H3ContinuumClipOverrides": "H3 Continuum Clip Overrides", + "H3ContinuumAdvanced": "H3 Continuum Advanced", + "H3ContinuumResult": "H3 Continuum Result", "H3ContinuumSamplerV2": "H3 Continuum Sampler V2", "H3ContinuumPromptPlanPreview": "H3 Continuum Prompt Plan", "H3ContinuumSaveSession": "H3 Continuum Save Session", diff --git a/v2/prompts.py b/v2/prompts.py index 7abf4b8..d9e2543 100644 --- a/v2/prompts.py +++ b/v2/prompts.py @@ -3,329 +3,131 @@ The parser is intentionally deterministic and side-effect free so saved workflows can be reproduced without depending on UI state. """ - from __future__ import annotations - -import hashlib -import json -import re +import hashlib, json, re from typing import Any - from ..constants import ( - PROMPT_FORMAT_AUTO, - PROMPT_FORMAT_FIXED, - PROMPT_FORMAT_LIST, - PROMPT_FORMAT_TIMELINE, - PROMPT_MODE_FIXED, - PROMPT_MODE_LIST, - PROMPT_MODE_TIMELINE, - PROMPT_PLAN_MAGIC, + PROMPT_FORMAT_AUTO,PROMPT_FORMAT_FIXED,PROMPT_FORMAT_LIST,PROMPT_FORMAT_TIMELINE, + PROMPT_MODE_FIXED,PROMPT_MODE_LIST,PROMPT_MODE_TIMELINE,PROMPT_PLAN_MAGIC, ) from ..version import PROMPT_PLAN_SCHEMA_VERSION - -_TIMELINE_HEADER = re.compile( - r"^\s*\[\s*(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*" - r"(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$", - re.IGNORECASE, -) -_CHUNK_HEADER = re.compile( - r"^\s*\[\s*(?:chunk|clip)\s*(?P\d+)\s*\]\s*$", - re.IGNORECASE, -) -_LIST_SEPARATOR = re.compile(r"^\s*---+\s*$", re.MULTILINE) - - -class PromptPlanError(ValueError): - pass - - -def prompt_hash(prompt: str) -> str: - return hashlib.sha256(prompt.encode("utf-8")).hexdigest() - - -def _normalize_prompt(value: str, *, label: str) -> str: - text = str(value).strip() - if not text: - raise PromptPlanError(f"{label} is empty") +_TIMELINE_HEADER=re.compile(r"^\s*\[\s*(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$",re.IGNORECASE) +_CHUNK_HEADER=re.compile(r"^\s*\[\s*(?:chunk|clip)\s*(?P\d+)\s*\]\s*$",re.IGNORECASE) +_LIST_SEPARATOR=re.compile(r"^\s*---+\s*$",re.MULTILINE) +class PromptPlanError(ValueError): pass +def prompt_hash(prompt:str)->str: return hashlib.sha256(prompt.encode("utf-8")).hexdigest() +def _normalize_prompt(value:str,*,label:str)->str: + text=str(value).strip() + if not text: raise PromptPlanError(f"{label} is empty") return text - - -def _parse_json_list(script: str) -> list[str] | None: - stripped = script.lstrip() - if not stripped.startswith("["): - return None - try: - value = json.loads(script) - except json.JSONDecodeError: - return None - if not isinstance(value, list) or not all(isinstance(item, str) for item in value): - raise PromptPlanError("JSON prompt list must be an array of strings") - return [_normalize_prompt(item, label=f"prompt {index + 1}") for index, item in enumerate(value)] - - -def _parse_list(script: str, chunks: int) -> tuple[list[str], list[str]]: - values = _parse_json_list(script) - if values is None: - values = [ - _normalize_prompt(part, label=f"prompt {index + 1}") - for index, part in enumerate(_LIST_SEPARATOR.split(script)) - if part.strip() - ] - if not values: - raise PromptPlanError("prompt list contains no prompts") - notes: list[str] = [] - if len(values) < chunks: - notes.append(f"repeated the last prompt for {chunks - len(values)} chunk(s)") - values.extend([values[-1]] * (chunks - len(values))) - elif len(values) > chunks: - notes.append(f"ignored {len(values) - chunks} extra prompt(s)") - values = values[:chunks] - return values, notes - - -def _parse_timeline_sections(script: str) -> list[dict[str, Any]]: - sections: list[dict[str, Any]] = [] - current: dict[str, Any] | None = None - body: list[str] = [] - - def finish() -> None: - nonlocal current, body +def _parse_json_list(script:str): + stripped=script.lstrip() + if not stripped.startswith("["): return None + try: value=json.loads(script) + except json.JSONDecodeError: return None + if not isinstance(value,list) or not all(isinstance(item,str) for item in value): raise PromptPlanError("JSON prompt list must be an array of strings") + return [_normalize_prompt(item,label=f"prompt {index+1}") for index,item in enumerate(value)] +def _parse_list(script,chunks): + values=_parse_json_list(script) + if values is None: values=[_normalize_prompt(part,label=f"prompt {index+1}") for index,part in enumerate(_LIST_SEPARATOR.split(script)) if part.strip()] + if not values: raise PromptPlanError("prompt list contains no prompts") + notes=[] + if len(values)chunks: notes.append(f"ignored {len(values)-chunks} extra prompt(s)"); values=values[:chunks] + return values,notes +def _parse_timeline_sections(script): + sections=[]; current=None; body=[] + def finish(): + nonlocal current,body if current is None: - if any(line.strip() for line in body): - raise PromptPlanError( - "timeline text before the first [0-5s] or [Chunk 1] header is not allowed" - ) - body = [] - return - current["prompt"] = _normalize_prompt("\n".join(body), label="timeline section") - sections.append(current) - current = None - body = [] - + if any(line.strip() for line in body): raise PromptPlanError("timeline text before the first [0-5s] or [Chunk 1] header is not allowed") + body=[]; return + current["prompt"]=_normalize_prompt("\n".join(body),label="timeline section"); sections.append(current); current=None; body=[] for line in str(script).splitlines(): - time_match = _TIMELINE_HEADER.match(line) - chunk_match = _CHUNK_HEADER.match(line) + time_match=_TIMELINE_HEADER.match(line); chunk_match=_CHUNK_HEADER.match(line) if time_match or chunk_match: finish() if time_match: - start = float(time_match.group("start")) - end = float(time_match.group("end")) - if not end > start: - raise PromptPlanError(f"timeline range must increase: {line.strip()}") - current = {"kind": "time", "start": start, "end": end} + start=float(time_match.group("start")); end=float(time_match.group("end")) + if not end>start: raise PromptPlanError(f"timeline range must increase: {line.strip()}") + current={"kind":"time","start":start,"end":end} else: - index = int(chunk_match.group("index")) - if index < 1: - raise PromptPlanError("chunk numbers are one-based") - current = {"kind": "chunk", "index": index} + index=int(chunk_match.group("index")) + if index<1: raise PromptPlanError("chunk numbers are one-based") + current={"kind":"chunk","index":index} continue body.append(line) finish() - if not sections: - raise PromptPlanError("timeline contains no sections") + if not sections: raise PromptPlanError("timeline contains no sections") return sections - - -def _parse_timeline(script: str, chunks: int, chunk_seconds: float) -> tuple[list[str], list[str]]: - sections = _parse_timeline_sections(script) - prompts: list[str] = [] - notes: list[str] = [] - for chunk_index in range(1, chunks + 1): - chunk_sections = [ - item for item in sections if item["kind"] == "chunk" and item["index"] == chunk_index - ] - if len(chunk_sections) > 1: - raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once") - if chunk_sections: - prompts.append(chunk_sections[0]["prompt"]) - continue - - start = (chunk_index - 1) * chunk_seconds - end = chunk_index * chunk_seconds - scored: list[tuple[float, int, dict[str, Any]]] = [] - for order, item in enumerate(sections): - if item["kind"] != "time": - continue - overlap = max(0.0, min(end, item["end"]) - max(start, item["start"])) - if overlap > 0: - scored.append((overlap, -order, item)) - if not scored: - raise PromptPlanError( - f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)" - ) - scored.sort(reverse=True, key=lambda row: (row[0], row[1])) - prompts.append(scored[0][2]["prompt"]) - if len(scored) > 1 and abs(scored[0][0] - scored[1][0]) < 1e-9: - notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section") - return prompts, notes - - -def detect_prompt_mode(script: str) -> str: - """Detect the stable Prompt Plan mode without changing the input text.""" - - text = str(script) +def _parse_timeline(script,chunks,chunk_seconds): + sections=_parse_timeline_sections(script); prompts=[]; notes=[] + for chunk_index in range(1,chunks+1): + chunk_sections=[item for item in sections if item["kind"]=="chunk" and item["index"]==chunk_index] + if len(chunk_sections)>1: raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once") + if chunk_sections: prompts.append(chunk_sections[0]["prompt"]); continue + start=(chunk_index-1)*chunk_seconds; end=chunk_index*chunk_seconds; scored=[] + for order,item in enumerate(sections): + if item["kind"]!="time": continue + overlap=max(0.0,min(end,item["end"])-max(start,item["start"])) + if overlap>0: scored.append((overlap,-order,item)) + if not scored: raise PromptPlanError(f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)") + scored.sort(reverse=True,key=lambda row:(row[0],row[1])); prompts.append(scored[0][2]["prompt"]) + if len(scored)>1 and abs(scored[0][0]-scored[1][0])<1e-9: notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section") + return prompts,notes +def detect_prompt_mode(script): + text=str(script) for line in text.splitlines(): - if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line): - return PROMPT_MODE_TIMELINE - if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text): - return PROMPT_MODE_LIST + if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line): return PROMPT_MODE_TIMELINE + if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text): return PROMPT_MODE_LIST return PROMPT_MODE_FIXED - - -def resolve_prompt_mode(mode: str, script: str) -> str: - explicit = { - PROMPT_FORMAT_FIXED: PROMPT_MODE_FIXED, - PROMPT_FORMAT_LIST: PROMPT_MODE_LIST, - PROMPT_FORMAT_TIMELINE: PROMPT_MODE_TIMELINE, - } - if mode == PROMPT_FORMAT_AUTO: - return detect_prompt_mode(script) - if mode in explicit: - return explicit[mode] - if mode in (PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE): - return mode +def resolve_prompt_mode(mode,script): + explicit={PROMPT_FORMAT_FIXED:PROMPT_MODE_FIXED,PROMPT_FORMAT_LIST:PROMPT_MODE_LIST,PROMPT_FORMAT_TIMELINE:PROMPT_MODE_TIMELINE} + if mode==PROMPT_FORMAT_AUTO: return detect_prompt_mode(script) + if mode in explicit: return explicit[mode] + if mode in (PROMPT_MODE_FIXED,PROMPT_MODE_LIST,PROMPT_MODE_TIMELINE): return mode raise PromptPlanError(f"unknown prompt format: {mode!r}") - - -def make_prompt_plan( - *, mode: str, script: str, chunks: int, chunk_seconds: float, -) -> dict[str, Any]: - chunks = int(chunks) - chunk_seconds = float(chunk_seconds) - if not (1 <= chunks <= 16): - raise PromptPlanError("chunks must be between 1 and 16") - if not (4.0 <= chunk_seconds <= 15.0): - raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3") - - resolved_mode = resolve_prompt_mode(mode, script) - notes: list[str] = [] - if mode == PROMPT_FORMAT_AUTO: - detected = { - PROMPT_MODE_FIXED: PROMPT_FORMAT_FIXED, - PROMPT_MODE_LIST: PROMPT_FORMAT_LIST, - PROMPT_MODE_TIMELINE: PROMPT_FORMAT_TIMELINE, - }[resolved_mode] - notes.append(f"Auto detected {detected}") - - if resolved_mode == PROMPT_MODE_FIXED: - prompt = _normalize_prompt(script, label="fixed prompt") - prompts = [prompt] * chunks - elif resolved_mode == PROMPT_MODE_LIST: - prompts, parse_notes = _parse_list(script, chunks) - notes.extend(parse_notes) - elif resolved_mode == PROMPT_MODE_TIMELINE: - prompts, parse_notes = _parse_timeline(script, chunks, chunk_seconds) - notes.extend(parse_notes) - else: - raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}") - - return { - "magic": PROMPT_PLAN_MAGIC, - "schema_version": PROMPT_PLAN_SCHEMA_VERSION, - "mode": resolved_mode, - "chunks": chunks, - "chunk_seconds": chunk_seconds, - "prompts": prompts, - "hashes": [prompt_hash(prompt) for prompt in prompts], - "notes": notes, - } - - -def validate_prompt_plan(plan: dict[str, Any]) -> dict[str, Any]: - if not isinstance(plan, dict) or plan.get("magic") != PROMPT_PLAN_MAGIC: - raise PromptPlanError("invalid H3 Continuum prompt plan") - if int(plan.get("schema_version", -1)) != PROMPT_PLAN_SCHEMA_VERSION: - raise PromptPlanError( - f"unsupported prompt-plan schema {plan.get('schema_version')}; " - f"expected {PROMPT_PLAN_SCHEMA_VERSION}" - ) - chunks = int(plan.get("chunks", 0)) - prompts = plan.get("prompts") - hashes = plan.get("hashes") - if not (1 <= chunks <= 16): - raise PromptPlanError("prompt-plan chunks are invalid") - if not isinstance(prompts, list) or len(prompts) != chunks: - raise PromptPlanError("prompt-plan prompt count does not match chunks") - if not isinstance(hashes, list) or len(hashes) != chunks: - raise PromptPlanError("prompt-plan hash count does not match chunks") - for index, prompt in enumerate(prompts): - _normalize_prompt(prompt, label=f"prompt {index + 1}") - if hashes[index] != prompt_hash(prompt): - raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index + 1}") +def make_prompt_plan(*,mode,script,chunks,chunk_seconds): + chunks=int(chunks); chunk_seconds=float(chunk_seconds) + if not 1<=chunks<=16: raise PromptPlanError("chunks must be between 1 and 16") + if not 4.0<=chunk_seconds<=15.0: raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3") + resolved_mode=resolve_prompt_mode(mode,script); notes=[] + if mode==PROMPT_FORMAT_AUTO: + detected={PROMPT_MODE_FIXED:PROMPT_FORMAT_FIXED,PROMPT_MODE_LIST:PROMPT_FORMAT_LIST,PROMPT_MODE_TIMELINE:PROMPT_FORMAT_TIMELINE}[resolved_mode]; notes.append(f"Auto detected {detected}") + if resolved_mode==PROMPT_MODE_FIXED: prompt=_normalize_prompt(script,label="fixed prompt"); prompts=[prompt]*chunks + elif resolved_mode==PROMPT_MODE_LIST: prompts,parse_notes=_parse_list(script,chunks); notes.extend(parse_notes) + elif resolved_mode==PROMPT_MODE_TIMELINE: prompts,parse_notes=_parse_timeline(script,chunks,chunk_seconds); notes.extend(parse_notes) + else: raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}") + return {"magic":PROMPT_PLAN_MAGIC,"schema_version":PROMPT_PLAN_SCHEMA_VERSION,"mode":resolved_mode,"chunks":chunks,"chunk_seconds":chunk_seconds,"prompts":prompts,"hashes":[prompt_hash(p) for p in prompts],"notes":notes} +def validate_prompt_plan(plan): + if not isinstance(plan,dict) or plan.get("magic")!=PROMPT_PLAN_MAGIC: raise PromptPlanError("invalid H3 Continuum prompt plan") + if int(plan.get("schema_version",-1))!=PROMPT_PLAN_SCHEMA_VERSION: raise PromptPlanError(f"unsupported prompt-plan schema {plan.get('schema_version')}; expected {PROMPT_PLAN_SCHEMA_VERSION}") + chunks=int(plan.get("chunks",0)); prompts=plan.get("prompts"); hashes=plan.get("hashes") + if not 1<=chunks<=16: raise PromptPlanError("prompt-plan chunks are invalid") + if not isinstance(prompts,list) or len(prompts)!=chunks: raise PromptPlanError("prompt-plan prompt count does not match chunks") + if not isinstance(hashes,list) or len(hashes)!=chunks: raise PromptPlanError("prompt-plan hash count does not match chunks") + for index,prompt in enumerate(prompts): + _normalize_prompt(prompt,label=f"prompt {index+1}") + if hashes[index]!=prompt_hash(prompt): raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index+1}") return plan - - -def build_sampler_prompt_plan( - *, - prompt_mode: str, - prompt_script: str, - sequence_prompt: str | None, - prompt_plan: dict[str, Any] | None, - chunks: int, - chunk_seconds: float, -) -> dict[str, Any]: - """Resolve the fixed Sequence socket while preserving legacy fallbacks.""" - - chunks = int(chunks) - chunk_seconds = float(chunk_seconds) - if sequence_prompt is not None: - return make_prompt_plan( - mode=prompt_mode, - script=sequence_prompt, - chunks=chunks, - chunk_seconds=chunk_seconds, - ) +def build_sampler_prompt_plan(*,prompt_mode,prompt_script,sequence_prompt,prompt_plan,chunks,chunk_seconds): + chunks=int(chunks); chunk_seconds=float(chunk_seconds) + if sequence_prompt is not None: return make_prompt_plan(mode=prompt_mode,script=sequence_prompt,chunks=chunks,chunk_seconds=chunk_seconds) if prompt_plan is not None: - plan = validate_prompt_plan(prompt_plan) - if int(plan["chunks"]) != chunks: - raise PromptPlanError( - "connected prompt_plan chunks do not match the Sampler chunks widget" - ) - if abs(float(plan["chunk_seconds"]) - chunk_seconds) > 1e-6: - raise PromptPlanError( - "connected prompt_plan chunk_seconds do not match the Sampler widget" - ) + plan=validate_prompt_plan(prompt_plan) + if int(plan["chunks"])!=chunks: raise PromptPlanError("connected prompt_plan chunks do not match the Sampler chunks widget") + if abs(float(plan["chunk_seconds"])-chunk_seconds)>1e-6: raise PromptPlanError("connected prompt_plan chunk_seconds do not match the Sampler widget") return plan - return make_prompt_plan( - mode=prompt_mode, - script=prompt_script, - chunks=chunks, - chunk_seconds=chunk_seconds, - ) - - -def apply_prompt_overrides( - plan: dict[str, Any], overrides: list[str | None] -) -> dict[str, Any]: - """Replace selected chunk prompts without changing the prompt-plan schema.""" - - plan = validate_prompt_plan(plan) - prompts = list(plan["prompts"]) - replaced: list[int] = [] - for index, value in enumerate(overrides[: int(plan["chunks"])]): - if value is None: - continue - prompts[index] = _normalize_prompt(value, label=f"Clip {index + 1} Prompt") - replaced.append(index + 1) - if not replaced: - return plan - - result = dict(plan) - result["mode"] = PROMPT_MODE_LIST - result["prompts"] = prompts - result["hashes"] = [prompt_hash(prompt) for prompt in prompts] - result["notes"] = list(plan.get("notes") or []) + [ - "external Clip Prompt input(s): " + ", ".join(map(str, replaced)) - ] + return make_prompt_plan(mode=prompt_mode,script=prompt_script,chunks=chunks,chunk_seconds=chunk_seconds) +def apply_prompt_overrides(plan,overrides): + plan=validate_prompt_plan(plan); prompts=list(plan["prompts"]); replaced=[] + for index,value in enumerate(overrides[:int(plan["chunks"])]): + if value is None: continue + prompts[index]=_normalize_prompt(value,label=f"Clip {index+1} Prompt"); replaced.append(index+1) + if not replaced: return plan + result=dict(plan); result["mode"]=PROMPT_MODE_LIST; result["prompts"]=prompts; result["hashes"]=[prompt_hash(p) for p in prompts]; result["notes"]=list(plan.get("notes") or [])+["external Clip Prompt input(s): "+", ".join(map(str,replaced))] return validate_prompt_plan(result) - - -def prompt_plan_report(plan: dict[str, Any]) -> str: - plan = validate_prompt_plan(plan) - unique = len(set(plan["hashes"])) - note = "; ".join(plan.get("notes") or []) - result = ( - f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; " - f"{unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk." - ) - return result + (" " + note + "." if note else "") +def prompt_plan_report(plan): + plan=validate_prompt_plan(plan); unique=len(set(plan["hashes"])); note="; ".join(plan.get("notes") or []) + result=f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; {unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk." + return result+(" "+note+"." if note else "") diff --git a/v2/seam_guard.py b/v2/seam_guard.py index 3fd9ed7..26151aa 100644 --- a/v2/seam_guard.py +++ b/v2/seam_guard.py @@ -1,551 +1,132 @@ """Conservative CPU-only decoded seam correction for H3 Continuum V2.1.""" - from __future__ import annotations - import math - import torch import torch.nn.functional as F - from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics +COMPARE_SIZE=96; MAX_CUT_REWIND=3; MIN_RELATIVE_IMPROVEMENT=0.10; MIN_ABSOLUTE_IMPROVEMENT=0.003; SAFE_SCORE_LIMIT=0.35 +WEIGHT_PIXEL=0.35; WEIGHT_LUMA=0.20; WEIGHT_CHROMA=0.10; WEIGHT_EDGE=0.15; WEIGHT_TEMPORAL=0.20 -COMPARE_SIZE = 96 -MAX_CUT_REWIND = 3 -MIN_RELATIVE_IMPROVEMENT = 0.10 -MIN_ABSOLUTE_IMPROVEMENT = 0.003 -SAFE_SCORE_LIMIT = 0.35 - -WEIGHT_PIXEL = 0.35 -WEIGHT_LUMA = 0.20 -WEIGHT_CHROMA = 0.10 -WEIGHT_EDGE = 0.15 -WEIGHT_TEMPORAL = 0.20 - - -def _video_shape(frames: torch.Tensor, label: str) -> None: - if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3: - raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]") - - -def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor: - _video_shape(frames, "video seam frames") - rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32) - rgb = rgb.permute(0, 3, 1, 2) - if tuple(rgb.shape[-2:]) != (COMPARE_SIZE, COMPARE_SIZE): - rgb = F.interpolate( - rgb, - size=(COMPARE_SIZE, COMPARE_SIZE), - mode="bilinear", - align_corners=False, - ) - rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0) - if not bool(torch.isfinite(rgb).all().item()): - raise ValueError("video seam comparison contains NaN or Inf") +def _video_shape(frames,label): + if not torch.is_tensor(frames) or frames.ndim!=4 or frames.shape[-1]<3: raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]") +def _downsample_rgb(frames): + _video_shape(frames,"video seam frames"); rgb=frames[...,:3].detach().to(device="cpu",dtype=torch.float32).permute(0,3,1,2) + if tuple(rgb.shape[-2:])!=(COMPARE_SIZE,COMPARE_SIZE): rgb=F.interpolate(rgb,size=(COMPARE_SIZE,COMPARE_SIZE),mode="bilinear",align_corners=False) + rgb=rgb.permute(0,2,3,1).clamp(0.0,1.0) + if not bool(torch.isfinite(rgb).all().item()): raise ValueError("video seam comparison contains NaN or Inf") return rgb - - -def _luma(frame: torch.Tensor) -> torch.Tensor: - weights = frame.new_tensor((0.2126, 0.7152, 0.0722)) - return torch.sum(frame * weights, dim=-1) - - -def _boundary_metrics( - previous_images: torch.Tensor, - next_images: torch.Tensor, -) -> VideoSeamMetrics: - if previous_images.shape[0] < 2 or next_images.shape[0] < 2: - raise ValueError("video seam scoring needs two frames on each side") - prev = _downsample_rgb(previous_images[-2:]) - nxt = _downsample_rgb(next_images[:2]) - prev_before, prev_edge = prev[0], prev[1] - next_edge, next_after = nxt[0], nxt[1] - - pixel = torch.mean(torch.abs(prev_edge - next_edge)).clamp(0.0, 1.0) - prev_y = _luma(prev_edge) - next_y = _luma(next_edge) - luma = torch.mean(torch.abs(prev_y - next_y)).clamp(0.0, 1.0) - prev_chroma = prev_edge - prev_y.unsqueeze(-1) - next_chroma = next_edge - next_y.unsqueeze(-1) - chroma = ( - torch.mean(torch.abs(prev_chroma - next_chroma)) / 2.0 - ).clamp(0.0, 1.0) - - prev_gx = prev_y[:, 1:] - prev_y[:, :-1] - next_gx = next_y[:, 1:] - next_y[:, :-1] - prev_gy = prev_y[1:, :] - prev_y[:-1, :] - next_gy = next_y[1:, :] - next_y[:-1, :] - edge = ( - ( - torch.mean(torch.abs(prev_gx - next_gx)) - + torch.mean(torch.abs(prev_gy - next_gy)) - ) - / 4.0 - ).clamp(0.0, 1.0) - - prev_delta = prev_edge - prev_before - next_delta = next_after - next_edge - temporal = ( - torch.mean(torch.abs(prev_delta - next_delta)) / 2.0 - ).clamp(0.0, 1.0) - motion = max( - float(torch.mean(torch.abs(prev_delta)).item()), - float(torch.mean(torch.abs(next_delta)).item()), - ) - score = ( - WEIGHT_PIXEL * float(pixel.item()) - + WEIGHT_LUMA * float(luma.item()) - + WEIGHT_CHROMA * float(chroma.item()) - + WEIGHT_EDGE * float(edge.item()) - + WEIGHT_TEMPORAL * float(temporal.item()) - ) - if not math.isfinite(score): - raise ValueError("video seam score is not finite") - return VideoSeamMetrics( - pixel_mae_norm=float(pixel.item()), - luma_norm=float(luma.item()), - chroma_norm=float(chroma.item()), - edge_norm=float(edge.item()), - temporal_norm=float(temporal.item()), - motion=motion, - score=max(0.0, min(1.0, score)), - ) - - -def analyze_video_seam( - previous_images: torch.Tensor, - next_raw_images: torch.Tensor, - *, - context_frames: int, - rewind_frames: int = 0, -) -> VideoSeamMetrics: - context_frames = int(context_frames) - rewind_frames = int(rewind_frames) - cut_index = context_frames - rewind_frames - previous_stop = int(previous_images.shape[0]) - rewind_frames - if rewind_frames < 0 or rewind_frames > MAX_CUT_REWIND: - raise ValueError("rewind_frames is outside the V2.1 safe range") - if cut_index < 0 or previous_stop < 2 or cut_index + 2 > next_raw_images.shape[0]: - raise ValueError("insufficient frames for the requested adaptive cut") - return _boundary_metrics( - previous_images[previous_stop - 2 : previous_stop], - next_raw_images[cut_index : cut_index + 2], - ) - - -def choose_adaptive_cut( - previous_images: torch.Tensor, - next_raw_images: torch.Tensor, - *, - context_frames: int, -) -> tuple[int, VideoSeamMetrics, VideoSeamMetrics]: - legacy = analyze_video_seam( - previous_images, - next_raw_images, - context_frames=context_frames, - rewind_frames=0, - ) - best_rewind = 0 - best = legacy - maximum = min( - MAX_CUT_REWIND, - max(0, int(context_frames)), - max(0, int(previous_images.shape[0]) - 2), - ) - for rewind in range(1, maximum + 1): - try: - candidate = analyze_video_seam( - previous_images, - next_raw_images, - context_frames=context_frames, - rewind_frames=rewind, - ) - except ValueError: - continue - if candidate.score < best.score: - best_rewind, best = rewind, candidate - - absolute = legacy.score - best.score - relative = absolute / max(legacy.score, 1e-12) - accepted = ( - best_rewind > 0 - and relative >= MIN_RELATIVE_IMPROVEMENT - and absolute >= MIN_ABSOLUTE_IMPROVEMENT - and best.score <= SAFE_SCORE_LIMIT - ) - return (best_rewind, legacy, best) if accepted else (0, legacy, legacy) - - -def apply_color_guard( - previous_images: torch.Tensor, - current_images: torch.Tensor, -) -> tuple[ - torch.Tensor, - float, - tuple[float, float, float], - VideoSeamMetrics, -]: - before = _boundary_metrics(previous_images[-2:], current_images[:2]) - count = min(4, int(current_images.shape[0])) - if count < 2: - return current_images, 1.0, (0.0, 0.0, 0.0), before - - prev_rgb = _downsample_rgb(previous_images[-2:]) - current_rgb = _downsample_rgb(current_images[:2]) - prev_luma = float(_luma(prev_rgb).mean().item()) - current_luma = float(_luma(current_rgb).mean().item()) - requested_gain = prev_luma / max(current_luma, 1e-6) - if not 0.85 <= requested_gain <= 1.15: - return current_images, 1.0, (0.0, 0.0, 0.0), before - gain = max(0.94, min(1.06, requested_gain)) - - prev_y = _luma(prev_rgb).unsqueeze(-1) - current_y = _luma(current_rgb).unsqueeze(-1) - requested_bias = ( - (prev_rgb - prev_y).mean(dim=(0, 1, 2)) - - (current_rgb - current_y).mean(dim=(0, 1, 2)) - ) - if float(torch.max(torch.abs(requested_bias)).item()) > 0.08: - return current_images, 1.0, (0.0, 0.0, 0.0), before - bias = requested_bias.clamp(-0.02, 0.02) - - result = current_images.clone().contiguous() - original = result[:count].clone() - fade = result.new_tensor((1.0, 0.65, 0.30, 0.0))[:count] - luma_weights = result.new_tensor((0.2126, 0.7152, 0.0722)) +def _luma(frame): return torch.sum(frame*frame.new_tensor((0.2126,0.7152,0.0722)),dim=-1) +def _boundary_metrics(previous_images,next_images): + if previous_images.shape[0]<2 or next_images.shape[0]<2: raise ValueError("video seam scoring needs two frames on each side") + prev=_downsample_rgb(previous_images[-2:]); nxt=_downsample_rgb(next_images[:2]); prev_before,prev_edge=prev[0],prev[1]; next_edge,next_after=nxt[0],nxt[1] + pixel=torch.mean(torch.abs(prev_edge-next_edge)).clamp(0,1); prev_y=_luma(prev_edge); next_y=_luma(next_edge); luma=torch.mean(torch.abs(prev_y-next_y)).clamp(0,1) + prev_chroma=prev_edge-prev_y.unsqueeze(-1); next_chroma=next_edge-next_y.unsqueeze(-1); chroma=(torch.mean(torch.abs(prev_chroma-next_chroma))/2.0).clamp(0,1) + prev_gx=prev_y[:,1:]-prev_y[:,:-1]; next_gx=next_y[:,1:]-next_y[:,:-1]; prev_gy=prev_y[1:,:]-prev_y[:-1,:]; next_gy=next_y[1:,:]-next_y[:-1,:] + edge=((torch.mean(torch.abs(prev_gx-next_gx))+torch.mean(torch.abs(prev_gy-next_gy)))/4.0).clamp(0,1) + prev_delta=prev_edge-prev_before; next_delta=next_after-next_edge; temporal=(torch.mean(torch.abs(prev_delta-next_delta))/2.0).clamp(0,1) + motion=max(float(torch.mean(torch.abs(prev_delta)).item()),float(torch.mean(torch.abs(next_delta)).item())) + score=WEIGHT_PIXEL*float(pixel.item())+WEIGHT_LUMA*float(luma.item())+WEIGHT_CHROMA*float(chroma.item())+WEIGHT_EDGE*float(edge.item())+WEIGHT_TEMPORAL*float(temporal.item()) + if not math.isfinite(score): raise ValueError("video seam score is not finite") + return VideoSeamMetrics(float(pixel.item()),float(luma.item()),float(chroma.item()),float(edge.item()),float(temporal.item()),motion,max(0.0,min(1.0,score))) +def analyze_video_seam(previous_images,next_raw_images,*,context_frames,rewind_frames=0): + context_frames=int(context_frames); rewind_frames=int(rewind_frames); cut_index=context_frames-rewind_frames; previous_stop=int(previous_images.shape[0])-rewind_frames + if rewind_frames<0 or rewind_frames>MAX_CUT_REWIND: raise ValueError("rewind_frames is outside the V2.1 safe range") + if cut_index<0 or previous_stop<2 or cut_index+2>next_raw_images.shape[0]: raise ValueError("insufficient frames for the requested adaptive cut") + return _boundary_metrics(previous_images[previous_stop-2:previous_stop],next_raw_images[cut_index:cut_index+2]) +def choose_adaptive_cut(previous_images,next_raw_images,*,context_frames): + legacy=analyze_video_seam(previous_images,next_raw_images,context_frames=context_frames,rewind_frames=0); best_rewind=0; best=legacy + maximum=min(MAX_CUT_REWIND,max(0,int(context_frames)),max(0,int(previous_images.shape[0])-2)) + for rewind in range(1,maximum+1): + try: candidate=analyze_video_seam(previous_images,next_raw_images,context_frames=context_frames,rewind_frames=rewind) + except ValueError: continue + if candidate.score0 and relative>=MIN_RELATIVE_IMPROVEMENT and absolute>=MIN_ABSOLUTE_IMPROVEMENT and best.score<=SAFE_SCORE_LIMIT + return (best_rewind,legacy,best) if accepted else (0,legacy,legacy) +def apply_color_guard(previous_images,current_images): + before=_boundary_metrics(previous_images[-2:],current_images[:2]); count=min(4,int(current_images.shape[0])) + if count<2: return current_images,1.0,(0.0,0.0,0.0),before + prev_rgb=_downsample_rgb(previous_images[-2:]); current_rgb=_downsample_rgb(current_images[:2]); prev_luma=float(_luma(prev_rgb).mean().item()); current_luma=float(_luma(current_rgb).mean().item()); requested_gain=prev_luma/max(current_luma,1e-6) + if not 0.85<=requested_gain<=1.15: return current_images,1.0,(0.0,0.0,0.0),before + gain=max(0.94,min(1.06,requested_gain)); prev_y=_luma(prev_rgb).unsqueeze(-1); current_y=_luma(current_rgb).unsqueeze(-1); requested_bias=(prev_rgb-prev_y).mean(dim=(0,1,2))-(current_rgb-current_y).mean(dim=(0,1,2)) + if float(torch.max(torch.abs(requested_bias)).item())>0.08: return current_images,1.0,(0.0,0.0,0.0),before + bias=requested_bias.clamp(-0.02,0.02); result=current_images.clone().contiguous(); original=result[:count].clone(); fade=result.new_tensor((1.0,0.65,0.30,0.0))[:count]; luma_weights=result.new_tensor((0.2126,0.7152,0.0722)) for index in range(count): - weight = fade[index] - rgb = result[index, ..., :3] - y = torch.sum(rgb * luma_weights, dim=-1, keepdim=True) - chroma = rgb - y - frame_gain = 1.0 + (gain - 1.0) * weight - frame_bias = bias.to(dtype=rgb.dtype) * weight - result[index, ..., :3] = ( - y * frame_gain + chroma + frame_bias - ).clamp(0.0, 1.0) - - after = _boundary_metrics(previous_images[-2:], result[:2]) - if after.score > before.score * 1.01: - result[:count].copy_(original) - return result, 1.0, (0.0, 0.0, 0.0), before - return ( - result, - gain, - tuple(float(value) for value in bias.tolist()), - after, - ) - - -def choose_video_blend(metrics: VideoSeamMetrics) -> int: - if metrics.motion <= 0.015: - return 2 - if metrics.motion <= 0.040: - return 1 + weight=fade[index]; rgb=result[index,...,:3]; y=torch.sum(rgb*luma_weights,dim=-1,keepdim=True); chroma=rgb-y; frame_gain=1.0+(gain-1.0)*weight; frame_bias=bias.to(dtype=rgb.dtype)*weight; result[index,...,:3]=(y*frame_gain+chroma+frame_bias).clamp(0.0,1.0) + after=_boundary_metrics(previous_images[-2:],result[:2]) + if after.score>before.score*1.01: result[:count].copy_(original); return result,1.0,(0.0,0.0,0.0),before + return result,gain,tuple(float(v) for v in bias.tolist()),after +def choose_video_blend(metrics): + if metrics.motion<=0.015: return 2 + if metrics.motion<=0.040: return 1 return 0 - - -def apply_video_blend( - previous_frames: torch.Tensor, - aligned_next_frames: torch.Tensor, - *, - blend_frames: int, -) -> torch.Tensor: - blend_frames = min( - int(blend_frames), - int(previous_frames.shape[0]), - int(aligned_next_frames.shape[0]), - 2, - ) - if blend_frames <= 0: - return previous_frames[:0].clone() - previous = previous_frames[-blend_frames:].clone() - current = aligned_next_frames[-blend_frames:].to(dtype=previous.dtype) +def apply_video_blend(previous_frames,aligned_next_frames,*,blend_frames): + blend_frames=min(int(blend_frames),int(previous_frames.shape[0]),int(aligned_next_frames.shape[0]),2) + if blend_frames<=0: return previous_frames[:0].clone() + previous=previous_frames[-blend_frames:].clone(); current=aligned_next_frames[-blend_frames:].to(dtype=previous.dtype) for index in range(blend_frames): - progress = float(index + 1) / float(blend_frames + 1) - alpha = 0.5 - 0.5 * math.cos(math.pi * progress) - previous[index].mul_(1.0 - alpha).add_(current[index], alpha=alpha) + progress=float(index+1)/float(blend_frames+1); alpha=0.5-0.5*math.cos(math.pi*progress); previous[index].mul_(1.0-alpha).add_(current[index],alpha=alpha) return previous +def correct_seam(previous_images,next_raw_images,*,context_frames,automatic=False): + rewind,legacy,selected=choose_adaptive_cut(previous_images,next_raw_images,context_frames=context_frames); cut_index=int(context_frames)-rewind; previous_stop=int(previous_images.shape[0])-rewind + if previous_stop<2 or cut_index<0: raise ValueError("adaptive cut would exhaust a seam side") + current=next_raw_images[cut_index:].clone().contiguous(); kept_previous=previous_images[:previous_stop]; current,gain,bias,corrected=apply_color_guard(kept_previous,current) + blend_frames=min(choose_video_blend(selected),cut_index,previous_stop); video_patch=None + if blend_frames>0: + video_patch=apply_video_blend(kept_previous[-blend_frames:],next_raw_images[cut_index-blend_frames:cut_index],blend_frames=blend_frames); score_previous=kept_previous[-2:].clone(); replaced=min(blend_frames,2); score_previous[-replaced:].copy_(video_patch[-replaced:]); corrected=_boundary_metrics(score_previous,current[:2]) + improvement=(legacy.score-corrected.score)/max(legacy.score,1e-12) + if corrected.score>legacy.score*1.01: + return next_raw_images[int(context_frames):].clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=legacy.score,corrected_score=legacy.score,fallback_reason="corrected score was worse than legacy"),None + absolute_improvement=legacy.score-corrected.score + if automatic and (improvement tuple[torch.Tensor, SeamDecision, torch.Tensor | None]: - rewind, legacy, selected = choose_adaptive_cut( - previous_images, - next_raw_images, - context_frames=context_frames, - ) - cut_index = int(context_frames) - rewind - previous_stop = int(previous_images.shape[0]) - rewind - if previous_stop < 2 or cut_index < 0: - raise ValueError("adaptive cut would exhaust a seam side") - - current = next_raw_images[cut_index:].clone().contiguous() - kept_previous = previous_images[:previous_stop] - current, gain, bias, corrected = apply_color_guard(kept_previous, current) - - blend_frames = min( - choose_video_blend(selected), - cut_index, - previous_stop, - ) - video_patch: torch.Tensor | None = None - if blend_frames > 0: - video_patch = apply_video_blend( - kept_previous[-blend_frames:], - next_raw_images[cut_index - blend_frames : cut_index], - blend_frames=blend_frames, - ) - score_previous = kept_previous[-2:].clone() - replaced = min(blend_frames, 2) - score_previous[-replaced:].copy_(video_patch[-replaced:]) - corrected = _boundary_metrics(score_previous, current[:2]) - - improvement = (legacy.score - corrected.score) / max(legacy.score, 1e-12) - if corrected.score > legacy.score * 1.01: - fallback = next_raw_images[int(context_frames) :].clone().contiguous() - return ( - fallback, - SeamDecision( - legacy_score=legacy.score, - cut_score=legacy.score, - corrected_score=legacy.score, - fallback_reason="corrected score was worse than legacy", - ), - None, - ) - - absolute_improvement = legacy.score - corrected.score - if automatic and ( - improvement < MIN_RELATIVE_IMPROVEMENT - or absolute_improvement < MIN_ABSOLUTE_IMPROVEMENT - ): - fallback = next_raw_images[int(context_frames) :].clone().contiguous() - return ( - fallback, - SeamDecision( - legacy_score=legacy.score, - cut_score=selected.score, - corrected_score=legacy.score, - fallback_reason="Auto kept the legacy video seam; improvement was below threshold", - ), - None, - ) - - decision = SeamDecision( - cut_rewind_frames=rewind, - legacy_score=legacy.score, - cut_score=selected.score, - corrected_score=corrected.score, - improvement=improvement, - video_blend_frames=blend_frames, - luma_gain=gain, - chroma_bias=bias, - ) - return current, decision, video_patch - - -def _audio_mid(waveform: torch.Tensor) -> torch.Tensor: - if not torch.is_tensor(waveform) or waveform.ndim != 3: - raise ValueError("audio seam waveform must be [B,C,S]") - value = waveform.detach().to(device="cpu", dtype=torch.float32) - if not bool(torch.isfinite(value).all().item()): - raise ValueError("audio seam waveform contains NaN or Inf") - return value.mean(dim=(0, 1)) - - -def _correlation(a: torch.Tensor, b: torch.Tensor) -> float: - length = min(int(a.numel()), int(b.numel())) - if length < 8: - return 0.0 - a = a[:length] - a[:length].mean() - b = b[:length] - b[:length].mean() - denominator = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12) - value = float((torch.sum(a * b) / denominator).item()) - return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0 - - -def analyze_audio_seam( - previous_waveform: torch.Tensor, - next_raw_waveform: torch.Tensor, - *, - sample_rate: int, - cut_sample: int, -) -> AudioSeamMetrics: - sample_rate = int(sample_rate) - cut_sample = int(cut_sample) - previous = _audio_mid(previous_waveform) - current = _audio_mid(next_raw_waveform) - max_lag = max(1, int(round(sample_rate * 0.020))) - window = min( - max(8, int(round(sample_rate * 0.080))), - int(previous.shape[-1]), - max(0, cut_sample - max_lag), - ) - if sample_rate <= 0 or window < 8: - raise ValueError("insufficient audio for seam alignment") - previous_tail = previous[-window:] - - def correlation_at(lag: int) -> float | None: - stop = cut_sample + int(lag) - start = stop - window - if start < 0 or stop > current.shape[-1]: - return None - return _correlation(previous_tail, current[start:stop]) - - baseline = correlation_at(0) - if baseline is None: - raise ValueError("audio seam cut is outside the decoded waveform") - best_lag, best_correlation = 0, baseline - for lag in range(-max_lag, max_lag + 1): - correlation = correlation_at(lag) - if correlation is not None and correlation > best_correlation: - best_lag, best_correlation = lag, correlation - - if best_correlation < 0.20 or best_correlation - baseline < 0.02: - best_lag, best_correlation = 0, baseline +def _audio_mid(waveform): + if not torch.is_tensor(waveform) or waveform.ndim!=3: raise ValueError("audio seam waveform must be [B,C,S]") + value=waveform.detach().to(device="cpu",dtype=torch.float32) + if not bool(torch.isfinite(value).all().item()): raise ValueError("audio seam waveform contains NaN or Inf") + return value.mean(dim=(0,1)) +def _correlation(a,b): + length=min(int(a.numel()),int(b.numel())) + if length<8: return 0.0 + a=a[:length]-a[:length].mean(); b=b[:length]-b[:length].mean(); denominator=torch.sqrt(torch.sum(a.square())*torch.sum(b.square())).clamp_min(1e-12); value=float((torch.sum(a*b)/denominator).item()); return max(-1.0,min(1.0,value)) if math.isfinite(value) else 0.0 +def analyze_audio_seam(previous_waveform,next_raw_waveform,*,sample_rate,cut_sample): + sample_rate=int(sample_rate); cut_sample=int(cut_sample); previous=_audio_mid(previous_waveform); current=_audio_mid(next_raw_waveform); max_lag=max(1,int(round(sample_rate*0.020))); window=min(max(8,int(round(sample_rate*0.080))),int(previous.shape[-1]),max(0,cut_sample-max_lag)) + if sample_rate<=0 or window<8: raise ValueError("insufficient audio for seam alignment") + previous_tail=previous[-window:] + def correlation_at(lag): + stop=cut_sample+int(lag); start=stop-window + if start<0 or stop>current.shape[-1]: return None + return _correlation(previous_tail,current[start:stop]) + baseline=correlation_at(0) + if baseline is None: raise ValueError("audio seam cut is outside the decoded waveform") + best_lag,best_correlation=0,baseline + for lag in range(-max_lag,max_lag+1): + corr=correlation_at(lag) + if corr is not None and corr>best_correlation: best_lag,best_correlation=lag,corr + if best_correlation<0.20 or best_correlation-baseline<0.02: best_lag,best_correlation=0,baseline else: - center = max(0, min(int(current.shape[-1]) - 1, cut_sample + best_lag)) - radius = max(1, int(round(sample_rate * 0.001))) - start = max(0, center - radius) - stop = min(int(current.shape[-1]), center + radius + 1) - if stop > start: - zero = int(torch.argmin(torch.abs(current[start:stop])).item()) + start - best_lag = zero - cut_sample - - before_jump = float( - torch.mean( - torch.abs(previous_waveform[..., -1] - next_raw_waveform[..., cut_sample]) - ).item() - ) - return AudioSeamMetrics( - correlation_before=baseline, - correlation_after=best_correlation, - boundary_jump_before=before_jump, - boundary_jump_after=before_jump, - offset_samples=int(best_lag), - ) - - -def _audio_crossfade_ms( - previous_waveform: torch.Tensor, - next_raw_waveform: torch.Tensor, - *, - cut_sample: int, - sample_rate: int, -) -> int: - probe = max(8, int(round(sample_rate * 0.020))) - if previous_waveform.shape[-1] < probe or cut_sample < probe: - return 10 - previous = _audio_mid(previous_waveform)[-probe:] - current = _audio_mid(next_raw_waveform)[cut_sample - probe : cut_sample] - rms = torch.sqrt(torch.mean(torch.cat((previous, current)).square())).clamp_min(1e-6) - transient = max( - float(torch.mean(torch.abs(torch.diff(previous))).item()), - float(torch.mean(torch.abs(torch.diff(current))).item()), - ) / float(rms.item()) - if transient > 0.80: - return 10 - if transient > 0.35: - return 20 - if transient > 0.15: - return 40 + center=max(0,min(int(current.shape[-1])-1,cut_sample+best_lag)); radius=max(1,int(round(sample_rate*0.001))); start=max(0,center-radius); stop=min(int(current.shape[-1]),center+radius+1) + if stop>start: best_lag=int(torch.argmin(torch.abs(current[start:stop])).item())+start-cut_sample + before_jump=float(torch.mean(torch.abs(previous_waveform[...,-1]-next_raw_waveform[...,cut_sample])).item()) + return AudioSeamMetrics(baseline,best_correlation,before_jump,before_jump,int(best_lag)) +def _audio_crossfade_ms(previous_waveform,next_raw_waveform,*,cut_sample,sample_rate): + probe=max(8,int(round(sample_rate*0.020))) + if previous_waveform.shape[-1]0.80:return 10 + if transient>0.35:return 20 + if transient>0.15:return 40 return 60 - - -def apply_audio_seam( - previous_waveform: torch.Tensor, - next_raw_waveform: torch.Tensor, - metrics: AudioSeamMetrics, - *, - sample_rate: int, - cut_sample: int, -) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: - if metrics.correlation_after < 0.20: - return None, metrics, 0, 1.0, 0.0 - fade_ms = _audio_crossfade_ms( - previous_waveform, - next_raw_waveform, - cut_sample=cut_sample, - sample_rate=sample_rate, - ) - fade_samples = min( - int(round(sample_rate * fade_ms / 1000.0)), - int(previous_waveform.shape[-1]), - int(cut_sample), - ) - if fade_samples < 2: - return None, metrics, 0, 1.0, 0.0 - - progress = torch.linspace( - 0.0, - 1.0, - fade_samples, - dtype=torch.float32, - device="cpu", - ) - base_indices = torch.arange(cut_sample - fade_samples, cut_sample, dtype=torch.long) - offsets = torch.round( - float(metrics.offset_samples) * (1.0 - progress) - ).to(dtype=torch.long) - indices = (base_indices + offsets).clamp(0, int(next_raw_waveform.shape[-1]) - 1) - current = next_raw_waveform.detach().to("cpu").index_select(-1, indices) - previous = previous_waveform[..., -fade_samples:].detach().to("cpu") - - previous_rms = torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6) - current_rms = torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6) - requested_db = 20.0 * math.log10(float((previous_rms / current_rms).item())) - level_db = max(-1.5, min(1.5, requested_db)) if abs(requested_db) <= 6.0 else 0.0 - level_gain = 10.0 ** (level_db / 20.0) - requested_dc = float((previous.float().mean() - current.float().mean()).item()) - dc_bias = max(-0.02, min(0.02, requested_dc)) if abs(requested_dc) <= 0.10 else 0.0 - current = current * level_gain + dc_bias - - shape = [1] * (previous.ndim - 1) + [fade_samples] - fade_in = torch.sin(progress * (math.pi / 2.0)).reshape(shape) - fade_out = torch.cos(progress * (math.pi / 2.0)).reshape(shape) - patch = previous * fade_out + current.to(dtype=previous.dtype) * fade_in - input_peak = max( - float(previous.abs().max().item()), - float(current.abs().max().item()), - 1e-6, - ) - patch_peak = float(patch.abs().max().item()) - if patch_peak > input_peak * 1.05: - patch = patch * ((input_peak * 1.05) / patch_peak) - after_jump = float( - torch.mean(torch.abs(patch[..., -1] - next_raw_waveform[..., cut_sample])).item() - ) - updated = AudioSeamMetrics( - correlation_before=metrics.correlation_before, - correlation_after=metrics.correlation_after, - boundary_jump_before=metrics.boundary_jump_before, - boundary_jump_after=after_jump, - offset_samples=metrics.offset_samples, - ) - return patch.contiguous(), updated, fade_samples, level_gain, dc_bias - - -def correct_audio_seam( - previous_waveform: torch.Tensor, - next_raw_waveform: torch.Tensor, - *, - sample_rate: int, - cut_sample: int, -) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: - metrics = analyze_audio_seam( - previous_waveform, - next_raw_waveform, - sample_rate=sample_rate, - cut_sample=cut_sample, - ) - return apply_audio_seam( - previous_waveform, - next_raw_waveform, - metrics, - sample_rate=sample_rate, - cut_sample=cut_sample, - ) +def apply_audio_seam(previous_waveform,next_raw_waveform,metrics,*,sample_rate,cut_sample): + if metrics.correlation_after<0.20:return None,metrics,0,1.0,0.0 + fade_ms=_audio_crossfade_ms(previous_waveform,next_raw_waveform,cut_sample=cut_sample,sample_rate=sample_rate); fade_samples=min(int(round(sample_rate*fade_ms/1000.0)),int(previous_waveform.shape[-1]),int(cut_sample)) + if fade_samples<2:return None,metrics,0,1.0,0.0 + progress=torch.linspace(0.0,1.0,fade_samples,dtype=torch.float32,device="cpu"); base_indices=torch.arange(cut_sample-fade_samples,cut_sample,dtype=torch.long); offsets=torch.round(float(metrics.offset_samples)*(1.0-progress)).to(dtype=torch.long); indices=(base_indices+offsets).clamp(0,int(next_raw_waveform.shape[-1])-1) + current=next_raw_waveform.detach().to("cpu").index_select(-1,indices); previous=previous_waveform[...,-fade_samples:].detach().to("cpu"); previous_rms=torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6); current_rms=torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6); requested_db=20.0*math.log10(float((previous_rms/current_rms).item())); level_db=max(-1.5,min(1.5,requested_db)) if abs(requested_db)<=6.0 else 0.0; level_gain=10.0**(level_db/20.0); requested_dc=float((previous.float().mean()-current.float().mean()).item()); dc_bias=max(-0.02,min(0.02,requested_dc)) if abs(requested_dc)<=0.10 else 0.0; current=current*level_gain+dc_bias + shape=[1]*(previous.ndim-1)+[fade_samples]; fade_in=torch.sin(progress*(math.pi/2.0)).reshape(shape); fade_out=torch.cos(progress*(math.pi/2.0)).reshape(shape); patch=previous*fade_out+current.to(dtype=previous.dtype)*fade_in; input_peak=max(float(previous.abs().max().item()),float(current.abs().max().item()),1e-6); patch_peak=float(patch.abs().max().item()) + if patch_peak>input_peak*1.05: patch=patch*((input_peak*1.05)/patch_peak) + after_jump=float(torch.mean(torch.abs(patch[...,-1]-next_raw_waveform[...,cut_sample])).item()); updated=AudioSeamMetrics(metrics.correlation_before,metrics.correlation_after,metrics.boundary_jump_before,after_jump,metrics.offset_samples); return patch.contiguous(),updated,fade_samples,level_gain,dc_bias +def correct_audio_seam(previous_waveform,next_raw_waveform,*,sample_rate,cut_sample): + metrics=analyze_audio_seam(previous_waveform,next_raw_waveform,sample_rate=sample_rate,cut_sample=cut_sample); return apply_audio_seam(previous_waveform,next_raw_waveform,metrics,sample_rate=sample_rate,cut_sample=cut_sample) diff --git a/v2/seam_types.py b/v2/seam_types.py index ead805d..1678186 100644 --- a/v2/seam_types.py +++ b/v2/seam_types.py @@ -1,10 +1,6 @@ """Transient types used by V2.1 decoded seam correction.""" - from __future__ import annotations - from dataclasses import dataclass - - @dataclass(frozen=True) class VideoSeamMetrics: pixel_mae_norm: float @@ -14,8 +10,6 @@ class VideoSeamMetrics: temporal_norm: float motion: float score: float - - @dataclass(frozen=True) class AudioSeamMetrics: correlation_before: float = 0.0 @@ -23,8 +17,6 @@ class AudioSeamMetrics: boundary_jump_before: float = 0.0 boundary_jump_after: float = 0.0 offset_samples: int = 0 - - @dataclass class SeamDecision: cut_rewind_frames: int = 0 @@ -34,7 +26,7 @@ class SeamDecision: improvement: float = 0.0 video_blend_frames: int = 0 luma_gain: float = 1.0 - chroma_bias: tuple[float, float, float] = (0.0, 0.0, 0.0) + chroma_bias: tuple[float,float,float] = (0.0,0.0,0.0) audio_metrics: AudioSeamMetrics = AudioSeamMetrics() audio_crossfade_samples: int = 0 audio_level_gain: float = 1.0 diff --git a/v2/sequence.py b/v2/sequence.py index 41e0c4d..8c7e622 100644 --- a/v2/sequence.py +++ b/v2/sequence.py @@ -5,538 +5,131 @@ orchestration only: all text/image conditioning is prepared first, H3 sampling runs for every chunk without intermediate VAE decoding, and decoding/assembly happens after the sampling phase. """ - from __future__ import annotations - -import copy -import logging +import copy, logging from typing import Any - import torch - from ..compatibility import accelerator_summary, check_comfy_h3_runtime from ..constants import ( - DIAGNOSTICS_FULL, - DIAGNOSTICS_OFF, - DIAGNOSTICS_OPTIONS, - FPS, - CONTINUUM_ACTUAL_PREFIX_STEPS, - SEAM_CORRECTION_AUTO, - SEAM_CORRECTION_OFF, - SEAM_CORRECTION_OPTIONS, - V2_CONTINUITY_AUTO, + DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, DIAGNOSTICS_OPTIONS, FPS, + CONTINUUM_ACTUAL_PREFIX_STEPS, SEAM_CORRECTION_AUTO, SEAM_CORRECTION_OFF, + SEAM_CORRECTION_OPTIONS, V2_CONTINUITY_AUTO, ) from ..continuation import POLICY_REPLACE, prepare_conditioning from ..model_patch import clone_model_for_chunk, patch_model -from ..state import ( - assert_context_unchanged, - context_fingerprint, - make_plan, - select_context, - validate_state, -) -from ..temporal import ( - align_frame_count_up, - largest_context_capacity, - make_extension_shape, -) +from ..state import assert_context_unchanged, context_fingerprint, make_plan, select_context, validate_state +from ..temporal import align_frame_count_up, largest_context_capacity, make_extension_shape from ..version import PACKAGE_VERSION from .decoder import decode_sequence, decode_sequence_with_seam, enforce_total_frames -from .h3_builder import ( - attach_keyframes, - empty_h3_latent, - encode_identity_latents, - encode_prompt_conditioning, - prepare_identity_assets, -) +from .h3_builder import attach_keyframes, empty_h3_latent, encode_identity_latents, encode_prompt_conditioning, prepare_identity_assets from .motion import choose_context_frames from .prompts import prompt_plan_report, validate_prompt_plan from .sampling import sample_chunk from .seeds import derive_chunk_seed from .session import ( - SessionValidationError, - entry_to_state, - make_chunk_entry, - make_session, - model_fingerprint, - session_summary, - validate_chunk_entry, - validate_session, + SessionValidationError, entry_to_state, make_chunk_entry, make_session, model_fingerprint, + session_summary, validate_chunk_entry, validate_session, ) +LOG=logging.getLogger("h3_continuum_join") +class SequenceRuntimeError(RuntimeError): pass -LOG = logging.getLogger("h3_continuum_join") - - -class SequenceRuntimeError(RuntimeError): - pass - - -def _check_decode_memory_budget( - *, width: int, height: int, chunks: int, chunk_seconds: float -) -> tuple[float, float | None]: - """Fail before sampling when the eventual CPU IMAGE output is unsafe. - - ComfyUI IMAGE tensors are normally float32. V2 preallocates the final output - once, but one raw VAE chunk and normal process headroom still coexist with it. - The estimate is conservative and intentionally protects long/high-resolution - jobs from failing only after every expensive H3 chunk has completed. - """ - - target_frames = max(1, int(round(chunks * chunk_seconds * FPS))) - max_chunk_frames = max(5, int(round(chunk_seconds * FPS)) + 39) - image_bytes = target_frames * width * height * 3 * 4 - transient_bytes = max_chunk_frames * width * height * 3 * 4 - required_bytes = image_bytes + int(transient_bytes * 1.5) + 1 * 1024**3 - estimate_gib = required_bytes / 1024**3 - available_gib: float | None = None +def _check_decode_memory_budget(*,width,height,chunks,chunk_seconds): + target_frames=max(1,int(round(chunks*chunk_seconds*FPS))); max_chunk_frames=max(5,int(round(chunk_seconds*FPS))+39) + image_bytes=target_frames*width*height*3*4; transient_bytes=max_chunk_frames*width*height*3*4 + required_bytes=image_bytes+int(transient_bytes*1.5)+1*1024**3; estimate_gib=required_bytes/1024**3; available_gib=None try: import psutil + available=int(psutil.virtual_memory().available); available_gib=available/1024**3 + if required_bytes>int(available*0.75): + raise SequenceRuntimeError(f"estimated decoded output requires about {estimate_gib:.1f} GiB including one-chunk headroom, but only {available_gib:.1f} GiB RAM is currently available. Reduce chunks, chunk_seconds, or resolution before sampling.") + except ImportError: pass + return estimate_gib,available_gib - available = int(psutil.virtual_memory().available) - available_gib = available / 1024**3 - if required_bytes > int(available * 0.75): - raise SequenceRuntimeError( - "estimated decoded output requires about " - f"{estimate_gib:.1f} GiB including one-chunk headroom, but only " - f"{available_gib:.1f} GiB RAM is currently available. Reduce chunks, " - "chunk_seconds, or resolution before sampling." - ) - except ImportError: - pass - return estimate_gib, available_gib +def _clone_entry_for_reuse(entry): + entry=validate_chunk_entry(entry); result=dict(entry); result["plan"]=copy.deepcopy(entry["plan"]); result["reused"]=True; return result - -def _clone_entry_for_reuse(entry: dict[str, Any]) -> dict[str, Any]: - entry = validate_chunk_entry(entry) - result = dict(entry) - result["plan"] = copy.deepcopy(entry["plan"]) - result["reused"] = True - return result - - -def _preserved_prefix( - *, - session: dict[str, Any] | None, - prompt_hashes: list[str], - chunks: int, - reroll_from_chunk: int, - width: int, - height: int, - chunk_seconds: float, - identity_hash: str, -) -> tuple[list[dict[str, Any]], list[str]]: - if session is None: - return [], [] - session = validate_session(session) - notes: list[str] = [] - if int(session["width"]) != int(width) or int(session["height"]) != int(height): - raise SessionValidationError( - f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}" - ) - if abs(float(session["chunk_seconds"]) - float(chunk_seconds)) > 1e-6: - raise SessionValidationError( - f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}" - ) - if str(session.get("identity_hash", "none")) != str(identity_hash): - raise SessionValidationError( - "the current first-frame identity image does not match the saved session" - ) - - if reroll_from_chunk < 0 or reroll_from_chunk > chunks: - raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index") - limit = min(len(session["chunks"]), chunks) - if reroll_from_chunk > 0: - limit = min(limit, reroll_from_chunk - 1) - - preserved: list[dict[str, Any]] = [] +def _preserved_prefix(*,session,prompt_hashes,chunks,reroll_from_chunk,width,height,chunk_seconds,identity_hash): + if session is None: return [],[] + session=validate_session(session); notes=[] + if int(session["width"])!=int(width) or int(session["height"])!=int(height): raise SessionValidationError(f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}") + if abs(float(session["chunk_seconds"])-float(chunk_seconds))>1e-6: raise SessionValidationError(f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}") + if str(session.get("identity_hash","none"))!=str(identity_hash): raise SessionValidationError("the current first-frame identity image does not match the saved session") + if reroll_from_chunk<0 or reroll_from_chunk>chunks: raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index") + limit=min(len(session["chunks"]),chunks) + if reroll_from_chunk>0: limit=min(limit,reroll_from_chunk-1) + preserved=[] for index in range(limit): - entry = validate_chunk_entry(session["chunks"][index]) - if entry["prompt_hash"] != prompt_hashes[index]: - notes.append( - f"session reuse stopped before chunk {index + 1}: prompt changed" - ) - break + entry=validate_chunk_entry(session["chunks"][index]) + if entry["prompt_hash"]!=prompt_hashes[index]: notes.append(f"session reuse stopped before chunk {index+1}: prompt changed"); break preserved.append(_clone_entry_for_reuse(entry)) - if reroll_from_chunk == 0 and len(preserved) == min(len(session["chunks"]), chunks): - if len(session["chunks"]) < chunks: - notes.append(f"resuming after accepted chunk {len(session['chunks'])}") - else: - notes.append("all requested chunks reused from the session") - elif reroll_from_chunk > 0: - notes.append( - f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}" - ) - return preserved, notes + if reroll_from_chunk==0 and len(preserved)==min(len(session["chunks"]),chunks): + notes.append(f"resuming after accepted chunk {len(session['chunks'])}" if len(session["chunks"])0: notes.append(f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}") + return preserved,notes - -def _conditioning_cache( - *, - clip: Any, - prompts: list[str], - assets: Any, - final_has_last_frame: bool, -) -> dict[tuple[str, bool], list]: - cache: dict[tuple[str, bool], list] = {} - final_index = len(prompts) - 1 - for index, prompt in enumerate(prompts): - include_last = bool(final_has_last_frame and index == final_index) - key = (prompt, include_last) - if key in cache: - continue - cache[key] = encode_prompt_conditioning( - clip, - prompt, - first_image=assets.first_image, - last_image=assets.last_image if include_last else None, - ) +def _conditioning_cache(*,clip,prompts,assets,final_has_last_frame): + cache={}; final_index=len(prompts)-1 + for index,prompt in enumerate(prompts): + include_last=bool(final_has_last_frame and index==final_index); key=(prompt,include_last) + if key in cache: continue + cache[key]=encode_prompt_conditioning(clip,prompt,first_image=assets.first_image,last_image=assets.last_image if include_last else None) return cache - -def run_sequence( - *, - model: Any, - clip: Any, - video_vae: Any, - audio_vae: Any, - sampler: Any, - sigmas: torch.Tensor, - first_frame: torch.Tensor | None, - last_frame: torch.Tensor | None, - prompt_plan: dict[str, Any], - width: int, - height: int, - continuity: str, - base_seed: int, - audio_continuity: bool, - exact_total_duration: bool, - diagnostics_mode: str, - reroll_from_chunk: int, - reroll_nonce: int, - strict_compatibility: bool, - debug: bool, - seam_correction: str = SEAM_CORRECTION_OFF, - enable_preview: bool = True, - session: dict[str, Any] | None = None, - initial_state: dict[str, Any] | None = None, -) -> tuple[torch.Tensor, dict[str, Any], dict[str, Any], dict[str, Any], str]: - """Generate or resume a complete Continuum sequence.""" - - plan = validate_prompt_plan(prompt_plan) - chunks = int(plan["chunks"]) - chunk_seconds = float(plan["chunk_seconds"]) - prompts = list(plan["prompts"]) - prompt_hashes = list(plan["hashes"]) - width, height = int(width), int(height) - if width <= 0 or height <= 0 or width % 32 or height % 32: - raise SequenceRuntimeError("width and height must be positive multiples of 32") - if session is not None and initial_state is not None: - raise SequenceRuntimeError("connect either a session or an initial_state, not both") - if diagnostics_mode not in DIAGNOSTICS_OPTIONS: - raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}") - if seam_correction not in SEAM_CORRECTION_OPTIONS: - raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}") - if not (0 <= int(reroll_from_chunk) <= chunks): - raise SequenceRuntimeError( - "reroll_from_chunk must be 0 or a valid one-based chunk index" - ) - if initial_state is not None and int(reroll_from_chunk) not in (0, 1): - raise SequenceRuntimeError( - "with initial_state, reroll_from_chunk can only be 0 or 1" - ) - decode_estimate_gib, available_ram_gib = _check_decode_memory_budget( - width=width, - height=height, - chunks=chunks, - chunk_seconds=chunk_seconds, - ) - - issues = check_comfy_h3_runtime() - if issues and strict_compatibility: - raise SequenceRuntimeError("H3 runtime is incompatible: " + "; ".join(issues)) - - # Phase 1: all CLIP and VAE encoding happens before H3 sampling. This is the - # key residency optimization: no text encoder or VAE is requested between - # H3 chunks. - assets = prepare_identity_assets( - video_vae, - width=width, - height=height, - first_frame=first_frame, - last_frame=last_frame, - encode_latents=False, - ) - - patched_model = patch_model( - model, - strict=bool(strict_compatibility), - debug=bool(debug), - ) - current_model_fingerprint = model_fingerprint(patched_model) - preserved, reuse_notes = _preserved_prefix( - session=session, - prompt_hashes=prompt_hashes, - chunks=chunks, - reroll_from_chunk=int(reroll_from_chunk), - width=width, - height=height, - chunk_seconds=chunk_seconds, - identity_hash=assets.identity_hash, - ) +def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,sigmas:torch.Tensor,first_frame:torch.Tensor|None,last_frame:torch.Tensor|None,prompt_plan:dict[str,Any],width:int,height:int,continuity:str,base_seed:int,audio_continuity:bool,exact_total_duration:bool,diagnostics_mode:str,reroll_from_chunk:int,reroll_nonce:int,strict_compatibility:bool,debug:bool,seam_correction:str=SEAM_CORRECTION_OFF,enable_preview:bool=True,session:dict[str,Any]|None=None,initial_state:dict[str,Any]|None=None): + plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height) + if width<=0 or height<=0 or width%32 or height%32: raise SequenceRuntimeError("width and height must be positive multiples of 32") + if session is not None and initial_state is not None: raise SequenceRuntimeError("connect either a session or an initial_state, not both") + if diagnostics_mode not in DIAGNOSTICS_OPTIONS: raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}") + if seam_correction not in SEAM_CORRECTION_OPTIONS: raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}") + if not 0<=int(reroll_from_chunk)<=chunks: raise SequenceRuntimeError("reroll_from_chunk must be 0 or a valid one-based chunk index") + if initial_state is not None and int(reroll_from_chunk) not in (0,1): raise SequenceRuntimeError("with initial_state, reroll_from_chunk can only be 0 or 1") + decode_estimate_gib,available_ram_gib=_check_decode_memory_budget(width=width,height=height,chunks=chunks,chunk_seconds=chunk_seconds) + issues=check_comfy_h3_runtime() + if issues and strict_compatibility: raise SequenceRuntimeError("H3 runtime is incompatible: "+"; ".join(issues)) + assets=prepare_identity_assets(video_vae,width=width,height=height,first_frame=first_frame,last_frame=last_frame,encode_latents=False) + patched_model=patch_model(model,strict=bool(strict_compatibility),debug=bool(debug)); current_model_fingerprint=model_fingerprint(patched_model) + preserved,reuse_notes=_preserved_prefix(session=session,prompt_hashes=prompt_hashes,chunks=chunks,reroll_from_chunk=int(reroll_from_chunk),width=width,height=height,chunk_seconds=chunk_seconds,identity_hash=assets.identity_hash) if session is not None: - old_fingerprint = str(session.get("model_fingerprint", "")) - if old_fingerprint and old_fingerprint != current_model_fingerprint: - reuse_notes.append( - "model/accelerator fingerprint differs from the saved session; accepted chunks were kept" - ) - - cache: dict[tuple[str, bool], list] = {} - if len(preserved) < chunks: - assets = encode_identity_latents(video_vae, assets) - cache = _conditioning_cache( - clip=clip, - prompts=prompts, - assets=assets, - final_has_last_frame=last_frame is not None, - ) - - entries: list[dict[str, Any]] = preserved[:] - previous_state = None - if entries: - previous_state = entry_to_state(entries[-1]) + old_fingerprint=str(session.get("model_fingerprint","")) + if old_fingerprint and old_fingerprint!=current_model_fingerprint: reuse_notes.append("model/accelerator fingerprint differs from the saved session; accepted chunks were kept") + cache={} + if len(preserved) str: - value = _SAFE_PREFIX.sub("_", str(prefix).strip()).strip("._") - return value or "h3_continuum_session" - - -def session_directory() -> Path: +LOG=logging.getLogger("h3_continuum_join") +_SAFE_PREFIX=re.compile(r"[^A-Za-z0-9._-]+") +_HEADER_KEY="h3_continuum_session_json" +def sanitize_prefix(prefix:str)->str: + value=_SAFE_PREFIX.sub("_",str(prefix).strip()).strip("._"); return value or "h3_continuum_session" +def session_directory()->Path: try: - import folder_paths - root = Path(folder_paths.get_output_directory()) - except Exception: - root = Path.cwd() - path = root / "h3_continuum_sessions" - path.mkdir(parents=True, exist_ok=True) - return path - - -def session_paths(prefix: str, slot: int) -> tuple[Path, Path]: - slot = int(slot) - if not (1 <= slot <= 9999): - raise ValueError("session slot must be between 1 and 9999") - name = f"{sanitize_prefix(prefix)}_slot{slot:04d}" - root = session_directory() - return root / f"{name}.safetensors", root / f"{name}.json" - - -def _fsync_file(path: Path) -> None: + import folder_paths; root=Path(folder_paths.get_output_directory()) + except Exception: root=Path.cwd() + path=root/"h3_continuum_sessions"; path.mkdir(parents=True,exist_ok=True); return path +def session_paths(prefix:str,slot:int)->tuple[Path,Path]: + slot=int(slot) + if not 1<=slot<=9999: raise ValueError("session slot must be between 1 and 9999") + name=f"{sanitize_prefix(prefix)}_slot{slot:04d}"; root=session_directory(); return root/f"{name}.safetensors",root/f"{name}.json" +def _fsync_file(path): try: - with path.open("rb") as handle: - os.fsync(handle.fileno()) - except OSError: - pass - - -def _fsync_directory(path: Path) -> None: - if os.name == "nt": - return + with path.open("rb") as handle: os.fsync(handle.fileno()) + except OSError: pass +def _fsync_directory(path): + if os.name=="nt": return try: - descriptor = os.open(str(path), os.O_RDONLY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - except OSError: - pass - - -def _metadata_without_tensors(session: dict[str, Any]) -> dict[str, Any]: - metadata = {key: value for key, value in session.items() if key != "chunks"} - chunks = [] - for entry in session["chunks"]: - chunks.append({key: value for key, value in entry.items() if key not in ("video", "audio")}) - metadata["chunks"] = chunks - return metadata - - -def save_session(session: dict[str, Any], *, prefix: str, slot: int) -> tuple[Path, Path]: - session = validate_session(session) - tensor_path, json_path = session_paths(prefix, slot) - token = uuid.uuid4().hex - tensor_tmp = tensor_path.with_name(f".{tensor_path.name}.{token}.tmp") - json_tmp = json_path.with_name(f".{json_path.name}.{token}.tmp") - - tensors = {} - for index, entry in enumerate(session["chunks"], start=1): - tensors[f"chunk_{index:04d}_video"] = entry["video"].detach().to("cpu").contiguous() - tensors[f"chunk_{index:04d}_audio"] = entry["audio"].detach().to("cpu").contiguous() - metadata = _metadata_without_tensors(session) - metadata["_session_commit_id"] = token - serialized = json.dumps(metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - + descriptor=os.open(str(path),os.O_RDONLY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + except OSError: pass +def _metadata_without_tensors(session): + metadata={k:v for k,v in session.items() if k!="chunks"}; chunks=[] + for entry in session["chunks"]: chunks.append({k:v for k,v in entry.items() if k not in ("video","audio")}) + metadata["chunks"]=chunks; return metadata +def save_session(session,*,prefix,slot): + session=validate_session(session); tensor_path,json_path=session_paths(prefix,slot); token=uuid.uuid4().hex + tensor_tmp=tensor_path.with_name(f".{tensor_path.name}.{token}.tmp"); json_tmp=json_path.with_name(f".{json_path.name}.{token}.tmp") + tensors={} + for index,entry in enumerate(session["chunks"],start=1): + tensors[f"chunk_{index:04d}_video"]=entry["video"].detach().to("cpu").contiguous(); tensors[f"chunk_{index:04d}_audio"]=entry["audio"].detach().to("cpu").contiguous() + metadata=_metadata_without_tensors(session); metadata["_session_commit_id"]=token + serialized=json.dumps(metadata,ensure_ascii=False,sort_keys=True,separators=(",",":")) try: - save_file(tensors, str(tensor_tmp), metadata={_HEADER_KEY: serialized}) - _fsync_file(tensor_tmp) - with json_tmp.open("w", encoding="utf-8", newline="\n") as handle: - json.dump(metadata, handle, ensure_ascii=False, indent=2, sort_keys=True) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(json_tmp, json_path) - os.replace(tensor_tmp, tensor_path) - _fsync_directory(tensor_path.parent) + save_file(tensors,str(tensor_tmp),metadata={_HEADER_KEY:serialized}); _fsync_file(tensor_tmp) + with json_tmp.open("w",encoding="utf-8",newline="\n") as handle: + json.dump(metadata,handle,ensure_ascii=False,indent=2,sort_keys=True); handle.write("\n"); handle.flush(); os.fsync(handle.fileno()) + os.replace(json_tmp,json_path); os.replace(tensor_tmp,tensor_path); _fsync_directory(tensor_path.parent) finally: - for path in (tensor_tmp, json_tmp): - try: - path.unlink(missing_ok=True) - except OSError: - pass - return tensor_path, json_path - - -def load_session(*, prefix: str, slot: int) -> dict[str, Any]: - tensor_path, json_path = session_paths(prefix, slot) - if not tensor_path.is_file(): - raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}") - with safe_open(str(tensor_path), framework="pt", device="cpu") as handle: - serialized = (handle.metadata() or {}).get(_HEADER_KEY) - if serialized is None: - raise ValueError("session safetensors has no embedded Continuum metadata") - session = json.loads(serialized) - for index, entry in enumerate(session.get("chunks") or [], start=1): - entry["video"] = handle.get_tensor(f"chunk_{index:04d}_video") - entry["audio"] = handle.get_tensor(f"chunk_{index:04d}_audio") - + for path in (tensor_tmp,json_tmp): + try: path.unlink(missing_ok=True) + except OSError: pass + return tensor_path,json_path +def load_session(*,prefix,slot): + tensor_path,json_path=session_paths(prefix,slot) + if not tensor_path.is_file(): raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}") + with safe_open(str(tensor_path),framework="pt",device="cpu") as handle: + serialized=(handle.metadata() or {}).get(_HEADER_KEY) + if serialized is None: raise ValueError("session safetensors has no embedded Continuum metadata") + session=json.loads(serialized) + for index,entry in enumerate(session.get("chunks") or [],start=1): + entry["video"]=handle.get_tensor(f"chunk_{index:04d}_video"); entry["audio"]=handle.get_tensor(f"chunk_{index:04d}_audio") if json_path.is_file(): try: - with json_path.open("r", encoding="utf-8") as handle: - mirror = json.load(handle) - if mirror.get("_session_commit_id") != session.get("_session_commit_id"): - LOG.warning("Ignoring stale Continuum session JSON mirror: %s", json_path.name) - except (OSError, ValueError, TypeError) as exc: - LOG.warning("Ignoring unreadable session JSON mirror %s: %s", json_path.name, exc) + with json_path.open("r",encoding="utf-8") as handle: mirror=json.load(handle) + if mirror.get("_session_commit_id")!=session.get("_session_commit_id"): LOG.warning("Ignoring stale Continuum session JSON mirror: %s",json_path.name) + except (OSError,ValueError,TypeError) as exc: LOG.warning("Ignoring unreadable session JSON mirror %s: %s",json_path.name,exc) return validate_session(session) - - -def session_mtime(*, prefix: str, slot: int) -> tuple[float, float]: - tensor_path, json_path = session_paths(prefix, slot) - return ( - tensor_path.stat().st_mtime if tensor_path.exists() else -1.0, - json_path.stat().st_mtime if json_path.exists() else -1.0, - ) +def session_mtime(*,prefix,slot): + tensor_path,json_path=session_paths(prefix,slot); return (tensor_path.stat().st_mtime if tensor_path.exists() else -1.0,json_path.stat().st_mtime if json_path.exists() else -1.0) diff --git a/version.py b/version.py index a65932c..97fb690 100644 --- a/version.py +++ b/version.py @@ -1,6 +1,6 @@ """Package version and public schema identifiers.""" -PACKAGE_VERSION = "2.1.5" +PACKAGE_VERSION = "2.1.7" STATE_SCHEMA_VERSION = 1 PLAN_SCHEMA_VERSION = 1 PUBLIC_API_VERSION = 2 diff --git a/web/h3_continuum_v2.js b/web/h3_continuum_v2.js deleted file mode 100644 index d68241d..0000000 --- a/web/h3_continuum_v2.js +++ /dev/null @@ -1,388 +0,0 @@ -import { app } from "../../../scripts/app.js"; - -const NODE_NAME = "H3ContinuumSamplerV2"; -const PREVIEW_SETTING = "H3Continuum.ShowLatentPreview"; -const PROMPT_INPUT = /^clip_(\d+)_prompt$/; -const SEQUENCE_PROMPT_INPUT = "sequence_prompt"; -const CLIP_OVERRIDES_PROPERTY = "h3_show_clip_overrides"; -const ADVANCED_PROPERTY = "h3_show_advanced"; -const ADVANCED_INPUTS = new Set(["last_frame", "session", "initial_state", "prompt_plan"]); -const ADVANCED_OUTPUTS = new Set(["last_state", "session", "report"]); -const MAX_CHUNKS = 16; -const VISIBILITY_STYLE_ID = "h3-continuum-v215-slot-visibility"; -const ADVANCED_WIDGETS = new Set([ - "control_after_generate", - "audio_continuity", - "exact_total_duration", - "diagnostics", - "reroll_from_chunk", - "reroll_nonce", - "strict_compatibility", - "debug", - "show_preview", -]); -const LEGACY_PROMPT_FORMATS = new Map([ - ["Fixed — one prompt", "Fixed"], - ["List — split with ---", "List"], - ["Timeline — [0-5s] sections", "Timeline"], -]); - -let showLatentPreview = true; - -function setWidgetCollapsed(widget, reason, collapsed) { - if (!widget) return; - if (!widget.__h3ContinuumCollapseState) { - widget.__h3ContinuumCollapseState = { - computeSize: widget.computeSize, - hiddenOption: widget.options?.hidden, - reasons: new Set(), - }; - } - const state = widget.__h3ContinuumCollapseState; - if (collapsed) { - state.reasons.add(reason); - } else { - state.reasons.delete(reason); - } - const hidden = state.reasons.size > 0; - widget.options ||= {}; - if (hidden) { - widget.options.hidden = true; - } else if (state.hiddenOption === undefined) { - delete widget.options.hidden; - } else { - widget.options.hidden = state.hiddenOption; - } - widget.computeSize = hidden ? () => [0, -4] : state.computeSize; - for (const element of [widget.element, widget.inputEl, widget.el]) { - if (element?.style) element.style.display = hidden ? "none" : ""; - } -} - -function serializableWidgets(widgets) { - return widgets.filter((widget) => widget.serialize !== false); -} - -function placeTransientAfter(node, transient, anchor) { - const original = node.widgets ?? []; - const serializedBefore = serializableWidgets(original); - const withoutTransient = original.filter((widget) => widget !== transient); - const anchorIndex = withoutTransient.findIndex((widget) => - typeof anchor === "string" ? widget.name === anchor : widget === anchor, - ); - if (anchorIndex < 0) return; - const next = [ - ...withoutTransient.slice(0, anchorIndex + 1), - transient, - ...withoutTransient.slice(anchorIndex + 1), - ]; - const serializedAfter = serializableWidgets(next); - const stable = serializedBefore.length === serializedAfter.length - && serializedBefore.every((widget, index) => widget === serializedAfter[index]); - if (!stable) throw new Error("H3 Continuum refused to reorder serialized widgets"); - node.widgets = next; -} - -function inputIsConnected(input) { - return input?.link != null || (Array.isArray(input?.links) && input.links.length > 0); -} - -function outputIsConnected(output) { - return Array.isArray(output?.links) && output.links.length > 0; -} - -function syncPromptFormat(node) { - const widget = node.widgets?.find((candidate) => candidate.name === "prompt_mode"); - if (!widget) return; - if (LEGACY_PROMPT_FORMATS.has(widget.value)) { - widget.value = LEGACY_PROMPT_FORMATS.get(widget.value); - } - widget.label = "Prompt Format"; -} - -function syncSequencePrompt(node) { - const input = node.inputs?.find((candidate) => candidate.name === SEQUENCE_PROMPT_INPUT); - if (input) input.label = "Sequence Prompt"; -} - -function clipOverridesExpanded(node) { - return node.properties?.[CLIP_OVERRIDES_PROPERTY] === true; -} - -function ensureClipOverridesButton(node) { - if (node.__h3ContinuumClipOverridesButton) { - return node.__h3ContinuumClipOverridesButton; - } - const button = node.addWidget( - "button", - "Individual Clip Overrides ▸", - null, - () => { - node.properties ||= {}; - node.properties[CLIP_OVERRIDES_PROPERTY] = !clipOverridesExpanded(node); - syncNode(node); - }, - { serialize: false }, - ); - button.serialize = false; - button.__h3ContinuumTransient = true; - node.__h3ContinuumClipOverridesButton = button; - return button; -} - -function syncPromptInputs(node) { - node.properties ||= {}; - if (typeof node.properties[CLIP_OVERRIDES_PROPERTY] !== "boolean") { - node.properties[CLIP_OVERRIDES_PROPERTY] = false; - } - const chunksWidget = node.widgets?.find((widget) => widget.name === "chunks"); - const chunks = Math.max(1, Math.min(MAX_CHUNKS, Number(chunksWidget?.value) || 1)); - const expanded = clipOverridesExpanded(node); - for (const input of node.inputs ?? []) { - const match = PROMPT_INPUT.exec(input.name); - if (!match) continue; - const clipIndex = Number(match[1]); - input.label = clipIndex <= chunks - ? "Clip " + clipIndex + " Prompt Override" - : "Clip " + clipIndex + " Override (inactive)"; - } - const button = ensureClipOverridesButton(node); - placeTransientAfter(node, button, "prompt_mode"); - button.name = expanded - ? "Individual Clip Overrides (" + chunks + ") ▾" - : "Individual Clip Overrides (" + chunks + ") ▸"; -} - -function advancedExpanded(node) { - return node.properties?.[ADVANCED_PROPERTY] === true; -} - -function ensureAdvancedButton(node) { - if (node.__h3ContinuumAdvancedButton) return node.__h3ContinuumAdvancedButton; - const button = node.addWidget( - "button", - "Advanced Settings ▸", - null, - () => { - node.properties ||= {}; - node.properties[ADVANCED_PROPERTY] = !advancedExpanded(node); - syncNode(node); - }, - { serialize: false }, - ); - button.serialize = false; - button.__h3ContinuumTransient = true; - node.__h3ContinuumAdvancedButton = button; - return button; -} - -function syncAdvancedWidgets(node) { - node.properties ||= {}; - if (typeof node.properties[ADVANCED_PROPERTY] !== "boolean") { - node.properties[ADVANCED_PROPERTY] = false; - } - if ( - (node.inputs ?? []).some((input) => ADVANCED_INPUTS.has(input.name) && inputIsConnected(input)) - || (node.outputs ?? []).some((output) => ADVANCED_OUTPUTS.has(output.name) && outputIsConnected(output)) - ) { - node.properties[ADVANCED_PROPERTY] = true; - } - const expanded = advancedExpanded(node); - for (const widget of node.widgets ?? []) { - if (ADVANCED_WIDGETS.has(widget.name)) { - setWidgetCollapsed(widget, "advanced", !expanded); - } - } - const button = ensureAdvancedButton(node); - button.name = expanded ? "Advanced Settings ▾" : "Advanced Settings ▸"; -} - -function syncPreviewSetting(node) { - const widget = node.widgets?.find((candidate) => candidate.name === "show_preview"); - if (!widget) return; - setWidgetCollapsed(widget, "preview-setting", true); - widget.value = showLatentPreview; -} - -function repairLegacySeamCorrection(node) { - const widget = node.widgets?.find((candidate) => candidate.name === "seam_correction"); - if (widget && typeof widget.value === "boolean") widget.value = "Off"; -} - -function ensureSeamCorrectionProxy(node) { - const source = node.widgets?.find((widget) => widget.name === "seam_correction"); - if (!source) return null; - if (typeof source.value === "boolean") source.value = "Off"; - setWidgetCollapsed(source, "seam-proxy", true); - - let proxy = node.__h3ContinuumSeamProxy; - if (!proxy) { - proxy = node.addWidget( - "combo", - "Seam Correction", - source.value, - (value) => { - source.value = value; - source.callback?.call(source, value); - }, - { values: source.options?.values ?? ["Auto", "Off", "Basic"], serialize: false }, - ); - proxy.serialize = false; - proxy.__h3ContinuumTransient = true; - node.__h3ContinuumSeamProxy = proxy; - } - proxy.value = source.value; - placeTransientAfter(node, proxy, "base_seed"); - return proxy; -} - -function cssAttributeValue(value) { - return String(value).replaceAll("\\", "\\\\").replaceAll('"', '\\"'); -} - -function shouldHideInput(node, input, chunks) { - const promptMatch = PROMPT_INPUT.exec(input.name); - if (promptMatch) { - const clipIndex = Number(promptMatch[1]); - return !inputIsConnected(input) - && (!clipOverridesExpanded(node) || clipIndex > chunks); - } - return ADVANCED_INPUTS.has(input.name) - && !advancedExpanded(node) - && !inputIsConnected(input); -} - -function slotVisibilityRules(node) { - const id = cssAttributeValue(node.id); - const root = `.lg-node[data-node-id="${id}"]`; - const chunksWidget = node.widgets?.find((widget) => widget.name === "chunks"); - const chunks = Math.max(1, Math.min(MAX_CHUNKS, Number(chunksWidget?.value) || 1)); - const rules = []; - const renderedInputs = (node.inputs ?? []).filter((input) => !input.widget); - renderedInputs.forEach((input, index) => { - if (shouldHideInput(node, input, chunks)) { - rules.push(`${root} .lg-slot--input:nth-child(${index + 1}) { display: none !important; }`); - } - }); - (node.outputs ?? []).forEach((output, index) => { - if ( - ADVANCED_OUTPUTS.has(output.name) - && !advancedExpanded(node) - && !outputIsConnected(output) - ) { - rules.push(`${root} .lg-slot--output:nth-child(${index + 1}) { display: none !important; }`); - } - }); - return rules; -} - -function syncSlotVisibilityStyles() { - let style = document.getElementById(VISIBILITY_STYLE_ID); - if (!style) { - style = document.createElement("style"); - style.id = VISIBILITY_STYLE_ID; - document.head.appendChild(style); - } - style.textContent = (app.graph?._nodes ?? []) - .filter((node) => node.comfyClass === NODE_NAME || node.type === NODE_NAME) - .flatMap(slotVisibilityRules) - .join("\n"); -} - -function resizeNode(node) { - node.setSize([Math.max(node.size?.[0] ?? 0, 320), 120]); - node.setDirtyCanvas(true, true); -} - -function syncNode(node) { - setWidgetCollapsed( - node.widgets?.find((widget) => widget.name === "prompt_script"), - "prompt-script", - true, - ); - syncPromptFormat(node); - syncSequencePrompt(node); - repairLegacySeamCorrection(node); - syncPromptInputs(node); - const seamProxy = ensureSeamCorrectionProxy(node); - syncAdvancedWidgets(node); - if (seamProxy) { - placeTransientAfter(node, node.__h3ContinuumAdvancedButton, seamProxy); - } - syncPreviewSetting(node); - syncSlotVisibilityStyles(); - resizeNode(node); -} - -function wrapChunksWidget(node) { - const widget = node.widgets?.find((candidate) => candidate.name === "chunks"); - if (!widget || widget.__h3ContinuumWrapped) return; - widget.__h3ContinuumWrapped = true; - const originalCallback = widget.callback; - widget.callback = function (...args) { - const result = originalCallback?.apply(this, args); - queueMicrotask(() => syncNode(node)); - return result; - }; -} - -function scheduleSync(node) { - clearTimeout(node.__h3ContinuumSyncTimer); - node.__h3ContinuumSyncTimer = setTimeout(() => { - wrapChunksWidget(node); - syncNode(node); - }, 0); -} - -function syncAllSamplerNodes() { - for (const node of app.graph?._nodes || []) { - if (node.comfyClass === NODE_NAME || node.type === NODE_NAME) { - syncNode(node); - } - } -} - -app.registerExtension({ - name: "H3Continuum.V215StableInterface", - - async setup() { - showLatentPreview = app.ui.settings.getSettingValue(PREVIEW_SETTING) ?? true; - app.ui.settings.addSetting({ - id: PREVIEW_SETTING, - name: "Show latent preview", - category: ["H3 Continuum", "Preview", "Show latent preview"], - tooltip: "Show the in-progress latent thumbnail inside H3 Continuum Sampler V2 nodes.", - type: "boolean", - defaultValue: true, - onChange(value) { - showLatentPreview = value !== false; - syncAllSamplerNodes(); - }, - }); - }, - - async beforeRegisterNodeDef(nodeType, nodeData) { - if (nodeData.name !== NODE_NAME) return; - - const originalCreated = nodeType.prototype.onNodeCreated; - nodeType.prototype.onNodeCreated = function () { - const result = originalCreated?.apply(this, arguments); - scheduleSync(this); - return result; - }; - - const originalConfigure = nodeType.prototype.onConfigure; - nodeType.prototype.onConfigure = function () { - const result = originalConfigure?.apply(this, arguments); - scheduleSync(this); - return result; - }; - - const originalConnectionsChange = nodeType.prototype.onConnectionsChange; - nodeType.prototype.onConnectionsChange = function () { - const result = originalConnectionsChange?.apply(this, arguments); - scheduleSync(this); - return result; - }; - }, -});