diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b38c06..f9f479e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 3.5.0 + +- Added `H3 Continuum Sampler V3.5` with a Continuum-aware refine-context output while preserving the V3.4 sampler path. +- Added `H3 Continuum Second Pass V3.5` as the advanced bridge for externally processed H3 video latents, including physical-group and Long Terminal Merge preservation. +- Added experimental `H3 Continuum Hi-Res Fix V3.5`, using a bounded pixel/VAE resize round trip before low-denoise Second Pass sampling. +- Added `H3 Continuum Latent Resize V3.5` as an advanced utility; direct high-ratio latent interpolation is not the recommended main Hi-Res Fix path. +- Added `H3 Continuum Assemble + Seam V3.5` with Auto, RAM, and Windows-safe disk-backed video buffers while keeping audio in RAM. +- Preserved Exact Duration, seam behavior, physical decode-group ordering, first-pass audio passthrough, and external Core VAE Decode contracts. +- Kept V3.4 nodes and saved-workflow compatibility unchanged. +- Validated a 9.49 GiB final IMAGE with hash-identical RAM/Disk-backed output and approximately 9.50 GiB lower Windows private memory in Disk-backed mode. +- GPU-accepted the integrated 1x5 576-to-1152 Main path and Hybrid/Reference 1x5 paths; documented that the 3x5 2x terminal 77T group exceeds the tested RTX 5060 Ti 16 GiB. + ## 3.4.0 - Completed the V3.4 runtime path from the public sampler through V3/V2 to `run_sequence()`. diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index c46d6a2..cdad928 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -1,29 +1,15 @@ +dca65fb1393e67f59c6f81f9411e3e745eb8f1097e9a2adf7ffc6f4e3901ca9f __init__.py 15b900d02624bea6cfe8ea25b22266a0c68b38b9f10c9089be0e38988bf2e34d .github/workflows/tests.yml 0a2f69d4cec89d440f5d5aa1ec68a7c9a741382d313f80dcddf96d13f70b3e68 .gitignore -e73cd6f60af63ecff74b447ca1913e8e23269d9f17758f5db78c76543dbb3404 AGENTS.md -08d6d4aafa2b7c014f3a8aeeb91d5eddbd22464367252812b7a0f516316b18b5 CHANGELOG.md -6abdbc29472c7f9d447085b189fc47b69e110b94ef83192f9689d04000408843 INSTALL_JA.txt -c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE -b92213d2d289d8495e879ad52b6109eb5a76314027915857e8fab79c0d1ac5d4 PACKAGE_INFO_JA.txt -6c8fe62d8856cd57fa3be611ced74fed3d79559d238dc562e39e029ca0aca8ca PACKAGE_VALIDATION.txt -a3801b2f24c6bafa49d3e6427ed03a32e0a1c6a3e21f022739a4d3b99a9e6a85 README.md -e20cd48d1786ab56bc68e0c8dcafa22be7b3b94396cc3c6fcefe47c4a725ccbd README_JA.md -0dbe8a5f71c92f3523942343b40787104c8b96781461ba3ba82934a6bdbe5834 THIRD_PARTY_NOTICES.md -dca65fb1393e67f59c6f81f9411e3e745eb8f1097e9a2adf7ffc6f4e3901ca9f __init__.py +16ea21aa8fe115ce8f6538a0268bc64762be2fd6931b3a0071a86a8cc9d218cf AGENTS.md +7b5c054c57940c7828c7ae5712cf796211b6a3bee0c50cc47fac03c3fffa0356 CHANGELOG.md e97e47c7585e81d9ec5fd32c0ca0b64699fd4610cd03e9d6575a91c30a10ca4e compatibility.py -33ed93cdf4b9e8f3f35b3f89d0d4ce60c34019683fc25502e669e1883dc9687c conditioning.py +4d62508f900af9f93ec320afb900c27cc9a300d1e0c0a94714258498c5ede5fc conditioning.py 71e3df269e226b1ecad597f9831c2a10b2215c8a3783652e9e9f536e99ce46c9 constants.py cf255b8a635471229aebadc926acfdcacb65963adbcb9e0f9505d04eada2c6b5 continuation.py f9375629fa1a2c4d3538442f0c3192d7ba2f46fe9544fb2e21f0c144b53d608c docs/AB_TEST_LOG_JA.md c93a3e52e4deabe5ef0db7f34a7605856a543089574ba17227e697e9a2c0139a docs/ARCHITECTURE.md 7660570840229b6fcbe62f0fbb52966d0099d64d7e6f01a739fbf9899293c8e4 docs/FUTURE_ROADMAP.md -93f07aa727b34802bce3d333426b8ba2af522500882a6d222d00cf577ca82e49 docs/SEAM_TEST_SOURCE_JA.md -2c13aefd137db92a3dbf5576e9442d58012b1f0de1240c07d039ceeb702e7c1f docs/V2_1_4_UI_SIMPLIFICATION.md -b08022df068576c759c8bf80ac6b6519693f3c65c917f1822aeb06ea359aa5b1 docs/V2_1_5_UI_SERIALIZATION_STABILITY.md -a032738b0bc5acb136e92d8bf7694643a5f518b9c7fc0fc878d66d81ba8b1b04 docs/V2_1_6_UNIFIED_UI.md -384e3841603f7c6c8286ef147e93aaf7d056a286503a2a4fc21f397e4dec4876 docs/V34_DRIVING_AUDIO_VALIDATION.md -f0806bb2b999794ddda072e98424dbd1b613df1cd8a096b12d979f124e066d8a docs/VALIDATION_JA.md -3fec541a675c1e4e5e287cc8e498036001f34f7a62460321e16f9ba27b5e4c18 docs/VALIDATION_RESULTS.md 9a634b4ff6c9ca98285f6b250402a3d0e67d29f031e134d9bfcab54c119ed337 docs/images/comfyui-manager-h3-continuum.png 6c72d9a9e35648355c4dd4c8f060a6b7667e322f4b6c86690df9d6769ba182b7 docs/images/h3-continuum-advanced.png b9f1dca35706a1cd963ba892d218568ea30173c6cf6a3bdc896f7a2266eb9407 docs/images/h3-continuum-detailed-report.png @@ -38,43 +24,59 @@ c9573d4393236276d032c4503f8ed8bd19c6ef24e84eec6f64f8a4c47aabc718 docs/images/v3 ff8d901d05aa89f9b698c60b3d11534ef97982b3e22e9ede6255a85149a669c4 docs/images/v34-video-reference-separate-audio.png 06c333b3dda41526da70d56f4aa25922aafa2f48ec2d99350b0ed6b428e92bad docs/images/v34-video-reference-with-audio.png 301afedc3e0922a4bf3bcf445734b5adcdd5c35384132b24a38098c66750f499 docs/images/v34-workflow-overview.png +a53796f7a904d675e80a2276c20d68502f03a3958529278e6ce9465baaf1c4af docs/images/v35-hires-fix-node.png +5a23f232a8b795226e8255e6e2e3248b0130f1c7ce276d50370d0333d6a62ef0 docs/images/v35-second-pass-node.png +93f07aa727b34802bce3d333426b8ba2af522500882a6d222d00cf577ca82e49 docs/SEAM_TEST_SOURCE_JA.md +2c13aefd137db92a3dbf5576e9442d58012b1f0de1240c07d039ceeb702e7c1f docs/V2_1_4_UI_SIMPLIFICATION.md +b08022df068576c759c8bf80ac6b6519693f3c65c917f1822aeb06ea359aa5b1 docs/V2_1_5_UI_SERIALIZATION_STABILITY.md +a032738b0bc5acb136e92d8bf7694643a5f518b9c7fc0fc878d66d81ba8b1b04 docs/V2_1_6_UNIFIED_UI.md +384e3841603f7c6c8286ef147e93aaf7d056a286503a2a4fc21f397e4dec4876 docs/V34_DRIVING_AUDIO_VALIDATION.md +1840c81b4090ff3c86c0c1b0acc2cc10d3d9839e383c81f262a5c59a67504cd6 docs/V35_HIRES_FIX.md +f0806bb2b999794ddda072e98424dbd1b613df1cd8a096b12d979f124e066d8a docs/VALIDATION_JA.md +3fec541a675c1e4e5e287cc8e498036001f34f7a62460321e16f9ba27b5e4c18 docs/VALIDATION_RESULTS.md cedd7b07901d6065079ef37380f0062d535a39c98c8f614e1b26582f549d7183 driving_audio.py 6771c9905d35666121c4892d2ec26202505f96d244ce814ad9a87a02878952cd examples/H3_CONTINUUM_V1_WIRING.md -c3cefcdf50df9d6609f583a008453b587c011a0fb41e290d5f1196ce62f58877 examples/H3_CONTINUUM_V2_WIRING_JA.md -20f4e7c4ad085791f8b578f575d03e52ef09d0d4550aa58ad086c9857b59bf29 examples/H3_Continuum_V2.json -f67febfdd7b2bfca2ad4e477c87a9a74dbe636231f9db3e1158b944d7b9f8397 examples/H3_Continuum_V2.zip f85ba3c69a473f0ac4bb96895c6d6532e8270121da65f858e364b19d31f05ef5 examples/H3_Continuum_V2_3x5s.json be419d6c773784823af7b58a3423ef2e21d3cc3e23c40e60e274560895ca034b examples/H3_Continuum_V2_README.txt 4b94f779a98bfc555f6999113a694da07746cc3cf400bfc70ebc580592034d52 examples/H3_Continuum_V2_reference.webp +c3cefcdf50df9d6609f583a008453b587c011a0fb41e290d5f1196ce62f58877 examples/H3_CONTINUUM_V2_WIRING_JA.md +20f4e7c4ad085791f8b578f575d03e52ef09d0d4550aa58ad086c9857b59bf29 examples/H3_Continuum_V2.json +f67febfdd7b2bfca2ad4e477c87a9a74dbe636231f9db3e1158b944d7b9f8397 examples/H3_Continuum_V2.zip 68e1a724d3b99458b86b8f643dcb916183e7e8dd12d7b4b3850991feb8077694 examples/recommended_settings_v1.json 5a26408c770381cdb259cb710de7dab12d7ddecca262bdbf35ba1566056050f5 examples/recommended_settings_v2.json b8275c8cecde12aee0418cea456d1e7f4b875cbb1bce32f1f3d9e06a44dd0be1 examples/v2_prompt_list_example.txt 0e59ea821081cb93165aa3f88c6949736dd225837fe1cdcf1173aa5cd54c6d29 examples/v2_prompt_timeline_example.txt -4e410aed8d7f6ebf11c98d4f71571e46855e4619c2bcbec921df70f29189b6b1 examples/workflows/MiniMax_H3_Continuum_V32.json 55bc60447d6cd783b9df049d04b6cebb6e4a633981434ac2b28e5a1527d4f4b4 examples/workflows/MiniMax_H3_Continuum_V32_turbo.json -4e410aed8d7f6ebf11c98d4f71571e46855e4619c2bcbec921df70f29189b6b1 examples/workflows/MiniMax_H3_Continuum_V33.json +4e410aed8d7f6ebf11c98d4f71571e46855e4619c2bcbec921df70f29189b6b1 examples/workflows/MiniMax_H3_Continuum_V32.json 55bc60447d6cd783b9df049d04b6cebb6e4a633981434ac2b28e5a1527d4f4b4 examples/workflows/MiniMax_H3_Continuum_V33_turbo.json -839236aa94771838bc7a7eecea0d451f713dfeca59278c38c3e98da6d3194b0f examples/workflows/MiniMax_H3_Continuum_V34.json -a70e5c3426b7a8a30663685713e63df9bef06c6c7aba341020b894bce33f78c6 examples/workflows/MiniMax_H3_Continuum_V34_turbo.json +4e410aed8d7f6ebf11c98d4f71571e46855e4619c2bcbec921df70f29189b6b1 examples/workflows/MiniMax_H3_Continuum_V33.json +5eab33c110e07ec08ac66f289f20c295a90ca026ea5231ad6121755a28a2ded8 examples/workflows/MiniMax_H3_Continuum_V34_turbo.json +660e955c983e41947f5abfd89e66fbe5759ffda01e6c1a7164d580b0c41b2120 examples/workflows/MiniMax_H3_Continuum_V34.json e8f41dee2240f1fcc6e8fd12448856513483ff3ae0427b2c554bb351f9ae8047 graph_contract.py -3d57276669e4ce95f8537c67c6437250b52dd843ee0817c2e262018ead200107 hardening.py +3d14acac341c63207995e2bd45cd407fdb3e2308273b1c90605adfe6f2e0bd8f hardening.py +6abdbc29472c7f9d447085b189fc47b69e110b94ef83192f9689d04000408843 INSTALL_JA.txt 23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat d74073e7ee292ebeba913af4288284f9f197cd91fceed6c62bf6c66b784c7cec install_windows.ps1 6aa3f5780baa7bb0c0672b5f01a2f77e5f209da93f6373468cb274ea2f643d4b layout_adapter.py +c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py -f9c65ddb289a53e52bd40c801d9216b96042d83d07ae4633c2bbb3fd9a2b9701 metadata.ini +84cc75b4583a2851e657f3142a1ec4e099091cb7a960be7355e376269e4d3048 metadata.ini ef705a476e3f5bbd05559e7a0e7d3dfb9275badef0f8f65fb1a36ce27fbcb537 model_patch.py -2aae3b9ef4fb7238b03f7d52af20fc091938be3b5f62a0923858cc769b99f72a nodes.py +01ea71240361725972e41746f5a1c4ff90653e6ccc673085f46f427fbf65e4be nodes.py +a230cd2f772021a683432e41f7028f6ea628e34dd19f617a3f98c174fdc97d58 PACKAGE_INFO_JA.txt +a7084bc1d12ab25a1ebe29a089dba72d6c1147e0f5556837e6321277562ad1f5 PACKAGE_VALIDATION.txt b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py -017f1472929b74ca60aca1cb96295f3a24181c8ad56ef4b4f29ccbf17a1f7002 pyproject.toml +16bc48b9ef6127af7e0e3ddd96791640cedf5159cd7c06452d72604f41998a52 pyproject.toml 94fdc2d545b2d4077a4d70229fd22ba3ccac21d49d1097d2873f76d04e9a3c7a pytest.ini -0f507812a3ed4490c0539a4b7fd1e9cd33de3908e72c09f82824c57a04ad3cf8 reference.py +365f1b1ecc89f8f2ced338ce467d02384ef612441202a15064e5d1a5c6cd1bcf README_JA.md +6a48d52e23aa594d79e5788265f17daa9eb3501beea1ae849a82a8bb2ff1ab12 README.md 5d89c366b4f076fa73a2695557e0af749bf95df1dad0d55dbb2fe5b1c2519ad7 reference_audio.py 8f1744dc99f1a3a222b7d6e986f6dac1edcc0abbbe8b61a1d2606b1ca3569b3d reference_video.py +2995924fcb7bac27b548b01187d8276d2ab103c6f43416cf85400de3846c6297 reference.py 467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt -6d2364fc023f36f9c8703cf52c115d70f6ac44f00383d8e12e6f7de7f66baae0 run_storage.py -3a359a0467725cc292ccb4c86cf0c3d30bb6a114d01f5345792b09ea3d45441d state.py +e02b523b5b1c1272c912c16d547050cc9bb9feb6c348743015bb5d059a4ead06 run_storage.py 6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py +3a359a0467725cc292ccb4c86cf0c3d30bb6a114d01f5345792b09ea3d45441d state.py 43ec4023dc06c941e5d7a04db4b8a89fb58d744ceb986f6eb9bf534d4fdcb87b temporal.py 3eeed61b7ee519631825e5f89e9a441644c66e552fce4207927cb8536867c8b1 tests/conftest.py 041289556cc621ea1857cbb8931ae41d817f69e084b2e8a7622e184a3197b61b tests/test_compatibility.py @@ -83,12 +85,17 @@ b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py f9faa77aef448971ae8ebd3447c52db2611f1dbab55e8a41b9c7fea82ba7e0bf tests/test_layout_adapter.py e480c599daf0ede07bba8b90efbae627461c1797a5ebacf228630fd74ebc48d4 tests/test_model_patch.py 45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py +b635d404ca67df69894c4fcf80d8782cc5699e946bfb55ae4c44fff9d930b018 tests/test_p20_file_backed_tensor_poc.py +c3ba92e6dc653ba9252a68e54df68a677132ab79f597e74005c9c0c746b01545 tests/test_p21_memory_attribution.py +2b097971258dd10f0df33e0f8cb5843ee6cd3e9ef9e7abb80b890c66baf22908 tests/test_p22_file_backed_buffer.py +d09d2bae2fac04a058ebce22fa0e761d90b267107923ab8cd01a99cc547624e6 tests/test_p23_assembly_v35.py +d2a02c120e01d360ddca17a92d5bc53b12c9f0734d06440b3fc2fad2e754e549 tests/test_p23_video_seam_patches.py 1ae0369943ba2fb13f21ad0cf56ee58621ba09bf20174379595e157a1775c7fd tests/test_public_api.py -98a33c75b33ff0c10a68e04bd856e4e43b3e70b6ae1ea458a2b21b4e4d74f46d tests/test_state.py +c1ed1671d126cd9969f4d896d89e0061fcab676aea961e3cd79b4433028d9f53 tests/test_release_metadata.py d4c53d2c260b635d70fa211029d608f63776e0a56c20253475fa47c58363f8a7 tests/test_state_io.py +98a33c75b33ff0c10a68e04bd856e4e43b3e70b6ae1ea458a2b21b4e4d74f46d tests/test_state.py 58c06925ccf06c4feee19b927b3515c540c0536aa5f3f8d29be35b8e919915a7 tests/test_temporal.py 970e8206c033c60d241732fc86f37018ffad2d66483ba235dd0eb39936ff718e tests/test_timeline_video.py -e4ec929e4227d3833887be72426054db4aff8a2219349a0683b0156b5a48271a tests/test_v217_stable_facade.py 912661d91c7f11f5fcaf0bd14d7d3ecf5c74c33b1507272f0ae72af6e5bffa6f tests/test_v2_decoder.py 36084a4f37e56786fa82200dad3e13a6a1a1ddde4f4eb5654117a88ef2659b99 tests/test_v2_prompts.py 5dfdd4b108ba7444c0f5f371c3b1e244b5735d23136127b9818fd60738c78b68 tests/test_v2_registration.py @@ -97,33 +104,56 @@ e4ec929e4227d3833887be72426054db4aff8a2219349a0683b0156b5a48271a tests/test_v21 b42bcc0983147462d96c6b71cc4a38e846995dafa6eebefc3ae0d9bb36caf359 tests/test_v2_sequence_orchestration.py 0b5b20a8318862d167f135720e0489b8d4f93406d67c6646e052e35b03dcb876 tests/test_v2_session.py 58438c3ed814adf40cefde098a37cae00ddcb703b7e5548bbbb295e54bed5a6d tests/test_v2_widget_migration.py -d8cecbeae7fa0af3f7549c718defa653d53e724325baed45ab84f3620b8bbaf4 tests/test_v31_run_storage.py -3af78ab85c8e19dee8d0a98bfd11a2c0e88c30d87256da1f9699c49a19a684e9 tests/test_v32_reference.py -f692476d0a7795b36bc5d9b97c10b7c6ba9686122b8f7987629948141e5dbab3 tests/test_v34_driving_audio.py +0410e2d10af76a83eee0e622d63b8d91662d9516c990aa657a15066e30604f58 tests/test_v217_stable_facade.py 78bc478a5e4edda902c80849c911782a99e75c06dfb04127391b6cc5b27b28e1 tests/test_v3_hardening.py eb5b27f8f0d79529c56ef344a5601703642c3cba5388c9768116223e2ed58a36 tests/test_v3_video_seam.py +eec67e2daddbbe4f5933a3ac0dd9c3188f750081d377b7d08fd6d5a02ff4a529 tests/test_v31_run_storage.py +3af78ab85c8e19dee8d0a98bfd11a2c0e88c30d87256da1f9699c49a19a684e9 tests/test_v32_reference.py +f692476d0a7795b36bc5d9b97c10b7c6ba9686122b8f7987629948141e5dbab3 tests/test_v34_driving_audio.py +8b75c23dccfd1ac302596187f2fbbd1c9b71fddcb5e666aa4f9eedfd8b5b4b93 tests/test_v35_compact_ui_frontend.py +c5a122cb9510320820f6057fc29e6bef4b42e19f141d4d9090c817702e7d151e tests/test_v35_context_sampler.py +c97d1323fa3d66c28dd1974340dbc17a3e6d5a79d4a5206b82d48bfb1a14b4ce tests/test_v35_hires_fix.py +9ceac8c3c6e4e57ec8b53b48f3b547c204df36a3f6281328aa9ab5ebc2e6a0b6 tests/test_v35_latent_resize.py +59a07f2172014dfdc0a447775324d7cd049f20b823276dfaf9b7b3f79628f5a8 tests/test_v35_pixel_vae_resize.py +4508d4d15bf48046b1664e385157f8f7f8cabdd7e6694726c8fa6e0b90ecbd1e tests/test_v35_refine_context.py +8ae5173eaec35adf9efcde2c464c95651c848248bdbdd6396ed2e3c3c1ac7965 tests/test_v35_refine_sampling.py +98a9006ebc4cab2b61b38904e0043fab24e088a25b28ffd38012c29991b8550c tests/test_v35_second_pass_contract.py +167d19efa7194c4dc52e39c3068d501023891e9a1dfba2ccf52440e7ca0d12fc tests/test_v35_second_pass_sampling.py +0dbe8a5f71c92f3523942343b40787104c8b96781461ba3ba82934a6bdbe5834 THIRD_PARTY_NOTICES.md f218e1a93c42467c6057c5c1d04d57cc82f6331b97ec9257e4e2dfb9c831876a timeline_video.py -172a16b396745ccd7d262dfbc077d6bfa33d15dfe50aa956c6c4aade76d8c099 tools/verify_runtime.py +e5a84569953ee63b9625f4f2e3195ec368204fddcfc746564cfbd0e8449bb8e9 tools/p20_file_backed_tensor_poc.py +2ff3f91da273cddefb47c9f4537326ae85642eaef3182b85d13497729d175a07 tools/verify_runtime.py ffd61f3f31c9b6430e1169007ee906c8f5000eba63ee44e2976ad4f5f1112671 v2/__init__.py 2003902c87bba09cfa143f5a9e4248f3a78c0fb544449a75fd40ba2cacb35655 v2/context_diagnostics.py cde27db19b677e318cc4dfb4235be7a11429c19945f4083052a91fe117e74c82 v2/decoder.py 31012ec48a64c7467ad869c79982b7f8502c1785c33f51ee1b5727ce8b4243fb v2/diagnostics.py 87b054be8754ebfcd85e106a7b221604611d3bf05de03380565ee3562d1c823b v2/h3_builder.py aea01a48a8c03a25ed97ea05e5b137f89e388832d1be7d38f2da2be3d1b9830b v2/motion.py -0e0fa65dcc68dfebdf5acdac2d66043906411742ff0ec4058677527760ffa742 v2/nodes.py +080e9d597e2586eb88c448c7db32b9e3661f5d21a4c270f163696966ffa6c7e0 v2/nodes.py 3c2145d47924b53f8f6ec50806f1b7113120ac314142d8b7809c4f2064ad6aef v2/prompts.py 4080f1875b958b316e8e76852c4be5527a6de5c71d604e7642d3098478eb4a36 v2/sampling.py 2fd811794c4cc0f2ce050250b80484876454926b9f20fce6af3a97d728e5acdf v2/seam_guard.py be0d8585fa57e0ccd6d02577bbc7299629e5bfe20e186747f571fccc98b67d7e v2/seam_types.py 12cfcc0c71ca087fe39a5ce2922959a62b2bb311bfe664f05c4480affce03708 v2/seeds.py -cbf26b823de838e4f2c3c68769ac817810a9c26a2f452b41ad4cc7e4b9500824 v2/sequence.py -3ea23dbce6fcd217b6e81235855479b9b16fed7a21693cf156f9014555c4fe13 v2/session.py +e6c15105fe66b9d7786602c8e6d51dd256428d80635cee686b0e35d66ab3b794 v2/sequence.py 13213e7709fd02bb810af3c18e4cd7d010f8cdfb121a4b99866a9f2333331089 v2/session_io.py +3ea23dbce6fcd217b6e81235855479b9b16fed7a21693cf156f9014555c4fe13 v2/session.py 78c830cdf49034c6e3b2cd3da5914d5cfc5638eb63ec6dbd1bab13c50d6d4caf v3/__init__.py -0823d872e152a07bf026ddb41a9339d56170f957e0bb349927be81fea803bd79 v3/assembly.py -f75497eb76a393ee0f68b7c98b8a7108f52a250084accaf3121e1d89ae154a3c v3/driving_nodes.py -9faa1274670060072b405f50c489fa5125bf4a61724b3b3709605712f7ee6e37 v3/nodes.py -66bd67c9ec7d8a7f32c7d1c1fc4ce1c19b42fda5d7cece67b171c3cffc49f0a8 v3/plan.py -892f319f0688a6f4fcac2e3a809601e63febd08065a34ff1a9f5f7521cef5dae v3/video_seam.py -67e9015be262bb29173a26f8653227836f2547c8c9582a5504a1f4812946282d version.py -44d56ea8df3265aea4d0066d6e4197265a3f7b312630cc75148acba5db20b6d5 web/project_id.js +b3dc61503d35d9ef0a0dc7fed1c9b2fdc00fcb2072d4e39f64ae0826f9ec7dca v3/assembly_v35.py +7815230e0a9540bc2edeaf818e940b7a1fdd817fd217ba8a4f9cb21aaf6a308e v3/assembly.py +1465a4cd53e4a90788c7cdbbbef5f7bc0dee2a8be04cc4c707863837077d9a47 v3/driving_nodes.py +6cc7ed74666084c55beb9075304e92a9defafd81b9c6678239ca3859a08cddd7 v3/file_backed_buffer.py +897343a4845f4ed59828bbfcfb16a22fcfce2c22907548bb3234c98dabcefff6 v3/hires_fix_nodes.py +0858487e2b8c0060a856273ffc65d23165b2f253d183f506e27e419aea0da561 v3/latent_resize_nodes.py +45bd1945b984b56369234053be2edb8455d4cff7bf976f7d9c7bef77a762b7eb v3/memory_attribution.py +f8708118f70fa2aae22951e025ed3b9499951088988c817c28f6ef7f388f2570 v3/nodes.py +b7c4a35a5e73f528492bc6946b97b687c412f1d3f5f976c19d45c6dc20288c9a v3/pixel_vae_resize.py +34ea38c03b6322288d88fc8595e3655a7a7afddb51709142a954eb7349f71ac2 v3/plan.py +10eaab14eb8a70dea3345bf8bb20069d78ec5464aef5542a9a80f1ccfdeeb9b0 v3/refine_context.py +519586b1b3f48c73d329f7a2aaa8a1bfc6fe31ac2ecb64f2b1fdf9c19026001a v3/refine_sampling.py +da5edf3b4392c41e12ef73a0cd21799be42a327b18bd5a6989447b75163dcab1 v3/second_pass_nodes.py +998d002c73c59ce88de1cf72afbb5f481a57cc31d365ba2fb4889ebf405dedc8 v3/second_pass.py +4178dcfa442abdb06ce276e05bd7bcbaa6da28469ba5e57d3da3c663de4e4a9d v3/video_seam_v35.py +2f07ace1462644f0bee553827b3571535de4d0c2d9ef847bd9e0ca73a4eed5cb v3/video_seam.py +343bf84d768b93a59c8a418635bcad4133961431c9c6f7975d7d65b0fac25eb4 version.py +ea79af984eb7d615f0d79384c8b505442af2b00cf7babfd53e82da2733927bcc web/project_id.js diff --git a/PACKAGE_INFO_JA.txt b/PACKAGE_INFO_JA.txt index 1d3700e..f180ddb 100644 --- a/PACKAGE_INFO_JA.txt +++ b/PACKAGE_INFO_JA.txt @@ -1,18 +1,25 @@ -ComfyUI-H3-Continuum V3.4.0 Stable +ComfyUI-H3-Continuum V3.5.0 MiniMax H3の5秒ChunkをAV latent contextで継続し、長尺映像として組み立てるComfyUIカスタムノードです。 -V3.4.0: -- 元音声を維持するDriving Audio -- 全Chunkで継続使用するVideo Reference -- Video Reference Size: 0.4 MP / 0.6 MP / Match Output -- Audio Seam Autoを既定化 -- Video Seam Autoによるguarded transient / micro-flash補正 -- Auto 2による実験的なexposure-ramp補正 -- Reference Image 1-3 -- Run Storage / Auto Resume -- 空欄や不完全なプロンプトを警告とfallbackで処理 -- 独自の互換性・モデル・上流ノード制限を撤廃 -- Spectrum Interop API v1 / Actual Prefix 2を維持 +V3.5.0: +- 外部加工したH3 latentをphysical group構造のまま再SamplingするSecond Pass +- Pixel/VAE round tripを使うHi-Res Fix(Experimental) +- Latent Resize utility(高倍率の直接latent補間はMain経路非推奨) +- RAM / Disk-backed / Autoを選べるAssemble + Seam V3.5 +- Exact Duration、Seam、Terminal Merge、元音声passthroughを維持 +- V3.4ノードと保存済みワークフローの互換性を維持 + +安定範囲: +- Advanced Second Pass Bridge +- Assemble + Seam V3.5のRAM / Disk-backed / Auto + +Experimental: +- Main Hi-Res Fix V3.5 +- Main Hi-Res Fix 3x5 2x(RTX 5060 Ti 16 GiBではTerminal 77TでCUDA OOM) + +GPU受入済み: +- Main Hi-Res Fix 1x5、576から1152 +- Hybrid FL2VA + Reference 1x5の統合Hi-Res FixとAdvanced Second Pass 詳細: README.md および docs/VALIDATION_RESULTS.md diff --git a/PACKAGE_VALIDATION.txt b/PACKAGE_VALIDATION.txt index 32aac9f..4a28f24 100644 --- a/PACKAGE_VALIDATION.txt +++ b/PACKAGE_VALIDATION.txt @@ -1,6 +1,6 @@ -ComfyUI-H3-Continuum V3.4.0 package validation +ComfyUI-H3-Continuum V3.5.0 package validation -Status: Stable release candidate validated for publication. +Status: V3.5.0 release. Stable V3.5 paths are validated; integrated Hi-Res Fix remains Experimental. Required automated checks: - python -m compileall -q . @@ -8,6 +8,14 @@ Required automated checks: - python tools/verify_runtime.py --comfy-root Validated runtime paths include: +- V3.5 refine-context contract and Advanced Second Pass Bridge +- 1x5 T2VA, 3x5 T2VA, and 3x5 FL2VA Long Terminal Merge Second Pass GPU acceptance +- First-pass audio LATENT object passthrough through Second Pass +- H3 Continuum Assemble + Seam V3.5 RAM / Disk-backed output equivalence +- Auto backend selection with RAM and 9.49 GiB disk-backed real-runtime cases +- Exact Duration and Video Seam direct-write behavior +- Experimental 1x5 FL2VA Pixel/VAE Hi-Res Fix GPU acceptance at 576 to 1152 +- Hybrid FL2VA + Reference 1x5 GPU acceptance through integrated 2x Hi-Res Fix and direct Advanced Second Pass - Production sampler and Core VAE Decode assembly - Reference Image 1-3 - Driving Audio absolute-time guide conditioning with original-source final output @@ -20,9 +28,8 @@ Validated runtime paths include: - Audio Seam Auto - Video Seam Auto guarded transient and micro-flash correction -Latest verified automated result: 192 passed. -Automated verification does not execute GPU checkpoint generation. +Latest verified automated result: 430 passed. -Auto 2 remains experimental. It is an optional fallback for exposure-ramp boundaries and is not the default. +Hi-Res Fix and Auto 2 remain experimental. Main 3x5 2x is not accepted on the tested RTX 5060 Ti 16 GiB because the terminal 77T Second Pass group exceeded VRAM; longer Reference/Hybrid acceptance remains pending. Detailed evidence: docs/VALIDATION_RESULTS.md diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index aca6e36..8f40dce 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,6 +1,16 @@ # Project State -Updated: 2026-08-20 +## E2-8 Second Pass contract (2026-08-22) + +- V3.4 remains the stable generation contract; UI, sockets, sampling, Run Storage identity, and Assembly behavior are unchanged. +- `assembly_plan.second_pass_contract` is additive V3.5 metadata organized by physical decode group. +- Prompt source is the stored physical prompt; a future Second Pass node must not reparse Sequence Prompt. +- Video B/C/T and physical group count/order are fixed; only latent H/W may be preserved or enlarged. +- Audio output is the original physical audio LATENT passed through bit-exact; refined audio is not adopted in V1. +- E2-8A Contract, E2-8B1 CPU/static validation, E2-8B2 external LBH GPU PoC, and E2-8C Second Pass GPU acceptance are formally PASS. The E2-8C CPU baseline is `260 passed`. +- External LATENT nodes may discard custom dictionary metadata. Identically shaped group reordering therefore cannot be cryptographically detected; count, position, T, geometry, and audio-shape checks are enforced. + +Updated: 2026-08-24 ## Baseline @@ -124,3 +134,154 @@ Static checks do not replace a real GPU generation. Record workflow, prompt, med - GPU acceptance passed for `MiniMax_H3_Continuum_V31_00008_.mp4`: 3 logical chunks, 2 physical sampling passes, decoded groups of 124 and 238 retained frames, 362-to-360 exact-duration adjustment, and 15.000-second output. - Core comparison `MiniMax_H3_00005_.mp4` showed the same short terminal still period. This is accepted as normal FL2VA Last Frame convergence, not a Continuum-specific defect. - Long Terminal Merge is formally adopted. Do not retune Conditioning, Physical Seed v2, physical decode grouping, or terminal prompt pairing without new contrary regression evidence. + +## V3.5 Hi-res Fix and Second Pass (2026-08-23) + +- The original V3.5 Main candidate was first-pass physical video latents -> lightweight Core spatial resize -> low-sigma Second Pass -> Core Decode -> Assemble + Seam. The 2026-08-24 GPU isolation below rejects plain latent interpolation as the general Main backend; the Advanced Second Pass bridge remains valid. +- E2-8A and E2-8B1 established additive `assembly_plan.second_pass_contract` metadata and static physical-group validation without changing V3.4 generation semantics. +- E2-8B2 GPU PoC is accepted: LBH 3D Upscaler preserved H3 24-channel physical groups, including the 3x5 FL2VA 124f + 260f Terminal Merge layout. +- E2-8C adds an experimental T2VA-only physical-group Second Pass node. It reads prompts from the plan, uses workflow-supplied SIGMAS, derives one refine seed per physical group, adopts refined video, and returns first-pass audio LATENT objects unchanged. +- `H3ContinuumLatentResizeV35` is the standalone lightweight Utility. It delegates to ComfyUI Core `common_upscale`, exposes the same nearest-exact/bilinear/area/bicubic/bislerp methods and a scale multiplier, preserves physical-group order and B/C/T, and changes only latent H/W. It is not an accepted general-purpose Main Hi-res backend. +- The final V3.5 node layout keeps three distinct roles: `H3ContinuumHiResFixV35` is the normal one-node Main path, `H3ContinuumSecondPassV35` is the Advanced entry for LBH or other externally processed latents, and `H3ContinuumLatentResizeV35` remains a standalone Utility. +- Hi-Res Fix ON performs an H3 Video VAE Decode -> CPU pixel resize -> same H3 Video VAE Encode for each physical group, then calls the existing context-aware physical-group Second Pass helper. Hi-Res Fix OFF returns the exact first-pass video/audio LATENT lists and original Assembly Plan object; all sampling and VAE inputs remain lazy and are not requested by the node. +- Second Pass itself has no enable switch and no duplicate first-pass input. Its public display name is `H3 Continuum Second Pass V3.5`; the Utility display name is `H3 Continuum Latent Resize V3.5`. +- E2-8C GPU acceptance is formally PASS for 1x5 T2VA, 3x5 T2VA, and 3x5 FL2VA Long Terminal Merge. +- The accepted FL2VA run preserved two physical groups as `37T + 77T`: group 1 mapped logical `[1]`, and group 2 mapped logical `[2,3]` with `paired_timeline_v1`. Each physical group was sampled exactly once by Second Pass. +- Final output was 768x768, 360 frames at 24 fps, and 15.000 seconds. First-pass audio LATENT passthrough, Core Video/Audio Decode, and Assemble succeeded; the 5-second and 10-second boundaries had no major discontinuity. +- The final approximately 0.5-second low-motion interval is accepted as natural Last Frame convergence, not temporal collapse. +- V3.4 Sampling, Terminal Merge, Assembly, Seam, and Run Storage were not changed by E2-8C. +- First/Last, Reference, Hybrid, Run Storage, and embedded LBH integration remain outside E2-8C. +- The current V3.5 working tree uses a Second Pass-only direct-sampling contract: video receives seed-derived random noise with mask 1, while first-pass audio receives zero noise with mask 0. Core applies workflow SIGMAS once; no external AddNoise, inverse noise scaling, or `audio / (1-sigma0)` compensation is used. +- This path is isolated in `v3/refine_sampling.py`; V3.4 `v2/sampling.py::sample_chunk()` remains unchanged. The Assembly Plan records execution as `t2va_physical_groups_audio_locked_v2` and audio sampling as `zero_noise_mask_locked`. +- CPU validation is `277 passed`. Core `reshape_mask` also expands the minimal video `[B,1,1,1,1]` and audio `[B,1,1,1]` masks correctly for real H3 stream shapes. +- The audio-locked v2 path is now GPU-confirmed for direct no-resize Second Pass: `V35_NoResize_SecondPass_Diagnostic_00001_.mp4` used the cached 576x576 first-pass latent, `res_multistep`, `simple` 10 steps / denoise 0.35, and refine seed 42. Its report confirmed video random noise/mask 1, audio zero noise/mask 0, one 37T physical group, and one sampling pass. +- The direct no-resize result is visually clean across the full 120-frame / 5.000-second output. The cyan/orange/white line and plate artifacts remain in the otherwise matched one-node Hi-Res Fix run `E2-8B2_01_1x5_T2VA_LBH3D_00017_.mp4`, which used bilinear 2x from 576x576 to 1152x1152. +- The fixed-condition GPU isolation therefore rejects random audio re-noise as the primary artifact cause and validates the basic direct Second Pass re-noise path. The remaining cause is scoped to the 2x spatial-resize/high-resolution re-entry path; this result alone does not yet distinguish interpolation method from scale/resolution. +- ComfyUI_W registration, native PackedLayout, and Fixed 3x5 prompt-plan verification passed before this isolation. The one-node Hi-Res Fix wrapper executes successfully on GPU, but its bilinear 2x path is visually rejected and is not an accepted standard Hi-res Fix path. +- Resolution/method isolation used the same cached first-pass latent, `res_multistep`, `simple` 10 steps / denoise 0.35, and refine seed 42 throughout. Both nearest-exact and bilinear were clean at 576x576 -> 768x768 (`36x36 -> 48x48` latent), while bilinear failed at 1024x1024 (`64x64`) and both bilinear and nearest-exact failed at 1152x1152 (`72x72`). +- This rejects bilinear-specific channel mixing as the primary cause within the direct interpolated-latent path. The tested safe square target for that legacy path is 768x768; its exact failure threshold between 768 and 1024 was not mapped. ComfyUI Core's H3 implementation independently defines `BASE_SHORT_EDGE = 768` and `MAX_PIXELS = 768 * 1344`, consistent with the observed direct-re-entry boundary. These limits do not describe the later Pixel/VAE roundtrip result. +- The current `H3ContinuumHiResFixV35` defaults to pixel Lanczos and `scale_by=2.0`; zero retains the H3 native-canvas Auto option and positive values retain manual multiplier semantics. Enabled execution requires the lazy `video_vae` input and never falls back to direct latent interpolation. Disabled execution does not evaluate the VAE or sampling inputs. +- Manual mode is not clamped or stopped when it exceeds the Core native canvas; the status reports the target as informational and execution continues through the Pixel/VAE backend. No V3.4 constraint or independent target policy was added. +- `V35_NativeAuto_576to768_Acceptance_00001_.mp4` remains historical evidence for the former direct interpolated-latent backend only. Main Hi-Res Fix no longer uses that backend; the explicit `H3ContinuumLatentResizeV35` Utility remains available separately. +- The earlier recommendation to move all 2x delivery to a post-decode video upscaler is superseded for Main Hi-res Fix by the validated Pixel/VAE roundtrip in the next section. Issue #8's external-latent Second Pass bridge remains valid independently. + +## V3.5 context-aware Hi-res isolation (2026-08-24) + +- Added `H3ContinuumSamplerV35` without changing the V3.4 public contract. Its first five outputs are byte-for-byte ordered like V3.4 (`video_latents`, `audio_latents`, `assembly_plan`, `status`, `driving_audio`); output six is runtime-only `H3_CONTINUUM_REFINE_CONTEXT`. +- The context captures the exact pre-sampling conditioning once per physical group, including Terminal Merge logical mapping, physical clip index, context frames, prepared First/Last images, and source H3 latent geometry. It is never stored in Assembly Plan, Session, logical entries, or Run Storage. +- A complete context is adapted to target H/W by re-encoding First/Last keyframes with the connected Video VAE and resizing only marked Continuum video-context latents. Ordinary MiniMax references, text conditioning, and all audio conditioning remain unchanged. Incomplete Run Storage capture falls back to the legacy prompt-only path with a warning; a corrupt typed context is a contract error. +- Second Pass now creates one call-local Continuum MODEL clone per physical group. The audio-lock contract remains video random noise/mask 1, audio zero noise/mask 0, one workflow SIGMAS application, one refine seed and one sampling call per physical group, and bit-exact first-pass audio LATENT passthrough. +- CPU/static validation after integration is `293 passed` with only the known `pynvml` FutureWarning. Runtime registration exposes `H3 Continuum Sampler V3.5` with six outputs. V3.4 Sampling, Terminal Merge, Assembly, Seam, Run Storage, and its five-output UI contract remain unchanged. +- GPU A/B `0adff3b7-1552-47e9-a167-7b6e4010b0a5` used the fixed FL2VA 1x5 condition, 576x576 -> 864x864 bilinear 1.5x, `res_multistep`, `simple` 10 steps / denoise 0.35, and refine seed 42. The report confirmed `conditioning_source=refine_context`, two keyframes re-encoded at target H/W, shape `(1, 24, 37, 54, 54)`, and first-pass audio passthrough. Output `V35_ContextAware_FL2VA_576to864_d035_00001_.mp4` still contains the persistent cyan/orange/white line and plate artifacts. Context mismatch is therefore not the primary cause. +- Resize-only GPU isolation `75249bbe-306a-45f9-a139-a79c0b17e536` decoded a plain bilinear 576x576 -> 864x864 latent as `V35_ResizeOnly_FL2VA_576to864_bilinear_00001_.mp4`. It removed the colored lines but produced severe person/background ghosting, confirming that channel-wise interpolation creates an invalid H3 video-latent representation even before refinement. +- Pixel/VAE roundtrip GPU isolation `2a03a1d4-2355-4d5a-a237-0ec3e0c09761` used First Pass Decode -> pixel Lanczos 864x864 -> H3 Video VAE Encode -> the same context-aware denoise 0.35 Second Pass. `V35_PixelRoundtrip_FL2VA_576to864_d035_00001_.mp4` completed at 864x864, 120 frames, 24 fps, 5.000 seconds with no colored-line artifacts or resize ghosting while retaining the orange jacket, subject identity, motion, and terminal composition. +- Pixel/VAE roundtrip 2x GPU acceptance `de11e383-8b3d-4d8a-806f-1ac3550091f1` used the same cached First Pass, pixel Lanczos 1152x1152, H3 Video VAE Encode, context-aware `res_multistep` / `simple` 10-step denoise 0.35 Second Pass, and refine seed 42. It completed without OOM on the 16 GiB GPU as `(1, 24, 37, 72, 72)`, one physical group / one sampling pass, two target keyframes VAE re-encoded, first-pass audio passthrough, Core Decode/Audio Decode, and Assemble. `V35_PixelRoundtrip_FL2VA_576to1152_d035_00001_.mp4` is 1152x1152, 120 frames, 24 fps, and 5.000 seconds; twelve representative time points are visually clean with no colored lines/plates, resize ghosting, or temporal collapse, while subject identity, orange clothing, motion, and composition remain coherent. +- The evidence-backed Main backend is implemented as a sequential Pixel/VAE roundtrip per physical group followed by the existing context-aware Second Pass. Pixel resize uses bounded 8-frame CPU batches; VAE Encode matches Core's 4D IMAGE contract by encoding each Continuum batch as `[F,H,W,C]` and then restoring B. Plain interpolated latent is never used as a Main fallback. `H3ContinuumSecondPassV35` remains the Advanced external-LBH/learned-upscaler bridge, and `H3ContinuumLatentResizeV35` remains an explicit Utility. +- CPU/static validation after the Main backend implementation is `308 passed`; ComfyUI_W runtime registration, native PackedLayout, Fixed 3x5 prompt-plan verification, `py_compile`, and `git diff --check` pass. The only test-environment messages are the known pytest cache permission warning and `pynvml` FutureWarning. +- Integrated Main GPU acceptance prompt `c5dbe55c-c846-47e8-9f49-8bb167887236` used FL2VA 1x5, 576x576 -> 1152x1152, pixel Lanczos, context-aware `res_multistep` / `simple` 10-step denoise 0.35 Second Pass, and refine seed input 42. It completed in 14:18 as one physical group / one sampling pass with `(1, 24, 37, 72, 72)`, two target keyframes re-encoded, and first-pass audio LATENT passthrough. +- `V35_MainSafePixelVAE_Core4D_FL2VA_576to1152_d035_00001_.mp4` is 1152x1152, 120 frames, 24 fps, and 5.000 seconds with 32 kHz stereo audio. Its decoded 120 video-frame hashes and all decoded audio-frame hashes are exactly identical to the accepted external four-node Pixel/VAE baseline. The prior colored line/plate and resize-ghosting artifacts are absent. Both baselines enter the same FL2VA Last Frame still interval at 3.666667 seconds; this is not an integrated-node regression. +- This acceptance is limited to the 1x5 FL2VA 2x case. Peak observed host use was approximately 47.7 GiB working set / 61.3 GiB private bytes with about 7.3 GiB system RAM still available; 3x5 Terminal Merge 2x remains a separate GPU/resource acceptance. V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, Advanced Second Pass, and the Latent Resize Utility were not changed by this Main backend implementation. + +## Phase 2A P2-0 file-backed Tensor backend PoC (2026-08-24) + +- P2-0 is accepted as a source-only Windows backend/lifetime PoC. It is not imported or registered by the Production node package, and neither ComfyUI_W nor ComfyUI_WAN runtime files were changed. +- The selected candidate is `torch.from_file(shared=True)`: it returns a contiguous CPU Tensor backed by a dedicated file while preserving the normal `IMAGE` Tensor contract. +- Backing files use strict allocation IDs plus JSON ownership sidecars. Cleanup is serialized per root across threads/processes, fails closed for invalid ownership metadata, retains live/cache-referenced storage, retries sharing violations, and removes dead-process stale files or eligible orphans. +- Storage lifetime is alias-aware through PyTorch `StorageWeakRef`. Future Production integration must use the process-global `get_process_lifetime_manager()`; a node-local manager is not an accepted integration contract. +- GC finalizers only enqueue cleanup intent. Filesystem I/O uses a consistent root-lock-before-manager-lock order, so cleanup/collect/finalizer interleavings cannot form the reproduced ABBA deadlock. Cancellation cleanup also protects a numeric file descriptor that has already been closed and reused by unrelated code. +- Primary validation: 26 focused tests and the standalone probe pass under ComfyUI_W Python/PyTorch; full repository validation is 334 passed. Secondary validation passes the same 26 tests and probe under ComfyUI_WAN Python 3.12.12 / PyTorch 2.13.0+cu130. Both probes are bit-exact, defer cleanup while an alias is alive, reclaim after the final alias is released, and leave zero managed backing files. +- This P2-0 acceptance does not claim end-to-end low-memory assembly. Real ComfyUI cache/requeue, Preview/VHS/Save consumption, RAM-vs-Disk assembler equivalence, and 9-15 GiB sequential-write/RSS behavior remain explicit later gates (P2-4/P2-5). Invalid sidecar quarantine/manual recovery is also not yet a public policy. +- P2-0 is now consumed by the independent V3.5 Production path described below. V3.4 remains fixed. + +## Phase 2A P2-1 through P2-3 CPU acceptance (2026-08-24) + +- P2-1 Memory Attribution is complete. V3.5 sequence reports capture chronological start, before CPU commit, after CPU commit, and completion snapshots; deduplicated retained AV/refine-context CPU/GPU bytes; projected natural/exact IMAGE bytes; process RSS/system availability; and read-only CUDA allocation/reservation counters. Collection is fail-soft and does not synchronize CUDA, reset peaks, clear caches, or alter V3.4 default behavior. +- P2-2 added `v3/file_backed_buffer.py`, backed by the accepted P2-0 `torch.from_file(shared=True)` lifetime contract. P2-6 now exposes `Auto` plus the original manual `RAM` and `Disk-backed` choices; there is still no silent Disk-to-RAM fallback. The first manager access performs root-scoped stale cleanup while live mappings remain protected by the ownership/lock contract. +- P2-3 adds the independent `v3/assembly_v35.py` and `v3/video_seam_v35.py` path used by `H3ContinuumAssembleSeamV35`. Video alone may be file-backed; Audio remains in RAM and preserves the V3.4 seam/sample-alignment result. +- Exact Duration allocates the final target-frame IMAGE shape first and writes trim, pad, and optional preserved final-frame spans directly. The assembler does not construct the final IMAGE with full-tensor `cat`, `stack`, `clone`, or `contiguous` copies. Audio remains intentionally RAM-backed and may use separate natural/target RAM buffers. +- Physical `decode_groups` retain their original order and terminal-prefix trim contract. Video Seam reuses the V3.4 analysis/formula/action semantics but copies only the affected 1-4 boundary frames plus required anchors; an analysis/patch error falls back transactionally to native frames. +- Disk-backed publication occurs only after assembly, reporting, and Driving Audio validation succeed. Unpublished early/late errors abort the backing pair. A published IMAGE remains valid while any Tensor/storage alias is retained, and the backing file becomes cleanup-eligible only after the final alias is released. +- P2-3 CPU acceptance covers RAM/Disk bit-exact output, Seam Off/Analyze/Auto/Auto 2, Exact Duration ON/OFF, trim/pad/preserve-final behavior, 1x5, 2x5 Terminal Merge, 3x5 Long Terminal Merge, 32 kHz cumulative audio rounding, Audio Seam, physical order/trim, file lifetime, stale recovery, and error cleanup. Its original Auto-rejection gate is superseded by the accepted P2-6 policy below. +- Canonical full validation is `412 passed` with only the known `pynvml` FutureWarning. `py_compile` and `git diff --check` pass. The ComfyUI_W runtime verifier passes registration, native PackedLayout, and Fixed 3x5 prompt-plan checks; its focused P2 regression is `141 passed`. +- Pre-change snapshots are `pre-phase2a-p2-1-memory-attribution-20260824_030858` and `pre-phase2a-p2-2-disk-backed-20260824_032923`; the latter contains 166/166 source files with zero SHA-256 mismatch at revision `d77a467796c048908f8b7b83736e226e736e0f98`, plus both runtime copies. +- Only the P2-1 through P2-3 target files were synchronized to primary ComfyUI_W; all 19 source-or-preserved-merge comparisons match SHA-256. Its pre-existing Video Reference tooltip addition was preserved. ComfyUI_WAN was deliberately not partially synchronized because its prior V3.5 dependency set is incomplete. +- P2-4 and the assembler-focused P2-5 acceptance are accepted below. Auto backend selection remains deliberately unimplemented; no new sampler/model generation acceptance is claimed by P2-5. + +## Phase 2A P2-4 ComfyUI integration acceptance (2026-08-24) + +- P2-4 is formally PASS using ComfyUI_W's installed `PromptExecutor` with the default CLASSIC `HierarchicalCache`, Core `SaveImage`/`PreviewImage`, and installed Video Helper Suite `VideoCombine`. The deterministic fixture is decoded CPU IMAGE/AUDIO only; no model, VAE, sampler, media input, CUDA allocation, or GPU generation is involved. +- `v3/assembly_v35.py` now polls ComfyUI's normal interrupt flag between bounded 8-frame direct-copy batches and immediately after the provisional output allocation. This changes only the independent V3.5 writer; V3.4 Assembly/Seam remains untouched. RAM and Disk-backed outputs remain bit-exact. +- A completed Disk-backed IMAGE remains alive after an intentional downstream error because the actual Core cache retains it. Requeue with unchanged assembler inputs reuses the same cached output and creates no additional backing pair. After cache eviction, the next allocation reclaims the released pair before creating its replacement, so managed pair count remains bounded; final managed files are zero. +- Core Save wrote 120 PNG frames and Core Preview wrote 120 temporary PNG frames from the mapped IMAGE. VHS streamed H.264 plus AAC from the same IMAGE/AUDIO. `ffprobe` verified 16x16, 120 frames, 24 fps, 5.000 seconds, and 32 kHz audio; this small geometry is intentional for compatibility/lifetime isolation. +- Core interrupt propagation is accepted: the probe raises the same `InterruptProcessingException` path used by ESC, PromptExecutor reports `execution_interrupted`, and the unpublished Disk-backed pair is removed. Separate-process crash/restart simulation leaves one stale pair and the next process removes it on first manager access. +- Reusable probe: `tools/p24_comfy_integration_probe.py`. Final runtime evidence is under `D:\Codex\_test_results\ComfyUI-H3-Continuum\P24-runtime-W-final-20260824_1105` (`p24_result.json`, exact prompt graphs, 120 Save frames, 120 Preview frames, and the VHS MP4). +- Validation: full canonical pytest `415 passed`; P2-focused ComfyUI_W runtime pytest `89 passed`; runtime registration/native PackedLayout/Fixed 3x5 prompt-plan verification, changed-file `py_compile`, and `git diff --check` pass. Only the known `pynvml` FutureWarning remains. +- Accepted pre-change snapshots are `pre-phase2a-p2-4-comfy-integration-source-verified-20260824_101909` (415 files, zero SHA-256 mismatch, revision `d77a467796c048908f8b7b83736e226e736e0f98`) and `pre-phase2a-p2-4-comfy-integration-comfyui-w-verified-20260824_101938` (324 files, zero mismatch). The standard snapshot script attempts failed only because the pre-existing `.pytest_cache` ACL denies traversal; those incomplete attempts are not accepted snapshots. +- Only `v3/assembly_v35.py`, the P2-4 probe, and its test file were synchronized to ComfyUI_W; source/runtime SHA-256 matches. P2-4 does not claim RAM-pressure/LRU cache modes, arbitrary downstream nodes, large-output memory behavior, manual browser UI interaction, or GPU/long-duration acceptance. + +## V3.5 compact UI parity (2026-08-24) + +- `web/project_id.js` now applies the established V3.4 compact/runtime-settings path to `H3ContinuumSamplerV35` and `H3ContinuumAssembleSeamV35`. No Python schema, socket, Sampling, Conditioning, Assembly, Seam, or Run Storage behavior changed. +- Sampler `Report Detail`, `debug`, and `show_preview` remain hidden widgets controlled by the existing H3-Continuum Settings entries. `strict_compatibility` remains loadable for legacy workflows but is hidden and forced `false` before queueing. +- `Auto Resume ID Override` remains always hidden and internally generated. `Run Name (Optional Override)` and `Regenerate From` are visible only when Run Storage is `Save + Auto Resume`; `Variation Nonce` is visible only for an explicit stored-chunk regeneration. +- The V3.5 assembler's `Report Detail` and `Exact Duration` controls use the same hidden Settings/fixed-contract path as V3.4. +- Accepted snapshots: source `pre-v35-compact-ui-source-verified-20260824_104010` (420 files, SHA-256 mismatch 0, excluding `.git` and inaccessible `.pytest_cache`) and ComfyUI_W `pre-v35-compact-ui-runtime-w-verified-20260824_104048` (330 files, mismatch 0, excluding `.pytest_cache`). The standard snapshot attempts are incomplete because that existing cache ACL denies traversal. +- Validation: dedicated frontend regression `2 passed`, JavaScript module syntax PASS, and full canonical pytest `417 passed`. The initial full-test attempts stopped only at inaccessible default pytest temp/cache paths; the canonical run used a fresh writable `D:\Codex\_test_temp` base. P2-5 is accepted below. +- Synchronized only `web/project_id.js` and its dedicated regression test to ComfyUI_W. Source/runtime SHA-256 matches (`EA79AF984EB7D615F0D79384C8B505442AF2B00CF7BABFD53E82DA2733927BCC` and `8B75C23DCCFD1AC302596187F2FBBD1C9B71FDDCB5E666AA4F9EEDFD8B5B4B93`). + +## Phase 2A P2-5 long-output acceptance (2026-08-24) + +- P2-5 is formally PASS for the manual `RAM` / `Disk-backed` V3.5 Assembly path. Testing was headless through ComfyUI_W's installed `PromptExecutor`; no browser, mouse, or front-end automation was used. +- The real 3x5 FL2VA Long Terminal Merge fixture reuses accepted Run Storage revision `1e30d8bf2154bbe7`: 3 logical chunks, 2 physical decode groups (`37T + 77T`), Core Video/Audio VAE Decode on GPU, 640x640, 360 frames, 24 fps, 15.000 seconds, and 32 kHz stereo audio. No new sampler/model generation was run. +- Independent fresh-process RAM and Disk-backed results are bit-exact. Final video SHA-256 is `95d5479bc5fee9c235ece7996c7c88dcc60981625dbedc330344403016ca9b9d`; audio SHA-256 is `eec0c6ff378c5bfcc3e71d02bab14230dea02a57e18ce1212f1dc1be4c88a3f9`. Exact Duration applies `362 -> 360` frames and `-2667` audio samples identically; Seam Auto and physical group order/trim match. +- Real-size Core Save and Preview each consumed all 360 mapped frames. VHS produced H.264/AAC at 640x640, 360 frames, 24 fps, 15.000 seconds, and 32 kHz. Disk-backed downstream peak RSS was about 12.10 GiB versus about 11.70 GiB for Assemble/hash alone; no catastrophic full-output duplicate was observed. +- Same-input requeue reports every node through assembler/digest cached, retains exactly one backing pair, and creates no additional pair. Cache release removes managed `.bin`/`.json` files. Core interrupt raised after backing allocation follows `execution_interrupted` and leaves zero managed files. A separate crash/restart recheck leaves one stale pair and removes it on first manager access in the next process. +- The 9.49 GiB stress uses a deterministic 1536x1536x360 float32 IMAGE while preserving the accepted two-group Long Terminal Merge plan. RAM and Disk-backed video/audio hashes are bit-exact; both are finite; Disk-backed holds one 10,192,158,720-byte file while cached and leaves zero managed files after release. +- On Windows, RSS/working set is not the acceptance metric for file-backed savings because resident mmap pages are counted in RSS. At the stress digest boundary, RAM and Disk-backed RSS were both about 10.44 GiB. The correct commitment metrics show the benefit: RAM private/USS were 13,322,362,880 / 11,045,199,872 bytes, while Disk-backed private/USS were 3,122,954,240 / 855,834,624 bytes. File backing therefore removed about 9.50 GiB of anonymous private commitment and about 9.49 GiB of unique resident commitment for a 9.49 GiB output. +- Reusable probe: `tools/p25_gpu_acceptance_probe.py`; focused P2-5/P2-4/P2-3 regression is `58 passed`, and full canonical pytest is `420 passed` with only the known `pynvml` FutureWarning. `py_compile` and `git diff --check` pass. +- Accepted snapshots are `pre-phase2a-p2-5-gpu-acceptance-source-verified-20260824_105605` (422 files, mismatch 0) and `pre-phase2a-p2-5-gpu-acceptance-runtime-w-verified-20260824_105626` (331 files, mismatch 0). P2-5 changed only the source-side probe, its tests, and project records; no Production runtime sync was required. +- Primary evidence roots: `P25-long-terminal-ab-20260824_110031`, `P25-downstream-disk-20260824_110515`, `P25-interrupt-disk-20260824_110815`, `P25-cache-requeue-disk-20260824_111126`, `P25-stress-1536-disk-private-20260824_111836`, `P25-stress-1536-ram-private-20260824_111907`, and `P25-lifetime-recheck-20260824_111947` under `D:\Codex\_test_results\ComfyUI-H3-Continuum`. +- Auto backend is accepted in P2-6 below. Arbitrary downstream nodes may still materialize their own full copies. New 3x5 sampler/model generation and 3x5 Main Hi-res Fix 2x GPU/resource acceptance remain separate work; P2-5 validates the assembler using already accepted physical latents and real Core GPU Decode. + +## Phase 2A P2-6 Auto Buffer Backend acceptance (2026-08-24) + +- P2-6 is formally PASS. `H3ContinuumAssembleSeamV35` now offers `Auto`, `RAM`, and `Disk-backed`; `Auto` is the default for newly created V3.5 assembler nodes. Existing saved string values for either manual backend remain valid and retain their exact manual behavior. +- Auto selects RAM only when the final IMAGE is at most 4 GiB and available physical memory is at least `final IMAGE + max(4 GiB, 10% of total physical RAM)`. Larger outputs or insufficient headroom select Disk-backed. If memory counters are unavailable, Auto chooses Disk-backed rather than assuming RAM is safe. +- A Disk-backed Auto decision requires free space for the final IMAGE plus a fixed 2 GiB disk reserve. Capacity-probe failure, insufficient disk, or allocation failure is explicit and never falls back to RAM. Users may still select manual RAM when they independently know it is safe. +- The V3.5 report records requested/resolved backend, final IMAGE bytes, available/total RAM, computed RAM reserve, 4 GiB RAM limit, disk free/reserve values, and the decision reason. Manual paths do not query or change Auto policy. +- Real headless acceptance `P26-auto-ram-long-terminal-20260824_113400` reused the accepted 3x5 FL2VA Long Terminal Merge revision and Core GPU Decode. The 1,769,472,000-byte output selected `Auto -> RAM` with about 39.2 GiB available and a 6.39 GiB RAM reserve. Its 640x640 / 360f / 15.000s video and 32 kHz audio hashes exactly match P2-5. +- Large-output acceptance `P26-auto-disk-stress-20260824_113610` used the 10,192,158,720-byte / 9.49 GiB IMAGE fixture and selected `Auto -> Disk-backed` because the final IMAGE exceeded 4 GiB. It retained one backing pair while cached, returned the accepted stress video/audio hashes, and left zero managed files after release. +- Focused Auto/P2 regression is `66 passed`; full canonical pytest is `428 passed`; changed-file and runtime `py_compile`, runtime registration/native PackedLayout/Fixed 3x5 verification, and `git diff --check` pass. Only the known `pynvml` FutureWarning remains. +- Accepted snapshots are `pre-phase2a-p2-6-auto-source-verified-20260824_112744` (426 files, mismatch 0) and `pre-phase2a-p2-6-auto-runtime-w-verified-20260824_112744` (331 files, mismatch 0), excluding the pre-existing inaccessible `.pytest_cache`. Only `v3/file_backed_buffer.py`, `v3/assembly_v35.py`, and the Auto portion of `v3/driving_nodes.py` were synchronized to ComfyUI_W; its pre-existing Video Reference tooltip addition remains preserved. +- V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, and every V3.4 socket remain unchanged. Auto does not guarantee that arbitrary downstream nodes avoid their own full copies. New 3x5 sampler/model generation and 3x5 Main Hi-res Fix 2x resource acceptance remain separate. + +## V3.5 release-candidate preparation (2026-08-24) + +- Public package version, Python package metadata, `metadata.ini`, Changelog, English/Japanese README, package information, and validation text now identify `3.5.0` consistently. +- Public scope is explicit: Sampler V3.5 refine-context, Advanced Second Pass Bridge, and Assemble + Seam V3.5 are release paths; integrated Main Hi-Res Fix remains Experimental. The standalone Latent Resize node is documented as an advanced utility rather than the recommended Main 2x backend. +- Added `docs/V35_HIRES_FIX.md` with standard Main and external-upscaler wiring, separate First/Second Pass scheduler roles, the accepted `simple / 10 steps / denoise 0.35` baseline, OFF passthrough, audio behavior, and current GPU limits. No diagnostic UI workflow is published as an accepted V3.5 template. +- Distribution Manifest includes the V3.5 Production modules, CPU regressions, release metadata test, public guide, and the Production-imported P2-0 backend. P2-4/P2-5 runtime probes remain repository validation tools rather than distribution Manifest payloads. No credentials or generated test media are included. +- Release metadata regression and targeted V3.5 tests pass (`75 passed`). Full canonical pytest is `430 passed`; only the known `pynvml` FutureWarning remains. Source and ComfyUI_W runtime verifier both report package `3.5.0`, native PackedLayout PASS, Fixed 3x5 prompt-plan PASS, and the complete expected node registry. +- Representative headless GPU revalidation `V35-release-candidate-auto-long-terminal-20260824` is PASS. Accepted 3x5 FL2VA Long Terminal Merge latents decoded through Core GPU Video/Audio VAE and Assemble V3.5 Auto as 3 logical / 2 physical groups (`37T + 77T`), 640x640, 360 frames, 24 fps, 15.000 seconds. Auto selected RAM for 1,769,472,000 bytes; video/audio SHA-256 exactly match P2-5/P2-6, all outputs are finite, and managed backing files after release are zero. +- Remaining publication limits: Main Hi-Res Fix 3x5 2x and Reference/Hybrid-specific Second Pass GPU acceptance are pending; arbitrary downstream full IMAGE copies are outside Continuum's guarantee. Existing V3.4 workflows remain supported. +- Verified release-preparation snapshots are `pre-v35-release-preparation-source-verified-20260824_115858` (426 files, mismatch 0) and `pre-v35-release-preparation-runtime-w-verified-20260824_115858` (331 files, mismatch 0). Commit/push remain pending explicit user instruction. + +## V3.5 pending GPU acceptance closure (2026-08-24) + +- Real ComfyUI API acceptance used an NVIDIA GeForce RTX 5060 Ti with 16,311 MiB VRAM. Production code was not changed. +- Main Hi-Res Fix `3 x 5s` FL2VA, 576x576 to 1152x1152 Lanczos, `res_multistep`, workflow `simple / 10 steps / denoise 0.35`, preserved 3 logical chunks and 2 physical groups (`37T + 77T`). First Pass and the 37T group's 10-step Second Pass completed. The terminal logical `[2,3]` / 77T group failed at its first Second Pass inference with CUDA OOM: 15.93 GiB device limit, 10.04 GiB allocated, 1.49 GiB requested, zero CUDA free. This condition is formally not accepted on the tested 16 GiB GPU; it is a resource boundary, not evidence that physical grouping changed. +- Hybrid FL2VA + Reference `1 x 5s`, 576x576 to 1152x1152, passed the integrated Main Hi-Res Fix. The node used `conditioning_source=refine_context`, one physical group / one sampling pass, actual refine seed `2946484852083492747`, output latent `(1, 24, 37, 72, 72)`, temporary refined audio discard, and bit-exact first-pass audio LATENT passthrough. Core Decode / Assemble produced 1152x1152, 120 frames, 24 fps, 5.000 seconds with 32 kHz stereo audio; visual inspection found no persistent colored line/plate artifact, flash, or temporal collapse. Output SHA-256 is `E5B27CABCD5FBC1422F169AC6E75570EF4D54010CD04A3439852D8DED15C1384`. +- The same Hybrid/Reference input also passed the public `H3ContinuumSecondPassV35` node directly at 576x576: `conditioning_source=refine_context`, actual refine seed `5765580629374593532`, shape `(1, 24, 37, 36, 36)`, one sampling pass, bit-exact first-pass audio passthrough, Core Decode / Assemble 120 frames / 5.000 seconds, and clean contact-sheet inspection. Output SHA-256 is `67311A797CD812584B4E2CCC21D7DF83AE8C1F1D72AB8BF5FA1C142D00894D2D`. +- The Hybrid prompt used the plain text `Picture 3` instead of the recommended explicit `` tag, so preflight emitted a non-blocking H3C-P103 warning. The connected reference still participated in First Pass conditioning; the run proves transport/execution acceptance, not a warning-free public prompt example. +- Evidence root: `D:\Codex\_test_results\ComfyUI-H3-Continuum\V35-pending-gpu-acceptance-20260824`. Remaining limits are Main 3x5 2x on larger VRAM or a lower-memory terminal-group implementation, and Reference/Hybrid durations longer than 1x5. +- Pre-documentation snapshots are `pre-v35-pending-gpu-results-source-verified-20260824_130824` (431 files) and `pre-v35-pending-gpu-results-runtime-w-verified-20260824_130824` (334 files), excluding `.git` and the pre-existing inaccessible `.pytest_cache`. + +## V3.5.0 formal release documentation (2026-08-24) + +- V3.5.0 is the current formal release. The public README now leads with the two major additions: Continuum-aware Second Pass / experimental integrated Hi-Res Fix, and low-memory Assemble + Seam V3.5. +- V3.4 node IDs remain registered intentionally for saved-workflow compatibility. V3.5 does not replace V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, or public sockets; existing V3.4 workflows can continue unchanged. +- Added separate public node images `docs/images/v35-hires-fix-node.png` and `docs/images/v35-second-pass-node.png`. BasicScheduler is omitted visually but the README explicitly requires workflow-side SIGMAS and records the accepted `res_multistep`, `simple / 10 steps / denoise 0.35` baseline. The Main image shows the accepted Lanczos 2x selection. +- Public memory claims are limited to measured Windows host-memory behavior: for the 9.49 GiB 1536x1536x360 stress IMAGE, Disk-backed reduced private memory by about 9.50 GiB and USS by about 9.49 GiB while preserving exact video/audio hashes. No sampler GPU-VRAM reduction is claimed. +- Public GPU limits now distinguish accepted 1x5 576-to-1152 Main and Hybrid/Reference paths from the unaccepted 3x5 2x RTX 5060 Ti 16 GiB condition, which failed at the terminal 77T group's first Second Pass inference after the 37T group completed. +- Final source validation is `430 passed`; all 159 Manifest entries verify by SHA-256; `git diff --check` passes. ComfyUI_W runtime verifier passes package 3.5.0 registration, native PackedLayout, and Fixed 3x5 planning. Nine changed public documentation/assets match the snapshotted runtime by SHA-256. +- Verified pre-release snapshots are `pre-v35-formal-readme-release-source-verified-20260824_141358` (431 files) and `pre-v35-formal-readme-release-runtime-w-verified-20260824_141359` (334 files), excluding `.git` and the pre-existing inaccessible `.pytest_cache`. diff --git a/README.md b/README.md index 53ad9e6..2c646cd 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,102 @@ -# ComfyUI-H3-Continuum 3.4.0 +# ComfyUI-H3-Continuum 3.5.0 -> **V3.4 hotfix notice:** The initial V3.4 repository package was incomplete. Although the public V3.4 nodes were present after the first hotfix, their parent sampler and sequence runtime did not yet accept the new Driving Audio and Video Reference contracts. We apologize for the incomplete release. The complete V3.4 runtime has now been synchronized. If you installed V3.4 earlier, run `git pull` or reinstall the node, then restart ComfyUI. +Long-form MiniMax H3 video generation for ComfyUI with Continuum-aware Second Pass refinement, optional Hi-Res Fix, low-memory disk-backed assembly, restartable chunks, persistent references, and optional Spectrum interoperability. -Long-form MiniMax H3 video generation for ComfyUI with restartable chunks, persistent references, Driving Audio, Video Reference, seam correction, and optional Spectrum interoperability. +> **V3.5.0 is the current release.** All V3.4 node IDs remain registered intentionally for saved-workflow compatibility. Existing V3.4 workflows continue unchanged; new workflows can opt into the independent V3.5 nodes. + +## What's new in V3.5 + +V3.5 has two major additions: + +1. **Continuum-aware Second Pass / Hi-Res Fix** — refine externally processed H3 latents or use the integrated pixel/VAE 2x path without changing V3.4 sampling. +2. **Low-memory Assemble + Seam** — write the final video IMAGE directly to RAM or a Windows-safe mapped file while preserving Exact Duration, Seam, Terminal Merge order, and audio behavior. + +### 1. Second Pass and Hi-Res Fix + +**H3 Continuum Hi-Res Fix V3.5** is the one-node Main path. It performs Video VAE Decode, bounded pixel resize, Video VAE Encode, and one low-denoise Second Pass. The integrated Main path remains **Experimental** because long 2x runs can exceed GPU memory. + +![H3 Continuum Hi-Res Fix V3.5 node](docs/images/v35-hires-fix-node.png) + +Normal Hi-Res Fix wiring: + +```text +H3 Continuum Sampler V3.5 + -> H3 Continuum Hi-Res Fix V3.5 + -> Core Video / Audio VAE Decode + -> H3 Continuum Assemble + Seam V3.5 +``` + +**H3 Continuum Second Pass V3.5** is the stable Advanced bridge requested in Issue #8. It accepts an externally processed H3 video latent and samples each Continuum physical group once while preserving prompt policy, seed reporting, group order, and the original first-pass audio LATENT object. + +![H3 Continuum Second Pass V3.5 node](docs/images/v35-second-pass-node.png) + +External latent processor wiring, such as LBH: + +```text +H3 Continuum Sampler V3.5.video_latents + -> external H3 latent processor + -> H3 Continuum Second Pass V3.5 + -> Core Video / Audio VAE Decode + -> H3 Continuum Assemble + Seam V3.5 +``` + +Connect the original `audio_latents`, `assembly_plan`, and `refine_context` from Sampler V3.5 to the Hi-Res Fix or Second Pass node. Use the same final MODEL and CLIP, plus a separate sampler and workflow-side SIGMAS for the refinement pass. The nodes do not contain an internal denoise value. + +The `BasicScheduler` is deliberately omitted from the screenshots. It is still required to create the Second Pass `SIGMAS`. The GPU-tested baseline is: + +```text +sampler: res_multistep +scheduler: simple +steps: 10 +denoise: 0.35 +``` + +Do not chain Hi-Res Fix and another Second Pass in the standard path: Hi-Res Fix already performs one Second Pass internally. `H3 Continuum Latent Resize V3.5` is an advanced utility, not the recommended Main 2x path; direct high-ratio interpolation of the 24-channel H3 latent produced persistent artifacts in testing. + +If Hi-Res Fix is not connected, V3.4 sampling remains on its original path. V3.5 does not add a forced CPU-offload or low-VRAM sampler mode. `Hi-Res Fix enabled=false` is a lazy exact passthrough and does not evaluate its resize, VAE, or Second Pass dependencies. + +See [V3.5 Hi-Res Fix and Second Pass workflow guide](docs/V35_HIRES_FIX.md) for exact sockets, bypass behavior, external-upscaler wiring, and limits. + +### 2. Low-memory Assemble + Seam + +**H3 Continuum Assemble + Seam V3.5** adds manual RAM / Disk-backed selection and a conservative Auto policy. Disk-backed mode writes the final video IMAGE directly to a mapped file instead of rebuilding the entire completed video in anonymous RAM. Audio remains in RAM. + +| Backend | Behavior | +|---|---| +| Auto | Uses RAM only when the final IMAGE is at most 4 GiB and physical-memory reserve is sufficient; otherwise selects Disk-backed. Missing memory counters fail safely to Disk-backed. | +| RAM | Existing in-memory output behavior. | +| Disk-backed | Maps the final video IMAGE to a temporary file; video only. Audio remains in RAM. | + +Disk-backed mode preserves the backing file while ComfyUI still references the returned Tensor. Downstream nodes can still allocate a full RAM copy, so Continuum's low-memory guarantee applies to its own assembly stage, not arbitrary downstream implementations. + +#### Measured memory reduction + +The validated stress case assembled a deterministic `1536 x 1536`, 360-frame float32 IMAGE with a final size of `9.49 GiB`. RAM and Disk-backed outputs were video/audio hash-identical. + +| Windows process metric | RAM backend | Disk-backed | Reduction | +|---|---:|---:|---:| +| Private memory | 12.41 GiB | 2.91 GiB | **9.50 GiB** | +| USS | 10.29 GiB | 0.80 GiB | **9.49 GiB** | + +This is a **system-RAM/private-commitment** reduction during Continuum Assembly, not a claim that sampler GPU VRAM is reduced. Windows RSS includes resident mapped-file pages and was therefore similar between the two backends; private memory and USS are the relevant measurements. Disk-backed I/O can make final assembly slower, but it does not slow the preceding model sampling path. + +### GPU acceptance at a glance + +| Path | Tested condition | Result | +|---|---|---| +| Main Hi-Res Fix | FL2VA 1 x 5s, 576 -> 1152 | PASS | +| Main Hi-Res Fix | Hybrid FL2VA + Reference 1 x 5s, 576 -> 1152 | PASS | +| Advanced Second Pass | Hybrid/Reference 1 x 5s, 576 x 576 | PASS | +| Main Hi-Res Fix | FL2VA 3 x 5s, 576 -> 1152, RTX 5060 Ti 16 GiB | **CUDA OOM at terminal 77T group** | + +For the failed 3 x 5s case, First Pass and the 37T Second Pass group completed. The final logical chunks `[2,3]` remained one 77T Long Terminal Merge physical group and exceeded the tested 16 GiB GPU at its first 1152x1152 Second Pass inference. This is a documented resource limit; the Continuum grouping contract did not change. + +## V3.4 compatibility baseline + +V3.4 nodes are retained specifically so saved workflows do not break. Their Node IDs, public sockets, Sampling, Conditioning, Terminal Merge, Assembly, Seam, and Run Storage behavior were not replaced by V3.5. Users who do not need Second Pass or disk-backed assembly can continue using V3.4 normally. ![H3 Continuum V3.4 workflow overview](docs/images/v34-workflow-overview.png) -> V3.4 is the current stable release. V3.3 legacy nodes remain available for existing workflows. - -## What's new in V3.4 - V3.4 focuses on the two reference workflows that are most useful in normal production: - **Driving Audio**: use an existing audio source as the preserved final audio while it guides the H3 generation across chunks. @@ -38,9 +125,11 @@ This is a usability and reliability decision, not a claim that the experimental - [V3.4 standard](examples/workflows/MiniMax_H3_Continuum_V34.json) - [V3.4 Turbo](examples/workflows/MiniMax_H3_Continuum_V34_turbo.json) +The V3.4 templates remain valid in V3.5. A release-grade V3.5 UI workflow JSON is not bundled yet; use the connection guide above rather than older diagnostic workflows. + The templates include optional external nodes such as Spectrum, Video Helper Suite, rgthree, EasyUse, and RTX Video Super Resolution. Install, replace, connect, or bypass them according to your installation. Media and acceleration choices remain under user control. -## What's new in V3.4 +## V3.4 feature details ### Driving Audio @@ -140,6 +229,28 @@ git clone https://github.com/ukr8b3g-cmyk/ComfyUI-H3-Continuum.git Restart ComfyUI after installation or update. +## V3.5 nodes + +### H3 Continuum Sampler V3.5 + +The V3.5 sampler preserves V3.4 generation behavior and adds `refine_context` as the sixth output. Connect it only to V3.5 Hi-Res Fix or Second Pass paths that need the original conditioning context. + +### H3 Continuum Hi-Res Fix V3.5 (Experimental) + +Main convenience node. When enabled, it performs the Video VAE pixel-resize round trip and Second Pass internally. When disabled, video/audio LATENT objects and the Assembly Plan pass through unchanged and the sampling/VAE dependencies remain lazy. + +### H3 Continuum Second Pass V3.5 + +Advanced Issue #8 bridge for an externally processed video latent. It samples once per physical group, reads the physical prompt contract from the Assembly Plan, and returns the original first-pass audio LATENT objects. + +### H3 Continuum Latent Resize V3.5 + +Utility that changes only video-latent H/W. It does not sample. Direct 2x interpolation at large square targets is not the accepted Main Hi-Res Fix method. + +### H3 Continuum Assemble + Seam V3.5 + +V3.4-compatible assembly with Auto, RAM, and Disk-backed video-buffer backends. Exact Duration and seam corrections write directly to the selected target buffer. + ## V3.4 nodes ### H3 Continuum Sampler V3.4 @@ -301,7 +412,9 @@ Use a 24 fps source for `Video Reference`. `Load Video (Upload)` may accept file ## Current validation status -V3.4 has been exercised locally with: +The V3.5.0 release passes `430` automated tests, source/runtime registration verification, native PackedLayout, Fixed 3x5 prompt planning, and release metadata consistency. Representative GPU revalidation passed with accepted 3x5 FL2VA Long Terminal Merge latents through Core Video/Audio VAE Decode and Assemble V3.5 Auto: 3 logical chunks, 2 physical groups (`37T + 77T`), 640x640, 360 frames, 24 fps, and 15.000 seconds, with the accepted video/audio hashes reproduced exactly. + +V3.4 compatibility paths have been exercised locally with: - 1, 2, and 3 chunks - standard and Turbo paths @@ -316,6 +429,15 @@ V3.4 has been exercised locally with: - Run Storage reuse and selected-chunk regeneration - Core VAE Decode and final assembly +Additional V3.5 acceptance includes: + +- Advanced Second Pass Bridge for 1x5 T2VA, 3x5 T2VA, and 3x5 FL2VA Long Terminal Merge +- first-pass audio LATENT object passthrough through Second Pass +- RAM / Disk-backed bit-exact assembly, cache/requeue, Preview, Save, VHS, interrupt, stale cleanup, and a 9.49 GiB mapped IMAGE stress +- Auto backend selection in both the real 1.65 GiB Long Terminal Merge case and the 9.49 GiB stress case +- Experimental integrated Main Hi-Res Fix for FL2VA 1x5, 576x576 to 1152x1152 +- Hybrid FL2VA + Reference 1x5 through both the integrated 576-to-1152 Main Hi-Res Fix and the direct 576x576 Advanced Second Pass node + No OOM was observed in the cited recent local V3.4 checks, including two-chunk 800 x 800 runs. This is not a universal memory guarantee. Model precision, LoRAs, source resolution, optional nodes, GPU, and RAM affect memory use. ## Limits @@ -325,6 +447,9 @@ No OOM was observed in the cited recent local V3.4 checks, including two-chunk 8 - Driving Audio preserves selected audio, but visual lip synchronization remains model-dependent. - FL2VA may settle on the supplied Last Frame before the requested duration ends; equivalent Core runs can show the same behavior. - Long, high-resolution sequences may exceed system RAM during final decode and assembly even when chunk sampling succeeds. +- Disk-backed assembly lowers anonymous/private memory commitment for the final Continuum IMAGE, but Core Decode and downstream nodes may still allocate large RAM copies. +- Main Hi-Res Fix 3x5 2x is not accepted on the tested RTX 5060 Ti 16 GiB: the 37T group completed at 1152x1152, but the terminal 77T group failed at its first Second Pass inference with CUDA OOM. Reference/Hybrid-specific 1x5 acceptance passed; longer Hybrid/Reference cases remain unverified. +- Direct high-ratio interpolation with Latent Resize can create persistent H3 artifacts; use the Pixel/VAE Main path or an appropriate external H3 latent processor. - Match Output can be substantially slower than 0.4 MP or 0.6 MP. - Seam correction may keep the native boundary when a proposed correction is not safer. - Optional template nodes must be installed, replaced, or bypassed by the user. diff --git a/README_JA.md b/README_JA.md index 6519295..e1f5e46 100644 --- a/README_JA.md +++ b/README_JA.md @@ -1,9 +1,79 @@ -# ComfyUI-H3-Continuum 3.4.0 +# ComfyUI-H3-Continuum 3.5.0 -**V3.4.0 Stable:** Driving Audio、Video Reference、Audio / Video Seam、Run Storageを含む安定版です。 +**V3.5.0正式版:** V3.4互換を維持し、Second Pass Bridge、実験的Hi-Res Fix、Disk-backed Assembleを追加します。V3.4のNode IDは保存済みワークフロー互換のため意図的に残しており、既存V3.4ワークフローはそのまま使用できます。 MiniMax H3を複数チャンクで連続生成し、直前チャンク末尾の**映像latent / 音声latentを直接**次チャンクへ継承するComfyUIカスタムノードです。チャンク間でVideo/Audio VAEのDecode→Encodeは行いません。 +## V3.5の主な追加機能 + +V3.5の大きな追加は2点です。 + +1. **Continuum対応Second Pass / Hi-Res Fix**: physical group構造を維持した再Samplingと、標準Pixel/VAE 2x経路。 +2. **低メモリAssemble + Seam**: Exact Duration、Seam、Terminal Merge、Audioを維持したRAM / Disk-backed / Auto出力。 + +### 1. Second Pass / Hi-Res Fix + +`H3 Continuum Hi-Res Fix V3.5`はVideo VAE Decode → pixel resize → Encode → low-denoise Second Passを1ノード化したMain経路です。長尺2xではVRAM上限に達する場合があるためExperimentalです。 + +![H3 Continuum Hi-Res Fix V3.5 node](docs/images/v35-hires-fix-node.png) + +通常のHi-Res Fix接続: + +```text +H3 Continuum Sampler V3.5 + → H3 Continuum Hi-Res Fix V3.5 + → Core Video / Audio VAE Decode + → H3 Continuum Assemble + Seam V3.5 +``` + +`H3 Continuum Second Pass V3.5`は外部加工したH3 video latentをphysical group構造のまま再Samplingする安定版Advanced入口です。Issue #8への本体回答です。 + +![H3 Continuum Second Pass V3.5 node](docs/images/v35-second-pass-node.png) + +外部latent upscaler接続: + +```text +H3 Continuum Sampler V3.5.video_latents + → LBH等の外部H3 latent処理 + → H3 Continuum Second Pass V3.5 + → Core Video / Audio VAE Decode + → H3 Continuum Assemble + Seam V3.5 +``` + +画像では`BasicScheduler`を省略していますが、Second Pass用`SIGMAS`の接続は必要です。GPU受入済み基準は`res_multistep`、`simple / 10 steps / denoise 0.35`、Lanczos pixel resize、FL2VA 1×5秒・576→1152です。Hybrid FL2VA + Referenceも1×5秒で、統合2x Hi-Res FixとAdvanced Second Pass直結の両方を受入しました。 + +Hi-Res Fixを接続しなければV3.4 Sampling経路は変わりません。強制CPU offloadや低VRAM samplerは追加していません。`enabled=false`ではResize、VAE、Second Passを評価せず、元のVideo/Audio LATENTとAssembly Planをそのまま返します。 + +### 2. 低メモリAssemble + Seam + +`H3 Continuum Assemble + Seam V3.5`はRAM / Disk-backed / Autoを選択できます。Disk-backedでは完成Video IMAGEをWindows対応mapped fileへ直接writeし、AudioはRAMに維持します。 + +1536×1536・360 frames・完成IMAGE 9.49 GiBの実測では、RAM版とDisk-backed版の映像/音声hashが完全一致しました。 + +| Windows process指標 | RAM | Disk-backed | 削減 | +|---|---:|---:|---:| +| Private memory | 12.41 GiB | 2.91 GiB | **9.50 GiB** | +| USS | 10.29 GiB | 0.80 GiB | **9.49 GiB** | + +これはContinuum Assembly時の**システムRAM/private commitment削減**であり、SamplerのGPU VRAM削減値ではありません。Disk I/Oにより最終Assemblyが遅くなる可能性はありますが、前段のモデルSampling速度は変えません。 + +### GPU受入範囲 + +| 経路 | 条件 | 結果 | +|---|---|---| +| Main Hi-Res Fix | FL2VA 1×5秒、576→1152 | PASS | +| Main Hi-Res Fix | Hybrid FL2VA + Reference 1×5秒、576→1152 | PASS | +| Advanced Second Pass | Hybrid/Reference 1×5秒、576×576 | PASS | +| Main Hi-Res Fix | FL2VA 3×5秒、576→1152、RTX 5060 Ti 16 GiB | **Terminal 77TでCUDA OOM** | + +3×5秒2xではFirst Passと37T groupのSecond Passは完了し、最後のlogical `[2,3]`を維持した77T physical groupが1152×1152の最初の推論でVRAM上限に達しました。group構造の破損ではなく、16 GiB実機でのリソース上限です。 + +詳しいsocket接続・OFF時のpassthrough・既知の制限は[英語版V3.5接続ガイド](docs/V35_HIRES_FIX.md)を参照してください。 + +## V3.4互換 + +V3.4ノードは保存済みワークフローを壊さないため残しています。Node ID、公開socket、Sampling、Conditioning、Terminal Merge、Assembly、Seam、Run StorageをV3.5へ置換していません。Second PassやDisk-backed Assembleを必要としない場合は、従来どおりV3.4を使用できます。 + ## V3.4.0 Stable V3.4.0は、最大3枚のReference Image、Run Storage、Spectrum Interop、Decode後のAudio / Video Seam補正を維持し、Driving AudioとVideo Referenceを正式な入力として追加します。 @@ -86,6 +156,10 @@ ZIPを展開して、`ComfyUI-H3-Continuum`フォルダーを`ComfyUI/custom_nod ## 検査 +V3.5.0正式版は全`430 passed`、source/runtime登録検証、PackedLayout、Fixed 3×5 Prompt Planを通過しています。代表GPU再検証では3×5秒FL2VA Long Terminal Mergeの受入済みlatentをCore Video/Audio VAE DecodeからAssemble V3.5 Autoへ通し、3 logical / 2 physical groups、640×640、360 frames、24 fps、15.000秒、既存受入済み映像/音声hashとの完全一致を確認しました。 + +Main Hi-Res Fixの3×5秒2xは、RTX 5060 Ti 16 GiBで37T groupの1152×1152 Second Pass完了後、Terminal Mergeの77T group最初の推論時にCUDA OOMとなり未受入です。Reference/Hybrid固有の1×5秒Second Passは受入済みですが、長尺Reference/Hybridは未確認です。Disk-backedが保証する低メモリ範囲はContinuum Assemblyであり、Core Decodeや下流ノードが別の全量copyを作る可能性は残ります。 + ```text python -m compileall -q . pytest -q diff --git a/WORKLOG.md b/WORKLOG.md index 4071348..8fcaa36 100644 --- a/WORKLOG.md +++ b/WORKLOG.md @@ -1,5 +1,14 @@ # Work Log +## 2026-08-22 - E2-8A / E2-8B1 implementation + +- Added additive physical-group Second Pass metadata to V3.4 Assembly Plan output. +- Added static validation, target-geometry update, audio passthrough, and namespaced refine-seed helpers. +- Added CPU regression coverage for 24ch, B/T/group preservation, terminal 260f preservation, H/W upscale, prompt policy, and audio passthrough. +- Production Second Pass sampling, LBH integration, GPU validation, UI, and node registration remain unimplemented. +- Validation: Second Pass contract `8 passed`; related V3.4 regressions `94 passed`; full suite `257 passed`. +- Updated two legacy mocked Assembly Plans with required geometry fields; Production generation code was not changed by this test-only correction. + Append concise completed-work records here. Do not paste full console logs or duplicate `PROJECT_STATE.md`. ## 2026-08-20 - Project guidance organization @@ -122,3 +131,206 @@ D:\Codex\_snapshots\ComfyUI-H3-Continuum\pre-rollback-after-00038-fail-20260821_ - GPU PASS: `MiniMax_H3_Continuum_V31_00008_.mp4` exercised 3x5s FL2VA with `terminal_pair=2-3`, `physical_frames=260`, `sampling_passes=1` for the pair, `paired_timeline_v1`, two physical decode groups, and final 360 frames / 15.000 seconds. - Core comparison `MiniMax_H3_00005_.mp4` reproduced the same short terminal still period. Long Terminal Merge was formally accepted; terminal stillness is Core-equivalent FL2VA convergence and is not a Continuum repair target. - Final release validation: full pytest `249 passed`; `tools/verify_runtime.py --comfy-root D:\StabilityMatrix\Data\Packages\ComfyUI_W` passed V3.4 registration, native PackedLayout, and Fixed 3x5s prompt-plan checks. GPU acceptance is recorded above. + +## 2026-08-23 - E2-8C experimental T2VA Second Pass + +- Snapshots: `pre-change-20260823_004307` for the repository and `comfyui-w-before-e2-8c-20260823_004341` for the active runtime files. +- Added a generic physical-group Second Pass node; LBH remains an external upstream latent upscaler. +- Reused the existing prompt encoder and ComfyUI-native `sample_chunk()` path with workflow-supplied SIGMAS. +- One namespaced refine seed is used per physical group; temporary refined audio is discarded and first-pass audio objects are returned unchanged. +- Existing V3.4 Sampling, Terminal Merge, Assembly, Seam, Run Storage, and public Sampler sockets were not changed. +- CPU regression and GPU 1x5 T2VA acceptance remain pending after implementation. + +## 2026-08-23 - E2-8C Second Pass GPU acceptance + +- E2-8C is formally PASS after 1x5 T2VA, 3x5 T2VA, and 3x5 FL2VA Long Terminal Merge GPU runs. +- Final acceptance artifact: `E2-8B2_01_1x5_T2VA_LBH3D_00011_.mp4`; embedded workflow used three 5-second timeline prompts, supplied First/Last images, the FL2VA INT8 ConvRot model, no enabled LoRA, `res_multistep`, First Pass `simple` 20 steps / denoise 1.0, external LBH FP16 CUDA upscale from 640x640 to 768x768, and Second Pass `simple` 4 steps / denoise 0.3. Execution time was 17 minutes 44 seconds. +- Long Terminal Merge preserved `37T + 77T` and two physical groups: group 1 logical `[1]`; group 2 logical `[2,3]` with `paired_timeline_v1`. Each physical group ran exactly one Second Pass sampling call with its own reported refine seed. +- Final media was 768x768, 360 frames, 24 fps, and 15.000 seconds. First-pass audio LATENT objects were returned bit-exact; Core Video/Audio Decode and Assemble completed successfully. +- The 5-second and 10-second boundaries had no major flash, ghosting, jump, or temporal collapse. The final approximately 0.5-second low-motion interval was accepted as natural Last Frame convergence. +- V3.4 Sampling, Terminal Merge, Assembly, Seam, and Run Storage were unchanged. No code or runtime file was modified during GPU acceptance classification. +- Final validation after documentation update: full pytest `260 passed in 4.41s`. + +## 2026-08-23 - V3.5 Continuum Latent Resize + +- Snapshots: `pre-v35-latent-resize-repo-20260823_153334` (345 files) and `comfyui-w-before-v35-latent-resize-20260823_153336` (308 files). +- Added `H3ContinuumLatentResizeV35`, a list-aware thin wrapper over ComfyUI Core `common_upscale` for the standard Continuum Hi-res fix path. +- The node exposes Core nearest-exact, bilinear, area, bicubic, and bislerp interpolation plus `scale_by`; it preserves LATENT metadata, physical-group order, and B/C/T while changing only spatial H/W. +- LBH and other learned latent upscalers remain optional external alternatives. No upscaler was embedded into Continuum. +- Updated runtime registration verification to include the existing V3.5 Second Pass and the new Latent Resize node. +- Synced only `nodes.py` and `v3/latent_resize_nodes.py` to ComfyUI_W and verified SHA-256 equality. +- Validation: Latent Resize tests `5 passed`; full pytest `265 passed in 5.09s`; ComfyUI_W runtime registration, native PackedLayout, and Fixed 3x5s prompt-plan verification passed. GPU generation with the new resize node remains pending. +- V3.4 Sampling, Terminal Merge, Conditioning, Assembly, Seam, Run Storage, and public Sampler sockets were unchanged. Commit/push not performed. + +## 2026-08-23 - V3.5 one-node Hi-Res Fix + +- Snapshots: `pre-v35-hires-fix-repo-20260823_174946` (349 files) and `comfyui-w-before-v35-hires-fix-20260823_174947` (310 files). +- Added `H3ContinuumHiResFixV35` as the normal Main workflow: ON performs Core-compatible spatial resize followed by the existing physical-group Second Pass; OFF returns the exact first-pass video/audio LATENT lists and original Assembly Plan object. +- MODEL/CLIP/SAMPLER/SIGMAS are lazy inputs, so the node does not request Second Pass dependencies while disabled. +- Preserved `H3ContinuumSecondPassV35` as the Advanced external-latent bridge required by Issue #8 and `H3ContinuumLatentResizeV35` as a standalone Utility. Removed only the `(Experimental)` suffixes from their display names. +- No V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, or Sampler socket behavior changed. +- Validation: V3.5 focused tests `13 passed`; full pytest `270 passed in 3.82s`; ComfyUI_W runtime registration, native PackedLayout, and Fixed 3x5 prompt-plan verification passed. GPU execution through the new wrapper remains pending. Commit/push not performed. + +## 2026-08-23 - V3.5 Second Pass audio-lock candidate + +- Snapshots: `pre-v35-second-pass-audio-lock-repo-20260823_203840` (357 files) and `comfyui-w-before-v35-second-pass-audio-lock-20260823_203843` (312 files). The earlier 45-file repository snapshot was incomplete after a `.pytest_cache` permission error and is not the accepted snapshot. +- Added `v3/refine_sampling.py` and switched only V3.5 Second Pass to direct Core sampling with seed-derived random video noise/mask 1 and zero audio noise/mask 0. No inverse noise scaling or manual sigma compensation is applied. +- Kept V3.4 `sample_chunk()`, UI sockets, Conditioning, Terminal Merge, Assembly, Seam, and Run Storage unchanged. First/Last/Reference conditioning remains out of scope. +- Validation: V3.5 focused tests `19 passed`; Core minimal-mask expansion PASS for representative 37T video and H3 audio shapes; ComfyUI_W venv full pytest `273 passed in 3.98s` using a writable dedicated basetemp (`pynvml` FutureWarning only). +- GPU A/B against the fixed 00015 artifact condition remains pending. This CPU result does not establish that audio re-noise was the artifact's primary cause. Commit/push not performed. + +## 2026-08-23 - V3.5 no-resize Second Pass GPU isolation + +- Snapshot: `pre-v35-resize-bypass-diagnostic-workflow-verified-20260823_211203` (356 repository files excluding the inaccessible `.pytest_cache`, plus the SHA-256-verified 00017 source MP4). +- Created `MiniMax_H3_Continuum_V35_NoResize_SecondPass_Diagnostic.json` by replacing the Hi-Res Fix wrapper with a direct `H3ContinuumSecondPassV35` connection and adding a status preview. No Production code or runtime file was changed. +- Reused the exact cached 576x576 first-pass latent from the 00017 condition. Second Pass remained `res_multistep`, `simple` 10 steps / denoise 0.35, refine seed 42; only spatial resize and 1152x1152 re-entry were removed. +- GPU execution succeeded as prompt `aa1faf7f-7353-49bd-8efb-23b1cf9c2462`. The report confirmed video random noise/mask 1, audio zero noise/mask 0, shape `(1, 24, 37, 36, 36)`, one physical group, one sampling pass, and first-pass audio LATENT passthrough. +- Output `V35_NoResize_SecondPass_Diagnostic_00001_.mp4` is 576x576, 120 frames, 24 fps, and 5.000 seconds. Twenty representative frames were visually clean with none of 00017's persistent cyan/orange/white line or plate artifacts. +- Result: audio-lock does not repair the 2x artifact and random audio re-noise is not its primary cause. The direct Second Pass path passes; the remaining failure is scoped to the 2x spatial-resize/high-resolution re-entry path. Interpolation method versus scale/resolution remains to be isolated. +- Commit/push not performed. + +## 2026-08-23 - V3.5 spatial method/resolution GPU isolation + +- Held the cached 576x576 first-pass latent, `res_multistep`, `simple` 10 steps / denoise 0.35, refine seed 42, prompt, audio, and Assembly Plan fixed. Only resize method or target H/W changed. +- FAIL `V35_NearestExact2x_Diagnostic_00001_.mp4` (`c35f5c9d-e1b7-43aa-8b09-7f4994aa6225`): nearest-exact 2x, 1152x1152, latent `(1, 24, 37, 72, 72)`. The same persistent colored line/plate artifact family remained, so bilinear-specific channel mixing is not the primary cause. +- PASS `V35_NearestExact_576to768_Diagnostic_00001_.mp4` (`676b23cb-79c8-4d03-9e24-0ecfa2288128`): nearest-exact 4/3x, 768x768, latent `(1, 24, 37, 48, 48)`. Twenty representative frames were clean. +- PASS `V35_Bilinear_576to768_Diagnostic_00001_.mp4` (`9a7b7383-06dd-4972-bb6c-82c010e64b1e`): bilinear 4/3x, 768x768, latent `(1, 24, 37, 48, 48)`. The target artifact did not recur. +- FAIL `V35_Bilinear_576to1024_Diagnostic_00001_.mp4` (`b0530d14-fb2c-42d4-b62c-08d73ec13c8d`): bilinear 16/9x, 1024x1024, latent `(1, 24, 37, 64, 64)`. Persistent line/plate artifacts returned across the sequence. +- Every run completed Core Decode and Assemble as 120 frames / 24 fps / 5.000 seconds with the audio-locked report and first-pass audio LATENT passthrough. Visual acceptance, not execution validity, separates PASS from FAIL. +- ComfyUI Core source defines H3 `BASE_SHORT_EDGE = 768` and `MAX_PIXELS = 768 * 1344`. The observed square boundary is consistent with the native canvas: 768 passed; 1024 and 1152 failed. The exact transition between 768 and 1024 remains untested. +- Added four importable diagnostic workflow JSON files under `D:\Codex\_test_workflows\ComfyUI-H3-Continuum`. No Production code, runtime file, V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, or Run Storage behavior changed. +- Commit/push not performed. + +## 2026-08-23 - V3.5 H3 native-canvas Auto default + +- Snapshots: `pre-v35-native-canvas-auto-repo-20260823_215429` (356 files, excluding the inaccessible `.pytest_cache`) and `pre-v35-native-canvas-auto-comfyui-w-20260823_215429` (309 files). Relevant source/runtime files matched by SHA-256 before the change. +- Preserved the existing `H3ContinuumHiResFixV35` node ID, sockets, widget order, and saved-workflow behavior. The new defaults are `bilinear` and `scale_by=0.0`; zero means H3 native-canvas Auto, while every positive value retains the previous manual scale multiplier. +- Auto calls ComfyUI Core `adapt_canvas()` and uses an exact-target resize helper so non-square Core targets do not depend on a rounded uniform multiplier. It never shrinks an already-larger latent. Manual targets beyond Core native canvas emit a status warning only; no clamp or hard stop was introduced. +- Added exact-target, multi-group metadata, native target, no-shrink, landscape 1024x576 -> 1344x768, manual-compatibility, and warning coverage. V3.5 focused tests: `28 passed`; full pytest: `277 passed` with only the known `pynvml` FutureWarning. +- Synchronized only `v3/hires_fix_nodes.py` and `v3/latent_resize_nodes.py` to ComfyUI_W and verified SHA-256 equality. Runtime registration reported `upscale_method=bilinear`, `scale_by=0.0`, and minimum 0.0 after restart. +- GPU acceptance prompt `26a7709d-0e4a-4c3f-ad34-3424e3dc7c69` produced `V35_NativeAuto_576to768_Acceptance_00001_.mp4`. The report selected 576x576 -> 768x768, sampled one `(1, 24, 37, 48, 48)` physical group once, retained first-pass audio bit-exact, and completed Core Decode/Assemble as 120 frames / 24 fps / 5.000 seconds. +- Ten-frame visual review found none of the 1024/1152 colored line/plate artifacts. Decoded video matched the accepted manual bilinear 576 -> 768 run exactly (`PSNR = inf`). The importable acceptance workflow is `D:\Codex\_test_workflows\ComfyUI-H3-Continuum\MiniMax_H3_Continuum_V35_NativeAuto_576to768_Acceptance.json`. +- V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, and public Sampler sockets were unchanged. Commit/push not performed. + +## 2026-08-24 - V3.5 context-aware conditioning implementation and GPU isolation + +- Verified snapshots before modification: `pre-v35-refine-context-repo-verified-20260823_233600` (356 repository files excluding `.git` and the inaccessible `.pytest_cache`) and `pre-v35-refine-context-comfyui-w-verified-20260823_233632` (309 runtime files with the same exclusions). +- Added runtime-only `v3/refine_context.py`, a new public `H3ContinuumSamplerV35`, physical-group capture immediately before First Pass sampling, and optional `refine_context` / `video_vae` inputs on Second Pass and Hi-Res Fix. V3.4 retains its original five outputs; V3.5 appends context as output six after Driving Audio. +- Context-aware Second Pass validates the capture against the Assembly Plan, re-encodes target First/Last keyframes through the Video VAE, resizes only marked Continuum video context, bypasses prompt re-encoding, and uses one call-local Continuum MODEL clone per physical group. Missing, incomplete, or target-adaptation-unavailable context warns and uses the legacy prompt-only path; corrupt typed context remains a contract error. +- Kept the existing direct refine sampling contract unchanged: video random noise/mask 1, audio zero noise/mask 0, workflow SIGMAS once, one derived seed and sampling call per physical group, and the original first-pass audio LATENT objects returned unchanged. +- Review found no P0/P1 integration defect. Known non-blocking risks are CPU RAM growth from immutable conditioning copies for long/reference-heavy runs, bilinear context resize when an Advanced external learned upscaler is used, and a still-required CLIP socket even when complete context bypasses prompt encode. +- Validation: targeted integration `40 passed`; full pytest `293 passed`; `git diff --check` PASS; source runtime verifier and ComfyUI_W runtime registration/native PackedLayout/Fixed prompt-plan checks PASS. Only the known `pynvml` warning remained. +- Synced the changed production files to ComfyUI_W after the verified snapshot. Eight files match source by SHA-256. `v3/driving_nodes.py` differs only by the pre-existing runtime Video Reference tooltip addition, which was deliberately preserved; runtime `py_compile` and registration passed. +- Context-aware GPU A/B prompt `0adff3b7-1552-47e9-a167-7b6e4010b0a5` used the same FL2VA 1x5 inputs and seeds as the prior artifact run, 576x576 -> 864x864 bilinear 1.5x, `res_multistep`, `simple` 10 steps / denoise 0.35, refine seed 42. Report: one physical group, `conditioning_source=refine_context`, two target keyframes VAE re-encoded, `(1, 24, 37, 54, 54)`, audio passthrough. `V35_ContextAware_FL2VA_576to864_d035_00001_.mp4` still shows the colored line/plate artifact family, so conditioning mismatch is not the primary cause. +- Resize-only prompt `75249bbe-306a-45f9-a139-a79c0b17e536` reused the cached First Pass and produced `V35_ResizeOnly_FL2VA_576to864_bilinear_00001_.mp4`. It is 864x864 / 120 frames / 24 fps / 5.000 seconds; colored lines are absent, but severe temporal/person/background ghosting proves that direct spatial interpolation is not a valid general H3 latent-upscale backend. +- Pixel/VAE roundtrip prompt `2a03a1d4-2355-4d5a-a237-0ec3e0c09761` used First Pass Core Decode -> pixel Lanczos 864x864 -> H3 Video VAE Encode -> the same context-aware denoise 0.35 Second Pass. Sampling, Core Decode/Audio Decode, and Assemble succeeded. `V35_PixelRoundtrip_FL2VA_576to864_d035_00001_.mp4` is 864x864 / 120 frames / 24 fps / 5.000 seconds and is visually clean: no colored lines, plates, or resize ghosting, with the orange jacket, subject identity, motion, and terminal composition retained. +- Pixel/VAE roundtrip 2x prompt `de11e383-8b3d-4d8a-806f-1ac3550091f1` used the same cached First Pass, pixel Lanczos 1152x1152, H3 Video VAE Encode, context-aware `res_multistep` / `simple` 10-step denoise 0.35 Second Pass, and refine seed 42. It completed without OOM on the 16 GiB GPU. Report: one physical group, `(1, 24, 37, 72, 72)`, one sampling pass, two target keyframes VAE re-encoded, first-pass audio passthrough, then successful Core Decode/Audio Decode and Assemble. `V35_PixelRoundtrip_FL2VA_576to1152_d035_00001_.mp4` is 1152x1152 / 120 frames / 24 fps / 5.000 seconds and is visually clean across twelve representative time points: no colored lines/plates, resize ghosting, or temporal collapse, with subject identity, orange clothing, motion, and composition retained. +- Result: retain the Advanced external-latent Second Pass bridge for Issue #8; do not promote plain Latent Resize as the Main Hi-res backend. The evidence-backed next code phase is a low-memory sequential pixel/VAE roundtrip inside Hi-Res Fix, one physical group at a time, followed by the existing context-aware Second Pass. That backend change is pending explicit approval. Commit/push not performed. + +## 2026-08-24 - V3.5 Main safe Pixel/VAE Hi-res Fix implementation and GPU acceptance + +- Pre-change snapshots: `pre-v35-pixel-vae-main-repo-20260824_004310` (159 files / 3,817,668 bytes) and `pre-v35-pixel-vae-main-comfyui-w-20260824_004310` (132 files / 1,660,144 bytes). Source/runtime hashes matched for the relevant pre-existing production files before modification. +- Added `v3/pixel_vae_resize.py` and changed only the V3.5 Main `H3ContinuumHiResFixV35` path to perform one physical group at a time as H3 Video VAE Decode -> bounded 8-frame CPU pixel resize -> same H3 Video VAE Encode -> existing context-aware Second Pass. Default Main controls are Lanczos and `scale_by=2.0`; zero remains native-canvas Auto. +- Enabled Main execution requires the lazy Video VAE and has no direct-latent interpolation fallback. OFF remains exact video/audio/Assembly Plan identity and does not evaluate VAE or sampling dependencies. Advanced Second Pass and the standalone Latent Resize Utility were not changed. +- The first integrated GPU attempt (`46f775ca-093c-4c1f-b5ce-1aa9a8f8616f`) stopped at the explicit output-shape contract before Second Pass. Root cause: passing `[B,F,H,W,C]` directly to Core `VAE.encode()` makes Core's 4D IMAGE crop path treat F as a spatial axis. The helper now calls Encode per B with `[F,H,W,C]`, validates each `[1,C,T,H,W]` result before concatenation, and restores B explicitly. No latent fallback ran during the failed attempt. +- Validation after the Core-boundary correction: focused pixel/VAE and Hi-Res tests are covered by the full `308 passed`; `py_compile`, `git diff --check`, and the ComfyUI_W runtime verifier all pass. Independent review found no remaining P0/P1/P2 defect. Runtime `v3/pixel_vae_resize.py` matches source SHA-256 `B7C4A35A5E73F528492BC6946B97B687C412F1D3F5F976C19D45C6DC20288C9A` after restart. +- Integrated Main GPU acceptance prompt `c5dbe55c-c846-47e8-9f49-8bb167887236` used FL2VA 1x5, 576x576 -> 1152x1152 pixel Lanczos, context-aware `res_multistep`, `simple` 10 steps / denoise 0.35, and refine seed input 42. Report: one physical group, one sampling pass, `(1, 24, 37, 72, 72)`, two target keyframes re-encoded, and first-pass audio LATENT passthrough. Core Video/Audio Decode and V3.4 Assemble+Seam completed in 14:18. +- `V35_MainSafePixelVAE_Core4D_FL2VA_576to1152_d035_00001_.mp4` is 1152x1152 / 120 frames / 24 fps / 5.000 seconds with 32 kHz stereo audio. All decoded video and audio frame hashes exactly match the accepted external Decode -> Lanczos -> Encode -> Second Pass baseline. Visual contact-sheet review found none of the prior colored line/plate or resize-ghosting artifacts. The shared still interval from 3.666667 seconds is the same FL2VA Last Frame convergence in both outputs. +- Host-memory use remains material: observed peak was about 47.7 GiB working set / 61.3 GiB private bytes, with about 7.3 GiB system RAM available during the run. The Main backend is accepted for this 1x5 FL2VA 2x condition only; 3x5 Terminal Merge 2x remains pending separate GPU/resource acceptance. +- V3.4 Sampling, Conditioning, Terminal Merge, Assembly, Seam, Run Storage, public V3.4 sockets, Advanced Second Pass, and Latent Resize Utility were unchanged. Commit/push not performed. + +## 2026-08-24 - Phase 2A P2-0 Windows file-backed Tensor PoC + +- Verified pre-change snapshot: `D:\Codex\_snapshots\ComfyUI-H3-Continuum\pre-phase2a-p2-0-backend-poc-verified-20260824_022242` (374/374 source files, SHA-256 mismatch 0, revision `d77a467796c048908f8b7b83736e226e736e0f98`). The earlier `pre-phase2a-p2-0-backend-poc-20260824_022217` attempt is incomplete because the repository `.pytest_cache` ACL denied traversal and is not the accepted snapshot. +- Added only the unregistered standalone PoC `tools/p20_file_backed_tensor_poc.py` and its tests. Production nodes and ComfyUI_W/ComfyUI_WAN runtime packages were not modified. +- Selected `torch.from_file(shared=True)` and implemented strict backing-path/sidecar validation, alias-aware Storage lifetime, a process-global manager, retryable deletion, dead-process stale cleanup, live/young retention, root-scoped thread/process locking, and fail-closed invalid metadata handling. +- Independent review reproduced and drove fixes for Windows sharing-violation retry, dropped unpublished allocations, allocation/cleanup races, GC/root-lock re-entry, cross-thread ABBA lock inversion, and close-after-cancel fd reuse. Finalizers now enqueue only; actual cleanup uses root-lock -> manager-lock ordering. +- Validation: ComfyUI_W focused `26 passed`; full repository `334 passed`; standalone probe PASS (`bit_exact=true`, 480/480 bytes, alias-live deferred=1, final-alias reclaimed=1, managed files remaining=0). ComfyUI_WAN independently passed the same 26 tests and probe under Python 3.12.12 / PyTorch 2.13.0+cu130. Only the known `pynvml` FutureWarning was emitted. +- Deferred by design: Production Assemble integration, real ComfyUI cache/requeue and VHS/Save compatibility, RAM/Disk full-assembler equivalence, cross-process application integration, and representative 9-15 GiB sequential-write/RSS acceptance. These remain P2-2 through P2-5 gates; P2-1 Memory Attribution is next. +- Commit/push not performed. + +## 2026-08-24 - Phase 2A P2-1 through P2-3 CPU/static acceptance + +- Verified the authoritative source and both runtime copies, preserved all prior uncommitted work, and created `pre-phase2a-p2-1-memory-attribution-20260824_030858` followed by the accepted implementation snapshot `D:\Codex\_snapshots\ComfyUI-H3-Continuum\pre-phase2a-p2-2-disk-backed-20260824_032923` (166/166 source files, zero SHA-256 mismatch, revision `d77a467796c048908f8b7b83736e226e736e0f98`). +- Completed P2-1 with fail-soft, read-only V3.5 Memory Attribution at sequence start/before CPU commit/after CPU commit/completion, including deduplicated retained AV/refine-context storage, projected final IMAGE sizes, RSS/system RAM, and CUDA allocated/reserved values. No synchronization, peak reset, cache clear, forced GC, model unload, or V3.4 default change was added. +- Completed P2-2 with the Production adapter `v3/file_backed_buffer.py`: manual RAM/Disk-backed selection only, no Auto and no fallback, process-global alias-aware lifetime management, lazy root-first stale cleanup, and Video-only mapping while Audio stays in RAM. +- Completed P2-3 with independent V3.5 assembly/seam modules. The final target IMAGE is allocated once and receives physical decode groups in order through direct trim/pad/final-anchor spans. Exact Duration and Terminal Merge contracts are retained without a final full IMAGE `cat`, `stack`, `clone`, or `contiguous` materialization. +- Reused the existing V3.4 Video Seam analysis and math while limiting temporary copies to the affected 1-4 boundary frames and anchors. Preserved Audio Seam and cumulative sample alignment in RAM. Disk-backed publication is delayed until late Driving Audio/report validation completes; regressions cover early/late abort, alias lifetime, and dead-process stale recovery. +- CPU regressions cover RAM/Disk bit-exact output; Seam Off/Analyze/Auto/Auto 2; Exact ON/OFF; trim, pad, and preserved final frame; 1x5, 2x5 Terminal Merge, and 3x5 Long Terminal Merge; 32 kHz rounding; Audio Seam; physical ordering; Auto rejection; and error/lifetime paths. +- Final canonical validation: full pytest `412 passed` with only the known `pynvml` FutureWarning; changed Production `py_compile` PASS; `git diff --check` PASS. ComfyUI_W runtime verification passed node registration, native PackedLayout, and Fixed 3x5 prompt plan; focused runtime regression `141 passed`. +- Synchronized only 19 P2-related source-or-preserved-merge files to primary ComfyUI_W and verified every SHA-256 comparison. Preserved its pre-existing Video Reference tooltip delta. ComfyUI_WAN was left unchanged rather than receiving a broken partial V3.5 dependency set. +- Stopped at the requested P2-3 boundary. Still pending for P2-4/P2-5: real cache/requeue lifetime, Preview/VHS/Save, ESC cancel, 9-15 GiB stress/RSS, downstream copy behavior, GPU generation, long-duration GPU acceptance, and Auto backend. Commit/push not performed. + +## 2026-08-24 - Phase 2A P2-4 ComfyUI cache/downstream integration + +- Accepted verified snapshots: source `pre-phase2a-p2-4-comfy-integration-source-verified-20260824_101909` (415 files, SHA-256 mismatch 0, revision `d77a467796c048908f8b7b83736e226e736e0f98`) and ComfyUI_W `pre-phase2a-p2-4-comfy-integration-comfyui-w-verified-20260824_101938` (324 files, mismatch 0). The two standard-script attempts and one intermediate verification attempt are incomplete because the existing `.pytest_cache` ACL denies traversal and are not accepted snapshots. +- Added bounded 8-frame interrupt polling to only the V3.5 direct IMAGE writer. ComfyUI's normal interrupt is checked after Disk-backed allocation and between copy batches, so immediate or mid-copy ESC uses the unpublished-allocation cleanup path. RAM/Disk output equality is unchanged; focused interrupt/P2-3 regression was `47 passed`. +- Added the reusable headless integration probe `tools/p24_comfy_integration_probe.py`. It registers deterministic decoded CPU fixtures only, then executes the real installed ComfyUI_W `PromptExecutor` CLASSIC cache, Core Save/Preview nodes, VHS h264-mp4, Core interrupt handling, and separate-process stale cleanup. No model/VAE/sampler/CUDA/GPU path is used. +- Actual cache evidence: an intentional downstream failure retains the completed mapped IMAGE; unchanged requeue reports assembler node 2 cached and leaves one backing pair; Save/Preview/VHS requeue reports nodes 1/2/4/5/6 cached without rewriting outputs; cache eviction followed by replacement allocation keeps pair count bounded at one; final pair count is zero. +- Downstream evidence: Core Save 120 PNGs, Core Preview 120 PNGs, and VHS H.264/AAC succeeded. `ffprobe` verified 16x16, 120 frames, 24/1 fps, 5.000-second video/container duration, and 32 kHz audio. Core interrupt produced `execution_interrupted` with zero unpublished files. Crash child left one pair; clean-start child removed it. +- Final runtime artifacts: `D:\Codex\_test_results\ComfyUI-H3-Continuum\P24-runtime-W-final-20260824_1105\p24_result.json`, `p24_prompt_graphs.json`, and `output\P24\VHS_00001-audio.mp4` (39,006 bytes). The expected intentional downstream-error traceback is part of the probe and not a failure. +- Final validation: full pytest `415 passed`; ComfyUI_W P2-focused `89 passed`; runtime verifier, `py_compile`, `git diff --check`, and source/runtime SHA-256 checks pass. Synced only `v3/assembly_v35.py`, the P2-4 probe, and its test. The known `pynvml` FutureWarning is unchanged. +- P2-4 is formally PASS and work stops here. P2-5 still covers representative 9-15 GiB/RSS stress, full-resolution and long-duration GPU execution, manual UI cancellation under large I/O, downstream memory peaks, and any future Auto backend policy. Commit/push not performed. + +## 2026-08-24 - V3.5 compact UI parity + +- Verified source/runtime `web/project_id.js` equality, preserved all existing uncommitted work, and accepted snapshots `pre-v35-compact-ui-source-verified-20260824_104010` (420 files) and `pre-v35-compact-ui-runtime-w-verified-20260824_104048` (330 files), both with zero SHA-256 mismatch. Standard snapshot attempts failed only on the pre-existing inaccessible `.pytest_cache`. +- Added the V3.5 Sampler and Assemble+Seam class IDs to the existing frontend compact/settings integration. This hides Report Detail/debug/preview/strict compatibility and the internal Auto Resume ID, while retaining the existing conditional Run Name/Regenerate/Variation Nonce behavior. +- `strict_compatibility` was not removed from the backend schema; legacy loading remains possible, while the frontend continues to force the ignored value to `false`. No backend or generation behavior changed. +- Added `tests/test_v35_compact_ui_frontend.py`. Dedicated regression `2 passed`, JavaScript module syntax PASS, and canonical full pytest `417 passed` using a fresh writable basetemp. Only the known `pynvml` and repository pytest-cache warnings remain. +- Synchronized only the frontend file and dedicated regression test to ComfyUI_W; both source/runtime SHA-256 comparisons match, and the runtime-side dedicated regression is `2 passed`. +- P2-5 was not started. Commit/push not performed. + +## 2026-08-24 - Phase 2A P2-5 long-output acceptance + +- Preserved every existing uncommitted change and accepted verified snapshots `pre-phase2a-p2-5-gpu-acceptance-source-verified-20260824_105605` (422 files) and `pre-phase2a-p2-5-gpu-acceptance-runtime-w-verified-20260824_105626` (331 files), both with zero SHA-256 mismatch after excluding the pre-existing inaccessible `.pytest_cache`. +- Added the reusable source-side `tools/p25_gpu_acceptance_probe.py` and focused tests. The probe runs ComfyUI_W headlessly through its installed `PromptExecutor`, reconstructs the accepted 3x5 FL2VA Long Terminal Merge physical decode groups from Run Storage revision `1e30d8bf2154bbe7`, and uses the installed Core Video/Audio VAE Decode and Production V3.5 assembler. No UI automation or new model/sampler generation was used. +- Fresh-process RAM/Disk 640x640 A/B passed: 3 logical chunks, 2 physical groups (`37T + 77T`), 360 frames at 24 fps / 15.000 seconds, 32 kHz stereo audio, identical Exact Duration `362 -> 360` and audio correction `-2667`, identical Seam Auto report, finite output, identical video SHA-256 `95d5479bc5fee9c235ece7996c7c88dcc60981625dbedc330344403016ca9b9d`, and identical audio SHA-256 `eec0c6ff378c5bfcc3e71d02bab14230dea02a57e18ce1212f1dc1be4c88a3f9`. +- Core Save/Preview each consumed 360 mapped frames. VHS H.264/AAC was verified as 640x640, 360 frames, 24 fps, 15.000 seconds, and 32 kHz. Same-input requeue cached every node through assembler/digest without growing beyond one backing pair; cache release reclaimed it. +- A post-allocation Core interrupt produced `execution_interrupted` and left zero managed files. The P2-4 crash/restart probe was rerun in the P2-5 environment: one crash-stale pair was created and the next process removed it on first manager access. Downstream-error retention/recovery and final cache pair count also remained correct. +- Ran a 1536x1536x360 float32 stress producing a 10,192,158,720-byte / 9.49 GiB final IMAGE. RAM and Disk-backed outputs were hash-identical and finite. Disk-backed retained one file only while cached and zero after release. +- Corrected the measurement contract: Windows RSS includes resident file-mapped pages, so stress RSS was about 10.44 GiB for both backends and is not a valid standalone proof of commitment savings. RAM private/USS measured 13,322,362,880 / 11,045,199,872 bytes; Disk-backed measured 3,122,954,240 / 855,834,624 bytes. The mapped backend therefore reduced anonymous private commitment by about 9.50 GiB and USS by about 9.49 GiB, matching the output size. +- Validation: focused acceptance regression `58 passed`; full pytest `420 passed`; probe `py_compile` and `git diff --check` PASS. The first pytest attempt failed only because the inherited default temp/cache roots denied access; reruns used fresh `D:\Codex\_pytest_temp` bases with cacheprovider disabled. Only the known `pynvml` FutureWarning remains. +- P2-5 is formally PASS for manual RAM/Disk-backed V3.5 Assembly. Auto backend, arbitrary downstream full-copy behavior, new 3x5 sampler/model generation, and 3x5 Main Hi-res Fix 2x GPU/resource acceptance remain separate. No Production runtime file required synchronization. Commit/push not performed. + +## 2026-08-24 - Phase 2A P2-6 Auto Buffer Backend + +- Preserved all existing uncommitted work and accepted snapshots `pre-phase2a-p2-6-auto-source-verified-20260824_112744` (426 files) and `pre-phase2a-p2-6-auto-runtime-w-verified-20260824_112744` (331 files), both with zero SHA-256 mismatch after excluding the pre-existing inaccessible `.pytest_cache`. +- Added `Auto` to only the independent V3.5 Assemble backend. New V3.5 assembler nodes default to Auto; saved manual `RAM` and `Disk-backed` values remain compatible. V3.4 paths and sockets were not changed. +- Fixed policy: RAM requires final IMAGE `<= 4 GiB` and available physical memory `>= final IMAGE + max(4 GiB, 10% total RAM)`. Otherwise Auto selects Disk-backed. Missing memory counters select Disk-backed. Disk requires final IMAGE plus a 2 GiB reserve; measurement/capacity/allocation failure is explicit with no RAM fallback. +- Added detailed decision reporting and CPU regression for safe RAM choice, size-limit Disk choice, memory-pressure Disk choice, missing counters, disk shortage, manual compatibility, actual selected writer, bit-exact output, and backing lifetime. +- Synchronized only `v3/file_backed_buffer.py`, `v3/assembly_v35.py`, and a preserved-runtime merge of `v3/driving_nodes.py` to ComfyUI_W. The first two match source SHA-256; the third matches the prepared merge exactly and retains the pre-existing runtime Video Reference tooltip addition. +- Headless real acceptance `P26-auto-ram-long-terminal-20260824_113400` used accepted 3x5 FL2VA Long Terminal Merge physical latents plus Core GPU Video/Audio Decode. Auto selected RAM for the 1,769,472,000-byte final IMAGE and reproduced the accepted P2-5 640x640 / 360f / 15.000s video/audio hashes exactly. +- Headless stress acceptance `P26-auto-disk-stress-20260824_113610` selected Disk-backed for the 10,192,158,720-byte / 9.49 GiB final IMAGE, reproduced the accepted stress hashes, retained one pair only while cached, and reclaimed all managed files after release. +- Validation: focused regression `66 passed`; full pytest `428 passed`; source/runtime `py_compile`, runtime verifier, and `git diff --check` PASS. Only the known `pynvml` FutureWarning remains. Commit/push not performed. + +## 2026-08-24 - V3.5 release-candidate preparation + +- Preserved all existing uncommitted changes and accepted verified snapshots `pre-v35-release-preparation-source-verified-20260824_115858` (426 files) and `pre-v35-release-preparation-runtime-w-verified-20260824_115858` (331 files), both with zero SHA-256 mismatch. +- Updated `version.py`, `pyproject.toml`, `metadata.ini`, Changelog, package information/validation, and English/Japanese README to `3.5.0`. Added an automated metadata-consistency regression. +- Promoted the prior local Hi-Res connection memo to `docs/V35_HIRES_FIX.md`. The public guide distinguishes standard integrated Pixel/VAE Hi-Res Fix from the external-latent Advanced Second Pass Bridge, documents workflow-side SIGMAS, and warns that direct high-ratio latent interpolation is not the accepted Main path. +- Classified public artifacts. Production modules, applicable CPU tests, the public guide, and Production-required `tools/p20_file_backed_tensor_poc.py` are included in `MANIFEST.sha256`. Generated media/results and diagnostic workflows are excluded; P2-4/P2-5 probes remain repository-only validation tools. +- Targeted release/V3.5 regression is `75 passed`; canonical full pytest is `430 passed`. Source and synced ComfyUI_W runtime verifier both report V3.5.0, native PackedLayout PASS, Fixed 3x5 prompt-plan PASS, and expected node registration. Only the known `pynvml` FutureWarning remains. +- Synchronized only release metadata, READMEs, Manifest, the public V3.5 guide, and its metadata regression to the already snapshotted ComfyUI_W runtime, with per-file SHA-256 equality. Runtime-specific Production differences were not overwritten. +- Representative release GPU revalidation used the accepted Long Terminal Merge revision through Core GPU Decode and Assemble V3.5 Auto. Result: PASS, 3 logical / 2 physical groups (`37T + 77T`), 640x640, 360f, 24 fps, 15.000s, exact accepted video/audio hashes, finite outputs, and zero managed backing files after release. Evidence: `D:\Codex\_test_results\ComfyUI-H3-Continuum\V35-release-candidate-auto-long-terminal-20260824\p25_backend_result.json`. +- Main Hi-Res Fix remains Experimental because 3x5 2x and Reference/Hybrid-specific GPU acceptance are pending. Commit/push not performed. + +## 2026-08-24 - V3.5 pending GPU acceptance + +- Preserved all uncommitted work. Verified pre-documentation snapshots are `pre-v35-pending-gpu-results-source-verified-20260824_130824` (431 files) and `pre-v35-pending-gpu-results-runtime-w-verified-20260824_130824` (334 files), excluding `.git` and the pre-existing inaccessible `.pytest_cache`. +- Started ComfyUI_W headlessly on `127.0.0.1:8188` and submitted workflows through its HTTP API. Test GPU was an NVIDIA GeForce RTX 5060 Ti with 16,311 MiB VRAM. No Production code was changed. +- Main Hi-Res Fix 3x5 FL2VA 2x used 576x576 First Pass, Lanczos 1152x1152 target, `res_multistep`, and workflow `simple / 10 steps / denoise 0.35`. First Pass retained the Long Terminal Merge contract: 3 logical chunks, 2 physical groups (`37T + 77T`), logical `[2,3]` shared terminal sample, and 362 pre-assembly frames. The 37T Second Pass group completed 10/10; the terminal 77T group failed at its first Second Pass inference with CUDA OOM (15.93 GiB limit, 10.04 GiB allocated, 1.49 GiB requested). Result: FAIL on the tested 16 GiB GPU; prompt id `e6042730-1618-4967-9e3b-06125ff8ee35`. +- Hybrid FL2VA + Reference 1x5 integrated Main Hi-Res Fix 2x passed. It used one captured context-aware physical group, one Second Pass sampling call, Lanczos 576-to-1152, actual refine seed `2946484852083492747`, and bit-exact first-pass audio LATENT passthrough. Core Decode / Assemble produced 1152x1152, 120 frames, 24 fps, 5.000 seconds, 32 kHz stereo. Visual contact-sheet inspection found no persistent colored line/plate artifact, flash, or temporal collapse. Output `V35_RC_HybridReference_MainHiRes_1x5_576to1152_d035_00001_.mp4`, SHA-256 `E5B27CABCD5FBC1422F169AC6E75570EF4D54010CD04A3439852D8DED15C1384`; prompt id `c48dd1de-dd45-4896-b39e-45478d971194`. +- Direct `H3ContinuumSecondPassV35` Hybrid/Reference 1x5 also passed independently at 576x576. It reported `conditioning_source=refine_context`, actual refine seed `5765580629374593532`, `(1, 24, 37, 36, 36)`, one sampling pass, temporary refined audio discard, and bit-exact first-pass audio passthrough. Core Decode / Assemble produced 120 frames / 5.000 seconds; visual inspection was clean. Output `V35_RC_HybridReference_MainHiRes_1x5_576to1152_d035_00002_.mp4` (filename retained from the reused workflow prefix), SHA-256 `67311A797CD812584B4E2CCC21D7DF83AE8C1F1D72AB8BF5FA1C142D00894D2D`; prompt id `01b8a2a6-245d-4d29-bc4d-0e92426308d2`. +- The Hybrid prompt omitted angle brackets around `Picture 3`, generating a non-blocking H3C-P103 warning. Connected Reference conditioning remained active; this is accepted for execution/transport but not a warning-free public prompt example. +- Final validation after documentation synchronization: full canonical pytest `430 passed`; ComfyUI_W runtime verification passed package 3.5.0 registration, native PackedLayout, and Fixed 3x5 planning; `git diff --check` passed. README English/Japanese, the V3.5 guide, package validation, and Manifest match ComfyUI_W by SHA-256. +- Remaining acceptance: Main 3x5 2x on more than 16 GiB VRAM or a lower-memory terminal 77T implementation, and Reference/Hybrid durations longer than 1x5. Hi-Res Fix remains Experimental. Commit/push not performed. + +## 2026-08-24 - V3.5.0 formal README and release publication + +- Preserved all accumulated uncommitted V3.4 Terminal Merge, E2-8, Phase 2A, V3.5, test, and release-preparation work. Verified snapshots are `pre-v35-formal-readme-release-source-verified-20260824_141358` (431 files) and `pre-v35-formal-readme-release-runtime-w-verified-20260824_141359` (334 files), excluding `.git` and the pre-existing inaccessible `.pytest_cache`. +- Reorganized English/Japanese README around the two V3.5 additions: Continuum-aware Second Pass / experimental integrated Hi-Res Fix, and low-memory Assemble + Seam. Added concise wiring, the workflow-side SIGMAS baseline, OFF/no-Hi-Res performance behavior, 16 GiB GPU acceptance limits, and an explicit V3.4 saved-workflow compatibility statement. +- Added separate Hi-Res Fix and Second Pass node images without BasicScheduler, wires, or group labels. The Hi-Res image uses the accepted `lanczos / 2.00` setting. Image generation/editing was limited to isolating the supplied UI and aligning that displayed option; README text remains authoritative for connections and SIGMAS. +- Added the validated 9.49 GiB stress numbers to both READMEs: private memory 12.41 -> 2.91 GiB and USS 10.29 -> 0.80 GiB, with hash-identical RAM/Disk output. Explicitly documented that this is host-RAM/private-commitment reduction, not sampler VRAM reduction, and that Disk-backed can affect final assembly I/O rather than model sampling speed. +- Updated Changelog, package information, package validation, V3.5 guide wording, and the 159-entry Manifest. Full canonical pytest is `430 passed`; every Manifest hash, `git diff --check`, and ComfyUI_W runtime verification pass. Nine changed public files/assets were synchronized to ComfyUI_W with per-file SHA-256 equality. +- User explicitly authorized committing and pushing the complete V3.5.0 release state to `origin/main`. diff --git a/docs/V35_HIRES_FIX.md b/docs/V35_HIRES_FIX.md new file mode 100644 index 0000000..8217a91 --- /dev/null +++ b/docs/V35_HIRES_FIX.md @@ -0,0 +1,154 @@ +# V3.5 Hi-Res Fix and Second Pass workflow guide + +Status: V3.5 public connection guide. The integrated Hi-Res Fix remains Experimental; the Advanced Second Pass Bridge is the accepted Issue #8 path. + +## Most important correction + +The accepted Second Pass scheduler values are: + +```text +BasicScheduler + scheduler = simple + steps = 10 + denoise = 0.35 +``` + +This means **10 steps and denoise 0.35**. It does not mean denoise 10. + +`H3 Continuum Hi-Res Fix V3.5` does not contain its own steps or denoise value. It receives the exact `SIGMAS` produced by the workflow-side `BasicScheduler`, so this connection controls how strongly and how many steps the resized latent is refined. + +The values above are the GPU-accepted baseline, not a claim that they are universally optimal for every model, sampler, resolution, or prompt. + +## Standard Main connection + +Normal users should use the two V3.5 nodes below: + +```text +H3 Continuum Sampler V3.5 + │ + ├─ video_latents ───────┐ + ├─ audio_latents ───────┤ + ├─ assembly_plan ───────┤ + └─ refine_context ──────┤ + ▼ + H3 Continuum Hi-Res Fix V3.5 + │ + ├─ video_latents ──→ Core Video VAE Decode + ├─ audio_latents ──→ Core Audio VAE Decode + └─ updated_assembly_plan → Assemble + Seam V3.5 +``` + +Additional connections to `H3 Continuum Hi-Res Fix V3.5`: + +| Hi-Res Fix input | Connect from | +|---|---| +| `model` | The same final H3 MODEL used by First Pass | +| `clip` | The same MiniMax H3 CLIP | +| `sampler` | Second Pass `KSamplerSelect`; accepted test used `res_multistep` | +| `sigmas` | A separate Second Pass `BasicScheduler`; accepted test used `simple`, 10 steps, denoise 0.35 | +| `video_latents` | `H3 Continuum Sampler V3.5.video_latents` | +| `audio_latents` | `H3 Continuum Sampler V3.5.audio_latents` | +| `assembly_plan` | `H3 Continuum Sampler V3.5.assembly_plan` | +| `refine_context` | `H3 Continuum Sampler V3.5.refine_context` | +| `video_vae` | The same H3 Video VAE used by First Pass/Core Decode | + +Accepted Main controls: + +```text +enabled = true +upscale_method = lanczos +scale_by = 2.0 +refine_seed = 42 in the acceptance run +``` + +The node derives one actual refine seed per physical group from `refine_seed`; the report shows that derived sampling seed. + +When `enabled=false`, the node returns the original video/audio LATENT objects and Assembly Plan unchanged. Sampling and VAE dependencies are lazy and are not evaluated. + +## First Pass scheduler versus Second Pass scheduler + +There are normally two independent scheduler roles: + +```text +First Pass BasicScheduler + → H3 Continuum Sampler V3.5.sigmas + +Second Pass BasicScheduler + → H3 Continuum Hi-Res Fix V3.5.sigmas +``` + +The accepted GPU run used: + +```text +First Pass: simple / 20 steps / denoise 1.0 +Second Pass: simple / 10 steps / denoise 0.35 +``` + +Only the Second Pass values describe the Hi-Res Fix refinement strength. + +## What Hi-Res Fix does internally + +```text +First Pass physical video latent + → H3 Video VAE Decode + → CPU pixel resize (Lanczos in the accepted run) + → same H3 Video VAE Encode + → context-aware physical-group Second Pass + → refined video latent +``` + +Audio is not adopted from the temporary Second Pass result. The original First Pass audio LATENT objects are returned bit-exact. + +The Main node never falls back to direct 24-channel latent interpolation. If the required Video VAE is missing or the VAE contract is violated, it raises an explicit error. + +## Advanced external-upscaler connection + +Use `H3 Continuum Second Pass V3.5` directly only when an external latent processor such as LBH is used: + +```text +H3 Continuum Sampler V3.5.video_latents + → LBH / external H3 latent processor + → H3 Continuum Second Pass V3.5.video_latents + +Second Pass BasicScheduler.SIGMAS + → H3 Continuum Second Pass V3.5.sigmas +``` + +Also connect the Sampler's original `audio_latents`, `assembly_plan`, and `refine_context`, plus the same MODEL, CLIP, sampler, and Video VAE. + +Do not place `H3 Continuum Hi-Res Fix V3.5` and a separate `H3 Continuum Second Pass V3.5` in series for the standard path. Hi-Res Fix already performs one Second Pass internally; chaining both would refine twice. + +## Latent Resize Utility + +`H3 Continuum Latent Resize V3.5` changes only the H/W of the 24-channel video latent. It does not sample and is not the standard Main backend. + +Direct 2x latent interpolation to 1024/1152 square produced persistent artifacts in testing. Therefore standalone Latent Resize `nearest-exact / scale_by=2.0`, or Main Hi-Res Fix `bislerp / scale_by=2.0`, must not be described as the accepted Main 2x configuration. The accepted Main 2x configuration is the Pixel/VAE path with pixel Lanczos. + +## GPU-accepted Main condition + +```text +Mode: FL2VA, 1 chunk × 5 seconds +First Pass canvas: 576 × 576 +Hi-Res target: 1152 × 1152 +Pixel resize: Lanczos +Second Pass sampler: res_multistep +Second Pass SIGMAS: simple / 10 steps / denoise 0.35 +Output latent: (1, 24, 37, 72, 72) +Physical groups: 1 +Sampling passes: 1 +Final media: 1152 × 1152, 120 frames, 24 fps, 5.000 seconds +``` + +This integrated Main output matched every decoded video and audio frame of the accepted external Decode -> Lanczos -> Encode -> Second Pass reference. + +Additional GPU results: + +- Hybrid FL2VA + Reference `1 x 5s` passed through the integrated 576-to-1152 Lanczos Main path. Context-aware conditioning used the captured `refine_context`, each physical group sampled once, first-pass audio was returned bit-exact, and Core Decode / Assemble produced 120 frames at 24 fps / 5.000 seconds without persistent line or plate artifacts. +- The same Hybrid/Reference input also passed through the direct Advanced Second Pass node at 576x576, confirming the public bridge node independently of the integrated wrapper. +- Main `3 x 5s` at 576-to-1152 is not accepted on the tested RTX 5060 Ti 16 GiB. The 37T group completed its 10-step Second Pass, then the terminal 77T group failed at its first inference with CUDA OOM (`15.93 GiB` device limit, `10.04 GiB` already allocated, `1.49 GiB` requested). + +Still unverified: + +- Main `3 x 5s` 2x on a GPU with more than 16 GiB VRAM, or a lower-memory implementation for its terminal 77T group +- Reference/Hybrid cases longer than `1 x 5s` +- Main 2x `bislerp` quality; it was not the accepted Lanczos test diff --git a/docs/images/v35-hires-fix-node.png b/docs/images/v35-hires-fix-node.png new file mode 100644 index 0000000..77223ba Binary files /dev/null and b/docs/images/v35-hires-fix-node.png differ diff --git a/docs/images/v35-second-pass-node.png b/docs/images/v35-second-pass-node.png new file mode 100644 index 0000000..adee5e3 Binary files /dev/null and b/docs/images/v35-second-pass-node.png differ diff --git a/hardening.py b/hardening.py index d519713..7f511cd 100644 --- a/hardening.py +++ b/hardening.py @@ -592,6 +592,57 @@ def run_sequence_with_hardening(base: Any, args: tuple[Any, ...], kwargs: dict[s return base(*args, **kwargs) video_vae = _first_argument(arguments, "video_vae") + memory_attribution = bool( + _first_argument(arguments, "memory_attribution") + ) + if memory_attribution: + from .v3.memory_attribution import ( + MemoryAttributionCollector, + capture_attribution_fail_soft, + ) + + collector = _first_argument( + arguments, + "_memory_attribution_collector", + ) + if collector is None: + collector = MemoryAttributionCollector() + capture_attribution_fail_soft( + collector, + "capture_sequence_start", + ) + call_kwargs = dict(kwargs) + call_kwargs["_memory_attribution_collector"] = collector + result = base(*args, **call_kwargs) + entries = ( + result[0] + if isinstance(result, (tuple, list)) and len(result) > 0 + else None + ) + session = ( + result[2] + if isinstance(result, (tuple, list)) and len(result) > 2 + else None + ) + refine_context = ( + result[4] + if isinstance(result, (tuple, list)) and len(result) > 4 + else None + ) + capture_attribution_fail_soft( + collector, + "capture_sequence_complete", + entries=entries, + retained_refine_context=refine_context, + ) + lines = [ + f"Core Video VAE chunked I/O: " + f"{core_video_vae_chunked_io_status(video_vae)}", + *collector.lines, + format_latent_storage_audit(entries, session), + ] + return _append_report(result, lines) + lines = [ f"Core Video VAE chunked I/O: {core_video_vae_chunked_io_status(video_vae)}", format_memory_snapshot("sequence start"), diff --git a/metadata.ini b/metadata.ini index 6326ab0..054f1e1 100644 --- a/metadata.ini +++ b/metadata.ini @@ -1,4 +1,4 @@ [project] name = H3 Continuum -version = 3.4.0 -description = Native MiniMax H3 continuation with Core VAE decode, Reference conditioning and crash-safe Run Storage. +version = 3.5.0 +description = Native MiniMax H3 continuation with Second Pass, experimental Hi-Res Fix, and disk-backed assembly. diff --git a/nodes.py b/nodes.py index 172abb8..cf7fc49 100644 --- a/nodes.py +++ b/nodes.py @@ -382,12 +382,41 @@ from .v3.driving_nodes import ( NODE_CLASS_MAPPINGS.update(V34_NODE_CLASS_MAPPINGS) NODE_DISPLAY_NAME_MAPPINGS.update(V34_NODE_DISPLAY_NAME_MAPPINGS) +from .v3.second_pass_nodes import ( + NODE_CLASS_MAPPINGS as V35_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as V35_NODE_DISPLAY_NAME_MAPPINGS, +) + +NODE_CLASS_MAPPINGS.update(V35_NODE_CLASS_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(V35_NODE_DISPLAY_NAME_MAPPINGS) + +from .v3.latent_resize_nodes import ( + NODE_CLASS_MAPPINGS as V35_RESIZE_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as V35_RESIZE_NODE_DISPLAY_NAME_MAPPINGS, +) + +NODE_CLASS_MAPPINGS.update(V35_RESIZE_NODE_CLASS_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(V35_RESIZE_NODE_DISPLAY_NAME_MAPPINGS) + +from .v3.hires_fix_nodes import ( + NODE_CLASS_MAPPINGS as V35_HIRES_FIX_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as V35_HIRES_FIX_NODE_DISPLAY_NAME_MAPPINGS, +) + +NODE_CLASS_MAPPINGS.update(V35_HIRES_FIX_NODE_CLASS_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(V35_HIRES_FIX_NODE_DISPLAY_NAME_MAPPINGS) + # The stable facade and its pack helpers are the public workflow surface. Keep # every older identifier registered so saved workflows still load, while # ComfyUI's native deprecated-node filter hides legacy building blocks. _primary_node_ids = { "H3ContinuumSamplerV34", + "H3ContinuumSamplerV35", "H3ContinuumAssembleSeamV34", + "H3ContinuumAssembleSeamV35", + "H3ContinuumHiResFixV35", + "H3ContinuumLatentResizeV35", + "H3ContinuumSecondPassV35", "H3ContinuumClipOverrides", "H3ContinuumResult", } diff --git a/pyproject.toml b/pyproject.toml index 3f40e5d..ef073e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "comfyui-h3-continuum" -version = "3.4.0" +version = "3.5.0" description = "Integrated N-chunk continuation for ComfyUI native MiniMax H3 video and audio" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_p20_file_backed_tensor_poc.py b/tests/test_p20_file_backed_tensor_poc.py new file mode 100644 index 0000000..9c4eaf0 --- /dev/null +++ b/tests/test_p20_file_backed_tensor_poc.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import gc +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +import weakref + +import pytest +import torch + + +MODULE_PATH = ( + Path(__file__).resolve().parents[1] / "tools" / "p20_file_backed_tensor_poc.py" +) +SPEC = importlib.util.spec_from_file_location("h3c_p20_file_backed_tensor_poc", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def _known_files(root: Path) -> list[Path]: + return sorted(path for path in root.iterdir() if path.name.startswith(MODULE.FILE_PREFIX)) + + +def test_shared_mapping_is_bit_exact_contiguous_and_write_through(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + reference = torch.arange(120, dtype=torch.float32).reshape(2, 3, 4, 5) + + with manager.allocate(tuple(reference.shape), dtype=reference.dtype) as allocation: + allocation.tensor.copy_(reference) + record = allocation.record + output = allocation.publish() + + assert output.shape == reference.shape + assert output.dtype == reference.dtype + assert output.device.type == "cpu" + assert output.is_contiguous() + assert output.contiguous().data_ptr() == output.data_ptr() + assert record.data_path.stat().st_size == reference.numel() * reference.element_size() + assert record.data_path.read_bytes() == reference.numpy().tobytes(order="C") + assert torch.equal(output, reference) + + storage_pointer = output.untyped_storage().data_ptr() + assert output[1].untyped_storage().data_ptr() == storage_pointer + assert output.detach().untyped_storage().data_ptr() == storage_pointer + assert output.reshape(6, 4, 5).untyped_storage().data_ptr() == storage_pointer + assert output.clone().untyped_storage().data_ptr() != storage_pointer + + del output + gc.collect() + result = manager.collect_ready() + assert result.reclaimed == 1 + assert result.deferred == 0 + assert not _known_files(tmp_path) + + +def test_lifetime_waits_for_last_storage_alias_and_not_root_tensor(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + with manager.allocate((4, 3, 2), dtype=torch.float32) as allocation: + allocation.tensor.copy_(torch.arange(24, dtype=torch.float32).reshape(4, 3, 2)) + record = allocation.record + output = allocation.publish() + + view = output[1:] + cache = {"images": view} + root_reference = weakref.ref(output) + del output + gc.collect() + + assert root_reference() is None + assert float(cache["images"].sum()) == pytest.approx(261.0) + first = manager.collect_ready() + assert first.reclaimed == 0 + assert first.deferred == 1 + assert record.data_path.exists() + assert record.metadata_path.exists() + + del view + cache.clear() + gc.collect() + second = manager.collect_ready() + assert second.reclaimed == 1 + assert second.deferred == 0 + assert not record.data_path.exists() + assert not record.metadata_path.exists() + + +@pytest.mark.parametrize("exception_type", [RuntimeError, MODULE.ProbeCancelled]) +def test_unpublished_exception_and_cancel_remove_provisional_files( + tmp_path, + exception_type, +): + manager = MODULE.StorageLifetimeManager(tmp_path) + with pytest.raises(exception_type): + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + allocation.tensor.fill_(1.0) + raise exception_type("stop before publish") + + assert manager.tracked_count() == 0 + assert not _known_files(tmp_path) + + +def test_mapping_creation_failure_removes_reserved_file(tmp_path, monkeypatch): + manager = MODULE.StorageLifetimeManager(tmp_path) + + def fail_from_file(*args, **kwargs): + raise RuntimeError("synthetic mapping failure") + + monkeypatch.setattr(MODULE.torch, "from_file", fail_from_file) + with pytest.raises(RuntimeError, match="synthetic mapping failure"): + manager.allocate((2, 2), dtype=torch.float32) + assert not _known_files(tmp_path) + + +def test_temporary_unlink_failure_preserves_cancel_and_is_retried( + tmp_path, + monkeypatch, +): + manager = MODULE.StorageLifetimeManager(tmp_path) + original_unlink = MODULE._unlink_known + fail_unlink = {"enabled": True} + + def sharing_violation(root, path): + if fail_unlink["enabled"]: + raise PermissionError(32, "synthetic sharing violation", str(path)) + return original_unlink(root, path) + + monkeypatch.setattr(MODULE, "_unlink_known", sharing_violation) + with pytest.raises(MODULE.ProbeCancelled, match="original cancel"): + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + allocation.tensor.fill_(1.0) + raise MODULE.ProbeCancelled("original cancel") + + assert manager.pending_count() == 1 + assert len(_known_files(tmp_path)) == 2 + fail_unlink["enabled"] = False + result = manager.collect_ready() + assert result.reclaimed == 1 + assert result.deferred == 0 + assert manager.pending_count() == 0 + assert not _known_files(tmp_path) + + +def test_cancel_after_backing_file_create_does_not_leave_orphan( + tmp_path, + monkeypatch, +): + manager = MODULE.StorageLifetimeManager(tmp_path) + original_close = MODULE.os.close + injected = False + sentinel_path = tmp_path / "unrelated-live-descriptor.bin" + sentinel_descriptor = None + + def cancel_after_real_close(descriptor): + nonlocal injected, sentinel_descriptor + original_close(descriptor) + if not injected: + injected = True + opened = MODULE.os.open( + sentinel_path, + MODULE.os.O_CREAT | MODULE.os.O_EXCL | MODULE.os.O_RDWR, + 0o600, + ) + if opened != descriptor: + MODULE.os.dup2(opened, descriptor) + original_close(opened) + opened = descriptor + sentinel_descriptor = opened + raise MODULE.ProbeCancelled("cancel after backing file create") + + try: + monkeypatch.setattr(MODULE.os, "close", cancel_after_real_close) + with pytest.raises(MODULE.ProbeCancelled, match="after backing file create"): + manager.allocate((2, 2), dtype=torch.float32) + finally: + monkeypatch.setattr(MODULE.os, "close", original_close) + + assert sentinel_descriptor is not None + MODULE.os.write(sentinel_descriptor, b"still-open") + original_close(sentinel_descriptor) + assert sentinel_path.read_bytes() == b"still-open" + manager.collect_ready() + assert manager.pending_count() == 0 + assert not _known_files(tmp_path) + + +def test_dropped_unpublished_allocation_is_finalized(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + allocation = manager.allocate((2, 2), dtype=torch.float32) + allocation.tensor.fill_(3.0) + allocation_reference = weakref.ref(allocation) + del allocation + gc.collect() + + assert allocation_reference() is None + manager.collect_ready() + assert manager.pending_count() == 0 + assert not _known_files(tmp_path) + + +def test_cache_requeue_keeps_backing_file_count_bounded(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + cache = {} + + for index in range(20): + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + allocation.tensor.fill_(float(index)) + output = allocation.publish() + cache["same_node_output"] = output + del output + gc.collect() + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) <= 2 + + same_requeue = cache["same_node_output"] + assert same_requeue is cache["same_node_output"] + manager.collect_ready() + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1 + cache.clear() + del same_requeue + gc.collect() + result = manager.collect_ready() + assert result.reclaimed == 1 + assert not _known_files(tmp_path) + + +def test_published_downstream_error_keeps_cache_then_reclaims_after_eviction(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + cache = {} + try: + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + allocation.tensor.fill_(4.0) + output = allocation.publish() + cache["assembler_output"] = output + raise RuntimeError("downstream node failed") + except RuntimeError: + pass + + del output + gc.collect() + held = manager.collect_ready() + assert held.reclaimed == 0 + assert held.deferred == 1 + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1 + + cache.clear() + gc.collect() + released = manager.collect_ready() + assert released.reclaimed == 1 + assert released.deferred == 0 + assert not _known_files(tmp_path) + + +def test_process_global_manager_survives_node_local_scope(tmp_path): + first = MODULE.get_process_lifetime_manager(tmp_path) + second = MODULE.get_process_lifetime_manager(tmp_path) + assert first is second + + with first.allocate((2, 2), dtype=torch.float32) as allocation: + output = allocation.publish() + manager_reference = weakref.ref(first) + del first + del second + gc.collect() + assert manager_reference() is not None + + del output + gc.collect() + manager = MODULE.get_process_lifetime_manager(tmp_path) + assert manager.collect_ready().reclaimed == 1 + assert not _known_files(tmp_path) + + +def test_stale_child_files_are_reclaimed_but_live_and_unknown_files_are_kept(tmp_path): + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--root", + str(tmp_path), + "--crash-child", + ], + check=True, + ) + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1 + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.json"))) == 1 + + unknown = tmp_path / "unrelated.bin" + unknown.write_bytes(b"do not delete") + outside = tmp_path.parent / "outside.bin" + outside.write_bytes(b"outside root") + malicious_id = "f" * 32 + malicious_metadata = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.json" + malicious_data = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.bin" + malicious_data.write_bytes(b"protected") + malicious_metadata.write_text( + json.dumps( + { + "schema": MODULE.SCHEMA, + "schema_version": MODULE.SCHEMA_VERSION, + "allocation_id": malicious_id, + "data_file": "..\\outside.bin", + "owner_pid": -1, + "created_at": 0, + } + ), + encoding="utf-8", + ) + + result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0) + assert result.removed_pairs == 1 + assert result.invalid_metadata == 1 + assert unknown.read_bytes() == b"do not delete" + assert outside.read_bytes() == b"outside root" + assert malicious_data.read_bytes() == b"protected" + assert malicious_metadata.exists() + + +@pytest.mark.parametrize( + ("field", "value", "remove"), + [ + ("owner_pid", None, True), + ("owner_pid", -1, False), + ("owner_create_time", float("nan"), False), + ("created_at", float("nan"), False), + ], +) +def test_invalid_ownership_metadata_never_deletes_live_mapping( + tmp_path, + field, + value, + remove, +): + manager = MODULE.StorageLifetimeManager(tmp_path) + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + record = allocation.record + output = allocation.publish() + + payload = json.loads(record.metadata_path.read_text(encoding="utf-8")) + if remove: + payload.pop(field) + else: + payload[field] = value + record.metadata_path.write_text(json.dumps(payload), encoding="utf-8") + + result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0) + assert result.invalid_metadata == 1 + assert result.removed_pairs == 0 + assert record.data_path.exists() + assert record.metadata_path.exists() + assert float(output.sum()) == pytest.approx(0.0) + + del output + gc.collect() + assert manager.collect_ready().reclaimed == 1 + assert not _known_files(tmp_path) + + +def test_cleanup_waits_for_in_progress_allocation_root_lock(tmp_path): + allocation_entered = threading.Event() + allow_allocation = threading.Event() + output_holder = [] + cleanup_holder = [] + + def allocation_hook(_path): + allocation_entered.set() + assert allow_allocation.wait(timeout=5.0) + + manager = MODULE.StorageLifetimeManager( + tmp_path, + allocation_hook=allocation_hook, + ) + + def allocate_output(): + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + output_holder.append(allocation.publish()) + + def cleanup(): + cleanup_holder.append( + MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0) + ) + + allocation_thread = threading.Thread(target=allocate_output) + allocation_thread.start() + assert allocation_entered.wait(timeout=5.0) + cleanup_thread = threading.Thread(target=cleanup) + cleanup_thread.start() + assert cleanup_thread.is_alive() + allow_allocation.set() + allocation_thread.join(timeout=5.0) + cleanup_thread.join(timeout=5.0) + + assert not allocation_thread.is_alive() + assert not cleanup_thread.is_alive() + assert cleanup_holder[0].retained_live == 1 + assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1 + + output_holder.clear() + gc.collect() + assert manager.collect_ready().reclaimed == 1 + assert not _known_files(tmp_path) + + +def test_root_lock_is_reentrant_when_finalizer_runs_during_allocation(tmp_path): + first_manager = MODULE.StorageLifetimeManager(tmp_path) + abandoned = first_manager.allocate((2, 2), dtype=torch.float32) + cycle = [abandoned] + cycle.append(cycle) + del abandoned + del cycle + + def collect_abandoned_allocation(_path): + gc.collect() + + second_manager = MODULE.StorageLifetimeManager( + tmp_path, + allocation_hook=collect_abandoned_allocation, + ) + started = time.perf_counter() + with second_manager.allocate((2, 2), dtype=torch.float32) as allocation: + output = allocation.publish() + elapsed = time.perf_counter() - started + + assert elapsed < 2.0 + first_manager.collect_ready() + del output + gc.collect() + second_manager.collect_ready() + assert not _known_files(tmp_path) + + +def test_cleanup_collect_and_gc_finalizer_have_no_lock_order_deadlock(tmp_path): + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--root", + str(tmp_path), + "--lock-order-child", + ], + check=True, + timeout=10.0, + ) + assert not _known_files(tmp_path) + + +def test_live_sidecar_is_retained_and_cleanup_is_idempotent(tmp_path): + manager = MODULE.StorageLifetimeManager(tmp_path) + with manager.allocate((2, 2), dtype=torch.float32) as allocation: + allocation.tensor.fill_(2.0) + record = allocation.record + output = allocation.publish() + + live = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0) + assert live.retained_live == 1 + assert record.data_path.exists() + assert record.metadata_path.exists() + + del output + gc.collect() + assert manager.collect_ready().reclaimed == 1 + empty = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0) + assert empty.removed_pairs == 0 + assert empty.removed_orphans == 0 + assert not _known_files(tmp_path) + + +def test_orphan_and_partial_cleanup_respects_grace_period(tmp_path): + allocation_id = "a" * 32 + orphan = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.bin" + partial = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.json.partial" + orphan.write_bytes(b"orphan") + partial.write_text("partial", encoding="utf-8") + now = time.time() + + young = MODULE.cleanup_stale_files( + tmp_path, + grace_seconds=60.0, + now=now, + ) + assert young.retained_young == 2 + assert orphan.exists() + assert partial.exists() + + old = MODULE.cleanup_stale_files( + tmp_path, + grace_seconds=60.0, + now=now + 61.0, + ) + assert old.removed_orphans == 2 + assert not orphan.exists() + assert not partial.exists() + + +@pytest.mark.parametrize("shape", [(2.5, 2), (True, 2), ("2", 2), ()]) +def test_shape_contract_rejects_lossy_or_empty_dimensions(tmp_path, shape): + manager = MODULE.StorageLifetimeManager(tmp_path) + with pytest.raises((TypeError, ValueError)): + manager.allocate(shape, dtype=torch.float32) + assert not _known_files(tmp_path) + + +def test_standalone_probe_passes(tmp_path): + result = MODULE.run_probe(tmp_path) + assert result.bit_exact + assert result.file_size == result.expected_bytes + assert result.root_released_while_alias_alive + assert result.deferred_while_alias_alive == 1 + assert result.reclaimed_after_last_alias == 1 + assert result.files_remaining == 0 diff --git a/tests/test_p21_memory_attribution.py b/tests/test_p21_memory_attribution.py new file mode 100644 index 0000000..942650a --- /dev/null +++ b/tests/test_p21_memory_attribution.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + DIAGNOSTICS_BASIC, + DIAGNOSTICS_FULL, + DIAGNOSTICS_OFF, +) +from ComfyUI_H3_Continuum_Join import hardening +from ComfyUI_H3_Continuum_Join.hardening import enrich_assembly_plan +from ComfyUI_H3_Continuum_Join.v3 import memory_attribution as memory +from ComfyUI_H3_Continuum_Join.v3 import driving_nodes as driving_nodes_module +from ComfyUI_H3_Continuum_Join.v3.driving_nodes import ( + H3ContinuumAssembleSeamV34, + H3ContinuumAssembleSeamV35, + NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS, +) +from ComfyUI_H3_Continuum_Join.v3.plan import ASSEMBLY_PLAN_MAGIC + + +def _terminal_plan(): + plan = enrich_assembly_plan( + { + "magic": ASSEMBLY_PLAN_MAGIC, + "schema_version": 1, + "fps": 24, + "width": 96, + "height": 64, + "chunk_seconds": 5.0, + "target_frames": 360, + "preserve_final_frame": True, + "chunks": [ + { + "sequence_index": 1, + "chunk_index": 1, + "total_frames": 124, + "trim_frames": 0, + "net_frames": 124, + "context_frames": 0, + "expected_video_latent_t": 37, + "expected_audio_latent_t": 207, + }, + { + "sequence_index": 2, + "chunk_index": 2, + "total_frames": 141, + "trim_frames": 22, + "net_frames": 119, + "context_frames": 22, + "expected_video_latent_t": 42, + "expected_audio_latent_t": 235, + }, + { + "sequence_index": 3, + "chunk_index": 3, + "total_frames": 141, + "trim_frames": 22, + "net_frames": 119, + "context_frames": 22, + "expected_video_latent_t": 42, + "expected_audio_latent_t": 235, + }, + ], + } + ) + plan["decode_groups"] = [ + { + "logical_chunk_indices": [1], + "terminal_merged": False, + "total_frames": 124, + "trim_frames": 0, + "net_frames": 124, + "context_frames": 0, + "expected_video_latent_t": 37, + "expected_audio_latent_t": 207, + "frame_start": 0, + "frame_stop": 124, + }, + { + "logical_chunk_indices": [2, 3], + "terminal_merged": True, + "total_frames": 260, + "trim_frames": 22, + "net_frames": 238, + "context_frames": 22, + "expected_video_latent_t": 77, + "expected_audio_latent_t": 433, + "frame_start": 124, + "frame_stop": 362, + }, + ] + plan["physical_decode_group_count"] = 2 + return plan + + +def _decoded(*, image_dtype=torch.float16, device=None): + images = [ + torch.empty((124, 4, 6, 3), dtype=image_dtype, device=device), + torch.empty((260, 4, 6, 3), dtype=image_dtype, device=device), + ] + audio = [ + { + "waveform": torch.empty( + (1, 2, 500000), + dtype=torch.float32, + device=device, + ), + "sample_rate": 32000, + }, + { + "waveform": torch.empty( + (1, 2, 500000), + dtype=torch.float32, + device=device, + ), + "sample_rate": 32000, + }, + ] + return images, audio + + +def test_snapshot_is_cpu_safe_read_only_and_deduplicates_retained_storage( + monkeypatch, +): + video_storage = torch.arange(24, dtype=torch.float32) + audio_storage = torch.arange(12, dtype=torch.float32) + entries = [ + {"video": video_storage[:16], "audio": audio_storage[:8]}, + {"video": video_storage[8:], "audio": audio_storage[4:]}, + ] + monkeypatch.setattr(memory.torch.cuda, "is_available", lambda: False) + for name in ( + "memory_allocated", + "memory_reserved", + "max_memory_allocated", + "synchronize", + "reset_peak_memory_stats", + "empty_cache", + ): + monkeypatch.setattr( + memory.torch.cuda, + name, + lambda *args, _name=name, **kwargs: (_ for _ in ()).throw( + AssertionError(f"unexpected CUDA call: {_name}") + ), + ) + + line = memory.format_attribution_snapshot( + "test", + retained_entries=entries, + ) + assert f"retained_video={video_storage.untyped_storage().nbytes()} bytes" in line + assert f"retained_audio={audio_storage.untyped_storage().nbytes()} bytes" in line + assert "CUDA_allocated=n/a" in line + assert "CUDA_peak_allocated_since_external_reset=n/a" in line + + +def test_snapshot_counter_failure_is_reported_as_unavailable(monkeypatch): + monkeypatch.setattr(memory, "_process_counters", lambda: (None, None)) + monkeypatch.setattr(memory, "_cuda_counters", lambda: (None, None, None)) + line = memory.format_attribution_snapshot("failure-safe") + assert "RSS=n/a" in line + assert "system_available=n/a" in line + assert "CUDA_reserved=n/a" in line + + +def test_snapshot_counts_refine_context_storage_once(): + shared = torch.arange(32, dtype=torch.float32) + refine_context = { + "groups": ( + { + "conditioning": ( + (shared[:8], {"reference": shared[8:16]}), + ), + "first_image": shared[16:24], + }, + ), + "alias": shared[4:28], + } + + line = memory.format_attribution_snapshot( + "refine-context", + retained_refine_context=refine_context, + ) + + expected = shared.untyped_storage().nbytes() + assert f"retained_refine_context_CPU={expected} bytes" in line + assert "retained_refine_context_GPU=0 bytes" in line + + +def test_cuda_snapshot_reads_counters_without_allocator_mutation(monkeypatch): + monkeypatch.setattr(memory.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(memory.torch.cuda, "memory_allocated", lambda: 11) + monkeypatch.setattr(memory.torch.cuda, "memory_reserved", lambda: 22) + monkeypatch.setattr(memory.torch.cuda, "max_memory_allocated", lambda: 33) + for name in ( + "synchronize", + "reset_peak_memory_stats", + "empty_cache", + ): + monkeypatch.setattr( + memory.torch.cuda, + name, + lambda *args, _name=name, **kwargs: (_ for _ in ()).throw( + AssertionError(f"unexpected CUDA mutation: {_name}") + ), + ) + + line = memory.format_attribution_snapshot("cuda-read-only") + + assert "CUDA_allocated=0.0 MiB" in line + assert "CUDA_reserved=0.0 MiB" in line + assert "CUDA_peak_allocated_since_external_reset=0.0 MiB" in line + + +@pytest.mark.parametrize( + "method", + ( + "capture_sequence_start", + "capture_group", + "capture_sequence_complete", + ), +) +def test_each_collector_stage_is_fail_soft(method): + class ExplodingCollector: + def __init__(self): + self.lines = [] + + def __getattr__(self, name): + if name.startswith("capture_"): + return lambda **_kwargs: (_ for _ in ()).throw( + RuntimeError(f"{name} failed") + ) + raise AttributeError(name) + + collector = ExplodingCollector() + memory.capture_attribution_fail_soft(collector, method) + + assert collector.lines == [ + f"Memory attribution [{method}]: unavailable " + f"(RuntimeError: {method} failed)" + ] + + +def test_sequence_start_and_complete_failures_do_not_replace_sampling_result(): + class ExplodingCollector: + def __init__(self): + self.lines = [] + + def capture_sequence_start(self): + raise RuntimeError("start failed") + + def capture_sequence_complete(self, **_kwargs): + raise RuntimeError("complete failed") + + collector = ExplodingCollector() + entries = [{"video": torch.zeros(1), "audio": torch.zeros(1)}] + session = {"chunks": entries} + refine_context = {"groups": ()} + base_result = (entries, "state", session, "base report", refine_context) + base_calls = [] + + def base( + *, + video_vae, + latent_only, + diagnostics_mode, + memory_attribution, + _memory_attribution_collector=None, + ): + base_calls.append(_memory_attribution_collector) + return base_result + + result = hardening.run_sequence_with_hardening( + base, + (), + { + "video_vae": object(), + "latent_only": True, + "diagnostics_mode": DIAGNOSTICS_FULL, + "memory_attribution": True, + "_memory_attribution_collector": collector, + }, + ) + + assert base_calls == [collector] + assert result[:3] == base_result[:3] + assert result[4] is refine_context + assert "base report" in result[3] + assert "capture_sequence_start" in result[3] + assert "capture_sequence_complete" in result[3] + + +@pytest.mark.parametrize("diagnostics", (DIAGNOSTICS_BASIC, DIAGNOSTICS_OFF)) +def test_v35_attribution_is_absent_outside_detailed_report(diagnostics): + base_result = ("entries", "state", "session", "base report", "context") + calls = [] + + def base( + *, + video_vae, + latent_only, + diagnostics_mode, + memory_attribution, + _memory_attribution_collector=None, + ): + calls.append(_memory_attribution_collector) + return base_result + + result = hardening.run_sequence_with_hardening( + base, + (), + { + "video_vae": object(), + "latent_only": True, + "diagnostics_mode": diagnostics, + "memory_attribution": True, + }, + ) + + assert result is base_result + assert calls == [None] + assert "Memory [V3.5" not in result[3] + + +def test_v34_private_flag_off_keeps_existing_detailed_report_path(): + entries = [{"video": torch.zeros(1), "audio": torch.zeros(1)}] + session = {"chunks": entries} + calls = [] + + def base( + *, + video_vae, + latent_only, + diagnostics_mode, + memory_attribution=False, + _memory_attribution_collector=None, + ): + calls.append((memory_attribution, _memory_attribution_collector)) + return entries, "state", session, "base report" + + result = hardening.run_sequence_with_hardening( + base, + (), + { + "video_vae": object(), + "latent_only": True, + "diagnostics_mode": DIAGNOSTICS_FULL, + "memory_attribution": False, + }, + ) + + assert calls == [(False, None)] + assert result[0] is entries + assert result[2] is session + assert "Memory [sequence start]" in result[3] + assert "Memory [sequence complete]" in result[3] + assert "Memory [V3.5" not in result[3] + + +def test_projection_uses_physical_groups_actual_dtypes_and_exact_target(): + images, audio = _decoded() + off = memory.project_assembly_buffers( + images=images, + audio=audio, + assembly_plan=_terminal_plan(), + exact_total_duration=False, + ) + assert off.natural_frames == 362 + assert off.target_frames == 362 + assert off.natural_image_bytes == 52128 + assert off.natural_audio_bytes == 3861336 + assert off.final_image_bytes == 52128 + assert off.final_audio_bytes == 3861336 + assert off.frame_count_adjustment_required is False + + exact = memory.project_assembly_buffers( + images=images, + audio=audio, + assembly_plan=_terminal_plan(), + exact_total_duration=True, + ) + assert exact.natural_frames == 362 + assert exact.target_frames == 360 + assert exact.natural_image_bytes == 52128 + assert exact.natural_audio_bytes == 3861336 + assert exact.final_image_bytes == 51840 + assert exact.final_audio_bytes == 3840000 + assert exact.frame_count_adjustment_required is True + + +def test_projection_uses_python_int_for_real_1152_square_output(): + images, audio = _decoded(device="meta") + images = [ + torch.empty((124, 1152, 1152, 3), dtype=torch.float32, device="meta"), + torch.empty((260, 1152, 1152, 3), dtype=torch.float32, device="meta"), + ] + projection = memory.project_assembly_buffers( + images=images, + audio=audio, + assembly_plan=_terminal_plan(), + exact_total_duration=True, + ) + assert projection.final_image_bytes == 5733089280 + assert isinstance(projection.final_image_bytes, int) + + +def test_v35_assembler_adds_projection_only_for_detailed_report(monkeypatch): + input_images, input_audio = _decoded() + output_images = torch.zeros((1, 1, 1, 3)) + output_audio = {"waveform": torch.zeros((1, 2, 1)), "sample_rate": 32000} + base_calls = [] + + def fake_v35_assemble(**kwargs): + base_calls.append(kwargs) + return ( + output_images, + output_audio, + "base report\nMemory [assemble preflight]: test\n" + "Memory [assemble complete]: test", + ) + + monkeypatch.setattr( + driving_nodes_module, + "assemble_decoded_chunks_v35", + fake_v35_assemble, + ) + node = H3ContinuumAssembleSeamV35() + detailed_images, detailed_audio, detailed_report = node.assemble( + images=input_images, + audio=input_audio, + assembly_plan=_terminal_plan(), + exact_total_duration=True, + audio_seam="Auto", + video_seam="Auto", + buffer_backend="RAM", + diagnostics=DIAGNOSTICS_FULL, + ) + assert detailed_images is output_images + assert detailed_audio is output_audio + assert "Projected assembly buffers [V3.5]" in detailed_report + assert detailed_report.index("Memory [assemble preflight]") < detailed_report.index( + "Projected assembly buffers [V3.5]" + ) + assert detailed_report.index("Projected assembly buffers [V3.5]") < ( + detailed_report.index("Memory [assemble complete]") + ) + + _, _, basic_report = node.assemble( + images=input_images, + audio=input_audio, + assembly_plan=_terminal_plan(), + exact_total_duration=True, + audio_seam="Auto", + video_seam="Auto", + buffer_backend="RAM", + diagnostics=DIAGNOSTICS_BASIC, + ) + assert "Projected assembly buffers [V3.5]" not in basic_report + assert len(base_calls) == 2 + + +def test_v35_assembler_projection_failure_never_replaces_base_execution(monkeypatch): + output_images = torch.zeros((1, 1, 1, 3)) + output_audio = {"waveform": torch.zeros((1, 2, 1)), "sample_rate": 32000} + monkeypatch.setattr( + driving_nodes_module, + "assemble_decoded_chunks_v35", + lambda **kwargs: (output_images, output_audio, "base report"), + ) + images, audio, report = H3ContinuumAssembleSeamV35().assemble( + images=[], + audio=[], + assembly_plan={}, + exact_total_duration=True, + audio_seam="Auto", + video_seam="Auto", + buffer_backend="RAM", + diagnostics=DIAGNOSTICS_FULL, + ) + assert images is output_images + assert audio is output_audio + assert "Projected assembly buffers [V3.5]: unavailable" in report + + +def test_v35_assembler_is_registered_without_replacing_v34(): + from ComfyUI_H3_Continuum_Join import nodes as root_nodes + + assert NODE_CLASS_MAPPINGS["H3ContinuumAssembleSeamV34"] is ( + H3ContinuumAssembleSeamV34 + ) + assert NODE_CLASS_MAPPINGS["H3ContinuumAssembleSeamV35"] is ( + H3ContinuumAssembleSeamV35 + ) + assert root_nodes.NODE_CLASS_MAPPINGS["H3ContinuumAssembleSeamV35"] is ( + H3ContinuumAssembleSeamV35 + ) + assert NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumAssembleSeamV35"] == ( + "H3 Continuum Assemble + Seam V3.5" + ) diff --git a/tests/test_p22_file_backed_buffer.py b/tests/test_p22_file_backed_buffer.py new file mode 100644 index 0000000..66e567a --- /dev/null +++ b/tests/test_p22_file_backed_buffer.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import gc +from pathlib import Path +import subprocess +import sys +from types import SimpleNamespace + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.v3 import file_backed_buffer as buffer + + +def _managed_files(root: Path) -> list[Path]: + return sorted( + path + for path in root.iterdir() + if path.suffix in {".bin", ".json", ".partial"} + ) + + +def test_backend_contract_accepts_auto_and_rejects_unknown_values(): + assert buffer.BUFFER_BACKEND_OPTIONS == ( + buffer.BUFFER_BACKEND_AUTO, + buffer.BUFFER_BACKEND_RAM, + buffer.BUFFER_BACKEND_DISK, + ) + assert buffer.validate_buffer_backend("Auto") == "Auto" + assert buffer.validate_buffer_backend("RAM") == "RAM" + assert buffer.validate_buffer_backend("Disk-backed") == "Disk-backed" + for value in ("disk-backed", "", None): + with pytest.raises(ValueError, match="unknown Buffer Backend"): + buffer.validate_buffer_backend(value) + + +def test_auto_selects_ram_only_with_size_and_memory_headroom(tmp_path, monkeypatch): + gib = 1024**3 + monkeypatch.setattr( + buffer, + "_system_memory_bytes", + lambda: (16 * gib, 64 * gib), + ) + monkeypatch.setattr( + buffer, + "_disk_free_bytes", + lambda _path: (_ for _ in ()).throw( + AssertionError("disk must not be queried for a safe RAM decision") + ), + ) + + decision = buffer.select_auto_buffer_backend(gib, backing_root=tmp_path) + + assert decision.selected_backend == buffer.BUFFER_BACKEND_RAM + assert decision.memory_reserve_bytes == int(64 * gib * 0.10) + assert decision.disk_free_bytes is None + assert "fits" in decision.reason + + +@pytest.mark.parametrize( + ("final_bytes", "available", "reason"), + ( + (5 * 1024**3, 48 * 1024**3, "exceeds"), + (1 * 1024**3, 5 * 1024**3, "below"), + ), +) +def test_auto_selects_disk_for_large_output_or_memory_pressure( + tmp_path, + monkeypatch, + final_bytes, + available, + reason, +): + gib = 1024**3 + monkeypatch.setattr( + buffer, + "_system_memory_bytes", + lambda: (available, 64 * gib), + ) + monkeypatch.setattr(buffer, "_disk_free_bytes", lambda _path: 100 * gib) + + decision = buffer.select_auto_buffer_backend( + final_bytes, + backing_root=tmp_path, + ) + + assert decision.selected_backend == buffer.BUFFER_BACKEND_DISK + assert decision.disk_free_bytes == 100 * gib + assert reason in decision.reason + + +def test_auto_uses_disk_when_memory_counters_are_unavailable(tmp_path, monkeypatch): + gib = 1024**3 + monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (None, None)) + monkeypatch.setattr(buffer, "_disk_free_bytes", lambda _path: 100 * gib) + + decision = buffer.select_auto_buffer_backend(gib, backing_root=tmp_path) + + assert decision.selected_backend == buffer.BUFFER_BACKEND_DISK + assert decision.memory_reserve_bytes is None + assert "unavailable" in decision.reason + + +def test_auto_disk_shortage_is_explicit_and_never_falls_back(tmp_path, monkeypatch): + gib = 1024**3 + monkeypatch.setattr( + buffer, + "_system_memory_bytes", + lambda: (48 * gib, 64 * gib), + ) + monkeypatch.setattr(buffer, "_disk_free_bytes", lambda _path: 6 * gib) + + with pytest.raises(RuntimeError, match="selected Disk-backed"): + buffer.select_auto_buffer_backend(5 * gib, backing_root=tmp_path) + + +def test_auto_decision_report_records_selection_inputs(tmp_path, monkeypatch): + gib = 1024**3 + monkeypatch.setattr( + buffer, + "_system_memory_bytes", + lambda: (16 * gib, 64 * gib), + ) + decision = buffer.select_auto_buffer_backend(gib, backing_root=tmp_path) + + report = buffer.format_auto_buffer_decision(decision) + + assert "selected=RAM" in report + assert f"final_IMAGE={gib} bytes" in report + assert "RAM_IMAGE_limit=4294967296 bytes" in report + assert "reason=" in report + + +def test_default_root_is_resolved_lazily_from_comfyui_temp(tmp_path, monkeypatch): + calls = [] + comfy_temp = tmp_path / "comfy-temp" + fake_folder_paths = SimpleNamespace( + get_temp_directory=lambda: calls.append(True) or str(comfy_temp) + ) + monkeypatch.setitem(sys.modules, "folder_paths", fake_folder_paths) + + assert calls == [] + resolved = buffer.resolve_backing_root() + assert calls == [True] + assert resolved == (comfy_temp / "h3-continuum-v35-file-backed").resolve() + + +def test_default_root_has_a_non_comfy_test_fallback(tmp_path, monkeypatch): + monkeypatch.setitem(sys.modules, "folder_paths", None) + monkeypatch.setattr(buffer.tempfile, "gettempdir", lambda: str(tmp_path)) + + assert buffer.resolve_backing_root() == ( + tmp_path / "h3-continuum-v35-file-backed" + ).resolve() + + +def test_explicit_root_bypasses_comfyui_resolution(tmp_path, monkeypatch): + fake_folder_paths = SimpleNamespace( + get_temp_directory=lambda: (_ for _ in ()).throw( + AssertionError("folder_paths must not be evaluated") + ) + ) + monkeypatch.setitem(sys.modules, "folder_paths", fake_folder_paths) + + assert buffer.resolve_backing_root(tmp_path) == tmp_path.resolve() + + +def test_file_backed_allocation_uses_process_global_manager_and_alias_lifetime( + tmp_path, +): + manager = buffer.get_file_backed_image_manager(tmp_path) + assert manager is buffer.get_file_backed_image_manager(tmp_path) + reference = torch.arange(4 * 3 * 2 * 3, dtype=torch.float32).reshape(4, 3, 2, 3) + + with buffer.allocate_file_backed_image( + tuple(reference.shape), + dtype=reference.dtype, + backing_root=tmp_path, + ) as allocation: + allocation.tensor.copy_(reference) + record = allocation.record + output = allocation.publish() + + assert output.device.type == "cpu" + assert output.is_contiguous() + assert torch.equal(output, reference) + assert record.data_path.parent == tmp_path.resolve() + assert record.data_path.stat().st_size == reference.numel() * reference.element_size() + assert record.data_path.exists() + assert record.metadata_path.exists() + + alias = output[1:] + del output + gc.collect() + held = manager.collect_ready() + assert held.reclaimed == 0 + assert held.deferred == 1 + assert float(alias.sum()) == pytest.approx(float(reference[1:].sum())) + assert record.data_path.exists() + + del alias + gc.collect() + released = manager.collect_ready() + assert released.reclaimed == 1 + assert released.deferred == 0 + assert not _managed_files(tmp_path) + + +def test_unpublished_error_aborts_provisional_files(tmp_path): + manager = buffer.get_file_backed_image_manager(tmp_path) + + with pytest.raises(RuntimeError, match="assembly failed"): + with buffer.allocate_file_backed_image( + (2, 2, 2, 3), + dtype=torch.float32, + backing_root=tmp_path, + ) as allocation: + allocation.tensor.fill_(1.0) + raise RuntimeError("assembly failed") + + assert manager.tracked_count() == 0 + assert manager.pending_count() == 0 + assert not _managed_files(tmp_path) + + +def test_disk_backend_failure_is_propagated_without_ram_fallback( + tmp_path, + monkeypatch, +): + class FailingManager: + def allocate(self, *args, **kwargs): + raise OSError("synthetic disk failure") + + monkeypatch.setattr( + buffer, + "get_file_backed_image_manager", + lambda backing_root=None: FailingManager(), + ) + with pytest.raises(OSError, match="synthetic disk failure"): + buffer.allocate_file_backed_image( + (2, 2, 2, 3), + dtype=torch.float32, + backing_root=tmp_path, + ) + + +def test_first_manager_access_reclaims_prior_dead_process_files(tmp_path): + probe = ( + Path(__file__).resolve().parents[1] + / "tools" + / "p20_file_backed_tensor_poc.py" + ) + subprocess.run( + [ + sys.executable, + str(probe), + "--root", + str(tmp_path), + "--crash-child", + ], + check=True, + ) + assert list(tmp_path.glob("*.bin")) + assert list(tmp_path.glob("*.json")) + + buffer.get_file_backed_image_manager(tmp_path) + + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) diff --git a/tests/test_p23_assembly_v35.py b/tests/test_p23_assembly_v35.py new file mode 100644 index 0000000..4973316 --- /dev/null +++ b/tests/test_p23_assembly_v35.py @@ -0,0 +1,796 @@ +from __future__ import annotations + +import gc +import inspect +from pathlib import Path + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + DIAGNOSTICS_BASIC, + DIAGNOSTICS_FULL, +) +from ComfyUI_H3_Continuum_Join.hardening import enrich_assembly_plan +from ComfyUI_H3_Continuum_Join.temporal import audio_latent_t, video_latent_t +from ComfyUI_H3_Continuum_Join.v3 import assembly_v35 +from ComfyUI_H3_Continuum_Join.v3.assembly import ( + AUDIO_SEAM_AUTO, + AUDIO_SEAM_OFF, + VIDEO_SEAM_ANALYZE, + VIDEO_SEAM_AUTO, + VIDEO_SEAM_AUTO_2, + VIDEO_SEAM_OFF, +) +from ComfyUI_H3_Continuum_Join.v3.driving_nodes import ( + H3ContinuumAssembleSeamV34, + H3ContinuumAssembleSeamV35, +) +from ComfyUI_H3_Continuum_Join.v3.file_backed_buffer import ( + AutoBufferDecision, + BUFFER_BACKEND_AUTO, + BUFFER_BACKEND_DISK, + BUFFER_BACKEND_RAM, + get_file_backed_image_manager, +) +from ComfyUI_H3_Continuum_Join.v3.plan import ASSEMBLY_PLAN_MAGIC + + +_FPS = 24 +_SAMPLE_RATE = 240 + + +def _logical_chunk( + sequence_index: int, + *, + total_frames: int, + trim_frames: int, +) -> dict: + return { + "sequence_index": sequence_index, + "chunk_index": sequence_index, + "total_frames": total_frames, + "trim_frames": trim_frames, + "net_frames": total_frames - trim_frames, + "context_frames": trim_frames, + "expected_video_latent_t": video_latent_t(total_frames), + "expected_audio_latent_t": audio_latent_t(total_frames), + } + + +def _physical_group( + logical_chunks: list[int], + *, + total_frames: int, + trim_frames: int, + frame_start: int, + terminal_merged: bool, +) -> dict: + net_frames = total_frames - trim_frames + return { + "logical_chunk_indices": logical_chunks, + "terminal_merged": terminal_merged, + "total_frames": total_frames, + "trim_frames": trim_frames, + "net_frames": net_frames, + "context_frames": trim_frames, + "expected_video_latent_t": video_latent_t(total_frames), + "expected_audio_latent_t": audio_latent_t(total_frames), + "frame_start": frame_start, + "frame_stop": frame_start + net_frames, + } + + +def _plan(kind: str, *, target_frames: int | None = None, preserve=True) -> dict: + if kind == "single": + chunks = [_logical_chunk(1, total_frames=124, trim_frames=0)] + groups = None + default_target = 120 + elif kind == "single_pad": + chunks = [_logical_chunk(1, total_frames=124, trim_frames=5)] + groups = None + default_target = 120 + elif kind == "terminal_2x5": + chunks = [ + _logical_chunk(1, total_frames=124, trim_frames=0), + _logical_chunk(2, total_frames=141, trim_frames=22), + ] + groups = [ + _physical_group( + [1, 2], + total_frames=243, + trim_frames=0, + frame_start=0, + terminal_merged=True, + ) + ] + default_target = 240 + elif kind == "long_terminal_3x5": + chunks = [ + _logical_chunk(1, total_frames=124, trim_frames=0), + _logical_chunk(2, total_frames=141, trim_frames=22), + _logical_chunk(3, total_frames=141, trim_frames=22), + ] + groups = [ + _physical_group( + [1], + total_frames=124, + trim_frames=0, + frame_start=0, + terminal_merged=False, + ), + _physical_group( + [2, 3], + total_frames=260, + trim_frames=22, + frame_start=124, + terminal_merged=True, + ), + ] + default_target = 360 + else: + raise AssertionError(f"unknown fixture kind: {kind}") + + plan = enrich_assembly_plan( + { + "magic": ASSEMBLY_PLAN_MAGIC, + "schema_version": 1, + "fps": _FPS, + "width": 8, + "height": 8, + "chunk_seconds": 5.0, + "target_frames": int( + default_target if target_frames is None else target_frames + ), + "preserve_final_frame": bool(preserve), + "chunks": chunks, + } + ) + if groups is not None: + plan["decode_groups"] = groups + plan["physical_decode_group_count"] = len(groups) + return plan + + +def _units(plan: dict) -> list[dict]: + return list(plan.get("decode_groups") or plan["chunks"]) + + +def _decoded( + plan: dict, + *, + seam_case: str = "ordered", + sample_rate: int = _SAMPLE_RATE, +) -> tuple[list[torch.Tensor], list[dict]]: + images: list[torch.Tensor] = [] + audio: list[dict] = [] + for group_index, unit in enumerate(_units(plan)): + total_frames = int(unit["total_frames"]) + if seam_case == "ordered": + frame_values = ( + group_index * 0.25 + + torch.arange(total_frames, dtype=torch.float32) / 4096.0 + ) + image = frame_values.view(-1, 1, 1, 1).expand(-1, 8, 8, 3).clone() + else: + image = torch.full((total_frames, 8, 8, 3), 0.25) + if group_index == 1: + trim = int(unit["trim_frames"]) + if seam_case == "transient": + image[trim] = 1.0 + elif seam_case == "exposure": + image[trim:] = 0.33 + image[trim : trim + 4] = torch.tensor( + [0.27, 0.29, 0.31, 0.33] + ).view(4, 1, 1, 1) + elif seam_case != "clean": + raise AssertionError(f"unknown seam case: {seam_case}") + + sample_count = int(round(total_frames / _FPS * sample_rate)) + time_values = ( + group_index * 10000 + + torch.arange(sample_count, dtype=torch.float32) + ) + waveform = torch.stack((time_values, -time_values), dim=0).unsqueeze(0) + images.append(image) + audio.append({"waveform": waveform, "sample_rate": sample_rate}) + return images, audio + + +def _v34( + images, + audio, + plan, + *, + exact: bool, + audio_seam: str = AUDIO_SEAM_OFF, + video_seam: str = VIDEO_SEAM_OFF, +): + return H3ContinuumAssembleSeamV34().assemble( + images=images, + audio=audio, + assembly_plan=plan, + exact_total_duration=exact, + audio_seam=audio_seam, + video_seam=video_seam, + diagnostics=DIAGNOSTICS_BASIC, + ) + + +def _v35( + images, + audio, + plan, + *, + exact: bool, + backend: str, + backing_root: Path, + audio_seam: str = AUDIO_SEAM_OFF, + video_seam: str = VIDEO_SEAM_OFF, +): + return assembly_v35.assemble_decoded_chunks_v35( + images=images, + audio=audio, + assembly_plan=plan, + exact_total_duration=exact, + audio_seam=audio_seam, + video_seam=video_seam, + diagnostics=DIAGNOSTICS_BASIC, + buffer_backend=backend, + backing_root=backing_root, + ) + + +def _assert_av_equal(left, right) -> None: + assert torch.equal(left[0], right[0]) + assert left[1]["sample_rate"] == right[1]["sample_rate"] + assert torch.equal(left[1]["waveform"], right[1]["waveform"]) + + +def _collect_released_disk_tensor(root: Path) -> None: + gc.collect() + get_file_backed_image_manager(root).collect_ready() + + +@pytest.mark.parametrize( + ("kind", "exact", "expected_frames"), + ( + ("single", False, 124), + ("single", True, 120), + ("terminal_2x5", False, 243), + ("terminal_2x5", True, 240), + ("long_terminal_3x5", False, 362), + ("long_terminal_3x5", True, 360), + ), +) +def test_ram_and_disk_are_bit_exact_with_v34_required_layouts( + tmp_path, + kind, + exact, + expected_frames, +): + plan = _plan(kind) + images, audio = _decoded(plan) + baseline = _v34(images, audio, plan, exact=exact) + ram = _v35( + images, + audio, + plan, + exact=exact, + backend=BUFFER_BACKEND_RAM, + backing_root=tmp_path, + ) + disk = _v35( + images, + audio, + plan, + exact=exact, + backend=BUFFER_BACKEND_DISK, + backing_root=tmp_path, + ) + + assert int(baseline[0].shape[0]) == expected_frames + _assert_av_equal(ram, baseline) + _assert_av_equal(disk, baseline) + disk_image = disk[0] + del disk + del disk_image + _collect_released_disk_tensor(tmp_path) + + +@pytest.mark.parametrize( + ("video_seam", "seam_case"), + ( + (VIDEO_SEAM_OFF, "transient"), + (VIDEO_SEAM_ANALYZE, "transient"), + (VIDEO_SEAM_AUTO, "transient"), + (VIDEO_SEAM_AUTO_2, "exposure"), + ), +) +@pytest.mark.parametrize("backend", (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK)) +@pytest.mark.parametrize("exact", (False, True)) +def test_all_video_seam_modes_are_bit_exact_with_v34( + tmp_path, + video_seam, + seam_case, + backend, + exact, +): + plan = _plan("long_terminal_3x5") + images, audio = _decoded(plan, seam_case=seam_case) + baseline = _v34( + images, + audio, + plan, + exact=exact, + video_seam=video_seam, + ) + candidate = _v35( + images, + audio, + plan, + exact=exact, + backend=backend, + backing_root=tmp_path, + video_seam=video_seam, + ) + + _assert_av_equal(candidate, baseline) + if backend == BUFFER_BACKEND_DISK: + disk_image = candidate[0] + del candidate + del disk_image + _collect_released_disk_tensor(tmp_path) + + +@pytest.mark.parametrize("backend", (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK)) +@pytest.mark.parametrize("exact", (False, True)) +def test_audio_seam_auto_is_bit_exact_with_v34(tmp_path, backend, exact): + plan = _plan("long_terminal_3x5") + images, audio = _decoded(plan, seam_case="clean") + previous_samples = int(round(124 / _FPS * _SAMPLE_RATE)) + context_samples = int(round(22 / _FPS * _SAMPLE_RATE)) + first_time = torch.arange(previous_samples, dtype=torch.float32) + second_count = int(round(260 / _FPS * _SAMPLE_RATE)) + second_time = torch.arange( + previous_samples - context_samples, + previous_samples - context_samples + second_count, + dtype=torch.float32, + ) + first_wave = torch.sin(first_time * (2.0 * torch.pi / 31.0)) + second_wave = torch.sin(second_time * (2.0 * torch.pi / 31.0)) + audio[0]["waveform"] = first_wave.view(1, 1, -1).expand(1, 2, -1).clone() + audio[1]["waveform"] = second_wave.view(1, 1, -1).expand(1, 2, -1).clone() + baseline = _v34( + images, + audio, + plan, + exact=exact, + audio_seam=AUDIO_SEAM_AUTO, + ) + candidate = _v35( + images, + audio, + plan, + exact=exact, + backend=backend, + backing_root=tmp_path, + audio_seam=AUDIO_SEAM_AUTO, + ) + + _assert_av_equal(candidate, baseline) + assert "audio seam 1->2" in baseline[2] + assert "fade=0 samples" not in baseline[2] + assert "fallback to native boundary" not in baseline[2] + if backend == BUFFER_BACKEND_DISK: + disk_image = candidate[0] + del candidate + del disk_image + _collect_released_disk_tensor(tmp_path) + + +@pytest.mark.parametrize("backend", (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK)) +@pytest.mark.parametrize("exact", (False, True)) +def test_32khz_cumulative_rounding_and_audio_seam_match_v34( + tmp_path, + backend, + exact, +): + plan = _plan("long_terminal_3x5") + images, audio = _decoded(plan, seam_case="clean", sample_rate=32000) + baseline = _v34( + images, + audio, + plan, + exact=exact, + audio_seam=AUDIO_SEAM_AUTO, + ) + candidate = _v35( + images, + audio, + plan, + exact=exact, + backend=backend, + backing_root=tmp_path, + audio_seam=AUDIO_SEAM_AUTO, + ) + + _assert_av_equal(candidate, baseline) + expected_frames = 360 if exact else 362 + assert candidate[1]["waveform"].shape[-1] == int( + round(expected_frames / _FPS * 32000) + ) + if backend == BUFFER_BACKEND_DISK: + disk_image = candidate[0] + del candidate + del disk_image + _collect_released_disk_tensor(tmp_path) + + +@pytest.mark.parametrize("backend", (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK)) +def test_padding_and_final_frame_preservation_match_v34(tmp_path, backend): + trim_plan = _plan("single", target_frames=120, preserve=True) + trim_images, trim_audio = _decoded(trim_plan) + trim_baseline = _v34(trim_images, trim_audio, trim_plan, exact=True) + trim_candidate = _v35( + trim_images, + trim_audio, + trim_plan, + exact=True, + backend=backend, + backing_root=tmp_path / "trim", + ) + _assert_av_equal(trim_candidate, trim_baseline) + assert torch.equal(trim_candidate[0][-1], trim_images[0][-1]) + + pad_plan = _plan("single_pad", preserve=False) + pad_images, pad_audio = _decoded(pad_plan) + pad_baseline = _v34(pad_images, pad_audio, pad_plan, exact=True) + pad_candidate = _v35( + pad_images, + pad_audio, + pad_plan, + exact=True, + backend=backend, + backing_root=tmp_path / "pad", + ) + _assert_av_equal(pad_candidate, pad_baseline) + assert torch.equal(pad_candidate[0][-1], pad_images[0][-1]) + + if backend == BUFFER_BACKEND_DISK: + trim_image = trim_candidate[0] + pad_image = pad_candidate[0] + del trim_candidate, pad_candidate + del trim_image, pad_image + _collect_released_disk_tensor(tmp_path / "trim") + _collect_released_disk_tensor(tmp_path / "pad") + + +@pytest.mark.parametrize("backend", (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK)) +def test_tail_trim_without_final_anchor_matches_v34(tmp_path, backend): + plan = _plan("single", target_frames=120, preserve=False) + images, audio = _decoded(plan) + baseline = _v34(images, audio, plan, exact=True) + candidate = _v35( + images, + audio, + plan, + exact=True, + backend=backend, + backing_root=tmp_path, + ) + + _assert_av_equal(candidate, baseline) + assert torch.equal(candidate[0][-1], images[0][119]) + assert not torch.equal(candidate[0][-1], images[0][-1]) + if backend == BUFFER_BACKEND_DISK: + disk_image = candidate[0] + del candidate + del disk_image + _collect_released_disk_tensor(tmp_path) + + +def test_long_terminal_uses_physical_group_order_and_trim_contract(tmp_path): + plan = _plan("long_terminal_3x5") + images, audio = _decoded(plan) + result = _v35( + images, + audio, + plan, + exact=False, + backend=BUFFER_BACKEND_RAM, + backing_root=tmp_path, + ) + + assert result[0].shape[0] == 362 + assert torch.equal(result[0][:124], images[0]) + assert torch.equal(result[0][124:], images[1][22:260]) + expected_samples = int(round(362 / _FPS * _SAMPLE_RATE)) + assert result[1]["waveform"].shape[-1] == expected_samples + + +def test_disk_storage_alias_keeps_backing_file_alive_until_last_view(tmp_path): + plan = _plan("single") + images, audio = _decoded(plan) + output, _, _ = _v35( + images, + audio, + plan, + exact=True, + backend=BUFFER_BACKEND_DISK, + backing_root=tmp_path, + ) + manager = get_file_backed_image_manager(tmp_path) + managed = list(tmp_path.glob("*.bin")) + assert len(managed) == 1 + assert managed[0].stat().st_size == output.numel() * output.element_size() + + alias = output[1:] + del output + gc.collect() + held = manager.collect_ready() + assert held.reclaimed == 0 + assert held.deferred == 1 + assert managed[0].exists() + assert torch.equal(alias[0], images[0][1]) + + del alias + gc.collect() + released = manager.collect_ready() + assert released.reclaimed == 1 + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) + + +def test_disk_write_error_aborts_unpublished_allocation(tmp_path, monkeypatch): + plan = _plan("single") + images, audio = _decoded(plan) + original = assembly_v35._copy_natural_interval + + def fail_after_allocation(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("synthetic direct-write failure") + + monkeypatch.setattr(assembly_v35, "_copy_natural_interval", fail_after_allocation) + with pytest.raises(RuntimeError, match="synthetic direct-write failure"): + _v35( + images, + audio, + plan, + exact=True, + backend=BUFFER_BACKEND_DISK, + backing_root=tmp_path, + ) + + manager = get_file_backed_image_manager(tmp_path) + assert manager.tracked_count() == 0 + assert manager.pending_count() == 0 + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) + + +def test_invalid_driving_audio_aborts_before_disk_publish(tmp_path, monkeypatch): + plan = _plan("single") + images, audio = _decoded(plan) + original_allocate = assembly_v35.allocate_file_backed_image + + def allocate_at_test_root(shape, *, dtype, backing_root=None): + return original_allocate(shape, dtype=dtype, backing_root=tmp_path) + + monkeypatch.setattr( + assembly_v35, + "allocate_file_backed_image", + allocate_at_test_root, + ) + invalid_driving = { + "waveform": torch.zeros((1, 1, 1240), dtype=torch.float32), + "sample_rate": 0, + } + with pytest.raises(ValueError, match="sample_rate"): + H3ContinuumAssembleSeamV35().assemble( + images, + audio, + plan, + True, + AUDIO_SEAM_OFF, + VIDEO_SEAM_OFF, + BUFFER_BACKEND_DISK, + DIAGNOSTICS_BASIC, + driving_audio=invalid_driving, + ) + + manager = get_file_backed_image_manager(tmp_path) + assert manager.tracked_count() == 0 + assert manager.pending_count() == 0 + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) + + +def test_auto_backend_is_default_while_manual_values_remain_available(): + schema = H3ContinuumAssembleSeamV35.INPUT_TYPES() + backend_options = schema["required"]["buffer_backend"][0] + backend_config = schema["required"]["buffer_backend"][1] + assert tuple(backend_options) == ( + BUFFER_BACKEND_AUTO, + BUFFER_BACKEND_RAM, + BUFFER_BACKEND_DISK, + ) + assert backend_config["default"] == BUFFER_BACKEND_AUTO + + +@pytest.mark.parametrize( + "selected_backend", + (BUFFER_BACKEND_RAM, BUFFER_BACKEND_DISK), +) +def test_auto_backend_uses_selected_writer_and_reports_decision( + tmp_path, + monkeypatch, + selected_backend, +): + gib = 1024**3 + plan = _plan("single") + images, audio = _decoded(plan) + decision = AutoBufferDecision( + selected_backend=selected_backend, + final_image_bytes=120 * 8 * 8 * 3 * 4, + backing_root=tmp_path, + available_memory_bytes=16 * gib, + total_memory_bytes=64 * gib, + memory_reserve_bytes=int(64 * gib * 0.10), + ram_image_limit_bytes=4 * gib, + disk_free_bytes=(100 * gib if selected_backend == BUFFER_BACKEND_DISK else None), + disk_reserve_bytes=2 * gib, + reason="synthetic test decision", + ) + monkeypatch.setattr( + assembly_v35, + "select_auto_buffer_backend", + lambda final_image_bytes, backing_root=None: decision, + ) + + candidate = _v35( + images, + audio, + plan, + exact=True, + backend=BUFFER_BACKEND_AUTO, + backing_root=tmp_path, + ) + baseline = _v35( + images, + audio, + plan, + exact=True, + backend=selected_backend, + backing_root=tmp_path / "manual", + ) + + _assert_av_equal(candidate, baseline) + assert f"Buffer Backend: Auto -> {selected_backend}" in candidate[2] + assert f"selected={selected_backend}" in candidate[2] + assert "reason=synthetic test decision" in candidate[2] + + candidate_image = candidate[0] + baseline_image = baseline[0] + del candidate, baseline, candidate_image, baseline_image + if selected_backend == BUFFER_BACKEND_DISK: + _collect_released_disk_tensor(tmp_path) + _collect_released_disk_tensor(tmp_path / "manual") + + +def test_direct_image_path_has_no_full_materialization_operations(): + source = inspect.getsource(assembly_v35) + for forbidden in ( + "torch.cat(", + "torch.stack(", + ".clone(", + ".contiguous(", + "correct_decoded_boundaries", + ): + assert forbidden not in source + + +def test_small_patch_copy_intersects_prefix_and_final_anchor_spans_only(): + destination = torch.full((5, 1, 1, 1), -1.0) + spans = ( + assembly_v35.CopySpan(0, 4, 0), + assembly_v35.CopySpan(9, 10, 4), + ) + crossing_patch = torch.tensor([30.0, 40.0, 50.0]).reshape(3, 1, 1, 1) + assembly_v35._copy_natural_interval( + destination, + crossing_patch, + natural_start=3, + spans=spans, + ) + anchor_patch = torch.tensor([90.0]).reshape(1, 1, 1, 1) + assembly_v35._copy_natural_interval( + destination, + anchor_patch, + natural_start=9, + spans=spans, + ) + + assert destination[:, 0, 0, 0].tolist() == [-1.0, -1.0, -1.0, 30.0, 90.0] + + +def test_node_accepts_comfy_positional_list_inputs(tmp_path, monkeypatch): + plan = _plan("single") + images, audio = _decoded(plan) + original_allocate = assembly_v35.allocate_file_backed_image + + def allocate_at_test_root(shape, *, dtype, backing_root=None): + return original_allocate(shape, dtype=dtype, backing_root=tmp_path) + + monkeypatch.setattr( + assembly_v35, + "allocate_file_backed_image", + allocate_at_test_root, + ) + node = H3ContinuumAssembleSeamV35() + result = node.assemble( + images, + audio, + [plan], + [True], + [AUDIO_SEAM_OFF], + [VIDEO_SEAM_OFF], + [BUFFER_BACKEND_DISK], + [DIAGNOSTICS_FULL], + ) + baseline = _v34(images, audio, plan, exact=True) + + assert node.INPUT_IS_LIST is True + _assert_av_equal(result, baseline) + assert "Buffer Backend: Disk-backed" in result[2] + assert "Projected assembly buffers [V3.5]" in result[2] + disk_image = result[0] + del result + del disk_image + _collect_released_disk_tensor(tmp_path) + + +@pytest.mark.parametrize("stored_in_plan", (False, True)) +def test_driving_audio_selection_is_bit_exact_with_v34( + tmp_path, + stored_in_plan, +): + plan = _plan("single", preserve=True) + images, audio = _decoded(plan) + driving_samples = int(round(124 / _FPS * _SAMPLE_RATE)) + driving = { + "waveform": torch.arange( + driving_samples, + dtype=torch.float32, + ).view(1, 1, -1), + "sample_rate": _SAMPLE_RATE, + } + direct = None if stored_in_plan else driving + if stored_in_plan: + plan = dict(plan) + plan["_h3_continuum_driving_audio_v1"] = driving + + baseline = H3ContinuumAssembleSeamV34().assemble( + images, + audio, + plan, + True, + AUDIO_SEAM_AUTO, + VIDEO_SEAM_OFF, + DIAGNOSTICS_BASIC, + driving_audio=direct, + ) + candidate = H3ContinuumAssembleSeamV35().assemble( + images, + audio, + plan, + True, + AUDIO_SEAM_AUTO, + VIDEO_SEAM_OFF, + BUFFER_BACKEND_RAM, + DIAGNOSTICS_BASIC, + driving_audio=direct, + ) + + _assert_av_equal(candidate, baseline) + assert "Driving Audio: preserved source selected" in candidate[2] diff --git a/tests/test_p23_video_seam_patches.py b/tests/test_p23_video_seam_patches.py new file mode 100644 index 0000000..4ec9ffb --- /dev/null +++ b/tests/test_p23_video_seam_patches.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from dataclasses import replace + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.hardening import enrich_assembly_plan +from ComfyUI_H3_Continuum_Join.v3.plan import ASSEMBLY_PLAN_MAGIC +from ComfyUI_H3_Continuum_Join.v3.video_seam import ( + analyze_decoded_boundaries, + correct_decoded_boundaries, +) +from ComfyUI_H3_Continuum_Join.v3.video_seam_v35 import ( + build_boundary_patches, +) + + +def _plan(): + return enrich_assembly_plan( + { + "magic": ASSEMBLY_PLAN_MAGIC, + "schema_version": 1, + "fps": 24, + "width": 8, + "height": 8, + "chunk_seconds": 5.0, + "target_frames": 240, + "preserve_final_frame": True, + "chunks": [ + { + "sequence_index": 1, + "chunk_index": 1, + "total_frames": 124, + "trim_frames": 0, + "net_frames": 124, + "context_frames": 0, + "expected_video_latent_t": 37, + "expected_audio_latent_t": 207, + }, + { + "sequence_index": 2, + "chunk_index": 2, + "total_frames": 141, + "trim_frames": 22, + "net_frames": 119, + "context_frames": 22, + "expected_video_latent_t": 42, + "expected_audio_latent_t": 235, + }, + ], + } + ) + + +def _decoded(case: str) -> list[torch.Tensor]: + images = [ + torch.full((124, 8, 8, 3), 0.25, dtype=torch.float32), + torch.full((141, 8, 8, 3), 0.25, dtype=torch.float32), + ] + if case == "transient": + images[1][22] = 1.0 + elif case == "micro": + images[1][22] = 0.258 + elif case == "exposure": + images[1][22:] = 0.33 + images[1][22:26] = torch.tensor( + [0.27, 0.29, 0.31, 0.33], + dtype=torch.float32, + ).view(4, 1, 1, 1) + elif case != "clean": + raise AssertionError(f"unknown test case: {case}") + return images + + +@pytest.mark.parametrize( + ("case", "enable_exposure_ramp", "expected_patch_frames"), + ( + ("transient", False, 1), + ("micro", False, 1), + ("exposure", False, 0), + ("exposure", True, 4), + ("clean", False, 0), + ), +) +def test_small_patches_are_bit_exact_with_v34_correction( + case, + enable_exposure_ramp, + expected_patch_frames, +): + plan = _plan() + images = _decoded(case) + originals = [value.clone() for value in images] + analyses = analyze_decoded_boundaries(images=images, assembly_plan=plan) + + legacy, legacy_actions = correct_decoded_boundaries( + images=images, + assembly_plan=plan, + analyses=analyses, + enable_exposure_ramp=enable_exposure_ramp, + ) + patches, actions = build_boundary_patches( + images, + plan, + analyses, + enable_exposure_ramp=enable_exposure_ramp, + ) + + assert actions == legacy_actions + assert all(torch.equal(value, original) for value, original in zip(images, originals)) + if expected_patch_frames == 0: + assert patches == {} + assert torch.equal(legacy[1], images[1]) + return + + assert set(patches) == {1} + patch = patches[1] + assert patch.device.type == "cpu" + assert patch.shape == (expected_patch_frames, 8, 8, 3) + assert 1 <= int(patch.shape[0]) <= 4 + trim = int(plan["chunks"][1]["trim_frames"]) + assert torch.equal( + patch, + legacy[1][trim : trim + expected_patch_frames], + ) + assert torch.equal(legacy[1][:trim], images[1][:trim]) + assert torch.equal( + legacy[1][trim + expected_patch_frames :], + images[1][trim + expected_patch_frames :], + ) + + +def test_guarded_candidate_returns_native_action_without_a_patch(): + plan = _plan() + images = _decoded("transient") + analysis = analyze_decoded_boundaries( + images=images, + assembly_plan=plan, + )[0] + guarded = replace(analysis, scene_cut=True) + + legacy, legacy_actions = correct_decoded_boundaries( + images=images, + assembly_plan=plan, + analyses=[guarded], + ) + patches, actions = build_boundary_patches(images, plan, [guarded]) + + assert patches == {} + assert actions == legacy_actions == { + 1: "kept native boundary; correction guard rejected candidate" + } + assert legacy == images + + +def test_patch_storage_is_independent_and_inputs_remain_unchanged(): + plan = _plan() + images = _decoded("transient") + original = images[1].clone() + analysis = analyze_decoded_boundaries( + images=images, + assembly_plan=plan, + )[0] + + patches, _ = build_boundary_patches(images, plan, [analysis]) + patches[1].zero_() + + assert torch.equal(images[1], original) + + +def test_planning_error_does_not_mutate_inputs_or_return_a_partial_result(): + plan = _plan() + images = _decoded("transient") + originals = [value.clone() for value in images] + analysis = analyze_decoded_boundaries( + images=images, + assembly_plan=plan, + )[0] + invalid = replace(analysis, boundary_index=2) + + with pytest.raises( + ValueError, + match="boundary index is outside decoded chunks", + ): + build_boundary_patches(images, plan, [analysis, invalid]) + + assert all(torch.equal(value, original) for value, original in zip(images, originals)) diff --git a/tests/test_p24_comfy_integration.py b/tests/test_p24_comfy_integration.py new file mode 100644 index 0000000..7808541 --- /dev/null +++ b/tests/test_p24_comfy_integration.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import gc + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.constants import DIAGNOSTICS_BASIC +from ComfyUI_H3_Continuum_Join.hardening import enrich_assembly_plan +from ComfyUI_H3_Continuum_Join.temporal import audio_latent_t, video_latent_t +from ComfyUI_H3_Continuum_Join.v3.assembly import ( + AUDIO_SEAM_OFF, + VIDEO_SEAM_OFF, +) +from ComfyUI_H3_Continuum_Join.v3.assembly_v35 import ( + assemble_decoded_chunks_v35, +) +from ComfyUI_H3_Continuum_Join.v3.file_backed_buffer import ( + BUFFER_BACKEND_DISK, + BUFFER_BACKEND_RAM, + get_file_backed_image_manager, +) +from ComfyUI_H3_Continuum_Join.v3.plan import ASSEMBLY_PLAN_MAGIC + + +class ProbeCancelled(BaseException): + pass + + +def _fixture(): + total_frames = 124 + target_frames = 120 + sample_rate = 240 + chunk = { + "sequence_index": 1, + "chunk_index": 1, + "total_frames": total_frames, + "trim_frames": 0, + "net_frames": total_frames, + "context_frames": 0, + "expected_video_latent_t": video_latent_t(total_frames), + "expected_audio_latent_t": audio_latent_t(total_frames), + } + plan = enrich_assembly_plan( + { + "magic": ASSEMBLY_PLAN_MAGIC, + "schema_version": 1, + "fps": 24, + "width": 8, + "height": 8, + "chunk_seconds": 5.0, + "target_frames": target_frames, + "preserve_final_frame": True, + "chunks": [chunk], + } + ) + values = torch.arange(total_frames, dtype=torch.float32) / total_frames + images = [values.reshape(-1, 1, 1, 1).expand(-1, 8, 8, 3).clone()] + samples = int(round(total_frames / 24 * sample_rate)) + audio = [ + { + "waveform": torch.linspace(0.0, 1.0, samples).reshape(1, 1, -1), + "sample_rate": sample_rate, + } + ] + return images, audio, plan + + +def _assemble(images, audio, plan, *, backend, root, interrupt_check): + return assemble_decoded_chunks_v35( + images=images, + audio=audio, + assembly_plan=plan, + exact_total_duration=True, + audio_seam=AUDIO_SEAM_OFF, + video_seam=VIDEO_SEAM_OFF, + diagnostics=DIAGNOSTICS_BASIC, + buffer_backend=backend, + backing_root=root, + interrupt_check=interrupt_check, + ) + + +def _collect(root, *values): + del values + gc.collect() + manager = get_file_backed_image_manager(root) + manager.collect_ready() + gc.collect() + manager.collect_ready() + + +def test_interrupt_polling_keeps_ram_and_disk_outputs_bit_exact(tmp_path): + images, audio, plan = _fixture() + checks = 0 + + def check(): + nonlocal checks + checks += 1 + + ram = _assemble( + images, + audio, + plan, + backend=BUFFER_BACKEND_RAM, + root=tmp_path, + interrupt_check=check, + ) + disk = _assemble( + images, + audio, + plan, + backend=BUFFER_BACKEND_DISK, + root=tmp_path, + interrupt_check=check, + ) + + assert checks >= 20 + assert torch.equal(disk[0], ram[0]) + assert torch.equal(disk[1]["waveform"], ram[1]["waveform"]) + assert disk[1]["sample_rate"] == ram[1]["sample_rate"] + + disk_image = disk[0] + del disk + del disk_image + _collect(tmp_path) + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) + + +def test_mid_copy_cancel_aborts_unpublished_disk_pair(tmp_path): + images, audio, plan = _fixture() + checks = 0 + + def cancel_during_copy(): + nonlocal checks + checks += 1 + if checks == 4: + raise ProbeCancelled("P2-4 synthetic ESC") + + with pytest.raises(ProbeCancelled, match="synthetic ESC"): + _assemble( + images, + audio, + plan, + backend=BUFFER_BACKEND_DISK, + root=tmp_path, + interrupt_check=cancel_during_copy, + ) + + manager = get_file_backed_image_manager(tmp_path) + assert checks == 4 + assert manager.tracked_count() == 0 + assert manager.pending_count() == 0 + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) + + +def test_cancel_immediately_after_allocation_aborts_disk_pair(tmp_path): + images, audio, plan = _fixture() + + def cancel_after_allocation(): + raise ProbeCancelled("P2-4 pre-copy ESC") + + with pytest.raises(ProbeCancelled, match="pre-copy ESC"): + _assemble( + images, + audio, + plan, + backend=BUFFER_BACKEND_DISK, + root=tmp_path, + interrupt_check=cancel_after_allocation, + ) + + manager = get_file_backed_image_manager(tmp_path) + assert manager.tracked_count() == 0 + assert manager.pending_count() == 0 + assert not list(tmp_path.glob("*.bin")) + assert not list(tmp_path.glob("*.json")) diff --git a/tests/test_p25_gpu_acceptance_probe.py b/tests/test_p25_gpu_acceptance_probe.py new file mode 100644 index 0000000..81750b6 --- /dev/null +++ b/tests/test_p25_gpu_acceptance_probe.py @@ -0,0 +1,49 @@ +import importlib.util +from pathlib import Path + +import torch + + +def _probe_module(): + path = Path(__file__).resolve().parents[1] / "tools" / "p25_gpu_acceptance_probe.py" + spec = importlib.util.spec_from_file_location("p25_gpu_acceptance_probe_test", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_p25_bounded_finite_check_and_hash_are_deterministic(): + probe = _probe_module() + value = torch.arange(48, dtype=torch.float32).reshape(4, 3, 4) + + assert probe._tensor_is_finite_batched(value, batch=1) is True + assert probe._hash_tensor(value, batch=1) == probe._hash_tensor(value, batch=3) + + invalid = value.clone() + invalid[2, 1, 1] = float("nan") + assert probe._tensor_is_finite_batched(invalid, batch=1) is False + + +def test_p25_stress_prompt_bypasses_vae_and_preserves_assembler_contract(): + probe = _probe_module() + prompt = probe._prompt("Disk-backed", stress=True) + + assert set(prompt) == {"1", "6", "7"} + assert prompt["1"]["class_type"] == "P25SyntheticDecodedStressFixture" + assert prompt["6"]["class_type"] == "H3ContinuumAssembleSeamV35" + assert prompt["6"]["inputs"]["images"] == ["1", 0] + assert prompt["6"]["inputs"]["audio"] == ["1", 1] + assert prompt["6"]["inputs"]["assembly_plan"] == ["1", 2] + assert prompt["6"]["inputs"]["buffer_backend"] == "Disk-backed" + assert prompt["6"]["inputs"]["exact_total_duration"] is True + assert prompt["6"]["inputs"]["video_seam"] == "Auto" + + +def test_p25_process_memory_snapshot_reports_windows_commit_fields(): + probe = _probe_module() + snapshot = probe._process_memory_snapshot() + + assert snapshot["rss"] > 0 + assert snapshot["private"] > 0 + assert snapshot["uss"] > 0 diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 0000000..f71cdb7 --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import configparser +from pathlib import Path +import tomllib + + +ROOT = Path(__file__).resolve().parents[1] +EXPECTED_VERSION = "3.5.0" + + +def test_v35_release_metadata_is_consistent(): + version_source = (ROOT / "version.py").read_text(encoding="utf-8") + pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + metadata = configparser.ConfigParser() + metadata.read(ROOT / "metadata.ini", encoding="utf-8") + + assert f'PACKAGE_VERSION = "{EXPECTED_VERSION}"' in version_source + assert pyproject["project"]["version"] == EXPECTED_VERSION + assert metadata["project"]["version"] == EXPECTED_VERSION + + +def test_v35_public_documents_identify_the_current_release(): + readme = (ROOT / "README.md").read_text(encoding="utf-8") + readme_ja = (ROOT / "README_JA.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + + assert readme.startswith(f"# ComfyUI-H3-Continuum {EXPECTED_VERSION}\n") + assert readme_ja.startswith(f"# ComfyUI-H3-Continuum {EXPECTED_VERSION}\n") + assert f"## {EXPECTED_VERSION}" in changelog diff --git a/tests/test_v217_stable_facade.py b/tests/test_v217_stable_facade.py index 9232abd..5361b5b 100644 --- a/tests/test_v217_stable_facade.py +++ b/tests/test_v217_stable_facade.py @@ -145,4 +145,4 @@ def test_v217_result_pack_expands_legacy_outputs(): def test_v217_has_no_frontend_display_controller(): assert not (ROOT / "web" / "h3_continuum_v2.js").exists() version_source = (ROOT / "version.py").read_text(encoding="utf-8") - assert 'PACKAGE_VERSION = "3.4.0"' in version_source + assert 'PACKAGE_VERSION = "3.5.0"' in version_source diff --git a/tests/test_v35_compact_ui_frontend.py b/tests/test_v35_compact_ui_frontend.py new file mode 100644 index 0000000..8a83653 --- /dev/null +++ b/tests/test_v35_compact_ui_frontend.py @@ -0,0 +1,45 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PROJECT_ID_JS = ROOT / "web" / "project_id.js" + + +def _function_source(source: str, name: str, next_name: str) -> str: + start = source.index(f"function {name}") + end = source.index(f"function {next_name}", start) + return source[start:end] + + +def test_v35_sampler_uses_existing_compact_runtime_settings_path(): + source = PROJECT_ID_JS.read_text(encoding="utf-8") + configure = _function_source(source, "configureNode", "configureNodeAfterSetup") + before_queue = source[source.index("async beforeQueuePrompt") :] + + assert 'const V35_NODE_CLASS = "H3ContinuumSamplerV35";' in source + assert "const isV35 = node.comfyClass === V35_NODE_CLASS;" in configure + assert "!isProduction && !isTimeline && !isV34 && !isV35" in configure + assert "isProduction || isV34 || isV35" in configure + assert "node.comfyClass === V35_NODE_CLASS" in before_queue + + for widget_name in ( + '"diagnostics"', + '"strict_compatibility"', + '"debug"', + '"show_preview"', + ): + assert f"hidePersistentWidget(findWidget(node, {widget_name}))" in configure + + +def test_v35_assembler_uses_existing_compact_runtime_settings_path(): + source = PROJECT_ID_JS.read_text(encoding="utf-8") + configure = _function_source(source, "configureAssembler", "configureNode") + before_queue = source[source.index("async beforeQueuePrompt") :] + + assert ( + 'const V35_ASSEMBLE_SEAM_NODE_CLASS = "H3ContinuumAssembleSeamV35";' + in source + ) + assert "node.comfyClass !== V35_ASSEMBLE_SEAM_NODE_CLASS" in configure + assert "node.comfyClass === V35_ASSEMBLE_SEAM_NODE_CLASS" in before_queue + assert 'hidePersistentWidget(findWidget(node, "diagnostics"))' in configure diff --git a/tests/test_v35_context_sampler.py b/tests/test_v35_context_sampler.py new file mode 100644 index 0000000..de521ad --- /dev/null +++ b/tests/test_v35_context_sampler.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + DIAGNOSTICS_BASIC, + DIAGNOSTICS_FULL, + PROMPT_MODE_FIXED, + PROMPT_MODE_LIST, + V2_CONTINUITY_OPTIONS, +) +from ComfyUI_H3_Continuum_Join.temporal import audio_latent_t, video_latent_t +from ComfyUI_H3_Continuum_Join.v2 import sequence +from ComfyUI_H3_Continuum_Join.v2.h3_builder import IdentityAssets +from ComfyUI_H3_Continuum_Join.v2.prompts import make_prompt_plan +from ComfyUI_H3_Continuum_Join.v3.driving_nodes import ( + H3ContinuumSamplerV34, + H3ContinuumSamplerV35, + NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS, +) +from ComfyUI_H3_Continuum_Join.v3.nodes import H3ContinuumSamplerProduction +from ComfyUI_H3_Continuum_Join.v3.plan import prepare_physical_decode_entries +from ComfyUI_H3_Continuum_Join.v3.refine_context import validate_refine_context + + +class _Nested: + def __init__(self, parts): + self.parts = list(parts) + + def unbind(self): + return self.parts + + +class _Clip: + def tokenize(self, prompt, **_kwargs): + return prompt + + def encode_from_tokens_scheduled(self, _tokens): + return [[torch.zeros((1, 1, 2)), {}]] + + +def _latent(width: int, height: int, frames: int): + return { + "samples": _Nested( + ( + torch.zeros( + (1, 24, video_latent_t(frames), height // 16, width // 16) + ), + torch.zeros((1, 32, 2, audio_latent_t(frames))), + ) + ) + } + + +def _model(): + return SimpleNamespace( + model=SimpleNamespace(diffusion_model=SimpleNamespace()), + model_options={}, + wrappers={}, + model_dtype=lambda: torch.bfloat16, + model_size=lambda: 0, + ) + + +def _capture_sequence( + monkeypatch, + *, + terminal: bool, + session=None, + diagnostics=DIAGNOSTICS_BASIC, + memory_attribution=False, +): + width, height = 96, 64 + first_image = torch.zeros((1, height, width, 3)) if terminal else None + last_image = torch.ones((1, height, width, 3)) if terminal else None + first_latent = torch.zeros((1, 24, 1, height // 16, width // 16)) if terminal else None + last_latent = torch.ones((1, 24, 1, height // 16, width // 16)) if terminal else None + assets = IdentityAssets( + first_image, + first_latent, + last_image, + last_latent, + "identity" if terminal else "none", + ) + monkeypatch.setattr(sequence, "check_comfy_h3_runtime", lambda: []) + monkeypatch.setattr(sequence, "prepare_identity_assets", lambda *a, **k: assets) + monkeypatch.setattr(sequence, "encode_identity_latents", lambda *a, **k: assets) + monkeypatch.setattr(sequence, "empty_h3_latent", _latent) + monkeypatch.setattr( + sequence, + "latent_from_cpu", + lambda video, audio: {"samples": _Nested((video, audio))}, + ) + monkeypatch.setattr(sequence, "accelerator_summary", lambda _model: "accelerators") + monkeypatch.setattr(sequence, "sample_chunk", lambda **kwargs: kwargs["latent"]) + monkeypatch.setattr( + sequence, + "clone_model_for_chunk", + lambda model, **_kwargs: model, + ) + chunks = 3 if terminal else 2 + mode = PROMPT_MODE_LIST if terminal else PROMPT_MODE_FIXED + script = "first\n---\nmiddle\n---\nlast" if terminal else "same prompt" + plan = make_prompt_plan( + mode=mode, + script=script, + chunks=chunks, + chunk_seconds=5.0, + ) + return sequence.run_sequence( + model=_model(), + clip=_Clip(), + video_vae=object(), + audio_vae=None, + sampler=object(), + sigmas=torch.tensor([1.0, 0.0]), + first_frame=first_image, + last_frame=last_image, + prompt_plan=plan, + width=width, + height=height, + continuity=V2_CONTINUITY_OPTIONS[1], + base_seed=42, + audio_continuity=True, + exact_total_duration=False, + diagnostics_mode=diagnostics, + reroll_from_chunk=0, + reroll_nonce=0, + strict_compatibility=False, + debug=False, + session=session, + latent_only=True, + capture_refine_context=True, + memory_attribution=memory_attribution, + ) + + +def test_v34_public_output_schema_is_unchanged(): + assert H3ContinuumSamplerV34.RETURN_TYPES == ( + "LATENT", + "LATENT", + "H3_CONTINUUM_ASSEMBLY_PLAN", + "STRING", + "AUDIO", + ) + assert H3ContinuumSamplerV34.RETURN_NAMES == ( + "video_latents", + "audio_latents", + "assembly_plan", + "status", + "driving_audio", + ) + assert H3ContinuumSamplerV34.OUTPUT_IS_LIST == (True, True, False, False, False) + + +def test_v35_is_registered_with_v34_outputs_then_refine_context(): + from ComfyUI_H3_Continuum_Join import nodes as root_nodes + + assert NODE_CLASS_MAPPINGS["H3ContinuumSamplerV35"] is H3ContinuumSamplerV35 + assert NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumSamplerV35"] == ( + "H3 Continuum Sampler V3.5" + ) + assert root_nodes.NODE_CLASS_MAPPINGS["H3ContinuumSamplerV35"] is ( + H3ContinuumSamplerV35 + ) + assert H3ContinuumSamplerV35.RETURN_TYPES[:5] == H3ContinuumSamplerV34.RETURN_TYPES + assert H3ContinuumSamplerV35.RETURN_NAMES[:5] == H3ContinuumSamplerV34.RETURN_NAMES + assert H3ContinuumSamplerV35.RETURN_TYPES[5] == "H3_CONTINUUM_REFINE_CONTEXT" + assert H3ContinuumSamplerV35.RETURN_NAMES[5] == "refine_context" + assert H3ContinuumSamplerV35.OUTPUT_IS_LIST == ( + True, + True, + False, + False, + False, + False, + ) + + +def test_v35_runtime_keeps_driving_audio_fifth_and_context_sixth(monkeypatch): + captured = {} + + def fake_production_run(_self, **kwargs): + captured["capture"] = kwargs["capture_refine_context"] + captured["memory"] = kwargs.get("memory_attribution", False) + base = ("video", "audio", {"plan": True}, "status") + return (*base, "context") if kwargs["capture_refine_context"] else base + + monkeypatch.setattr(H3ContinuumSamplerProduction, "run", fake_production_run) + v34_outputs = H3ContinuumSamplerV34().run( + chunks=1, + chunk_seconds=5.0, + width=96, + height=64, + ) + assert captured["capture"] is False + assert captured["memory"] is False + assert v34_outputs == ( + "video", + "audio", + {"plan": True}, + "status", + None, + ) + outputs = H3ContinuumSamplerV35().run( + chunks=1, + chunk_seconds=5.0, + width=96, + height=64, + ) + assert captured["capture"] is True + assert captured["memory"] is True + assert outputs == ( + "video", + "audio", + {"plan": True}, + "status", + None, + "context", + ) + + +def test_capture_records_normal_physical_groups_in_sampling_order(monkeypatch): + outputs = _capture_sequence(monkeypatch, terminal=False) + assert len(outputs) == 5 + context = outputs[4] + assert context["complete"] is True + assert context["conditioning_mode"] == "t2va" + groups = context["groups"] + assert [group["logical_chunks"] for group in groups] == [(1,), (2,)] + assert [group["physical_clip_index"] for group in groups] == [1, 2] + assert [group["context_frames"] for group in groups] == [0, 22] + assert [group["prompt_policy"] for group in groups] == ["single", "single"] + assert [group["source_video_shape"] for group in groups] == [ + (1, 24, 37, 4, 6), + (1, 24, 42, 4, 6), + ] + _, assembly_plan = prepare_physical_decode_entries( + outputs[0], + chunk_seconds=5.0, + preserve_final_frame=False, + terminal_merged=False, + ) + assert validate_refine_context(context, assembly_plan) is context + + +def test_capture_records_terminal_pair_as_one_physical_group(monkeypatch): + outputs = _capture_sequence(monkeypatch, terminal=True) + context = outputs[4] + assert context["complete"] is True + assert context["conditioning_mode"] == "fl2va" + groups = context["groups"] + assert len(groups) == 2 + assert groups[0]["logical_chunks"] == (1,) + assert groups[1]["logical_chunks"] == (2, 3) + assert groups[1]["physical_frames"] == 260 + assert groups[1]["physical_clip_index"] == 2 + assert groups[1]["context_frames"] == 22 + assert groups[1]["prompt_policy"] == "paired_timeline_v1" + assert groups[1]["physical_prompt"] == ( + "[0-5s]\nmiddle\n\n[5-10s]\nlast" + ) + assert groups[1]["source_video_shape"] == (1, 24, 77, 4, 6) + assert torch.equal(groups[1]["first_image"], torch.zeros((1, 64, 96, 3))) + assert torch.equal(groups[1]["last_image"], torch.ones((1, 64, 96, 3))) + _, assembly_plan = prepare_physical_decode_entries( + outputs[0], + chunk_seconds=5.0, + preserve_final_frame=True, + terminal_merged=True, + ) + assert validate_refine_context(context, assembly_plan) is context + + +def test_reused_session_returns_incomplete_context_without_stopping(monkeypatch): + first_outputs = _capture_sequence(monkeypatch, terminal=False) + reused_outputs = _capture_sequence( + monkeypatch, + terminal=False, + session=first_outputs[2], + ) + context = reused_outputs[4] + assert context["complete"] is False + assert context["groups"] == () + assert len(reused_outputs[0]) == 2 + assert any("Run Storage" in note for note in context["notes"]) + + +def test_v35_full_memory_events_follow_physical_terminal_group_order(monkeypatch): + outputs = _capture_sequence( + monkeypatch, + terminal=True, + diagnostics=DIAGNOSTICS_FULL, + memory_attribution=True, + ) + memory_lines = [ + line for line in outputs[3].splitlines() if line.startswith("Memory [V3.5") + ] + labels = [line.split("]: RSS", 1)[0] + "]" for line in memory_lines] + assert labels == [ + "Memory [V3.5 sequence start]", + "Memory [V3.5 physical group 1 logical=[1] before sampling]", + "Memory [V3.5 physical group 1 logical=[1] after sampling]", + "Memory [V3.5 physical group 1 logical=[1] after CPU commit]", + "Memory [V3.5 physical group 2 logical=[2,3] before sampling]", + "Memory [V3.5 physical group 2 logical=[2,3] after sampling]", + "Memory [V3.5 physical group 2 logical=[2,3] after CPU commit]", + "Memory [V3.5 sequence complete]", + ] + + +def test_memory_attribution_is_absent_when_private_flag_is_off(monkeypatch): + outputs = _capture_sequence( + monkeypatch, + terminal=False, + diagnostics=DIAGNOSTICS_FULL, + memory_attribution=False, + ) + assert "Memory [V3.5" not in outputs[3] + assert "Memory [sequence start]" in outputs[3] + assert "Memory [sequence complete]" in outputs[3] diff --git a/tests/test_v35_hires_fix.py b/tests/test_v35_hires_fix.py new file mode 100644 index 0000000..8d36879 --- /dev/null +++ b/tests/test_v35_hires_fix.py @@ -0,0 +1,443 @@ +from __future__ import annotations + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.nodes import ( + NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS, +) +from ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes import ( + H3ContinuumHiResFixV35, + resolve_h3_native_latent_target, +) +from ComfyUI_H3_Continuum_Join.v3.second_pass import SecondPassContractError + + +def _video(marker: str, *, height: int = 4, width: int = 4) -> dict: + return { + "samples": torch.zeros((1, 24, 3, height, width)), + "marker": marker, + } + + +def _audio(marker: str) -> dict: + return { + "samples": torch.zeros((1, 8, 12)), + "marker": marker, + } + + +def _plan( + *, + width: int = 64, + height: int = 64, + latent_width: int = 4, + latent_height: int = 4, +) -> dict: + return { + "width": width, + "height": height, + "second_pass_contract": { + "version": 1, + "physical_groups": [ + { + "group_id": 0, + "source_width": width, + "source_height": height, + "source_batch": 1, + "latent_channels": 24, + "source_latent_t": 3, + "source_latent_w": latent_width, + "source_latent_h": latent_height, + } + ] + }, + } + + +def test_disabled_returns_first_pass_objects_without_resize_or_sampling(monkeypatch): + def unexpected_call(*_args, **_kwargs): + raise AssertionError("disabled Hi-res fix must not resize or sample") + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.resize_video_latents_via_vae", + unexpected_call, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.run_second_pass_groups", + unexpected_call, + ) + videos = [_video("video")] + audios = [_audio("audio")] + plan = {"width": 640, "height": 640} + + output_video, output_audio, output_plan, status = H3ContinuumHiResFixV35().apply( + None, + None, + None, + None, + videos, + audios, + [plan], + [False], + ["bislerp"], + [2.0], + [7], + ["must", "not be unwrapped"], + ["must", "not be unwrapped"], + ) + + assert output_video is videos + assert output_audio is audios + assert output_plan is plan + assert "disabled" in status + + +def test_enabled_pixel_vae_roundtrip_then_reuses_second_pass_contract(monkeypatch): + videos = [_video("first-pass", height=40, width=40)] + resized = [_video("reencoded", height=80, width=80)] + audios = [_audio("first-pass")] + plan = _plan(width=640, height=640, latent_width=40, latent_height=40) + sentinel = ( + [_video("refined")], + audios, + {"width": 1280, "height": 1280}, + "sampled", + ) + calls = [] + + def fake_roundtrip(video_latents, assembly_plan, **kwargs): + calls.append(("roundtrip", video_latents, assembly_plan, kwargs)) + return resized + + def fake_second_pass(**kwargs): + calls.append(("second_pass", kwargs)) + return sentinel + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.resize_video_latents_via_vae", + fake_roundtrip, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.run_second_pass_groups", + fake_second_pass, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.h3_core_native_canvas", + lambda *_args, **_kwargs: (768, 768), + ) + + result = H3ContinuumHiResFixV35().apply( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + videos, + audios, + [plan], + [True], + ["lanczos"], + [2.0], + [17], + [{"context": True}], + ["video-vae"], + ) + + assert result[:3] == sentinel[:3] + assert "mode=manual_scale_by" in result[3] + assert "backend=sequential_pixel_vae_roundtrip" in result[3] + assert "target=1280x1280" in result[3] + assert "INFO" in result[3] + assert calls[0] == ( + "roundtrip", + videos, + plan, + { + "video_vae": "video-vae", + "target_width": 1280, + "target_height": 1280, + "upscale_method": "lanczos", + }, + ) + second_pass = calls[1][1] + assert second_pass["model"] == "model" + assert second_pass["clip"] == "clip" + assert second_pass["sampler"] == "sampler" + assert second_pass["video_latents"] is resized + assert second_pass["audio_latents"] is audios + assert second_pass["assembly_plan"] is plan + assert second_pass["refine_seed"] == 17 + assert second_pass["refine_context"] == {"context": True} + assert second_pass["video_vae"] == "video-vae" + assert second_pass["conditioning_upscale_method"] == "bilinear" + + +def test_native_auto_resizes_to_exact_core_target_then_samples(monkeypatch): + videos = [_video("first-pass")] + resized = [_video("native", height=6, width=6)] + audios = [_audio("first-pass")] + plan = _plan() + calls = [] + + def fake_roundtrip(video_latents, assembly_plan, **kwargs): + calls.append(("roundtrip", video_latents, assembly_plan, kwargs)) + return resized + + def fake_second_pass(**kwargs): + calls.append(("second_pass", kwargs)) + return ( + [_video("refined", height=6, width=6)], + audios, + {"width": 96, "height": 96}, + "sampled", + ) + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.h3_core_native_canvas", + lambda *_args, **_kwargs: (96, 96), + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.resize_video_latents_via_vae", + fake_roundtrip, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.run_second_pass_groups", + fake_second_pass, + ) + + result = H3ContinuumHiResFixV35().apply( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + videos, + audios, + [plan], + [True], + ["bilinear"], + [0.0], + [17], + None, + ["video-vae"], + ) + + assert calls[0] == ( + "roundtrip", + videos, + plan, + { + "video_vae": "video-vae", + "upscale_method": "bilinear", + "target_width": 96, + "target_height": 96, + }, + ) + assert calls[1][1]["video_latents"] is resized + assert calls[1][1]["conditioning_upscale_method"] == "bilinear" + assert result[2]["width"] == 96 + assert "mode=h3_native_canvas_auto" in result[3] + assert "source=64x64" in result[3] + assert "target=96x96" in result[3] + assert "WARNING" not in result[3] + + +def test_native_target_uses_core_canvas_and_never_shrinks_existing_latent(): + source = [_video("source", height=36, width=36)] + plan = _plan(width=576, height=576, latent_width=36, latent_height=36) + + native = resolve_h3_native_latent_target( + source, + plan, + adapt_canvas_fn=lambda _width, _height: (768, 768), + ) + + assert native["target_latent_width"] == 48 + assert native["target_latent_height"] == 48 + assert native["target_width"] == 768 + assert native["target_height"] == 768 + assert native["preserved_current"] is False + + oversized = [_video("oversized", height=64, width=64)] + preserved = resolve_h3_native_latent_target( + oversized, + plan, + adapt_canvas_fn=lambda _width, _height: (768, 768), + ) + + assert preserved["target_latent_width"] == 64 + assert preserved["target_latent_height"] == 64 + assert preserved["target_width"] == 1024 + assert preserved["target_height"] == 1024 + assert preserved["preserved_current"] is True + + +def test_native_target_uses_exact_core_landscape_geometry(): + source = [_video("landscape", height=36, width=64)] + plan = _plan(width=1024, height=576, latent_width=64, latent_height=36) + + native = resolve_h3_native_latent_target( + source, + plan, + adapt_canvas_fn=lambda _width, _height: (1344, 768), + ) + + assert native["target_latent_width"] == 84 + assert native["target_latent_height"] == 48 + assert native["target_width"] == 1344 + assert native["target_height"] == 768 + + +def test_enabled_requires_video_vae_without_legacy_latent_fallback(monkeypatch): + def unexpected_call(*_args, **_kwargs): + raise AssertionError("missing VAE must fail before resize or sampling") + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.resize_video_latents_via_vae", + unexpected_call, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.run_second_pass_groups", + unexpected_call, + ) + + with pytest.raises(SecondPassContractError, match="requires video_vae"): + H3ContinuumHiResFixV35().apply( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + [_video("first-pass")], + [_audio("first-pass")], + [_plan()], + [True], + ["lanczos"], + [2.0], + [17], + ) + + +def test_manual_scale_below_one_rejects_before_vae_or_sampling(monkeypatch): + def unexpected_call(*_args, **_kwargs): + raise AssertionError("invalid manual scale must fail before expensive work") + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.resize_video_latents_via_vae", + unexpected_call, + ) + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.hires_fix_nodes.run_second_pass_groups", + unexpected_call, + ) + + with pytest.raises(SecondPassContractError, match="at least 1.0"): + H3ContinuumHiResFixV35().apply( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + [_video("first-pass")], + [_audio("first-pass")], + [_plan()], + [True], + ["lanczos"], + [0.5], + [17], + None, + ["video-vae"], + ) + + +def test_lazy_inputs_are_requested_only_when_enabled(): + node = H3ContinuumHiResFixV35() + + assert node.check_lazy_status([False]) == [] + assert node.check_lazy_status( + [False], + refine_context=[None], + video_vae=[None], + ) == [] + assert node.check_lazy_status([True]) == [ + "model", + "clip", + "sampler", + "sigmas", + ] + assert node.check_lazy_status( + [True], + model=(None,), + clip=(None,), + sampler=(None,), + sigmas=(None,), + ) == ["model", "clip", "sampler", "sigmas"] + assert node.check_lazy_status( + [True], + model=["model"], + clip=["clip"], + sampler=["sampler"], + sigmas=["sigmas"], + ) == [] + assert node.check_lazy_status( + [True], + model=["model"], + clip=["clip"], + sampler=["sampler"], + sigmas=["sigmas"], + refine_context=[None], + video_vae=[None], + ) == ["refine_context", "video_vae"] + + +def test_three_v35_nodes_are_registered_with_final_display_names(): + assert NODE_CLASS_MAPPINGS["H3ContinuumHiResFixV35"] is H3ContinuumHiResFixV35 + assert NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumHiResFixV35"] == ( + "H3 Continuum Hi-Res Fix V3.5" + ) + assert NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumSecondPassV35"] == ( + "H3 Continuum Second Pass V3.5" + ) + assert NODE_DISPLAY_NAME_MAPPINGS["H3ContinuumLatentResizeV35"] == ( + "H3 Continuum Latent Resize V3.5" + ) + assert H3ContinuumHiResFixV35.INPUT_IS_LIST is True + assert H3ContinuumHiResFixV35.OUTPUT_IS_LIST == (True, True, False, False) + + +def test_hires_fix_schema_exposes_one_node_controls_and_lazy_sampling_inputs(): + schema = H3ContinuumHiResFixV35.INPUT_TYPES() + required = schema["required"] + + assert required["enabled"][1]["default"] is True + assert required["upscale_method"][0][0] == "lanczos" + assert required["upscale_method"][1]["default"] == "lanczos" + assert required["scale_by"][1] == { + "default": 2.0, + "min": 0.0, + "max": 4.0, + "step": 0.01, + "tooltip": ( + "2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the " + "ComfyUI Core H3 native canvas automatically. Manual " + "values follow the existing Second Pass contract and must " + "be 1.0 or greater." + ), + } + assert all(required[name][1]["lazy"] is True for name in ( + "model", + "clip", + "sampler", + "sigmas", + )) + assert schema["optional"] == { + "refine_context": ("H3_CONTINUUM_REFINE_CONTEXT", {"lazy": True}), + "video_vae": ( + "VAE", + { + "lazy": True, + "tooltip": ( + "Required while enabled for the safe Pixel/VAE roundtrip; " + "not evaluated while disabled." + ), + }, + ), + } diff --git a/tests/test_v35_latent_resize.py b/tests/test_v35_latent_resize.py new file mode 100644 index 0000000..8cac70a --- /dev/null +++ b/tests/test_v35_latent_resize.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from ComfyUI_H3_Continuum_Join.nodes import NODE_CLASS_MAPPINGS +from ComfyUI_H3_Continuum_Join.v3.latent_resize_nodes import ( + H3ContinuumLatentResizeV35, + UPSCALE_METHODS, + resize_video_latents, + resize_video_latents_to_size, +) + + +def _latent(shape: tuple[int, ...], *, marker: str) -> dict: + return { + "samples": torch.arange(torch.tensor(shape).prod().item(), dtype=torch.float32).reshape(shape), + "marker": marker, + } + + +def _test_upscale(samples, width, height, upscale_method, crop): + assert crop == "disabled" + original = samples.shape + flattened = samples.movedim(2, 1).reshape( + -1, + original[1], + original[-2], + original[-1], + ) + mode = "bilinear" if upscale_method == "bislerp" else upscale_method + output = F.interpolate(flattened, size=(height, width), mode=mode) + return output.reshape((original[0], -1, original[1], height, width)).movedim( + 2, + 1, + ) + + +def test_resize_preserves_physical_groups_bct_order_and_metadata() -> None: + inputs = [ + _latent((1, 24, 37, 4, 5), marker="group-1"), + _latent((1, 24, 77, 4, 5), marker="group-2"), + ] + + outputs = resize_video_latents( + inputs, + upscale_method="bilinear", + scale_by=2.0, + upscale_fn=_test_upscale, + ) + + assert [item["samples"].shape for item in outputs] == [ + (1, 24, 37, 8, 10), + (1, 24, 77, 8, 10), + ] + assert [item["marker"] for item in outputs] == ["group-1", "group-2"] + assert all(after is not before for before, after in zip(inputs, outputs, strict=True)) + assert [item["samples"].shape for item in inputs] == [ + (1, 24, 37, 4, 5), + (1, 24, 77, 4, 5), + ] + + +def test_resize_delegates_dimensions_method_and_crop_to_core_boundary() -> None: + latent = _latent((1, 24, 3, 4, 5), marker="core") + calls = [] + + def capture(samples, width, height, upscale_method, crop): + calls.append((samples, width, height, upscale_method, crop)) + return _test_upscale(samples, width, height, upscale_method, crop) + + output = resize_video_latents( + [latent], + upscale_method="bicubic", + scale_by=1.5, + upscale_fn=capture, + )[0] + + assert calls == [(latent["samples"], 8, 6, "bicubic", "disabled")] + assert output["samples"].shape == (1, 24, 3, 6, 8) + + +def test_resize_to_exact_latent_geometry_preserves_groups_and_metadata() -> None: + inputs = [ + _latent((1, 24, 37, 4, 5), marker="group-1"), + _latent((1, 24, 77, 4, 5), marker="group-2"), + ] + + outputs = resize_video_latents_to_size( + inputs, + upscale_method="bilinear", + target_width=9, + target_height=7, + upscale_fn=_test_upscale, + ) + + assert [item["samples"].shape for item in outputs] == [ + (1, 24, 37, 7, 9), + (1, 24, 77, 7, 9), + ] + assert [item["marker"] for item in outputs] == ["group-1", "group-2"] + + +def test_bislerp_preserves_dtype_and_returns_finite_video_latent() -> None: + latent = {"samples": torch.randn((1, 24, 2, 3, 3), dtype=torch.float32)} + output = resize_video_latents( + [latent], + upscale_method="bislerp", + scale_by=2.0, + upscale_fn=_test_upscale, + )[0]["samples"] + + assert output.shape == (1, 24, 2, 6, 6) + assert output.dtype == latent["samples"].dtype + assert torch.isfinite(output).all() + + +def test_latent_resize_node_is_registered_and_core_compatible() -> None: + node_class = NODE_CLASS_MAPPINGS["H3ContinuumLatentResizeV35"] + assert node_class is H3ContinuumLatentResizeV35 + assert node_class.DEPRECATED is False + assert node_class.INPUT_IS_LIST is True + assert node_class.OUTPUT_IS_LIST == (True,) + assert list(UPSCALE_METHODS) == [ + "nearest-exact", + "bilinear", + "area", + "bicubic", + "bislerp", + ] + + +def test_node_unwraps_widget_lists_but_preserves_physical_latent_list(monkeypatch) -> None: + monkeypatch.setitem( + H3ContinuumLatentResizeV35.resize.__globals__, + "resize_video_latents", + lambda video_latents, **kwargs: resize_video_latents( + video_latents, + upscale_fn=_test_upscale, + **kwargs, + ), + ) + inputs = [_latent((1, 24, 2, 2, 2), marker="one")] + (outputs,) = H3ContinuumLatentResizeV35().resize( + inputs, + ["nearest-exact"], + [2.0], + ) + + assert len(outputs) == 1 + assert outputs[0]["samples"].shape == (1, 24, 2, 4, 4) + assert outputs[0]["marker"] == "one" diff --git a/tests/test_v35_pixel_vae_resize.py b/tests/test_v35_pixel_vae_resize.py new file mode 100644 index 0000000..8e2d74f --- /dev/null +++ b/tests/test_v35_pixel_vae_resize.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import pytest +import torch +import torch.nn.functional as F + +from ComfyUI_H3_Continuum_Join.v3.pixel_vae_resize import ( + PIXEL_UPSCALE_METHODS, + PixelVaeResizeError, + resolve_pixel_latent_scale, + resize_video_latents_via_vae, +) + + +def _plan( + *, + temporals=(3, 5), + batch=1, + scale_width=6, + scale_height=4, +) -> dict: + latent_width = 3 + latent_height = 2 + width = latent_width * scale_width + height = latent_height * scale_height + return { + "width": width, + "height": height, + "second_pass_contract": { + "version": 1, + "physical_groups": [ + { + "group_id": position, + "source_width": width, + "source_height": height, + "source_batch": batch, + "latent_channels": 24, + "source_latent_t": temporal, + "source_latent_h": latent_height, + "source_latent_w": latent_width, + } + for position, temporal in enumerate(temporals) + ], + }, + } + + +def _latents(temporals=(3, 5), *, batch=1) -> list[dict]: + return [ + { + "samples": torch.full( + (batch, 24, temporal, 2, 3), + float(position + 1), + ), + "marker": f"group-{position}", + } + for position, temporal in enumerate(temporals) + ] + + +class _RoundTripVae: + def __init__(self, log, *, scale_width=6, scale_height=4): + self.log = log + self.scale_width = scale_width + self.scale_height = scale_height + self.pending_temporal = None + self.pending_batches = 0 + + def decode(self, samples): + temporal = int(samples.shape[2]) + assert self.pending_temporal is None + assert self.pending_batches == 0 + self.pending_temporal = temporal + self.pending_batches = int(samples.shape[0]) + self.log.append(("decode", temporal, tuple(samples.shape))) + frames = temporal * 2 + 1 + return torch.full( + (int(samples.shape[0]), frames, 8, 18, 3), + float(temporal), + ) + + def encode(self, pixels): + temporal = self.pending_temporal + assert temporal is not None + assert pixels.ndim == 4 + assert self.pending_batches > 0 + self.log.append(("encode", temporal, tuple(pixels.shape))) + self.pending_batches -= 1 + if self.pending_batches == 0: + self.pending_temporal = None + return torch.full( + ( + 1, + 24, + temporal, + int(pixels.shape[1]) // self.scale_height, + int(pixels.shape[2]) // self.scale_width, + ), + float(temporal), + ) + + +def test_plan_derives_non_hardcoded_pixel_latent_scale() -> None: + assert resolve_pixel_latent_scale( + _plan(scale_width=6, scale_height=4) + ) == (6, 4) + + +def test_sequential_roundtrip_preserves_groups_bct_metadata_and_input() -> None: + log = [] + vae = _RoundTripVae(log) + inputs = _latents() + original_samples = [item["samples"] for item in inputs] + + def upscale(samples, width, height, method, crop): + log.append(("upscale", tuple(samples.shape), width, height, method, crop)) + assert samples.ndim == 4 + return F.interpolate(samples, size=(height, width), mode="bilinear") + + outputs = resize_video_latents_via_vae( + inputs, + _plan(), + video_vae=vae, + target_width=36, + target_height=16, + upscale_method="lanczos", + upscale_fn=upscale, + ) + + assert [tuple(item["samples"].shape) for item in outputs] == [ + (1, 24, 3, 4, 6), + (1, 24, 5, 4, 6), + ] + assert [item["marker"] for item in outputs] == ["group-0", "group-1"] + assert all( + output is not source + for output, source in zip(outputs, inputs, strict=True) + ) + assert all( + item["samples"] is original + for item, original in zip(inputs, original_samples, strict=True) + ) + assert all(output["samples"].device.type == "cpu" for output in outputs) + assert all(torch.isfinite(output["samples"]).all() for output in outputs) + assert log == [ + ("decode", 3, (1, 24, 3, 2, 3)), + ("upscale", (7, 3, 8, 18), 36, 16, "lanczos", "disabled"), + ("encode", 3, (7, 16, 36, 3)), + ("decode", 5, (1, 24, 5, 2, 3)), + ("upscale", (8, 3, 8, 18), 36, 16, "lanczos", "disabled"), + ("upscale", (3, 3, 8, 18), 36, 16, "lanczos", "disabled"), + ("encode", 5, (11, 16, 36, 3)), + ] + + +def test_core_4d_encode_boundary_preserves_continuum_batch() -> None: + log = [] + vae = _RoundTripVae(log) + + outputs = resize_video_latents_via_vae( + _latents((3,), batch=2), + _plan(temporals=(3,), batch=2), + video_vae=vae, + target_width=36, + target_height=16, + upscale_method="lanczos", + upscale_fn=lambda samples, width, height, *_args: F.interpolate( + samples, size=(height, width), mode="bilinear" + ), + ) + + assert tuple(outputs[0]["samples"].shape) == (2, 24, 3, 4, 6) + assert [entry for entry in log if entry[0] == "encode"] == [ + ("encode", 3, (7, 16, 36, 3)), + ("encode", 3, (7, 16, 36, 3)), + ] + + +def test_each_core_encode_batch_is_validated_before_concat() -> None: + class BadSecondBatchVae(_RoundTripVae): + def __init__(self): + super().__init__([]) + self.encode_calls = 0 + + def encode(self, pixels): + encoded = super().encode(pixels) + self.encode_calls += 1 + if self.encode_calls == 2: + return torch.zeros((1, 24, 4, 4, 6)) + return encoded + + with pytest.raises(PixelVaeResizeError, match="for batch 1"): + resize_video_latents_via_vae( + _latents((3,), batch=2), + _plan(temporals=(3,), batch=2), + video_vae=BadSecondBatchVae(), + target_width=36, + target_height=16, + upscale_fn=lambda samples, width, height, *_args: F.interpolate( + samples, size=(height, width), mode="bilinear" + ), + ) + + +def test_lanczos_is_a_supported_pixel_method() -> None: + assert PIXEL_UPSCALE_METHODS == ( + "nearest-exact", + "bilinear", + "area", + "bicubic", + "bislerp", + "lanczos", + ) + + +def test_decode_requires_core_video_image_layout() -> None: + class BadDecode: + def decode(self, _samples): + return torch.zeros((1, 8, 18, 3)) + + def encode(self, _pixels): + raise AssertionError("encode must not run") + + with pytest.raises(PixelVaeResizeError, match=r"\[B,F,H,W,C\]"): + resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=BadDecode(), + target_width=36, + target_height=16, + upscale_fn=lambda *args: args[0], + ) + + +def test_encode_rejects_changed_bct_or_target_geometry() -> None: + class BadEncode(_RoundTripVae): + def encode(self, pixels): + temporal = self.pending_temporal + assert temporal is not None + return torch.zeros((1, 24, temporal + 1, 4, 6)) + + with pytest.raises(PixelVaeResizeError, match="changed B/C/T"): + resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=BadEncode([]), + target_width=36, + target_height=16, + upscale_fn=lambda samples, width, height, *_args: F.interpolate( + samples, size=(height, width), mode="bilinear" + ), + ) + + +def test_target_geometry_uses_plan_ratio_and_must_be_aligned() -> None: + class MustNotRun: + def decode(self, _samples): + raise AssertionError("decode must not run") + + def encode(self, _pixels): + raise AssertionError("encode must not run") + + with pytest.raises(PixelVaeResizeError, match="plan-derived latent scale"): + resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=MustNotRun(), + target_width=35, + target_height=16, + upscale_fn=lambda *args: args[0], + ) + + +def test_manual_target_is_not_independently_restricted_to_upscale_only() -> None: + outputs = resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=_RoundTripVae([]), + target_width=12, + target_height=4, + upscale_method="bilinear", + upscale_fn=lambda samples, width, height, *_args: F.interpolate( + samples, size=(height, width), mode="bilinear" + ), + ) + + assert tuple(outputs[0]["samples"].shape) == (1, 24, 3, 1, 2) + + +def test_vae_failure_propagates_without_latent_fallback() -> None: + sentinel = RuntimeError("VAE encode sentinel") + + class FailingVae(_RoundTripVae): + def encode(self, _pixels): + raise sentinel + + with pytest.raises(RuntimeError) as captured: + resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=FailingVae([]), + target_width=36, + target_height=16, + upscale_fn=lambda samples, width, height, *_args: F.interpolate( + samples, size=(height, width), mode="bilinear" + ), + ) + + assert captured.value is sentinel + + +@pytest.mark.parametrize("stage", ["decode", "resize", "encode"]) +def test_nonfinite_values_are_rejected_at_each_stage(stage) -> None: + class NonFiniteVae(_RoundTripVae): + def decode(self, samples): + pixels = super().decode(samples) + if stage == "decode": + pixels[0, 0, 0, 0, 0] = torch.nan + return pixels + + def encode(self, pixels): + encoded = super().encode(pixels) + if stage == "encode": + encoded[0, 0, 0, 0, 0] = torch.inf + return encoded + + def upscale(samples, width, height, *_args): + output = F.interpolate(samples, size=(height, width), mode="bilinear") + if stage == "resize": + output[0, 0, 0, 0] = torch.nan + return output + + with pytest.raises(PixelVaeResizeError, match="NaN or Inf"): + resize_video_latents_via_vae( + _latents((3,)), + _plan(temporals=(3,)), + video_vae=NonFiniteVae([]), + target_width=36, + target_height=16, + upscale_fn=upscale, + ) diff --git a/tests/test_v35_refine_context.py b/tests/test_v35_refine_context.py new file mode 100644 index 0000000..7ff1a6d --- /dev/null +++ b/tests/test_v35_refine_context.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from collections.abc import Mapping + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.constants import MARK_VIDEO_CONTEXT +from ComfyUI_H3_Continuum_Join.v3 import refine_context as rc + + +def _upscale(samples, width, height, method, crop): + del method, crop + return torch.nn.functional.interpolate( + samples, size=(height, width), mode="nearest" + ) + + +def _conditioning(): + text = torch.arange(6, dtype=torch.float32, device="cpu").reshape(1, 2, 3) + first = torch.full((1, 24, 1, 2, 3), 1.0) + last = torch.full((1, 24, 1, 2, 3), 2.0) + ordinary_video = torch.full((1, 24, 2, 4, 5), 3.0) + ordinary_audio = torch.full((1, 32, 2, 7), 4.0) + context_video = torch.arange(1 * 24 * 2 * 2 * 3, dtype=torch.float32).reshape( + 1, 24, 2, 2, 3 + ) + context_audio = torch.full((1, 32, 2, 9), 5.0) + return [ + [ + text, + { + "minimax_token_tags": torch.tensor([1, 0]), + "minimax_frame_count": 124, + "minimax_keyframes": [ + {"resolved_frame_index": 0, "latent": first}, + {"resolved_frame_index": 123, "latent": last}, + ], + "minimax_refs": [ + { + "kind": "video_audio", + "latent_t": 2, + "latent_h": 4, + "latent_w": 5, + "latent": ordinary_video, + "ref_audio_t": 7, + "audio_latent": ordinary_audio, + }, + { + "kind": "video_audio", + "latent_t": 2, + "latent_h": 2, + "latent_w": 3, + "latent": context_video, + "ref_audio_t": 9, + "audio_latent": context_audio, + MARK_VIDEO_CONTEXT: True, + }, + ], + }, + ] + ] + + +def _group(**overrides): + values = { + "conditioning": _conditioning(), + "group_id": 0, + "logical_chunks": [1], + "physical_frames": 124, + "prompt_policy": "single", + "physical_prompt": "prompt", + "source_video_shape": (1, 24, 37, 2, 3), + "physical_clip_index": 1, + "context_frames": 0, + "first_image": torch.ones((1, 32, 48, 3)), + "last_image": torch.zeros((1, 32, 48, 3)), + } + values.update(overrides) + return rc.make_refine_group(**values) + + +def test_freeze_conditioning_owns_cpu_tensors_and_immutable_containers(): + source = _conditioning() + frozen = rc.freeze_conditioning(source) + assert isinstance(frozen, tuple) + assert isinstance(frozen[0], tuple) + assert isinstance(frozen[0][1], Mapping) + assert isinstance(frozen[0][1]["minimax_refs"], tuple) + assert frozen[0][0].device.type == "cpu" + assert frozen[0][0] is not source[0][0] + assert frozen[0][0].data_ptr() != source[0][0].data_ptr() + source[0][0].fill_(99) + source[0][1]["minimax_refs"][0]["audio_latent"].fill_(99) + assert not bool((frozen[0][0] == 99).any()) + assert not bool( + (frozen[0][1]["minimax_refs"][0]["audio_latent"] == 99).any() + ) + with pytest.raises(TypeError): + frozen[0][1]["new"] = True + + +def test_adapt_resizes_keyframes_and_only_marked_video_context(monkeypatch): + monkeypatch.setattr(rc, "_common_upscale", _upscale) + group = _group(first_image=None, last_image=None) + frozen_meta = group["conditioning"][0][1] + ordinary_ref = frozen_meta["minimax_refs"][0] + context_ref = frozen_meta["minimax_refs"][1] + ordinary_video = ordinary_ref["latent"] + ordinary_audio = ordinary_ref["audio_latent"] + context_audio = context_ref["audio_latent"] + + adapted, stats = rc.adapt_group_conditioning(group, 4, 6) + metadata = adapted[0][1] + assert adapted[0][0] is group["conditioning"][0][0] + assert [tuple(item["latent"].shape[-2:]) for item in metadata["minimax_keyframes"]] == [ + (4, 6), + (4, 6), + ] + assert metadata["minimax_refs"][0] is ordinary_ref + assert metadata["minimax_refs"][0]["latent"] is ordinary_video + assert metadata["minimax_refs"][0]["audio_latent"] is ordinary_audio + assert tuple(metadata["minimax_refs"][1]["latent"].shape[-2:]) == (4, 6) + expected_context = torch.nn.functional.interpolate( + context_ref["latent"].permute(0, 2, 1, 3, 4).reshape(2, 24, 2, 3), + size=(4, 6), + mode="nearest", + ).reshape(1, 2, 24, 4, 6).permute(0, 2, 1, 3, 4) + assert torch.equal(metadata["minimax_refs"][1]["latent"], expected_context) + assert metadata["minimax_refs"][1]["latent_h"] == 4 + assert metadata["minimax_refs"][1]["latent_w"] == 6 + assert metadata["minimax_refs"][1]["audio_latent"] is context_audio + assert context_ref["latent_h"] == 2 + assert context_ref["latent_w"] == 3 + assert tuple(context_ref["latent"].shape[-2:]) == (2, 3) + assert stats["keyframes_latent_fallback"] == 2 + assert stats["context_refs_resized"] == 1 + assert len(stats["warnings"]) == 2 + + +class _FakeVAE: + def __init__(self): + self.inputs = [] + + def encode(self, image): + self.inputs.append(image.clone()) + batch, height, width, _ = image.shape + return torch.full((batch, 24, 1, height // 16, width // 16), len(self.inputs)) + + +def test_keyframes_reencode_images_with_core_first_last_crop_rules(monkeypatch): + calls = [] + + def upscale(samples, width, height, method, crop): + calls.append((width, height, method, crop)) + return torch.nn.functional.interpolate(samples, size=(height, width), mode="nearest") + + monkeypatch.setattr(rc, "_common_upscale", upscale) + vae = _FakeVAE() + adapted, stats = rc.adapt_group_conditioning(_group(), 4, 6, video_vae=vae) + keyframes = adapted[0][1]["minimax_keyframes"] + assert torch.all(keyframes[0]["latent"] == 1) + assert torch.all(keyframes[1]["latent"] == 2) + assert calls[:2] == [ + (96, 64, "lanczos", "disabled"), + (96, 64, "lanczos", "center"), + ] + assert stats["keyframes_reencoded"] == 2 + assert stats["keyframes_latent_fallback"] == 0 + assert stats["warnings"] == () + + +def test_context_validation_matches_physical_group_assembly_contract(): + group = _group() + context = rc.make_refine_context( + [group], + source_width=48, + source_height=32, + conditioning_mode="fl2va", + ) + plan = { + "width": 48, + "height": 32, + "second_pass_contract": { + "physical_groups": [ + { + "group_id": 0, + "logical_chunks": [1], + "physical_frames": 124, + "prompt_policy": "single", + "physical_prompt": "prompt", + "trim_prefix_frames": 0, + "source_batch": 1, + "latent_channels": 24, + "source_latent_t": 37, + "source_latent_h": 2, + "source_latent_w": 3, + } + ] + }, + } + assert rc.validate_refine_context(context, plan) is context + assert context["groups"] == (group,) + assert context["complete"] is True + bad = {**plan, "width": 64} + with pytest.raises(rc.RefineContextError, match="dimensions differ"): + rc.validate_refine_context(context, bad) + + +def test_incomplete_context_may_be_empty_but_complete_context_may_not(): + context = rc.make_refine_context( + [], + source_width=48, + source_height=32, + conditioning_mode="t2va", + complete=False, + notes=("Run Storage reused groups were not recaptured",), + ) + assert context["groups"] == () + assert context["notes"] + partial = rc.make_refine_context( + [ + _group( + logical_chunks=[3], + physical_clip_index=3, + ) + ], + source_width=48, + source_height=32, + conditioning_mode="t2va", + complete=False, + ) + # Partial ids are capture-local. They must not be paired by index with a + # plan group, because an earlier Run Storage group may be absent. + plan = { + "width": 48, + "height": 32, + "second_pass_contract": {"physical_groups": [{"logical_chunks": [1]}]}, + } + assert rc.validate_refine_context(partial, plan) is partial + with pytest.raises(rc.RefineContextError, match="no physical groups"): + rc.make_refine_context( + [], + source_width=48, + source_height=32, + conditioning_mode="t2va", + complete=True, + ) diff --git a/tests/test_v35_refine_sampling.py b/tests/test_v35_refine_sampling.py new file mode 100644 index 0000000..b4bf645 --- /dev/null +++ b/tests/test_v35_refine_sampling.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import torch + +from ComfyUI_H3_Continuum_Join.v3 import refine_sampling + + +class _Nested: + is_nested = True + + def __init__(self, tensors): + self.tensors = list(tensors) + + def unbind(self): + return self.tensors + + def to(self, *_args, **_kwargs): + return self + + +def test_refine_noise_is_random_for_video_zero_for_audio_with_minimal_masks(): + video = torch.zeros((1, 24, 3, 4, 5), dtype=torch.float16) + audio = torch.ones((1, 32, 2, 12), dtype=torch.float16) + calls = [] + + def prepare_noise(samples, seed, batch_inds): + calls.append((samples, seed, batch_inds)) + return torch.full_like(samples, 0.75, device="cpu") + + noise, mask = refine_sampling._prepare_refine_noise_and_mask( + video, + audio, + seed=123, + batch_inds=[0], + prepare_noise_fn=prepare_noise, + nested_builder=_Nested, + ) + + video_noise, audio_noise = noise.unbind() + video_mask, audio_mask = mask.unbind() + assert calls == [(video, 123, [0])] + assert torch.equal(video_noise, torch.full_like(video_noise, 0.75)) + assert torch.count_nonzero(audio_noise) == 0 + assert audio_noise.dtype == audio.dtype + assert tuple(video_mask.shape) == (1, 1, 1, 1, 1) + assert tuple(audio_mask.shape) == (1, 1, 1, 1) + assert torch.all(video_mask == 1) + assert torch.all(audio_mask == 0) + + +def test_refine_sampler_passes_original_av_latent_and_workflow_sigmas_to_core(monkeypatch): + video = torch.randn((1, 24, 3, 4, 4)) + audio = torch.randn((1, 32, 2, 12)) + latent_samples = _Nested((video, audio)) + latent = {"samples": latent_samples} + sigmas = torch.tensor([0.35, 0.2, 0.0]) + captured = {} + + class _Guider: + def sample(self, noise, latent_image, sampler, supplied_sigmas, **kwargs): + captured.update( + noise=noise, + latent_image=latent_image, + sampler=sampler, + sigmas=supplied_sigmas, + **kwargs, + ) + return latent_image + + def prepare_noise(samples, seed, batch_inds): + captured["prepare_noise"] = (samples, seed, batch_inds) + return torch.full_like(samples, 0.5, device="cpu") + + fake_sample = SimpleNamespace( + fix_empty_latent_channels=lambda _model, samples, _spatial, _temporal: samples, + prepare_noise=prepare_noise, + ) + fake_runtime = ( + SimpleNamespace(intermediate_device=lambda: torch.device("cpu")), + SimpleNamespace(NestedTensor=_Nested), + fake_sample, + SimpleNamespace(PROGRESS_BAR_ENABLED=False), + SimpleNamespace(prepare_callback=lambda *_args: None), + ) + monkeypatch.setattr(refine_sampling, "_load_runtime_modules", lambda: fake_runtime) + monkeypatch.setattr(refine_sampling, "_make_basic_guider", lambda *_args: _Guider()) + + sampler = object() + output = refine_sampling.sample_refine_chunk( + model=object(), + conditioning=["conditioning"], + latent=latent, + sampler=sampler, + sigmas=sigmas, + seed=987, + enable_preview=False, + ) + + assert captured["prepare_noise"] == (video, 987, None) + assert captured["latent_image"] is latent_samples + assert captured["latent_image"].unbind()[1] is audio + assert captured["sigmas"] is sigmas + assert captured["sampler"] is sampler + video_noise, audio_noise = captured["noise"].unbind() + video_mask, audio_mask = captured["denoise_mask"].unbind() + assert torch.equal(video_noise, torch.full_like(video_noise, 0.5)) + assert torch.count_nonzero(audio_noise) == 0 + assert torch.all(video_mask == 1) + assert torch.all(audio_mask == 0) + assert output["samples"] is latent_samples diff --git a/tests/test_v35_second_pass_contract.py b/tests/test_v35_second_pass_contract.py new file mode 100644 index 0000000..51f889d --- /dev/null +++ b/tests/test_v35_second_pass_contract.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import copy + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.v3.plan import _attach_second_pass_contract +from ComfyUI_H3_Continuum_Join.v3.second_pass import ( + SecondPassContractError, + derive_refine_seed, + passthrough_audio_latents, + update_second_pass_geometry, + validate_second_pass_inputs, +) + + +def _latent(samples: torch.Tensor) -> dict[str, torch.Tensor]: + return {"samples": samples} + + +def _physical_entry(t: int, audio_t: int, *, prompt: str = "") -> dict: + return { + "video": torch.zeros((1, 24, t, 40, 40)), + "audio": torch.zeros((1, 8, audio_t)), + "prompt": prompt, + } + + +def _plan_and_latents(*, terminal: bool = False): + if terminal: + logical = [ + _physical_entry(31, 100, prompt="first"), + _physical_entry(35, 110, prompt="middle"), + _physical_entry(35, 110, prompt="last"), + ] + physical = [logical[0], _physical_entry(65, 220)] + plan = { + "width": 640, + "height": 640, + "chunks": [ + {"total_frames": 124, "trim_frames": 0}, + {"total_frames": 141, "trim_frames": 22}, + {"total_frames": 141, "trim_frames": 22}, + ], + "decode_groups": [ + { + "logical_chunk_indices": [1], + "terminal_merged": False, + "total_frames": 124, + "trim_frames": 0, + }, + { + "logical_chunk_indices": [2, 3], + "terminal_merged": True, + "total_frames": 260, + "trim_frames": 22, + }, + ], + } + else: + logical = [_physical_entry(31, 100, prompt="single")] + physical = list(logical) + plan = { + "width": 640, + "height": 640, + "chunks": [{"total_frames": 124, "trim_frames": 0}], + } + _attach_second_pass_contract( + plan, + logical_entries=logical, + physical_entries=physical, + chunk_seconds=5.0, + ) + videos = [_latent(entry["video"]) for entry in physical] + audios = [_latent(entry["audio"]) for entry in physical] + return plan, videos, audios + + +def test_terminal_contract_preserves_physical_group_and_prompt() -> None: + plan, videos, audios = _plan_and_latents(terminal=True) + result = validate_second_pass_inputs(videos, audios, plan) + groups = plan["second_pass_contract"]["physical_groups"] + assert result["physical_group_count"] == 2 + assert groups[1]["logical_chunks"] == [2, 3] + assert groups[1]["physical_frames"] == 260 + assert groups[1]["trim_prefix_frames"] == 22 + assert groups[1]["prompt_policy"] == "paired_timeline_v1" + assert groups[1]["physical_prompt"] == "[0-5s]\nmiddle\n\n[5-10s]\nlast" + + +def test_spatial_upscale_updates_only_target_geometry() -> None: + plan, videos, audios = _plan_and_latents(terminal=True) + upscaled = [ + _latent(item["samples"].new_zeros((1, 24, item["samples"].shape[2], 60, 60))) + for item in videos + ] + validate_second_pass_inputs(upscaled, audios, plan) + updated = update_second_pass_geometry(plan, upscaled) + assert (plan["width"], plan["height"]) == (640, 640) + assert (updated["width"], updated["height"]) == (960, 960) + assert updated["second_pass_contract"]["target_width"] == 960 + + +@pytest.mark.parametrize("channels", [23, 25]) +def test_rejects_non_h3_video_channels(channels: int) -> None: + plan, videos, audios = _plan_and_latents() + videos[0] = _latent(torch.zeros((1, channels, 31, 40, 40))) + with pytest.raises(SecondPassContractError, match="24 channels"): + validate_second_pass_inputs(videos, audios, plan) + + +def test_rejects_temporal_or_group_count_changes() -> None: + plan, videos, audios = _plan_and_latents(terminal=True) + changed_t = copy.deepcopy(videos) + changed_t[1] = _latent(torch.zeros((1, 24, 64, 40, 40))) + with pytest.raises(SecondPassContractError, match="temporal length"): + validate_second_pass_inputs(changed_t, audios, plan) + with pytest.raises(SecondPassContractError, match="group count"): + validate_second_pass_inputs(videos[:1], audios, plan) + + +def test_rejects_group_order_and_inconsistent_spatial_sizes() -> None: + plan, videos, audios = _plan_and_latents(terminal=True) + with pytest.raises(SecondPassContractError): + validate_second_pass_inputs(list(reversed(videos)), list(reversed(audios)), plan) + videos[1] = _latent(torch.zeros((1, 24, 65, 48, 48))) + with pytest.raises(SecondPassContractError, match="same target H/W"): + validate_second_pass_inputs(videos, audios, plan) + + +def test_audio_passthrough_is_bit_exact_and_keeps_latent_objects() -> None: + _, _, audios = _plan_and_latents(terminal=True) + output = passthrough_audio_latents(audios) + assert all(before is after for before, after in zip(audios, output, strict=True)) + assert all( + torch.equal(before["samples"], after["samples"]) + for before, after in zip(audios, output, strict=True) + ) + + +def test_refine_seed_is_deterministic_and_namespaced_per_group() -> None: + assert derive_refine_seed(42, 0) == derive_refine_seed(42, 0) + assert derive_refine_seed(42, 0) != derive_refine_seed(42, 1) + assert derive_refine_seed(42, 0) != derive_refine_seed(43, 0) diff --git a/tests/test_v35_second_pass_sampling.py b/tests/test_v35_second_pass_sampling.py new file mode 100644 index 0000000..b189984 --- /dev/null +++ b/tests/test_v35_second_pass_sampling.py @@ -0,0 +1,499 @@ +from __future__ import annotations + +import torch +import pytest + +import ComfyUI_H3_Continuum_Join.v3.second_pass as second_pass_module +from ComfyUI_H3_Continuum_Join.nodes import NODE_CLASS_MAPPINGS +from ComfyUI_H3_Continuum_Join.v3.second_pass import ( + SecondPassContractError, + derive_refine_seed, + run_second_pass_groups, +) +from ComfyUI_H3_Continuum_Join.v3.refine_context import ( + MAGIC as REFINE_CONTEXT_MAGIC, + RefineConditioningAdaptationError, +) +from ComfyUI_H3_Continuum_Join.v3.second_pass_nodes import ( + H3ContinuumSecondPassV35, +) + + +def _latent(samples: torch.Tensor) -> dict[str, torch.Tensor]: + return {"samples": samples} + + +def _plan(groups: list[dict]) -> dict: + return { + "width": 640, + "height": 640, + "second_pass_contract": { + "version": 1, + "physical_groups": groups, + }, + } + + +def _group(group_id: int, *, temporal: int, prompt: str, terminal: bool = False) -> dict: + return { + "group_id": group_id, + "logical_chunks": [group_id + 1] if not terminal else [2, 3], + "physical_prompt": prompt, + "prompt_policy": "single" if not terminal else "paired_timeline_v1", + "physical_frames": 124 if not terminal else 260, + "trim_prefix_frames": 0 if not terminal else 22, + "terminal_merged": terminal, + "source_width": 640, + "source_height": 640, + "source_batch": 1, + "latent_channels": 24, + "source_latent_t": temporal, + "source_latent_h": 40, + "source_latent_w": 40, + "source_audio_shape": [1, 8, temporal * 4], + } + + +def _run_fixture(groups: list[dict]): + videos = [ + _latent(torch.zeros((1, 24, group["source_latent_t"], 48, 48))) + for group in groups + ] + audios = [ + _latent(torch.full(tuple(group["source_audio_shape"]), float(index + 1))) + for index, group in enumerate(groups) + ] + calls: list[dict] = [] + + def encode_prompt(_clip, prompt, **kwargs): + assert kwargs["first_image"] is None + assert kwargs["last_image"] is None + calls.append({"prompt": prompt}) + return [prompt] + + def build_latent(video, audio): + return {"video": video, "audio": audio} + + def clone_model(model, **kwargs): + calls[-1]["clone"] = kwargs + return model + + def sample(**kwargs): + calls[-1]["seed"] = kwargs["seed"] + calls[-1]["conditioning"] = kwargs["conditioning"] + nested = kwargs["latent"] + return { + "video": nested["video"] + 1, + "audio": nested["audio"] + 99, + } + + def extract(sampled): + return sampled["video"], sampled["audio"] + + result = run_second_pass_groups( + model=object(), + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=videos, + audio_latents=audios, + assembly_plan=_plan(groups), + refine_seed=42, + encode_prompt_fn=encode_prompt, + latent_builder=build_latent, + sample_fn=sample, + stream_extractor=extract, + clone_model_fn=clone_model, + ) + return videos, audios, calls, result + + +def test_second_pass_uses_plan_prompts_one_seed_per_physical_group_and_audio_passthrough(): + groups = [ + _group(0, temporal=31, prompt="first physical prompt"), + _group(1, temporal=65, prompt="terminal physical prompt", terminal=True), + ] + videos, audios, calls, (refined, output_audio, updated, report) = _run_fixture(groups) + + assert [call["prompt"] for call in calls] == [ + "first physical prompt", + "terminal physical prompt", + ] + assert [call["seed"] for call in calls] == [ + derive_refine_seed(42, 0), + derive_refine_seed(42, 1), + ] + assert [call["clone"] for call in calls] == [ + { + "strict": False, + "debug": False, + "chunk_index": 1, + "context_frames": None, + }, + { + "strict": False, + "debug": False, + "chunk_index": 2, + "context_frames": 22, + }, + ] + assert all(after is before for before, after in zip(audios, output_audio, strict=True)) + assert all(torch.equal(item["samples"], torch.ones_like(item["samples"])) for item in refined) + assert [item["samples"].shape[2] for item in refined] == [31, 65] + assert updated["width"] == 768 + assert updated["height"] == 768 + assert updated["second_pass_contract"]["audio_output"] == ( + "bit_exact_first_pass_passthrough" + ) + assert updated["second_pass_contract"]["execution"] == ( + "t2va_physical_groups_audio_locked_v2" + ) + assert updated["second_pass_contract"]["audio_sampling"] == ( + "zero_noise_mask_locked" + ) + assert "temporary_audio_discarded=true" in report + assert "audio zero noise/mask=0" in report + assert "refine_seed=" in report + assert videos[1]["samples"].shape[2] == 65 + + +def test_second_pass_defaults_to_audio_locked_refine_sampler(monkeypatch): + group = _group(0, temporal=31, prompt="physical prompt") + video = _latent(torch.zeros((1, 24, 31, 48, 48))) + audio = _latent(torch.ones(tuple(group["source_audio_shape"]))) + captured = {} + + def fake_refine_sample(**kwargs): + captured.update(kwargs) + return kwargs["latent"] + + monkeypatch.setattr( + second_pass_module, + "sample_refine_chunk", + fake_refine_sample, + ) + run_second_pass_groups( + model=object(), + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=[video], + audio_latents=[audio], + assembly_plan=_plan([group]), + refine_seed=17, + encode_prompt_fn=lambda *_args, **_kwargs: ["conditioning"], + latent_builder=lambda video_samples, audio_samples: { + "video": video_samples, + "audio": audio_samples, + }, + stream_extractor=lambda sampled: (sampled["video"], sampled["audio"]), + clone_model_fn=lambda source, **_kwargs: source, + ) + + assert captured["latent"]["video"] is video["samples"] + assert captured["latent"]["audio"] is audio["samples"] + assert captured["seed"] == derive_refine_seed(17, 0) + + +def test_complete_refine_context_skips_prompt_encode_and_clones_each_group_model(): + groups = [ + _group(0, temporal=31, prompt="unused first prompt"), + _group(1, temporal=65, prompt="unused terminal prompt", terminal=True), + ] + videos = [ + _latent(torch.zeros((1, 24, group["source_latent_t"], 48, 48))) + for group in groups + ] + audios = [ + _latent(torch.ones(tuple(group["source_audio_shape"]))) for group in groups + ] + context_groups = ( + { + "group_id": 0, + "logical_chunks": (1,), + "physical_clip_index": 4, + "context_frames": 0, + }, + { + "group_id": 1, + "logical_chunks": (2, 3), + "physical_clip_index": 5, + "context_frames": 22, + }, + ) + refine_context = { + "magic": REFINE_CONTEXT_MAGIC, + "complete": True, + "groups": context_groups, + } + adapted_calls = [] + clone_calls = [] + sample_calls = [] + video_vae = object() + + def unexpected_prompt_encode(*_args, **_kwargs): + raise AssertionError("complete refine_context must bypass prompt encoding") + + def validate_context(value, *, assembly_plan): + assert value is refine_context + assert assembly_plan is plan + return value + + def adapt(group, **kwargs): + adapted_calls.append((group, kwargs)) + return [f"captured-{group['group_id']}"], { + "keyframes_resized": group["group_id"], + "keyframes_reencoded": 0, + "context_refs_resized": 0, + "warnings": (), + } + + def clone(source, **kwargs): + clone_calls.append((source, kwargs)) + return f"group-model-{kwargs['chunk_index']}" + + def sample(**kwargs): + sample_calls.append(kwargs) + return kwargs["latent"] + + plan = _plan(groups) + refined, output_audio, updated, report = run_second_pass_groups( + model="source-model", + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=videos, + audio_latents=audios, + assembly_plan=plan, + refine_seed=42, + refine_context=refine_context, + video_vae=video_vae, + encode_prompt_fn=unexpected_prompt_encode, + latent_builder=lambda video, audio: {"video": video, "audio": audio}, + sample_fn=sample, + stream_extractor=lambda value: (value["video"], value["audio"]), + clone_model_fn=clone, + validate_refine_context_fn=validate_context, + adapt_group_conditioning_fn=adapt, + ) + + assert [call[0] for call in adapted_calls] == list(context_groups) + assert all(call[1] == { + "target_latent_h": 48, + "target_latent_w": 48, + "video_vae": video_vae, + "upscale_method": "bilinear", + } for call in adapted_calls) + assert [call[1] for call in clone_calls] == [ + { + "strict": False, + "debug": False, + "chunk_index": 4, + "context_frames": None, + }, + { + "strict": False, + "debug": False, + "chunk_index": 5, + "context_frames": 22, + }, + ] + assert [call["model"] for call in sample_calls] == [ + "group-model-4", + "group-model-5", + ] + assert [call["conditioning"] for call in sample_calls] == [ + ["captured-0"], + ["captured-1"], + ] + assert all(before is after for before, after in zip(audios, output_audio, strict=True)) + assert all( + item["samples"] is before["samples"] + for item, before in zip(refined, videos, strict=True) + ) + assert updated["second_pass_contract"]["execution"] == ( + "context_aware_physical_groups_audio_locked_v3" + ) + assert "conditioning_source=refine_context" in report + + +def test_unadaptable_complete_context_warns_and_uses_prompt_only_fallback(): + group = _group(0, temporal=31, prompt="fallback prompt") + video = _latent(torch.zeros((1, 24, 31, 48, 48))) + audio = _latent(torch.ones(tuple(group["source_audio_shape"]))) + refine_context = { + "magic": REFINE_CONTEXT_MAGIC, + "complete": True, + "groups": ({"group_id": 0, "logical_chunks": (1,)},), + } + prompts = [] + + def unavailable(*_args, **_kwargs): + raise RefineConditioningAdaptationError("target conversion unavailable") + + _refined, output_audio, updated, report = run_second_pass_groups( + model=object(), + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=[video], + audio_latents=[audio], + assembly_plan=_plan([group]), + refine_seed=5, + refine_context=refine_context, + encode_prompt_fn=lambda _clip, prompt, **_kwargs: prompts.append(prompt) or [prompt], + latent_builder=lambda video_samples, audio_samples: { + "video": video_samples, + "audio": audio_samples, + }, + sample_fn=lambda **kwargs: kwargs["latent"], + stream_extractor=lambda value: (value["video"], value["audio"]), + clone_model_fn=lambda source, **_kwargs: source, + validate_refine_context_fn=lambda value, **_kwargs: value, + adapt_group_conditioning_fn=unavailable, + ) + + assert prompts == ["fallback prompt"] + assert output_audio[0] is audio + assert updated["second_pass_contract"]["execution"] == ( + "t2va_physical_groups_audio_locked_v2" + ) + assert "target adaptation was unavailable" in report + assert "conditioning_source=prompt_only_fallback" in report + + +def test_incomplete_refine_context_warns_and_preserves_legacy_prompt_path(): + group = _group(0, temporal=31, prompt="legacy prompt") + video = _latent(torch.zeros((1, 24, 31, 48, 48))) + audio = _latent(torch.ones(tuple(group["source_audio_shape"]))) + incomplete_context = { + "magic": REFINE_CONTEXT_MAGIC, + "complete": False, + "groups": (), + } + prompts = [] + + def unexpected_adaptation(*_args, **_kwargs): + raise AssertionError("incomplete context must not be adapted") + + _refined, output_audio, updated, report = run_second_pass_groups( + model=object(), + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=[video], + audio_latents=[audio], + assembly_plan=_plan([group]), + refine_seed=5, + refine_context=incomplete_context, + encode_prompt_fn=lambda _clip, prompt, **_kwargs: prompts.append(prompt) or [prompt], + latent_builder=lambda video_samples, audio_samples: { + "video": video_samples, + "audio": audio_samples, + }, + sample_fn=lambda **kwargs: kwargs["latent"], + stream_extractor=lambda value: (value["video"], value["audio"]), + clone_model_fn=lambda source, **_kwargs: source, + validate_refine_context_fn=lambda value, **_kwargs: value, + adapt_group_conditioning_fn=unexpected_adaptation, + ) + + assert prompts == ["legacy prompt"] + assert output_audio[0] is audio + assert updated["second_pass_contract"]["execution"] == ( + "t2va_physical_groups_audio_locked_v2" + ) + assert "capture is incomplete" in report + + +def test_corrupt_typed_refine_context_is_a_contract_error(): + group = _group(0, temporal=31, prompt="prompt") + with pytest.raises(SecondPassContractError, match="refine_context contract"): + run_second_pass_groups( + model=object(), + clip=object(), + sampler=object(), + sigmas=torch.tensor([0.3, 0.0]), + video_latents=[_latent(torch.zeros((1, 24, 31, 48, 48)))], + audio_latents=[_latent(torch.ones(tuple(group["source_audio_shape"])))], + assembly_plan=_plan([group]), + refine_seed=5, + refine_context={"magic": REFINE_CONTEXT_MAGIC, "version": 999}, + ) + + +def test_experimental_node_unwraps_single_inputs_but_preserves_physical_lists(monkeypatch): + captured = {} + sentinel = (["video"], ["audio"], {"plan": True}, "status") + + def fake_run(**kwargs): + captured.update(kwargs) + return sentinel + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.second_pass_nodes.run_second_pass_groups", + fake_run, + ) + node = H3ContinuumSecondPassV35() + videos = [{"samples": torch.zeros((1, 24, 31, 40, 40))}] + audios = [{"samples": torch.zeros((1, 8, 124))}] + result = node.refine( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + videos, + audios, + [{"second_pass_contract": {}}], + [7], + ) + assert result == sentinel + assert captured["video_latents"] is videos + assert captured["audio_latents"] is audios + assert captured["refine_seed"] == 7 + assert captured["refine_context"] is None + assert captured["video_vae"] is None + + +def test_second_pass_node_forwards_optional_refine_context_and_video_vae(monkeypatch): + captured = {} + sentinel = ([], [], {"plan": True}, "status") + + def fake_run(**kwargs): + captured.update(kwargs) + return sentinel + + monkeypatch.setattr( + "ComfyUI_H3_Continuum_Join.v3.second_pass_nodes.run_second_pass_groups", + fake_run, + ) + context = {"context": True} + video_vae = object() + result = H3ContinuumSecondPassV35().refine( + ["model"], + ["clip"], + ["sampler"], + [torch.tensor([0.3, 0.0])], + [{"samples": torch.zeros((1, 24, 31, 40, 40))}], + [{"samples": torch.zeros((1, 8, 124))}], + [{"second_pass_contract": {}}], + [7], + [context], + [video_vae], + ) + + assert result == sentinel + assert captured["refine_context"] is context + assert captured["video_vae"] is video_vae + + +def test_second_pass_node_is_registered_and_not_deprecated(): + node_class = NODE_CLASS_MAPPINGS["H3ContinuumSecondPassV35"] + assert node_class is H3ContinuumSecondPassV35 + assert node_class.DEPRECATED is False + assert node_class.INPUT_IS_LIST is True + assert node_class.OUTPUT_IS_LIST == (True, True, False, False) + assert node_class.INPUT_TYPES()["optional"] == { + "refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",), + "video_vae": ("VAE",), + } diff --git a/tests/test_v3_physical_decode_groups.py b/tests/test_v3_physical_decode_groups.py index 251408a..7143959 100644 --- a/tests/test_v3_physical_decode_groups.py +++ b/tests/test_v3_physical_decode_groups.py @@ -8,6 +8,8 @@ from ComfyUI_H3_Continuum_Join.v3 import plan as plan_module def _logical_plan(): return { + "width": 1, + "height": 1, "chunks": [ { "sequence_index": 0, @@ -116,6 +118,8 @@ def test_three_logical_chunks_use_two_physical_decode_groups(monkeypatch): "audio": torch.zeros((1, 1, 207), dtype=torch.float32), } logical_plan = { + "width": 1, + "height": 1, "chunks": [ {"sequence_index": 0, "chunk_index": 1, "total_frames": 124, "trim_frames": 0, "net_frames": 124, "context_frames": 0, "expected_video_latent_t": 37, "expected_audio_latent_t": 207, "frame_start": 0, "frame_stop": 124}, {"sequence_index": 1, "chunk_index": 2, "total_frames": 141, "trim_frames": 22, "net_frames": 119, "context_frames": 22, "expected_video_latent_t": 42, "expected_audio_latent_t": 235, "frame_start": 124, "frame_stop": 243}, diff --git a/tools/p20_file_backed_tensor_poc.py b/tools/p20_file_backed_tensor_poc.py new file mode 100644 index 0000000..567cd66 --- /dev/null +++ b/tools/p20_file_backed_tensor_poc.py @@ -0,0 +1,960 @@ +"""Phase 2A P2-0 probe for a Windows-safe file-backed IMAGE buffer. + +The accepted storage/lifetime implementation remains here as the single tested +source and is re-exported through ``v3.file_backed_buffer`` for the V3.5 +assembler. The standalone probes below continue to exercise the same code. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from dataclasses import asdict, dataclass +import gc +import hashlib +import json +import math +import os +from pathlib import Path +import queue +import re +import sys +import threading +import time +from typing import Any, Callable, Iterator +import uuid +import weakref + +import torch + + +SCHEMA = "h3-continuum-p20-file-backed-tensor" +SCHEMA_VERSION = 1 +FILE_PREFIX = "h3c-p20-" +_ID_PATTERN = re.compile(r"^h3c-p20-([0-9a-f]{32})\.(bin|json)$") +_PARTIAL_PATTERN = re.compile(r"^h3c-p20-([0-9a-f]{32})\.json\.partial$") +_LOCK_FILE_NAME = ".h3c-p20.lock" +_ROOT_THREAD_LOCKS: dict[str, threading.RLock] = {} +_ROOT_THREAD_LOCKS_GUARD = threading.Lock() +_ROOT_LOCK_STATE = threading.local() +_PROCESS_MANAGERS: dict[str, "StorageLifetimeManager"] = {} +_PROCESS_MANAGERS_GUARD = threading.Lock() + + +class FileBackedPoCError(RuntimeError): + """Raised when the P2-0 backend cannot uphold its safety contract.""" + + +class StorageTrackingUnavailable(FileBackedPoCError): + """Raised when the current PyTorch cannot weakly track Storage lifetime.""" + + +class ProbeCancelled(BaseException): + """BaseException used to verify cancellation cleanup in the standalone PoC.""" + + +@dataclass(frozen=True) +class BackingRecord: + allocation_id: str + data_path: Path + metadata_path: Path + shape: tuple[int, ...] + dtype: torch.dtype + numel: int + nbytes: int + + +@dataclass(frozen=True) +class CollectionResult: + reclaimed: int + deferred: int + errors: tuple[str, ...] + + +@dataclass(frozen=True) +class StaleCleanupResult: + removed_pairs: int + removed_orphans: int + retained_live: int + retained_young: int + invalid_metadata: int + deferred: int + errors: tuple[str, ...] + + +@dataclass(frozen=True) +class ProbeResult: + torch_version: str + bit_exact: bool + file_size: int + expected_bytes: int + root_released_while_alias_alive: bool + deferred_while_alias_alive: int + reclaimed_after_last_alias: int + files_remaining: int + + +def _storage_weak_ref(tensor: torch.Tensor) -> Any: + """Create an alias-aware weak reference to the underlying Storage. + + ``weakref.finalize(tensor, ...)`` is not sufficient: a Tensor view can keep + the Storage alive after that particular Tensor object has died. The PoC + feature-detects PyTorch's StorageWeakRef and fails closed when unavailable. + """ + + try: + from torch.multiprocessing.reductions import StorageWeakRef + + reference = StorageWeakRef(tensor.untyped_storage()) + if not callable(getattr(reference, "expired", None)): + raise TypeError("StorageWeakRef.expired is unavailable") + return reference + except Exception as exc: + raise StorageTrackingUnavailable( + "this PyTorch build cannot track the lifetime of file-backed " + "Tensor storage aliases" + ) from exc + + +def _process_create_time(pid: int) -> float | None: + try: + import psutil + + return float(psutil.Process(int(pid)).create_time()) + except Exception: + return None + + +_CURRENT_PROCESS_CREATE_TIME = _process_create_time(os.getpid()) + + +def _same_live_process(pid: int, expected_create_time: float | None) -> bool: + """Return True conservatively when a sidecar may belong to a live process.""" + + pid = int(pid) + if pid <= 0: + return False + try: + import psutil + + process = psutil.Process(pid) + actual_create_time = float(process.create_time()) + if expected_create_time is None: + return bool(process.is_running()) + return bool(process.is_running()) and abs( + actual_create_time - float(expected_create_time) + ) <= 1.0 + except ImportError: + pass + except Exception as exc: + try: + import psutil + + if isinstance(exc, psutil.NoSuchProcess): + return False + if isinstance(exc, psutil.AccessDenied): + return True + except Exception: + return True + return True + + if pid == os.getpid(): + if expected_create_time is None or _CURRENT_PROCESS_CREATE_TIME is None: + return True + return abs( + float(expected_create_time) - _CURRENT_PROCESS_CREATE_TIME + ) <= 1.0 + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None: + partial_path = path.with_suffix(path.suffix + ".partial") + with partial_path.open("x", encoding="utf-8", newline="\n") as handle: + json.dump(payload, handle, ensure_ascii=False, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(partial_path, path) + + +def _dtype_name(dtype: torch.dtype) -> str: + value = str(dtype) + return value[6:] if value.startswith("torch.") else value + + +def _validate_shape(shape: tuple[int, ...]) -> tuple[int, ...]: + normalized_values: list[int] = [] + for value in shape: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("file-backed Tensor dimensions must be integers") + normalized_values.append(value) + normalized = tuple(normalized_values) + if not normalized or any(value <= 0 for value in normalized): + raise ValueError("file-backed Tensor shape must contain positive dimensions") + return normalized + + +def _known_path(root: Path, path: Path) -> bool: + try: + return path.parent.resolve() == root.resolve() and bool( + _ID_PATTERN.fullmatch(path.name) or _PARTIAL_PATTERN.fullmatch(path.name) + ) + except OSError: + return False + + +def _unlink_known(root: Path, path: Path) -> bool: + if not _known_path(root, path): + raise FileBackedPoCError(f"refusing to unlink an unknown path: {path}") + try: + path.unlink() + return True + except FileNotFoundError: + return False + + +def _thread_lock_for_root(root: Path) -> threading.RLock: + key = os.path.normcase(str(root.resolve())) + with _ROOT_THREAD_LOCKS_GUARD: + lock = _ROOT_THREAD_LOCKS.get(key) + if lock is None: + lock = threading.RLock() + _ROOT_THREAD_LOCKS[key] = lock + return lock + + +@contextmanager +def _root_lock(root: Path): + """Serialize allocation and cleanup across threads and ComfyUI processes.""" + + root.mkdir(parents=True, exist_ok=True) + key = os.path.normcase(str(root.resolve())) + thread_lock = _thread_lock_for_root(root) + with thread_lock: + depths = getattr(_ROOT_LOCK_STATE, "depths", None) + if depths is None: + depths = {} + _ROOT_LOCK_STATE.depths = depths + depth = int(depths.get(key, 0)) + if depth > 0: + depths[key] = depth + 1 + try: + yield + finally: + remaining = int(depths[key]) - 1 + if remaining > 0: + depths[key] = remaining + else: + depths.pop(key, None) + return + + lock_path = root / _LOCK_FILE_NAME + try: + with lock_path.open("a+b") as handle: + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + if os.name == "nt": + import msvcrt + + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + depths[key] = 1 + try: + yield + finally: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + depths[key] = 1 + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + finally: + depths.pop(key, None) + + +class _TrackedStorage: + __slots__ = ("record", "storage_reference") + + def __init__(self, record: BackingRecord, tensor: torch.Tensor): + self.record = record + self.storage_reference = _storage_weak_ref(tensor) + + def expired(self) -> bool: + return bool(self.storage_reference.expired()) + + +class FileBackedAllocation: + """Provisional allocation that is deleted unless explicitly published.""" + + def __init__( + self, + manager: "StorageLifetimeManager", + tensor: torch.Tensor, + record: BackingRecord, + ): + self._manager = manager + self._tensor: torch.Tensor | None = tensor + self.record = record + self._published = False + self._cleanup_finalizer = weakref.finalize( + self, + manager._discard_unpublished, + record, + ) + + @property + def tensor(self) -> torch.Tensor: + if self._tensor is None: + raise FileBackedPoCError("file-backed allocation is no longer available") + return self._tensor + + def publish(self) -> torch.Tensor: + if self._published: + raise FileBackedPoCError("file-backed allocation was already published") + tensor = self.tensor + try: + self._manager._register(self.record, tensor) + except BaseException: + self.abort() + raise + self._published = True + self._cleanup_finalizer.detach() + self._tensor = None + return tensor + + def abort(self) -> None: + if self._published: + raise FileBackedPoCError("cannot abort a published file-backed Tensor") + self._tensor = None + if self._cleanup_finalizer.alive: + self._cleanup_finalizer() + # Explicit cancellation should attempt cleanup now. The finalizer only + # enqueues an intent so GC never performs filesystem I/O or waits on a + # Windows byte lock. Cleanup failures remain queued for a later sweep + # and must not replace the exception that caused __exit__ to abort. + try: + self._manager.collect_ready() + except Exception: + pass + + def __enter__(self) -> "FileBackedAllocation": + return self + + def __exit__(self, exc_type, exc, traceback) -> bool: + if not self._published: + self.abort() + return False + + +class StorageLifetimeManager: + """Track backing files without retaining their Tensor/storage strongly.""" + + def __init__( + self, + root: str | os.PathLike[str], + *, + allocation_hook: Callable[[Path], None] | None = None, + ): + self.root = Path(root).expanduser().resolve() + self.root.mkdir(parents=True, exist_ok=True) + self._tracked: dict[str, _TrackedStorage] = {} + self._pending: dict[str, tuple[Path, ...]] = {} + self._abandoned: queue.SimpleQueue[BackingRecord] = queue.SimpleQueue() + self._lock = threading.RLock() + self._allocation_hook = allocation_hook + + def allocate( + self, + shape: tuple[int, ...], + *, + dtype: torch.dtype = torch.float32, + ) -> FileBackedAllocation: + with _root_lock(self.root): + with self._lock: + self._drain_abandoned_locked() + self._collect_ready_locked() + return self._allocate_locked(shape, dtype=dtype) + + def _allocate_locked( + self, + shape: tuple[int, ...], + *, + dtype: torch.dtype, + ) -> FileBackedAllocation: + normalized_shape = _validate_shape(shape) + if not isinstance(dtype, torch.dtype): + raise TypeError("dtype must be a torch.dtype") + numel = math.prod(normalized_shape) + element_size = torch.empty((), dtype=dtype).element_size() + allocation_id = uuid.uuid4().hex + data_path = self.root / f"{FILE_PREFIX}{allocation_id}.bin" + metadata_path = self.root / f"{FILE_PREFIX}{allocation_id}.json" + descriptor: int | None = None + tensor: torch.Tensor | None = None + try: + descriptor = os.open( + data_path, + os.O_CREAT | os.O_EXCL | os.O_RDWR, + 0o600, + ) + try: + os.close(descriptor) + finally: + # close() may raise after the OS already released the handle. + # Never retry that numeric fd: another thread could already + # have reused it for an unrelated file. + descriptor = None + flat = torch.from_file( + str(data_path), + shared=True, + size=numel, + dtype=dtype, + ) + tensor = flat.reshape(normalized_shape) + del flat + if tensor.device.type != "cpu" or not tensor.is_contiguous(): + raise FileBackedPoCError( + "torch.from_file did not create a contiguous CPU Tensor" + ) + if self._allocation_hook is not None: + self._allocation_hook(data_path) + record = BackingRecord( + allocation_id=allocation_id, + data_path=data_path, + metadata_path=metadata_path, + shape=normalized_shape, + dtype=dtype, + numel=numel, + nbytes=numel * element_size, + ) + _atomic_write_json( + metadata_path, + { + "schema": SCHEMA, + "schema_version": SCHEMA_VERSION, + "allocation_id": allocation_id, + "data_file": data_path.name, + "shape": list(normalized_shape), + "dtype": _dtype_name(dtype), + "numel": numel, + "nbytes": record.nbytes, + "owner_pid": os.getpid(), + "owner_create_time": _CURRENT_PROCESS_CREATE_TIME, + "created_at": time.time(), + }, + ) + return FileBackedAllocation(self, tensor, record) + except BaseException: + tensor = None + if descriptor is not None: + try: + os.close(descriptor) + except BaseException: + # Preserve the original cancellation/error. On Windows an + # actually-live descriptor will also make unlink fail, so + # the backing paths remain in the retryable pending set. + pass + paths = ( + metadata_path.with_suffix(metadata_path.suffix + ".partial"), + metadata_path, + data_path, + ) + failed = False + for path in paths: + try: + _unlink_known(self.root, path) + except BaseException: + failed = True + if failed: + self._pending[allocation_id] = paths + raise + + def _register(self, record: BackingRecord, tensor: torch.Tensor) -> None: + with _root_lock(self.root): + with self._lock: + if record.allocation_id in self._tracked: + raise FileBackedPoCError("duplicate file-backed allocation ID") + if not record.data_path.is_file() or not record.metadata_path.is_file(): + raise FileBackedPoCError("file-backed allocation files are missing") + self._tracked[record.allocation_id] = _TrackedStorage(record, tensor) + + def _discard_unpublished(self, record: BackingRecord) -> None: + """Queue GC cleanup without taking locks or touching the filesystem.""" + + self._abandoned.put(record) + + def _drain_abandoned_locked(self) -> None: + """Move finalizer intents into the retryable pending set.""" + + while True: + try: + record = self._abandoned.get_nowait() + except queue.Empty: + return + paths = ( + record.metadata_path.with_suffix( + record.metadata_path.suffix + ".partial" + ), + record.metadata_path, + record.data_path, + ) + if record.allocation_id not in self._tracked: + self._pending[record.allocation_id] = paths + + def collect_ready(self) -> CollectionResult: + with _root_lock(self.root): + with self._lock: + self._drain_abandoned_locked() + return self._collect_ready_locked() + + def _collect_ready_locked(self) -> CollectionResult: + reclaimed = 0 + deferred = 0 + errors: list[str] = [] + for allocation_id, paths in list(self._pending.items()): + failed = False + for path in paths: + try: + _unlink_known(self.root, path) + except OSError as exc: + failed = True + errors.append(f"{path.name}: {type(exc).__name__}: {exc}") + if failed: + deferred += 1 + continue + del self._pending[allocation_id] + reclaimed += 1 + for allocation_id, tracked in list(self._tracked.items()): + try: + expired = tracked.expired() + except Exception as exc: + deferred += 1 + errors.append( + f"{allocation_id}: storage lifetime unavailable: " + f"{type(exc).__name__}: {exc}" + ) + continue + if not expired: + deferred += 1 + continue + failed = False + for path in (tracked.record.data_path, tracked.record.metadata_path): + try: + _unlink_known(self.root, path) + except OSError as exc: + failed = True + errors.append( + f"{path.name}: {type(exc).__name__}: {exc}" + ) + if failed: + deferred += 1 + continue + del self._tracked[allocation_id] + reclaimed += 1 + return CollectionResult(reclaimed, deferred, tuple(errors)) + + def tracked_count(self) -> int: + with self._lock: + return len(self._tracked) + + def pending_count(self) -> int: + with self._lock: + return len(self._pending) + + +def get_process_lifetime_manager( + root: str | os.PathLike[str], +) -> StorageLifetimeManager: + """Return a process-global manager so cache lifetime outlives one node call.""" + + resolved = Path(root).expanduser().resolve() + key = os.path.normcase(str(resolved)) + with _PROCESS_MANAGERS_GUARD: + manager = _PROCESS_MANAGERS.get(key) + if manager is None: + manager = StorageLifetimeManager(resolved) + _PROCESS_MANAGERS[key] = manager + return manager + + +def _sidecar_age(metadata_path: Path, payload: dict[str, Any], now: float) -> float: + created_at = payload.get("created_at") + if isinstance(created_at, (int, float)) and math.isfinite(float(created_at)): + return max(0.0, now - float(created_at)) + return max(0.0, now - metadata_path.stat().st_mtime) + + +def _valid_sidecar( + metadata_path: Path, + payload: dict[str, Any], +) -> tuple[bool, Path]: + match = _ID_PATTERN.fullmatch(metadata_path.name) + if match is None or match.group(2) != "json": + return False, metadata_path.with_suffix(".bin") + expected_data = metadata_path.with_suffix(".bin") + shape = payload.get("shape") + owner_pid = payload.get("owner_pid") + owner_create_time = payload.get("owner_create_time") + created_at = payload.get("created_at") + numel = payload.get("numel") + nbytes = payload.get("nbytes") + dtype = payload.get("dtype") + shape_valid = ( + isinstance(shape, list) + and bool(shape) + and all(type(value) is int and value > 0 for value in shape) + ) + ownership_valid = ( + type(owner_pid) is int + and owner_pid > 0 + and "owner_create_time" in payload + and ( + owner_create_time is None + or ( + isinstance(owner_create_time, (int, float)) + and not isinstance(owner_create_time, bool) + and math.isfinite(float(owner_create_time)) + and float(owner_create_time) > 0.0 + ) + ) + and isinstance(created_at, (int, float)) + and not isinstance(created_at, bool) + and math.isfinite(float(created_at)) + and float(created_at) > 0.0 + ) + size_valid = ( + type(numel) is int + and numel > 0 + and shape_valid + and numel == math.prod(shape) + and type(nbytes) is int + and nbytes > 0 + and isinstance(dtype, str) + and bool(dtype) + ) + valid = ( + payload.get("schema") == SCHEMA + and payload.get("schema_version") == SCHEMA_VERSION + and payload.get("allocation_id") == match.group(1) + and payload.get("data_file") == expected_data.name + and ownership_valid + and size_valid + ) + return bool(valid), expected_data + + +def cleanup_stale_files( + root: str | os.PathLike[str], + *, + grace_seconds: float, + now: float | None = None, + process_is_live: Callable[[int, float | None], bool] = _same_live_process, +) -> StaleCleanupResult: + """Remove only dead-process artifacts inside the dedicated P2-0 root.""" + + root_path = Path(root).expanduser().resolve() + root_path.mkdir(parents=True, exist_ok=True) + with _root_lock(root_path): + return _cleanup_stale_files_locked( + root_path, + grace_seconds=grace_seconds, + now=now, + process_is_live=process_is_live, + ) + + +def _cleanup_stale_files_locked( + root_path: Path, + *, + grace_seconds: float, + now: float | None, + process_is_live: Callable[[int, float | None], bool], +) -> StaleCleanupResult: + grace_seconds = max(0.0, float(grace_seconds)) + current_time = time.time() if now is None else float(now) + removed_pairs = 0 + removed_orphans = 0 + retained_live = 0 + retained_young = 0 + invalid_metadata = 0 + deferred = 0 + errors: list[str] = [] + referenced_data: set[str] = set() + protected_data: set[str] = set() + + for metadata_path in sorted(root_path.glob(f"{FILE_PREFIX}*.json")): + if not _known_path(root_path, metadata_path): + continue + expected_data = metadata_path.with_suffix(".bin") + protected_data.add(expected_data.name) + try: + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError("metadata root is not an object") + except Exception as exc: + invalid_metadata += 1 + errors.append( + f"{metadata_path.name}: invalid metadata: {type(exc).__name__}: {exc}" + ) + continue + valid, data_path = _valid_sidecar(metadata_path, payload) + if not valid: + invalid_metadata += 1 + errors.append(f"{metadata_path.name}: metadata contract mismatch") + continue + referenced_data.add(data_path.name) + try: + age = _sidecar_age(metadata_path, payload, current_time) + except OSError as exc: + deferred += 1 + errors.append(f"{metadata_path.name}: {type(exc).__name__}: {exc}") + continue + try: + pid = int(payload.get("owner_pid")) + except (TypeError, ValueError): + pid = -1 + create_time_value = payload.get("owner_create_time") + create_time = ( + float(create_time_value) + if isinstance(create_time_value, (int, float)) + else None + ) + if process_is_live(pid, create_time): + retained_live += 1 + continue + if age < grace_seconds: + retained_young += 1 + continue + pair_failed = False + for path in (data_path, metadata_path): + try: + _unlink_known(root_path, path) + except OSError as exc: + pair_failed = True + errors.append(f"{path.name}: {type(exc).__name__}: {exc}") + if pair_failed: + deferred += 1 + else: + removed_pairs += 1 + + for data_path in sorted(root_path.glob(f"{FILE_PREFIX}*.bin")): + if not _known_path(root_path, data_path): + continue + if data_path.name in referenced_data or data_path.name in protected_data: + continue + try: + age = max(0.0, current_time - data_path.stat().st_mtime) + if age < grace_seconds: + retained_young += 1 + continue + _unlink_known(root_path, data_path) + removed_orphans += 1 + except OSError as exc: + deferred += 1 + errors.append(f"{data_path.name}: {type(exc).__name__}: {exc}") + + for partial_path in sorted(root_path.glob(f"{FILE_PREFIX}*.json.partial")): + if not _known_path(root_path, partial_path): + continue + try: + age = max(0.0, current_time - partial_path.stat().st_mtime) + if age < grace_seconds: + retained_young += 1 + continue + _unlink_known(root_path, partial_path) + removed_orphans += 1 + except OSError as exc: + deferred += 1 + errors.append(f"{partial_path.name}: {type(exc).__name__}: {exc}") + + return StaleCleanupResult( + removed_pairs=removed_pairs, + removed_orphans=removed_orphans, + retained_live=retained_live, + retained_young=retained_young, + invalid_metadata=invalid_metadata, + deferred=deferred, + errors=tuple(errors), + ) + + +def _raw_float32_bytes(tensor: torch.Tensor) -> bytes: + value = tensor.detach().to(device="cpu", dtype=torch.float32) + if not value.is_contiguous(): + value = value.contiguous() + return value.numpy().tobytes(order="C") + + +def _remaining_known_files(root: Path) -> Iterator[Path]: + for path in root.iterdir(): + if _known_path(root, path): + yield path + + +def run_probe(root: str | os.PathLike[str]) -> ProbeResult: + manager = get_process_lifetime_manager(root) + cleanup_stale_files(manager.root, grace_seconds=0.0) + reference = torch.arange(2 * 3 * 4 * 5, dtype=torch.float32).reshape(2, 3, 4, 5) + with manager.allocate(tuple(reference.shape), dtype=reference.dtype) as allocation: + allocation.tensor.copy_(reference) + record = allocation.record + output = allocation.publish() + + bit_exact = torch.equal(output, reference) and ( + record.data_path.read_bytes() == _raw_float32_bytes(reference) + ) + actual_file_size = record.data_path.stat().st_size + alias = output[1] + cache = {"output": alias} + output_reference = weakref.ref(output) + del output + gc.collect() + deferred = manager.collect_ready() + root_released = output_reference() is None and float(cache["output"].sum()) >= 0.0 + del alias + cache.clear() + gc.collect() + reclaimed = manager.collect_ready() + return ProbeResult( + torch_version=str(torch.__version__), + bit_exact=bool(bit_exact), + file_size=actual_file_size, + expected_bytes=reference.numel() * reference.element_size(), + root_released_while_alias_alive=root_released, + deferred_while_alias_alive=deferred.deferred, + reclaimed_after_last_alias=reclaimed.reclaimed, + files_remaining=sum(1 for _ in _remaining_known_files(manager.root)), + ) + + +def _crash_child(root: str | os.PathLike[str]) -> None: + manager = StorageLifetimeManager(root) + with manager.allocate((2, 3, 4), dtype=torch.float32) as allocation: + allocation.tensor.fill_(1.0) + output = allocation.publish() + if float(output.sum()) <= 0.0: + os._exit(3) + os._exit(0) + + +def _lock_order_child(root: str | os.PathLike[str]) -> None: + """Regression child for root/manager lock order and GC finalizer safety.""" + + root_path = Path(root).expanduser().resolve() + manager = StorageLifetimeManager(root_path) + gc.disable() + allocation = manager.allocate((2, 2), dtype=torch.float32) + abandoned_cycle: list[Any] = [allocation] + abandoned_cycle.append(abandoned_cycle) + del allocation + del abandoned_cycle + + manager_lock_acquired = threading.Event() + cleanup_callback_entered = threading.Event() + collector_started = threading.Event() + failures: list[BaseException] = [] + original_manager_lock = manager._lock + + class SignalingLock: + def __enter__(self): + original_manager_lock.acquire() + manager_lock_acquired.set() + return self + + def __exit__(self, exc_type, exc, traceback): + original_manager_lock.release() + return False + + manager._lock = SignalingLock() # type: ignore[assignment] + + def process_is_live(_pid: int, _create_time: float | None) -> bool: + cleanup_callback_entered.set() + if not collector_started.wait(timeout=2.0): + raise RuntimeError("collector did not start") + # With root-before-manager ordering the collector must still be waiting + # for the root lock here. The GC finalizer must only enqueue cleanup. + manager_lock_acquired.wait(timeout=0.25) + gc.collect() + return True + + def run_cleanup() -> None: + try: + cleanup_stale_files( + root_path, + grace_seconds=0.0, + process_is_live=process_is_live, + ) + except BaseException as exc: + failures.append(exc) + + def run_collector() -> None: + collector_started.set() + try: + manager.collect_ready() + except BaseException as exc: + failures.append(exc) + + cleanup_thread = threading.Thread(target=run_cleanup, daemon=True) + cleanup_thread.start() + if not cleanup_callback_entered.wait(timeout=2.0): + os._exit(90) + collector_thread = threading.Thread(target=run_collector, daemon=True) + collector_thread.start() + cleanup_thread.join(timeout=3.0) + collector_thread.join(timeout=3.0) + if cleanup_thread.is_alive() or collector_thread.is_alive(): + os._exit(91) + if failures: + os._exit(92) + manager.collect_ready() + if any(_remaining_known_files(root_path)): + os._exit(93) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--crash-child", action="store_true", help=argparse.SUPPRESS) + parser.add_argument( + "--lock-order-child", + action="store_true", + help=argparse.SUPPRESS, + ) + arguments = parser.parse_args(argv) + if arguments.crash_child: + _crash_child(arguments.root) + return 0 + if arguments.lock_order_child: + _lock_order_child(arguments.root) + return 0 + result = run_probe(arguments.root) + print(json.dumps(asdict(result), ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if all( + ( + result.bit_exact, + result.file_size == result.expected_bytes, + result.root_released_while_alias_alive, + result.deferred_while_alias_alive == 1, + result.reclaimed_after_last_alias == 1, + result.files_remaining == 0, + ) + ) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/p24_comfy_integration_probe.py b/tools/p24_comfy_integration_probe.py new file mode 100644 index 0000000..8d80b80 --- /dev/null +++ b/tools/p24_comfy_integration_probe.py @@ -0,0 +1,676 @@ +"""Phase 2A P2-4 probe against the installed ComfyUI execution/cache stack. + +The probe uses deterministic decoded CPU IMAGE/AUDIO fixtures. It does not +load a model, run a VAE, or use CUDA. Its purpose is to exercise the real +ComfyUI cache, Core Preview/Save nodes, installed VHS Video Combine, Core ESC +interrupt propagation, and restart-style stale cleanup around the V3.5 +Disk-backed assembler. +""" + +from __future__ import annotations + +import argparse +import gc +import importlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +from types import SimpleNamespace +from typing import Any + +import torch + + +class ProbeFailure(RuntimeError): + pass + + +class _Server: + def __init__(self): + self.client_id = None + self.last_node_id = None + self.messages: list[tuple[str, dict[str, Any], Any]] = [] + + def send_sync(self, event, data, client_id): + self.messages.append((event, data, client_id)) + + +def _load_package(package_root: Path): + name = "h3_continuum_p24_probe" + spec = importlib.util.spec_from_file_location( + name, + package_root / "__init__.py", + submodule_search_locations=[str(package_root)], + ) + if spec is None or spec.loader is None: + raise ProbeFailure("cannot create Continuum package import specification") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def _install_paths(comfy_root: Path) -> None: + for value in ( + comfy_root, + comfy_root / "custom_nodes" / "comfyui-videohelpersuite", + ): + text = str(value) + if text not in sys.path: + sys.path.insert(0, text) + + +def _single_plan(package_name: str) -> dict[str, Any]: + hardening = importlib.import_module(f"{package_name}.hardening") + temporal = importlib.import_module(f"{package_name}.temporal") + plan_module = importlib.import_module(f"{package_name}.v3.plan") + total_frames = 124 + chunk = { + "sequence_index": 1, + "chunk_index": 1, + "total_frames": total_frames, + "trim_frames": 0, + "net_frames": total_frames, + "context_frames": 0, + "expected_video_latent_t": temporal.video_latent_t(total_frames), + "expected_audio_latent_t": temporal.audio_latent_t(total_frames), + } + return hardening.enrich_assembly_plan( + { + "magic": plan_module.ASSEMBLY_PLAN_MAGIC, + "schema_version": 1, + "fps": 24, + "width": 16, + "height": 16, + "chunk_seconds": 5.0, + "target_frames": 120, + "preserve_final_frame": True, + "chunks": [chunk], + } + ) + + +def _register_probe_nodes(core_nodes, package_module, video_combine): + package_name = package_module.__name__ + + class P24DecodedFixture: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "nonce": ("INT", {"default": 0}), + "interrupt_before_assembly": ( + "BOOLEAN", + {"default": False}, + ), + } + } + + RETURN_TYPES = ( + "IMAGE", + "AUDIO", + "H3_CONTINUUM_ASSEMBLY_PLAN", + ) + RETURN_NAMES = ("images", "audio", "assembly_plan") + FUNCTION = "build" + CATEGORY = "P2-4 Probe" + + def build(self, nonce, interrupt_before_assembly): + total_frames = 124 + values = ( + torch.arange(total_frames, dtype=torch.float32) + + float(int(nonce) % 7) + ) / float(total_frames + 7) + images = values.reshape(-1, 1, 1, 1).expand( + -1, + 16, + 16, + 3, + ) + sample_rate = 32_000 + samples = int(round(total_frames / 24 * sample_rate)) + audio = { + "waveform": torch.linspace( + -0.05, + 0.05, + samples, + dtype=torch.float32, + ).reshape(1, 1, -1), + "sample_rate": sample_rate, + } + plan = _single_plan(package_name) + if bool(interrupt_before_assembly): + core_nodes.interrupt_processing(True) + return images, audio, plan + + class P24FailOrPass: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "fail": ("BOOLEAN", {"default": True}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "consume" + OUTPUT_NODE = True + CATEGORY = "P2-4 Probe" + + def consume(self, images, fail): + if bool(fail): + raise RuntimeError("P2-4 intentional downstream failure") + return { + "ui": {"p24": ["downstream pass"]}, + "result": (images,), + } + + class P24ClearCacheOutput: + @classmethod + def INPUT_TYPES(cls): + return {"required": {"token": ("INT", {"default": 0})}} + + RETURN_TYPES = ("STRING",) + FUNCTION = "finish" + OUTPUT_NODE = True + CATEGORY = "P2-4 Probe" + + def finish(self, token): + return (f"clear-{int(token)}",) + + core_nodes.NODE_CLASS_MAPPINGS.update(package_module.NODE_CLASS_MAPPINGS) + core_nodes.NODE_CLASS_MAPPINGS.update( + { + "P24DecodedFixture": P24DecodedFixture, + "P24FailOrPass": P24FailOrPass, + "P24ClearCacheOutput": P24ClearCacheOutput, + "P24VHSVideoCombine": video_combine, + } + ) + + +def _base_prompt(*, nonce: int, interrupt_before_assembly: bool) -> dict[str, Any]: + return { + "1": { + "class_type": "P24DecodedFixture", + "inputs": { + "nonce": int(nonce), + "interrupt_before_assembly": bool(interrupt_before_assembly), + }, + }, + "2": { + "class_type": "H3ContinuumAssembleSeamV35", + "inputs": { + "images": ["1", 0], + "audio": ["1", 1], + "assembly_plan": ["1", 2], + "exact_total_duration": True, + "audio_seam": "Off", + "video_seam": "Off", + "buffer_backend": "Disk-backed", + "diagnostics": "Basic", + }, + }, + } + + +def _failure_prompt(*, fail: bool, nonce: int = 0) -> dict[str, Any]: + prompt = _base_prompt(nonce=nonce, interrupt_before_assembly=False) + prompt["3"] = { + "class_type": "P24FailOrPass", + "inputs": {"images": ["2", 0], "fail": bool(fail)}, + } + return prompt + + +def _compatibility_prompt() -> dict[str, Any]: + prompt = _base_prompt(nonce=0, interrupt_before_assembly=False) + prompt.update( + { + "4": { + "class_type": "SaveImage", + "inputs": { + "images": ["2", 0], + "filename_prefix": "P24/Save", + }, + }, + "5": { + "class_type": "PreviewImage", + "inputs": {"images": ["2", 0]}, + }, + "6": { + "class_type": "P24VHSVideoCombine", + "inputs": { + "images": ["2", 0], + "audio": ["2", 1], + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "P24/VHS", + "format": "video/h264-mp4", + "pingpong": False, + "save_output": True, + }, + }, + } + ) + return prompt + + +def _interrupt_prompt() -> dict[str, Any]: + prompt = _base_prompt(nonce=99, interrupt_before_assembly=True) + prompt["3"] = { + "class_type": "P24FailOrPass", + "inputs": {"images": ["2", 0], "fail": False}, + } + return prompt + + +def _clear_prompt(token: int) -> dict[str, Any]: + return { + "9": { + "class_type": "P24ClearCacheOutput", + "inputs": {"token": int(token)}, + } + } + + +def _cached_nodes(executor) -> set[str]: + for event, data in executor.status_messages: + if event == "execution_cached": + return {str(value) for value in data.get("nodes", [])} + return set() + + +def _has_event(executor, wanted: str) -> bool: + return any(event == wanted for event, _ in executor.status_messages) + + +def _execute(executor, prompt, prompt_id: str, outputs: list[str]) -> None: + executor.execute( + prompt, + prompt_id, + extra_data={}, + execute_outputs=outputs, + ) + + +def _managed_files(backing_root: Path, suffix: str) -> list[Path]: + return sorted(backing_root.glob(f"h3c-p20-*.{suffix}")) + + +def _all_output_files(root: Path) -> set[str]: + if not root.exists(): + return set() + return { + str(path.relative_to(root)) + for path in root.rglob("*") + if path.is_file() + } + + +def _assert_output_paths(output_root: Path, temp_root: Path, history: dict) -> dict: + outputs = history.get("outputs", {}) + save_items = outputs.get("4", {}).get("images", []) + preview_items = outputs.get("5", {}).get("images", []) + gifs = outputs.get("6", {}).get("gifs", []) + if len(save_items) != 120: + raise ProbeFailure(f"Core SaveImage wrote {len(save_items)} frames, expected 120") + if len(preview_items) != 120: + raise ProbeFailure( + f"Core PreviewImage wrote {len(preview_items)} frames, expected 120" + ) + if len(gifs) != 1: + raise ProbeFailure(f"VHS returned {len(gifs)} previews, expected 1") + + for item in save_items: + path = output_root / item.get("subfolder", "") / item["filename"] + if not path.is_file() or path.stat().st_size <= 0: + raise ProbeFailure(f"Core SaveImage output missing: {path}") + for item in preview_items: + path = temp_root / item.get("subfolder", "") / item["filename"] + if not path.is_file() or path.stat().st_size <= 0: + raise ProbeFailure(f"Core PreviewImage output missing: {path}") + video_path = Path(gifs[0]["fullpath"]) + if not video_path.is_file() or video_path.stat().st_size <= 0: + raise ProbeFailure(f"VHS output missing: {video_path}") + ffprobe = shutil.which("ffprobe") + if ffprobe is None: + raise ProbeFailure("ffprobe is required for P2-4 VHS acceptance") + inspected = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-count_frames", + "-show_entries", + ( + "stream=codec_type,codec_name,width,height,r_frame_rate," + "nb_read_frames,sample_rate,channels,duration:" + "format=duration,size" + ), + "-of", + "json", + str(video_path), + ], + check=True, + capture_output=True, + text=True, + ) + media = json.loads(inspected.stdout) + video_stream = next( + (item for item in media.get("streams", []) if item.get("codec_type") == "video"), + None, + ) + audio_stream = next( + (item for item in media.get("streams", []) if item.get("codec_type") == "audio"), + None, + ) + if video_stream is None or audio_stream is None: + raise ProbeFailure("VHS output does not contain both video and audio") + if ( + video_stream.get("codec_name") != "h264" + or int(video_stream.get("nb_read_frames", 0)) != 120 + or video_stream.get("r_frame_rate") != "24/1" + or int(video_stream.get("width", 0)) != 16 + or int(video_stream.get("height", 0)) != 16 + or int(audio_stream.get("sample_rate", 0)) != 32_000 + ): + raise ProbeFailure(f"unexpected VHS media contract: {media}") + return { + "save_frames": len(save_items), + "preview_frames": len(preview_items), + "vhs_file": str(video_path), + "vhs_bytes": int(video_path.stat().st_size), + "vhs_video_frames": int(video_stream["nb_read_frames"]), + "vhs_video_fps": video_stream["r_frame_rate"], + "vhs_video_duration": float(video_stream["duration"]), + "vhs_audio_sample_rate": int(audio_stream["sample_rate"]), + "vhs_container_duration": float(media["format"]["duration"]), + } + + +def _collect_until_empty(manager, backing_root: Path) -> None: + for _ in range(8): + gc.collect() + manager.collect_ready() + if not _managed_files(backing_root, "bin") and not _managed_files( + backing_root, + "json", + ): + return + time.sleep(0.05) + raise ProbeFailure("released cache storage did not become reclaimable") + + +def _run_child_mode(args, package_root: Path) -> int: + _install_paths(args.comfy_root) + package = _load_package(package_root) + buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer") + root = args.backing_root.resolve() + if args.child_mode == "create-stale": + manager = buffers.get_file_backed_image_manager(root) + allocation = manager.allocate((2, 8, 8, 3), dtype=torch.float32) + tensor = allocation.publish() + tensor.fill_(0.25) + os._exit(0) + if args.child_mode == "clean-stale": + buffers.get_file_backed_image_manager(root) + payload = { + "bin": len(_managed_files(root, "bin")), + "json": len(_managed_files(root, "json")), + } + print(json.dumps(payload, sort_keys=True)) + return 0 + raise ProbeFailure(f"unknown child mode: {args.child_mode}") + + +def _restart_stale_probe(args, package_root: Path, stale_root: Path) -> dict: + common = [ + sys.executable, + str(Path(__file__).resolve()), + "--comfy-root", + str(args.comfy_root), + "--package-root", + str(package_root), + "--backing-root", + str(stale_root), + ] + subprocess.run( + [*common, "--child-mode", "create-stale"], + check=True, + capture_output=True, + text=True, + ) + created = ( + len(_managed_files(stale_root, "bin")), + len(_managed_files(stale_root, "json")), + ) + if created != (1, 1): + raise ProbeFailure(f"crash child left unexpected files: {created}") + cleaned = subprocess.run( + [*common, "--child-mode", "clean-stale"], + check=True, + capture_output=True, + text=True, + ) + line = next( + (value for value in reversed(cleaned.stdout.splitlines()) if value.startswith("{")), + "", + ) + payload = json.loads(line) + if payload != {"bin": 0, "json": 0}: + raise ProbeFailure(f"restart stale cleanup failed: {payload}") + return {"created_pairs": 1, "remaining_pairs_after_restart": 0} + + +def run_probe(args, package_root: Path) -> dict: + _install_paths(args.comfy_root) + import folder_paths + import nodes as core_nodes + import server as comfy_server + from execution import CacheType, PromptExecutor + + # A normal ComfyUI process installs this singleton before loading custom + # nodes. The headless probe supplies only the queue fields VHS imports; + # Video Combine itself does not use them without a meta-batch. + if not hasattr(comfy_server.PromptServer, "instance"): + comfy_server.PromptServer.instance = SimpleNamespace( + prompt_queue=SimpleNamespace( + currently_running={}, + put=lambda _item: None, + ), + number=0, + ) + from videohelpersuite.nodes import VideoCombine + + package = _load_package(package_root) + _register_probe_nodes(core_nodes, package, VideoCombine) + buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer") + + work_root = args.work_root.resolve() + output_root = work_root / "output" + temp_root = work_root / "temp" + backing_root = temp_root / "h3-continuum-v35-file-backed" + stale_root = work_root / "restart-stale" + for path in (output_root, temp_root, backing_root, stale_root): + path.mkdir(parents=True, exist_ok=True) + + old_output = folder_paths.get_output_directory() + old_temp = folder_paths.get_temp_directory() + folder_paths.set_output_directory(str(output_root)) + folder_paths.set_temp_directory(str(temp_root)) + prompt_graphs = { + "downstream_error": _failure_prompt(fail=True), + "recovery": _failure_prompt(fail=False), + "compatibility": _compatibility_prompt(), + "interrupt": _interrupt_prompt(), + "clear": _clear_prompt(1), + } + (work_root / "p24_prompt_graphs.json").write_text( + json.dumps(prompt_graphs, indent=2, sort_keys=True), + encoding="utf-8", + ) + + server = _Server() + executor = PromptExecutor( + server, + cache_type=CacheType.CLASSIC, + cache_args={"ram": 16.0, "ram_inactive": 0.0}, + ) + manager = buffers.get_file_backed_image_manager(backing_root) + started = time.perf_counter() + try: + _execute(executor, prompt_graphs["downstream_error"], "p24-error", ["3"]) + if executor.success or not _has_event(executor, "execution_error"): + raise ProbeFailure("intentional downstream error was not reported") + if (len(_managed_files(backing_root, "bin")), len(_managed_files(backing_root, "json"))) != (1, 1): + raise ProbeFailure("completed assembler output was not retained by Core cache") + retained = manager.collect_ready() + if retained.deferred < 1: + raise ProbeFailure("Core cache did not keep mapped storage alive after error") + + _execute(executor, prompt_graphs["recovery"], "p24-recovery", ["3"]) + recovery_cached = _cached_nodes(executor) + if not executor.success or "2" not in recovery_cached: + raise ProbeFailure("requeue did not reuse cached V3.5 assembler output") + if len(_managed_files(backing_root, "bin")) != 1: + raise ProbeFailure("same-input requeue created another backing file") + + _execute( + executor, + prompt_graphs["compatibility"], + "p24-compatibility", + ["4", "5", "6"], + ) + if not executor.success or "2" not in _cached_nodes(executor): + raise ProbeFailure("downstream compatibility run did not reuse assembler cache") + downstream = _assert_output_paths( + output_root, + temp_root, + executor.history_result, + ) + output_files_before = _all_output_files(output_root) + temp_files_before = _all_output_files(temp_root) + + _execute( + executor, + prompt_graphs["compatibility"], + "p24-compatibility-requeue", + ["4", "5", "6"], + ) + second_cached = _cached_nodes(executor) + if not {"2", "4", "5", "6"}.issubset(second_cached): + raise ProbeFailure( + f"compatibility requeue cache miss: {sorted(second_cached)}" + ) + if output_files_before != _all_output_files(output_root): + raise ProbeFailure("cached requeue unexpectedly rewrote output files") + if temp_files_before != _all_output_files(temp_root): + raise ProbeFailure("cached requeue unexpectedly rewrote temp files") + + _execute(executor, prompt_graphs["clear"], "p24-clear", ["9"]) + gc.collect() + if manager.tracked_count() != 1: + raise ProbeFailure("cache eviction lost lifetime-manager tracking state") + + replacement = _failure_prompt(fail=False, nonce=2) + _execute(executor, replacement, "p24-replacement", ["3"]) + if not executor.success or len(_managed_files(backing_root, "bin")) != 1: + raise ProbeFailure("post-eviction allocation did not keep backing pairs bounded") + _execute(executor, _clear_prompt(2), "p24-final-clear", ["9"]) + _collect_until_empty(manager, backing_root) + + _execute(executor, prompt_graphs["interrupt"], "p24-interrupt", ["3"]) + if executor.success or not _has_event(executor, "execution_interrupted"): + raise ProbeFailure("Core interrupt was not propagated as execution_interrupted") + if _managed_files(backing_root, "bin") or _managed_files(backing_root, "json"): + raise ProbeFailure("interrupted unpublished allocation left backing files") + _execute(executor, _clear_prompt(3), "p24-interrupt-clear", ["9"]) + _collect_until_empty(manager, backing_root) + + stale = _restart_stale_probe(args, package_root, stale_root) + elapsed = time.perf_counter() - started + result = { + "phase": "P2-4", + "status": "PASS", + "gpu_used": False, + "fixture": { + "source": "deterministic decoded CPU IMAGE/AUDIO", + "frames": 124, + "assembled_frames": 120, + "width": 16, + "height": 16, + "fps": 24, + "audio_sample_rate": 32_000, + "model": "not applicable", + "lora": "not applicable", + "sampler": "not applicable", + "prompt": "not applicable", + "media_inputs": "none", + }, + "cache": { + "implementation": "ComfyUI PromptExecutor CLASSIC / HierarchicalCache", + "downstream_error_retained": True, + "same_input_requeue_cached": True, + "recovery_cached_nodes": sorted(recovery_cached), + "compatibility_requeue_cached_nodes": sorted(second_cached), + "post_eviction_pair_count_bounded": True, + "final_pairs": 0, + }, + "downstream": downstream, + "interrupt": { + "core_execution_interrupted": True, + "unpublished_pairs_remaining": 0, + }, + "restart_stale_cleanup": stale, + "elapsed_seconds": round(elapsed, 3), + "work_root": str(work_root), + } + (work_root / "p24_result.json").write_text( + json.dumps(result, indent=2, sort_keys=True), + encoding="utf-8", + ) + return result + finally: + core_nodes.interrupt_processing(False) + folder_paths.set_output_directory(old_output) + folder_paths.set_temp_directory(old_temp) + + +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--comfy-root", type=Path, required=True) + parser.add_argument("--package-root", type=Path) + parser.add_argument("--work-root", type=Path) + parser.add_argument("--child-mode", choices=("create-stale", "clean-stale")) + parser.add_argument("--backing-root", type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + args.comfy_root = args.comfy_root.resolve() + package_root = ( + args.package_root.resolve() + if args.package_root is not None + else Path(__file__).resolve().parents[1] + ) + if args.child_mode is not None: + if args.backing_root is None: + raise ProbeFailure("--backing-root is required in child mode") + return _run_child_mode(args, package_root) + if args.work_root is None: + raise ProbeFailure("--work-root is required") + result = run_probe(args, package_root) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/p25_gpu_acceptance_probe.py b/tools/p25_gpu_acceptance_probe.py new file mode 100644 index 0000000..3cc7000 --- /dev/null +++ b/tools/p25_gpu_acceptance_probe.py @@ -0,0 +1,963 @@ +"""Phase 2A P2-5 GPU/RSS acceptance probe for the V3.5 assembler. + +The probe reuses a completed three-chunk Run Storage revision. It restores +the logical AV latents, losslessly reconstructs the accepted Long Terminal +Merge physical decode groups, runs the installed Core Video/Audio VAE Decode +nodes, and compares independent RAM and Disk-backed assembler processes. + +No sampler/model/CLIP execution is performed and no browser interaction is +required. Each backend runs in a fresh process so its RSS peak is not +contaminated by the other backend's ComfyUI cache. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import importlib +import importlib.util +import json +from pathlib import Path +import shutil +import subprocess +import sys +import threading +import time +from types import SimpleNamespace +from typing import Any + +import psutil +import torch +from safetensors.torch import load_file + + +class ProbeFailure(RuntimeError): + pass + + +class _Server: + def __init__(self): + self.client_id = None + self.last_node_id = None + self.messages: list[tuple[str, dict[str, Any], Any]] = [] + + def send_sync(self, event, data, client_id): + self.messages.append((event, data, client_id)) + + +class _ResourceMonitor: + def __init__(self, interval: float = 0.05): + self.interval = float(interval) + self.process = psutil.Process() + self.stop_event = threading.Event() + self.thread: threading.Thread | None = None + self.samples = 0 + self.peak_rss = 0 + self.minimum_system_available = 1 << 63 + self.minimum_cuda_free = 1 << 63 + + def _sample(self) -> None: + try: + self.peak_rss = max(self.peak_rss, int(self.process.memory_info().rss)) + self.minimum_system_available = min( + self.minimum_system_available, + int(psutil.virtual_memory().available), + ) + if torch.cuda.is_available(): + free, _total = torch.cuda.mem_get_info() + self.minimum_cuda_free = min(self.minimum_cuda_free, int(free)) + self.samples += 1 + except Exception: + pass + + def _run(self) -> None: + while not self.stop_event.wait(self.interval): + self._sample() + + def start(self) -> None: + self._sample() + self.thread = threading.Thread(target=self._run, daemon=True) + self.thread.start() + + def stop(self) -> dict[str, Any]: + self._sample() + self.stop_event.set() + if self.thread is not None: + self.thread.join(timeout=5.0) + return { + "samples": int(self.samples), + "peak_rss_bytes": int(self.peak_rss), + "minimum_system_available_bytes": int(self.minimum_system_available), + "minimum_cuda_free_bytes": ( + int(self.minimum_cuda_free) + if self.minimum_cuda_free != 1 << 63 + else None + ), + } + + +def _install_paths(comfy_root: Path) -> None: + for value in ( + comfy_root, + comfy_root / "custom_nodes" / "comfyui-videohelpersuite", + ): + text = str(value) + if text not in sys.path: + sys.path.insert(0, text) + + +def _load_package(package_root: Path): + name = "h3_continuum_p25_probe" + spec = importlib.util.spec_from_file_location( + name, + package_root / "__init__.py", + submodule_search_locations=[str(package_root)], + ) + if spec is None or spec.loader is None: + raise ProbeFailure("cannot create Continuum package import specification") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def _hash_tensor(tensor: torch.Tensor, batch: int = 8) -> str: + digest = hashlib.sha256() + value = tensor.detach().cpu() + if value.ndim == 0: + digest.update(value.numpy().tobytes(order="C")) + return digest.hexdigest() + for start in range(0, int(value.shape[0]), int(batch)): + block = value[start : start + int(batch)].contiguous() + digest.update(block.numpy().tobytes(order="C")) + return digest.hexdigest() + + +def _tensor_is_finite_batched(tensor: torch.Tensor, batch: int = 8) -> bool: + value = tensor.detach().cpu() + if value.ndim == 0: + return bool(torch.isfinite(value).item()) + for start in range(0, int(value.shape[0]), int(batch)): + if not bool(torch.isfinite(value[start : start + int(batch)]).all().item()): + return False + return True + + +def _process_memory_snapshot() -> dict[str, int]: + values = psutil.Process().memory_full_info()._asdict() + return {key: int(value) for key, value in values.items()} + + +def _managed_files(root: Path, suffix: str) -> list[Path]: + return sorted(root.glob(f"h3c-p20-*.{suffix}")) + + +def _collect_until_empty(manager, root: Path) -> None: + for _ in range(20): + gc.collect() + manager.collect_ready() + if not _managed_files(root, "bin") and not _managed_files(root, "json"): + return + time.sleep(0.1) + raise ProbeFailure("released Disk-backed output was not reclaimed") + + +def _register_probe_nodes( + core_nodes, + package_module, + revision_root: Path, + results, + video_combine=None, + stress_size: int = 0, +): + plan_module = importlib.import_module(f"{package_module.__name__}.v3.plan") + + class P25StoredLongTerminalFixture: + @classmethod + def INPUT_TYPES(cls): + return {"required": {"nonce": ("INT", {"default": 0})}} + + RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN") + RETURN_NAMES = ("video_latents", "audio_latents", "assembly_plan") + OUTPUT_IS_LIST = (True, True, False) + FUNCTION = "load" + CATEGORY = "P2-5 Probe" + + def load(self, nonce): + del nonce + manifest_path = revision_root / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + records = sorted( + manifest["chunks"], + key=lambda item: int(item["sequence_index"]), + ) + entries = [] + for record in records: + tensors = load_file( + str(revision_root / "chunks" / record["filename"]), + device="cpu", + ) + entry = dict(record["entry"]) + entry["video"] = tensors["video"] + entry["audio"] = tensors["audio"] + entry["prompt"] = f"stored:{entry.get('prompt_hash', '')}" + entries.append(entry) + decode_entries, plan = plan_module.prepare_physical_decode_entries( + entries, + chunk_seconds=float(manifest["contract"]["global"]["chunk_seconds"]), + preserve_final_frame=manifest.get("last_frame_hash") not in (None, "none"), + terminal_merged=True, + ) + results["fixture"] = { + "revision_id": manifest["revision_id"], + "logical_chunks": len(entries), + "physical_decode_groups": len(decode_entries), + "video_shapes": [list(item["video"].shape) for item in decode_entries], + "audio_shapes": [list(item["audio"].shape) for item in decode_entries], + "width": int(plan["width"]), + "height": int(plan["height"]), + "target_frames": int(plan["target_frames"]), + "prompt_mode": manifest["contract"]["prompt_mode"], + "conditioning_mode": manifest["contract"]["global"]["conditioning_mode"], + "sampler": manifest["contract"]["global"]["sampler"]["function"], + "sigma_count": int(manifest["contract"]["global"]["sigmas"]["shape"][0]), + "base_seed": int(manifest["contract"]["global"]["base_seed"]), + } + return ( + [{"samples": item["video"]} for item in decode_entries], + [{"samples": item["audio"]} for item in decode_entries], + plan, + ) + + class P25DigestOutput: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "audio": ("AUDIO",), + "report": ("STRING",), + "backend": (["Auto", "RAM", "Disk-backed"],), + } + } + + RETURN_TYPES = ("STRING",) + FUNCTION = "consume" + OUTPUT_NODE = True + CATEGORY = "P2-5 Probe" + + def consume(self, images, audio, report, backend): + waveform = audio["waveform"] + hash_batch = 1 if results.get("stress") else 8 + results["pre_digest_memory"] = _process_memory_snapshot() + results["digest"] = { + "backend": str(backend), + "video_shape": list(images.shape), + "video_dtype": str(images.dtype), + "video_sha256": _hash_tensor(images, batch=hash_batch), + "video_finite": _tensor_is_finite_batched( + images, batch=hash_batch + ), + "audio_shape": list(waveform.shape), + "audio_dtype": str(waveform.dtype), + "audio_sha256": _hash_tensor(waveform), + "audio_finite": _tensor_is_finite_batched(waveform), + "audio_sample_rate": int(audio["sample_rate"]), + "report": str(report), + } + return {"ui": {"p25": [str(backend)]}, "result": (str(backend),)} + + class P25SyntheticDecodedStressFixture: + """Large decoded IMAGE fixture with O(H*W), not O(T*H*W), source RAM.""" + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"nonce": ("INT", {"default": 0})}} + + RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN") + RETURN_NAMES = ("images", "audio", "assembly_plan") + OUTPUT_IS_LIST = (True, True, False) + FUNCTION = "load" + CATEGORY = "P2-5 Probe" + + def load(self, nonce): + del nonce + if stress_size <= 0: + raise ProbeFailure("synthetic stress fixture requires --stress-size") + + # Reuse the accepted Long Terminal Merge plan instead of inventing a + # synthetic grouping contract. Only the already-decoded pixel frames + # are replaced. Expanded frame views keep source RAM bounded while + # the assembler must still write every byte of the final IMAGE. + _video, _audio, plan = P25StoredLongTerminalFixture().load(0) + del _video, _audio + gc.collect() + plan = dict(plan) + plan["width"] = int(stress_size) + plan["height"] = int(stress_size) + units = list(plan.get("decode_groups") or plan["chunks"]) + + axis = torch.linspace(0.0, 1.0, stress_size, dtype=torch.float32) + yy = axis.view(stress_size, 1) + xx = axis.view(1, stress_size) + frames = [] + waveforms = [] + sample_rate = 32000 + for index, unit in enumerate(units): + base = torch.empty( + (1, stress_size, stress_size, 3), dtype=torch.float32 + ) + base[0, ..., 0].copy_((xx + index * 0.07).remainder(1.0)) + base[0, ..., 1].copy_((yy + index * 0.11).remainder(1.0)) + base[0, ..., 2].copy_(((xx + yy) * 0.5 + index * 0.13).remainder(1.0)) + total_frames = int(unit["total_frames"]) + frames.append(base.expand(total_frames, -1, -1, -1)) + + total_samples = int(round(total_frames / 24.0 * sample_rate)) + phase = torch.linspace( + 0.0, + (index + 1) * 24.0, + total_samples, + dtype=torch.float32, + ) + wave = (0.05 * torch.sin(phase)).reshape(1, 1, -1) + waveforms.append( + { + "waveform": wave.expand(1, 2, -1), + "sample_rate": sample_rate, + } + ) + + final_bytes = ( + int(plan["target_frames"]) + * stress_size + * stress_size + * 3 + * torch.tensor([], dtype=torch.float32).element_size() + ) + results["stress"] = { + "size": int(stress_size), + "target_frames": int(plan["target_frames"]), + "final_image_bytes": int(final_bytes), + "final_image_gib": round(final_bytes / (1024**3), 3), + "source_frame_storage_bytes": int( + sum(frame.untyped_storage().nbytes() for frame in frames) + ), + "source_frames_are_expanded_views": all( + int(frame.stride(0)) == 0 for frame in frames + ), + } + return frames, waveforms, plan + + class P25ClearOutput: + @classmethod + def INPUT_TYPES(cls): + return {"required": {"token": ("INT", {"default": 0})}} + + RETURN_TYPES = ("STRING",) + FUNCTION = "finish" + OUTPUT_NODE = True + CATEGORY = "P2-5 Probe" + + def finish(self, token): + return (f"clear-{int(token)}",) + + class P25InterruptAfterAllocation: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "audio": ("AUDIO",), + "assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",), + } + } + + RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_ASSEMBLY_PLAN") + RETURN_NAMES = ("images", "audio", "assembly_plan") + INPUT_IS_LIST = True + OUTPUT_IS_LIST = (True, True, False) + FUNCTION = "arm" + CATEGORY = "P2-5 Probe" + + def arm(self, images, audio, assembly_plan): + backing_root = Path(results["backing_root"]) + + def trigger(): + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + if _managed_files(backing_root, "bin"): + results["interrupt_observed_allocation"] = True + time.sleep(0.05) + core_nodes.interrupt_processing(True) + return + time.sleep(0.005) + results["interrupt_observed_allocation"] = False + core_nodes.interrupt_processing(True) + + threading.Thread(target=trigger, daemon=True).start() + return images, audio, assembly_plan[0] + + from comfy_extras.nodes_audio import VAEDecodeAudio + + core_nodes.NODE_CLASS_MAPPINGS.update(package_module.NODE_CLASS_MAPPINGS) + core_nodes.NODE_CLASS_MAPPINGS.update( + { + "VAEDecodeAudio": VAEDecodeAudio, + "P25StoredLongTerminalFixture": P25StoredLongTerminalFixture, + "P25SyntheticDecodedStressFixture": P25SyntheticDecodedStressFixture, + "P25DigestOutput": P25DigestOutput, + "P25ClearOutput": P25ClearOutput, + "P25InterruptAfterAllocation": P25InterruptAfterAllocation, + } + ) + if video_combine is not None: + core_nodes.NODE_CLASS_MAPPINGS["P25VHSVideoCombine"] = video_combine + + +def _prompt( + backend: str, + *, + downstream: bool = False, + interrupt_after_allocation: bool = False, + stress: bool = False, +) -> dict[str, Any]: + if stress: + return { + "1": { + "class_type": "P25SyntheticDecodedStressFixture", + "inputs": {"nonce": 0}, + }, + "6": { + "class_type": "H3ContinuumAssembleSeamV35", + "inputs": { + "images": ["1", 0], + "audio": ["1", 1], + "assembly_plan": ["1", 2], + "exact_total_duration": True, + "audio_seam": "Auto", + "video_seam": "Auto", + "buffer_backend": str(backend), + "diagnostics": "Detailed Report", + }, + }, + "7": { + "class_type": "P25DigestOutput", + "inputs": { + "images": ["6", 0], + "audio": ["6", 1], + "report": ["6", 2], + "backend": str(backend), + }, + }, + } + prompt = { + "1": { + "class_type": "P25StoredLongTerminalFixture", + "inputs": {"nonce": 0}, + }, + "2": { + "class_type": "VAELoader", + "inputs": {"vae_name": "minimax_h3_video_vae_fp16.safetensors"}, + }, + "3": { + "class_type": "VAELoader", + "inputs": {"vae_name": "minimax_h3_audio_vae_fp32.safetensors"}, + }, + "4": { + "class_type": "VAEDecode", + "inputs": {"samples": ["1", 0], "vae": ["2", 0]}, + }, + "5": { + "class_type": "VAEDecodeAudio", + "inputs": {"samples": ["1", 1], "vae": ["3", 0]}, + }, + "6": { + "class_type": "H3ContinuumAssembleSeamV35", + "inputs": { + "images": ["4", 0], + "audio": ["5", 0], + "assembly_plan": ["1", 2], + "exact_total_duration": True, + "audio_seam": "Auto", + "video_seam": "Auto", + "buffer_backend": str(backend), + "diagnostics": "Detailed Report", + }, + }, + "7": { + "class_type": "P25DigestOutput", + "inputs": { + "images": ["6", 0], + "audio": ["6", 1], + "report": ["6", 2], + "backend": str(backend), + }, + }, + } + if interrupt_after_allocation: + prompt["11"] = { + "class_type": "P25InterruptAfterAllocation", + "inputs": { + "images": ["4", 0], + "audio": ["5", 0], + "assembly_plan": ["1", 2], + }, + } + prompt["6"]["inputs"].update( + { + "images": ["11", 0], + "audio": ["11", 1], + "assembly_plan": ["11", 2], + } + ) + if downstream: + prompt.update( + { + "8": { + "class_type": "SaveImage", + "inputs": { + "images": ["6", 0], + "filename_prefix": "P25/Save", + }, + }, + "9": { + "class_type": "PreviewImage", + "inputs": {"images": ["6", 0]}, + }, + "10": { + "class_type": "P25VHSVideoCombine", + "inputs": { + "images": ["6", 0], + "audio": ["6", 1], + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "P25/VHS", + "format": "video/h264-mp4", + "pingpong": False, + "save_output": True, + }, + }, + } + ) + return prompt + + +def _downstream_result(history: dict[str, Any]) -> dict[str, Any]: + outputs = history.get("outputs", {}) + save_items = outputs.get("8", {}).get("images", []) + preview_items = outputs.get("9", {}).get("images", []) + video_items = outputs.get("10", {}).get("gifs", []) + if len(save_items) != 360 or len(preview_items) != 360 or len(video_items) != 1: + raise ProbeFailure( + "unexpected downstream output counts: " + f"save={len(save_items)} preview={len(preview_items)} vhs={len(video_items)}" + ) + video_path = Path(video_items[0]["fullpath"]) + if not video_path.is_file() or video_path.stat().st_size <= 0: + raise ProbeFailure(f"VHS output is missing: {video_path}") + ffprobe = shutil.which("ffprobe") + if ffprobe is None: + raise ProbeFailure("ffprobe is required for P2-5 downstream acceptance") + inspected = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-count_frames", + "-show_entries", + ( + "stream=codec_type,codec_name,width,height,r_frame_rate," + "nb_read_frames,sample_rate,channels,duration:format=duration,size" + ), + "-of", + "json", + str(video_path), + ], + check=True, + capture_output=True, + text=True, + ) + media = json.loads(inspected.stdout) + video_stream = next( + (item for item in media.get("streams", []) if item.get("codec_type") == "video"), + None, + ) + audio_stream = next( + (item for item in media.get("streams", []) if item.get("codec_type") == "audio"), + None, + ) + if video_stream is None or audio_stream is None: + raise ProbeFailure("VHS output does not contain both video and audio") + expected = ( + int(video_stream.get("width", 0)) == 640 + and int(video_stream.get("height", 0)) == 640 + and int(video_stream.get("nb_read_frames", 0)) == 360 + and video_stream.get("r_frame_rate") == "24/1" + and int(audio_stream.get("sample_rate", 0)) == 32_000 + ) + if not expected: + raise ProbeFailure(f"unexpected VHS media contract: {media}") + return { + "save_frames": len(save_items), + "preview_frames": len(preview_items), + "vhs_file": str(video_path), + "vhs_bytes": int(video_path.stat().st_size), + "video_codec": video_stream.get("codec_name"), + "video_frames": int(video_stream["nb_read_frames"]), + "video_fps": video_stream["r_frame_rate"], + "video_duration": float(video_stream["duration"]), + "audio_codec": audio_stream.get("codec_name"), + "audio_sample_rate": int(audio_stream["sample_rate"]), + "container_duration": float(media["format"]["duration"]), + } + + +def _clear_prompt() -> dict[str, Any]: + return {"9": {"class_type": "P25ClearOutput", "inputs": {"token": 1}}} + + +def _has_event(executor, wanted: str) -> bool: + return any(event == wanted for event, _data in executor.status_messages) + + +def _cached_nodes(executor) -> set[str]: + for event, data in executor.status_messages: + if event == "execution_cached": + return {str(value) for value in data.get("nodes", [])} + return set() + + +def _run_backend(args, package_root: Path) -> dict[str, Any]: + _install_paths(args.comfy_root) + import folder_paths + import nodes as core_nodes + import server as comfy_server + from execution import CacheType, PromptExecutor + + if not hasattr(comfy_server.PromptServer, "instance"): + comfy_server.PromptServer.instance = SimpleNamespace( + prompt_queue=SimpleNamespace(currently_running={}, put=lambda _item: None), + number=0, + ) + + work_root = args.work_root.resolve() + output_root = work_root / "output" + temp_root = work_root / "temp" + backing_root = temp_root / "h3-continuum-v35-file-backed" + for path in (output_root, temp_root, backing_root): + path.mkdir(parents=True, exist_ok=True) + + package = _load_package(package_root) + results: dict[str, Any] = {"backing_root": str(backing_root)} + video_combine = None + if args.downstream: + from videohelpersuite.nodes import VideoCombine + + video_combine = VideoCombine + _register_probe_nodes( + core_nodes, + package, + args.revision_root, + results, + video_combine=video_combine, + stress_size=int(args.stress_size), + ) + buffers = importlib.import_module(f"{package.__name__}.v3.file_backed_buffer") + + folder_paths.set_output_directory(str(output_root)) + folder_paths.set_temp_directory(str(temp_root)) + + prompt = _prompt( + args.backend, + downstream=bool(args.downstream), + interrupt_after_allocation=bool(args.interrupt_after_allocation), + stress=bool(args.stress_size), + ) + (work_root / "p25_prompt_graph.json").write_text( + json.dumps(prompt, indent=2, sort_keys=True), + encoding="utf-8", + ) + executor = PromptExecutor( + _Server(), + cache_type=CacheType.CLASSIC, + cache_args={"ram": 64.0, "ram_inactive": 0.0}, + ) + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats() + monitor = _ResourceMonitor() + monitor.start() + started = time.perf_counter() + execute_outputs = ["7", "8", "9", "10"] if args.downstream else ["7"] + executor.execute( + prompt, + f"p25-{args.backend.lower()}", + extra_data={}, + execute_outputs=execute_outputs, + ) + cache_requeue = None + if args.cache_requeue and executor.success: + first_pairs = ( + len(_managed_files(backing_root, "bin")), + len(_managed_files(backing_root, "json")), + ) + executor.execute( + prompt, + f"p25-{args.backend.lower()}-requeue", + extra_data={}, + execute_outputs=execute_outputs, + ) + cached = _cached_nodes(executor) + second_pairs = ( + len(_managed_files(backing_root, "bin")), + len(_managed_files(backing_root, "json")), + ) + cache_requeue = { + "cached_nodes": sorted(cached), + "assembler_cached": "6" in cached, + "first_pair_count": list(first_pairs), + "second_pair_count": list(second_pairs), + "pair_count_bounded": first_pairs == second_pairs, + } + if not executor.success or not cache_requeue["assembler_cached"]: + raise ProbeFailure(f"large-output requeue cache miss: {cache_requeue}") + if not cache_requeue["pair_count_bounded"]: + raise ProbeFailure(f"large-output requeue grew backing files: {cache_requeue}") + elapsed = time.perf_counter() - started + resources = monitor.stop() + manager = buffers.get_file_backed_image_manager(backing_root) + if args.interrupt_after_allocation: + interrupted = _has_event(executor, "execution_interrupted") + core_nodes.interrupt_processing(False) + executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"]) + del executor + _collect_until_empty(manager, backing_root) + result = { + "backend": args.backend, + "stage": "interrupt-after-allocation", + "status": "PASS" if interrupted else "FAIL", + "execution_interrupted": bool(interrupted), + "allocation_observed_before_interrupt": bool( + results.get("interrupt_observed_allocation", False) + ), + "fixture": results.get("fixture"), + "elapsed_seconds": round(elapsed, 3), + "managed_bin_after_interrupt": len(_managed_files(backing_root, "bin")), + "managed_json_after_interrupt": len(_managed_files(backing_root, "json")), + "resources": resources, + "work_root": str(work_root), + } + if not interrupted or not result["allocation_observed_before_interrupt"]: + raise ProbeFailure(f"post-allocation interrupt was not proven: {result}") + (work_root / "p25_interrupt_result.json").write_text( + json.dumps(result, indent=2, sort_keys=True), + encoding="utf-8", + ) + return result + if not executor.success or _has_event(executor, "execution_error"): + raise ProbeFailure(f"{args.backend} PromptExecutor failed") + if "digest" not in results or "fixture" not in results: + raise ProbeFailure(f"{args.backend} probe outputs were not captured") + + bin_files = _managed_files(backing_root, "bin") + json_files = _managed_files(backing_root, "json") + resources.update( + { + "elapsed_seconds": round(elapsed, 3), + "cuda_peak_allocated_bytes": ( + int(torch.cuda.max_memory_allocated()) if torch.cuda.is_available() else 0 + ), + "cuda_peak_reserved_bytes": ( + int(torch.cuda.max_memory_reserved()) if torch.cuda.is_available() else 0 + ), + "backing_bin_count_while_cached": len(bin_files), + "backing_json_count_while_cached": len(json_files), + "backing_bytes_while_cached": sum(item.stat().st_size for item in bin_files), + "pre_digest_memory": results.get("pre_digest_memory"), + } + ) + downstream = _downstream_result(executor.history_result) if args.downstream else None + executor.execute(_clear_prompt(), "p25-clear", extra_data={}, execute_outputs=["9"]) + del executor + _collect_until_empty(manager, backing_root) + resources["managed_bin_after_cache_release"] = len( + _managed_files(backing_root, "bin") + ) + resources["managed_json_after_cache_release"] = len( + _managed_files(backing_root, "json") + ) + resources["nonmanaged_root_entries_after_cache_release"] = len( + [ + item + for item in backing_root.iterdir() + if item.name != ".h3c-p20.lock" + ] + ) + result = { + "backend": args.backend, + "status": "PASS", + "fixture": results["fixture"], + "stress": results.get("stress"), + "digest": results["digest"], + "resources": resources, + "cache_requeue": cache_requeue, + "downstream": downstream, + "work_root": str(work_root), + } + (work_root / "p25_backend_result.json").write_text( + json.dumps(result, indent=2, sort_keys=True), + encoding="utf-8", + ) + return result + + +def _last_json(stdout: str) -> dict[str, Any]: + marker = "P25_RESULT_JSON=" + line = next( + (value for value in reversed(stdout.splitlines()) if value.startswith(marker)), + "", + ) + if not line: + raise ProbeFailure("backend child did not emit a result marker") + return json.loads(line[len(marker) :]) + + +def _summarize_results(work_root: Path, results: dict[str, Any] | None = None): + work_root = work_root.resolve() + if results is None: + results = { + "RAM": json.loads( + (work_root / "ram" / "p25_backend_result.json").read_text( + encoding="utf-8" + ) + ), + "Disk-backed": json.loads( + (work_root / "disk" / "p25_backend_result.json").read_text( + encoding="utf-8" + ) + ), + } + ram = results["RAM"] + disk = results["Disk-backed"] + disk_backing_root = ( + work_root / "disk" / "temp" / "h3-continuum-v35-file-backed" + ) + managed_bin_after = len(_managed_files(disk_backing_root, "bin")) + managed_json_after = len(_managed_files(disk_backing_root, "json")) + disk["resources"]["managed_bin_after_cache_release"] = managed_bin_after + disk["resources"]["managed_json_after_cache_release"] = managed_json_after + comparisons = { + "fixture_equal": ram["fixture"] == disk["fixture"], + "video_shape_equal": ram["digest"]["video_shape"] == disk["digest"]["video_shape"], + "video_sha256_equal": ram["digest"]["video_sha256"] == disk["digest"]["video_sha256"], + "audio_shape_equal": ram["digest"]["audio_shape"] == disk["digest"]["audio_shape"], + "audio_sha256_equal": ram["digest"]["audio_sha256"] == disk["digest"]["audio_sha256"], + "audio_sample_rate_equal": ram["digest"]["audio_sample_rate"] == disk["digest"]["audio_sample_rate"], + "both_finite": bool( + ram["digest"]["video_finite"] + and disk["digest"]["video_finite"] + and ram["digest"]["audio_finite"] + and disk["digest"]["audio_finite"] + ), + "disk_backing_reclaimed": managed_bin_after == 0 and managed_json_after == 0, + } + if not all(comparisons.values()): + raise ProbeFailure(f"RAM/Disk-backed comparison failed: {comparisons}") + peak_delta = ( + int(ram["resources"]["peak_rss_bytes"]) + - int(disk["resources"]["peak_rss_bytes"]) + ) + result = { + "phase": "P2-5", + "stage": "small-long-terminal-ab", + "status": "PASS", + "gpu_used": True, + "comparisons": comparisons, + "rss_peak_reduction_bytes": peak_delta, + "rss_peak_reduction_percent": round( + peak_delta / max(1, int(ram["resources"]["peak_rss_bytes"])) * 100.0, + 3, + ), + "backends": results, + "work_root": str(work_root), + } + (work_root / "p25_result.json").write_text( + json.dumps(result, indent=2, sort_keys=True), + encoding="utf-8", + ) + return result + + +def _run_parent(args, package_root: Path) -> dict[str, Any]: + work_root = args.work_root.resolve() + work_root.mkdir(parents=True, exist_ok=True) + common = [ + sys.executable, + str(Path(__file__).resolve()), + "--comfy-root", + str(args.comfy_root), + "--package-root", + str(package_root), + "--revision-root", + str(args.revision_root), + ] + results = {} + for backend, folder in (("RAM", "ram"), ("Disk-backed", "disk")): + child = subprocess.run( + [*common, "--work-root", str(work_root / folder), "--backend", backend], + check=True, + capture_output=True, + text=True, + ) + (work_root / f"{folder}_stdout.txt").write_text(child.stdout, encoding="utf-8") + (work_root / f"{folder}_stderr.txt").write_text(child.stderr, encoding="utf-8") + results[backend] = _last_json(child.stdout) + return _summarize_results(work_root, results) + + +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--comfy-root", type=Path, required=True) + parser.add_argument("--package-root", type=Path) + parser.add_argument("--revision-root", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + parser.add_argument("--backend", choices=("Auto", "RAM", "Disk-backed")) + parser.add_argument("--summarize-existing", action="store_true") + parser.add_argument("--downstream", action="store_true") + parser.add_argument("--interrupt-after-allocation", action="store_true") + parser.add_argument("--cache-requeue", action="store_true") + parser.add_argument( + "--stress-size", + type=int, + default=0, + help="Use a synthetic decoded square fixture of this size; 1536 is 9.49 GiB.", + ) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + args.comfy_root = args.comfy_root.resolve() + args.revision_root = args.revision_root.resolve() + args.work_root = args.work_root.resolve() + package_root = ( + args.package_root.resolve() + if args.package_root is not None + else Path(__file__).resolve().parents[1] + ) + if args.backend is not None: + result = _run_backend(args, package_root) + print("P25_RESULT_JSON=" + json.dumps(result, sort_keys=True)) + return 0 + if args.summarize_existing: + result = _summarize_results(args.work_root) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + result = _run_parent(args, package_root) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_runtime.py b/tools/verify_runtime.py index 9a571b8..38a0f64 100644 --- a/tools/verify_runtime.py +++ b/tools/verify_runtime.py @@ -67,7 +67,12 @@ def main() -> int: current_required = { "H3ContinuumSamplerV34", + "H3ContinuumSamplerV35", "H3ContinuumAssembleSeamV34", + "H3ContinuumAssembleSeamV35", + "H3ContinuumHiResFixV35", + "H3ContinuumLatentResizeV35", + "H3ContinuumSecondPassV35", "H3ContinuumClipOverrides", "H3ContinuumResult", } diff --git a/v2/nodes.py b/v2/nodes.py index 92ae064..b06c070 100644 --- a/v2/nodes.py +++ b/v2/nodes.py @@ -311,6 +311,8 @@ class H3ContinuumSamplerV2: driving_audio_vae=None, reference_video_source=None, timeline_video_source=None, + capture_refine_context=False, + memory_attribution=False, **clip_prompt_inputs, ): ( @@ -383,6 +385,8 @@ class H3ContinuumSamplerV2: driving_audio_vae=driving_audio_vae, reference_video_source=reference_video_source, timeline_video_source=timeline_video_source, + capture_refine_context=bool(capture_refine_context), + memory_attribution=bool(memory_attribution), ) diff --git a/v2/sequence.py b/v2/sequence.py index 07e9c48..b96b3f5 100644 --- a/v2/sequence.py +++ b/v2/sequence.py @@ -339,11 +339,22 @@ def _attach_terminal_flf_keyframes( metadata["minimax_frame_count"] = int(frame_count) return conditioning -def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,sigmas:torch.Tensor,first_frame:torch.Tensor|None,last_frame:torch.Tensor|None,prompt_plan:dict[str,Any],width:int,height:int,continuity:str,base_seed:int,audio_continuity:bool,exact_total_duration:bool,diagnostics_mode:str,reroll_from_chunk:int,reroll_nonce:int,strict_compatibility:bool,debug:bool,seam_correction:str=SEAM_CORRECTION_OFF,enable_preview:bool=True,session:dict[str,Any]|None=None,initial_state:dict[str,Any]|None=None,latent_only:bool=False,reference_assets=None,reference_audio_source=None,reference_audio_vae=None,driving_audio_source=None,driving_audio_vae=None,reference_video_source=None,timeline_video_source=None): +def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,sigmas:torch.Tensor,first_frame:torch.Tensor|None,last_frame:torch.Tensor|None,prompt_plan:dict[str,Any],width:int,height:int,continuity:str,base_seed:int,audio_continuity:bool,exact_total_duration:bool,diagnostics_mode:str,reroll_from_chunk:int,reroll_nonce:int,strict_compatibility:bool,debug:bool,seam_correction:str=SEAM_CORRECTION_OFF,enable_preview:bool=True,session:dict[str,Any]|None=None,initial_state:dict[str,Any]|None=None,latent_only:bool=False,reference_assets=None,reference_audio_source=None,reference_audio_vae=None,driving_audio_source=None,driving_audio_vae=None,reference_video_source=None,timeline_video_source=None,capture_refine_context:bool=False,memory_attribution:bool=False,_memory_attribution_collector:Any=None): from ..conditioning import detect_conditioning_mode, conditioning_display_label from ..run_storage import get_active_run_storage storage_controller=get_active_run_storage() - diagnostics_mode=normalize_diagnostics_mode(diagnostics_mode); plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height) + capture_refine_context=bool(capture_refine_context and latent_only) + memory_attribution=bool(memory_attribution and capture_refine_context) + memory_collector=None + refine_groups=[] + make_refine_group=make_refine_context=None + if bool(capture_refine_context): + from ..v3.refine_context import make_refine_context, make_refine_group + diagnostics_mode=normalize_diagnostics_mode(diagnostics_mode); memory_collector=_memory_attribution_collector if memory_attribution and diagnostics_mode==DIAGNOSTICS_FULL else None; capture_memory=None + if memory_collector is not None: + from ..v3.memory_attribution import capture_attribution_fail_soft + capture_memory=capture_attribution_fail_soft + plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height) # Legacy workflow input only. Runtime compatibility is advisory in V3.4. strict_compatibility=False if width<=0 or height<=0 or width%32 or height%32: raise SequenceRuntimeError("width and height must be positive multiples of 32") @@ -504,7 +515,42 @@ def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,si driving_audio_latent=slice_driving_audio_latent(driving_audio_assets,cumulative_retained_before=retained_frames,total_frames=int(chunk_plan["total_frames"]),trim_frames=int(chunk_plan["trim_frames"]),fps=FPS) conditioning=attach_driving_audio(conditioning,driving_audio_latent) chunk_model=clone_model_for_chunk(model,strict=bool(strict_compatibility),debug=bool(debug),chunk_index=clip_index,context_frames=context_frames if previous_state is not None else None) + if bool(capture_refine_context): + from ..state import extract_av_streams + source_video,_=extract_av_streams(latent) + refine_groups.append(make_refine_group( + conditioning=conditioning, + group_id=len(refine_groups), + logical_chunks=(sequence_index+1,), + physical_frames=int(chunk_plan["total_frames"]), + prompt_policy="single", + physical_prompt=prompt, + source_video_shape=tuple(int(value) for value in source_video.shape), + physical_clip_index=int(clip_index), + context_frames=int(context_frames), + first_image=assets.first_image, + last_image=assets.last_image if include_chunk_last else None, + )) + del source_video,_ + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=sequence_index+1, + logical_chunks=(sequence_index+1,), + stage="before sampling", + retained_entries=entries, + observed_latent=latent, + retained_refine_context=refine_groups, + ) sampled=sample_chunk(model=chunk_model,conditioning=conditioning,latent=latent,sampler=sampler,sigmas=sigmas,seed=seed,enable_preview=bool(enable_preview)) + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=sequence_index+1, + logical_chunks=(sequence_index+1,), + stage="after sampling", + retained_entries=entries, + observed_latent=sampled, + retained_refine_context=refine_groups, + ) if context_before is not None and video_context is not None: assert_context_unchanged(video_context,audio_context,context_before) entry=make_chunk_entry(latent=sampled,plan=chunk_plan,prompt=prompt,prompt_hash=prompt_hash_value,seed=seed,context_frames=context_frames,motion_score=motion_score,reused=False); previous_state=entry_to_state(entry); entries.append(entry) _record_context_diagnostics(tracker=context_diagnostics,reports=sampling_reports,state=previous_state,continuity=continuity,reused=False) @@ -513,6 +559,14 @@ def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,si sampling_reports.append(f"chunk {sequence_index+1}/{chunks}: seed={seed}, frames={chunk_plan['total_frames']}, trim={chunk_plan['trim_frames']}, retained_total={retained_frames}, context={context_frames} ({reason}), motion={motion_score:.6f}, "+(f"interop=emitted actual_prefix={CONTINUUM_ACTUAL_PREFIX_STEPS} consumer=not_observable" if context_before is not None else "interop=not_emitted")) del sampled,latent,conditioning,chunk_model,chunk_cache if timeline_video_source is not None and timeline_video_assets is not None: del timeline_video_assets + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=sequence_index+1, + logical_chunks=(sequence_index+1,), + stage="after CPU commit", + retained_entries=entries, + retained_refine_context=refine_groups, + ) if terminal_merge_pending: pair_start=chunks-2 if len(entries)!=pair_start: @@ -543,7 +597,44 @@ def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,si driving_audio_latent=slice_driving_audio_latent(driving_audio_assets,cumulative_retained_before=retained_frames,total_frames=physical_frames,trim_frames=physical_context_frames,fps=FPS) conditioning=attach_driving_audio(conditioning,driving_audio_latent) chunk_model=clone_model_for_chunk(model,strict=bool(strict_compatibility),debug=bool(debug),chunk_index=physical_clip_index,context_frames=physical_context_frames if not initial_pair else None) + if bool(capture_refine_context): + from ..state import extract_av_streams + source_video,_=extract_av_streams(latent) + refine_groups.append(make_refine_group( + conditioning=conditioning, + group_id=len(refine_groups), + logical_chunks=(pair_start+1,chunks), + physical_frames=physical_frames, + prompt_policy=str(terminal_prompt_policy), + physical_prompt=str(prompt), + source_video_shape=tuple(int(value) for value in source_video.shape), + physical_clip_index=int(physical_clip_index), + context_frames=int(physical_context_frames), + first_image=assets.first_image, + last_image=assets.last_image, + )) + del source_video,_ + terminal_physical_group=pair_start+1 + terminal_logical_chunks=(pair_start+1,chunks) + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=terminal_physical_group, + logical_chunks=terminal_logical_chunks, + stage="before sampling", + retained_entries=entries, + observed_latent=latent, + retained_refine_context=refine_groups, + ) sampled=sample_chunk(model=chunk_model,conditioning=conditioning,latent=latent,sampler=sampler,sigmas=sigmas,seed=physical_seed,enable_preview=bool(enable_preview)) + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=terminal_physical_group, + logical_chunks=terminal_logical_chunks, + stage="after sampling", + retained_entries=entries, + observed_latent=sampled, + retained_refine_context=refine_groups, + ) if context_before is not None and video_context is not None: assert_context_unchanged(video_context,audio_context,context_before) sampled_video,sampled_audio=latent_to_cpu(sampled) @@ -569,6 +660,15 @@ def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,si sampling_reports.append(f"chunk {sequence_index+1}/{chunks}: seed={logical_seed}, frames={total_frames}, trim={trim_frames}, retained_total={retained_frames}, context={trim_frames}, shared_physical_sample=terminal_10s_seed_v2") del logical_latent del sampled,sampled_video,sampled_audio,logical_parts,latent,conditioning,chunk_model + del video_part,audio_part + if memory_collector is not None: + capture_memory(memory_collector, "capture_group", + physical_group=terminal_physical_group, + logical_chunks=terminal_logical_chunks, + stage="after CPU commit", + retained_entries=entries, + retained_refine_context=refine_groups, + ) if len(entries)!=chunks: raise SequenceRuntimeError(f"internal sequence length mismatch: expected {chunks}, got {len(entries)}") if latent_only: last_state=entry_to_state(entries[-1]); parent_id=session.get("session_id") if session is not None else None @@ -582,7 +682,27 @@ def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,si report_lines=[f"H3 Continuum V3 {PACKAGE_VERSION}",f"Conditioning mode: {conditioning_display}.",prompt_plan_report(plan),"Decode: external ComfyUI Core VAE nodes; full raw AV chunks retained.",accelerators,"Execution: conditioning precomputed; single call-local MODEL clone per chunk; no internal VAE decode.",*reuse_notes] if diagnostics_mode!=DIAGNOSTICS_OFF: report_lines.extend(sampling_reports) report_lines.extend([session_summary(new_session),f"Output: {len(entries)} raw AV latent chunk(s); connect Core VAE Decode nodes, then H3 Continuum Assemble V3."]) - return entries,last_state,new_session,"\n".join(report_lines) + outputs=(entries,last_state,new_session,"\n".join(report_lines)) + if bool(capture_refine_context): + expected_groups=chunks-(1 if terminal_merge_enabled else 0) + complete=len(refine_groups)==expected_groups + notes=[] + if not complete: + notes.append( + "Refine context is incomplete because accepted session/Run Storage " + f"entries were reused; captured {len(refine_groups)} of " + f"{expected_groups} physical sampling groups." + ) + raw_refine_context=make_refine_context( + refine_groups, + source_width=width, + source_height=height, + conditioning_mode=conditioning_mode, + complete=complete, + notes=notes, + ) + return (*outputs,raw_refine_context) + return outputs if seam_correction==SEAM_CORRECTION_OFF: images,audio,decode_reports=decode_sequence(entries=entries,video_vae=video_vae,audio_vae=audio_vae,diagnostics_full=diagnostics_mode==DIAGNOSTICS_FULL) else: images,audio,decode_reports=decode_sequence_with_seam(entries=entries,video_vae=video_vae,audio_vae=audio_vae,diagnostics_mode=diagnostics_mode,automatic=seam_correction==SEAM_CORRECTION_AUTO) duration_report="" diff --git a/v3/assembly_v35.py b/v3/assembly_v35.py new file mode 100644 index 0000000..c4dc776 --- /dev/null +++ b/v3/assembly_v35.py @@ -0,0 +1,720 @@ +"""V3.5 direct-write assembly with a manual RAM or file-backed IMAGE buffer.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from dataclasses import dataclass +import math +from pathlib import Path +import time +from typing import Any, Callable, Mapping + +import torch + +from ..constants import ( + DIAGNOSTICS_FULL, + DIAGNOSTICS_OFF, + normalize_diagnostics_mode, +) +from ..hardening import assemble_with_hardening +from ..media import validate_audio +from ..v2.seam_guard import correct_audio_seam +from .assembly import ( + AUDIO_SEAM_AUTO, + AUDIO_SEAM_OPTIONS, + VIDEO_SEAM_ANALYZE, + VIDEO_SEAM_ANALYSIS_OPTIONS, + VIDEO_SEAM_AUTO, + VIDEO_SEAM_AUTO_2, + VIDEO_SEAM_OFF, +) +from .file_backed_buffer import ( + BUFFER_BACKEND_AUTO, + BUFFER_BACKEND_DISK, + BUFFER_BACKEND_OPTIONS, + BUFFER_BACKEND_RAM, + allocate_file_backed_image, + format_auto_buffer_decision, + select_auto_buffer_backend, +) +from .plan import FPS, validate_assembly_plan + + +_COPY_BATCH_FRAMES = 8 + + +def _check_interrupted( + interrupt_check: Callable[[], None] | None, +) -> None: + """Honor ComfyUI's normal ESC interrupt without requiring Core in tests.""" + + if interrupt_check is not None: + interrupt_check() + return + try: + from comfy.model_management import throw_exception_if_processing_interrupted + except ModuleNotFoundError: + return + throw_exception_if_processing_interrupted() + + +@dataclass(frozen=True) +class CopySpan: + """Map a half-open natural-timeline interval into the output buffer.""" + + source_start: int + source_stop: int + destination_start: int + + +class _RamAllocation(AbstractContextManager): + """Match the provisional file-backed allocation interface for RAM.""" + + def __init__(self, shape: tuple[int, ...], dtype: torch.dtype): + self.tensor = torch.empty(shape, dtype=dtype, device="cpu") + self.record = None + self._published = False + + def publish(self) -> torch.Tensor: + if self._published: + raise RuntimeError("RAM assembly buffer was already published") + self._published = True + return self.tensor + + def __exit__(self, exc_type, exc, traceback) -> bool: + return False + + +def _video_copy_spans( + *, + natural_frames: int, + output_frames: int, + preserve_final_frame: bool, +) -> tuple[CopySpan, ...]: + if output_frames >= natural_frames: + return (CopySpan(0, natural_frames, 0),) + if preserve_final_frame and output_frames >= 2: + return ( + CopySpan(0, output_frames - 1, 0), + CopySpan(natural_frames - 1, natural_frames, output_frames - 1), + ) + return (CopySpan(0, output_frames, 0),) + + +def _copy_natural_interval( + destination: torch.Tensor, + source: torch.Tensor, + *, + natural_start: int, + spans: tuple[CopySpan, ...], + interrupt_check: Callable[[], None] | None = None, +) -> None: + natural_stop = natural_start + int(source.shape[0]) + for span in spans: + overlap_start = max(natural_start, int(span.source_start)) + overlap_stop = min(natural_stop, int(span.source_stop)) + if overlap_stop <= overlap_start: + continue + source_start = overlap_start - natural_start + destination_start = int(span.destination_start) + ( + overlap_start - int(span.source_start) + ) + frame_count = overlap_stop - overlap_start + for offset in range(0, frame_count, _COPY_BATCH_FRAMES): + _check_interrupted(interrupt_check) + batch_frames = min(_COPY_BATCH_FRAMES, frame_count - offset) + source_batch_start = source_start + offset + destination_batch_start = destination_start + offset + destination[ + destination_batch_start : destination_batch_start + batch_frames + ].copy_( + source[source_batch_start : source_batch_start + batch_frames] + ) + + +def _repeat_final_video_frame( + destination: torch.Tensor, + *, + natural_frames: int, + output_frames: int, + interrupt_check: Callable[[], None] | None, +) -> None: + for destination_start in range( + natural_frames, + output_frames, + _COPY_BATCH_FRAMES, + ): + _check_interrupted(interrupt_check) + destination_stop = min( + output_frames, + destination_start + _COPY_BATCH_FRAMES, + ) + destination[destination_start:destination_stop].copy_( + destination[natural_frames - 1 : natural_frames].expand( + destination_stop - destination_start, + *destination.shape[1:], + ) + ) + + +def _copy_audio_segment( + destination: torch.Tensor, + waveform: torch.Tensor, + *, + trim_samples: int, +) -> None: + """Reproduce timeline slicing/padding without concatenating a waveform.""" + + wanted = int(destination.shape[-1]) + trim_samples = max(0, int(trim_samples)) + available = max(0, min(wanted, int(waveform.shape[-1]) - trim_samples)) + if available > 0: + destination[..., :available].copy_( + waveform[..., trim_samples : trim_samples + available] + ) + if available >= wanted: + return + if available > 0: + destination[..., available:].copy_( + destination[..., available - 1 : available].expand( + *destination.shape[:-1], + wanted - available, + ) + ) + else: + destination.zero_() + + +def adjust_audio_duration_direct( + audio: Mapping[str, Any], + *, + target_frames: int, + preserve_final_audio_anchor: bool, +) -> tuple[dict[str, Any], int]: + """Apply the legacy audio trim/pad values using direct allocation and copy.""" + + waveform, sample_rate = validate_audio(audio) + waveform = waveform.detach().to(device="cpu") + sample_rate = int(sample_rate) + target_samples = int(round(int(target_frames) / FPS * sample_rate)) + original_samples = int(waveform.shape[-1]) + delta_samples = target_samples - original_samples + if delta_samples == 0: + return {"waveform": waveform, "sample_rate": sample_rate}, 0 + + output = torch.empty( + (*waveform.shape[:-1], target_samples), + dtype=waveform.dtype, + device="cpu", + ) + if delta_samples < 0: + if preserve_final_audio_anchor and target_samples > 1: + tail_samples = min( + max(1, int(round(sample_rate / FPS))), + target_samples, + original_samples, + ) + head_samples = target_samples - tail_samples + if head_samples > 0: + output[..., :head_samples].copy_(waveform[..., :head_samples]) + output[..., head_samples:].copy_(waveform[..., -tail_samples:]) + fade = min( + int(round(sample_rate * 0.005)), + head_samples, + tail_samples, + ) + if fade > 0: + start = output[..., head_samples - 1 : head_samples].to( + dtype=output.dtype + ) + alpha = torch.linspace( + 0.0, + 1.0, + fade, + dtype=output.dtype, + device=output.device, + ).reshape(*([1] * (output.ndim - 1)), fade) + tail_head = output[..., head_samples : head_samples + fade] + tail_head.copy_(start * (1.0 - alpha) + tail_head * alpha) + else: + output.copy_(waveform[..., :target_samples]) + else: + if original_samples > 0: + output[..., :original_samples].copy_(waveform) + output[..., original_samples:].copy_( + waveform[..., -1:].expand( + *waveform.shape[:-1], + delta_samples, + ) + ) + else: + output.zero_() + return {"waveform": output, "sample_rate": sample_rate}, delta_samples + + +def _prepare_video_seam( + *, + images: list[Any], + plan: dict[str, Any], + mode: str, +) -> tuple[list[Any], dict[int, torch.Tensor], dict[int, str], Exception | None]: + if mode == VIDEO_SEAM_OFF: + return [], {}, {}, None + analyses: list[Any] = [] + try: + from .video_seam import analyze_decoded_boundaries + + analyses = analyze_decoded_boundaries( + images=images, + assembly_plan=plan, + ) + patches: dict[int, torch.Tensor] = {} + actions: dict[int, str] = {} + if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2): + from .video_seam_v35 import build_boundary_patches + + patches, actions = build_boundary_patches( + images=images, + assembly_plan=plan, + analyses=analyses, + enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2, + ) + return analyses, patches, actions, None + except Exception as exc: + # Match V3.4's all-native fallback: no partial patch is retained. + return analyses, {}, {}, exc + + +def _video_seam_status(mode: str) -> str: + if mode == VIDEO_SEAM_OFF: + return "Video seam correction is disabled." + if mode == VIDEO_SEAM_ANALYZE: + return "Video Seam: Analyze Only; decoded frames and audio are unchanged." + if mode == VIDEO_SEAM_AUTO: + return "Video Seam: Auto; guarded transient and micro-flash correction enabled." + return ( + "Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, " + "and exposure-ramp correction enabled." + ) + + +def _format_video_seam_lines( + *, + mode: str, + analyses: list[Any], + actions: dict[int, str], + error: Exception | None, +) -> list[str]: + if mode == VIDEO_SEAM_OFF: + return [] + if error is not None: + return [ + f"Video Seam {mode}: native output preserved; analysis unavailable " + f"({type(error).__name__}: {error})" + ] + from .video_seam import format_video_boundary_analysis + + lines = [ + format_video_boundary_analysis( + item, + action=( + "analysis only" + if mode == VIDEO_SEAM_ANALYZE + else actions.get(item.boundary_index, "kept native boundary") + ), + ) + for item in analyses + ] + if not lines: + lines.append(f"Video Seam {mode}: no decoded chunk boundary to analyze.") + return lines + + +def _format_elapsed(seconds: float) -> str: + total = max(0.0, float(seconds)) + hours, remainder = divmod(total, 3600.0) + minutes, seconds = divmod(remainder, 60.0) + return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}" + + +def _write_assembly( + *, + image_buffer: torch.Tensor, + images: list[Any], + audio: list[Any], + plan: dict[str, Any], + exact_total_duration: bool, + audio_seam: str, + video_seam: str, + diagnostics: str, + natural_frames: int, + output_frames: int, + video_spans: tuple[CopySpan, ...], + seam_patches: dict[int, torch.Tensor], + seam_analyses: list[Any], + seam_actions: dict[int, str], + seam_error: Exception | None, + buffer_backend: str, + auto_decision_report: str | None, + final_audio_override: Mapping[str, Any] | None, + final_audio_source: str | None, + interrupt_check: Callable[[], None] | None, +) -> tuple[dict[str, Any], str]: + diagnostics_mode = normalize_diagnostics_mode(diagnostics) + units = list(plan.get("decode_groups") or plan["chunks"]) + first_waveform, audio_rate = validate_audio(audio[0]) + first_waveform = first_waveform.detach().to(device="cpu") + audio_rate = int(audio_rate) + natural_samples = int(round(natural_frames / FPS * audio_rate)) + audio_buffer = torch.empty( + (*first_waveform.shape[:-1], natural_samples), + dtype=first_waveform.dtype, + device="cpu", + ) + reports = [ + "H3 Continuum Assemble V3.5", + f"Audio Seam: {audio_seam}. {_video_seam_status(video_seam)}", + f"Buffer Backend: {buffer_backend}; IMAGE direct-write " + f"frames={output_frames}, bytes={image_buffer.numel() * image_buffer.element_size()}; " + "no final full IMAGE copy.", + ] + if auto_decision_report is not None: + reports.append(auto_decision_report) + + frame_cursor = 0 + for index, (raw_images, raw_audio, unit) in enumerate( + zip(images, audio, units), + start=1, + ): + _check_interrupted(interrupt_check) + total_frames = int(unit["total_frames"]) + trim_frames = int(unit["trim_frames"]) + net_frames = int(unit["net_frames"]) + raw_images_cpu = raw_images.detach().to(device="cpu") + retained_images = raw_images_cpu[trim_frames:total_frames] + if int(retained_images.shape[0]) != net_frames: + raise ValueError( + f"decoded image chunk {index} retained {retained_images.shape[0]} " + f"frames; plan expects {net_frames}" + ) + _copy_natural_interval( + image_buffer, + retained_images, + natural_start=frame_cursor, + spans=video_spans, + interrupt_check=interrupt_check, + ) + + _check_interrupted(interrupt_check) + waveform, sample_rate = validate_audio(raw_audio) + waveform = waveform.detach().to(device="cpu") + sample_rate = int(sample_rate) + if sample_rate != audio_rate: + raise ValueError( + f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}" + ) + frame_stop = frame_cursor + net_frames + sample_start = int(round(frame_cursor / FPS * sample_rate)) + sample_stop = int(round(frame_stop / FPS * sample_rate)) + seam_report = None + if index > 1 and audio_seam == AUDIO_SEAM_AUTO: + try: + cut_sample = int(round(trim_frames / FPS * sample_rate)) + patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam( + audio_buffer[..., :sample_start], + waveform, + sample_rate=sample_rate, + cut_sample=cut_sample, + ) + if patch is not None and int(patch.shape[-1]) > 0: + patch_start = sample_start - int(patch.shape[-1]) + if patch_start < 0: + raise ValueError("audio seam patch starts before the output") + audio_buffer[..., patch_start:sample_start].copy_(patch) + seam_report = ( + f"audio seam {index-1}->{index}: " + f"corr={metrics.correlation_before:.4f}->" + f"{metrics.correlation_after:.4f}, " + f"jump={metrics.boundary_jump_before:.6f}->" + f"{metrics.boundary_jump_after:.6f}, " + f"fade={fade_samples} samples" + ) + if diagnostics_mode == DIAGNOSTICS_FULL: + seam_report += ( + f", offset={metrics.offset_samples:+d}, " + f"gain={level_gain:.4f}, dc={dc_bias:+.6f}" + ) + except Exception as exc: + seam_report = ( + f"audio seam {index-1}->{index}: fallback to native boundary " + f"({type(exc).__name__}: {exc})" + ) + + trim_samples = int(round(trim_frames / FPS * sample_rate)) + _copy_audio_segment( + audio_buffer[..., sample_start:sample_stop], + waveform, + trim_samples=trim_samples, + ) + if diagnostics_mode != DIAGNOSTICS_OFF: + reports.append( + f"assembled decoded chunk {index}: {net_frames} retained frames, " + f"cumulative {frame_stop}/{natural_frames}" + ) + if seam_report: + reports.append(seam_report) + frame_cursor = frame_stop + + if frame_cursor != natural_frames: + raise RuntimeError( + f"V3.5 assembly cursor mismatch: {frame_cursor} != {natural_frames}" + ) + + unit_starts: list[int] = [] + cursor = 0 + for unit in units: + unit_starts.append(cursor) + cursor += int(unit["net_frames"]) + for boundary, patch in seam_patches.items(): + _check_interrupted(interrupt_check) + if boundary <= 0 or boundary >= len(unit_starts): + raise ValueError("V3.5 video seam patch boundary is outside decode groups") + _copy_natural_interval( + image_buffer, + patch, + natural_start=unit_starts[boundary], + spans=video_spans, + interrupt_check=interrupt_check, + ) + + if output_frames > natural_frames: + _repeat_final_video_frame( + image_buffer, + natural_frames=natural_frames, + output_frames=output_frames, + interrupt_check=interrupt_check, + ) + + _check_interrupted(interrupt_check) + result_audio: dict[str, Any] = { + "waveform": audio_buffer, + "sample_rate": audio_rate, + } + duration_report = "" + if exact_total_duration: + preserve_anchor = bool( + plan.get("preserve_final_frame", False) + and output_frames < natural_frames + and output_frames >= 2 + ) + result_audio, delta_samples = adjust_audio_duration_direct( + result_audio, + target_frames=output_frames, + preserve_final_audio_anchor=preserve_anchor, + ) + adjustment = output_frames - natural_frames + mode = ( + "final anchor preserved; pre-tail trim" + if preserve_anchor + else "tail trim/pad" + ) + duration_report = ( + f"Exact-duration adjustment: frames {natural_frames}->{output_frames} " + f"({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}." + ) + + driving_audio_report = "" + if final_audio_override is not None: + result_audio = dict(final_audio_override) + if exact_total_duration: + result_audio, _ = adjust_audio_duration_direct( + result_audio, + target_frames=output_frames, + preserve_final_audio_anchor=False, + ) + waveform, selected_rate = validate_audio(result_audio) + result_audio = { + "waveform": waveform, + "sample_rate": int(selected_rate), + } + driving_audio_report = ( + "Driving Audio: preserved source selected from " + f"{final_audio_source or 'direct input'}; " + f"sample_rate={int(selected_rate)}, samples={int(waveform.shape[-1])}; " + "generated audio and Audio Seam bypassed." + ) + + reports.append( + f"Assembled {len(units)} physical decoded group(s) into " + f"{output_frames} frames with cumulative sample-boundary alignment." + ) + if duration_report: + reports.append(duration_report) + if driving_audio_report: + reports.append(driving_audio_report) + runtime_started_at = plan.get("_runtime_started_at") + if isinstance(runtime_started_at, (int, float)): + elapsed = time.perf_counter() - float(runtime_started_at) + if math.isfinite(elapsed) and elapsed >= 0.0: + reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}") + reports.extend( + _format_video_seam_lines( + mode=video_seam, + analyses=seam_analyses, + actions=seam_actions, + error=seam_error, + ) + ) + return result_audio, "\n".join(reports) + + +def _assemble_decoded_chunks_v35_core( + *, + images: list[Any], + audio: list[Any], + assembly_plan: dict[str, Any], + exact_total_duration: bool, + audio_seam: str, + video_seam: str, + diagnostics: str, + buffer_backend: str, + backing_root: str | Path | None = None, + final_audio_override: Mapping[str, Any] | None = None, + final_audio_source: str | None = None, + interrupt_check: Callable[[], None] | None = None, +): + plan = validate_assembly_plan(assembly_plan) + if audio_seam not in AUDIO_SEAM_OPTIONS: + raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}") + if video_seam not in VIDEO_SEAM_ANALYSIS_OPTIONS: + raise ValueError(f"unknown Video Seam mode: {video_seam!r}") + if buffer_backend not in BUFFER_BACKEND_OPTIONS: + raise ValueError(f"unknown Buffer Backend: {buffer_backend!r}") + units = list(plan.get("decode_groups") or plan["chunks"]) + natural_frames = sum(int(unit["net_frames"]) for unit in units) + output_frames = ( + int(plan["target_frames"]) + if bool(exact_total_duration) + else natural_frames + ) + if natural_frames <= 0 or output_frames <= 0: + raise ValueError("V3.5 assembly frame counts must be positive") + first_image = images[0] + shape = (output_frames, *tuple(int(value) for value in first_image.shape[1:])) + video_spans = _video_copy_spans( + natural_frames=natural_frames, + output_frames=output_frames, + preserve_final_frame=bool(plan.get("preserve_final_frame", False)), + ) + analyses, patches, actions, analysis_error = _prepare_video_seam( + images=images, + plan=plan, + mode=video_seam, + ) + + resolved_backend = buffer_backend + resolved_backing_root = backing_root + auto_decision_report = None + if buffer_backend == BUFFER_BACKEND_AUTO: + final_image_bytes = ( + math.prod(shape) * first_image.element_size() + ) + decision = select_auto_buffer_backend( + final_image_bytes, + backing_root=backing_root, + ) + resolved_backend = decision.selected_backend + resolved_backing_root = decision.backing_root + auto_decision_report = format_auto_buffer_decision(decision) + + allocation: Any + if resolved_backend == BUFFER_BACKEND_RAM: + allocation = _RamAllocation(shape, first_image.dtype) + else: + allocation = allocate_file_backed_image( + shape, + dtype=first_image.dtype, + backing_root=resolved_backing_root, + ) + with allocation: + # The file exists at this point, so an ESC received immediately before + # assembly exercises the same unpublished cleanup path as a mid-copy + # cancellation. + _check_interrupted(interrupt_check) + result_audio, report = _write_assembly( + image_buffer=allocation.tensor, + images=images, + audio=audio, + plan=plan, + exact_total_duration=bool(exact_total_duration), + audio_seam=audio_seam, + video_seam=video_seam, + diagnostics=diagnostics, + natural_frames=natural_frames, + output_frames=output_frames, + video_spans=video_spans, + seam_patches=patches, + seam_analyses=analyses, + seam_actions=actions, + seam_error=analysis_error, + buffer_backend=( + resolved_backend + if buffer_backend != BUFFER_BACKEND_AUTO + else f"Auto -> {resolved_backend}" + ), + auto_decision_report=auto_decision_report, + final_audio_override=final_audio_override, + final_audio_source=final_audio_source, + interrupt_check=interrupt_check, + ) + result_images = allocation.publish() + return result_images, result_audio, report + + +def assemble_decoded_chunks_v35( + *, + images: list[Any], + audio: list[Any], + assembly_plan: dict[str, Any], + exact_total_duration: bool, + audio_seam: str, + video_seam: str, + diagnostics: str, + buffer_backend: str, + backing_root: str | Path | None = None, + final_audio_override: Mapping[str, Any] | None = None, + final_audio_source: str | None = None, + interrupt_check: Callable[[], None] | None = None, +): + """Preflight all inputs, then execute the independent V3.5 writer.""" + + kwargs = { + "images": images, + "audio": audio, + "assembly_plan": assembly_plan, + "exact_total_duration": bool(exact_total_duration), + "audio_seam": str(audio_seam), + "video_seam": str(video_seam), + "diagnostics": str(diagnostics), + "buffer_backend": str(buffer_backend), + "backing_root": backing_root, + "final_audio_override": final_audio_override, + "final_audio_source": final_audio_source, + "interrupt_check": interrupt_check, + } + return assemble_with_hardening( + _assemble_decoded_chunks_v35_core, + (), + kwargs, + ) + + +__all__ = [ + "BUFFER_BACKEND_DISK", + "BUFFER_BACKEND_AUTO", + "BUFFER_BACKEND_OPTIONS", + "BUFFER_BACKEND_RAM", + "CopySpan", + "adjust_audio_duration_direct", + "assemble_decoded_chunks_v35", +] diff --git a/v3/driving_nodes.py b/v3/driving_nodes.py index 7da0cbc..8395265 100644 --- a/v3/driving_nodes.py +++ b/v3/driving_nodes.py @@ -6,12 +6,26 @@ import torch from ..constants import FPS from ..driving_audio import prepare_driving_audio_source +from ..hardening import diagnostics_is_full from ..reference_video import ( REFERENCE_VIDEO_SIZE_EFFICIENT, REFERENCE_VIDEO_SIZE_OPTIONS, ) from ..v2.decoder import enforce_total_frames -from .assembly import H3ContinuumAssembleSeamExperimental +from .assembly import ( + H3ContinuumAssembleSeamExperimental, + _chunk_list, + _singleton, +) +from .assembly_v35 import ( + BUFFER_BACKEND_AUTO, + BUFFER_BACKEND_OPTIONS, + assemble_decoded_chunks_v35, +) +from .memory_attribution import ( + format_assembly_projection, + project_assembly_buffers, +) from .nodes import CATEGORY as CONTINUUM_CATEGORY, H3ContinuumSamplerProduction @@ -136,6 +150,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction): audio_vae=None, reference_video_1=None, video_reference_size=REFERENCE_VIDEO_SIZE_EFFICIENT, + capture_refine_context=False, **kwargs, ): from ..reference_video import prepare_reference_video_source @@ -165,6 +180,7 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction): driving_audio_source=source, driving_audio_vae=audio_vae, reference_video_source=reference_video_source, + capture_refine_context=bool(capture_refine_context), **kwargs, ) selected_audio = _copy_audio(source.source_audio) if source is not None else None @@ -172,9 +188,46 @@ class H3ContinuumSamplerV34(H3ContinuumSamplerProduction): assembly_plan = dict(outputs[2]) assembly_plan[_DRIVING_AUDIO_PLAN_KEY] = _copy_audio(selected_audio) outputs = (*outputs[:2], assembly_plan, *outputs[3:]) + if bool(capture_refine_context): + return (*outputs[:4], selected_audio, outputs[4]) return (*outputs, selected_audio) +class H3ContinuumSamplerV35(H3ContinuumSamplerV34): + """V3.5 sampler with a physical-group First Pass refine context output.""" + + DEPRECATED = False + CATEGORY = CONTINUUM_CATEGORY + DESCRIPTION = ( + "H3 Continuum V3.5 sampler. It preserves the V3.4 outputs and adds one " + "physical-group refine context for context-aware Hi-Res Fix or an " + "external latent Second Pass." + ) + SEARCH_ALIASES = [ + "H3 Continuum Sampler V3.5", + "MiniMax H3 context-aware Hi-Res Fix", + ] + RETURN_TYPES = ( + *H3ContinuumSamplerV34.RETURN_TYPES, + "H3_CONTINUUM_REFINE_CONTEXT", + ) + RETURN_NAMES = ( + *H3ContinuumSamplerV34.RETURN_NAMES, + "refine_context", + ) + OUTPUT_IS_LIST = (*H3ContinuumSamplerV34.OUTPUT_IS_LIST, False) + + def run(self, *args, **kwargs): + kwargs.pop("capture_refine_context", None) + kwargs.pop("memory_attribution", None) + return super().run( + *args, + capture_refine_context=True, + memory_attribution=True, + **kwargs, + ) + + class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental): """Assemble decoded chunks and select original Driving Audio when connected.""" @@ -239,12 +292,128 @@ class H3ContinuumAssembleSeamV34(H3ContinuumAssembleSeamExperimental): return images, selected, report +def _assembly_argument(args, kwargs, name, index): + if name in kwargs: + return kwargs[name] + if len(args) > index: + return args[index] + return None + + +def _insert_projection_report(report, line): + lines = str(report).splitlines() + for index, value in enumerate(lines): + if value.startswith("Memory [assemble preflight]"): + lines.insert(index + 1, line) + return "\n".join(lines) + lines.append(line) + return "\n".join(lines) + + +class H3ContinuumAssembleSeamV35(H3ContinuumAssembleSeamV34): + """Independent V3.5 direct-write assembly with selectable IMAGE storage.""" + + DEPRECATED = False + CATEGORY = CONTINUUM_CATEGORY + DESCRIPTION = ( + "V3.5 decoded-chunk assembler with Auto, RAM, or Disk-backed IMAGE " + "storage, direct Exact Duration writes, and small-patch Video Seam." + ) + + @classmethod + def INPUT_TYPES(cls): + schema = super().INPUT_TYPES() + required = dict(schema["required"]) + diagnostics = required.pop("diagnostics") + required["buffer_backend"] = ( + BUFFER_BACKEND_OPTIONS, + { + "default": BUFFER_BACKEND_AUTO, + "display_name": "Buffer Backend", + "tooltip": ( + "Auto keeps outputs up to 4 GiB in RAM only when physical " + "memory has a safety reserve; otherwise it maps the final " + "IMAGE to disk. Manual RAM and Disk-backed remain available." + ), + }, + ) + required["diagnostics"] = diagnostics + schema["required"] = required + return schema + + def assemble( + self, + images, + audio, + assembly_plan, + exact_total_duration, + audio_seam, + video_seam, + buffer_backend, + diagnostics, + driving_audio=None, + ): + image_chunks = _chunk_list(images) + audio_chunks = _chunk_list(audio) + plan = _singleton(assembly_plan, "assembly_plan") + exact = bool(_singleton(exact_total_duration, "exact_total_duration")) + audio_seam_mode = str(_singleton(audio_seam, "audio_seam")) + video_seam_mode = str(_singleton(video_seam, "video_seam")) + backend = str(_singleton(buffer_backend, "buffer_backend")) + diagnostics_mode = str(_singleton(diagnostics, "diagnostics")) + preserved_audio = _copy_audio(plan.get(_DRIVING_AUDIO_PLAN_KEY)) + selected_audio = preserved_audio or _copy_audio(driving_audio) + selected_audio_source = None + if selected_audio is not None: + selected_audio_source = ( + "assembly plan" if preserved_audio is not None else "direct input" + ) + projection_line = None + if diagnostics_is_full(diagnostics_mode): + try: + projection = project_assembly_buffers( + images=image_chunks, + audio=audio_chunks, + assembly_plan=plan, + exact_total_duration=exact, + ) + projection_line = format_assembly_projection(projection) + ( + "; V3.5 Exact Duration IMAGE write=direct; " + "additional final IMAGE copy=no" + ) + except Exception as exc: + projection_line = ( + "Projected assembly buffers [V3.5]: unavailable " + f"({type(exc).__name__}: {exc})" + ) + + result_images, result_audio, report = assemble_decoded_chunks_v35( + images=image_chunks, + audio=audio_chunks, + assembly_plan=plan, + exact_total_duration=exact, + audio_seam=audio_seam_mode, + video_seam=video_seam_mode, + diagnostics=diagnostics_mode, + buffer_backend=backend, + final_audio_override=selected_audio, + final_audio_source=selected_audio_source, + ) + if projection_line is not None: + report = _insert_projection_report(report, projection_line) + return result_images, result_audio, report + + NODE_CLASS_MAPPINGS = { "H3ContinuumSamplerV34": H3ContinuumSamplerV34, + "H3ContinuumSamplerV35": H3ContinuumSamplerV35, "H3ContinuumAssembleSeamV34": H3ContinuumAssembleSeamV34, + "H3ContinuumAssembleSeamV35": H3ContinuumAssembleSeamV35, } NODE_DISPLAY_NAME_MAPPINGS = { "H3ContinuumSamplerV34": "H3 Continuum Sampler V3.4", + "H3ContinuumSamplerV35": "H3 Continuum Sampler V3.5", "H3ContinuumAssembleSeamV34": "H3 Continuum Assemble + Seam V3.4", + "H3ContinuumAssembleSeamV35": "H3 Continuum Assemble + Seam V3.5", } diff --git a/v3/file_backed_buffer.py b/v3/file_backed_buffer.py new file mode 100644 index 0000000..5b25e5b --- /dev/null +++ b/v3/file_backed_buffer.py @@ -0,0 +1,257 @@ +"""V3.5 adapter for the accepted P2-0 file-backed Tensor backend. + +The storage and lifetime implementation remains in the P2-0 module so the +Windows-tested locking, sidecar, cancellation, and alias-lifetime contracts +have one implementation. This module only exposes the small Production API +needed by the V3.5 assembler. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import shutil +import tempfile +import threading + +import torch + +from ..tools.p20_file_backed_tensor_poc import ( + FileBackedAllocation, + StorageLifetimeManager, + cleanup_stale_files, + get_process_lifetime_manager, +) + + +BUFFER_BACKEND_AUTO = "Auto" +BUFFER_BACKEND_RAM = "RAM" +BUFFER_BACKEND_DISK = "Disk-backed" +BUFFER_BACKEND_OPTIONS = ( + BUFFER_BACKEND_AUTO, + BUFFER_BACKEND_RAM, + BUFFER_BACKEND_DISK, +) +AUTO_RAM_IMAGE_LIMIT_BYTES = 4 * 1024**3 +AUTO_RAM_RESERVE_MIN_BYTES = 4 * 1024**3 +AUTO_RAM_RESERVE_FRACTION = 0.10 +AUTO_DISK_RESERVE_BYTES = 2 * 1024**3 +_BACKING_DIRECTORY_NAME = "h3-continuum-v35-file-backed" +_CLEANED_ROOTS: set[str] = set() +_CLEANED_ROOTS_LOCK = threading.Lock() + + +def validate_buffer_backend(value: str) -> str: + """Validate the explicit or automatic V3.5 IMAGE backend request.""" + + if value not in BUFFER_BACKEND_OPTIONS: + raise ValueError(f"unknown Buffer Backend: {value!r}") + return value + + +@dataclass(frozen=True) +class AutoBufferDecision: + selected_backend: str + final_image_bytes: int + backing_root: Path + available_memory_bytes: int | None + total_memory_bytes: int | None + memory_reserve_bytes: int | None + ram_image_limit_bytes: int + disk_free_bytes: int | None + disk_reserve_bytes: int + reason: str + + +def _system_memory_bytes() -> tuple[int | None, int | None]: + try: + import psutil + + counters = psutil.virtual_memory() + return int(counters.available), int(counters.total) + except Exception: + return None, None + + +def _existing_disk_anchor(path: Path) -> Path: + candidate = path + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + if not candidate.exists(): + raise OSError(f"no existing filesystem anchor for backing root: {path}") + return candidate + + +def _disk_free_bytes(path: Path) -> int: + return int(shutil.disk_usage(_existing_disk_anchor(path)).free) + + +def select_auto_buffer_backend( + final_image_bytes: int, + *, + backing_root: str | os.PathLike[str] | None = None, +) -> AutoBufferDecision: + """Choose RAM only when both output size and physical headroom are safe. + + Auto never converts a Disk-backed decision into RAM after a disk-space or + allocation failure. Users can still explicitly select either manual path. + """ + + final_bytes = int(final_image_bytes) + if final_bytes <= 0: + raise ValueError("Auto Buffer Backend requires a positive final IMAGE size") + root = resolve_backing_root(backing_root) + available, total = _system_memory_bytes() + reserve = None + if available is not None and total is not None and total > 0: + reserve = max( + AUTO_RAM_RESERVE_MIN_BYTES, + int(total * AUTO_RAM_RESERVE_FRACTION), + ) + + if ( + reserve is not None + and final_bytes <= AUTO_RAM_IMAGE_LIMIT_BYTES + and available >= final_bytes + reserve + ): + return AutoBufferDecision( + selected_backend=BUFFER_BACKEND_RAM, + final_image_bytes=final_bytes, + backing_root=root, + available_memory_bytes=available, + total_memory_bytes=total, + memory_reserve_bytes=reserve, + ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES, + disk_free_bytes=None, + disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES, + reason="final IMAGE fits the Auto RAM size limit and safety reserve", + ) + + if reserve is None: + reason = "physical-memory counters are unavailable" + elif final_bytes > AUTO_RAM_IMAGE_LIMIT_BYTES: + reason = "final IMAGE exceeds the Auto RAM size limit" + else: + reason = "available physical memory is below final IMAGE plus reserve" + + try: + disk_free = _disk_free_bytes(root) + except Exception as exc: + raise RuntimeError( + "Auto Buffer Backend selected Disk-backed but disk capacity could " + f"not be measured for {root}: {type(exc).__name__}: {exc}. " + "Select RAM manually only if sufficient memory is known to be available." + ) from exc + required_disk = final_bytes + AUTO_DISK_RESERVE_BYTES + if disk_free < required_disk: + raise RuntimeError( + "Auto Buffer Backend selected Disk-backed but the backing volume " + f"has {disk_free} free bytes; {required_disk} bytes are required " + f"for the final IMAGE plus the {AUTO_DISK_RESERVE_BYTES}-byte safety " + "reserve. Free disk space or select RAM manually only if sufficient " + "memory is known to be available." + ) + return AutoBufferDecision( + selected_backend=BUFFER_BACKEND_DISK, + final_image_bytes=final_bytes, + backing_root=root, + available_memory_bytes=available, + total_memory_bytes=total, + memory_reserve_bytes=reserve, + ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES, + disk_free_bytes=disk_free, + disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES, + reason=reason, + ) + + +def format_auto_buffer_decision(decision: AutoBufferDecision) -> str: + def value(number: int | None) -> str: + return "n/a" if number is None else str(int(number)) + + return ( + "Auto Buffer Decision: " + f"selected={decision.selected_backend}, " + f"final_IMAGE={decision.final_image_bytes} bytes, " + f"system_available={value(decision.available_memory_bytes)} bytes, " + f"system_total={value(decision.total_memory_bytes)} bytes, " + f"RAM_reserve={value(decision.memory_reserve_bytes)} bytes, " + f"RAM_IMAGE_limit={decision.ram_image_limit_bytes} bytes, " + f"disk_free={value(decision.disk_free_bytes)} bytes, " + f"disk_reserve={decision.disk_reserve_bytes} bytes; " + f"reason={decision.reason}." + ) + + +def resolve_backing_root( + backing_root: str | os.PathLike[str] | None = None, +) -> Path: + """Resolve the dedicated backing root without importing ComfyUI eagerly.""" + + if backing_root is not None: + return Path(backing_root).expanduser().resolve() + + try: + import folder_paths + except ImportError: + base = Path(tempfile.gettempdir()) + else: + value = folder_paths.get_temp_directory() + if value is None or not str(value).strip(): + raise RuntimeError("ComfyUI returned an empty temporary directory") + base = Path(value) + return (base.expanduser().resolve() / _BACKING_DIRECTORY_NAME).resolve() + + +def get_file_backed_image_manager( + backing_root: str | os.PathLike[str] | None = None, +) -> StorageLifetimeManager: + """Return the accepted process-global manager for one resolved root.""" + + root = resolve_backing_root(backing_root) + key = os.path.normcase(str(root)) + with _CLEANED_ROOTS_LOCK: + if key not in _CLEANED_ROOTS: + # The P2-0 root lock prevents cleanup from observing an allocation + # between its data-file and sidecar publication. A live owner is + # always retained; only prior abnormal-exit artifacts are removed. + cleanup_stale_files(root, grace_seconds=0.0) + _CLEANED_ROOTS.add(key) + return get_process_lifetime_manager(root) + + +def allocate_file_backed_image( + shape: tuple[int, ...], + *, + dtype: torch.dtype, + backing_root: str | os.PathLike[str] | None = None, +) -> FileBackedAllocation: + """Create a provisional Disk-backed IMAGE allocation. + + The caller must use the returned context manager and call ``publish()`` + only after every output element has been written. Backend failures are + propagated; this manual Disk-backed path never falls back to RAM. + """ + + manager = get_file_backed_image_manager(backing_root) + return manager.allocate(shape, dtype=dtype) + + +__all__ = [ + "AUTO_DISK_RESERVE_BYTES", + "AUTO_RAM_IMAGE_LIMIT_BYTES", + "AUTO_RAM_RESERVE_FRACTION", + "AUTO_RAM_RESERVE_MIN_BYTES", + "AutoBufferDecision", + "BUFFER_BACKEND_AUTO", + "BUFFER_BACKEND_DISK", + "BUFFER_BACKEND_OPTIONS", + "BUFFER_BACKEND_RAM", + "allocate_file_backed_image", + "format_auto_buffer_decision", + "get_file_backed_image_manager", + "resolve_backing_root", + "select_auto_buffer_backend", + "validate_buffer_backend", +] diff --git a/v3/hires_fix_nodes.py b/v3/hires_fix_nodes.py new file mode 100644 index 0000000..dc61355 --- /dev/null +++ b/v3/hires_fix_nodes.py @@ -0,0 +1,463 @@ +"""V3.5 Continuum-aware Hi-res fix convenience node.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +from .pixel_vae_resize import ( + resize_video_latents_via_vae, + resolve_pixel_latent_scale, +) +from .second_pass import SecondPassContractError, run_second_pass_groups + + +_MISSING = object() +HIRES_UPSCALE_METHODS = ( + "lanczos", + "nearest-exact", + "bilinear", + "area", + "bicubic", + "bislerp", +) +_CONDITIONING_UPSCALE_METHOD = "bilinear" + + +def _single_list_input(name: str, value: Any) -> Any: + if not isinstance(value, list) or len(value) != 1: + raise ValueError(f"{name} must contain exactly one value") + return value[0] + + +def _enabled_value(value: Any) -> bool: + if isinstance(value, list): + value = _single_list_input("enabled", value) + return bool(value) + + +def _lazy_missing(value: Any) -> bool: + return value is None or ( + isinstance(value, (list, tuple)) + and len(value) == 1 + and value[0] is None + ) + + +def h3_core_native_canvas( + width: int, + height: int, + *, + adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None, +) -> tuple[int, int]: + """Return the native canvas selected by ComfyUI Core for this aspect ratio.""" + + if width < 1 or height < 1: + raise SecondPassContractError("source pixel width and height must be positive") + if adapt_canvas_fn is None: + from comfy_extras.nodes_minimax_h3 import adapt_canvas + + adapt_canvas_fn = adapt_canvas + target = adapt_canvas_fn(int(width), int(height)) + if not isinstance(target, Sequence) or len(target) != 2: + raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry") + target_width, target_height = (int(value) for value in target) + if target_width < 1 or target_height < 1: + raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry") + return target_width, target_height + + +def resolve_h3_native_latent_target( + video_latents: Sequence[Mapping[str, Any]], + assembly_plan: Mapping[str, Any], + *, + adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None, +) -> dict[str, int | bool]: + """Resolve Core's native canvas without shrinking an existing video latent.""" + + contract = assembly_plan.get("second_pass_contract") + if not isinstance(contract, Mapping): + raise SecondPassContractError("assembly_plan has no second_pass_contract") + groups = contract.get("physical_groups") + if not isinstance(groups, list) or not groups: + raise SecondPassContractError( + "second_pass_contract.physical_groups must be non-empty" + ) + if not isinstance(video_latents, Sequence) or not video_latents: + raise SecondPassContractError("video_latents must be a non-empty sequence") + if len(video_latents) != len(groups): + raise SecondPassContractError("video latent group count changed") + + first_group = groups[0] + if not isinstance(first_group, Mapping): + raise SecondPassContractError("physical group metadata must be a mapping") + source_width = int(first_group.get("source_width", -1)) + source_height = int(first_group.get("source_height", -1)) + source_latent_width = int(first_group.get("source_latent_w", -1)) + source_latent_height = int(first_group.get("source_latent_h", -1)) + if min(source_width, source_height, source_latent_width, source_latent_height) < 1: + raise SecondPassContractError("source pixel/latent geometry is invalid") + if source_width % source_latent_width or source_height % source_latent_height: + raise SecondPassContractError("source pixel/latent geometry is not integral") + pixel_scale_width = source_width // source_latent_width + pixel_scale_height = source_height // source_latent_height + + current_geometry: tuple[int, int] | None = None + for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)): + if not isinstance(group, Mapping): + raise SecondPassContractError("physical group metadata must be a mapping") + if int(group.get("source_width", -1)) != source_width or int( + group.get("source_height", -1) + ) != source_height: + raise SecondPassContractError("source geometry differs between physical groups") + if int(group.get("source_latent_w", -1)) != source_latent_width or int( + group.get("source_latent_h", -1) + ) != source_latent_height: + raise SecondPassContractError( + "source latent geometry differs between physical groups" + ) + if not isinstance(latent, Mapping) or "samples" not in latent: + raise SecondPassContractError( + f"video_latents[{position}] must be a LATENT mapping with samples" + ) + samples = latent["samples"] + if not hasattr(samples, "shape") or len(samples.shape) != 5: + raise SecondPassContractError("video latent must have shape [B,C,T,H,W]") + geometry = (int(samples.shape[-2]), int(samples.shape[-1])) + if current_geometry is None: + current_geometry = geometry + elif geometry != current_geometry: + raise SecondPassContractError( + "all physical groups must use the same current H/W" + ) + + assert current_geometry is not None + native_width, native_height = h3_core_native_canvas( + source_width, + source_height, + adapt_canvas_fn=adapt_canvas_fn, + ) + if native_width % pixel_scale_width or native_height % pixel_scale_height: + raise SecondPassContractError( + "ComfyUI Core native canvas is not aligned to the H3 latent scale" + ) + native_latent_width = native_width // pixel_scale_width + native_latent_height = native_height // pixel_scale_height + current_height, current_width = current_geometry + preserve_current = ( + native_latent_width < current_width + or native_latent_height < current_height + ) + if preserve_current: + target_latent_width = current_width + target_latent_height = current_height + else: + target_latent_width = native_latent_width + target_latent_height = native_latent_height + + return { + "source_width": source_width, + "source_height": source_height, + "native_width": native_width, + "native_height": native_height, + "target_width": target_latent_width * pixel_scale_width, + "target_height": target_latent_height * pixel_scale_height, + "target_latent_width": target_latent_width, + "target_latent_height": target_latent_height, + "preserved_current": preserve_current, + } + + +def _prepend_resize_status( + second_pass_status: str, + *, + backend: str, + mode: str, + upscale_method: str, + scale_by: float, + source_width: int, + source_height: int, + target_width: int, + target_height: int, + native_width: int, + native_height: int, +) -> str: + lines = [ + "H3 Continuum Hi-Res Fix V3.5", + "Pixel/VAE resize: " + f"backend={backend}, mode={mode}, method={upscale_method}, " + f"scale_by={scale_by:g}, " + f"source={source_width}x{source_height}, " + f"target={target_width}x{target_height}, " + f"core_native={native_width}x{native_height}.", + ] + if target_width > native_width or target_height > native_height: + lines.append( + "INFO: target exceeds the ComfyUI Core H3 base canvas; the Pixel/VAE " + "roundtrip is preserved and resource use may increase." + ) + lines.append(second_pass_status) + return "\n".join(lines) + + +class H3ContinuumHiResFixV35: + """Pixel/VAE resize and low-sigma refine complete physical video groups.""" + + DEPRECATED = False + DESCRIPTION = ( + "Continuum-aware Hi-res fix. When enabled, each first-pass physical video " + "LATENT is decoded with the connected H3 Video VAE, resized in pixel space, " + "re-encoded with the same VAE, and refined once with the workflow SIGMAS. " + "scale_by 0 uses ComfyUI Core's H3 native canvas; positive values use the " + "manual multiplier. Connect the optional Continuum refine context to preserve " + "First/Last/Reference conditioning at the target geometry. " + "When disabled, first-pass video/audio LATENT objects and the original assembly " + "plan are returned unchanged." + ) + + @classmethod + def INPUT_TYPES(cls): + lazy = {"lazy": True} + return { + "required": { + "model": ("MODEL", lazy), + "clip": ("CLIP", lazy), + "sampler": ("SAMPLER", lazy), + "sigmas": ("SIGMAS", lazy), + "video_latents": ("LATENT",), + "audio_latents": ("LATENT",), + "assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",), + "enabled": ( + "BOOLEAN", + { + "default": True, + "tooltip": ( + "Enable Resize + Second Pass. Off returns the first-pass " + "video/audio LATENT objects and assembly plan unchanged." + ), + }, + ), + "upscale_method": ( + HIRES_UPSCALE_METHODS, + {"default": "lanczos"}, + ), + "scale_by": ( + "FLOAT", + { + "default": 2.0, + "min": 0.0, + "max": 4.0, + "step": 0.01, + "tooltip": ( + "2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the " + "ComfyUI Core H3 native canvas automatically. Manual " + "values follow the existing Second Pass contract and must " + "be 1.0 or greater." + ), + }, + ), + "refine_seed": ( + "INT", + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Base seed independently derived once per physical group.", + }, + ), + }, + "optional": { + "refine_context": ( + "H3_CONTINUUM_REFINE_CONTEXT", + {"lazy": True}, + ), + "video_vae": ( + "VAE", + { + "lazy": True, + "tooltip": ( + "Required while enabled for the safe Pixel/VAE roundtrip; " + "not evaluated while disabled." + ), + }, + ), + }, + } + + INPUT_IS_LIST = True + RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING") + RETURN_NAMES = ( + "video_latents", + "audio_latents", + "updated_assembly_plan", + "status", + ) + OUTPUT_IS_LIST = (True, True, False, False) + FUNCTION = "apply" + CATEGORY = "MiniMax H3/Continuum" + + def check_lazy_status( + self, + enabled, + model=None, + clip=None, + sampler=None, + sigmas=None, + refine_context=_MISSING, + video_vae=_MISSING, + **_kwargs, + ): + if not _enabled_value(enabled): + return [] + required = [ + name + for name, value in ( + ("model", model), + ("clip", clip), + ("sampler", sampler), + ("sigmas", sigmas), + ) + if _lazy_missing(value) + ] + required.extend( + name + for name, value in ( + ("refine_context", refine_context), + ("video_vae", video_vae), + ) + if value is not _MISSING and _lazy_missing(value) + ) + return required + + def apply( + self, + model, + clip, + sampler, + sigmas, + video_latents, + audio_latents, + assembly_plan, + enabled, + upscale_method, + scale_by, + refine_seed, + refine_context=None, + video_vae=None, + ): + plan = _single_list_input("assembly_plan", assembly_plan) + if not _enabled_value(enabled): + return ( + video_latents, + audio_latents, + plan, + "H3 Continuum Hi-Res Fix V3.5: disabled; first-pass video/audio " + "LATENT objects and assembly_plan returned unchanged.", + ) + + method = str(_single_list_input("upscale_method", upscale_method)) + scale = float(_single_list_input("scale_by", scale_by)) + if 0.0 < scale < 1.0: + raise SecondPassContractError( + "scale_by must be 0 for H3 native Auto or at least 1.0; the " + "existing Second Pass contract preserves or enlarges spatial size" + ) + vae = ( + None + if video_vae is None + else _single_list_input("video_vae", video_vae) + ) + if vae is None: + raise SecondPassContractError( + "Hi-Res Fix enabled requires video_vae for the safe Pixel/VAE " + "roundtrip; legacy direct latent interpolation is disabled" + ) + source_width = int(plan["width"]) + source_height = int(plan["height"]) + native_width, native_height = h3_core_native_canvas( + source_width, + source_height, + ) + first_samples = video_latents[0]["samples"] + current_geometry = ( + int(first_samples.shape[-2]), + int(first_samples.shape[-1]), + ) + if scale == 0.0: + native_target = resolve_h3_native_latent_target(video_latents, plan) + target_latent_width = int(native_target["target_latent_width"]) + target_latent_height = int(native_target["target_latent_height"]) + target_width = int(native_target["target_width"]) + target_height = int(native_target["target_height"]) + resize_mode = "h3_native_canvas_auto" + else: + pixel_scale_width, pixel_scale_height = resolve_pixel_latent_scale(plan) + target_latent_width = round(current_geometry[1] * scale) + target_latent_height = round(current_geometry[0] * scale) + if target_latent_width < 1 or target_latent_height < 1: + raise SecondPassContractError( + "scale_by produced a non-positive target geometry" + ) + target_width = target_latent_width * pixel_scale_width + target_height = target_latent_height * pixel_scale_height + resize_mode = "manual_scale_by" + + if current_geometry == (target_latent_height, target_latent_width): + resized_video_latents = list(video_latents) + resize_backend = "direct_second_pass_no_resize" + else: + resized_video_latents = resize_video_latents_via_vae( + video_latents, + plan, + video_vae=vae, + target_width=target_width, + target_height=target_height, + upscale_method=method, + ) + resize_backend = "sequential_pixel_vae_roundtrip" + + refined_video, output_audio, updated_plan, status = run_second_pass_groups( + model=_single_list_input("model", model), + clip=_single_list_input("clip", clip), + sampler=_single_list_input("sampler", sampler), + sigmas=_single_list_input("sigmas", sigmas), + video_latents=resized_video_latents, + audio_latents=audio_latents, + assembly_plan=plan, + refine_seed=int(_single_list_input("refine_seed", refine_seed)), + refine_context=( + None + if refine_context is None + else _single_list_input("refine_context", refine_context) + ), + video_vae=vae, + conditioning_upscale_method=_CONDITIONING_UPSCALE_METHOD, + ) + return ( + refined_video, + output_audio, + updated_plan, + _prepend_resize_status( + status, + backend=resize_backend, + mode=resize_mode, + upscale_method=method, + scale_by=scale, + source_width=source_width, + source_height=source_height, + target_width=int(updated_plan["width"]), + target_height=int(updated_plan["height"]), + native_width=native_width, + native_height=native_height, + ), + ) + + +NODE_CLASS_MAPPINGS = { + "H3ContinuumHiResFixV35": H3ContinuumHiResFixV35, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumHiResFixV35": "H3 Continuum Hi-Res Fix V3.5", +} diff --git a/v3/latent_resize_nodes.py b/v3/latent_resize_nodes.py new file mode 100644 index 0000000..4224e52 --- /dev/null +++ b/v3/latent_resize_nodes.py @@ -0,0 +1,140 @@ +"""V3.5 Core-compatible spatial resize for physical video latents.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +UPSCALE_METHODS = ( + "nearest-exact", + "bilinear", + "area", + "bicubic", + "bislerp", +) + + +def _single_list_input(name: str, value: Any) -> Any: + if not isinstance(value, list) or len(value) != 1: + raise ValueError(f"{name} must contain exactly one value") + return value[0] + + +def resize_video_latents( + video_latents: list[dict[str, Any]], + *, + upscale_method: str, + scale_by: float, + upscale_fn: Callable[..., Any] | None = None, +) -> list[dict[str, Any]]: + """Resize only spatial H/W through the ComfyUI Core latent implementation.""" + + if upscale_fn is None: + from comfy.utils import common_upscale + + upscale_fn = common_upscale + + resized: list[dict[str, Any]] = [] + for latent in video_latents: + samples = latent["samples"] + width = round(samples.shape[-1] * scale_by) + height = round(samples.shape[-2] * scale_by) + output = latent.copy() + output["samples"] = upscale_fn( + samples, + width, + height, + upscale_method, + "disabled", + ) + resized.append(output) + return resized + + +def resize_video_latents_to_size( + video_latents: list[dict[str, Any]], + *, + upscale_method: str, + target_width: int, + target_height: int, + upscale_fn: Callable[..., Any] | None = None, +) -> list[dict[str, Any]]: + """Resize every physical video group to one exact latent H/W.""" + + if target_width < 1 or target_height < 1: + raise ValueError("target latent width and height must be positive") + if upscale_fn is None: + from comfy.utils import common_upscale + + upscale_fn = common_upscale + + resized: list[dict[str, Any]] = [] + for latent in video_latents: + samples = latent["samples"] + output = latent.copy() + output["samples"] = upscale_fn( + samples, + int(target_width), + int(target_height), + upscale_method, + "disabled", + ) + resized.append(output) + return resized + + +class H3ContinuumLatentResizeV35: + """List-aware Core spatial resize for Continuum physical video groups.""" + + DEPRECATED = False + DESCRIPTION = ( + "V3.5 Hi-res fix resize. Spatially resizes every Continuum " + "physical video LATENT with the selected ComfyUI Core interpolation method. " + "B/C/T, physical group order, and LATENT metadata are preserved." + ) + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_latents": ("LATENT",), + "upscale_method": (UPSCALE_METHODS,), + "scale_by": ( + "FLOAT", + { + "default": 2.0, + "min": 0.01, + "max": 8.0, + "step": 0.01, + }, + ), + } + } + + INPUT_IS_LIST = True + RETURN_TYPES = ("LATENT",) + RETURN_NAMES = ("resized_video_latents",) + OUTPUT_IS_LIST = (True,) + FUNCTION = "resize" + CATEGORY = "MiniMax H3/Continuum/Utilities" + + def resize(self, video_latents, upscale_method, scale_by): + return ( + resize_video_latents( + video_latents, + upscale_method=str( + _single_list_input("upscale_method", upscale_method) + ), + scale_by=float(_single_list_input("scale_by", scale_by)), + ), + ) + + +NODE_CLASS_MAPPINGS = { + "H3ContinuumLatentResizeV35": H3ContinuumLatentResizeV35, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumLatentResizeV35": "H3 Continuum Latent Resize V3.5", +} diff --git a/v3/memory_attribution.py b/v3/memory_attribution.py new file mode 100644 index 0000000..7d6cf8e --- /dev/null +++ b/v3/memory_attribution.py @@ -0,0 +1,294 @@ +"""V3.5-only, non-synchronizing memory attribution helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +import math +from typing import Any, Mapping + +import torch + +from ..constants import FPS +from ..hardening import tensor_storage_stats +from ..state import extract_av_streams +from .plan import validate_assembly_plan + + +_MIB = 1024**2 + + +def _mib(value: int | None) -> str: + if value is None: + return "n/a" + return f"{int(value) / _MIB:.1f} MiB" + + +def _bytes_and_mib(value: int) -> str: + return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)" + + +def _process_counters() -> tuple[int | None, int | None]: + try: + import psutil + + return ( + int(psutil.Process().memory_info().rss), + int(psutil.virtual_memory().available), + ) + except Exception: + return None, None + + +def _cuda_counters() -> tuple[int | None, int | None, int | None]: + """Read counters only; never synchronize, clear caches, or reset peaks.""" + + try: + if torch.cuda.is_available(): + return ( + int(torch.cuda.memory_allocated()), + int(torch.cuda.memory_reserved()), + int(torch.cuda.max_memory_allocated()), + ) + except Exception: + pass + return None, None, None + + +def _retained_stream_bytes( + entries: Any, + stream: str, +) -> int: + values: list[Any] = [] + if isinstance(entries, (list, tuple)): + for entry in entries: + if isinstance(entry, Mapping): + values.append(entry.get(stream)) + stats = tensor_storage_stats(values) + return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"]) + + +def _observed_streams(latent: Any) -> tuple[Any, ...]: + if latent is None: + return () + if isinstance(latent, Mapping) and "samples" in latent: + try: + return tuple(extract_av_streams(latent)) + except Exception: + return () + return (latent,) + + +def format_attribution_snapshot( + label: str, + *, + retained_entries: Any = None, + observed_latent: Any = None, + retained_refine_context: Any = None, +) -> str: + """Format one fail-soft snapshot without changing allocator state.""" + try: + rss, available = _process_counters() + cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters() + retained_video = _retained_stream_bytes(retained_entries, "video") + retained_audio = _retained_stream_bytes(retained_entries, "audio") + observed = tensor_storage_stats(_observed_streams(observed_latent)) + refine_context = tensor_storage_stats(retained_refine_context) + return ( + f"Memory [{label}]: RSS={_mib(rss)}, " + f"system_available={_mib(available)}, " + f"CUDA_allocated={_mib(cuda_allocated)}, " + f"CUDA_reserved={_mib(cuda_reserved)}, " + f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, " + f"retained_video={_bytes_and_mib(retained_video)}, " + f"retained_audio={_bytes_and_mib(retained_audio)}, " + "retained_refine_context_CPU=" + f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, " + "retained_refine_context_GPU=" + f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, " + f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, " + f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}" + ) + except Exception as exc: + return ( + f"Memory [{label}]: unavailable " + f"({type(exc).__name__}: {exc})" + ) + + +class MemoryAttributionCollector: + """Keep V3.5 memory events in chronological order until report assembly.""" + + def __init__(self): + self.lines: list[str] = [] + + def capture_sequence_start(self) -> None: + self.lines.append(format_attribution_snapshot("V3.5 sequence start")) + + def capture_group( + self, + *, + physical_group: int, + logical_chunks: tuple[int, ...], + stage: str, + retained_entries: Any, + observed_latent: Any = None, + retained_refine_context: Any = None, + ) -> None: + logical = ",".join(str(int(value)) for value in logical_chunks) + self.lines.append( + format_attribution_snapshot( + f"V3.5 physical group {int(physical_group)} " + f"logical=[{logical}] {stage}", + retained_entries=retained_entries, + observed_latent=observed_latent, + retained_refine_context=retained_refine_context, + ) + ) + + def capture_sequence_complete( + self, + entries: Any, + retained_refine_context: Any = None, + ) -> None: + self.lines.append( + format_attribution_snapshot( + "V3.5 sequence complete", + retained_entries=entries, + retained_refine_context=retained_refine_context, + ) + ) + + +def capture_attribution_fail_soft( + collector: Any, + method: str, + **kwargs: Any, +) -> None: + """A diagnostics failure must never replace a sampling result.""" + + try: + getattr(collector, method)(**kwargs) + except Exception as exc: + try: + collector.lines.append( + f"Memory attribution [{method}]: unavailable " + f"({type(exc).__name__}: {exc})" + ) + except Exception: + pass + + +@dataclass(frozen=True) +class AssemblyBufferProjection: + natural_frames: int + target_frames: int + natural_image_bytes: int + natural_audio_bytes: int + final_image_bytes: int + final_audio_bytes: int + frame_count_adjustment_required: bool + + +def _as_list(value: Any) -> list[Any]: + if isinstance(value, list): + if len(value) == 1 and isinstance(value[0], list): + return list(value[0]) + return list(value) + if isinstance(value, tuple): + return list(value) + return [value] + + +def _singleton(value: Any, name: str) -> Any: + while isinstance(value, list): + if len(value) != 1: + raise ValueError(f"{name} must contain exactly one value") + value = value[0] + return value + + +def project_assembly_buffers( + *, + images: Any, + audio: Any, + assembly_plan: Any, + exact_total_duration: Any, +) -> AssemblyBufferProjection: + """Calculate natural and final buffer sizes without allocating a buffer.""" + + plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan")) + image_chunks = _as_list(images) + audio_chunks = _as_list(audio) + if not image_chunks or not audio_chunks: + raise ValueError("decoded image/audio chunks are required for projection") + + first_image = _singleton(image_chunks[0], "decoded image chunk") + if not torch.is_tensor(first_image) or first_image.ndim != 4: + raise ValueError("decoded image chunk must be [frames,H,W,C]") + frame_shape = tuple(int(value) for value in first_image.shape[1:]) + for index, value in enumerate(image_chunks, start=1): + tensor = _singleton(value, f"decoded image chunk {index}") + if ( + not torch.is_tensor(tensor) + or tensor.ndim != 4 + or tuple(int(item) for item in tensor.shape[1:]) != frame_shape + or tensor.dtype != first_image.dtype + ): + raise ValueError("decoded image geometry or dtype changed between chunks") + + first_audio = _singleton(audio_chunks[0], "decoded audio chunk") + if not isinstance(first_audio, Mapping): + raise ValueError("decoded audio chunk must be an AUDIO mapping") + waveform = first_audio.get("waveform") + sample_rate = first_audio.get("sample_rate") + if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None: + raise ValueError("decoded audio chunk has an invalid waveform or sample rate") + sample_rate = int(sample_rate) + if sample_rate <= 0: + raise ValueError("decoded audio sample rate must be positive") + + decode_units = list(plan.get("decode_groups") or plan["chunks"]) + natural_frames = sum(int(unit["net_frames"]) for unit in decode_units) + exact = bool(_singleton(exact_total_duration, "exact_total_duration")) + target_frames = int(plan["target_frames"]) if exact else natural_frames + if natural_frames <= 0 or target_frames <= 0: + raise ValueError("assembly frame counts must be positive") + + frame_elements = math.prod(frame_shape) + audio_streams = math.prod(int(value) for value in waveform.shape[:-1]) + natural_samples = int(round(natural_frames / FPS * sample_rate)) + target_samples = int(round(target_frames / FPS * sample_rate)) + return AssemblyBufferProjection( + natural_frames=natural_frames, + target_frames=target_frames, + natural_image_bytes=( + natural_frames * frame_elements * first_image.element_size() + ), + natural_audio_bytes=( + natural_samples * audio_streams * waveform.element_size() + ), + final_image_bytes=( + target_frames * frame_elements * first_image.element_size() + ), + final_audio_bytes=( + target_samples * audio_streams * waveform.element_size() + ), + frame_count_adjustment_required=bool( + exact and target_frames != natural_frames + ), + ) + + +def format_assembly_projection(projection: AssemblyBufferProjection) -> str: + return ( + "Projected assembly buffers [V3.5]: " + f"natural_frames={projection.natural_frames}, " + f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, " + "projected_natural_generated_AUDIO=" + f"{_bytes_and_mib(projection.natural_audio_bytes)}, " + f"target_frames={projection.target_frames}, " + f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, " + f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, " + "frame_count_adjustment_required=" + f"{'yes' if projection.frame_count_adjustment_required else 'no'}" + ) diff --git a/v3/nodes.py b/v3/nodes.py index 1ab2133..0cb3c6c 100644 --- a/v3/nodes.py +++ b/v3/nodes.py @@ -256,6 +256,8 @@ class H3ContinuumSamplerV3: driving_audio_vae=None, reference_video_source=None, timeline_video_source=None, + capture_refine_context=False, + memory_attribution=False, ): if prompt_overrides is not None and not isinstance(prompt_overrides, dict): prompt_overrides = None @@ -297,7 +299,7 @@ class H3ContinuumSamplerV3: for index, prompt in sparse_overrides.items(): clip_prompt_inputs[f"clip_{index}_prompt"] = prompt - entries, last_state, session, report = H3ContinuumSamplerV2().run( + sequence_outputs = H3ContinuumSamplerV2().run( model=model, clip=clip, video_vae=video_vae, @@ -335,8 +337,14 @@ class H3ContinuumSamplerV3: driving_audio_vae=driving_audio_vae, reference_video_source=reference_video_source, timeline_video_source=timeline_video_source, + capture_refine_context=bool(capture_refine_context), + memory_attribution=bool(memory_attribution), **clip_prompt_inputs, ) + if bool(capture_refine_context): + entries, last_state, session, report, refine_context = sequence_outputs + else: + entries, last_state, session, report = sequence_outputs from ..v2.sequence import _terminal_flf_merge_enabled from .plan import prepare_physical_decode_entries @@ -370,7 +378,10 @@ class H3ContinuumSamplerV3: "session": session, "report": report, } - return video_latents, audio_latents, assembly_plan, result + outputs = (video_latents, audio_latents, assembly_plan, result) + if bool(capture_refine_context): + return (*outputs, refine_context) + return outputs class H3ContinuumSamplerProduction(H3ContinuumSamplerV3): @@ -650,6 +661,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3): driving_audio_vae=None, reference_video_source=None, timeline_video_source=None, + capture_refine_context=False, + memory_attribution=False, ): runtime_started_at = time.perf_counter() from ..reference import prepare_reference_assets @@ -699,6 +712,8 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3): driving_audio_vae=driving_audio_vae, reference_video_source=reference_video_source, timeline_video_source=timeline_video_source, + capture_refine_context=bool(capture_refine_context), + memory_attribution=bool(memory_attribution), advanced={ "audio_continuity": bool(audio_continuity), "diagnostics": diagnostics, @@ -713,13 +728,20 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3): if run_storage == "Off": _validate_regenerate_storage(run_storage, regenerate_from) - video_latents, audio_latents, assembly_plan, result = execute() - return ( + execute_outputs = execute() + if bool(capture_refine_context): + video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs + else: + video_latents, audio_latents, assembly_plan, result = execute_outputs + outputs = ( video_latents, audio_latents, mark_runtime_start(assembly_plan), str(result["report"]), ) + if bool(capture_refine_context): + return (*outputs, refine_context) + return outputs if run_storage != "Save + Auto Resume": raise ValueError(f"unknown Run Storage mode: {run_storage!r}") from ..run_storage import ( @@ -735,14 +757,26 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3): with run_storage_scope( storage_name, prompt=prompt, unique_id=unique_id ) as storage: - video_latents, audio_latents, assembly_plan, result = execute() + execute_outputs = execute() + if bool(capture_refine_context): + video_latents, audio_latents, assembly_plan, result, refine_context = execute_outputs + else: + video_latents, audio_latents, assembly_plan, result = execute_outputs result = dict(result) report = str(result["report"]) + "\n" + storage.summary( detailed=diagnostics == DIAGNOSTICS_FULL ) result["report"] = report storage.finalize(session=result["session"], report=report) - return video_latents, audio_latents, mark_runtime_start(assembly_plan), report + outputs = ( + video_latents, + audio_latents, + mark_runtime_start(assembly_plan), + report, + ) + if bool(capture_refine_context): + return (*outputs, refine_context) + return outputs class H3ContinuumSamplerTimelineVideo(H3ContinuumSamplerProduction): diff --git a/v3/pixel_vae_resize.py b/v3/pixel_vae_resize.py new file mode 100644 index 0000000..60edf5c --- /dev/null +++ b/v3/pixel_vae_resize.py @@ -0,0 +1,367 @@ +"""Sequential pixel/VAE resize for Continuum physical video groups.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +import torch + +from .plan import SECOND_PASS_CONTRACT_VERSION + + +PIXEL_UPSCALE_METHODS = ( + "nearest-exact", + "bilinear", + "area", + "bicubic", + "bislerp", + "lanczos", +) +_PIXEL_RESIZE_FRAME_BATCH = 8 + + +class PixelVaeResizeError(ValueError): + """Raised when a pixel/VAE resize would violate the physical-group contract.""" + + +def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]: + if not isinstance(assembly_plan, Mapping): + raise PixelVaeResizeError("assembly_plan must be a mapping") + contract = assembly_plan.get("second_pass_contract") + if not isinstance(contract, Mapping): + raise PixelVaeResizeError("assembly_plan has no second_pass_contract") + if contract.get("version") != SECOND_PASS_CONTRACT_VERSION: + raise PixelVaeResizeError("unsupported second_pass_contract version") + groups = contract.get("physical_groups") + if ( + not isinstance(groups, list) + or not groups + or not all(isinstance(group, Mapping) for group in groups) + ): + raise PixelVaeResizeError( + "second_pass_contract.physical_groups must be non-empty" + ) + return groups + + +def resolve_pixel_latent_scale( + assembly_plan: Mapping[str, Any], +) -> tuple[int, int]: + """Return plan-derived pixel-per-latent ratios as ``(width, height)``.""" + + groups = _contract_groups(assembly_plan) + scale: tuple[int, int] | None = None + for position, group in enumerate(groups): + if int(group.get("group_id", -1)) != position: + raise PixelVaeResizeError("physical group order or group_id changed") + source_width = int(group.get("source_width", -1)) + source_height = int(group.get("source_height", -1)) + source_latent_width = int(group.get("source_latent_w", -1)) + source_latent_height = int(group.get("source_latent_h", -1)) + if min( + source_width, + source_height, + source_latent_width, + source_latent_height, + ) < 1: + raise PixelVaeResizeError("source pixel/latent geometry is invalid") + if ( + source_width % source_latent_width + or source_height % source_latent_height + ): + raise PixelVaeResizeError( + "source pixel/latent geometry is not integral" + ) + group_scale = ( + source_width // source_latent_width, + source_height // source_latent_height, + ) + if scale is None: + scale = group_scale + elif scale != group_scale: + raise PixelVaeResizeError( + "source pixel/latent scale differs between physical groups" + ) + + assert scale is not None + return scale + + +def _latent_samples( + latent: Any, + *, + position: int, + group: Mapping[str, Any], +) -> torch.Tensor: + if not isinstance(latent, Mapping) or "samples" not in latent: + raise PixelVaeResizeError( + f"video_latents[{position}] must be a LATENT mapping with samples" + ) + samples = latent["samples"] + if not isinstance(samples, torch.Tensor) or samples.ndim != 5: + raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]") + batch, channels, temporal, height, width = ( + int(value) for value in samples.shape + ) + expected = ( + int(group.get("source_batch", -1)), + int(group.get("latent_channels", -1)), + int(group.get("source_latent_t", -1)), + int(group.get("source_latent_h", -1)), + int(group.get("source_latent_w", -1)), + ) + if (batch, channels, temporal, height, width) != expected: + raise PixelVaeResizeError( + "video latent shape or physical group order changed" + ) + if channels != 24: + raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels") + if not bool(torch.isfinite(samples).all()): + raise PixelVaeResizeError("input video latent contains NaN or Inf") + return samples + + +def _resize_one_group( + samples: torch.Tensor, + *, + video_vae: Any, + source_width: int, + source_height: int, + target_width: int, + target_height: int, + target_latent_width: int, + target_latent_height: int, + upscale_method: str, + upscale_fn: Callable[..., Any], +) -> torch.Tensor: + """Decode, resize, and encode one group so its pixel buffer dies per call.""" + + decoded_pixels = video_vae.decode(samples) + if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5: + raise PixelVaeResizeError( + "Video VAE decode must return pixels with shape [B,F,H,W,C]" + ) + batch, frames, pixel_height, pixel_width, channels = ( + int(value) for value in decoded_pixels.shape + ) + if batch != int(samples.shape[0]): + raise PixelVaeResizeError("Video VAE decode changed the batch size") + if frames < 1 or channels != 3: + raise PixelVaeResizeError( + "Video VAE decode returned invalid frame or channel geometry" + ) + if (pixel_width, pixel_height) != (source_width, source_height): + raise PixelVaeResizeError( + "Video VAE decode geometry differs from the assembly plan" + ) + for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH): + if not bool( + torch.isfinite( + decoded_pixels[ + :, + frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH, + ] + ).all() + ): + raise PixelVaeResizeError("Video VAE decode contains NaN or Inf") + + # Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation + # is CPU/PIL-based already, and this also avoids retaining a full video-sized + # resize workspace in VRAM under --gpu-only configurations. + pixels = decoded_pixels.detach().to("cpu").contiguous() + del decoded_pixels + + # Core common_upscale is a 4D image boundary. Make B*F explicit so video + # frame order is unambiguous. Resize bounded frame batches into one CPU + # [B,F,H,W,C] buffer required by the temporal H3 VAE encoder. + source_flat = pixels.reshape( + batch * frames, pixel_height, pixel_width, channels + ) + resized_pixels = torch.empty( + (batch, frames, target_height, target_width, channels), + dtype=pixels.dtype, + device="cpu", + ) + target_flat = resized_pixels.reshape( + batch * frames, target_height, target_width, channels + ) + for frame_start in range( + 0, + batch * frames, + _PIXEL_RESIZE_FRAME_BATCH, + ): + frame_stop = min( + frame_start + _PIXEL_RESIZE_FRAME_BATCH, + batch * frames, + ) + pixel_nchw = ( + source_flat[frame_start:frame_stop] + .movedim(-1, 1) + .contiguous() + ) + resized_nchw = upscale_fn( + pixel_nchw, + int(target_width), + int(target_height), + upscale_method, + "disabled", + ) + expected_resize_shape = ( + frame_stop - frame_start, + channels, + target_height, + target_width, + ) + if not isinstance(resized_nchw, torch.Tensor) or tuple( + resized_nchw.shape + ) != expected_resize_shape: + raise PixelVaeResizeError("Core pixel resize returned invalid geometry") + resized_nchw = resized_nchw.detach().to("cpu") + if not bool(torch.isfinite(resized_nchw).all()): + raise PixelVaeResizeError("Core pixel resize contains NaN or Inf") + target_flat[frame_start:frame_stop].copy_( + resized_nchw.movedim(1, -1) + ) + del pixel_nchw, resized_nchw + + del pixels, source_flat, target_flat + + # Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch + # and promotes its frame axis to one temporal video batch. Passing the 5D + # [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial. + # Encode each Continuum batch independently, then restore B explicitly. + encoded_batches: list[torch.Tensor] = [] + expected_batch_shape = ( + 1, + int(samples.shape[1]), + int(samples.shape[2]), + int(target_latent_height), + int(target_latent_width), + ) + for batch_index in range(batch): + encoded_batch = video_vae.encode(resized_pixels[batch_index]) + if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5: + raise PixelVaeResizeError( + "Video VAE encode must return a latent with shape [B,C,T,H,W]" + ) + if int(encoded_batch.shape[0]) != 1: + raise PixelVaeResizeError( + "Core Video VAE Encode must return one batch per IMAGE sequence" + ) + encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape) + if encoded_batch_shape != expected_batch_shape: + raise PixelVaeResizeError( + "Video VAE encode changed B/C/T or returned the wrong target H/W " + f"for batch {batch_index}: expected {expected_batch_shape}, " + f"got {encoded_batch_shape}" + ) + encoded_batches.append(encoded_batch) + encoded = ( + encoded_batches[0] + if len(encoded_batches) == 1 + else torch.cat(encoded_batches, dim=0) + ) + if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5: + raise PixelVaeResizeError( + "Video VAE encode must return a latent with shape [B,C,T,H,W]" + ) + expected_shape = ( + int(samples.shape[0]), + int(samples.shape[1]), + int(samples.shape[2]), + int(target_latent_height), + int(target_latent_width), + ) + if tuple(int(value) for value in encoded.shape) != expected_shape: + raise PixelVaeResizeError( + "Video VAE encode changed B/C/T or returned the wrong target H/W: " + f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}" + ) + if not bool(torch.isfinite(encoded).all()): + raise PixelVaeResizeError("Video VAE encode contains NaN or Inf") + + # Continuum stores accepted physical latents on CPU; do not retain one GPU + # allocation per physical group while later groups are being converted. + output = encoded.detach().to("cpu").contiguous() + del resized_pixels, encoded_batches, encoded + return output + + +@torch.inference_mode() +def resize_video_latents_via_vae( + video_latents: Sequence[Mapping[str, Any]], + assembly_plan: Mapping[str, Any], + *, + video_vae: Any, + target_width: int, + target_height: int, + upscale_method: str = "lanczos", + upscale_fn: Callable[..., Any] | None = None, +) -> list[dict[str, Any]]: + """Resize physical groups sequentially through the connected H3 Video VAE. + + ``target_width`` and ``target_height`` are pixel dimensions. The expected + encoded latent geometry is derived exclusively from assembly-plan metadata. + VAE failures are intentionally not caught: Core owns its OOM/tiled fallback, + and this path must never fall back to direct latent interpolation. + """ + + groups = _contract_groups(assembly_plan) + if ( + not isinstance(video_latents, Sequence) + or isinstance(video_latents, (str, bytes)) + or len(video_latents) != len(groups) + ): + raise PixelVaeResizeError("video latent group count changed") + if ( + video_vae is None + or not callable(getattr(video_vae, "decode", None)) + or not callable(getattr(video_vae, "encode", None)) + ): + raise PixelVaeResizeError("a usable H3 Video VAE is required") + if upscale_method not in PIXEL_UPSCALE_METHODS: + raise PixelVaeResizeError( + f"unsupported pixel upscale method {upscale_method!r}" + ) + target_width = int(target_width) + target_height = int(target_height) + if target_width < 1 or target_height < 1: + raise PixelVaeResizeError("target pixel width and height must be positive") + + scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan) + if target_width % scale_width or target_height % scale_height: + raise PixelVaeResizeError( + "target pixel geometry is not aligned to the plan-derived latent scale" + ) + target_latent_width = target_width // scale_width + target_latent_height = target_height // scale_height + if upscale_fn is None: + from comfy.utils import common_upscale + + upscale_fn = common_upscale + + outputs: list[dict[str, Any]] = [] + for position, (latent, group) in enumerate( + zip(video_latents, groups, strict=True) + ): + samples = _latent_samples(latent, position=position, group=group) + source_width = int(group["source_width"]) + source_height = int(group["source_height"]) + resized_samples = _resize_one_group( + samples, + video_vae=video_vae, + source_width=source_width, + source_height=source_height, + target_width=target_width, + target_height=target_height, + target_latent_width=target_latent_width, + target_latent_height=target_latent_height, + upscale_method=upscale_method, + upscale_fn=upscale_fn, + ) + output = dict(latent) + output["samples"] = resized_samples + outputs.append(output) + + return outputs diff --git a/v3/plan.py b/v3/plan.py index d3e3d32..2eec276 100644 --- a/v3/plan.py +++ b/v3/plan.py @@ -11,6 +11,7 @@ from ..state import validate_plan ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN" ASSEMBLY_PLAN_SCHEMA_VERSION = 1 FPS = 24 +SECOND_PASS_CONTRACT_VERSION = 1 class AssemblyPlanError(ValueError): @@ -199,6 +200,89 @@ def _recombine_terminal_tensor( return output.contiguous() +def _terminal_physical_prompt( + prompts: list[str], + *, + chunk_seconds: float, +) -> tuple[str, str]: + """Reproduce the established terminal-pair prompt without reparsing it later.""" + first, second = prompts + if first == second: + return first, "shared_prompt_v1" + split = f"{float(chunk_seconds):g}" + stop = f"{float(chunk_seconds) * 2:g}" + return ( + f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}", + "paired_timeline_v1", + ) + + +def _attach_second_pass_contract( + plan: dict[str, Any], + *, + logical_entries: list[dict[str, Any]], + physical_entries: list[dict[str, Any]], + chunk_seconds: float, +) -> None: + """Attach additive V3.5 metadata without changing V3.4 generation semantics.""" + decode_groups = plan.get("decode_groups") + if not isinstance(decode_groups, list): + decode_groups = [ + { + "logical_chunk_indices": [index + 1], + "terminal_merged": False, + "total_frames": plan["chunks"][index]["total_frames"], + "trim_frames": plan["chunks"][index]["trim_frames"], + } + for index in range(len(physical_entries)) + ] + + groups: list[dict[str, Any]] = [] + for group_id, (entry, decode_group) in enumerate( + zip(physical_entries, decode_groups, strict=True) + ): + video = entry["video"] + audio = entry["audio"] + logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]] + prompts = [str(logical_entries[index - 1].get("prompt", "")) for index in logical_chunks] + if len(prompts) == 1: + physical_prompt = prompts[0] + prompt_policy = "single" + elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")): + physical_prompt, prompt_policy = _terminal_physical_prompt( + prompts, + chunk_seconds=chunk_seconds, + ) + else: + physical_prompt = "\n\n".join(prompts) + prompt_policy = "physical_group_v1" + + groups.append( + { + "group_id": group_id, + "logical_chunks": logical_chunks, + "physical_prompt": physical_prompt, + "prompt_policy": prompt_policy, + "physical_frames": int(decode_group["total_frames"]), + "trim_prefix_frames": int(decode_group["trim_frames"]), + "terminal_merged": bool(decode_group.get("terminal_merged", False)), + "source_width": int(plan["width"]), + "source_height": int(plan["height"]), + "source_batch": int(video.shape[0]), + "latent_channels": int(video.shape[1]), + "source_latent_t": int(video.shape[2]), + "source_latent_h": int(video.shape[3]), + "source_latent_w": int(video.shape[4]), + "source_audio_shape": [int(value) for value in audio.shape], + } + ) + + plan["second_pass_contract"] = { + "version": SECOND_PASS_CONTRACT_VERSION, + "physical_groups": groups, + } + + def prepare_physical_decode_entries( entries, *, @@ -215,6 +299,12 @@ def prepare_physical_decode_entries( preserve_final_frame=bool(preserve_final_frame), ) if not terminal_merged: + _attach_second_pass_contract( + plan, + logical_entries=logical_entries, + physical_entries=logical_entries, + chunk_seconds=float(chunk_seconds), + ) return logical_entries, plan if len(logical_entries) < 2: raise ValueError("terminal physical decode requires two logical entries") @@ -287,5 +377,11 @@ def prepare_physical_decode_entries( plan["logical_chunk_count"] = len(logical_chunks) plan["physical_decode_group_count"] = len(decode_groups) plan["decode_groups"] = decode_groups + _attach_second_pass_contract( + plan, + logical_entries=logical_entries, + physical_entries=decode_entries, + chunk_seconds=float(chunk_seconds), + ) validate_assembly_plan(plan) return decode_entries, plan diff --git a/v3/refine_context.py b/v3/refine_context.py new file mode 100644 index 0000000..0b47286 --- /dev/null +++ b/v3/refine_context.py @@ -0,0 +1,546 @@ +"""Runtime-only First Pass conditioning for Continuum-aware refinement. + +The context deliberately remains outside the persisted Assembly Plan contract. It +owns CPU copies of the First Pass conditioning tensors and records them in physical +sampling order. Second Pass callers may then derive a target-canvas conditioning +without mutating the captured source context. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Any + +import torch + +from ..constants import MARK_VIDEO_CONTEXT + + +MAGIC = "H3_CONTINUUM_REFINE_CONTEXT" +VERSION = 1 + + +class RefineContextError(ValueError): + """Raised when the runtime-only refine context is structurally invalid.""" + + +class RefineConditioningAdaptationError(RefineContextError): + """Raised when a valid captured conditioning cannot be adapted safely.""" + + +def _cpu_tensor(value: torch.Tensor) -> torch.Tensor: + return value.detach().to(device="cpu").clone().contiguous() + + +def _freeze(value: Any) -> Any: + if torch.is_tensor(value): + return _cpu_tensor(value) + if isinstance(value, Mapping): + return MappingProxyType({key: _freeze(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_freeze(item) for item in value) + if isinstance(value, (set, frozenset)): + return frozenset(_freeze(item) for item in value) + return value + + +def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]: + """Return a CPU-owned conditioning clone with immutable containers. + + Tensors are detached and cloned, so later mutation or device movement of the + original First Pass conditioning cannot affect the refine context. Container + immutability makes the result safe to share until target adaptation performs + copy-on-write replacement of visual fields. + """ + + if not isinstance(conditioning, Sequence) or isinstance( + conditioning, (str, bytes, bytearray) + ): + raise RefineContextError("conditioning must be a non-empty sequence") + output: list[tuple[torch.Tensor, Mapping]] = [] + for index, entry in enumerate(conditioning): + if not isinstance(entry, Sequence) or isinstance( + entry, (str, bytes, bytearray) + ) or len(entry) != 2: + raise RefineContextError( + f"conditioning entry {index} is not [tensor, metadata]" + ) + tensor, metadata = entry + if not torch.is_tensor(tensor): + raise RefineContextError( + f"conditioning entry {index} text value is not a tensor" + ) + if not isinstance(metadata, Mapping): + raise RefineContextError( + f"conditioning entry {index} metadata is not a mapping" + ) + output.append((_cpu_tensor(tensor), _freeze(metadata))) + if not output: + raise RefineContextError("conditioning is empty") + return tuple(output) + + +def _optional_image(value: Any, name: str) -> torch.Tensor | None: + if value is None: + return None + if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3: + raise RefineContextError( + f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels" + ) + return _cpu_tensor(value) + + +def _validate_optional_image(value: Any, name: str) -> None: + if value is not None and ( + not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3 + ): + raise RefineContextError( + f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels" + ) + + +def make_refine_group( + *, + conditioning: Any, + group_id: int, + logical_chunks: Sequence[int], + physical_frames: int, + prompt_policy: str, + physical_prompt: str, + source_video_shape: Sequence[int], + physical_clip_index: int, + context_frames: int, + first_image: torch.Tensor | None = None, + last_image: torch.Tensor | None = None, +) -> dict[str, Any]: + """Capture one physical First Pass sampling group's effective conditioning.""" + + group_id = int(group_id) + physical_frames = int(physical_frames) + physical_clip_index = int(physical_clip_index) + context_frames = int(context_frames) + chunks = tuple(int(value) for value in logical_chunks) + shape = tuple(int(value) for value in source_video_shape) + if group_id < 0: + raise RefineContextError("refine group id must be non-negative") + if not chunks or any(value <= 0 for value in chunks): + raise RefineContextError("refine group logical chunks are invalid") + if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks): + raise RefineContextError("refine group logical chunks must be ordered and unique") + if physical_frames <= 0: + raise RefineContextError("refine group physical frame count is invalid") + if not str(prompt_policy): + raise RefineContextError("refine group prompt policy is empty") + if len(shape) != 5 or any(value <= 0 for value in shape): + raise RefineContextError("refine group source video shape must be [B,C,T,H,W]") + if physical_clip_index <= 0: + raise RefineContextError("refine group physical clip index is invalid") + if context_frames < 0: + raise RefineContextError("refine group context frame count is invalid") + return { + "group_id": group_id, + "logical_chunks": chunks, + "physical_frames": physical_frames, + "prompt_policy": str(prompt_policy), + "physical_prompt": str(physical_prompt), + "source_video_shape": shape, + "physical_clip_index": physical_clip_index, + "context_frames": context_frames, + "conditioning": freeze_conditioning(conditioning), + "first_image": _optional_image(first_image, "first_image"), + "last_image": _optional_image(last_image, "last_image"), + } + + +def make_refine_context( + groups: Sequence[Mapping[str, Any]], + *, + source_width: int, + source_height: int, + conditioning_mode: str, + complete: bool = True, + notes: Sequence[str] = (), +) -> dict[str, Any]: + """Build and validate a physical-order runtime refine context.""" + + context = { + "magic": MAGIC, + "version": VERSION, + "source_width": int(source_width), + "source_height": int(source_height), + "conditioning_mode": str(conditioning_mode), + "complete": bool(complete), + "groups": tuple(dict(group) for group in groups), + "notes": tuple(str(note) for note in notes), + } + return validate_refine_context(context) + + +def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None: + if not isinstance(value, tuple) or not value: + raise RefineContextError(f"refine group {group_id} conditioning is invalid") + for index, entry in enumerate(value): + if not isinstance(entry, tuple) or len(entry) != 2: + raise RefineContextError( + f"refine group {group_id} conditioning entry {index} is invalid" + ) + tensor, metadata = entry + if not torch.is_tensor(tensor) or tensor.device.type != "cpu": + raise RefineContextError( + f"refine group {group_id} conditioning tensor {index} is not CPU-owned" + ) + if not isinstance(metadata, Mapping): + raise RefineContextError( + f"refine group {group_id} conditioning metadata {index} is invalid" + ) + + +def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]: + value = group.get(name) + if not isinstance(value, (tuple, list)): + raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid") + try: + return tuple(int(item) for item in value) + except (TypeError, ValueError) as exc: + raise RefineContextError( + f"refine group {group.get('group_id')} {name} is invalid" + ) from exc + + +def validate_refine_context( + context: Any, assembly_plan: Mapping[str, Any] | None = None +) -> dict[str, Any]: + """Validate a runtime context and, when supplied, its Assembly Plan pairing.""" + + if not isinstance(context, dict) or context.get("magic") != MAGIC: + raise RefineContextError("invalid H3 Continuum refine context") + if type(context.get("version")) is not int or context["version"] != VERSION: + raise RefineContextError( + f"unsupported refine context version {context.get('version')!r}" + ) + if int(context.get("source_width", 0)) <= 0 or int( + context.get("source_height", 0) + ) <= 0: + raise RefineContextError("refine context source dimensions are invalid") + if not str(context.get("conditioning_mode", "")): + raise RefineContextError("refine context conditioning mode is empty") + if type(context.get("complete")) is not bool: + raise RefineContextError("refine context complete flag is invalid") + groups = context.get("groups") + if not isinstance(groups, tuple): + raise RefineContextError("refine context groups must be an immutable tuple") + if context["complete"] and not groups: + raise RefineContextError("complete refine context contains no physical groups") + notes = context.get("notes") + if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes): + raise RefineContextError("refine context notes must be an immutable string tuple") + + previous_clip_index = 0 + for expected_id, group in enumerate(groups): + if not isinstance(group, Mapping): + raise RefineContextError(f"refine group {expected_id} is not a mapping") + if type(group.get("group_id")) is not int or group["group_id"] != expected_id: + raise RefineContextError("refine context physical group order is not contiguous") + logical_chunks = _group_field_tuple(group, "logical_chunks") + if not logical_chunks or any(value <= 0 for value in logical_chunks): + raise RefineContextError(f"refine group {expected_id} logical chunks are invalid") + if tuple(sorted(logical_chunks)) != logical_chunks or len( + set(logical_chunks) + ) != len(logical_chunks): + raise RefineContextError( + f"refine group {expected_id} logical chunks are not ordered and unique" + ) + if int(group.get("physical_frames", 0)) <= 0: + raise RefineContextError(f"refine group {expected_id} physical frames are invalid") + if not str(group.get("prompt_policy", "")): + raise RefineContextError(f"refine group {expected_id} prompt policy is empty") + if "physical_prompt" not in group: + raise RefineContextError(f"refine group {expected_id} physical prompt is absent") + shape = _group_field_tuple(group, "source_video_shape") + if len(shape) != 5 or any(value <= 0 for value in shape): + raise RefineContextError(f"refine group {expected_id} source shape is invalid") + clip_index = int(group.get("physical_clip_index", 0)) + if clip_index <= previous_clip_index: + raise RefineContextError( + "refine context physical clip indices are not strictly increasing" + ) + previous_clip_index = clip_index + if int(group.get("context_frames", -1)) < 0: + raise RefineContextError(f"refine group {expected_id} context frames are invalid") + _validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id) + _validate_optional_image(group.get("first_image"), "first_image") + _validate_optional_image(group.get("last_image"), "last_image") + + if assembly_plan is not None: + if not isinstance(assembly_plan, Mapping): + raise RefineContextError("assembly plan paired with refine context is invalid") + if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int( + assembly_plan.get("height", 0) + ) != int(context["source_height"]): + raise RefineContextError("refine context source dimensions differ from assembly plan") + contract = assembly_plan.get("second_pass_contract") + plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None + if not isinstance(plan_groups, Sequence): + raise RefineContextError("assembly plan has no Second Pass physical-group contract") + # An incomplete context is diagnostic-only (for example, Run Storage may + # have reused an earlier group that was never sampled in this execution). + # Its local group ids describe capture order, not Assembly Plan position; + # validating those partial entries by index would incorrectly turn the + # intended prompt-only fallback into a broken typed-contract hard error. + if not context["complete"]: + return context + if context["complete"] and len(plan_groups) != len(groups): + raise RefineContextError( + "complete refine context physical-group count differs from assembly plan" + ) + for group in groups: + group_id = int(group["group_id"]) + if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping): + raise RefineContextError( + f"refine group {group_id} has no matching assembly-plan group" + ) + planned = plan_groups[group_id] + comparisons = ( + ("logical_chunks", tuple(int(v) for v in group["logical_chunks"]), + tuple(int(v) for v in planned.get("logical_chunks", ()))), + ("physical_frames", int(group["physical_frames"]), + int(planned.get("physical_frames", 0))), + ("prompt_policy", str(group["prompt_policy"]), + str(planned.get("prompt_policy", ""))), + ("physical_prompt", str(group["physical_prompt"]), + str(planned.get("physical_prompt", ""))), + ("context_frames", int(group["context_frames"]), + int(planned.get("trim_prefix_frames", -1))), + ) + for name, actual, expected in comparisons: + if actual != expected: + raise RefineContextError( + f"refine group {group_id} {name} differs from assembly plan" + ) + expected_shape = ( + int(planned.get("source_batch", 0)), + int(planned.get("latent_channels", 0)), + int(planned.get("source_latent_t", 0)), + int(planned.get("source_latent_h", 0)), + int(planned.get("source_latent_w", 0)), + ) + if tuple(group["source_video_shape"]) != expected_shape: + raise RefineContextError( + f"refine group {group_id} source shape differs from assembly plan" + ) + return context + + +def _common_upscale( + samples: torch.Tensor, + width: int, + height: int, + method: str, + crop: str, +) -> torch.Tensor: + try: + import comfy.utils + except Exception as exc: # pragma: no cover - ComfyUI runtime only + raise RefineConditioningAdaptationError( + f"ComfyUI Core resize support is unavailable: {exc}" + ) from exc + return comfy.utils.common_upscale(samples, width, height, method, crop) + + +def _resize_latent( + latent: Any, + *, + target_h: int, + target_w: int, + method: str, +) -> torch.Tensor: + if not torch.is_tensor(latent) or latent.ndim != 5: + raise RefineConditioningAdaptationError( + "H3 visual conditioning latent must be [B,C,T,H,W]" + ) + batch, channels, temporal, _, _ = (int(value) for value in latent.shape) + flat = latent.permute(0, 2, 1, 3, 4).reshape( + batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1]) + ) + try: + resized = _common_upscale(flat, target_w, target_h, method, "disabled") + except RefineConditioningAdaptationError: + raise + except Exception as exc: + raise RefineConditioningAdaptationError( + f"visual conditioning latent resize failed: {exc}" + ) from exc + if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != ( + target_h, + target_w, + ): + raise RefineConditioningAdaptationError( + "Core resize returned an incompatible visual conditioning shape" + ) + return ( + resized.reshape(batch, temporal, channels, target_h, target_w) + .permute(0, 2, 1, 3, 4) + .contiguous() + ) + + +def _encode_keyframe_image( + image: torch.Tensor, + *, + target_h: int, + target_w: int, + crop: str, + video_vae: Any, +) -> torch.Tensor: + pixels_h = target_h * 16 + pixels_w = target_w * 16 + samples = image[:1, ..., :3].movedim(-1, 1) + try: + resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop) + latent = video_vae.encode(resized.movedim(1, -1).contiguous()) + except RefineConditioningAdaptationError: + raise + except Exception as exc: + raise RefineConditioningAdaptationError( + f"target keyframe VAE encode failed: {exc}" + ) from exc + if not torch.is_tensor(latent) or latent.ndim != 5 or tuple( + int(value) for value in latent.shape[-2:] + ) != (target_h, target_w): + raise RefineConditioningAdaptationError( + "target keyframe VAE encode returned an incompatible latent shape" + ) + return latent.contiguous() + + +def adapt_group_conditioning( + group: Mapping[str, Any], + target_latent_h: int, + target_latent_w: int, + video_vae: Any = None, + upscale_method: str = "bilinear", +) -> tuple[list[list[Any]], dict[str, Any]]: + """Copy-on-write target adaptation for keyframes and Continuum video context. + + Ordinary MiniMax references, all audio latents, and text conditioning tensors + remain shared with the frozen source. Only keyframe video latents and + ``MARK_VIDEO_CONTEXT`` reference video latents are target-sized. + """ + + if not isinstance(group, Mapping): + raise RefineConditioningAdaptationError("refine group is not a mapping") + group_id = int(group.get("group_id", -1)) + _validate_frozen_conditioning(group.get("conditioning"), group_id=group_id) + target_h = int(target_latent_h) + target_w = int(target_latent_w) + if target_h <= 0 or target_w <= 0: + raise RefineConditioningAdaptationError("target latent dimensions are invalid") + physical_frames = int(group.get("physical_frames", 0)) + if physical_frames <= 0: + raise RefineConditioningAdaptationError("refine group physical frames are invalid") + + warnings: list[str] = [] + stats: dict[str, Any] = { + "group_id": group_id, + "target_latent_h": target_h, + "target_latent_w": target_w, + "keyframes_resized": 0, + "keyframes_reencoded": 0, + "keyframes_latent_fallback": 0, + "context_refs_resized": 0, + } + output: list[list[Any]] = [] + for text_tensor, frozen_metadata in group["conditioning"]: + metadata = dict(frozen_metadata) + keyframes = frozen_metadata.get("minimax_keyframes") + if keyframes is not None: + adapted_keyframes = [] + for frozen_keyframe in keyframes: + if not isinstance(frozen_keyframe, Mapping): + raise RefineConditioningAdaptationError( + f"refine group {group_id} contains an invalid keyframe" + ) + keyframe = dict(frozen_keyframe) + latent = keyframe.get("latent") + if not torch.is_tensor(latent) or latent.ndim != 5: + raise RefineConditioningAdaptationError( + f"refine group {group_id} keyframe latent is invalid" + ) + if tuple(int(value) for value in latent.shape[-2:]) != ( + target_h, + target_w, + ): + frame_index = int(keyframe.get("resolved_frame_index", -1)) + image = None + crop = "disabled" + if frame_index == 0: + image = group.get("first_image") + elif frame_index == physical_frames - 1: + image = group.get("last_image") + crop = "center" + if video_vae is not None and torch.is_tensor(image): + keyframe["latent"] = _encode_keyframe_image( + image, + target_h=target_h, + target_w=target_w, + crop=crop, + video_vae=video_vae, + ) + stats["keyframes_reencoded"] += 1 + else: + keyframe["latent"] = _resize_latent( + latent, + target_h=target_h, + target_w=target_w, + method=str(upscale_method), + ) + stats["keyframes_latent_fallback"] += 1 + warning = ( + f"group {group_id} keyframe {frame_index} used latent resize " + "because its source image and video VAE were not both available" + ) + warnings.append(warning) + stats["keyframes_resized"] += 1 + adapted_keyframes.append(keyframe) + metadata["minimax_keyframes"] = adapted_keyframes + + refs = frozen_metadata.get("minimax_refs") + if refs is not None: + adapted_refs = [] + refs_changed = False + for frozen_ref in refs: + if not isinstance(frozen_ref, Mapping): + raise RefineConditioningAdaptationError( + f"refine group {group_id} contains an invalid MiniMax reference" + ) + if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)): + latent = frozen_ref.get("latent") + if not torch.is_tensor(latent) or latent.ndim != 5: + raise RefineConditioningAdaptationError( + f"refine group {group_id} Continuum video context is invalid" + ) + if tuple(int(value) for value in latent.shape[-2:]) != ( + target_h, + target_w, + ): + adapted_ref = dict(frozen_ref) + adapted_ref["latent"] = _resize_latent( + latent, + target_h=target_h, + target_w=target_w, + method=str(upscale_method), + ) + adapted_ref["latent_h"] = target_h + adapted_ref["latent_w"] = target_w + adapted_refs.append(adapted_ref) + refs_changed = True + stats["context_refs_resized"] += 1 + continue + adapted_refs.append(frozen_ref) + if refs_changed: + metadata["minimax_refs"] = adapted_refs + + output.append([text_tensor, metadata]) + stats["warnings"] = tuple(warnings) + return output, stats diff --git a/v3/refine_sampling.py b/v3/refine_sampling.py new file mode 100644 index 0000000..b43f8fb --- /dev/null +++ b/v3/refine_sampling.py @@ -0,0 +1,158 @@ +"""V3.5 Second Pass sampling with the first-pass audio stream locked.""" + +from __future__ import annotations + +from typing import Any, Callable + +import torch + +from ..state import extract_av_streams + + +class RefineSamplingRuntimeError(RuntimeError): + """Raised when the ComfyUI sampling runtime cannot satisfy the refine contract.""" + + +def _make_basic_guider(model: Any, conditioning: list): + try: + import comfy.samplers + except Exception as exc: # pragma: no cover + raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc + + class _BasicGuider(comfy.samplers.CFGGuider): + def set_positive(self, positive): + self.inner_set_conds({"positive": positive}) + + guider = _BasicGuider(model) + guider.set_positive(conditioning) + return guider + + +def _load_runtime_modules(): + try: + import comfy.model_management + import comfy.nested_tensor + import comfy.sample + import comfy.utils + import latent_preview + except Exception as exc: # pragma: no cover + raise RefineSamplingRuntimeError( + f"ComfyUI refine sampling runtime unavailable: {exc}" + ) from exc + return ( + comfy.model_management, + comfy.nested_tensor, + comfy.sample, + comfy.utils, + latent_preview, + ) + + +def _prepare_refine_noise_and_mask( + video: torch.Tensor, + audio: torch.Tensor, + *, + seed: int, + batch_inds: Any = None, + prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor], + nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any], +) -> tuple[Any, Any]: + """Build random video noise and a Core-expandable audio-lock mask.""" + + if video.ndim != 5 or audio.ndim != 4: + raise RefineSamplingRuntimeError( + "refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]" + ) + if int(video.shape[0]) != int(audio.shape[0]): + raise RefineSamplingRuntimeError("refine video/audio batch sizes differ") + + video_noise = prepare_noise_fn(video, int(seed), batch_inds) + if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape): + raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry") + + # Core prepare_noise creates CPU noise. Keep both streams on that same handoff + # device so NestedTensor packing remains valid before Core moves it to the model. + audio_noise = torch.zeros_like(audio, device="cpu") + + # These masks deliberately use the smallest shapes accepted by Core reshape_mask. + # Core expands them to the full stream geometry on the sampling device. + video_mask = torch.ones( + (int(video.shape[0]), 1, 1, 1, 1), + dtype=torch.float32, + device="cpu", + ) + audio_mask = torch.zeros( + (int(audio.shape[0]), 1, 1, 1), + dtype=torch.float32, + device="cpu", + ) + noise = nested_builder((video_noise, audio_noise)) + noise_mask = nested_builder((video_mask, audio_mask)) + return noise, noise_mask + + +def sample_refine_chunk( + *, + model: Any, + conditioning: list, + latent: dict[str, Any], + sampler: Any, + sigmas: torch.Tensor, + seed: int, + enable_preview: bool = True, +) -> dict[str, Any]: + """Refine video while Core carries the first-pass audio as a locked AV stream.""" + + if not isinstance(latent, dict) or "samples" not in latent: + raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary") + if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2: + raise RefineSamplingRuntimeError("sigmas must contain at least two values") + + model_management, nested_tensor, comfy_sample, comfy_utils, preview = ( + _load_runtime_modules() + ) + working = latent.copy() + latent_image = comfy_sample.fix_empty_latent_channels( + model, + working["samples"], + working.get("downscale_ratio_spacial"), + working.get("downscale_ratio_temporal"), + ) + working["samples"] = latent_image + video, audio = extract_av_streams(working) + noise, noise_mask = _prepare_refine_noise_and_mask( + video, + audio, + seed=int(seed), + batch_inds=working.get("batch_index"), + prepare_noise_fn=comfy_sample.prepare_noise, + nested_builder=nested_tensor.NestedTensor, + ) + guider = _make_basic_guider(model, conditioning) + + x0_output: dict[str, Any] = {} + callback = None + if enable_preview: + callback = preview.prepare_callback( + model, + int(sigmas.shape[-1]) - 1, + x0_output, + ) + disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED + samples = guider.sample( + noise, + latent_image, + sampler, + sigmas, + denoise_mask=noise_mask, + callback=callback, + disable_pbar=disable_pbar, + seed=int(seed), + ) + samples = samples.to(model_management.intermediate_device()) + output = working.copy() + output.pop("downscale_ratio_spacial", None) + output.pop("downscale_ratio_temporal", None) + output["samples"] = samples + extract_av_streams(output) + return output diff --git a/v3/second_pass.py b/v3/second_pass.py new file mode 100644 index 0000000..97540af --- /dev/null +++ b/v3/second_pass.py @@ -0,0 +1,395 @@ +"""V3.5 Second Pass contracts and the physical-group sampling runtime.""" + +from __future__ import annotations + +import copy +import hashlib +from collections.abc import Mapping, Sequence +from typing import Any + +import torch + +from ..model_patch import clone_model_for_chunk +from ..state import extract_av_streams +from ..v2.h3_builder import encode_prompt_conditioning +from ..v2.sampling import latent_from_cpu +from .plan import SECOND_PASS_CONTRACT_VERSION +from .refine_context import ( + MAGIC as REFINE_CONTEXT_MAGIC, + RefineConditioningAdaptationError, + RefineContextError, + adapt_group_conditioning, + validate_refine_context, +) +from .refine_sampling import sample_refine_chunk + + +class SecondPassContractError(ValueError): + """Raised only when a Second Pass structural contract is impossible.""" + + +def derive_refine_seed(refine_seed: int, physical_group_index: int) -> int: + """Derive one deterministic seed per physical group in an independent namespace.""" + if type(refine_seed) is not int or refine_seed < 0: + raise SecondPassContractError("refine_seed must be a non-negative integer") + if type(physical_group_index) is not int or physical_group_index < 0: + raise SecondPassContractError("physical_group_index must be a non-negative integer") + payload = f"h3-continuum-refine-v1:{refine_seed}:{physical_group_index}".encode("ascii") + return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big") + + +def _latent_samples(value: Any, *, name: str) -> torch.Tensor: + if not isinstance(value, Mapping) or "samples" not in value: + raise SecondPassContractError(f"{name} must be a LATENT mapping with samples") + samples = value["samples"] + if not isinstance(samples, torch.Tensor): + raise SecondPassContractError(f"{name}.samples must be a tensor") + return samples + + +def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]: + if not isinstance(assembly_plan, Mapping): + raise SecondPassContractError("assembly_plan must be a mapping") + contract = assembly_plan.get("second_pass_contract") + if not isinstance(contract, Mapping): + raise SecondPassContractError("assembly_plan has no second_pass_contract") + if contract.get("version") != SECOND_PASS_CONTRACT_VERSION: + raise SecondPassContractError("unsupported second_pass_contract version") + groups = contract.get("physical_groups") + if not isinstance(groups, list) or not groups or not all(isinstance(g, Mapping) for g in groups): + raise SecondPassContractError("second_pass_contract.physical_groups must be non-empty") + return groups + + +def _validate_video_groups( + video_latents: Sequence[Any], + assembly_plan: Mapping[str, Any], +) -> tuple[list[Mapping[str, Any]], tuple[int, int]]: + groups = _contract_groups(assembly_plan) + if not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)): + raise SecondPassContractError("video_latents must be a sequence") + if len(video_latents) != len(groups): + raise SecondPassContractError("video latent group count changed") + + target_geometry: tuple[int, int] | None = None + for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)): + if group.get("group_id") != position: + raise SecondPassContractError("physical group order or group_id changed") + samples = _latent_samples(latent, name=f"video_latents[{position}]") + if samples.ndim != 5: + raise SecondPassContractError("video latent must have shape [B,C,T,H,W]") + batch, channels, temporal, height, width = (int(value) for value in samples.shape) + if channels != 24: + raise SecondPassContractError("MiniMax H3 video latent must have 24 channels") + if batch != int(group.get("source_batch", -1)): + raise SecondPassContractError("video latent batch changed") + if temporal != int(group.get("source_latent_t", -1)): + raise SecondPassContractError("video latent temporal length changed") + source_h = int(group.get("source_latent_h", -1)) + source_w = int(group.get("source_latent_w", -1)) + if source_h < 1 or source_w < 1 or height < source_h or width < source_w: + raise SecondPassContractError("video latent spatial size must be preserved or enlarged") + if target_geometry is None: + target_geometry = (height, width) + elif target_geometry != (height, width): + raise SecondPassContractError("all physical groups must use the same target H/W") + + assert target_geometry is not None + return groups, target_geometry + + +def validate_second_pass_inputs( + video_latents: Sequence[Any], + audio_latents: Sequence[Any], + assembly_plan: Mapping[str, Any], +) -> dict[str, Any]: + """Validate only structural invariants required by a future Second Pass node.""" + groups, target_geometry = _validate_video_groups(video_latents, assembly_plan) + if not isinstance(audio_latents, Sequence) or isinstance(audio_latents, (str, bytes)): + raise SecondPassContractError("audio_latents must be a sequence") + if len(audio_latents) != len(groups): + raise SecondPassContractError("audio latent group count changed") + + for position, (latent, group) in enumerate(zip(audio_latents, groups, strict=True)): + samples = _latent_samples(latent, name=f"audio_latents[{position}]") + expected_shape = group.get("source_audio_shape") + if not isinstance(expected_shape, list) or tuple(expected_shape) != tuple(samples.shape): + raise SecondPassContractError("audio latent shape or physical group order changed") + + return { + "physical_group_count": len(groups), + "target_latent_h": target_geometry[0], + "target_latent_w": target_geometry[1], + "audio_passthrough": True, + } + + +def passthrough_audio_latents(audio_latents: Sequence[Any]) -> list[Any]: + """Return original LATENT objects; temporary refined audio is not adopted in V1.""" + return list(audio_latents) + + +def update_second_pass_geometry( + assembly_plan: Mapping[str, Any], + video_latents: Sequence[Any], +) -> dict[str, Any]: + """Return a copied plan whose output geometry follows the upscaled physical latents.""" + groups, (target_h, target_w) = _validate_video_groups(video_latents, assembly_plan) + scale_h: int | None = None + scale_w: int | None = None + for group in groups: + source_h = int(group["source_latent_h"]) + source_w = int(group["source_latent_w"]) + source_height = int(group["source_height"]) + source_width = int(group["source_width"]) + if source_height % source_h or source_width % source_w: + raise SecondPassContractError("source pixel/latent geometry is not integral") + group_scale_h = source_height // source_h + group_scale_w = source_width // source_w + if scale_h is None: + scale_h, scale_w = group_scale_h, group_scale_w + elif (scale_h, scale_w) != (group_scale_h, group_scale_w): + raise SecondPassContractError("source geometry scale differs between groups") + + assert scale_h is not None and scale_w is not None + target_height = target_h * scale_h + target_width = target_w * scale_w + updated = copy.deepcopy(dict(assembly_plan)) + updated["width"] = target_width + updated["height"] = target_height + contract = updated["second_pass_contract"] + contract["target_width"] = target_width + contract["target_height"] = target_height + for group in contract["physical_groups"]: + group["target_latent_h"] = target_h + group["target_latent_w"] = target_w + group["target_width"] = target_width + group["target_height"] = target_height + return updated + + +def run_second_pass_groups( + *, + model: Any, + clip: Any, + sampler: Any, + sigmas: torch.Tensor, + video_latents: Sequence[Any], + audio_latents: Sequence[Any], + assembly_plan: Mapping[str, Any], + refine_seed: int, + refine_context: Any = None, + video_vae: Any = None, + conditioning_upscale_method: str = "bilinear", + enable_preview: bool = True, + encode_prompt_fn=None, + latent_builder=None, + sample_fn=None, + stream_extractor=None, + clone_model_fn=None, + validate_refine_context_fn=None, + adapt_group_conditioning_fn=None, +) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]: + """Refine complete physical groups while preserving first-pass audio exactly.""" + + validation = validate_second_pass_inputs(video_latents, audio_latents, assembly_plan) + groups = _contract_groups(assembly_plan) + encode_prompt_fn = encode_prompt_fn or encode_prompt_conditioning + latent_builder = latent_builder or latent_from_cpu + sample_fn = sample_fn or sample_refine_chunk + stream_extractor = stream_extractor or extract_av_streams + clone_model_fn = clone_model_fn or clone_model_for_chunk + validate_refine_context_fn = validate_refine_context_fn or validate_refine_context + adapt_group_conditioning_fn = ( + adapt_group_conditioning_fn or adapt_group_conditioning + ) + + captured_groups: Sequence[Mapping[str, Any]] | None = None + context_warnings: list[str] = [] + if refine_context is None: + context_warnings.append( + "WARNING: refine_context is not connected; using legacy prompt-only " + "conditioning." + ) + else: + typed_context = ( + isinstance(refine_context, Mapping) + and refine_context.get("magic") == REFINE_CONTEXT_MAGIC + ) + try: + validated_context = validate_refine_context_fn( + refine_context, + assembly_plan=assembly_plan, + ) + except RefineContextError as exc: + if typed_context: + raise SecondPassContractError( + f"refine_context contract is invalid: {exc}" + ) from exc + context_warnings.append( + "WARNING: optional refine_context was not a typed Continuum context; " + f"using legacy prompt-only conditioning ({exc})." + ) + else: + if bool(validated_context.get("complete")): + captured_groups = validated_context["groups"] + else: + context_warnings.append( + "WARNING: refine_context capture is incomplete; using legacy " + "prompt-only conditioning." + ) + + refined_videos: list[dict[str, Any]] = [] + group_seeds: list[int] = [] + conditioning_sources: list[str] = [] + report_lines = [ + "H3 Continuum Second Pass V3.5", + "Mode: context-aware physical-group refine when a complete First Pass " + "refine_context is available.", + f"Physical groups: {validation['physical_group_count']}.", + "Sampling contract: video random noise/mask=1; audio zero noise/mask=0; " + "workflow SIGMAS are applied directly by Core.", + *context_warnings, + ] + + for group_index, (video_latent, audio_latent, group) in enumerate( + zip(video_latents, audio_latents, groups, strict=True) + ): + video_samples = _latent_samples(video_latent, name=f"video_latents[{group_index}]") + audio_samples = _latent_samples(audio_latent, name=f"audio_latents[{group_index}]") + captured_group = ( + captured_groups[group_index] if captured_groups is not None else None + ) + model_group = captured_group if captured_group is not None else group + adaptation_stats: Mapping[str, Any] | None = None + context_conditioning_available = False + if captured_group is not None: + try: + conditioning, adaptation_stats = adapt_group_conditioning_fn( + captured_group, + target_latent_h=validation["target_latent_h"], + target_latent_w=validation["target_latent_w"], + video_vae=video_vae, + upscale_method=str(conditioning_upscale_method), + ) + conditioning_source = "refine_context" + context_conditioning_available = True + except RefineConditioningAdaptationError as exc: + report_lines.append( + f"WARNING: group {group_index + 1} refine_context target " + f"adaptation was unavailable; using legacy prompt-only " + f"conditioning ({exc})." + ) + except RefineContextError as exc: + raise SecondPassContractError( + f"refine_context group {group_index} is invalid: {exc}" + ) from exc + + if not context_conditioning_available: + prompt = str(group.get("physical_prompt", "")) + conditioning = encode_prompt_fn( + clip, + prompt, + first_image=None, + last_image=None, + reference_audio_assets=None, + timeline_video_assets=None, + ) + conditioning_source = "prompt_only_fallback" + + logical_chunks = model_group.get("logical_chunks") + if not isinstance(logical_chunks, Sequence) or not logical_chunks: + logical_chunks = group.get("logical_chunks") + if not isinstance(logical_chunks, Sequence) or not logical_chunks: + raise SecondPassContractError( + f"physical group {group_index} has no logical chunk identity" + ) + physical_clip_index = int( + model_group.get("physical_clip_index", logical_chunks[0]) + ) + context_frames = int( + model_group.get( + "context_frames", + group.get("trim_prefix_frames", 0), + ) + ) + if physical_clip_index <= 0 or context_frames < 0: + raise SecondPassContractError( + f"physical group {group_index} MODEL context is invalid" + ) + chunk_model = clone_model_fn( + model, + strict=False, + debug=False, + chunk_index=physical_clip_index, + context_frames=context_frames if context_frames > 0 else None, + ) + physical_seed = derive_refine_seed(int(refine_seed), group_index) + group_seeds.append(physical_seed) + conditioning_sources.append(conditioning_source) + nested_latent = latent_builder(video_samples, audio_samples) + sampled = sample_fn( + model=chunk_model, + conditioning=conditioning, + latent=nested_latent, + sampler=sampler, + sigmas=sigmas, + seed=physical_seed, + enable_preview=bool(enable_preview), + ) + refined_video, _temporary_audio = stream_extractor(sampled) + if tuple(refined_video.shape) != tuple(video_samples.shape): + raise SecondPassContractError( + f"refined video group {group_index} changed B/C/T/H/W geometry" + ) + if not bool(torch.isfinite(refined_video.float()).all().item()): + raise SecondPassContractError( + f"refined video group {group_index} contains NaN or Inf" + ) + output_latent = dict(video_latent) + output_latent["samples"] = refined_video + refined_videos.append(output_latent) + report_lines.append( + "group " + f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, " + f"prompt_policy={group.get('prompt_policy', 'unknown')}, " + f"conditioning_source={conditioning_source}, " + f"physical_clip_index={physical_clip_index}, " + f"context_frames={context_frames}, " + f"refine_seed={physical_seed}, shape={tuple(refined_video.shape)}, " + "sampling_passes=1, temporary_audio_discarded=true" + ) + if adaptation_stats is not None: + report_lines.append( + "group " + f"{group_index + 1} target conditioning: " + f"keyframes_resized={int(adaptation_stats.get('keyframes_resized', 0))}, " + f"keyframes_reencoded={int(adaptation_stats.get('keyframes_reencoded', 0))}, " + f"context_refs_resized={int(adaptation_stats.get('context_refs_resized', 0))}" + ) + report_lines.extend( + f"WARNING: {warning}" + for warning in adaptation_stats.get("warnings", ()) + ) + + passthrough_audio = passthrough_audio_latents(audio_latents) + updated_plan = update_second_pass_geometry(assembly_plan, refined_videos) + contract = updated_plan["second_pass_contract"] + if conditioning_sources and all( + source == "refine_context" for source in conditioning_sources + ): + contract["execution"] = "context_aware_physical_groups_audio_locked_v3" + elif any(source == "refine_context" for source in conditioning_sources): + contract["execution"] = "mixed_context_physical_groups_audio_locked_v3" + else: + contract["execution"] = "t2va_physical_groups_audio_locked_v2" + contract["conditioning_sources"] = conditioning_sources + contract["refine_seed_base"] = int(refine_seed) + contract["refine_group_seeds"] = group_seeds + contract["audio_output"] = "bit_exact_first_pass_passthrough" + contract["audio_sampling"] = "zero_noise_mask_locked" + report_lines.append( + "Audio output: first-pass physical audio LATENT objects returned bit-exact; " + "temporary refined audio discarded." + ) + return refined_videos, passthrough_audio, updated_plan, "\n".join(report_lines) diff --git a/v3/second_pass_nodes.py b/v3/second_pass_nodes.py new file mode 100644 index 0000000..a7f8ca5 --- /dev/null +++ b/v3/second_pass_nodes.py @@ -0,0 +1,110 @@ +"""V3.5 Second Pass public node.""" + +from __future__ import annotations + +from typing import Any + +from .second_pass import run_second_pass_groups + + +def _single_list_input(name: str, value: Any) -> Any: + if not isinstance(value, list) or len(value) != 1: + raise ValueError(f"{name} must contain exactly one value") + return value[0] + + +class H3ContinuumSecondPassV35: + """Low-sigma refinement over complete physical decode groups.""" + + DEPRECATED = False + DESCRIPTION = ( + "V3.5 context-aware Second Pass. Connect externally upscaled H3 video " + "latents, the matching first-pass audio latents, and the original assembly plan. " + "Connect the optional Continuum refine context to preserve the physical group's " + "First/Last/Reference conditioning. " + "SIGMAS controls the low-denoise refine schedule; audio output is preserved " + "bit-exact from the first pass." + ) + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL",), + "clip": ("CLIP",), + "sampler": ("SAMPLER",), + "sigmas": ("SIGMAS",), + "video_latents": ("LATENT",), + "audio_latents": ("LATENT",), + "assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",), + "refine_seed": ( + "INT", + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + "tooltip": "Base seed independently derived once per physical group.", + }, + ), + }, + "optional": { + "refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",), + "video_vae": ("VAE",), + }, + } + + INPUT_IS_LIST = True + RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING") + RETURN_NAMES = ( + "refined_video_latents", + "audio_latents", + "updated_assembly_plan", + "status", + ) + OUTPUT_IS_LIST = (True, True, False, False) + FUNCTION = "refine" + CATEGORY = "MiniMax H3/Continuum/Advanced" + + def refine( + self, + model, + clip, + sampler, + sigmas, + video_latents, + audio_latents, + assembly_plan, + refine_seed, + refine_context=None, + video_vae=None, + ): + return run_second_pass_groups( + model=_single_list_input("model", model), + clip=_single_list_input("clip", clip), + sampler=_single_list_input("sampler", sampler), + sigmas=_single_list_input("sigmas", sigmas), + video_latents=video_latents, + audio_latents=audio_latents, + assembly_plan=_single_list_input("assembly_plan", assembly_plan), + refine_seed=int(_single_list_input("refine_seed", refine_seed)), + refine_context=( + None + if refine_context is None + else _single_list_input("refine_context", refine_context) + ), + video_vae=( + None + if video_vae is None + else _single_list_input("video_vae", video_vae) + ), + ) + + +NODE_CLASS_MAPPINGS = { + "H3ContinuumSecondPassV35": H3ContinuumSecondPassV35, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumSecondPassV35": "H3 Continuum Second Pass V3.5", +} diff --git a/v3/video_seam_v35.py b/v3/video_seam_v35.py new file mode 100644 index 0000000..10c4c08 --- /dev/null +++ b/v3/video_seam_v35.py @@ -0,0 +1,131 @@ +"""Small decoded-boundary patches for the V3.5 direct-write assembler. + +The established V3.4 seam analyzer remains the source of all boundary +decisions. This module reproduces only the correction step while avoiding the +full physical-group clone performed by ``correct_decoded_boundaries``. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .plan import validate_assembly_plan +from .video_seam import VideoBoundaryAnalysis, _validate_frames + + +def build_boundary_patches( + images: list[Any], + assembly_plan: dict[str, Any], + analyses: list[VideoBoundaryAnalysis], + *, + enable_exposure_ramp: bool = False, +) -> tuple[dict[int, torch.Tensor], dict[int, str]]: + """Return qualified 1--4 frame corrections without changing ``images``. + + Patch keys are the existing one-based ``boundary_index`` values. They are + also the zero-based index of the current physical decode group, so the + direct writer can add its natural-frame cursor without this module knowing + the final buffer layout. + + The result is transactional: all patches and actions are built in local + containers and are returned only after every requested boundary succeeds. + Any exception therefore leaves the caller with no partial correction set. + """ + + plan = validate_assembly_plan(assembly_plan) + groups = list( + plan["decode_groups"] if "decode_groups" in plan else plan["chunks"] + ) + if len(images) != len(groups): + raise ValueError("decoded image count does not match the assembly plan") + + patches: dict[int, torch.Tensor] = {} + actions: dict[int, str] = {} + for analysis in analyses: + boundary = int(analysis.boundary_index) + correction_kind = "transient flash" + replace_frames = int(analysis.recommended_rewind) + if analysis.micro_flash_candidate: + correction_kind = "micro-flash" + replace_frames = 1 + elif enable_exposure_ramp and analysis.exposure_ramp_candidate: + correction_kind = "exposure ramp" + replace_frames = 4 + + if replace_frames <= 0: + actions[boundary] = "kept native boundary" + continue + if analysis.scene_cut or not ( + analysis.transient_flash_candidate + or analysis.micro_flash_candidate + or (enable_exposure_ramp and analysis.exposure_ramp_candidate) + ): + actions[boundary] = ( + "kept native boundary; correction guard rejected candidate" + ) + continue + + previous_index = boundary - 1 + current_index = boundary + if previous_index < 0 or current_index >= len(images): + raise ValueError("video seam boundary index is outside decoded chunks") + + previous_raw = images[previous_index] + current_raw = images[current_index] + _validate_frames( + previous_raw, + f"decoded image chunk {previous_index + 1}", + ) + _validate_frames( + current_raw, + f"decoded image chunk {current_index + 1}", + ) + previous_plan = groups[previous_index] + current_plan = groups[current_index] + previous_total = int(previous_plan["total_frames"]) + previous_trim = int(previous_plan["trim_frames"]) + current_total = int(current_plan["total_frames"]) + current_trim = int(current_plan["trim_frames"]) + if current_trim + replace_frames >= current_total: + raise ValueError("video seam correction has no retained recovery frame") + + previous_segment = previous_raw[:previous_total][previous_trim:] + if int(previous_segment.shape[0]) < 1: + raise ValueError("video seam correction has no previous retained frame") + + # Only the frames that may be changed are cloned. The recovery frame + # and previous anchor remain read-only views of their physical groups. + patch = current_raw[ + current_trim : current_trim + replace_frames + ].detach().to(device="cpu").clone() + if int(patch.shape[0]) != replace_frames: + raise ValueError("video seam correction frames are outside the current chunk") + previous_anchor = previous_segment[-1].detach().to( + device="cpu", + dtype=torch.float32, + ) + recovery_anchor = current_raw[current_trim + replace_frames].detach().to( + device="cpu", + dtype=torch.float32, + ) + if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape): + raise ValueError("decoded image geometry changed at video seam") + + for offset in range(replace_frames): + alpha = float(offset + 1) / float(replace_frames + 1) + original = patch[offset].to(dtype=torch.float32) + target = torch.lerp(previous_anchor, recovery_anchor, alpha) + original_mean = original.mean(dim=(0, 1), keepdim=True) + target_mean = target.mean(dim=(0, 1), keepdim=True) + normalized = (original + target_mean - original_mean).clamp(0.0, 1.0) + patch[offset].copy_(normalized.to(dtype=patch.dtype)) + + patches[boundary] = patch + actions[boundary] = ( + f"normalized exposure/color on {replace_frames} {correction_kind} " + "boundary frame(s)" + ) + + return patches, actions diff --git a/version.py b/version.py index 60b6394..0878b10 100644 --- a/version.py +++ b/version.py @@ -1,6 +1,6 @@ """Package version and public schema identifiers.""" -PACKAGE_VERSION = "3.4.0" +PACKAGE_VERSION = "3.5.0" STATE_SCHEMA_VERSION = 1 PLAN_SCHEMA_VERSION = 1 PUBLIC_API_VERSION = 3 diff --git a/web/project_id.js b/web/project_id.js index 8c714ae..a6f6416 100644 --- a/web/project_id.js +++ b/web/project_id.js @@ -5,6 +5,8 @@ const TIMELINE_NODE_CLASS = "H3ContinuumSamplerTimelineVideo"; const ASSEMBLE_SEAM_NODE_CLASS = "H3ContinuumAssembleSeamExperimental"; const V34_NODE_CLASS = "H3ContinuumSamplerV34"; const V34_ASSEMBLE_SEAM_NODE_CLASS = "H3ContinuumAssembleSeamV34"; +const V35_NODE_CLASS = "H3ContinuumSamplerV35"; +const V35_ASSEMBLE_SEAM_NODE_CLASS = "H3ContinuumAssembleSeamV35"; const PROJECT_WIDGET = "project_id"; const LEGACY_RUN_NAME_WIDGET = "run_name"; const CHUNKS_WIDGET = "chunks"; @@ -206,6 +208,7 @@ function configureAssembler(node) { if ( node.comfyClass !== ASSEMBLE_SEAM_NODE_CLASS && node.comfyClass !== V34_ASSEMBLE_SEAM_NODE_CLASS + && node.comfyClass !== V35_ASSEMBLE_SEAM_NODE_CLASS ) { return false; } @@ -222,12 +225,13 @@ function configureNode(node) { const isProduction = node.comfyClass === PRODUCTION_NODE_CLASS; const isTimeline = node.comfyClass === TIMELINE_NODE_CLASS; const isV34 = node.comfyClass === V34_NODE_CLASS; - if (!isProduction && !isTimeline && !isV34) { + const isV35 = node.comfyClass === V35_NODE_CLASS; + if (!isProduction && !isTimeline && !isV34 && !isV35) { configureAssembler(node); return null; } const projectWidget = findWidget(node, PROJECT_WIDGET); - if (isProduction || isV34) { + if (isProduction || isV34 || isV35) { if (projectWidget && !String(projectWidget.value || "").trim()) { projectWidget.value = createProjectId(); } @@ -306,6 +310,7 @@ app.registerExtension({ node.comfyClass === PRODUCTION_NODE_CLASS || node.comfyClass === TIMELINE_NODE_CLASS || node.comfyClass === V34_NODE_CLASS + || node.comfyClass === V35_NODE_CLASS ) { apiNode.inputs.diagnostics = settingValue(SETTINGS.detailedReport, false) ? "Detailed Report" @@ -316,6 +321,7 @@ app.registerExtension({ } else if ( node.comfyClass === ASSEMBLE_SEAM_NODE_CLASS || node.comfyClass === V34_ASSEMBLE_SEAM_NODE_CLASS + || node.comfyClass === V35_ASSEMBLE_SEAM_NODE_CLASS ) { apiNode.inputs.diagnostics = settingValue(SETTINGS.detailedReport, false) ? "Detailed Report"