commit 83590a3935effac0d6f51b8c918c48ec956df6b7 Author: ukr8b3g-cmyk Date: Mon Aug 10 16:28:20 2026 +0900 Release V2.1.4 UI simplification diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc5c780 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +.pytest_cache/ +*.py[cod] +*.safetensors +*.tmp diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ca715cf --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS.md + +## Scope +Native MiniMax H3 audio/video continuation for ComfyUI. V1 supplies manual Join/Finish nodes; V2 integrates N chunks over the same continuation core. + +## Non-negotiable rules +- Do not globally replace `PackedLayout`, `MiniMaxH3.extra_conds`, or other ComfyUI classes. +- Keep SageAttention, Sol-Attn, and Spectrum external; integrate through public ComfyUI wrappers and H3 payload contracts. +- Preserve the identity of `layout.position_ids` when changing MM-RoPE coordinates. +- Never silently resize a continuation State or Session to a different resolution. +- Unknown H3 layout contracts must fail clearly instead of rendering a subtly incorrect join. +- State and Session files remain safetensors + JSON and are written atomically. +- UI nodes call reusable non-UI modules. V1 and V2 must use the same temporal, state, continuation, and layout logic. +- V2 must not decode Video/Audio VAE between H3 chunks. +- Accepted full chunk latents remain on CPU; do not accumulate generated chunks in VRAM. +- Spectrum history must remain run-scoped through its public sampler wrappers; never import Spectrum private runtime functions. +- Changes must preserve V1 node identifiers and State schema unless an explicit migration is included. + +## Validation + +```bash +python -m compileall -q . +pytest -q +``` + +Runtime installer verification must check native PackedLayout behavior and all registered nodes. + +GPU checks must cover Sage only, Sage+Sol, Sage+Spectrum, and Sage+Sol+Spectrum with identical prompts and seeds. V2 must also be compared against the equivalent V1 3×5-second graph. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..60d5dfe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,89 @@ +# Changelog + +## 2.1.4 + +- Make the integrated V2 sampler the only primary Continuum workflow node. +- Mark V1 building blocks and auxiliary plan/state nodes as deprecated Legacy nodes while preserving saved-workflow identifiers. +- Collapse continuation, session, prompt-plan, diagnostics, and advanced generation controls by default. +- Automatically expose Advanced settings when an existing workflow has an advanced connection. +- Preserve output ordering, State/Session schemas, sampling, and accelerator behavior. + +## 2.1.3 + +- Added one stable external `Sequence Prompt` STRING socket for the complete sequence. +- Added `Auto / Fixed / List / Timeline` Prompt Format selection without changing socket connections. +- Added deterministic Auto detection for `---` lists and `[0-5s]` / `[Chunk 1]` timeline sections. +- Kept individual `Clip N Prompt` inputs as optional overrides, collapsed by default while preserving connected legacy sockets. +- Preserved connected Prompt Plan and saved prompt widget fallbacks without changing Prompt Plan schema. + +## 2.1.2 + +- Added fail-closed Native Continuity preflight for raw AV tails, native video-cycle phase, signed 40/24 audio-grid placement, finite Context tensors, and packed Context/Target row separation. +- Added branch-local immutable PackedLayout signatures on native Actual calls while allowing Spectrum Forecast calls to bypass native layout execution. +- Added per-chunk MODEL copy-on-write isolation so Spectrum runtime history, archives, and disable state cannot leak between chunks. +- Added read-only Spectrum Interop API v1 hints only when a valid previous Context exists; initial chunks emit no hint. +- Added Spectrum-compatible Actual Prefix 2 requests with strict metadata types, invalid/unknown fail-open behavior, total-step clamping, and no prefix during offline replay. +- Preserved State, Session, Prompt Plan, Seam Correction, node identifiers, and the raw-latent-only continuation path. + +## 2.1.0 + +- Added opt-in `Seam Correction: Off / Basic` while preserving the complete V2.0.2 decode path for `Off`. +- Added normalized CPU-only video seam scoring and conservative adaptive cuts from 0 to 3 frames before the legacy boundary. +- Added bounded luminance gain, chroma bias, and motion-aware 0–2 frame cosine blending with score-based rollback. +- Added boundary-local stereo-safe audio correlation alignment, level/DC guards, and 10–60ms equal-power crossfades without shifting the following clip. +- Added legacy, cut, and corrected seam scores plus video/audio decisions to Basic and Full diagnostics. +- Added boundary-level fallback without changing Sampling, Context, State, Session, Prompt Plan, V1 nodes, or accelerator composition. + +## 2.0.2 + +- Added external `Clip N Prompt` STRING inputs for 1–16 chunks, intended for ComfyUI's `Text (Multiline)` node. +- Made the visible prompt inputs follow the `chunks` widget while preserving the internal Fixed/List/Timeline values as a legacy fallback. +- Added deterministic per-clip prompt overrides with regenerated prompt hashes and unchanged Prompt Plan schema. +- Added `Settings > H3 Continuum > Preview > Show latent preview`, enabled by default. +- Kept latent-preview control out of the sampler UI and passed the setting to the existing sampling callback without changing final AV output. +- Added regression coverage for selective external prompt overrides, hashes, and the 16 connectable prompt inputs. +- Preserved all V1/V2 node identifiers and State, Session, and Prompt Plan schemas. + +## 2.0.1 + +- Fixed the bundled V2 example workflow widget serialization for `base_seed` with `control_after_generate`. +- Added an in-memory migration guard for the malformed 2.0.0 example where later widgets were shifted by one slot (`diagnostics` arrived as `0`). +- Preserved valid 2.0.0 workflows and all V1/V2 node identifiers and schemas. +- Added regression coverage for the legacy shifted-widget signature. + +## 2.0.0 + +- Added production **H3 Continuum Sampler V2** for integrated N-chunk generation. +- Kept the V1 continuation algorithm and all V1 node identifiers unchanged. +- Added Fixed, List, and Timeline prompt planning plus a connectable Prompt Plan node. +- Cached each unique Qwen prompt conditioning before H3 sampling. +- Reused one externally accelerator-patched MODEL, sampler, and sigma schedule across all chunks. +- Deferred Video/Audio VAE decoding until every H3 chunk finished. +- Captured full raw AV chunk latents and next-state tails on CPU before decoding. +- Removed the second device-to-host state transfer by deriving next-state tails from committed CPU entries. +- Added conservative latent-motion Auto Context selection with 5/22/39-frame profiles. +- Added deterministic independent 64-bit per-chunk seeds and reroll nonces. +- Added V2 Session resume, accepted-prefix reuse, branch-safe reroll, and atomic safetensors/JSON persistence. +- Added exact cumulative audio sample-boundary assembly to prevent long-chain rounding drift. +- Added exact total-duration correction. +- Added single-allocation decoded IMAGE assembly and preflight RAM safety checks. +- Added Basic and Full diagnostics, including video overlap MAE/PSNR and audio correlation/boundary jump. +- Added optional V1 State entry point and V2 Session-to-V1 State conversion. +- Added V2 public API modules and regression/orchestration tests. +- Updated installer runtime verification for all ten registered nodes. + +## 1.0.1 + +- Fixed a false strict-compatibility failure when Sol-Attn wraps MiniMax H3 + `PackedLayout.__init__` with `*args` / `**kwargs` for Morton/span tracking. +- Strict checking still fails closed when the native H3 constructor genuinely + removes required positional or keyword capabilities. +- Added regression tests for the native signature, Sol-Attn forwarding wrapper, + and a genuinely incompatible constructor. + +## 1.0.0 + +- Added **H3 Continuum Join** for initial and continuation clips. +- Added direct native video/audio latent state capture with 5/22/39-frame profiles. +- Represented continuation history as one native H3 `video` / `video_audio` reference block. +- Added Finish, Assemble, atomic State Save/Load, signed audio-grid offsets, in-place MM-RoPE adjustment, strict compatibility tests, and accelerator composition. diff --git a/INSTALL_JA.txt b/INSTALL_JA.txt new file mode 100644 index 0000000..dd1d23d --- /dev/null +++ b/INSTALL_JA.txt @@ -0,0 +1,47 @@ +H3 Continuum Join 2.1.0 — インストール + +1. ComfyUIを終了します。 +2. ZIPを展開します。 +3. フォルダーを次へ配置します。 + + D:\StabilityMatrix\Data\Packages\ComfyUI_W\custom_nodes\ComfyUI-H3-Continuum-Join\ + +4. ComfyUIを再起動します。 +5. ノード検索で「H3 Continuum Sampler V2」を確認します。 + +自動インストール: + + install_windows.bat + +または: + + powershell -ExecutionPolicy Bypass -File .\install_windows.ps1 + +V2標準配線: + +H3 UNET → SageAttention → Sol-Attn → LoRA → Spectrum + ↓ +CLIP / Video VAE / Audio VAE / Sampler / Sigmas + ↓ + H3 Continuum Sampler V2 + ↓ + IMAGE + AUDIO → CreateVideo + +標準値: +- chunks: 3 +- chunk_seconds: 5.0 +- continuity: Balanced — 22 frames +- audio_continuity: true +- exact_total_duration: true +- diagnostics: Basic +- strict_compatibility: true +- seam_correction: Off(2.0.2互換)/ Basic(V2.1境界補正) + +Prompt入力: +- chunksの数だけ表示されるClip N Promptへ、ComfyUI標準のText (Multiline)を接続します。 + +生成中サムネイル: +- Settings > H3 Continuum > Preview > Show latent preview +- 初期値はONです。OFFにしても最終生成結果は変更されません。 + +V1ワークフローも引き続き使用できます。 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 new file mode 100644 index 0000000..021cfd4 --- /dev/null +++ b/MANIFEST.sha256 @@ -0,0 +1,68 @@ +85eada29c12706d55b78e4d8100899d1dadde4d9867406b8bac4eec22b81c6b6 .gitignore +0005be4e1e56a88d5b2168dc3fe203d5f87944922aa400d87e8c840805d2a4e1 AGENTS.md +bc060532bfba782fa2657032566c98c3e406cb5008eec07622a393b21363440b CHANGELOG.md +3db03a23a8f659d1725c8d1ec868d0db7b51604b43f538a488895b9f2b2abe44 INSTALL_JA.txt +3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986 LICENSE +f92943c9a1e53b470d04c9b07dc93eb568e1a3b07b05f8482eff79eba73269dc PACKAGE_INFO_JA.txt +128b77935e21a6489e679c2f34c8dcd28609c804058a03297774e70a16597fc3 PACKAGE_VALIDATION.txt +6d5b51d8111c8651480959ab046bf0a980bbe1f3dbcf490472ae3b6993747caa README.md +4d0d31c3f88eaa6722ed11b03b0c35d76f2ddb2dd8184adae08d0cfb2ae78cb8 README_JA.md +0dbe8a5f71c92f3523942343b40787104c8b96781461ba3ba82934a6bdbe5834 THIRD_PARTY_NOTICES.md +3711de49fc67ff7014eab5dc1bba8d25801de460bcd10f6eb1cf70e00093e60d __init__.py +7dbbc8ce1018d6ac07c53eaa40f2d8d144babf9c6fe6bd94b7ec1d08adffa215 compatibility.py +6fbc2b9e93b034f1b9201fffb18c8940c830e0ce7466fd2ca52984050ca1c063 constants.py +b16f68bf701ef20a39ccfb334cd3bb67c0b245b4296a83cba8252e3b1874cc80 continuation.py +c93a3e52e4deabe5ef0db7f34a7605856a543089574ba17227e697e9a2c0139a docs/ARCHITECTURE.md +7660570840229b6fcbe62f0fbb52966d0099d64d7e6f01a739fbf9899293c8e4 docs/FUTURE_ROADMAP.md +f0806bb2b999794ddda072e98424dbd1b613df1cd8a096b12d979f124e066d8a docs/VALIDATION_JA.md +6771c9905d35666121c4892d2ec26202505f96d244ce814ad9a87a02878952cd examples/H3_CONTINUUM_V1_WIRING.md +c3cefcdf50df9d6609f583a008453b587c011a0fb41e290d5f1196ce62f58877 examples/H3_CONTINUUM_V2_WIRING_JA.md +f85ba3c69a473f0ac4bb96895c6d6532e8270121da65f858e364b19d31f05ef5 examples/H3_Continuum_V2_3x5s.json +68e1a724d3b99458b86b8f643dcb916183e7e8dd12d7b4b3850991feb8077694 examples/recommended_settings_v1.json +5a26408c770381cdb259cb710de7dab12d7ddecca262bdbf35ba1566056050f5 examples/recommended_settings_v2.json +b8275c8cecde12aee0418cea456d1e7f4b875cbb1bce32f1f3d9e06a44dd0be1 examples/v2_prompt_list_example.txt +0e59ea821081cb93165aa3f88c6949736dd225837fe1cdcf1173aa5cd54c6d29 examples/v2_prompt_timeline_example.txt +10c26a052ae8ee8f6fa9168971b92e27da9f0b838cfaf64170e228fbb8e131fa install_windows.bat +c0f1ccebc07cbee0159a2e0fd04683f5725d54e30edd386759e6964b46d84308 install_windows.ps1 +25025ac7e4cf3413b168ced55c1aac0a643a45d51315d1d326cdfe49bf372c24 layout_adapter.py +d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py +385935c0ba3cddf0417bf288dc50bc00b3bdc9aa54e625aef9c2621b6cc68ea0 metadata.ini +3527916298beb55abb5a9d95fcb5056d48259032cc708bb5f66fb29fe010a6c5 model_patch.py +ab770097398609196bbfb9236d3d925337d3e477c63de377689ecdd5323f9979 nodes.py +b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py +524e0b5e7a9f6901e93e3f60fc18ca88484fc3e3345a149b9a193c6237d040f0 pyproject.toml +94fdc2d545b2d4077a4d70229fd22ba3ccac21d49d1097d2873f76d04e9a3c7a pytest.ini +467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt +e3114656ebd5b7be14de50d2a803ac2a727d22c8d0f5befed4401661fd10db2b state.py +6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py +343b3bb0ffd9882a5abde9e3b6e9fc851517ab1b2937cde06ba749c7ee1b91cf temporal.py +3eeed61b7ee519631825e5f89e9a441644c66e552fce4207927cb8536867c8b1 tests/conftest.py +041289556cc621ea1857cbb8931ae41d817f69e084b2e8a7622e184a3197b61b tests/test_compatibility.py +e6fecb17e373333b315f0ce89e8dda3bda9dfdd34dac2656ae8785d26256c422 tests/test_layout_adapter.py +396fc6f02d6bc57758557ba0c84eb35ea067f8962ba633628c79dff975adad67 tests/test_model_patch.py +45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py +a9a4c10770a4a1b50db421d79d58f7aca8efa46232030ea9bd9495199f6e0cf8 tests/test_public_api.py +98a33c75b33ff0c10a68e04bd856e4e43b3e70b6ae1ea458a2b21b4e4d74f46d tests/test_state.py +d4c53d2c260b635d70fa211029d608f63776e0a56c20253475fa47c58363f8a7 tests/test_state_io.py +06ae7327e491a324c7eec4d26b2ae053c96cfa489dc018c07e33eb63facde30c tests/test_temporal.py +912661d91c7f11f5fcaf0bd14d7d3ecf5c74c33b1507272f0ae72af6e5bffa6f tests/test_v2_decoder.py +fcf068c7a7e606b084fa8483b9b50506d05fa370b747114ac958240fd1a73757 tests/test_v2_prompts.py +34bb5b023c504e6ebcaaa52f37a5dd3e7ab592696848f38b44322095bff0cb84 tests/test_v2_registration.py +74e623120a963cb96b68651b44f47a836ea8449d30ce89d749c71ffad4766d4e tests/test_v2_seeds_motion.py +6678a2407cf3d1ffb781cbca8ce8d531a2c055dd82a43cd209e796f9d99020bb tests/test_v2_sequence_orchestration.py +a6e200c21036521383a838070f2fd6e822e1928d06c0fab52b52cc01f64951bb tests/test_v2_session.py +58438c3ed814adf40cefde098a37cae00ddcb703b7e5548bbbb295e54bed5a6d tests/test_v2_widget_migration.py +dbf44a536c550e0cdffc01f8c3725b1b9af60dbeb16f45ff735f5d682276d389 tools/verify_runtime.py +ffd61f3f31c9b6430e1169007ee906c8f5000eba63ee44e2976ad4f5f1112671 v2/__init__.py +f3395f2dc1c595c2318c1cfc800437ddf0539e75dac40050c9552d501d59f118 v2/decoder.py +9acbeff9a811bcff5ae1b9e4ba134b1c54b5975cc7516c740bb348458d8ff309 v2/diagnostics.py +5543ec570f18f1d97327ffcf07f9eeaf078e7c7d3b3d722559d65a19d11dcb84 v2/h3_builder.py +aea01a48a8c03a25ed97ea05e5b137f89e388832d1be7d38f2da2be3d1b9830b v2/motion.py +29a15eee9296b6e1a27761fbd39b9f823798ff8856ced3470551ade2ad6b151e v2/nodes.py +b82dc37d950cd47939f830410207bcfd9fb6c7d8a229c3962b95416678ca9b0d v2/prompts.py +4080f1875b958b316e8e76852c4be5527a6de5c71d604e7642d3098478eb4a36 v2/sampling.py +12cfcc0c71ca087fe39a5ce2922959a62b2bb311bfe664f05c4480affce03708 v2/seeds.py +1371286a3e4b91fb8736c17dff8f615fd2cfa869e21cc09badf9e2e4b1efd931 v2/sequence.py +af89216e549f0a549340645a401b5dccc6aefb73e9f4cbc5e02285117ae46b3a v2/session.py +f3a94dfd2adbc97e2747941d5e50ab07b8b572e9f2684b9e3e827d78dfed86fa v2/session_io.py +d2b789bad2d5a99a28950486008c5cf6523bf6fb803912c95b6346b8787acffd version.py diff --git a/PACKAGE_INFO_JA.txt b/PACKAGE_INFO_JA.txt new file mode 100644 index 0000000..4673890 --- /dev/null +++ b/PACKAGE_INFO_JA.txt @@ -0,0 +1,31 @@ +名称: H3 Continuum Join +バージョン: 2.1.0 +対象: ComfyUI native MiniMax H3 +主ノード: H3 Continuum Sampler V2 +ライセンス: GPL-3.0-or-later +追加pip依存: なし(ComfyUI同梱のPyTorch / safetensorsを使用) + +V2ノード: +- H3 Continuum Sampler V2 +- H3 Continuum Prompt Plan +- H3 Continuum Save Session +- H3 Continuum Load Session +- H3 Continuum Session Info + +V1互換ノード: +- H3 Continuum Join +- H3 Continuum Finish +- H3 Continuum Assemble +- H3 Continuum Save State +- H3 Continuum Load State + +設計: +- V1のnative video/audio latent continuation coreを再利用 +- Nチャンクを1ノードで統合管理 +- Prompt/Identity conditioningをSampling前にキャッシュ +- H3チャンク間のVAE Decodeなし +- CPU Session、再開、指定チャンク以降のリロール +- 累積フレーム基準の音声sample境界 +- SageAttention / Sol-Attn / Spectrumは外部ノードとして併用 +- Spectrumのprivate APIには依存しない +- グローバルH3 Monkey Patchなし diff --git a/PACKAGE_VALIDATION.txt b/PACKAGE_VALIDATION.txt new file mode 100644 index 0000000..4378f2c --- /dev/null +++ b/PACKAGE_VALIDATION.txt @@ -0,0 +1,48 @@ +H3 Continuum Join 2.1.0 — Package Validation +Date: 2026-08-10 + +Release scope: +- Integrated H3 Continuum Sampler V2 for 1–16 native MiniMax H3 chunks. +- V1 Join / Finish / Assemble / State nodes preserved with unchanged node IDs. +- External SageAttention, Sol-Attn, LoRA, and Spectrum composition retained. + +Completed checks: +- Python bytecode syntax compilation: PASS +- Python 3.10 grammar parse: PASS (42 Python files) +- Unit and orchestration tests: PASS (44 tests) +- Package import and node registration: PASS (10 nodes) +- V1 node-ID compatibility: PASS +- V1 State schema compatibility: PASS +- Native H3 17k+5 temporal-grid tests: PASS +- 5 / 22 / 39-frame latent-context mapping: PASS +- Signed 40 Hz audio-grid offset tests: PASS +- PackedLayout in-place MM-RoPE patch tests: PASS +- Sol-Attn position_ids identity preservation: PASS +- Sol-Attn-style *args/**kwargs PackedLayout wrapper compatibility: PASS +- Genuine native-H3 API removal still fails closed: PASS +- Spectrum/public-wrapper marker detection: PASS +- V2 Fixed / List / Timeline prompt parsing: PASS +- Deterministic per-chunk seed and reroll derivation: PASS +- Conservative Auto-context selection: PASS +- Session validation, branch-safe reuse, and V1 State extraction: PASS +- Atomic safetensors + JSON State/Session round-trip: PASS +- Cumulative 24 fps / PCM sample-boundary alignment: PASS +- Exact-duration AV correction and final-anchor preservation tests: PASS +- Decode-after-all-sampling orchestration test: PASS +- Shared MODEL / shared conditioning-cache orchestration test: PASS +- V2 2.0.0 malformed widget-serialization migration regression: PASS +- Example V2 3x5s workflow structural validation: PASS (17 nodes, 20 links) +- Release ZIP manifest verification: performed after archive creation +- Release ZIP extraction and test rerun: performed after archive creation + +Runtime verifier included: +- tools/verify_runtime.py checks the installed ComfyUI native H3 API. +- It constructs and patches a real native PackedLayout without loading the H3 checkpoint. +- It verifies all 10 registered node IDs and the V2 prompt-plan core. + +Not performed in this build container: +- Full MiniMax H3 checkpoint generation on an RTX GPU. +- Perceptual A/B validation of Sage, Sol-Attn, and Spectrum combinations. +- Timing comparison against the equivalent V1 3x5-second workflow on the user's RTX 5060 Ti. + +The first-machine GPU validation matrix is documented in docs/VALIDATION_JA.md. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3516754 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# H3 Continuum Join 2.1 + +Integrated long-form continuation for ComfyUI's native MiniMax H3 audio/video model. + +**H3 Continuum Sampler V2** runs N native H3 chunks through the proven V1 continuation core, keeps text/image preparation outside the H3-only sampling loop, captures AV latent state before decoding, and decodes/assembles the sequence only after all sampling is complete. All V1 node identifiers remain available. + +## V2 highlights + +- One integrated N-chunk sampler node +- Native video/audio latent continuation with 5, 22, 39-frame and conservative Auto profiles +- One externally patched MODEL input with an isolated call-local clone per chunk +- Unique prompt conditioning cached before sampling +- One stable `Sequence Prompt` socket with Auto, Fixed, List, and Timeline parsing +- Optional `Clip N Prompt` overrides follow `chunks` from 1–16 and are collapsed by default +- Local latent-preview toggle under `Settings > H3 Continuum > Preview` +- Optional V2.1 Basic seam correction after deferred decode; Off keeps the V2.0.2 path +- CPU-only adaptive cut, bounded color/luma guard, short motion-aware blend, and boundary-local audio crossfade +- No Video/Audio VAE decode between H3 chunks +- Full chunk latents stored on CPU; VRAM does not grow with accepted chunks +- Session resume, branch-safe reroll, atomic safetensors + JSON persistence +- Cumulative 24fps audio boundaries, preventing per-chunk rounding drift +- Single-allocation final image assembly and preflight RAM safety check +- External SageAttention, Sol-Attn, and Spectrum composition +- No global ComfyUI H3 monkey patch and no accelerator private-API dependency +- No additional pip dependencies + +See [README_JA.md](README_JA.md) for the complete guide. + +## Recommended graph + +```text +H3 UNET → SageAttention → Sol-Attn → LoRA → Spectrum + ↓ +CLIP / Video VAE / Audio VAE / Sampler / Sigmas + ↓ + H3 Continuum Sampler V2 + ↓ + IMAGE + AUDIO → CreateVideo +``` + +## V2 nodes + +- H3 Continuum Sampler V2 +- H3 Continuum Prompt Plan +- H3 Continuum Save Session +- H3 Continuum Load Session +- H3 Continuum Session Info + +V1 compatibility nodes remain registered: Join, Finish, Assemble, Save State, and Load State. + +## Development checks + +```bash +python -m compileall -q . +pytest -q +``` + +## License + +GPL-3.0-or-later. This package was independently implemented against public ComfyUI H3 runtime contracts. Optional accelerator source code is not bundled. +## V2.1.4 Feature Summary + +- One integrated `H3 Continuum Sampler V2` node generates 1 to 16 sequential chunks. +- `Sequence Prompt` accepts one external Text (Multiline) input. +- Prompt formats: `Auto`, `Fixed`, `List`, and `Timeline`. +- Optional per-clip prompt overrides remain collapsed until needed. +- Native video and audio latents pass directly between chunks without VAE decode/re-encode. +- Continuity presets: `Fast - 5 frames`, `Balanced - 22 frames` (default), and `Strong - 39 frames`. +- Seam Correction provides `Off` and `Basic`; `Basic` is the default safe post-decode correction path. +- Exact total duration and audio continuity are enabled by default. +- SageAttention, Sol-Attn, LoRA, and Spectrum remain external and composable. +- Spectrum Interop API v1 requests a two-step Actual prefix only when continuation context exists. +- Advanced continuation, State, Session, Prompt Plan, reroll, and diagnostic controls are collapsed by default. +- Existing advanced connections automatically reopen the Advanced section. +- V1 and auxiliary nodes remain load-compatible but are marked deprecated and moved to the Legacy category. +- Existing State and Session schemas, output ordering, and saved workflow node identifiers are preserved. +- Repository: https://github.com/ukr8b3g-cmyk/ComfyUI-H3-Continuum diff --git a/README_JA.md b/README_JA.md new file mode 100644 index 0000000..c733875 --- /dev/null +++ b/README_JA.md @@ -0,0 +1,368 @@ +# H3 Continuum Join 2.1 + +MiniMax H3を4〜15秒のチャンク単位で連続生成し、前チャンク末尾の**映像latentと音声latent**を次チャンクへ直接継承するComfyUIカスタムノードです。 + +V2の主要ノード **H3 Continuum Sampler V2** は、V1で実機確認されたContinuation処理を変更せず、Nチャンクの準備・Sampling・State管理・Decode・結合を1ノードへ統合します。V1の5ノードもそのまま残るため、既存ワークフローは継続利用できます。 + +## V2の主な特徴 + +- `chunks`を指定するだけでNチャンクを順番に生成 +- V1と同じnative H3 video/audio latent continuation coreを使用 +- 同一MODEL、Sampler、Sigmasを全チャンクで共有 +- CLIP conditioningと入力画像latentをSampling前にまとめて準備 +- H3チャンク間でVideo/Audio VAE Decodeを行わず、全Sampling終了後にDecode +- 全チャンクのraw latentをCPU RAMへ退避し、VRAMに蓄積しない +- Spectrumは1ノードのみ。各チャンクでSpectrum自身のrunを開始・終了 +- SageAttention、Sol-Attn、Spectrumのprivate APIを内蔵・複製しない +- 固定`Sequence Prompt`端子とAuto / Fixed / List / Timeline解析 +- `chunks`に連動する1〜16個の`Clip N Prompt`は折りたたみ式の任意Override +- `Settings > H3 Continuum > Preview`で生成中latentサムネイルをON/OFF +- `Seam Correction: Off / Basic`。Offは2.0.2経路、BasicはDecode後だけの境界補正 +- Basicでは最大3fのAdaptive Cut、制限付き色・輝度補正、最大2f Blend、境界限定Audio補正 +- 同じPromptのQwen conditioningを1回だけ作成して再利用 +- 5 / 22 / 39フレーム、および保守的なAuto Context +- 64bitの決定的なチャンク別Seed +- Session保存、再開、指定チャンク以降の安全なリロール +- Decode時に最終IMAGEを1回だけ確保し、反復`torch.cat`を回避 +- 音声境界を累積フレーム数から計算し、チャンクごとの丸め誤差を蓄積しない +- `chunks × chunk_seconds × 24fps`への最終尺補正 +- 大きすぎる出力はSampling前のRAM見積りで安全停止 +- 追加pip依存なし + +## 対応環境 + +- ComfyUI v0.31.0以降のnative MiniMax H3 +- MiniMax H3 FL2VA / Image-to-Videoモデル +- Python 3.10以降 +- batch size 1 +- Windows / Linux + +本パッケージはComfyUI本体、Spectrum、Sol-Attn、SageAttentionのソースコードを同梱しません。各アクセラレータは別途導入してください。 + +## インストール + +1. ComfyUIを終了します。 +2. ZIPを展開します。 +3. フォルダー全体を次へ配置します。 + +```text +D:\StabilityMatrix\Data\Packages\ComfyUI_W\custom_nodes\ComfyUI-H3-Continuum-Join\ +``` + +または展開後に`install_windows.bat`を実行します。インストーラーは既存フォルダーを日時付きバックアップへ移し、native H3のAPI、PackedLayout、ノード登録を検査します。 + +```powershell +powershell -ExecutionPolicy Bypass -File .\install_windows.ps1 +``` + +導入後、ComfyUIを再起動してください。 + +## V2の最小ワークフロー + +```text +H3 UNET Loader + → SageAttention + → Sol-Attn + → 通常LoRA(必要時) + → Spectrum Apply MiniMax H3 + → H3 Continuum Sampler V2 + +CLIP Loader ───────────────→ H3 Continuum Sampler V2 +Video VAE Loader ──────────→ H3 Continuum Sampler V2 +Audio VAE Loader ──────────→ H3 Continuum Sampler V2 +KSampler Select ───────────→ H3 Continuum Sampler V2 +Basic Scheduler ───────────→ H3 Continuum Sampler V2 +Load Image / scaled IMAGE ─→ first_frame + +H3 Continuum Sampler V2 + ├─ images ─→ CreateVideo + ├─ audio ─→ CreateVideo + ├─ session ─→ Save Session(任意) + └─ report +``` + +MODELへの推奨順は次です。 + +```text +UNET → Sage → Sol → LoRA → Spectrum → H3 Continuum Sampler V2 +``` + +ContinuumはSampler内部でMODELを1回だけ追加cloneし、MODEL単位の`APPLY_MODEL` wrapperを付加します。グローバルなH3クラス変更は行いません。 + +## H3 Continuum Sampler V2 + +### 必須入力 + +- `model`:Sage / Sol / LoRA / Spectrumを適用済みのH3 MODEL +- `clip`:MiniMax用Qwen3-VL CLIP +- `video_vae`:H3 Video VAE +- `audio_vae`:H3 Audio VAE +- `sampler`:標準は`res_multistep` +- `sigmas`:標準は`simple / 20 steps / denoise 1.0` +- `Sequence Prompt`:標準`Text (Multiline)`を1個接続する固定STRING入力 +- `Prompt Format`:Auto / Fixed / List / Timeline。形式変更時も接続を維持 +- `Clip N Prompt`:折りたたみ内の任意Override。接続済み旧端子は折りたたんでも維持 +- `prompt_script`:画面上では非表示。旧Workflow用の内部フォールバックとして保持 +- `chunks`:1〜16 +- `chunk_seconds`:4.0〜15.0。標準5.0 +- `width` / `height`:32の倍数 +- `continuity` +- `base_seed` +- `audio_continuity` +- `exact_total_duration` +- `diagnostics` +- `reroll_from_chunk` / `reroll_nonce` +- `strict_compatibility` +- `seam_correction`:`Off`は2.0.2互換、`Basic`はV2.1の非破壊境界補正 + +### 任意入力 + +- `first_frame`:全チャンクの長期Identity cue。画像ロードは1回だけでよい +- `last_frame`:最終チャンク末尾のFL2VA anchor +- `session`:保存済みV2 Sessionの再開・リロール +- `initial_state`:V1の`H3_CONTINUUM_STATE`からV2を開始 +- `prompt_plan`:`H3 Continuum Prompt Plan`出力を接続可能 + +`session`と`initial_state`は同時接続できません。 + +Promptの優先順位は次のとおりです。 + +```text +外部 Clip N Prompt Override +→ Sequence Prompt +→ prompt_plan +→ 保存済み prompt_mode / prompt_script +``` + +Clip別Overrideがないチャンクは、`Sequence Prompt`、接続済み`prompt_plan`、内部フォールバックの順に対応Promptを維持します。生成中サムネイルは`Settings > H3 Continuum > Preview > Show latent preview`で切り替えます。初期値はONで、OFFにしても最終IMAGE / AUDIOには影響しません。 + +## Promptモード + +### Fixed — one prompt + +同一Promptを全チャンクで使用します。Qwen conditioningは1回だけ作成されます。 + +```text +The same woman continues singing in one uninterrupted shot... +``` + +### List — split with --- + +```text +Chunk 1 prompt... +--- +Chunk 2 prompt... +--- +Chunk 3 prompt... +``` + +Prompt数が不足すると最後のPromptを繰り返し、余分なPromptは無視してReportへ記録します。JSON文字列配列も使用できます。 + +### Timeline — [0-5s] sections + +```text +[0-5s] +First section prompt... + +[5-10s] +Second section prompt... + +[10-15s] +Third section prompt... +``` + +`[Chunk 1]` / `[Clip 2]`形式も使えます。時間範囲がチャンクをまたぐ場合は、重なり時間が最大のセクションを採用します。 + +## Continuity + +- `Auto — conservative`:latent motion量を測定。境界付近は22fへ戻す +- `Balanced — 22 frames`:V1標準。最初の実機検証に推奨 +- `Fast — 5 frames`:静かな動き、速度優先 +- `Strong — 39 frames (Experimental)`:激しい動き・カメラ、計算量増加 + +Autoは前Stateのvideo latent時間差を正規化して判定します。判定不能、State容量不足、しきい値付近では安全側の22fを選びます。39fは実験的です。 + +## IdentityとMotionの分離 + +V2は次を別に扱います。 + +```text +Long-term Identity + = first_frameをQwen visual tokenとして全チャンクで共有 + +Short-term Motion / Audio + = 直前チャンク末尾の5 / 22 / 39f video latent + + 対応するaudio latent +``` + +継続チャンクでは旧first-frameの空間keyframeを外しますが、Qwen visual tokenは人物・衣装・背景のIdentity cueとして残します。前チャンクの姿勢、移動方向、口の動き、カメラ運動、音声はnative latent Contextから継続します。 + +## Sessionとリロール + +V2出力の`session`には、各チャンクのraw AV latent、Prompt hash、Seed、Plan、Context情報をCPUで保持します。 + +```text +H3 Continuum Sampler V2.session + → H3 Continuum Save Session +``` + +保存形式: + +```text +h3_continuum_session_slot0001.safetensors +h3_continuum_session_slot0001.json +``` + +### 再開 + +保存済みSessionをLoadし、Samplerの`session`へ接続します。Prompt、解像度、first_frame fingerprintが一致する採用済みチャンクは再生成しません。追加したチャンクだけ生成します。 + +### リロール + +```text +reroll_from_chunk = 3 +reroll_nonce = 1 +``` + +この場合、Chunk 1〜2を保持し、Chunk 3以降を新しい決定的Seedで生成します。途中チャンクを変更すると、それ以降は旧Continuationと一致しないため自動的に新しいSession branchになります。元Sessionは上書きされません。 + +`reroll_from_chunk=0`は、接続Sessionの一致する採用済みチャンクをすべて再利用します。 + +## SamplingとDecodeの実行順 + +```text +Phase 1 + first/last image resize・VAE encode + unique PromptのQwen encode + +Phase 2 + H3 Chunk 1 Sampling → CPU latent / State + H3 Chunk 2 Sampling → CPU latent / State + H3 Chunk 3 Sampling → CPU latent / State + ※この間Video/Audio Decodeなし + +Phase 3 + Video/Audio VAE Decode + overlap Trim + 累積sample境界で音声を配置 + final IMAGE/AUDIOを組立 +``` + +ComfyUIの動的VRAM管理は引き続き有効です。RTX 5060 Ti 16GBでH3全重みを完全VRAM常駐させるという意味ではありません。V2は**同じMODEL objectを使い、H3チャンク間にCLIP/VAE処理を挟まない**ことで、不要なモデル切替を抑えます。 + +## Spectrum互換 + +SpectrumはV2の外部MODELノードとして1回だけ適用します。V2はSpectrumのprivate APIを呼びません。 + +各チャンクの`guider.sample()`がSpectrumの通常wrapperを通るため、Spectrumはチャンクごとに独立したrunを開始・終了します。前チャンクのSpectrum forecast historyは次チャンクへ流用されません。継承するのはnative video/audio Contextだけです。 + +推奨初期値: + +```text +blend_weight: 0.50 +degree: 1 +warmup_steps: 1 +offline_smoothing_replay: true +audio_blend_weight: 0.00 +``` + +`offline_smoothing_replay=true`ではSpectrumが1チャンクにつきcapture/replayの2 passを行います。V2はこの動作を変更しません。 + +## SageAttention / Sol-Attn互換 + +- SageAttention:Attention kernelを変更せず、そのまま使用 +- Sol-Attn:Continuumは`PackedLayout.position_ids` Tensorを置換せずin-place補正 +- SolからSageへのfallback chainを維持 +- Contextはnative H3 `video` / `video_audio` reference blockとして渡す + +本ノードからSolのtau、sink、Morton設定は変更しません。GPU、解像度、内容ごとにA/B比較してください。 + +## Audio / exact duration + +V2は各セグメントを個別に丸めて連結しません。累積フレーム境界から、次を直接計算します。 + +```text +sample_start = round(cumulative_frames / 24 × sample_rate) +sample_stop = round(next_cumulative_frames / 24 × sample_rate) +``` + +そのため119フレーム区間を多数つないでも、1sample単位の丸め誤差が累積しません。 + +`exact_total_duration=true`では、最終結果を次へ合わせます。 + +```text +target_frames = round(chunks × chunk_seconds × 24) +``` + +余分なフレームは通常末尾からTrimします。`last_frame`が接続されている場合は最終アンカーを失わないよう、余剰を最終フレーム直前から除去し、音声側も短い補間でクリックを抑えながら同じ長さへ合わせます。不足が小さい場合のみ最終フレームと最終音声sampleを最小限反復します。 + +## RAM安全設計 + +長尺の最終`IMAGE`はCPU float tensorになるため、解像度と長さに比例してRAMを消費します。V2はSampling開始前に、 + +- 最終IMAGE +- 1チャンク分のraw Decode +- 基本headroom + +を見積もります。現在利用可能RAMの安全範囲を超える場合は、すべてのH3 Samplingを終えた後で失敗しないよう事前停止します。 + +Decode時は最終IMAGE tensorを1回だけ確保し、各チャンクを所定範囲へcopyします。反復`torch.cat`による全長コピーは行いません。 + +## Diagnostics + +- `Off`:診断を最小化 +- `Basic`:標準。Seed、Context、motion score、アクセラレータ検出、尺、RAM見積り +- `Full`:Basicに加え、接続部のvideo MAE/PSNR、audio correlation、boundary jump + +FullはDecode後の測定であり、外部の顔認識モデルやoptical-flowモデルには依存しません。 + +## V1互換ノード + +次のV1ノードは2.0でも同じIDで残ります。 + +- H3 Continuum Join +- H3 Continuum Finish +- H3 Continuum Assemble +- H3 Continuum Save State +- H3 Continuum Load State + +V1の`H3_CONTINUUM_STATE`はV2の`initial_state`へ直接接続できます。V2 Sessionからは`H3 Continuum Session Info`で最終V1 Stateを取得できます。 + +## 制限 + +- SVI専用学習済みモデルではなく、native H3 conditioningを利用した独自Continuationです +- H3モデル自体の再生成による長期Identity・音色のドリフトを完全には排除できません +- 39fとAutoは内容ごとの実機比較が必要です +- 解像度をまたぐState/Session継続は安全停止します +- Spectrumは近似高速化です。同一SeedでON/OFF比較してください +- 本配布環境ではH3 checkpointを使ったRTX実生成は実施できません。実機検証手順は`docs/VALIDATION_JA.md`を参照してください + +## 同梱ノード + +### V2 + +- H3 Continuum Sampler V2 +- H3 Continuum Prompt Plan +- H3 Continuum Save Session +- H3 Continuum Load Session +- H3 Continuum Session Info + +### V1互換 + +- H3 Continuum Join +- H3 Continuum Finish +- H3 Continuum Assemble +- H3 Continuum Save State +- H3 Continuum Load State + +## 開発・検査 + +```bash +python -m compileall -q . +pytest -q +python tools/verify_runtime.py --comfy-root D:\StabilityMatrix\Data\Packages\ComfyUI_W +``` + +## ライセンス + +GPL-3.0-or-later。 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..4426373 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,12 @@ +# Third-party notices + +H3 Continuum Join interoperates with, but does not bundle source code from: + +- ComfyUI native MiniMax H3 implementation +- ComfyUI-Spectrum-MiniMax-H3 +- ComfyUI-SolAttn_triton +- SageAttention / ComfyUI-KJNodes + +The package calls public ComfyUI sampling, VAE, ModelPatcher wrapper, NestedTensor, and native H3 payload/layout contracts. Optional accelerators must be installed separately and remain governed by their own licenses. + +This package was independently implemented. No source files from the optional accelerator repositories are included. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..325e21e --- /dev/null +++ b/__init__.py @@ -0,0 +1,29 @@ +"""H3 Continuum Join — native MiniMax H3 chunk continuation.""" + +from __future__ import annotations + +import logging + +WEB_DIRECTORY = "./web" + +# ComfyUI loads custom-node folders as packages. Standalone test collection may +# import this file as a top-level module; keep that path inert because ComfyUI is +# intentionally not a unit-test dependency. +if __package__: + from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + from .temporal import run_temporal_self_test + from .version import PACKAGE_VERSION + + try: + run_temporal_self_test() + except Exception as exc: + raise RuntimeError(f"H3 Continuum Join self-test failed: {exc}") from exc + logging.getLogger("h3_continuum_join").info( + "H3 Continuum Join %s loaded (V2 integrated sampler + hidden legacy workflow compatibility)", + PACKAGE_VERSION, + ) +else: # pragma: no cover + NODE_CLASS_MAPPINGS = {} + NODE_DISPLAY_NAME_MAPPINGS = {} + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/compatibility.py b/compatibility.py new file mode 100644 index 0000000..5a1b1d0 --- /dev/null +++ b/compatibility.py @@ -0,0 +1,269 @@ +"""ComfyUI native-H3 and accelerator compatibility checks.""" + +from __future__ import annotations + +import inspect +from typing import Any, Iterable + + +class CompatibilityError(RuntimeError): + """Raised when a MODEL is not the supported native MiniMax H3 runtime.""" + + +def _missing_callable_parameters( + callable_obj: Any, + *, + positional: Iterable[str] = (), + keywords: Iterable[str] = (), +) -> list[str]: + """Return parameters that a callable can no longer accept. + + Compatibility wrappers are allowed to replace a concrete signature with + ``*args``/``**kwargs``. Sol-Attn's MiniMax-H3 Morton integration does this + for ``PackedLayout.__init__`` while transparently forwarding the original + arguments. Treating the wrapped signature as an API removal produces a + false incompatibility even though construction remains valid. + + Explicit parameters are preferred when available. ``*args`` satisfies + missing positional parameters and ``**kwargs`` satisfies missing keyword + parameters. If ``inspect.signature`` itself is unavailable, the caller + still receives the exception and may fail closed. + """ + + params = inspect.signature(callable_obj).parameters + has_var_positional = any( + parameter.kind is inspect.Parameter.VAR_POSITIONAL + for parameter in params.values() + ) + has_var_keyword = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in params.values() + ) + + missing: list[str] = [] + for name in positional: + if name not in params and not has_var_positional: + missing.append(name) + for name in keywords: + if name not in params and not has_var_keyword: + missing.append(name) + return missing + + +def check_comfy_h3_runtime() -> list[str]: + """Return human-readable incompatibilities with ComfyUI's native H3 API. + + The check deliberately validates public/runtime contracts only. It does not + import optional accelerators and therefore cannot make their installation a + hard dependency. + """ + + issues: list[str] = [] + try: + import comfy.ldm.minimax.model as h3_model + except Exception as exc: # pragma: no cover - requires a broken Comfy install + return [f"cannot import native MiniMax H3: {exc}"] + + layout_cls = getattr(h3_model, "PackedLayout", None) + model_cls = getattr(h3_model, "MiniMaxH3Model", None) + if layout_cls is None: + issues.append("PackedLayout is missing") + else: + try: + missing = _missing_callable_parameters( + layout_cls.__init__, + positional=( + "text_len", + "latent_t", + "latent_h", + "latent_w", + "audio_t", + ), + keywords=( + "keyframes", + "refs", + "frame_count", + ), + ) + for name in missing: + issues.append(f"PackedLayout no longer accepts '{name}'") + except (TypeError, ValueError) as exc: + issues.append(f"cannot inspect PackedLayout: {exc}") + + if model_cls is None: + issues.append("MiniMaxH3Model is missing") + else: + for name in ( + "forward", + "rope_freqs", + "_cond_video_rows", + "_cond_audio_rows", + ): + if not hasattr(model_cls, name): + issues.append(f"MiniMaxH3Model.{name} is missing") + + if tuple(getattr(h3_model, "FRAME_PER_TOKEN", ())) != (1, 4, 4, 4, 4): + issues.append("MiniMax H3 temporal compression grid changed") + + try: + import comfy.model_base as model_base + + base_cls = getattr(model_base, "MiniMaxH3", None) + if base_cls is None or not hasattr(base_cls, "extra_conds"): + issues.append("native MiniMaxH3 BaseModel.extra_conds is missing") + except Exception as exc: + issues.append(f"cannot inspect MiniMaxH3 BaseModel: {exc}") + + try: + from comfy.patcher_extension import WrappersMP + + for name in ("APPLY_MODEL", "DIFFUSION_MODEL"): + if not hasattr(WrappersMP, name): + issues.append(f"WrappersMP.{name} is missing") + except Exception as exc: + issues.append(f"cannot import model wrapper API: {exc}") + + try: + import comfy.nested_tensor + + if not hasattr(comfy.nested_tensor, "NestedTensor"): + issues.append("comfy.nested_tensor.NestedTensor is missing") + except Exception as exc: + issues.append(f"cannot import NestedTensor API: {exc}") + return issues + + + +def run_native_layout_self_test() -> str: + """Exercise the installed native H3 PackedLayout without loading a model. + + This catches the most dangerous class of upstream change: a layout that can + still be constructed but no longer maps reference/target rows the way the + Continuum adapter expects. + """ + + import torch + import comfy.ldm.minimax.model as h3_model + + from .constants import ( + MARK_AUDIO_CONTEXT, + MARK_AUDIO_END_FRAME, + MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, + MARK_VIDEO_CONTEXT, + ) + from .layout_adapter import normalize_condition_latents, patch_layout_in_place + + video = torch.zeros(1, 24, 2, 2, 2) # five-frame context + audio = torch.zeros(1, 32, 2, 8) + ref = { + "kind": "video_audio", + "latent_t": 2, + "latent_h": 2, + "latent_w": 2, + "ref_audio_t": 8, + "latent": video, + "audio_latent": audio, + MARK_VIDEO_CONTEXT: True, + MARK_CONTEXT_FRAMES: 5, + MARK_AUDIO_CONTEXT: True, + MARK_AUDIO_END_FRAME: 5.0, + MARK_AUDIO_OVERHANG: -1.0 / 3.0, + } + layout = h3_model.PackedLayout( + 3, 7, 2, 2, 10, keyframes=None, refs=[ref], frame_count=22 + ) + position_id = id(layout.position_ids) + payload = {"layout": layout, "keyframes": [], "refs": [ref]} + normalize_condition_latents(payload) + result = patch_layout_in_place(payload, strict=True, debug=False) + if id(layout.position_ids) != position_id: + raise CompatibilityError("layout self-test replaced position_ids") + ref_video = next((a, b) for a, b, kind in layout.segments if kind == "ref_img") + target_video = next((a, b) for a, b, kind in layout.segments if kind == "video") + ra, rb = ref_video + va, _vb = target_video + if not torch.equal(layout.position_ids[ra:rb], layout.position_ids[va : va + (rb - ra)]): + raise CompatibilityError("layout self-test could not align video context") + return ( + f"native PackedLayout self-test passed; rows={layout.position_ids.shape[0]}, " + f"position_ids_id={result['position_ids_id']}" + ) + +def ensure_native_h3_base_model(base_model: Any) -> None: + """Validate a ComfyUI BaseModel instance before adding our wrapper.""" + + inner = getattr(base_model, "diffusion_model", None) + if inner is None: + raise CompatibilityError("MODEL does not expose diffusion_model") + actual = f"{type(inner).__module__}.{type(inner).__name__}" + if actual != "comfy.ldm.minimax.model.MiniMaxH3Model": + raise CompatibilityError(f"native MiniMaxH3Model required; got {actual}") + missing = [ + name + for name in ( + "blocks", + "final_layer", + "patch_size", + "latents_dim", + "audio_latents_dim", + "rope_freqs", + ) + if not hasattr(inner, name) + ] + if missing: + raise CompatibilityError( + "MiniMaxH3Model contract changed; missing " + ", ".join(missing) + ) + + +def _wrapper_keys(model: Any) -> tuple[str, ...]: + """Collect ModelPatcher wrapper keys without depending on enum values.""" + + wrappers = getattr(model, "wrappers", {}) or {} + found: list[str] = [] + if not isinstance(wrappers, dict): + return () + for group in wrappers.values(): + if not isinstance(group, dict): + continue + found.extend(str(key) for key in group.keys()) + return tuple(found) + + +def _contains_any(values: Iterable[str], *needles: str) -> bool: + text = " ".join(values).lower() + return any(needle.lower() in text for needle in needles) + + +def accelerator_summary(model: Any) -> str: + """Best-effort, non-authoritative summary of attached accelerators. + + Optional custom nodes expose different public markers across releases. The + summary therefore reports only what is observable and never gates execution. + """ + + options = getattr(model, "model_options", {}) or {} + transformer = options.get("transformer_options", {}) or {} + wrapper_keys = _wrapper_keys(model) + parts: list[str] = [] + + if _contains_any(wrapper_keys, "spectrum"): + parts.append("Spectrum wrapper detected") + + sol_marker = transformer.get("sol_compose") is not None or _contains_any( + wrapper_keys, "sol_attn", "sol-attn", "solattn" + ) + if sol_marker: + parts.append("Sol-Attn marker detected") + + if transformer.get("optimized_attention_override") is not None: + if sol_marker: + parts.append("chained attention override present (Sage/other fallback possible)") + else: + parts.append("attention override present (Sage/other)") + + if options.get("h3_continuum_join"): + parts.append("Continuum APPLY_MODEL wrapper installed") + + return "; ".join(parts) if parts else "accelerator markers not detected (informational only)" diff --git a/constants.py b/constants.py new file mode 100644 index 0000000..9427c36 --- /dev/null +++ b/constants.py @@ -0,0 +1,74 @@ +"""Constants shared by H3 Continuum Join.""" + +from __future__ import annotations + +FPS = 24.0 +AUDIO_SAMPLE_RATE = 32_000 +AUDIO_LATENT_FPS = 40.0 +FRAME_RESCALE = AUDIO_LATENT_FPS / FPS +FRAME_PER_TOKEN = (1, 4, 4, 4, 4) + +STATE_MAGIC = "H3_CONTINUUM_STATE" +PLAN_MAGIC = "H3_CONTINUUM_PLAN" +SESSION_MAGIC = "H3_CONTINUUM_SESSION" +PROMPT_PLAN_MAGIC = "H3_CONTINUUM_PROMPT_PLAN" + +MARK_SCHEMA = "_h3cj_schema" +MARK_VIDEO_SLOT = "_h3cj_video_slot" # legacy/internal-slot compatibility +MARK_PIXEL_START = "_h3cj_pixel_start" +MARK_VIDEO_CONTEXT = "_h3cj_video_context" +MARK_CONTEXT_FRAMES = "_h3cj_context_frames" +MARK_AUDIO_CONTEXT = "_h3cj_audio_context" +MARK_AUDIO_END_FRAME = "_h3cj_audio_end_frame" +MARK_AUDIO_OVERHANG = "_h3cj_audio_overhang" +CONTINUUM_REFERENCE_METADATA_KEY = "_h3_continuum" +CONTINUUM_INTEROP_KEY = "h3_continuum" +CONTINUUM_INTEROP_API = 1 +CONTINUUM_ACTUAL_PREFIX_STEPS = 2 + +MODEL_WRAPPER_KEY = "h3_continuum_join.apply_model.v1" +V2_RUNTIME_KEY = "h3_continuum_v2" +LAYOUT_ORIGINAL_TIME_ATTR = "_h3cj_original_time_v1" +LAYOUT_SIGNATURE_ATTR = "_h3cj_patch_signature_v1" +LAYOUT_DEVICE_CACHE_ATTR = "_h3cj_device_cache_v1" + +CONTINUITY_OPTIONS = ( + "Balanced — 22 frames", + "Fast — 5 frames", + "Strong — 39 frames (Experimental)", +) +CONTINUITY_FRAMES = { + CONTINUITY_OPTIONS[0]: 22, + CONTINUITY_OPTIONS[1]: 5, + CONTINUITY_OPTIONS[2]: 39, +} +DEFAULT_STATE_CAPACITY_FRAMES = 39 + + +V2_CONTINUITY_AUTO = "Auto — conservative" +V2_CONTINUITY_OPTIONS = (V2_CONTINUITY_AUTO,) + CONTINUITY_OPTIONS + +PROMPT_MODE_FIXED = "Fixed — one prompt" +PROMPT_MODE_LIST = "List — split with ---" +PROMPT_MODE_TIMELINE = "Timeline — [0-5s] sections" +PROMPT_MODE_OPTIONS = (PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE) + +PROMPT_FORMAT_AUTO = "Auto" +PROMPT_FORMAT_FIXED = "Fixed" +PROMPT_FORMAT_LIST = "List" +PROMPT_FORMAT_TIMELINE = "Timeline" +PROMPT_FORMAT_OPTIONS = ( + PROMPT_FORMAT_AUTO, + PROMPT_FORMAT_FIXED, + PROMPT_FORMAT_LIST, + PROMPT_FORMAT_TIMELINE, +) + +DIAGNOSTICS_OFF = "Off" +DIAGNOSTICS_BASIC = "Basic" +DIAGNOSTICS_FULL = "Full" +DIAGNOSTICS_OPTIONS = (DIAGNOSTICS_BASIC, DIAGNOSTICS_FULL, DIAGNOSTICS_OFF) + +SEAM_CORRECTION_OFF = "Off" +SEAM_CORRECTION_BASIC = "Basic" +SEAM_CORRECTION_OPTIONS = (SEAM_CORRECTION_OFF, SEAM_CORRECTION_BASIC) diff --git a/continuation.py b/continuation.py new file mode 100644 index 0000000..a4718d4 --- /dev/null +++ b/continuation.py @@ -0,0 +1,183 @@ +"""Reusable native-H3 continuation preparation independent of ComfyUI UI nodes.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .constants import ( + CONTINUUM_INTEROP_API, + CONTINUUM_REFERENCE_METADATA_KEY, + MARK_AUDIO_CONTEXT, + MARK_AUDIO_END_FRAME, + MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, + MARK_SCHEMA, + MARK_VIDEO_CONTEXT, +) +from .state import extract_av_streams +from .version import STATE_SCHEMA_VERSION + +POLICY_REPLACE = "Use Continuum context; keep visual identity tokens (Recommended)" +POLICY_ERROR = "Error if a first-frame keyframe exists" +POLICY_KEEP = "Keep first-frame keyframe too (Advanced)" +FIRST_FRAME_POLICIES = (POLICY_REPLACE, POLICY_ERROR, POLICY_KEEP) + + +def clone_conditioning(conditioning): + """Clone the outer conditioning list and metadata dictionaries.""" + + output = [] + for index, item in enumerate(conditioning): + if not isinstance(item, (tuple, list)) or len(item) != 2: + raise ValueError(f"conditioning entry {index} is not [tensor, metadata]") + if not isinstance(item[1], dict): + raise ValueError(f"conditioning entry {index} metadata is not a dictionary") + output.append([item[0], dict(item[1])]) + if not output: + raise ValueError("conditioning is empty") + return output + + +def conditioning_diagnostics(conditioning) -> dict[str, int]: + """Return non-authoritative cues used in the Join report.""" + + first_keyframes = 0 + visual_tokens = 0 + for item in conditioning: + if not isinstance(item, (tuple, list)) or len(item) != 2: + continue + metadata = item[1] if isinstance(item[1], dict) else {} + for keyframe in metadata.get("minimax_keyframes") or (): + if int(keyframe.get("resolved_frame_index", 0)) == 0: + first_keyframes += 1 + tags = metadata.get("minimax_token_tags") + if torch.is_tensor(tags): + visual_tokens += int((tags == 0).sum().item()) + return {"first_keyframes": first_keyframes, "visual_tokens": visual_tokens} + + +def prepare_conditioning( + conditioning, + *, + video_context: torch.Tensor, + audio_context: torch.Tensor | None, + audio_grid_offset: float, + context_frames: int, + new_frame_count: int, + first_frame_policy: str, + preserve_last_frame: bool, +): + """Add one native H3 video/video-audio reference as timeline context. + + A single multi-slot reference keeps the previous motion latent intact and + avoids relying on ComfyUI's first/last-only keyframe constructor. The + per-model wrapper moves the reference's MM-RoPE rows onto target time zero. + """ + + if first_frame_policy not in FIRST_FRAME_POLICIES: + raise ValueError(f"unknown first-frame policy: {first_frame_policy!r}") + if not torch.is_tensor(video_context) or video_context.ndim != 5: + raise ValueError("video_context must be [1,24,T,H,W]") + if tuple(video_context.shape[:2]) != (1, 24): + raise ValueError("video_context must have shape [1,24,T,H,W]") + if audio_context is not None: + if not torch.is_tensor(audio_context) or audio_context.ndim != 4: + raise ValueError("audio_context must be [1,32,2,T]") + if tuple(audio_context.shape[:3]) != (1, 32, 2): + raise ValueError("audio_context must have shape [1,32,2,T]") + + context_t = int(video_context.shape[2]) + context_ref: dict[str, Any] = { + "kind": "video_audio" if audio_context is not None else "video", + "latent_t": context_t, + "latent_h": int(video_context.shape[-2]), + "latent_w": int(video_context.shape[-1]), + "ref_audio_t": int(audio_context.shape[-1]) if audio_context is not None else 0, + "latent": video_context.contiguous(), + MARK_SCHEMA: STATE_SCHEMA_VERSION, + MARK_VIDEO_CONTEXT: True, + MARK_CONTEXT_FRAMES: int(context_frames), + CONTINUUM_REFERENCE_METADATA_KEY: { + "api": CONTINUUM_INTEROP_API, + "role": "video_context", + "audio_role": "audio_context" if audio_context is not None else None, + }, + } + if audio_context is not None: + context_ref.update( + { + "audio_latent": audio_context.contiguous(), + MARK_AUDIO_CONTEXT: True, + MARK_AUDIO_END_FRAME: float(context_frames), + # Schema-v1 name retained; value is a signed grid offset. + MARK_AUDIO_OVERHANG: float(audio_grid_offset), + } + ) + + output = clone_conditioning(conditioning) + for _, metadata in output: + old_frame_count = metadata.get("minimax_frame_count") + old_keyframes = [dict(item) for item in (metadata.get("minimax_keyframes") or [])] + kept = [] + for keyframe in old_keyframes: + frame_index = int(keyframe.get("resolved_frame_index", 0)) + if frame_index == 0: + if first_frame_policy == POLICY_ERROR: + raise ValueError( + "conditioning already has a first-frame keyframe. Disconnect the image " + "for continuation, or use the recommended identity-token policy." + ) + if first_frame_policy == POLICY_KEEP: + kept.append(keyframe) + # POLICY_REPLACE drops the spatial keyframe. Qwen visual tokens, + # if present, remain in conditioning as a global identity cue. + continue + + is_last = old_frame_count is not None and frame_index == int(old_frame_count) - 1 + if is_last and preserve_last_frame: + keyframe["resolved_frame_index"] = int(new_frame_count) - 1 + kept.append(keyframe) + elif is_last: + continue + else: + raise ValueError( + f"unsupported existing H3 keyframe index {frame_index}; only first/last are safe" + ) + + if kept: + metadata["minimax_keyframes"] = kept + else: + metadata.pop("minimax_keyframes", None) + metadata["minimax_frame_count"] = int(new_frame_count) + + refs = [ + dict(item) + for item in (metadata.get("minimax_refs") or []) + if not (item.get(MARK_VIDEO_CONTEXT) or item.get(MARK_AUDIO_CONTEXT)) + ] + refs.append(dict(context_ref)) + metadata["minimax_refs"] = refs + return output + + +def new_h3_latent(template: dict[str, Any], *, video_t: int, audio_t: int): + """Create an empty native H3 AV latent matching the template geometry/dtype.""" + + try: + import comfy.model_management + import comfy.nested_tensor + except Exception as exc: # pragma: no cover - requires ComfyUI runtime + raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc + video, audio = extract_av_streams(template) + device = comfy.model_management.intermediate_device() + new_video = torch.zeros( + (1, 24, int(video_t), int(video.shape[-2]), int(video.shape[-1])), + device=device, + dtype=video.dtype, + ) + new_audio = torch.zeros( + (1, 32, 2, int(audio_t)), device=device, dtype=audio.dtype + ) + return {"samples": comfy.nested_tensor.NestedTensor((new_video, new_audio))} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..bf973ea --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,177 @@ +# Architecture — H3 Continuum Join 2.0 + +## 1. Design goals + +1. Preserve the V1 native H3 continuation behavior. +2. Integrate N chunks without duplicating V1 temporal/layout logic. +3. Keep SageAttention, Sol-Attn, and Spectrum external and independently updatable. +4. Avoid global ComfyUI class mutation. +5. Fail closed when native H3 layout contracts change. +6. Prevent H3/VAE/CLIP model switching between continuation chunks. +7. Keep accepted chunk latents on CPU and bound transient decode memory. +8. Preserve exact long-chain A/V timing. + +## 2. Module layers + +```text +UI + nodes.py V1 compatibility nodes + v2/nodes.py integrated V2 nodes + +V2 orchestration + v2/sequence.py phases, session reuse, reroll, reports + v2/sampling.py ComfyUI-native sampler path + v2/h3_builder.py empty AV latent, prompt and keyframe conditioning + v2/decoder.py deferred decode, preallocated assembly + v2/prompts.py Fixed/List/Timeline parser + v2/session.py in-memory accepted chunk/session model + v2/session_io.py atomic session persistence + v2/motion.py conservative latent motion analysis + v2/seeds.py deterministic per-chunk seed derivation + v2/diagnostics.py lightweight seam measurements + +Shared continuation core + continuation.py native H3 video/video_audio reference construction + model_patch.py per-MODEL APPLY_MODEL wrapper + layout_adapter.py in-place MM-RoPE placement and payload normalization + temporal.py H3 17k+5 / 24fps / 40Hz grid + state.py V1 State and Plan schemas + media.py V1 trim/assembly helpers + compatibility.py runtime contract checks +``` + +V2 calls the shared continuation core. It does not implement a second continuation algorithm. + +## 3. Three execution phases + +### Phase 1 — conditioning preparation + +- Resize first/last images once. +- VAE-encode their keyframe latents once. +- Tokenize and encode each unique Prompt once. +- Clone the externally patched MODEL once and add the Continuum APPLY_MODEL wrapper. + +No H3 diffusion sampling begins until this phase completes. + +### Phase 2 — H3-only loop + +For each chunk: + +1. Create native H3 NestedTensor AV latent. +2. Select 5/22/39-frame Context or Auto. +3. Add one native `video` / `video_audio` reference block. +4. Call the standard ComfyUI guider/sampler path. +5. Let Spectrum create and tear down its own run through its public wrappers. +6. Copy the sampled full AV latent to CPU once. +7. Derive the next V1-compatible State from that CPU chunk. + +Video and Audio VAE decode are deliberately absent from this loop. + +### Phase 3 — deferred decode + +- Decode each CPU chunk after all H3 sampling. +- Immediately move decoded tensors to CPU. +- Remove the overlap. +- Copy images into one preallocated final IMAGE tensor. +- Place audio into cumulative 24fps-derived sample ranges. +- Release each raw chunk before decoding the next. +- Apply exact final duration if enabled. + +## 4. Native H3 continuation representation + +V1 and V2 both represent the previous tail as one reference block: + +```text +kind: video or video_audio +latent: native H3 video latent tail +latent_t/h/w: native geometry +audio_latent: optional native H3 audio latent tail +ref_audio_t: audio latent length +_h3cj_* metadata: timeline placement contract +``` + +The outer APPLY_MODEL wrapper receives the payload built by ComfyUI, moves only marked Continuum reference rows onto the target timeline, and normalizes keyframe/reference latent lists. + +## 5. Accelerator composition + +### SageAttention + +Continuum never replaces the attention kernel. Existing attention overrides remain intact. + +### Sol-Attn + +Sol-Attn registers H3 target spans by the identity of `PackedLayout.position_ids`. Continuum modifies this tensor in place and never replaces it. Sol conditioning sink and Morton tracking therefore retain their registration. + +### Spectrum + +Spectrum remains an external OUTER_SAMPLE / PREDICT_NOISE / SAMPLER_SAMPLE / DIFFUSION_MODEL wrapper stack. V2 calls the normal ComfyUI `guider.sample()` once per chunk. Spectrum starts and ends an independent runtime run for each call. Continuum imports no Spectrum private functions. + +The recommended model chain is: + +```text +UNET → Sage → Sol → LoRA → Spectrum → V2 Sampler +``` + +V2 then clones this ModelPatcher once; Spectrum's ON_CLONE callback creates an independent runtime binding on the clone. + +## 6. State and Session + +### V1 State + +Contains only the maximum retained continuation tail and temporal metadata. It can start V2 through `initial_state`. + +### V2 Session + +Contains all accepted raw chunk AV latents plus: + +- Prompt and SHA-256 Prompt hash +- per-chunk Seed +- Continuum Plan +- selected Context frames and motion score +- model/accelerator fingerprint +- branch parent/session identifiers + +Tensors remain on CPU. Save Session writes safetensors and a JSON mirror atomically. The authoritative metadata is embedded in safetensors. + +## 7. Reroll semantics + +- `reroll_from_chunk=0`: reuse every matching accepted prefix. +- `reroll_from_chunk=N`: preserve chunks before N and regenerate N onward. +- A Prompt mismatch automatically stops reuse before the changed chunk. +- A first-frame fingerprint mismatch fails explicitly. +- A model/accelerator fingerprint mismatch is reported but accepted chunks remain fixed, because their latent output is already approved. +- The new output creates a child Session; the input Session is not mutated. + +## 8. Audio timeline + +H3 video is 24fps and audio latent is 40Hz. Decoded PCM is placed using cumulative integer boundaries: + +```text +start_sample = round(cumulative_frames / 24 * sample_rate) +stop_sample = round(next_cumulative_frames / 24 * sample_rate) +``` + +This avoids the one-sample drift that can occur when every 119-frame segment is rounded independently. + +## 9. Memory model + +Full accepted AV latents are comparatively small and stored on CPU. The final decoded IMAGE is large, so V2: + +- estimates final float32 IMAGE + one raw chunk + headroom before sampling; +- refuses unsafe jobs before expensive H3 work; +- allocates the final IMAGE exactly once; +- copies each decoded segment into its final range; +- avoids progressive whole-sequence concatenation. + +## 10. Compatibility policy + +The package checks: + +- native `MiniMaxH3Model` class and required attributes; +- `PackedLayout` capability, including transparent `*args/**kwargs` wrappers; +- H3 temporal grid `(1,4,4,4,4)`; +- APPLY_MODEL and DIFFUSION_MODEL wrapper APIs; +- NestedTensor support; +- real PackedLayout row placement through the installer self-test. + +Marked Continuum payload failures never silently fall back to stock reference placement, because that could render a plausible but semantically wrong join. diff --git a/docs/FUTURE_ROADMAP.md b/docs/FUTURE_ROADMAP.md new file mode 100644 index 0000000..95e1f64 --- /dev/null +++ b/docs/FUTURE_ROADMAP.md @@ -0,0 +1,16 @@ +# Future roadmap after V2 + +V2 is the integrated production path. Future changes must preserve V1/V2 schemas and remain optional. + +Possible additions: + +- streaming decode/save that does not retain the complete final IMAGE tensor; +- interactive per-chunk approval UI with a frontend timeline; +- persisted multiple child branches and branch comparison; +- richer identity diagnostics using optional external encoders; +- adaptive Spectrum seam policy only if Spectrum exposes a stable public hook; +- reference-video / Ref2VA identity-memory mode; +- multi-resolution continuation through explicit regeneration, never silent resize; +- batch-independent server/API orchestration. + +These are not promises for a specific release. The current V2 avoids private accelerator APIs so updates remain maintainable. diff --git a/docs/V2_1_4_UI_SIMPLIFICATION.md b/docs/V2_1_4_UI_SIMPLIFICATION.md new file mode 100644 index 0000000..aa22014 --- /dev/null +++ b/docs/V2_1_4_UI_SIMPLIFICATION.md @@ -0,0 +1,35 @@ +# V2.1.4 UI Simplification + +The integrated `H3 Continuum Sampler V2` is the primary workflow node. + +## Default workflow + +```text +Text (Multiline) -> Sequence Prompt +Image -> first_frame +Sampler images/audio -> Create Video +``` + +The default node surface shows Sequence Prompt, first frame, chunk timing, +continuity, seed, and seam correction. Individual clip overrides and Advanced +settings are collapsed. + +## Advanced settings + +Advanced settings expose `last_frame`, `initial_state`, `session`, +`prompt_plan`, reroll controls, diagnostics, `last_state`, session output, and +report output. A saved workflow with an existing Advanced connection expands +the section automatically. + +The Advanced Settings button remains the section header when expanded. The +button is placed before the expanded controls instead of moving below them. + +## Legacy nodes + +V1 Join, Finish, Assemble, state utilities, and auxiliary V2 plan/session nodes +remain registered for saved-workflow compatibility. They are marked deprecated +and moved to `MiniMax H3/Continuum/Legacy`, so ComfyUI can omit them from the +normal node picker. + +No sampling, State/Session schema, output order, or accelerator behavior changes +in V2.1.4. diff --git a/docs/VALIDATION_JA.md b/docs/VALIDATION_JA.md new file mode 100644 index 0000000..962222b --- /dev/null +++ b/docs/VALIDATION_JA.md @@ -0,0 +1,150 @@ +# H3 Continuum Join 2.0 — 実機検証 + +## 1. インストール検査 + +ComfyUIを終了後、展開フォルダーで実行します。 + +```powershell +.\install_windows.bat +``` + +または: + +```powershell +D:\StabilityMatrix\Data\Packages\ComfyUI_W\venv\Scripts\python.exe ` + .\tools\verify_runtime.py ` + --comfy-root D:\StabilityMatrix\Data\Packages\ComfyUI_W +``` + +確認項目: + +- native H3 API +- Sol-Attn形式の`*args/**kwargs` PackedLayout wrapper互換 +- real PackedLayout行構造とin-place座標補正 +- V1/V2計10ノード登録 +- Fixed 3×5秒Prompt Plan + +## 2. 最初のV2基準生成 + +```text +chunks: 3 +chunk_seconds: 5.0 +continuity: Balanced — 22 frames +audio_continuity: true +exact_total_duration: true +diagnostics: Basic +reroll_from_chunk: 0 +reroll_nonce: 0 +strict_compatibility: true +``` + +Model: + +```text +H3 INT8 ConvRot +20 steps / CFG 1 +Turbo LoRA OFF +``` + +同じ入力画像とPromptで、既存V1の3×5秒ワークフローとV2を比較します。 + +## 3. アクセラレータ比較 + +同じBase Seed、Prompt、解像度で次を順番に確認します。 + +1. SageAttentionのみ +2. Sage + Sol-Attn +3. Sage + Spectrum +4. Sage + Sol-Attn + Spectrum + +RTX 5060 Ti環境で既に動作したSage backendを基準にします。クラッシュしたbackendへ切り替えません。 + +確認: + +- 全チャンクが順番に完了 +- Spectrumのrunが各チャンクで開始・終了 +- `Sol-Attn takes first refusal...`ログ +- 黒出力、NaN、CUDA errorなし +- VAE Decodeが全H3 Sampling後に始まる +- 最終出力が指定尺になる + +## 4. 接続品質 + +Basic確認: + +- 人物ID、髪、服、背景 +- 前チャンクからの姿勢・移動方向・カメラ速度 +- 口形、歌唱・発話の継続 +- 音声tick、無音、語尾重複 +- 最終フレーム数と音声samples + +Full DiagnosticsではReportに次を出します。 + +```text +video overlap MAE +video overlap PSNR +audio overlap correlation +audio boundary jump +``` + +絶対的な合格値ではなく、V1と各アクセラレータ構成の相対比較に使います。 + +## 5. Session再開 + +1. 3チャンク生成 +2. `Save Session slot 1` +3. ComfyUI再起動 +4. `Load Session slot 1` +5. chunksを4へ変更し、sessionへ接続 +6. Chunk 1〜3がreused、Chunk 4だけSamplingされることをReportで確認 + +## 6. リロール + +```text +reroll_from_chunk: 3 +reroll_nonce: 1 +``` + +確認: + +- Chunk 1〜2はSamplingされない +- Chunk 3以降だけ新Seed +- 入力Sessionファイルは変更されない +- 出力Sessionの`parent_session_id`が入力Sessionを示す + +## 7. Promptモード + +- Fixed:Qwen encodeが1回 +- List:`---`で3 Prompt +- Timeline:`[0-5s] / [5-10s] / [10-15s]` +- Prompt変更時、変更前チャンクだけSession再利用される + +## 8. Auto Context + +Balanced 22fを品質基準にします。その後Autoと比較します。 + +- 静かな動き:5fを選ぶ可能性 +- 通常:22f +- 強いlatent motionかつ39f State:39f + +Autoの結果が悪い内容ではBalancedへ固定します。 + +## 9. RAM guard + +高解像度・多数チャンクで、Sampling前にRAM見積りエラーが出ることを確認します。これは意図した安全停止です。chunks、秒数、解像度を下げます。 + +## 10. 報告時に必要な情報 + +```text +ComfyUI version / revision +Python / Torch / CUDA +GPU / VRAM / RAM +H3 checkpoint +Sage backend +Sol settings +Spectrum settings +chunks / seconds / context +steps / scheduler / sampler +Console log +V2 report +``` diff --git a/examples/H3_CONTINUUM_V1_WIRING.md b/examples/H3_CONTINUUM_V1_WIRING.md new file mode 100644 index 0000000..3ffc1f1 --- /dev/null +++ b/examples/H3_CONTINUUM_V1_WIRING.md @@ -0,0 +1,33 @@ +# H3 Continuum Join V1 — Manual Wiring + +## Clip 1 + +```text +MiniMax H3 Image to Video ── conditioning/latent ─┐ +H3 model → Sage → Sol ───────────────────────────┼→ H3 Continuum Join (state empty) + └→ Spectrum → Basic Guider → Sampler +Sampler output → Video VAE Decode / Audio VAE Decode + └────────────────────────────────────→ H3 Continuum Finish + └→ STATE 1 +``` + +## Clip 2 + +Duplicate the H3 conditioning/sampling group and connect `STATE 1` to the next `H3 Continuum Join.previous_state`. Use another seed for the next clip. + +Two conditioning choices are supported: + +1. **Identity anchored:** keep the original image connected. Join removes its spatial first-frame keyframe but retains the Qwen visual tokens as identity guidance. +2. **Context only:** disconnect the image from the second MiniMax H3 Image to Video node. The previous AV state becomes the only visual timeline anchor. + +## Clip 3+ + +Repeat the same pattern. State Save/Load is optional when every clip is in one graph. Use numbered state slots when clips are approved across separate runs. + +## Recommended accelerator order + +```text +MODEL → SageAttention → Sol-Attn → H3 Continuum Join → Spectrum → Basic Guider +``` + +The order is shown for clarity. Continuum uses `APPLY_MODEL`, Spectrum uses `DIFFUSION_MODEL`, and Sage/Sol remain attention overrides. diff --git a/examples/H3_CONTINUUM_V2_WIRING_JA.md b/examples/H3_CONTINUUM_V2_WIRING_JA.md new file mode 100644 index 0000000..f263431 --- /dev/null +++ b/examples/H3_CONTINUUM_V2_WIRING_JA.md @@ -0,0 +1,69 @@ +# H3 Continuum Sampler V2 — 配線 + +## MODEL + +```text +UNET Loader + → Patch Sage Attention KJ + → Sol-Attn Patch + → Power LoRA Loader(任意) + → Spectrum Apply MiniMax H3 + → H3 Continuum Sampler V2.model +``` + +## 共有入力 + +```text +CLIP Loader → clip +Video VAE → video_vae +Audio VAE → audio_vae +KSampler Select → sampler +Basic Scheduler → sigmas +Load Image → ImageScaleToTotalPixels → first_frame +GetImageSize.width/height → width/height(widgetをinputへ変換して接続する場合) +``` + +## 出力 + +```text +Sampler V2.images ─→ CreateVideo.images +Sampler V2.audio ─→ CreateVideo.audio +CreateVideo ─→ SaveVideo + +Sampler V2.session ─→ Save Session(任意) +``` + +## 3×5秒の初期値 + +```text +Prompt Format: Auto +Sequence Prompt: Text (Multiline)を1個接続 +chunks: 3 +chunk_seconds: 5.0 +continuity: Balanced — 22 frames +base_seed: 固定値 +audio_continuity: true +exact_total_duration: true +diagnostics: Basic +reroll_from_chunk: 0 +reroll_nonce: 0 +strict_compatibility: true +``` + +## 再開 + +```text +Load Session.session → Sampler V2.session +chunks: 保存時より大きい値 +reroll_from_chunk: 0 +``` + +## 最終チャンクだけリロール + +```text +Load Session.session → Sampler V2.session +reroll_from_chunk: 最終チャンク番号 +reroll_nonce: 1以上へ変更 +``` + +同梱の`H3_Continuum_V2_3x5s.json`は、上記配線を1本のV2ノードにまとめた編集可能なComfyUI workflow例です。モデル名、入力画像、Prompt、LoRAを自分の環境に合わせて変更してください。 diff --git a/examples/H3_Continuum_V2_3x5s.json b/examples/H3_Continuum_V2_3x5s.json new file mode 100644 index 0000000..982f6e3 --- /dev/null +++ b/examples/H3_Continuum_V2_3x5s.json @@ -0,0 +1,1106 @@ +{ + "id": "ea864939-c34d-4853-b8f4-33214b9f7383", + "revision": 0, + "last_node_id": 181, + "last_link_id": 20, + "nodes": [ + { + "id": 114, + "type": "LoadImage", + "pos": [ + -1900, + 0 + ], + "size": [ + 450, + 630 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "20260613183320-ernie-image-turbo-QT-v2-FP8_scaled-1306799849-Euler.webp", + "image" + ] + }, + { + "id": 119, + "type": "ImageScaleToTotalPixels", + "pos": [ + -1400, + 80 + ], + "size": [ + 280, + 150 + ], + "flags": {}, + "order": 1, + "mode": 0, + "showAdvanced": true, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 1 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2, + 3 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "ImageScaleToTotalPixels" + }, + "widgets_values": [ + "area", + 0.5, + 32 + ] + }, + { + "id": 120, + "type": "GetImageSize", + "pos": [ + -1050, + 80 + ], + "size": [ + 240, + 150 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 2 + } + ], + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 4 + ] + }, + { + "name": "height", + "type": "INT", + "links": [ + 5 + ] + }, + { + "name": "batch_size", + "type": "INT", + "links": [] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "GetImageSize" + }, + "widgets_values": [] + }, + { + "id": 147, + "type": "UNETLoader", + "pos": [ + -1900, + 800 + ], + "size": [ + 640, + 90 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 6 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "UNETLoader", + "models": [ + { + "name": "minimax_h3_fl2va_pruned_int8_convrot.safetensors", + "url": "https://huggingface.co/Comfy-Org/MiniMax-H3/resolve/main/diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "minimax\\minimax_h3_fl2va_pruned_int8_convrot.safetensors", + "default" + ], + "title": "H3 Model" + }, + { + "id": 149, + "type": "PathchSageAttentionKJ", + "pos": [ + -1250, + 800 + ], + "size": [ + 370, + 90 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 6 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7 + ] + } + ], + "properties": { + "cnr_id": "comfyui-kjnodes", + "ver": "1.5.0", + "Node name for S&R": "PathchSageAttentionKJ" + }, + "widgets_values": [ + "sageattn_qk_int8_pv_fp8_cuda++", + false + ], + "title": "SageAttention" + }, + { + "id": 151, + "type": "SolAttnPatch", + "pos": [ + -800, + 800 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "tau_profile", + "shape": 7, + "type": "STRING", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 8 + ] + } + ], + "properties": { + "aux_id": "kijai/ComfyUI-SolAttn_triton", + "ver": "842c4eaa7d91dbaef3fee3ccdbf36a39521e82fc", + "Node name for S&R": "SolAttnPatch" + }, + "widgets_values": [ + 1.7, + 0.2, + 0.9, + 4096, + true, + "exact_kv", + false, + "2d_frame", + true, + false, + true, + "" + ], + "title": "Sol-Attn" + }, + { + "id": 154, + "type": "Power Lora Loader (rgthree)", + "pos": [ + -400, + 800 + ], + "size": [ + 610, + 190 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "dir": 3, + "name": "model", + "type": "MODEL", + "link": 8 + }, + { + "dir": 3, + "name": "clip", + "type": "CLIP", + "link": null + } + ], + "outputs": [ + { + "dir": 4, + "name": "MODEL", + "shape": 3, + "type": "MODEL", + "links": [ + 9 + ] + }, + { + "dir": 4, + "name": "CLIP", + "shape": 3, + "type": "CLIP", + "links": [] + } + ], + "properties": { + "cnr_id": "rgthree-comfy", + "ver": "1.0.2607232129", + "Show Strengths": "Single Strength", + "Match": "" + }, + "widgets_values": [ + {}, + { + "type": "PowerLoraLoaderHeaderWidget" + }, + { + "on": false, + "lora": "minimax\\HMNSFW_AIO_V2.safetensors", + "strength": 0.5, + "strengthTwo": null + }, + { + "on": false, + "lora": "minimax\\minimax_h3_turbo_v4_step600_ema.safetensors", + "strength": 0.8, + "strengthTwo": null + }, + { + "on": false, + "lora": "minimax\\minimax_h3_turbo_4step_ckpt500.safetensors", + "strength": 0.8, + "strengthTwo": null + }, + {}, + "" + ], + "title": "LoRA (optional)" + }, + { + "id": 150, + "type": "SpectrumApplyMiniMaxH3", + "pos": [ + 300, + 800 + ], + "size": [ + 330, + 420 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 9 + } + ], + "outputs": [ + { + "name": "model", + "type": "MODEL", + "links": [ + 10, + 15 + ] + } + ], + "title": "Spectrum", + "properties": { + "cnr_id": "comfyui-spectrum-minimax-h3", + "ver": "9395bf98fc60a04c5f588de7b2bb33516a0b622f", + "Node name for S&R": "SpectrumApplyMiniMaxH3" + }, + "widgets_values": [ + true, + 0.5, + 1, + 0.1, + 2, + 0.75, + 1, + 1, + 8, + false, + "system_ram", + true, + false, + false, + true, + 0 + ] + }, + { + "id": 142, + "type": "CLIPLoader", + "pos": [ + -1900, + 1050 + ], + "size": [ + 640, + 120 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 11 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "url": "https://huggingface.co/Comfy-Org/MiniMax-H3/resolve/main/text_encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "minimax", + "default" + ], + "title": "MiniMax H3 Text Encoder" + }, + { + "id": 135, + "type": "VAELoader", + "pos": [ + -1900, + 1250 + ], + "size": [ + 640, + 70 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 12 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "minimax_h3_video_vae_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/MiniMax-H3/resolve/main/vae/minimax_h3_video_vae_fp16.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "minimax_h3_video_vae_fp16.safetensors" + ], + "title": "Video VAE" + }, + { + "id": 136, + "type": "VAELoader", + "pos": [ + -1900, + 1400 + ], + "size": [ + 650, + 70 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 13 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "minimax_h3_audio_vae_fp32.safetensors", + "url": "https://huggingface.co/Comfy-Org/MiniMax-H3/resolve/main/vae/minimax_h3_audio_vae_fp32.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "minimax_h3_audio_vae_fp32.safetensors" + ], + "title": "Audio VAE" + }, + { + "id": 139, + "type": "KSamplerSelect", + "pos": [ + -1250, + 1300 + ], + "size": [ + 370, + 70 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 14 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "res_multistep" + ], + "title": "Sampler" + }, + { + "id": 153, + "type": "BasicScheduler", + "pos": [ + -800, + 1300 + ], + "size": [ + 370, + 130 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 15 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 16 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 20, + 1 + ], + "title": "Scheduler" + }, + { + "id": 148, + "type": "CreateVideo", + "pos": [ + 1550, + 300 + ], + "size": [ + 270, + 110 + ], + "flags": { + "collapsed": false + }, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 17 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 18 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 20 + ] + } + ], + "title": "Create 3x5s Video", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24, + 8 + ] + }, + { + "id": 92, + "type": "SaveVideo", + "pos": [ + 1950, + 300 + ], + "size": [ + 570, + 938 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 20 + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save 3x5s Video", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.30.0", + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/MiniMax_H3_Continuum_V2_3x5s", + "auto", + "auto" + ] + }, + { + "id": 180, + "type": "H3ContinuumSamplerV2", + "pos": [ + 850, + 0 + ], + "size": [ + 650, + 720 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "clip", + "type": "CLIP", + "link": 11 + }, + { + "name": "video_vae", + "type": "VAE", + "link": 12 + }, + { + "name": "audio_vae", + "type": "VAE", + "link": 13 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 14 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 16 + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 4 + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 5 + }, + { + "name": "first_frame", + "shape": 7, + "type": "IMAGE", + "link": 3 + }, + { + "name": "last_frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "session", + "shape": 7, + "type": "H3_CONTINUUM_SESSION", + "link": null + }, + { + "name": "initial_state", + "shape": 7, + "type": "H3_CONTINUUM_STATE", + "link": null + }, + { + "name": "prompt_plan", + "shape": 7, + "type": "H3_CONTINUUM_PROMPT_PLAN", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 17 + ] + }, + { + "name": "audio", + "type": "AUDIO", + "links": [ + 18 + ] + }, + { + "name": "last_state", + "type": "H3_CONTINUUM_STATE", + "links": [] + }, + { + "name": "session", + "type": "H3_CONTINUUM_SESSION", + "links": [ + 19 + ] + }, + { + "name": "report", + "type": "STRING", + "links": [] + } + ], + "title": "H3 Continuum Sampler V2 — Integrated 3x5s", + "properties": { + "Node name for S&R": "H3ContinuumSamplerV2" + }, + "widgets_values": [ + "Fixed — one prompt", + "Cinematic single continuous shot. Preserve the exact person, face, hairstyle, clothing, background, lighting, camera direction, motion, vocal timbre, and music across every chunk. The performance continues through each boundary without a cut, reset, repeated opening, fade, or pose jump. Audio and lip movement remain synchronized.", + 3, + 5.0, + 1344, + 768, + "Balanced — 22 frames", + 123456789, + "fixed", + true, + true, + "Basic", + 0, + 0, + true, + false + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 181, + "type": "H3ContinuumSaveSession", + "pos": [ + 1550, + 520 + ], + "size": [ + 300, + 120 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "session", + "type": "H3_CONTINUUM_SESSION", + "link": 19 + } + ], + "outputs": [ + { + "name": "session", + "type": "H3_CONTINUUM_SESSION", + "links": [] + }, + { + "name": "path", + "type": "STRING", + "links": [] + } + ], + "title": "Save V2 Session (optional)", + "properties": { + "Node name for S&R": "H3ContinuumSaveSession" + }, + "widgets_values": [ + "h3_continuum_session", + 1 + ], + "color": "#232", + "bgcolor": "#353" + } + ], + "links": [ + [ + 1, + 114, + 0, + 119, + 0, + "IMAGE" + ], + [ + 2, + 119, + 0, + 120, + 0, + "IMAGE" + ], + [ + 3, + 119, + 0, + 180, + 8, + "IMAGE" + ], + [ + 4, + 120, + 0, + 180, + 6, + "INT" + ], + [ + 5, + 120, + 1, + 180, + 7, + "INT" + ], + [ + 6, + 147, + 0, + 149, + 0, + "MODEL" + ], + [ + 7, + 149, + 0, + 151, + 0, + "MODEL" + ], + [ + 8, + 151, + 0, + 154, + 0, + "MODEL" + ], + [ + 9, + 154, + 0, + 150, + 0, + "MODEL" + ], + [ + 10, + 150, + 0, + 180, + 0, + "MODEL" + ], + [ + 11, + 142, + 0, + 180, + 1, + "CLIP" + ], + [ + 12, + 135, + 0, + 180, + 2, + "VAE" + ], + [ + 13, + 136, + 0, + 180, + 3, + "VAE" + ], + [ + 14, + 139, + 0, + 180, + 4, + "SAMPLER" + ], + [ + 15, + 150, + 0, + 153, + 0, + "MODEL" + ], + [ + 16, + 153, + 0, + 180, + 5, + "SIGMAS" + ], + [ + 17, + 180, + 0, + 148, + 0, + "IMAGE" + ], + [ + 18, + 180, + 1, + 148, + 1, + "AUDIO" + ], + [ + 19, + 180, + 3, + 181, + 0, + "H3_CONTINUUM_SESSION" + ], + [ + 20, + 148, + 0, + 92, + 0, + "VIDEO" + ] + ], + "groups": [ + { + "id": 1, + "title": "Shared H3 Models and Accelerators", + "bounding": [ + -1950, + 720, + 2600, + 850 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Integrated V2 Sequence", + "bounding": [ + 800, + -80, + 1100, + 820 + ], + "color": "#6b5b95", + "flags": {} + }, + { + "id": 3, + "title": "Final Video", + "bounding": [ + 1500, + 180, + 1100, + 620 + ], + "color": "#3f789e", + "flags": {} + } + ], + "config": {}, + "extra": { + "frontendVersion": "1.48.7", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0 + }, + "version": 0.4 +} diff --git a/examples/recommended_settings_v1.json b/examples/recommended_settings_v1.json new file mode 100644 index 0000000..59b1e1a --- /dev/null +++ b/examples/recommended_settings_v1.json @@ -0,0 +1,24 @@ +{ + "h3_continuum_join": { + "continuity": "Balanced — 22 frames", + "extend_seconds": 5.0, + "audio_continuity": true, + "first_frame_policy": "Use Continuum context; keep visual identity tokens (Recommended)", + "preserve_last_frame": true, + "strict_compatibility": true, + "debug": false + }, + "spectrum": { + "blend_weight": 0.5, + "degree": 1, + "warmup_steps": 1, + "offline_smoothing_replay": true, + "audio_blend_weight": 0.0 + }, + "validation_matrix": [ + "Sage only", + "Sage + Sol", + "Sage + Spectrum", + "Sage + Sol + Spectrum" + ] +} diff --git a/examples/recommended_settings_v2.json b/examples/recommended_settings_v2.json new file mode 100644 index 0000000..f77d3ed --- /dev/null +++ b/examples/recommended_settings_v2.json @@ -0,0 +1,29 @@ +{ + "h3_continuum_sampler_v2": { + "chunks": 3, + "chunk_seconds": 5.0, + "continuity": "Balanced — 22 frames", + "audio_continuity": true, + "exact_total_duration": true, + "diagnostics": "Basic", + "reroll_from_chunk": 0, + "reroll_nonce": 0, + "strict_compatibility": true, + "debug": false + }, + "sampling": { + "sampler": "res_multistep", + "scheduler": "simple", + "steps": 20, + "denoise": 1.0, + "turbo_lora": false + }, + "spectrum": { + "enabled": true, + "blend_weight": 0.5, + "degree": 1, + "warmup_steps": 1, + "offline_smoothing_replay": true, + "audio_blend_weight": 0.0 + } +} diff --git a/examples/v2_prompt_list_example.txt b/examples/v2_prompt_list_example.txt new file mode 100644 index 0000000..add15b8 --- /dev/null +++ b/examples/v2_prompt_list_example.txt @@ -0,0 +1,5 @@ +The same woman begins singing in one continuous shot. Preserve identity, clothing, background, lighting, camera direction, vocal timbre, and piano tempo. End while the phrase is still continuing. +--- +Seamless continuation of the immediately preceding clip. Continue the exact mouth motion, sustained note, breathing state, camera movement, vocal key, tempo, and piano phrase. No cut or reset. +--- +Seamless final continuation. Preserve all identity and motion. Complete the vocal phrase and piano cadence naturally during the final second, then release a soft breath. No scene change. diff --git a/examples/v2_prompt_timeline_example.txt b/examples/v2_prompt_timeline_example.txt new file mode 100644 index 0000000..4daefa5 --- /dev/null +++ b/examples/v2_prompt_timeline_example.txt @@ -0,0 +1,8 @@ +[0-5s] +The same woman begins singing in a single continuous shot. The melody and camera movement remain unresolved at the boundary. + +[5-10s] +Continue the exact performance, mouth motion, breath, vocal key, piano tempo, and camera movement with no reset. + +[10-15s] +Continue seamlessly, then resolve the final vocal note and piano phrase naturally in the last second. diff --git a/install_windows.bat b/install_windows.bat new file mode 100644 index 0000000..ae5c461 --- /dev/null +++ b/install_windows.bat @@ -0,0 +1,14 @@ +@echo off +setlocal +set "ROOT=%~1" +if "%ROOT%"=="" set "ROOT=D:\StabilityMatrix\Data\Packages\ComfyUI_W" +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0install_windows.ps1" -ComfyUIRoot "%ROOT%" +if errorlevel 1 ( + echo. + echo Installation failed. + pause + exit /b 1 +) +echo. +echo Installation completed. +pause diff --git a/install_windows.ps1 b/install_windows.ps1 new file mode 100644 index 0000000..704c7bd --- /dev/null +++ b/install_windows.ps1 @@ -0,0 +1,46 @@ +param( + [string]$ComfyUIRoot = "D:\StabilityMatrix\Data\Packages\ComfyUI_W" +) + +$ErrorActionPreference = "Stop" +$source = (Resolve-Path $PSScriptRoot).Path +$customNodes = Join-Path $ComfyUIRoot "custom_nodes" +$destination = Join-Path $customNodes "ComfyUI-H3-Continuum-Join" + +if (-not (Test-Path $ComfyUIRoot -PathType Container)) { + throw "ComfyUI root not found: $ComfyUIRoot" +} +if (-not (Test-Path (Join-Path $ComfyUIRoot "comfy") -PathType Container)) { + throw "The selected folder does not look like a ComfyUI installation: $ComfyUIRoot" +} +New-Item -ItemType Directory -Force -Path $customNodes | Out-Null + +if ([System.StringComparer]::OrdinalIgnoreCase.Equals($source, $destination)) { + Write-Host "Already installed at: $destination" + exit 0 +} + +if (Test-Path $destination) { + $backup = "$destination.backup-$(Get-Date -Format 'yyyyMMdd-HHmmss')" + Move-Item -LiteralPath $destination -Destination $backup + Write-Host "Existing installation moved to: $backup" +} + +New-Item -ItemType Directory -Force -Path $destination | Out-Null +$exclude = @(".git", ".pytest_cache", "__pycache__") +Get-ChildItem -LiteralPath $source -Force | Where-Object { $exclude -notcontains $_.Name } | ForEach-Object { + Copy-Item -LiteralPath $_.FullName -Destination $destination -Recurse -Force +} + +$python = Join-Path $ComfyUIRoot "venv\Scripts\python.exe" +if (Test-Path $python) { + & $python (Join-Path $destination "tools\verify_runtime.py") --comfy-root $ComfyUIRoot + if ($LASTEXITCODE -ne 0) { + throw "Runtime verification failed. See the output above." + } +} else { + Write-Warning "venv Python was not found; installation was copied but runtime verification was skipped." +} + +Write-Host "Installed H3 Continuum Join 2.0 to: $destination" +Write-Host "Restart ComfyUI." diff --git a/layout_adapter.py b/layout_adapter.py new file mode 100644 index 0000000..e6f76dc --- /dev/null +++ b/layout_adapter.py @@ -0,0 +1,566 @@ +"""Per-run H3 payload normalization and MM-RoPE timeline adaptation. + +No ComfyUI class is monkey-patched. ComfyUI builds a normal H3 PackedLayout; +this module then changes only its coordinates in place. Keeping the same +``position_ids`` Tensor identity preserves Sol-Attn's native-H3 span registry. + +The preferred Continuum representation is one native H3 ``video`` or +``video_audio`` reference block. Its latent rows are moved onto the beginning +of the target timeline. The older marked-keyframe form remains supported as an +internal compatibility path, but the UI does not emit it. +""" + +from __future__ import annotations + +import logging +from typing import Any + +import torch + +from .constants import ( + CONTINUUM_INTEROP_API, + CONTINUUM_REFERENCE_METADATA_KEY, + FRAME_RESCALE, + LAYOUT_DEVICE_CACHE_ATTR, + LAYOUT_ORIGINAL_TIME_ATTR, + LAYOUT_SIGNATURE_ATTR, + MARK_AUDIO_CONTEXT, + MARK_AUDIO_END_FRAME, + MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, + MARK_VIDEO_CONTEXT, + MARK_VIDEO_SLOT, +) + +LOG = logging.getLogger("h3_continuum_join") + + +class LayoutCompatibilityError(RuntimeError): + pass + + +def _reference_metadata(ref: dict[str, Any]) -> dict[str, Any]: + value = ref.get(CONTINUUM_REFERENCE_METADATA_KEY) + return value if isinstance(value, dict) else {} + + +def _is_video_context(ref: dict[str, Any]) -> bool: + return bool( + ref.get(MARK_VIDEO_CONTEXT) + or _reference_metadata(ref).get("role") == "video_context" + ) + + +def _is_audio_context(ref: dict[str, Any]) -> bool: + metadata = _reference_metadata(ref) + return bool( + ref.get(MARK_AUDIO_CONTEXT) + or metadata.get("role") == "audio_context" + or metadata.get("audio_role") == "audio_context" + ) + + +def payload_has_continuum(payload: Any) -> bool: + if not isinstance(payload, dict): + return False + keyframes = payload.get("keyframes") or () + refs = payload.get("refs") or () + return any(MARK_VIDEO_SLOT in item for item in keyframes) or any( + _is_video_context(item) or _is_audio_context(item) for item in refs + ) + + +def normalize_condition_latents(payload: dict[str, Any]) -> None: + """Keep keyframe rows first and reference rows second, matching PackedLayout.""" + keyframes = list(payload.get("keyframes") or ()) + refs = list(payload.get("refs") or ()) + video_latents = [item["latent"] for item in keyframes if item.get("latent") is not None] + video_latents.extend(item["latent"] for item in refs if item.get("latent") is not None) + audio_latents = [ + item["audio_latent"] for item in refs if item.get("audio_latent") is not None + ] + payload["cond_video_latents"] = video_latents + payload["cond_audio_latents"] = audio_latents + + +def _input_device(x: Any) -> torch.device | None: + device = getattr(x, "device", None) + if isinstance(device, torch.device): + return device + if hasattr(x, "unbind"): + try: + parts = x.unbind() + if parts: + part_device = getattr(parts[0], "device", None) + if isinstance(part_device, torch.device): + return part_device + except Exception: + return None + return None + + +def materialize_continuum_latents( + payload: dict[str, Any], x: Any, *, debug: bool = False, +) -> None: + """Move only Continuum context tensors to the active device once per run. + + Native H3 intentionally keeps payload dictionaries outside BaseModel's + generic dtype/device conversion. Without this cache, a CPU continuation + state would be transferred again on every solver step. Existing user refs + are left untouched to avoid unexpectedly pinning large Ref2VA inputs in + VRAM. + """ + layout = payload.get("layout") + device = _input_device(x) + if layout is None or device is None or device.type == "cpu": + return + + keyframes = [dict(item) for item in (payload.get("keyframes") or ())] + refs = [dict(item) for item in (payload.get("refs") or ())] + marked_tensors: list[tuple[str, int, torch.Tensor]] = [] + for index, item in enumerate(keyframes): + if MARK_VIDEO_SLOT in item and torch.is_tensor(item.get("latent")): + marked_tensors.append(("keyframe", index, item["latent"])) + for index, item in enumerate(refs): + if _is_video_context(item) and torch.is_tensor(item.get("latent")): + marked_tensors.append(("ref_video", index, item["latent"])) + if _is_audio_context(item) and torch.is_tensor(item.get("audio_latent")): + marked_tensors.append(("ref_audio", index, item["audio_latent"])) + if not marked_tensors: + return + + cache_key = ( + str(device), + tuple( + (kind, index, id(tensor), tuple(tensor.shape), str(tensor.dtype)) + for kind, index, tensor in marked_tensors + ), + ) + cached = getattr(layout, LAYOUT_DEVICE_CACHE_ATTR, None) + if not isinstance(cached, dict) or cached.get("key") != cache_key: + values: dict[tuple[str, int], torch.Tensor] = {} + for kind, index, tensor in marked_tensors: + values[(kind, index)] = tensor.to( + device=device, dtype=torch.float32, non_blocking=True + ).contiguous() + cached = {"key": cache_key, "values": values} + setattr(layout, LAYOUT_DEVICE_CACHE_ATTR, cached) + if debug: + total_bytes = sum(t.numel() * t.element_size() for t in values.values()) + LOG.info( + "Continuum context cached on %s: %d tensors, %.2f MiB", + device, + len(values), + total_bytes / (1024 * 1024), + ) + + values = cached["values"] + for index, item in enumerate(keyframes): + replacement = values.get(("keyframe", index)) + if replacement is not None: + item["latent"] = replacement + for index, item in enumerate(refs): + replacement = values.get(("ref_video", index)) + if replacement is not None: + item["latent"] = replacement + replacement = values.get(("ref_audio", index)) + if replacement is not None: + item["audio_latent"] = replacement + payload["keyframes"] = keyframes + payload["refs"] = refs + + +def _single_segment(layout: Any, kind: str) -> tuple[int, int]: + matches = [(int(a), int(b)) for a, b, k in layout.segments if k == kind] + if len(matches) != 1: + raise LayoutCompatibilityError(f"expected one H3 '{kind}' segment, found {len(matches)}") + return matches[0] + + +def _map_refs_to_segments( + layout: Any, refs: list[dict[str, Any]], +) -> list[dict[str, tuple[int, int] | None]]: + available = [ + (int(a), int(b), str(kind)) + for a, b, kind in layout.segments + if kind in ("ref_img", "ref_audio") + ] + cursor = 0 + result: list[dict[str, tuple[int, int] | None]] = [] + + def consume(expected: str) -> tuple[int, int]: + nonlocal cursor + if cursor >= len(available): + raise LayoutCompatibilityError(f"layout ended while mapping '{expected}'") + a, b, kind = available[cursor] + cursor += 1 + if kind != expected: + raise LayoutCompatibilityError( + f"reference layout mismatch: expected '{expected}', found '{kind}'" + ) + return a, b + + for ref in refs: + kind = ref.get("kind") + mapping: dict[str, tuple[int, int] | None] = {"audio": None, "video": None} + if kind == "image": + mapping["video"] = consume("ref_img") + elif kind == "audio": + if int(ref.get("ref_audio_t", 0)) > 0: + mapping["audio"] = consume("ref_audio") + elif kind in ("video", "video_audio"): + if int(ref.get("ref_audio_t", 0)) > 0: + mapping["audio"] = consume("ref_audio") + mapping["video"] = consume("ref_img") + else: + raise LayoutCompatibilityError(f"unsupported H3 reference kind: {kind!r}") + result.append(mapping) + if cursor != len(available): + raise LayoutCompatibilityError( + f"layout contains {len(available)-cursor} unmapped reference segments" + ) + return result + + +def _patch_signature(payload: dict[str, Any], layout: Any) -> tuple[Any, ...]: + return ( + tuple( + ( + int(kf.get("resolved_frame_index", -1)), + int(kf.get(MARK_VIDEO_SLOT, -1)), + ) + for kf in (payload.get("keyframes") or ()) + ), + tuple( + ( + bool(ref.get(MARK_VIDEO_CONTEXT)), + tuple(sorted(_reference_metadata(ref).items())), + int(ref.get("latent_t", 0)), + int(ref.get(MARK_CONTEXT_FRAMES, 0)), + bool(ref.get(MARK_AUDIO_CONTEXT)), + int(ref.get("ref_audio_t", 0)), + float(ref.get(MARK_AUDIO_END_FRAME, 0.0)), + float(ref.get(MARK_AUDIO_OVERHANG, 0.0)), + ) + for ref in (payload.get("refs") or ()) + ), + tuple(getattr(layout, "signature", ())), + tuple((int(a), int(b), str(k)) for a, b, k in layout.segments), + ) + + +def _branch_key(transformer_options: dict[str, Any]) -> tuple[Any, ...]: + conds = transformer_options.get("cond_or_uncond") + uuids = transformer_options.get("uuids") + if conds is None or uuids is None: + return ("default",) + try: + cond_values = tuple(int(value) for value in conds) + uuid_values = tuple(str(value) for value in uuids) + except (TypeError, ValueError): + return ("default",) + if not cond_values or len(cond_values) != len(uuid_values): + return ("default",) + return tuple( + (cond_values[index], uuid_values[index]) + for index in range(len(cond_values)) + ) + + +def _sampled_tensor_signature(tensor: torch.Tensor | None) -> tuple[Any, ...] | None: + if tensor is None: + return None + value = tensor.detach().to(dtype=torch.float32) + flat = value.reshape(-1) + if flat.numel() <= 64: + sample = flat + else: + indices = torch.linspace( + 0, + flat.numel() - 1, + 64, + dtype=torch.long, + device=flat.device, + ) + sample = flat.index_select(0, indices) + sample = sample.to("cpu") + finite = bool(torch.isfinite(sample).all().item()) + if not finite: + raise LayoutCompatibilityError("Continuum layout/context contains NaN or Inf") + weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32) + return ( + tuple(int(part) for part in tensor.shape), + str(tensor.dtype), + finite, + float(sample.mean().item()), + float(sample.std(unbiased=False).item()), + float((sample * weights).sum().item()), + ) + + +def _ranges_overlap(left: tuple[int, int], right: tuple[int, int]) -> bool: + return max(left[0], right[0]) < min(left[1], right[1]) + + +def validate_native_continuity_layout( + payload: dict[str, Any], + *, + transformer_options: dict[str, Any] | None, + branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]], +) -> dict[str, Any]: + """Validate immutable native-H3 Context/Target topology per logical branch.""" + + layout = payload.get("layout") + if layout is None: + raise LayoutCompatibilityError("Native Continuity payload has no PackedLayout") + position_ids = getattr(layout, "position_ids", None) + segments = tuple( + (int(start), int(stop), str(kind)) + for start, stop, kind in getattr(layout, "segments", ()) + ) + if not torch.is_tensor(position_ids) or position_ids.ndim != 2: + raise LayoutCompatibilityError("Native Continuity position_ids are unavailable") + seq_len = int(getattr(layout, "seq_len", position_ids.shape[0])) + if seq_len != int(position_ids.shape[0]): + raise LayoutCompatibilityError("PackedLayout seq_len does not match position_ids") + + previous_stop = 0 + for start, stop, kind in segments: + if start < previous_stop or stop <= start or stop > seq_len: + raise LayoutCompatibilityError( + f"PackedLayout segment {kind!r} has an overlapping or invalid row range" + ) + previous_stop = stop + + target_audio = [(a, b) for a, b, kind in segments if kind == "audio"] + target_video = [(a, b) for a, b, kind in segments if kind == "video"] + if len(target_audio) != 1 or len(target_video) != 1: + raise LayoutCompatibilityError( + "Native Continuity requires one target audio and one target video segment" + ) + audio_range, video_range = target_audio[0], target_video[0] + if audio_range[1] != video_range[0] or video_range[1] != seq_len: + raise LayoutCompatibilityError( + "target segments must be the contiguous packed tail [audio | video]" + ) + + refs = list(payload.get("refs") or ()) + mappings = _map_refs_to_segments(layout, refs) + context_ranges: list[tuple[int, int, str]] = [] + context_descriptors: list[tuple[Any, ...]] = [] + for ref, mapping in zip(refs, mappings): + metadata = _reference_metadata(ref) + if metadata: + if ( + type(metadata.get("api")) is not int + or int(metadata["api"]) != CONTINUUM_INTEROP_API + ): + raise LayoutCompatibilityError("unsupported Continuum reference metadata") + if _is_video_context(ref) and metadata.get("role") != "video_context": + raise LayoutCompatibilityError("Continuum video reference role is invalid") + if _is_audio_context(ref) and metadata.get("audio_role") not in ( + None, + "audio_context", + ): + raise LayoutCompatibilityError("Continuum audio reference role is invalid") + + if _is_video_context(ref): + row_range = mapping.get("video") + if row_range is None: + raise LayoutCompatibilityError("Continuum video context has no reference rows") + context_ranges.append((*row_range, "video_context")) + if _is_audio_context(ref): + row_range = mapping.get("audio") + if row_range is None: + raise LayoutCompatibilityError("Continuum audio context has no reference rows") + context_ranges.append((*row_range, "audio_context")) + if _is_video_context(ref) or _is_audio_context(ref): + context_descriptors.append( + ( + ref.get("kind"), + int(ref.get("latent_t", 0)), + int(ref.get("latent_h", 0)), + int(ref.get("latent_w", 0)), + int(ref.get("ref_audio_t", 0)), + tuple(sorted(metadata.items())), + _sampled_tensor_signature(ref.get("latent")), + _sampled_tensor_signature(ref.get("audio_latent")), + ) + ) + + if not context_ranges: + raise LayoutCompatibilityError("Native Continuity payload has no context rows") + for start, stop, role in context_ranges: + row_range = (start, stop) + if _ranges_overlap(row_range, audio_range) or _ranges_overlap(row_range, video_range): + raise LayoutCompatibilityError(f"{role} rows overlap target rows") + + topology = ( + tuple(getattr(layout, "signature", ())), + segments, + audio_range, + video_range, + tuple(context_ranges), + tuple(context_descriptors), + int(payload.get("frame_count", 0) or 0), + _sampled_tensor_signature(position_ids), + ) + options = transformer_options if isinstance(transformer_options, dict) else {} + if options.get("spectrum_h3_actual") is False: + return {"status": "forecast_validated", "branch": _branch_key(options)} + + branch = _branch_key(options) + baseline = branch_baselines.get(branch) + if baseline is None: + branch_baselines[branch] = topology + return {"status": "baseline_created", "branch": branch} + if baseline != topology: + raise LayoutCompatibilityError( + f"Native Continuity topology changed during sampling for branch {branch!r}" + ) + return {"status": "baseline_matched", "branch": branch} + + +def patch_layout_in_place( + payload: dict[str, Any], *, strict: bool = True, debug: bool = False, +) -> dict[str, Any]: + layout = payload.get("layout") + if layout is None: + raise LayoutCompatibilityError( + "H3 payload has no prebuilt PackedLayout; use ComfyUI v0.31.0 or newer" + ) + missing = [name for name in ("position_ids", "segments", "signature") if not hasattr(layout, name)] + if missing: + raise LayoutCompatibilityError("PackedLayout is missing: " + ", ".join(missing)) + position_ids = layout.position_ids + if not torch.is_tensor(position_ids) or position_ids.ndim != 2 or position_ids.shape[1] < 3: + raise LayoutCompatibilityError("position_ids must be an [S,3] Tensor") + + signature = _patch_signature(payload, layout) + if getattr(layout, LAYOUT_SIGNATURE_ATTR, None) == signature: + return {"status": "already_patched", "position_ids_id": id(position_ids)} + + if not hasattr(layout, LAYOUT_ORIGINAL_TIME_ATTR): + setattr(layout, LAYOUT_ORIGINAL_TIME_ATTR, position_ids[:, 0].clone()) + original_time = getattr(layout, LAYOUT_ORIGINAL_TIME_ATTR) + if not torch.is_tensor(original_time) or original_time.shape != position_ids[:, 0].shape: + raise LayoutCompatibilityError("stored layout baseline is incompatible") + position_ids[:, 0].copy_(original_time) + + keyframes = list(payload.get("keyframes") or ()) + refs = list(payload.get("refs") or ()) + cond_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "cond"] + if len(cond_segments) != len(keyframes): + raise LayoutCompatibilityError( + f"layout has {len(cond_segments)} cond segments for {len(keyframes)} keyframes" + ) + + video_start, video_stop = _single_segment(layout, "video") + _single_segment(layout, "audio") + text_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "text"] + if len(text_segments) != 1 or text_segments[0][0] != 0: + raise LayoutCompatibilityError("unexpected H3 text segment") + text_len = text_segments[0][1] + + latent_t = int(layout.signature[1]) + video_rows = video_stop - video_start + if latent_t <= 0 or video_rows % latent_t: + raise LayoutCompatibilityError( + f"video rows {video_rows} are incompatible with latent T={latent_t}" + ) + frame_rows = video_rows // latent_t + target_origin = float(position_ids[video_start, 0]) + reference_shift = target_origin - float(text_len) + + # Existing first/last-frame keyframes are text-relative in stock H3. Once + # refs are present, move them by the target-origin shift. The legacy marked + # slot path is retained for compatibility with early development states. + patched_video_slots = 0 + for (start, stop), keyframe in zip(cond_segments, keyframes): + if stop - start != frame_rows: + raise LayoutCompatibilityError("keyframe rows no longer equal one target slot") + if MARK_VIDEO_SLOT in keyframe: + slot = int(keyframe[MARK_VIDEO_SLOT]) + if not (0 <= slot < latent_t): + raise LayoutCompatibilityError( + f"context slot {slot} is outside target latent T={latent_t}" + ) + target_row = video_start + slot * frame_rows + position_ids[start:stop].copy_( + position_ids[target_row : target_row + frame_rows] + ) + patched_video_slots += 1 + elif reference_shift: + position_ids[start:stop, 0].add_(reference_shift) + + ref_mappings = _map_refs_to_segments(layout, refs) + patched_video_refs = 0 + patched_audio = 0 + for ref, mapping in zip(refs, ref_mappings): + if _is_video_context(ref): + segment = mapping.get("video") + if segment is None: + raise LayoutCompatibilityError("Continuum video ref has no video segment") + start, stop = segment + context_t = int(ref.get("latent_t", 0)) + expected_rows = context_t * frame_rows + if context_t <= 0 or stop - start != expected_rows: + raise LayoutCompatibilityError( + f"video-context rows {stop-start} do not match latent T={context_t}" + ) + if context_t > latent_t: + raise LayoutCompatibilityError( + f"video context T={context_t} exceeds target latent T={latent_t}" + ) + target_stop = video_start + expected_rows + # Copy all (t,h,w) coordinates. Resolution equality is already + # enforced by state validation; this also fails safely if row geometry changes. + position_ids[start:stop].copy_(position_ids[video_start:target_stop]) + patched_video_refs += 1 + + if _is_audio_context(ref): + segment = mapping.get("audio") + if segment is None: + raise LayoutCompatibilityError("Continuum audio ref has no audio segment") + start, stop = segment + rt = int(ref.get("ref_audio_t", 0)) + if rt <= 0 or stop - start != rt * 2: + raise LayoutCompatibilityError( + f"audio rows {stop-start} do not match {rt} stereo steps" + ) + end_frame = float(ref.get(MARK_AUDIO_END_FRAME, 0.0)) + grid_offset = float(ref.get(MARK_AUDIO_OVERHANG, 0.0)) + desired_end = target_origin + FRAME_RESCALE * end_frame + grid_offset + desired_start = desired_end - float(rt) + old_start = float(position_ids[start, 0]) + position_ids[start:stop, 0].add_(desired_start - old_start) + patched_audio += 1 + + patched_video = patched_video_slots + patched_video_refs + if patched_video == 0: + message = "no marked Continuum video context was found" + if strict: + raise LayoutCompatibilityError(message) + LOG.warning(message) + + setattr(layout, LAYOUT_SIGNATURE_ATTR, signature) + if debug: + LOG.info( + "Continuum layout: video_refs=%d legacy_slots=%d audio_windows=%d " + "origin=%.6f rows=%d pos_id=%s", + patched_video_refs, + patched_video_slots, + patched_audio, + target_origin, + position_ids.shape[0], + id(position_ids), + ) + return { + "status": "patched", + "video_contexts": patched_video, + "video_refs": patched_video_refs, + "legacy_video_slots": patched_video_slots, + "audio_windows": patched_audio, + "target_origin": target_origin, + "position_ids_id": id(position_ids), + } diff --git a/media.py b/media.py new file mode 100644 index 0000000..3899df8 --- /dev/null +++ b/media.py @@ -0,0 +1,91 @@ +"""Reusable video/audio trim and assembly helpers.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as F + +from .constants import FPS + + +def validate_audio(audio: dict[str, Any], *, name: str = "AUDIO") -> tuple[torch.Tensor, int]: + if not isinstance(audio, dict) or "waveform" not in audio or "sample_rate" not in audio: + raise ValueError(f"{name} must contain waveform and sample_rate") + waveform = audio["waveform"] + sample_rate = int(audio["sample_rate"]) + if not torch.is_tensor(waveform) or waveform.ndim != 3: + raise ValueError(f"{name} waveform must be [B,C,samples]") + if waveform.shape[0] != 1: + raise ValueError(f"{name} batch size must be 1") + if sample_rate <= 0: + raise ValueError(f"{name} sample_rate must be positive") + return waveform, sample_rate + + +def trim_audio(audio: dict[str, Any], *, trim_frames: int, output_frames: int): + """Trim a decoded stream by video-frame time and normalize final duration.""" + + waveform, sample_rate = validate_audio(audio) + trim_samples = int(round(float(trim_frames) / FPS * sample_rate)) + wanted = int(round(float(output_frames) / FPS * sample_rate)) + result = waveform[..., trim_samples : trim_samples + wanted] + if result.shape[-1] < wanted: + result = F.pad(result, (0, wanted - result.shape[-1])) + return {**audio, "waveform": result.contiguous(), "sample_rate": sample_rate} + + +def assemble_segments(previous_images, previous_audio, next_images, next_audio): + """Append two Finish-normalized AV segments and correct rounding drift.""" + + if not all( + torch.is_tensor(value) and value.ndim == 4 + for value in (previous_images, next_images) + ): + raise ValueError("both IMAGE inputs must be [frames,H,W,C]") + if tuple(previous_images.shape[1:]) != tuple(next_images.shape[1:]): + raise ValueError( + f"image segment shapes differ: {tuple(previous_images.shape[1:])} vs " + f"{tuple(next_images.shape[1:])}" + ) + previous_waveform, previous_rate = validate_audio(previous_audio, name="previous_audio") + next_waveform, next_rate = validate_audio(next_audio, name="next_audio") + if previous_rate != next_rate: + raise ValueError("audio sample rates differ; resample before assembling") + if tuple(previous_waveform.shape[:2]) != tuple(next_waveform.shape[:2]): + raise ValueError("audio batch/channel shapes differ") + if previous_waveform.device != next_waveform.device: + raise ValueError("audio segments are on different devices") + if previous_images.device != next_images.device: + raise ValueError("image segments are on different devices") + + expected_previous = int(round(float(previous_images.shape[0]) / FPS * previous_rate)) + expected_next = int(round(float(next_images.shape[0]) / FPS * previous_rate)) + tolerance = max(2, int(round(previous_rate * 0.020))) # 20 ms safety bound + for name, waveform_part, expected in ( + ("previous", previous_waveform, expected_previous), + ("next", next_waveform, expected_next), + ): + delta = int(waveform_part.shape[-1]) - expected + if abs(delta) > tolerance: + raise ValueError( + f"{name} segment audio differs from its video duration by " + f"{delta} samples; pass segments through H3 Continuum Finish first" + ) + + images = torch.cat((previous_images, next_images), dim=0).contiguous() + waveform = torch.cat((previous_waveform, next_waveform), dim=-1).contiguous() + expected_total = int(round(float(images.shape[0]) / FPS * previous_rate)) + correction = expected_total - int(waveform.shape[-1]) + if correction < 0: + waveform = waveform[..., :expected_total].contiguous() + elif correction > 0: + waveform = F.pad(waveform, (0, correction)).contiguous() + audio = {**previous_audio, "waveform": waveform, "sample_rate": previous_rate} + report = ( + f"Assembled {previous_images.shape[0]} + {next_images.shape[0]} = " + f"{images.shape[0]} frames ({images.shape[0]/FPS:.3f}s), " + f"audio samples {waveform.shape[-1]} (sync correction {correction:+d})." + ) + return images, audio, report diff --git a/metadata.ini b/metadata.ini new file mode 100644 index 0000000..c4cfe00 --- /dev/null +++ b/metadata.ini @@ -0,0 +1,4 @@ +[project] +name = H3 Continuum Join +version = 2.1.4 +description = Integrated native MiniMax H3 long-video continuation with V1 compatibility and external SageAttention, Sol-Attn and Spectrum composition. diff --git a/model_patch.py b/model_patch.py new file mode 100644 index 0000000..ca9d3af --- /dev/null +++ b/model_patch.py @@ -0,0 +1,128 @@ +"""Per-MODEL ComfyUI wrapper for H3 Continuum Join.""" + +from __future__ import annotations + +from typing import Any, Callable + +from .compatibility import CompatibilityError, ensure_native_h3_base_model +from .constants import ( + CONTINUUM_ACTUAL_PREFIX_STEPS, + CONTINUUM_INTEROP_API, + CONTINUUM_INTEROP_KEY, + MODEL_WRAPPER_KEY, +) +from .layout_adapter import ( + LayoutCompatibilityError, + materialize_continuum_latents, + normalize_condition_latents, + patch_layout_in_place, + payload_has_continuum, + validate_native_continuity_layout, +) + + + +def _wrapper_factory(*, strict: bool, debug: bool) -> Callable[..., Any]: + branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]] = {} + + def apply_model_wrapper(executor, *args, **kwargs): + payload = kwargs.get("minimax_payload") + if not payload_has_continuum(payload): + return executor(*args, **kwargs) + try: + ensure_native_h3_base_model(executor.class_obj) + patched_payload = dict(payload) + # Patch coordinates before any accelerator sees the layout, then + # cache only the small Continuum state tensors on the active device. + patch_layout_in_place(patched_payload, strict=True, debug=debug) + transformer_options = kwargs.get("transformer_options") + validate_native_continuity_layout( + patched_payload, + transformer_options=( + transformer_options + if isinstance(transformer_options, dict) + else {} + ), + branch_baselines=branch_baselines, + ) + input_x = args[0] if args else None + materialize_continuum_latents(patched_payload, input_x, debug=debug) + normalize_condition_latents(patched_payload) + kwargs["minimax_payload"] = patched_payload + except (CompatibilityError, LayoutCompatibilityError, KeyError, TypeError, ValueError) as exc: + # A stock-time reference would render a plausible but incorrect join. + # Never silently bypass a marked Continuum payload, even when the + # proactive compatibility toggle is disabled. + raise RuntimeError(f"H3 Continuum Join compatibility failure: {exc}") from exc + return executor(*args, **kwargs) + return apply_model_wrapper + + +def patch_model(model: Any, *, strict: bool, debug: bool): + try: + from comfy.patcher_extension import WrappersMP + except Exception as exc: # pragma: no cover + raise RuntimeError(f"ComfyUI model-wrapper API unavailable: {exc}") from exc + required_api = ("clone", "add_wrapper_with_key", "remove_wrappers_with_key") + missing_api = [name for name in required_api if not hasattr(model, name)] + if missing_api: + raise RuntimeError( + "MODEL lacks the wrapper API required by H3 Continuum Join: " + + ", ".join(missing_api) + ) + base_model = getattr(model, "model", None) + if base_model is None: + raise RuntimeError("MODEL does not expose the ComfyUI BaseModel") + ensure_native_h3_base_model(base_model) + patched = model.clone() + patched.remove_wrappers_with_key(WrappersMP.APPLY_MODEL, MODEL_WRAPPER_KEY) + patched.add_wrapper_with_key( + WrappersMP.APPLY_MODEL, + MODEL_WRAPPER_KEY, + _wrapper_factory(strict=bool(strict), debug=bool(debug)), + ) + model_options = dict(getattr(patched, "model_options", None) or {}) + join_options = dict(model_options.get("h3_continuum_join") or {}) + join_options.update({"api_version": 1, "strict": bool(strict)}) + model_options["h3_continuum_join"] = join_options + patched.model_options = model_options + return patched + + +def continuum_interop_request( + *, + chunk_index: int, + context_frames: int, +) -> dict[str, Any]: + return { + "api": CONTINUUM_INTEROP_API, + "active": True, + "min_actual_prefix_steps": CONTINUUM_ACTUAL_PREFIX_STEPS, + "chunk_index": int(chunk_index), + "context_frames": int(context_frames), + } + + +def clone_model_for_chunk( + model: Any, + *, + strict: bool, + debug: bool, + chunk_index: int, + context_frames: int | None, +): + """Create a call-local MODEL and attach an optional read-only Spectrum hint.""" + + chunk_model = patch_model(model, strict=bool(strict), debug=bool(debug)) + model_options = dict(getattr(chunk_model, "model_options", None) or {}) + transformer_options = dict(model_options.get("transformer_options") or {}) + if context_frames is None: + transformer_options.pop(CONTINUUM_INTEROP_KEY, None) + else: + transformer_options[CONTINUUM_INTEROP_KEY] = continuum_interop_request( + chunk_index=int(chunk_index), + context_frames=int(context_frames), + ) + model_options["transformer_options"] = transformer_options + chunk_model.model_options = model_options + return chunk_model diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..42e0e7c --- /dev/null +++ b/nodes.py @@ -0,0 +1,376 @@ +"""ComfyUI nodes for H3 Continuum Join.""" + +from __future__ import annotations + +import logging +import torch + +from .compatibility import accelerator_summary, check_comfy_h3_runtime +from .constants import ( + CONTINUITY_FRAMES, + CONTINUITY_OPTIONS, +) +from .continuation import ( + POLICY_ERROR, + POLICY_KEEP, + POLICY_REPLACE, + conditioning_diagnostics as _conditioning_diagnostics, + new_h3_latent as _new_h3_latent, + prepare_conditioning as _prepare_conditioning, +) +from .media import assemble_segments, trim_audio as _trim_audio +from .model_patch import patch_model +from .state import ( + StateValidationError, + capture_state, + extract_av_streams, + make_plan, + select_context, + validate_plan, + validate_state, +) +from .state_io import load_state, save_state, state_mtime, state_paths +from .temporal import ( + is_valid_frame_count, + largest_context_capacity, + make_extension_shape, + pixel_frames_for_latent_t, +) +from .version import PACKAGE_VERSION + +LOG = logging.getLogger("h3_continuum_join") +CATEGORY = "MiniMax H3/Continuum" + + +class H3ContinuumJoin: + DESCRIPTION = ( + "Prepare a native MiniMax H3 continuation clip from a previous AV latent state. " + "Uses one timeline-positioned video/audio context reference, per-MODEL wrappers only, " + "and composes with external SageAttention, Sol-Attn, and Spectrum." + ) + SEARCH_ALIASES = ["H3 extend", "MiniMax H3 continuation", "H3 long video"] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL",), + "conditioning": ("CONDITIONING",), + "latent": ("LATENT",), + "continuity": (CONTINUITY_OPTIONS, {"default": CONTINUITY_OPTIONS[0]}), + "extend_seconds": ( + "FLOAT", + { + "default": 5.0, + "min": 0.5, + "max": 15.0, + "step": 0.1, + "tooltip": "Requested NEW duration. H3 snaps to the nearest 17k+5 frame count.", + }, + ), + "audio_continuity": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Carry the previous native audio latent and place it on the new clip timeline.", + }, + ), + "first_frame_policy": ( + [POLICY_REPLACE, POLICY_ERROR, POLICY_KEEP], + { + "default": POLICY_REPLACE, + "tooltip": ( + "Recommended: replace only the old spatial keyframe. If the prompt was encoded " + "with an input image, its Qwen visual tokens remain as an identity cue." + ), + }, + ), + "preserve_last_frame": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Move an existing FL2VA last-frame anchor to the end of the continuation clip.", + }, + ), + "strict_compatibility": ( + "BOOLEAN", + { + "default": True, + "tooltip": ( + "Run proactive native-H3 API checks. Marked Continuum payload failures always stop; " + "they are never silently bypassed." + ), + }, + ), + "debug": ("BOOLEAN", {"default": False}), + }, + "optional": {"previous_state": ("H3_CONTINUUM_STATE",)}, + } + + RETURN_TYPES = ("MODEL", "CONDITIONING", "LATENT", "H3_CONTINUUM_PLAN", "STRING") + RETURN_NAMES = ("model", "conditioning", "latent", "plan", "report") + FUNCTION = "join" + CATEGORY = CATEGORY + + def join( + self, + model, + conditioning, + latent, + continuity, + extend_seconds, + audio_continuity, + first_frame_policy, + preserve_last_frame, + strict_compatibility, + debug, + previous_state=None, + ): + issues = check_comfy_h3_runtime() + if issues and strict_compatibility: + raise RuntimeError("H3 runtime is incompatible: " + "; ".join(issues)) + patched_model = patch_model(model, strict=strict_compatibility, debug=debug) + video, _audio = extract_av_streams(latent) + current_frames = pixel_frames_for_latent_t(int(video.shape[2])) + if not is_valid_frame_count(current_frames): + raise StateValidationError( + f"input latent represents {current_frames} frames, not a native H3 17k+5 length" + ) + width, height = int(video.shape[-1]) * 16, int(video.shape[-2]) * 16 + context_frames = CONTINUITY_FRAMES[continuity] + + if previous_state is None: + plan = make_plan( + continuation=False, + clip_index=1, + total_frames=current_frames, + trim_frames=0, + width=width, + height=height, + context_frames=context_frames, + state_capacity_frames=largest_context_capacity(current_frames), + requested_extend_seconds=current_frames / 24.0, + debug=debug, + ) + report = ( + f"H3 Continuum Join {PACKAGE_VERSION}: initial clip; {current_frames} frames " + f"({current_frames/24.0:.3f}s). {accelerator_summary(patched_model)}" + ) + return patched_model, conditioning, latent, plan, report + + state = validate_state(previous_state) + if int(state["width"]) != width or int(state["height"]) != height: + raise StateValidationError( + f"state is {state['width']}x{state['height']} but latent is {width}x{height}; " + "automatic state resizing is disabled" + ) + diagnostics = _conditioning_diagnostics(conditioning) + video_context, audio_context, grid_offset = select_context( + state, context_frames, include_audio=bool(audio_continuity) + ) + shape = make_extension_shape(context_frames, float(extend_seconds)) + new_latent = _new_h3_latent( + latent, video_t=shape.video_latent_t, audio_t=shape.audio_latent_t + ) + new_conditioning = _prepare_conditioning( + conditioning, + video_context=video_context, + audio_context=audio_context, + audio_grid_offset=grid_offset, + context_frames=context_frames, + new_frame_count=shape.total_frames, + first_frame_policy=first_frame_policy, + preserve_last_frame=bool(preserve_last_frame), + ) + clip_index = int(state.get("clip_index", 0)) + 1 + plan = make_plan( + continuation=True, + clip_index=clip_index, + total_frames=shape.total_frames, + trim_frames=context_frames, + width=width, + height=height, + context_frames=context_frames, + state_capacity_frames=largest_context_capacity(shape.net_new_frames), + requested_extend_seconds=float(extend_seconds), + debug=debug, + ) + plan["visual_identity_tokens"] = diagnostics["visual_tokens"] + report_parts = [ + f"Continuation clip {clip_index}: generate {shape.total_frames}, trim {context_frames}, " + f"append {shape.net_new_frames} frames ({shape.actual_extend_seconds:.3f}s).", + f"Audio context {'ON' if audio_context is not None else 'OFF'}.", + f"Next-state capacity {plan['state_capacity_frames']} frames.", + ] + if diagnostics["visual_tokens"]: + report_parts.append( + f"Qwen visual identity tokens retained: {diagnostics['visual_tokens']}." + ) + report_parts.append(accelerator_summary(patched_model)) + return patched_model, new_conditioning, new_latent, plan, " ".join(report_parts) + + +class H3ContinuumFinish: + DESCRIPTION = ( + "Trim the overlap, sample-align audio, and capture a CPU AV latent state for the next Join." + ) + SEARCH_ALIASES = ["H3 trim", "H3 continuation state"] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "samples": ("LATENT",), + "images": ("IMAGE",), + "audio": ("AUDIO",), + "plan": ("H3_CONTINUUM_PLAN",), + } + } + + RETURN_TYPES = ("IMAGE", "AUDIO", "H3_CONTINUUM_STATE", "STRING") + RETURN_NAMES = ("images", "audio", "state", "report") + FUNCTION = "finish" + CATEGORY = CATEGORY + + def finish(self, samples, images, audio, plan): + plan = validate_plan(plan) + total_frames = int(plan["total_frames"]) + trim_frames = int(plan["trim_frames"]) + if not torch.is_tensor(images) or images.ndim != 4: + raise ValueError("IMAGE input must be [frames,H,W,C]") + if images.shape[0] < total_frames: + raise ValueError( + f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}" + ) + trimmed_images = images[:total_frames][trim_frames:].contiguous() + trimmed_audio = _trim_audio( + audio, trim_frames=trim_frames, output_frames=int(trimmed_images.shape[0]) + ) + state = capture_state( + samples, + source_frame_count=total_frames, + clip_index=int(plan["clip_index"]), + capacity_frames=int(plan["state_capacity_frames"]), + ) + report = ( + f"Clip {plan['clip_index']} finished: {trimmed_images.shape[0]} retained frames " + f"({trimmed_images.shape[0]/24.0:.3f}s); next-state capacity " + f"{plan['state_capacity_frames']} frames; " + f"audio samples {trimmed_audio['waveform'].shape[-1]}." + ) + return trimmed_images, trimmed_audio, state, report + + +class H3ContinuumAssemble: + DESCRIPTION = ( + "Append two already-trimmed Continuum segments while preserving exact audio sample order." + ) + SEARCH_ALIASES = ["H3 concatenate", "H3 join clips", "H3 append"] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "previous_images": ("IMAGE",), + "previous_audio": ("AUDIO",), + "next_images": ("IMAGE",), + "next_audio": ("AUDIO",), + } + } + + RETURN_TYPES = ("IMAGE", "AUDIO", "STRING") + RETURN_NAMES = ("images", "audio", "report") + FUNCTION = "assemble" + CATEGORY = CATEGORY + + def assemble(self, previous_images, previous_audio, next_images, next_audio): + return assemble_segments(previous_images, previous_audio, next_images, next_audio) + + +class H3ContinuumSaveState: + DESCRIPTION = "Atomically save a Continuum state as safetensors + JSON." + OUTPUT_NODE = True + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "state": ("H3_CONTINUUM_STATE",), + "prefix": ("STRING", {"default": "h3_continuum"}), + "slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_STATE", "STRING") + RETURN_NAMES = ("state", "path") + FUNCTION = "save" + CATEGORY = CATEGORY + + def save(self, state, prefix, slot): + tensor_path, _ = save_state(state, prefix=prefix, slot=slot) + return state, str(tensor_path) + + +class H3ContinuumLoadState: + DESCRIPTION = "Load an explicitly numbered Continuum state slot." + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prefix": ("STRING", {"default": "h3_continuum"}), + "slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_STATE", "STRING") + RETURN_NAMES = ("state", "path") + FUNCTION = "load" + CATEGORY = CATEGORY + + @classmethod + def IS_CHANGED(cls, prefix, slot): + return state_mtime(prefix=prefix, slot=slot) + + def load(self, prefix, slot): + state = load_state(prefix=prefix, slot=slot) + tensor_path, _ = state_paths(prefix, slot) + return state, str(tensor_path) + + +NODE_CLASS_MAPPINGS = { + "H3ContinuumJoin": H3ContinuumJoin, + "H3ContinuumFinish": H3ContinuumFinish, + "H3ContinuumAssemble": H3ContinuumAssemble, + "H3ContinuumSaveState": H3ContinuumSaveState, + "H3ContinuumLoadState": H3ContinuumLoadState, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumJoin": "H3 Continuum Join", + "H3ContinuumFinish": "H3 Continuum Finish", + "H3ContinuumAssemble": "H3 Continuum Assemble", + "H3ContinuumSaveState": "H3 Continuum Save State", + "H3ContinuumLoadState": "H3 Continuum Load State", +} + +# V2 is an orchestration layer over the same V1 continuation core. Keeping the +# mapping merge here preserves every V1 workflow and node identifier. +from .v2.nodes import ( + NODE_CLASS_MAPPINGS as V2_NODE_CLASS_MAPPINGS, + NODE_DISPLAY_NAME_MAPPINGS as V2_NODE_DISPLAY_NAME_MAPPINGS, +) + +NODE_CLASS_MAPPINGS.update(V2_NODE_CLASS_MAPPINGS) +NODE_DISPLAY_NAME_MAPPINGS.update(V2_NODE_DISPLAY_NAME_MAPPINGS) + +# V2's integrated sampler is the only primary workflow node. Keep every older +# identifier registered so saved workflows still load, while ComfyUI's native +# deprecated-node filter keeps legacy building blocks out of the normal menu. +for _node_id, _node_class in NODE_CLASS_MAPPINGS.items(): + if _node_id == "H3ContinuumSamplerV2": + continue + _node_class.DEPRECATED = True + _node_class.CATEGORY = f"{CATEGORY}/Legacy" + _display_name = NODE_DISPLAY_NAME_MAPPINGS.get(_node_id, _node_id) + if not _display_name.startswith("[Legacy]"): + NODE_DISPLAY_NAME_MAPPINGS[_node_id] = f"[Legacy] {_display_name}" diff --git a/public_api.py b/public_api.py new file mode 100644 index 0000000..c58dfed --- /dev/null +++ b/public_api.py @@ -0,0 +1,55 @@ +"""Stable non-UI API shared by V1 and the integrated V2 sampler. + +External orchestration code should call these functions instead of importing node classes. The +schema/version constants allow saved workflows and state files to evolve +without binding a sampler implementation to the UI layer. +""" + +from __future__ import annotations + +from .continuation import ( + FIRST_FRAME_POLICIES, + POLICY_ERROR, + POLICY_KEEP, + POLICY_REPLACE, + conditioning_diagnostics, + new_h3_latent, + prepare_conditioning, +) +from .layout_adapter import normalize_condition_latents, patch_layout_in_place +from .media import assemble_segments, trim_audio +from .state import capture_state, select_context, validate_plan, validate_state +from .temporal import make_extension_shape +from .version import PUBLIC_API_VERSION +from .v2.motion import choose_context_frames, latent_motion_score +from .v2.prompts import make_prompt_plan, validate_prompt_plan +from .v2.seeds import derive_chunk_seed +from .v2.session import entry_to_state, make_session, validate_session + +__all__ = [ + "PUBLIC_API_VERSION", + "FIRST_FRAME_POLICIES", + "POLICY_REPLACE", + "POLICY_ERROR", + "POLICY_KEEP", + "make_extension_shape", + "conditioning_diagnostics", + "prepare_conditioning", + "new_h3_latent", + "capture_state", + "select_context", + "validate_state", + "validate_plan", + "normalize_condition_latents", + "patch_layout_in_place", + "trim_audio", + "assemble_segments", + "make_prompt_plan", + "validate_prompt_plan", + "derive_chunk_seed", + "latent_motion_score", + "choose_context_frames", + "make_session", + "validate_session", + "entry_to_state", +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..173df49 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "comfyui-h3-continuum-join" +version = "2.1.4" +description = "Integrated N-chunk continuation for ComfyUI native MiniMax H3 video and audio" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "GPL-3.0-or-later" } +authors = [{ name = "ukr8b3g-cmyk" }] +dependencies = [] +keywords = ["comfyui", "minimax-h3", "video", "audio", "continuation"] + +[project.urls] +Repository = "https://github.com/ukr8b3g-cmyk/ComfyUI-H3-Continuum-Join" + +[tool.comfy] +PublisherId = "ukr8b3g-cmyk" +DisplayName = "H3 Continuum Join" +Icon = "" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6474f54 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = --import-mode=importlib diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fd5d7c2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +# No extra pip dependencies. Uses PyTorch and safetensors bundled with ComfyUI. diff --git a/state.py b/state.py new file mode 100644 index 0000000..55bf1b6 --- /dev/null +++ b/state.py @@ -0,0 +1,286 @@ +"""State and plan containers for H3 Continuum Join.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .constants import DEFAULT_STATE_CAPACITY_FRAMES, PLAN_MAGIC, STATE_MAGIC +from .temporal import ( + audio_latent_t_from_grid_offset, + audio_grid_offset, + audio_latent_t, + context_slots, + is_valid_frame_count, + make_av_context_window, + pixel_frames_for_latent_t, + video_latent_t, +) +from .version import PLAN_SCHEMA_VERSION, STATE_SCHEMA_VERSION + + +class StateValidationError(ValueError): + pass + + +def extract_av_streams(latent: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]: + if not isinstance(latent, dict) or "samples" not in latent: + raise StateValidationError("expected LATENT dictionary containing 'samples'") + samples = latent["samples"] + if hasattr(samples, "unbind"): + parts = list(samples.unbind()) + elif isinstance(samples, (tuple, list)): + parts = list(samples) + else: + raise StateValidationError( + "expected MiniMax H3 NestedTensor samples; got " + type(samples).__name__ + ) + if len(parts) < 2: + raise StateValidationError("MiniMax H3 latent must contain video and audio") + video, audio = parts[0], parts[1] + if video.ndim == 4: + video = video.unsqueeze(0) + if audio.ndim == 3: + audio = audio.unsqueeze(0) + if video.ndim != 5 or audio.ndim != 4: + raise StateValidationError( + f"unexpected H3 latent shapes: video={tuple(video.shape)}, audio={tuple(audio.shape)}" + ) + if video.shape[0] != 1 or audio.shape[0] != 1: + raise StateValidationError("H3 Continuum Join supports batch size 1") + if video.shape[1] != 24: + raise StateValidationError(f"expected 24 video channels, got {video.shape[1]}") + if audio.shape[1] != 32 or audio.shape[2] != 2: + raise StateValidationError(f"expected stereo [1,32,2,T] audio, got {tuple(audio.shape)}") + return video, audio + + +def make_plan( + *, continuation: bool, clip_index: int, total_frames: int, trim_frames: int, + width: int, height: int, context_frames: int, state_capacity_frames: int, + requested_extend_seconds: float, debug: bool, +) -> dict[str, Any]: + return { + "magic": PLAN_MAGIC, + "schema_version": PLAN_SCHEMA_VERSION, + "continuation": bool(continuation), + "clip_index": int(clip_index), + "total_frames": int(total_frames), + "trim_frames": int(trim_frames), + "net_frames": int(total_frames) - int(trim_frames), + "width": int(width), + "height": int(height), + "context_frames": int(context_frames), + "state_capacity_frames": int(state_capacity_frames), + "requested_extend_seconds": float(requested_extend_seconds), + "debug": bool(debug), + } + + +def validate_plan(plan: dict[str, Any]) -> dict[str, Any]: + if not isinstance(plan, dict) or plan.get("magic") != PLAN_MAGIC: + raise StateValidationError("invalid H3 Continuum plan") + if int(plan.get("schema_version", -1)) != PLAN_SCHEMA_VERSION: + raise StateValidationError( + f"unsupported plan schema {plan.get('schema_version')}; expected {PLAN_SCHEMA_VERSION}" + ) + for key in ( + "total_frames", + "trim_frames", + "net_frames", + "width", + "height", + "clip_index", + "state_capacity_frames", + ): + if not isinstance(plan.get(key), int): + raise StateValidationError(f"plan field '{key}' is missing or invalid") + if plan["total_frames"] < 5 or plan["width"] <= 0 or plan["height"] <= 0: + raise StateValidationError("plan dimensions or frame count are invalid") + if plan["trim_frames"] < 0 or plan["trim_frames"] >= plan["total_frames"]: + raise StateValidationError("plan trim range is invalid") + if plan["net_frames"] != plan["total_frames"] - plan["trim_frames"]: + raise StateValidationError("plan net_frames does not match total_frames - trim_frames") + if plan["clip_index"] < 1: + raise StateValidationError("plan clip_index must be at least 1") + if plan["state_capacity_frames"] not in (5, 22, 39): + raise StateValidationError("plan state_capacity_frames must be 5, 22, or 39") + if plan["state_capacity_frames"] > plan["net_frames"]: + raise StateValidationError("plan state capacity exceeds retained output frames") + continuation = bool(plan.get("continuation", False)) + if continuation and plan["trim_frames"] <= 0: + raise StateValidationError("continuation plan must trim a positive overlap") + if not continuation and plan["trim_frames"] != 0: + raise StateValidationError("initial plan cannot trim an overlap") + return plan + + +def capture_state( + latent: dict[str, Any], *, source_frame_count: int, clip_index: int, + capacity_frames: int = DEFAULT_STATE_CAPACITY_FRAMES, +) -> dict[str, Any]: + video, audio = extract_av_streams(latent) + actual_video_t = int(video.shape[2]) + actual_frames = pixel_frames_for_latent_t(actual_video_t) + if actual_frames != int(source_frame_count): + raise StateValidationError( + f"plan says {source_frame_count} frames but latent represents {actual_frames}" + ) + if not is_valid_frame_count(actual_frames) or video_latent_t(actual_frames) != actual_video_t: + raise StateValidationError( + f"sampler video latent T={actual_video_t} is not on the native H3 temporal grid" + ) + try: + window = make_av_context_window( + source_frame_count, + int(audio.shape[-1]), + capacity_frames, + ) + except ValueError as exc: + raise StateValidationError(str(exc)) from exc + slots = context_slots(capacity_frames) + audio_steps = window.audio_steps + if video.shape[2] < slots or audio.shape[-1] < audio_steps: + raise StateValidationError("sampler output is too short for requested state capacity") + grid_offset = audio_grid_offset(source_frame_count, int(audio.shape[-1])) + if not (-0.500001 <= grid_offset <= 0.500001): + raise StateValidationError(f"unexpected signed audio-grid offset {grid_offset:.6f}") + video_tail = video[:, :, -slots:].detach().to("cpu").contiguous().clone() + audio_tail = audio[..., -audio_steps:].detach().to("cpu").contiguous().clone() + if not bool(torch.isfinite(video_tail.float()).all().item()): + raise StateValidationError("video continuation state contains NaN or Inf") + if not bool(torch.isfinite(audio_tail.float()).all().item()): + raise StateValidationError("audio continuation state contains NaN or Inf") + return { + "magic": STATE_MAGIC, + "schema_version": STATE_SCHEMA_VERSION, + "clip_index": int(clip_index), + "source_frame_count": int(source_frame_count), + "capacity_frames": int(capacity_frames), + "width": int(video.shape[-1]) * 16, + "height": int(video.shape[-2]) * 16, + "video_tail": video_tail, + "audio_tail": audio_tail, + # Schema-v1 field name retained. The value is signed and represents + # audio-grid end offset, not only positive overhang. + "audio_overhang": float(grid_offset), + "source_mode": "latent_direct", + } + + +def validate_state(state: dict[str, Any]) -> dict[str, Any]: + if not isinstance(state, dict) or state.get("magic") != STATE_MAGIC: + raise StateValidationError("invalid H3 Continuum state") + if int(state.get("schema_version", -1)) != STATE_SCHEMA_VERSION: + raise StateValidationError( + f"unsupported state schema {state.get('schema_version')}; expected {STATE_SCHEMA_VERSION}" + ) + video, audio = state.get("video_tail"), state.get("audio_tail") + if not torch.is_tensor(video) or video.ndim != 5 or tuple(video.shape[:2]) != (1, 24): + raise StateValidationError("state video_tail must be [1,24,T,H,W]") + if not torch.is_tensor(audio) or audio.ndim != 4 or tuple(audio.shape[:3]) != (1, 32, 2): + raise StateValidationError("state audio_tail must be [1,32,2,T]") + if int(state.get("width", 0)) != int(video.shape[-1]) * 16: + raise StateValidationError("state width does not match video_tail") + if int(state.get("height", 0)) != int(video.shape[-2]) * 16: + raise StateValidationError("state height does not match video_tail") + capacity = int(state.get("capacity_frames", 0)) + if capacity not in (5, 22, 39): + raise StateValidationError("state capacity must be 5, 22, or 39 frames") + expected_video_t = context_slots(capacity) + expected_audio_t = audio_latent_t(capacity) + if int(video.shape[2]) != expected_video_t or int(audio.shape[-1]) != expected_audio_t: + raise StateValidationError( + "state tensor lengths do not match declared capacity: " + f"video T={video.shape[2]} (expected {expected_video_t}), " + f"audio T={audio.shape[-1]} (expected {expected_audio_t})" + ) + source_frames = int(state.get("source_frame_count", 0)) + if not is_valid_frame_count(source_frames): + raise StateValidationError("state source_frame_count is not on the H3 17k+5 grid") + if int(state.get("clip_index", 0)) < 1: + raise StateValidationError("state clip_index must be at least 1") + grid_offset = float(state.get("audio_overhang", 0.0)) + if not (-0.500001 <= grid_offset <= 0.500001): + raise StateValidationError("state signed audio-grid offset is outside [-0.5,0.5]") + if not bool(torch.isfinite(video.float()).all().item()): + raise StateValidationError("state video_tail contains NaN or Inf") + if not bool(torch.isfinite(audio.float()).all().item()): + raise StateValidationError("state audio_tail contains NaN or Inf") + return state + + +def select_context( + state: dict[str, Any], context_frames: int, *, include_audio: bool, +) -> tuple[torch.Tensor, torch.Tensor | None, float]: + state = validate_state(state) + if int(context_frames) > int(state["capacity_frames"]): + raise StateValidationError( + f"state retains {state['capacity_frames']} frames; {context_frames} requested" + ) + source_frames = int(state["source_frame_count"]) + grid_offset = float(state["audio_overhang"]) + try: + source_audio_t = audio_latent_t_from_grid_offset(source_frames, grid_offset) + window = make_av_context_window( + source_frames, + source_audio_t, + int(context_frames), + ) + except ValueError as exc: + raise StateValidationError( + "Native Continuity unavailable for this state: " + str(exc) + ) from exc + slots = context_slots(int(context_frames)) + video = state["video_tail"][:, :, -slots:].contiguous() + audio = None + if include_audio: + if int(state["audio_tail"].shape[-1]) < window.audio_steps: + raise StateValidationError( + "Native Continuity state does not retain the required source audio-grid tail" + ) + audio = state["audio_tail"][..., -window.audio_steps:].contiguous() + return video, audio, grid_offset + + +def _tensor_fingerprint(tensor: torch.Tensor | None) -> tuple[Any, ...] | None: + if tensor is None: + return None + value = tensor.detach().to(device="cpu", dtype=torch.float32) + finite = bool(torch.isfinite(value).all().item()) + if not finite: + raise StateValidationError("continuation context contains NaN or Inf") + flat = value.reshape(-1) + if flat.numel() <= 64: + sample = flat + else: + indices = torch.linspace(0, flat.numel() - 1, 64, dtype=torch.long) + sample = flat.index_select(0, indices) + weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32) + return ( + tuple(int(part) for part in tensor.shape), + str(tensor.dtype), + finite, + float(value.mean().item()), + float(value.std(unbiased=False).item()), + float((sample * weights).sum().item()), + ) + + +def context_fingerprint( + video: torch.Tensor, + audio: torch.Tensor | None, +) -> tuple[Any, ...]: + return (_tensor_fingerprint(video), _tensor_fingerprint(audio)) + + +def assert_context_unchanged( + video: torch.Tensor, + audio: torch.Tensor | None, + expected: tuple[Any, ...], +) -> None: + if context_fingerprint(video, audio) != expected: + raise StateValidationError( + "Native Continuity context tensor changed during sampling" + ) diff --git a/state_io.py b/state_io.py new file mode 100644 index 0000000..210ddde --- /dev/null +++ b/state_io.py @@ -0,0 +1,161 @@ +"""Crash-resistant safetensors state persistence with a JSON mirror. + +The safetensors file is the authoritative single-file commit. All non-tensor +metadata is embedded in its header, while the sidecar JSON remains human +readable. Save replaces the JSON mirror first and the safetensors file last, so +a process interruption cannot commit tensors without their matching metadata. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import uuid +from pathlib import Path +from typing import Any + +from safetensors import safe_open +from safetensors.torch import save_file + +from .state import validate_state + +LOG = logging.getLogger("h3_continuum_join") +_SAFE_PREFIX = re.compile(r"[^A-Za-z0-9._-]+") +_HEADER_KEY = "h3_continuum_state_json" + + +def sanitize_prefix(prefix: str) -> str: + value = _SAFE_PREFIX.sub("_", str(prefix).strip()).strip("._") + return value or "h3_continuum" + + +def state_directory() -> Path: + try: + import folder_paths + root = Path(folder_paths.get_output_directory()) + except Exception: + root = Path.cwd() + path = root / "h3_continuum_states" + path.mkdir(parents=True, exist_ok=True) + return path + + +def state_paths(prefix: str, slot: int) -> tuple[Path, Path]: + slot = int(slot) + if not (1 <= slot <= 9999): + raise ValueError("state slot must be between 1 and 9999") + name = f"{sanitize_prefix(prefix)}_slot{slot:04d}" + root = state_directory() + return root / f"{name}.safetensors", root / f"{name}.json" + + + +def _fsync_file(path: Path) -> None: + """Best-effort durability before atomic replacement.""" + + try: + with path.open("rb") as handle: + os.fsync(handle.fileno()) + except OSError: + # Atomic replace still protects against partial readers; some filesystems + # or Windows configurations do not expose meaningful fsync semantics. + pass + + +def _fsync_directory(path: Path) -> None: + if os.name == "nt": + return + try: + descriptor = os.open(str(path), os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError: + pass + +def save_state(state: dict[str, Any], *, prefix: str, slot: int) -> tuple[Path, Path]: + state = validate_state(state) + tensor_path, json_path = state_paths(prefix, slot) + token = uuid.uuid4().hex + tensor_tmp = tensor_path.with_name(f".{tensor_path.name}.{token}.tmp") + json_tmp = json_path.with_name(f".{json_path.name}.{token}.tmp") + tensors = { + "video_tail": state["video_tail"].detach().to("cpu").contiguous(), + "audio_tail": state["audio_tail"].detach().to("cpu").contiguous(), + } + metadata = {k: v for k, v in state.items() if k not in ("video_tail", "audio_tail")} + metadata["_state_id"] = token + serialized = json.dumps(metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + try: + save_file(tensors, str(tensor_tmp), metadata={_HEADER_KEY: serialized}) + _fsync_file(tensor_tmp) + with json_tmp.open("w", encoding="utf-8", newline="\n") as handle: + json.dump(metadata, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + # The sidecar is informational. Commit the authoritative safetensors + # file last so its embedded metadata always matches its tensor payload. + os.replace(json_tmp, json_path) + os.replace(tensor_tmp, tensor_path) + _fsync_directory(tensor_path.parent) + finally: + for path in (tensor_tmp, json_tmp): + try: + path.unlink(missing_ok=True) + except OSError: + pass + return tensor_path, json_path + + +def _load_authoritative(tensor_path: Path) -> dict[str, Any]: + with safe_open(str(tensor_path), framework="pt", device="cpu") as handle: + header = handle.metadata() or {} + serialized = header.get(_HEADER_KEY) + if serialized is None: + raise ValueError("state safetensors has no embedded Continuum metadata") + state = json.loads(serialized) + state["video_tail"] = handle.get_tensor("video_tail") + state["audio_tail"] = handle.get_tensor("audio_tail") + return state + + +def load_state(*, prefix: str, slot: int) -> dict[str, Any]: + tensor_path, json_path = state_paths(prefix, slot) + if not tensor_path.is_file(): + raise FileNotFoundError(f"state slot does not exist: {tensor_path.name}") + try: + state = _load_authoritative(tensor_path) + except ValueError as exc: + # Backward-compatible path for early development states that stored + # metadata only in the JSON sidecar. + if not json_path.is_file(): + raise + LOG.warning("Loading legacy Continuum state without embedded metadata: %s", exc) + with json_path.open("r", encoding="utf-8") as handle: + state = json.load(handle) + with safe_open(str(tensor_path), framework="pt", device="cpu") as tensors: + state["video_tail"] = tensors.get_tensor("video_tail") + state["audio_tail"] = tensors.get_tensor("audio_tail") + + # The JSON mirror is not authoritative, but report stale/mismatched files. + if json_path.is_file(): + try: + with json_path.open("r", encoding="utf-8") as handle: + mirror = json.load(handle) + if mirror.get("_state_id") != state.get("_state_id"): + LOG.warning("Ignoring stale Continuum JSON mirror: %s", json_path.name) + except (OSError, ValueError, TypeError) as exc: + LOG.warning("Ignoring unreadable Continuum JSON mirror %s: %s", json_path.name, exc) + return validate_state(state) + + +def state_mtime(*, prefix: str, slot: int) -> tuple[float, float]: + tensor_path, json_path = state_paths(prefix, slot) + return ( + tensor_path.stat().st_mtime if tensor_path.exists() else -1.0, + json_path.stat().st_mtime if json_path.exists() else -1.0, + ) diff --git a/temporal.py b/temporal.py new file mode 100644 index 0000000..ab5697a --- /dev/null +++ b/temporal.py @@ -0,0 +1,225 @@ +"""MiniMax H3 temporal-grid helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from fractions import Fraction + +from .constants import AUDIO_LATENT_FPS, FPS, FRAME_PER_TOKEN + + +_AUDIO_TICKS_PER_FRAME = Fraction(int(AUDIO_LATENT_FPS), int(FPS)) + + +@dataclass(frozen=True, slots=True) +class ExtensionShape: + context_frames: int + requested_new_frames: int + total_frames: int + net_new_frames: int + video_latent_t: int + audio_latent_t: int + + @property + def actual_extend_seconds(self) -> float: + return self.net_new_frames / FPS + + +@dataclass(frozen=True, slots=True) +class AVContextWindow: + source_frame_count: int + context_frames: int + video_start_slot: int + video_stop_slot: int + audio_start_tick: int + audio_stop_tick: int + audio_grid_offset: float + + @property + def audio_steps(self) -> int: + return self.audio_stop_tick - self.audio_start_tick + + +def is_valid_frame_count(frame_count: int) -> bool: + return frame_count >= 5 and frame_count % 17 == 5 + + +def align_frame_count_up(frame_count: int) -> int: + n = max(5, int(frame_count)) + remainder = (n - 5) % 17 + return n if remainder == 0 else n + (17 - remainder) + + +def align_frame_count_nearest(frame_count: int, *, minimum: int = 5) -> int: + target = max(int(frame_count), int(minimum), 5) + min_valid = align_frame_count_up(minimum) + k_floor = max(0, (target - 5) // 17) + low = max(5 + 17 * k_floor, min_valid) + high = max(low + 17, min_valid) + return low if abs(target - low) < abs(high - target) else high + + +def video_latent_t(frame_count: int) -> int: + if not is_valid_frame_count(frame_count): + raise ValueError(f"H3 frame count must satisfy 17k+5; got {frame_count}") + return 2 if frame_count <= 5 else ((frame_count - 5) // 17) * 5 + 2 + + +def pixel_frames_for_latent_t(latent_t: int) -> int: + if int(latent_t) < 1: + raise ValueError("video latent T must be positive") + return sum(FRAME_PER_TOKEN[index % 5] for index in range(int(latent_t))) + + +def latent_slot_offsets(latent_t: int) -> tuple[int, ...]: + if int(latent_t) < 1: + raise ValueError("video latent T must be positive") + out: list[int] = [] + cursor = 0 + for index in range(int(latent_t)): + out.append(cursor) + cursor += FRAME_PER_TOKEN[index % 5] + return tuple(out) + + +def context_slots(context_frames: int) -> int: + target = int(context_frames) + for slots in range(1, 256): + frames = pixel_frames_for_latent_t(slots) + if frames == target: + return slots + if frames > target: + break + raise ValueError( + f"context_frames={context_frames} is not representable; use 5, 22, or 39" + ) + + +def audio_latent_t(frame_count: int) -> int: + return int(round(Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME)) + + +def audio_grid_offset(frame_count: int, actual_audio_t: int) -> float: + """Signed audio-grid end offset in 40 Hz latent steps. + + Native H3 uses ``round(duration * 40)``. Depending on the valid 17k+5 + frame count, the audio grid can end up to half a latent step before or + after the exact video end. Keeping the sign is required for sample-accurate + continuation placement. + """ + + value = Fraction(int(actual_audio_t)) - Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME + value = float(value) + return 0.0 if abs(value) < 1e-9 else value + + +# Backward-compatible public name used by state schema v1. +def audio_overhang(frame_count: int, actual_audio_t: int) -> float: + return audio_grid_offset(frame_count, actual_audio_t) + + +def audio_latent_t_from_grid_offset(frame_count: int, grid_offset: float) -> int: + """Recover the actual source audio-grid end without guessing its phase.""" + + exact_end = float(Fraction(int(frame_count)) * _AUDIO_TICKS_PER_FRAME) + candidate = int(round(exact_end + float(grid_offset))) + recovered = audio_grid_offset(int(frame_count), candidate) + if abs(recovered - float(grid_offset)) > 1e-6: + raise ValueError( + "source audio-grid phase cannot be recovered from frame count and signed offset" + ) + return candidate + + +def make_av_context_window( + source_frame_count: int, + actual_audio_t: int, + context_frames: int, +) -> AVContextWindow: + """Resolve a raw AV tail on the native 5-step video and 40/24 audio grids.""" + + source_frames = int(source_frame_count) + context = int(context_frames) + if not is_valid_frame_count(source_frames): + raise ValueError( + f"source_frame_count={source_frames} is not on the native H3 17k+5 grid" + ) + if context not in (5, 22, 39) or context > source_frames: + raise ValueError(f"unsupported {context}-frame context for {source_frames} source frames") + + source_video_t = video_latent_t(source_frames) + context_video_t = context_slots(context) + video_start = source_video_t - context_video_t + if video_start < 0 or video_start % len(FRAME_PER_TOKEN): + raise ValueError( + "video context tail does not begin at a uniquely recoverable H3 latent-cycle phase" + ) + + audio_stop = int(actual_audio_t) + grid_offset = audio_grid_offset(source_frames, audio_stop) + if not (-0.500001 <= grid_offset <= 0.500001): + raise ValueError(f"source audio-grid offset {grid_offset:.6f} is outside [-0.5,0.5]") + audio_steps = audio_latent_t(context) + audio_start = audio_stop - audio_steps + if audio_start < 0: + raise ValueError("source audio latent is too short for the requested context") + + return AVContextWindow( + source_frame_count=source_frames, + context_frames=context, + video_start_slot=video_start, + video_stop_slot=source_video_t, + audio_start_tick=audio_start, + audio_stop_tick=audio_stop, + audio_grid_offset=grid_offset, + ) + + +def make_extension_shape(context_frames: int, extend_seconds: float) -> ExtensionShape: + context_frames = int(context_frames) + context_slots(context_frames) + requested_new = max(1, int(round(float(extend_seconds) * FPS))) + total = align_frame_count_nearest( + context_frames + requested_new, minimum=context_frames + 1 + ) + net = total - context_frames + if net < 1: + raise RuntimeError("computed continuation contains no new frames") + return ExtensionShape( + context_frames=context_frames, + requested_new_frames=requested_new, + total_frames=total, + net_new_frames=net, + video_latent_t=video_latent_t(total), + audio_latent_t=audio_latent_t(total), + ) + + +def largest_context_capacity(frame_count: int) -> int: + """Largest supported continuation profile contained in ``frame_count``.""" + + available = int(frame_count) + for capacity in (39, 22, 5): + if available >= capacity: + return capacity + raise ValueError( + f"at least 5 retained frames are required for a continuation state; got {available}" + ) + + +def run_temporal_self_test() -> None: + expected = { + 5: (2, (0, 1)), + 22: (7, (0, 1, 5, 9, 13, 17, 18)), + 39: (12, (0, 1, 5, 9, 13, 17, 18, 22, 26, 30, 34, 35)), + } + for frames, (slots, offsets) in expected.items(): + assert context_slots(frames) == slots + assert latent_slot_offsets(slots) == offsets + assert pixel_frames_for_latent_t(slots) == frames + shape = make_extension_shape(22, 5.0) + assert (shape.total_frames, shape.net_new_frames) == (141, 119) + assert (shape.video_latent_t, shape.audio_latent_t) == (42, 235) + window = make_av_context_window(141, 235, 22) + assert (window.video_start_slot, window.video_stop_slot) == (35, 42) + assert (window.audio_start_tick, window.audio_stop_tick) == (198, 235) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b827ae9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE_NAME = "ComfyUI_H3_Continuum_Join" +if PACKAGE_NAME not in sys.modules: + spec = importlib.util.spec_from_file_location( + PACKAGE_NAME, + ROOT / "__init__.py", + submodule_search_locations=[str(ROOT)], + ) + module = importlib.util.module_from_spec(spec) + module.__path__ = [str(ROOT)] + sys.modules[PACKAGE_NAME] = module diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py new file mode 100644 index 0000000..219122b --- /dev/null +++ b/tests/test_compatibility.py @@ -0,0 +1,83 @@ +from types import SimpleNamespace + +from ComfyUI_H3_Continuum_Join.compatibility import ( + _missing_callable_parameters, + accelerator_summary, +) + + +def test_signature_contract_accepts_native_packed_layout_signature(): + def native( + self, + text_len, + latent_t, + latent_h, + latent_w, + audio_t, + keyframes=None, + refs=None, + frame_count=None, + ): + pass + + assert _missing_callable_parameters( + native, + positional=("text_len", "latent_t", "latent_h", "latent_w", "audio_t"), + keywords=("keyframes", "refs", "frame_count"), + ) == [] + + +def test_signature_contract_accepts_sol_attn_style_forwarding_wrapper(): + # Sol-Attn's H3 Morton registration wraps PackedLayout.__init__ with this + # effective shape: five named base args followed by *args/**kwargs. + def sol_wrapper( + self, text_len, latent_t, latent_h, latent_w, audio_t, *args, **kwargs + ): + pass + + assert _missing_callable_parameters( + sol_wrapper, + positional=("text_len", "latent_t", "latent_h", "latent_w", "audio_t"), + keywords=("keyframes", "refs", "frame_count"), + ) == [] + + +def test_signature_contract_still_fails_closed_for_real_keyword_removal(): + def incompatible(self, text_len, latent_t, latent_h, latent_w, audio_t): + pass + + assert _missing_callable_parameters( + incompatible, + positional=("text_len", "latent_t", "latent_h", "latent_w", "audio_t"), + keywords=("keyframes", "refs", "frame_count"), + ) == ["keyframes", "refs", "frame_count"] + + +def test_accelerator_summary_reads_modelpatcher_wrappers_and_transformer_markers(): + model = SimpleNamespace( + wrappers={ + "diffusion_model": { + "spectrum_h3.diffusion_model": [object()], + }, + "apply_model": { + "h3_continuum_join.apply_model.v1": [object()], + }, + }, + model_options={ + "transformer_options": { + "optimized_attention_override": object(), + "sol_compose": {"min_tokens": 4096}, + }, + "h3_continuum_join": {"api_version": 1}, + }, + ) + summary = accelerator_summary(model) + assert "Spectrum" in summary + assert "Sol-Attn" in summary + assert "attention override" in summary + assert "Continuum" in summary + + +def test_accelerator_summary_is_informational_when_no_markers_exist(): + model = SimpleNamespace(wrappers={}, model_options={}) + assert "informational only" in accelerator_summary(model) diff --git a/tests/test_layout_adapter.py b/tests/test_layout_adapter.py new file mode 100644 index 0000000..9e1fe71 --- /dev/null +++ b/tests/test_layout_adapter.py @@ -0,0 +1,156 @@ +from types import SimpleNamespace + +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + MARK_AUDIO_CONTEXT, + MARK_AUDIO_END_FRAME, + MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, + MARK_VIDEO_CONTEXT, +) +from ComfyUI_H3_Continuum_Join.layout_adapter import ( + normalize_condition_latents, + patch_layout_in_place, +) + + +def _video_times(origin: float): + offsets = (0, 1, 5, 9, 13, 17, 18) + return [origin + value * 5.0 / 3.0 for value in offsets] + + +def _fake_layout(*, with_last_keyframe=False): + # text=3, optional one stock keyframe, a 22-frame video/audio context ref, + # target audio=10 steps, target video=7 slots x 2 rows. + row = 0 + segments = [(row, row + 3, "text")] + row += 3 + if with_last_keyframe: + segments.append((row, row + 2, "cond")) + row += 2 + audio_ref = (row, row + 74) + segments.append((*audio_ref, "ref_audio")) + row += 74 + video_ref = (row, row + 14) + segments.append((*video_ref, "ref_img")) + row += 14 + target_audio = (row, row + 20) + segments.append((*target_audio, "audio")) + row += 20 + target_video = (row, row + 14) + segments.append((*target_video, "video")) + row += 14 + + pos = torch.zeros(row, 3, dtype=torch.float64) + pos[:3, 0] = torch.arange(3, dtype=torch.float64) + if with_last_keyframe: + # Stock last keyframe is text-relative before refs shift the target. + pos[3:5, 0] = 3.0 + 35.0 + # Stock context ref lives in its own pre-target coordinate span. + a0, a1 = audio_ref + pos[a0 : a0 + 37, 0] = torch.arange(37, dtype=torch.float64) + 3.0 + pos[a0 + 37 : a1, 0] = torch.arange(37, dtype=torch.float64) + 3.0 + v0, _v1 = video_ref + for slot, value in enumerate(_video_times(3.0)): + pos[v0 + slot * 2 : v0 + (slot + 1) * 2, 0] = value + # Target origin = text 3 + max(audio 37, video span 36 2/3) = 40. + ta0, ta1 = target_audio + pos[ta0 : ta0 + 10, 0] = torch.arange(10, dtype=torch.float64) + 40.0 + pos[ta0 + 10 : ta1, 0] = torch.arange(10, dtype=torch.float64) + 40.0 + tv0, _tv1 = target_video + for slot, value in enumerate(_video_times(40.0)): + pos[tv0 + slot * 2 : tv0 + (slot + 1) * 2, 0] = value + # Give every target video row distinct h/w values so the full-coordinate + # copy (not only time) is tested. + pos[tv0 : tv0 + 14, 1] = torch.arange(14, dtype=torch.float64) + pos[tv0 : tv0 + 14, 2] = torch.arange(14, dtype=torch.float64) + 100 + + return SimpleNamespace( + segments=segments, + position_ids=pos, + signature=(3, 7, 2, 2, 10), + ), video_ref, target_video + + +def _context_ref(video, audio): + return { + "kind": "video_audio", + "latent_t": 7, + "latent_h": 2, + "latent_w": 2, + "ref_audio_t": 37, + "latent": video, + "audio_latent": audio, + MARK_VIDEO_CONTEXT: True, + MARK_CONTEXT_FRAMES: 22, + MARK_AUDIO_CONTEXT: True, + MARK_AUDIO_END_FRAME: 22.0, + MARK_AUDIO_OVERHANG: 1.0 / 3.0, + } + + +def test_layout_patches_video_audio_ref_in_place_and_is_idempotent(): + layout, video_ref_span, target_video_span = _fake_layout() + tensor_id = id(layout.position_ids) + video = torch.zeros(1, 24, 7, 2, 2) + audio = torch.zeros(1, 32, 2, 37) + payload = {"layout": layout, "keyframes": [], "refs": [_context_ref(video, audio)]} + normalize_condition_latents(payload) + result = patch_layout_in_place(payload) + assert result["video_refs"] == 1 + assert result["audio_windows"] == 1 + assert id(layout.position_ids) == tensor_id + + rv0, rv1 = video_ref_span + tv0, tv1 = target_video_span + assert torch.equal(layout.position_ids[rv0:rv1], layout.position_ids[tv0:tv1]) + # 22 frames (36 2/3 audio steps) plus source overhang 1/3 = 37, + # so the audio context begins exactly at target origin 40. + audio_start = next(a for a, _b, kind in layout.segments if kind == "ref_audio") + assert torch.isclose( + layout.position_ids[audio_start, 0], torch.tensor(40.0, dtype=torch.float64) + ) + + before = layout.position_ids.clone() + second = patch_layout_in_place(payload) + assert second["status"] == "already_patched" + assert torch.equal(before, layout.position_ids) + assert payload["cond_video_latents"] == [video] + assert payload["cond_audio_latents"] == [audio] + + +def test_existing_last_keyframe_is_shifted_with_context_ref(): + layout, _video_ref_span, _target_video_span = _fake_layout(with_last_keyframe=True) + image = torch.zeros(1, 24, 1, 2, 2) + video = torch.zeros(1, 24, 7, 2, 2) + audio = torch.zeros(1, 32, 2, 37) + payload = { + "layout": layout, + "keyframes": [{"resolved_frame_index": 140, "latent": image}], + "refs": [_context_ref(video, audio)], + } + normalize_condition_latents(payload) + patch_layout_in_place(payload) + cond_start = next(a for a, _b, kind in layout.segments if kind == "cond") + # Reference cursor shift is 37. + assert torch.all(layout.position_ids[cond_start : cond_start + 2, 0] == 75.0) + assert payload["cond_video_latents"] == [image, video] + + +def test_negative_audio_grid_offset_places_context_before_target_origin(): + layout, _video_ref_span, _target_video_span = _fake_layout() + video = torch.zeros(1, 24, 7, 2, 2) + audio = torch.zeros(1, 32, 2, 37) + ref = _context_ref(video, audio) + ref[MARK_AUDIO_OVERHANG] = -1.0 / 3.0 + payload = {"layout": layout, "keyframes": [], "refs": [ref]} + normalize_condition_latents(payload) + patch_layout_in_place(payload) + audio_start = next(a for a, _b, kind in layout.segments if kind == "ref_audio") + # target origin is 40; 22 frames = 36 2/3 audio steps, signed offset -1/3, + # and the retained window is 37 steps, so it starts 2/3 step before origin. + assert torch.isclose( + layout.position_ids[audio_start, 0], + torch.tensor(40.0 - 2.0 / 3.0, dtype=torch.float64), + ) diff --git a/tests/test_model_patch.py b/tests/test_model_patch.py new file mode 100644 index 0000000..a344275 --- /dev/null +++ b/tests/test_model_patch.py @@ -0,0 +1,219 @@ +from types import SimpleNamespace + +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + CONTINUUM_INTEROP_KEY, + MARK_AUDIO_CONTEXT, + MARK_AUDIO_END_FRAME, + MARK_AUDIO_OVERHANG, + MARK_CONTEXT_FRAMES, + MARK_VIDEO_CONTEXT, +) +from ComfyUI_H3_Continuum_Join.model_patch import ( + _wrapper_factory, + clone_model_for_chunk, + patch_model, +) + + +class Executor: + def __init__(self): + inner_type = type( + "MiniMaxH3Model", + (), + { + "__module__": "comfy.ldm.minimax.model", + "blocks": (), + "final_layer": object(), + "patch_size": (1, 2, 2), + "latents_dim": 24, + "audio_latents_dim": 32, + "rope_freqs": lambda self, position_ids, device: position_ids, + }, + ) + self.class_obj = SimpleNamespace(diffusion_model=inner_type()) + + def __call__(self, *args, **kwargs): + return kwargs["minimax_payload"] + + +def _layout(): + # text 1; ref audio 2 rows; ref video 1 row; target audio 2 rows; target video 1 row + segments = [ + (0, 1, "text"), + (1, 3, "ref_audio"), + (3, 4, "ref_img"), + (4, 6, "audio"), + (6, 7, "video"), + ] + pos = torch.zeros(7, 3, dtype=torch.float64) + pos[1:3, 0] = 1 + pos[3, 0] = 1 + pos[4:6, 0] = 2 + pos[6, 0] = 2 + return SimpleNamespace(segments=segments, position_ids=pos, signature=(1, 1, 1, 1, 1)) + + +def test_apply_model_wrapper_repairs_payload_before_executor(): + video = torch.zeros(1, 24, 1, 1, 1) + audio = torch.zeros(1, 32, 2, 1) + ref = { + "kind": "video_audio", + "latent_t": 1, + "latent_h": 1, + "latent_w": 1, + "ref_audio_t": 1, + "latent": video, + "audio_latent": audio, + MARK_VIDEO_CONTEXT: True, + MARK_CONTEXT_FRAMES: 1, + MARK_AUDIO_CONTEXT: True, + MARK_AUDIO_END_FRAME: 1.0, + MARK_AUDIO_OVERHANG: 0.0, + } + payload = {"layout": _layout(), "keyframes": [], "refs": [ref]} + wrapper = _wrapper_factory(strict=True, debug=False) + result = wrapper(Executor(), torch.zeros(1), minimax_payload=payload) + assert result["cond_video_latents"] == [video] + assert result["cond_audio_latents"] == [audio] + assert torch.equal(result["layout"].position_ids[3:4], result["layout"].position_ids[6:7]) + + +def test_apply_model_wrapper_fails_when_actual_topology_changes(): + video = torch.zeros(1, 24, 1, 1, 1) + audio = torch.zeros(1, 32, 2, 1) + ref = { + "kind": "video_audio", + "latent_t": 1, + "latent_h": 1, + "latent_w": 1, + "ref_audio_t": 1, + "latent": video, + "audio_latent": audio, + MARK_VIDEO_CONTEXT: True, + MARK_CONTEXT_FRAMES: 1, + MARK_AUDIO_CONTEXT: True, + MARK_AUDIO_END_FRAME: 1.0, + MARK_AUDIO_OVERHANG: 0.0, + } + payload = {"layout": _layout(), "keyframes": [], "refs": [ref]} + wrapper = _wrapper_factory(strict=True, debug=False) + options = { + "spectrum_h3_actual": True, + "cond_or_uncond": [0], + "uuids": ["positive"], + } + wrapper( + Executor(), + torch.zeros(1), + transformer_options=options, + minimax_payload=payload, + ) + payload["layout"].position_ids[-1, 1] = 1.0 + with pytest.raises(RuntimeError, match="topology changed"): + wrapper( + Executor(), + torch.zeros(1), + transformer_options=options, + minimax_payload=payload, + ) + + +class FakeModelPatcher: + def __init__(self): + inner_type = type( + "MiniMaxH3Model", + (), + { + "__module__": "comfy.ldm.minimax.model", + "blocks": (), + "final_layer": object(), + "patch_size": (1, 2, 2), + "latents_dim": 24, + "audio_latents_dim": 32, + "rope_freqs": lambda self, position_ids, device: position_ids, + }, + ) + self.model = SimpleNamespace(diffusion_model=inner_type()) + self.model_options = {} + self.added = [] + self.removed = [] + + def clone(self): + clone = FakeModelPatcher() + clone.model = self.model + clone.model_options = {k: dict(v) if isinstance(v, dict) else v for k, v in self.model_options.items()} + return clone + + def add_wrapper_with_key(self, wrapper_type, key, wrapper): + self.added.append((wrapper_type, key, wrapper)) + + def remove_wrappers_with_key(self, wrapper_type, key): + self.removed.append((wrapper_type, key)) + + +def test_patch_model_validates_and_installs_per_model_wrapper(monkeypatch): + import sys + from types import ModuleType + + extension = ModuleType("comfy.patcher_extension") + extension.WrappersMP = SimpleNamespace(APPLY_MODEL="apply_model") + comfy = sys.modules.get("comfy") or ModuleType("comfy") + monkeypatch.setitem(sys.modules, "comfy", comfy) + monkeypatch.setitem(sys.modules, "comfy.patcher_extension", extension) + + patched = patch_model(FakeModelPatcher(), strict=True, debug=False) + assert patched.removed[0][1] == "h3_continuum_join.apply_model.v1" + assert patched.added[0][1] == "h3_continuum_join.apply_model.v1" + assert patched.model_options["h3_continuum_join"]["api_version"] == 1 + + +def test_chunk_model_copy_on_write_and_context_scoped_hint(monkeypatch): + import sys + from types import ModuleType + + extension = ModuleType("comfy.patcher_extension") + extension.WrappersMP = SimpleNamespace(APPLY_MODEL="apply_model") + comfy = sys.modules.get("comfy") or ModuleType("comfy") + monkeypatch.setitem(sys.modules, "comfy", comfy) + monkeypatch.setitem(sys.modules, "comfy.patcher_extension", extension) + + source = FakeModelPatcher() + source.model_options = {"transformer_options": {"existing": "kept"}} + base = patch_model(source, strict=True, debug=False) + first = clone_model_for_chunk( + base, + strict=True, + debug=False, + chunk_index=1, + context_frames=None, + ) + second = clone_model_for_chunk( + base, + strict=True, + debug=False, + chunk_index=2, + context_frames=22, + ) + third = clone_model_for_chunk( + base, + strict=True, + debug=False, + chunk_index=3, + context_frames=22, + ) + + assert source.model_options == {"transformer_options": {"existing": "kept"}} + assert CONTINUUM_INTEROP_KEY not in first.model_options["transformer_options"] + request = second.model_options["transformer_options"][CONTINUUM_INTEROP_KEY] + assert request == { + "api": 1, + "active": True, + "min_actual_prefix_steps": 2, + "chunk_index": 2, + "context_frames": 22, + } + assert request is not third.model_options["transformer_options"][CONTINUUM_INTEROP_KEY] + assert CONTINUUM_INTEROP_KEY not in base.model_options["transformer_options"] diff --git a/tests/test_nodes_logic.py b/tests/test_nodes_logic.py new file mode 100644 index 0000000..b73ff6e --- /dev/null +++ b/tests/test_nodes_logic.py @@ -0,0 +1,95 @@ +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + MARK_AUDIO_CONTEXT, + MARK_VIDEO_CONTEXT, +) +from ComfyUI_H3_Continuum_Join.nodes import ( + POLICY_REPLACE, + H3ContinuumAssemble, + _conditioning_diagnostics, + _prepare_conditioning, + _trim_audio, +) + + +def test_prepare_conditioning_uses_one_video_audio_ref_and_moves_last(): + old_first = torch.zeros(1, 24, 1, 2, 2) + old_last = torch.ones(1, 24, 1, 2, 2) + tags = torch.tensor([1, 0, 0, 1]) + conditioning = [ + [ + torch.zeros(1, 4, 3), + { + "minimax_token_tags": tags, + "minimax_frame_count": 124, + "minimax_keyframes": [ + {"resolved_frame_index": 0, "latent": old_first}, + {"resolved_frame_index": 123, "latent": old_last}, + ], + }, + ] + ] + video_context = torch.randn(1, 24, 7, 2, 2) + audio_context = torch.randn(1, 32, 2, 37) + out = _prepare_conditioning( + conditioning, + video_context=video_context, + audio_context=audio_context, + audio_grid_offset=1 / 3, + context_frames=22, + new_frame_count=141, + first_frame_policy=POLICY_REPLACE, + preserve_last_frame=True, + ) + meta = out[0][1] + assert len(meta["minimax_keyframes"]) == 1 + assert meta["minimax_keyframes"][0]["resolved_frame_index"] == 140 + assert len(meta["minimax_refs"]) == 1 + ref = meta["minimax_refs"][0] + assert ref["kind"] == "video_audio" + assert ref[MARK_VIDEO_CONTEXT] is True + assert ref[MARK_AUDIO_CONTEXT] is True + assert ref["latent"] is video_context + assert ref["audio_latent"] is audio_context + assert _conditioning_diagnostics(conditioning)["visual_tokens"] == 2 + + +def test_prepare_video_only_context(): + conditioning = [[torch.zeros(1, 2, 3), {}]] + video_context = torch.randn(1, 24, 2, 2, 2) + out = _prepare_conditioning( + conditioning, + video_context=video_context, + audio_context=None, + audio_grid_offset=0.0, + context_frames=5, + new_frame_count=124, + first_frame_policy=POLICY_REPLACE, + preserve_last_frame=True, + ) + ref = out[0][1]["minimax_refs"][0] + assert ref["kind"] == "video" + assert ref["ref_audio_t"] == 0 + assert MARK_AUDIO_CONTEXT not in ref + + +def test_trim_audio_matches_video_duration(): + audio = {"waveform": torch.arange(1 * 2 * 4000).reshape(1, 2, 4000), "sample_rate": 1000} + out = _trim_audio(audio, trim_frames=24, output_frames=48) + assert out["waveform"].shape == (1, 2, 2000) + assert out["waveform"][0, 0, 0] == audio["waveform"][0, 0, 1000] + + +def test_assemble_segments(): + node = H3ContinuumAssemble() + a_images = torch.zeros(24, 4, 6, 3) + b_images = torch.ones(12, 4, 6, 3) + a_audio = {"waveform": torch.zeros(1, 2, 1000), "sample_rate": 1000} + # One extra sample is tolerated and corrected at the final total boundary. + b_audio = {"waveform": torch.ones(1, 2, 501), "sample_rate": 1000} + images, audio, report = node.assemble(a_images, a_audio, b_images, b_audio) + assert images.shape[0] == 36 + assert audio["waveform"].shape[-1] == 1500 + assert torch.all(images[-12:] == 1) + assert "sync correction -1" in report diff --git a/tests/test_public_api.py b/tests/test_public_api.py new file mode 100644 index 0000000..264d285 --- /dev/null +++ b/tests/test_public_api.py @@ -0,0 +1,20 @@ +from ComfyUI_H3_Continuum_Join import public_api + + +def test_public_api_is_v2_ready_and_complete(): + assert public_api.PUBLIC_API_VERSION == 2 + for name in ( + "make_extension_shape", + "prepare_conditioning", + "new_h3_latent", + "capture_state", + "select_context", + "patch_layout_in_place", + "trim_audio", + "assemble_segments", + "make_prompt_plan", + "derive_chunk_seed", + "choose_context_frames", + "validate_session", + ): + assert callable(getattr(public_api, name)) diff --git a/tests/test_state.py b/tests/test_state.py new file mode 100644 index 0000000..b12118b --- /dev/null +++ b/tests/test_state.py @@ -0,0 +1,69 @@ +import torch + +from ComfyUI_H3_Continuum_Join.constants import PLAN_MAGIC +from ComfyUI_H3_Continuum_Join.state import ( + capture_state, + select_context, + validate_plan, + validate_state, +) + + +class FakeNested: + def __init__(self, parts): + self.parts = parts + + def unbind(self): + return tuple(self.parts) + + +def _latent(): + return { + "samples": FakeNested( + ( + torch.randn(1, 24, 37, 4, 6), + torch.randn(1, 32, 2, 207), + ) + ) + } + + +def test_capture_and_select(): + state = capture_state(_latent(), source_frame_count=124, clip_index=1) + validate_state(state) + assert state["video_tail"].shape == (1, 24, 12, 4, 6) + assert state["audio_tail"].shape == (1, 32, 2, 65) + video, audio, grid_offset = select_context(state, 22, include_audio=True) + assert video.shape[2] == 7 + assert audio.shape[-1] == 37 + assert abs(grid_offset - 1.0 / 3.0) < 1e-6 + + +def test_plan_validation(): + plan = { + "magic": PLAN_MAGIC, + "schema_version": 1, + "total_frames": 141, + "trim_frames": 22, + "net_frames": 119, + "width": 1344, + "height": 768, + "clip_index": 2, + "continuation": True, + "state_capacity_frames": 39, + } + assert validate_plan(plan) is plan + + +def test_signed_audio_grid_offset_is_preserved_for_round_down_lengths(): + latent = { + "samples": FakeNested( + ( + torch.randn(1, 24, 47, 4, 6), # 158 frames + torch.randn(1, 32, 2, 263), + ) + ) + } + state = capture_state(latent, source_frame_count=158, clip_index=2) + assert abs(state["audio_overhang"] + 1.0 / 3.0) < 1e-6 + validate_state(state) diff --git a/tests/test_state_io.py b/tests/test_state_io.py new file mode 100644 index 0000000..1630d86 --- /dev/null +++ b/tests/test_state_io.py @@ -0,0 +1,49 @@ +import torch + +from ComfyUI_H3_Continuum_Join import state_io +from ComfyUI_H3_Continuum_Join.constants import STATE_MAGIC + + +def test_state_roundtrip(tmp_path, monkeypatch): + monkeypatch.setattr(state_io, "state_directory", lambda: tmp_path) + state = { + "magic": STATE_MAGIC, + "schema_version": 1, + "clip_index": 2, + "source_frame_count": 141, + "capacity_frames": 39, + "width": 96, + "height": 64, + "video_tail": torch.randn(1, 24, 12, 4, 6), + "audio_tail": torch.randn(1, 32, 2, 65), + "audio_overhang": 0.0, + "source_mode": "latent_direct", + } + tensor_path, json_path = state_io.save_state(state, prefix="test state", slot=3) + assert tensor_path.exists() and json_path.exists() + loaded = state_io.load_state(prefix="test state", slot=3) + assert torch.equal(loaded["video_tail"], state["video_tail"]) + assert torch.equal(loaded["audio_tail"], state["audio_tail"]) + assert loaded["clip_index"] == 2 + + +def test_state_loads_when_json_mirror_is_missing(tmp_path, monkeypatch): + monkeypatch.setattr(state_io, "state_directory", lambda: tmp_path) + state = { + "magic": STATE_MAGIC, + "schema_version": 1, + "clip_index": 1, + "source_frame_count": 124, + "capacity_frames": 39, + "width": 96, + "height": 64, + "video_tail": torch.randn(1, 24, 12, 4, 6), + "audio_tail": torch.randn(1, 32, 2, 65), + "audio_overhang": 1.0 / 3.0, + "source_mode": "latent_direct", + } + _tensor_path, json_path = state_io.save_state(state, prefix="atomic", slot=1) + json_path.unlink() + loaded = state_io.load_state(prefix="atomic", slot=1) + assert torch.equal(loaded["video_tail"], state["video_tail"]) + assert loaded["clip_index"] == 1 diff --git a/tests/test_temporal.py b/tests/test_temporal.py new file mode 100644 index 0000000..54c7832 --- /dev/null +++ b/tests/test_temporal.py @@ -0,0 +1,82 @@ +import pytest + +from ComfyUI_H3_Continuum_Join.temporal import ( + align_frame_count_nearest, + audio_grid_offset, + audio_latent_t, + context_slots, + latent_slot_offsets, + largest_context_capacity, + make_av_context_window, + make_extension_shape, + pixel_frames_for_latent_t, + video_latent_t, +) + + +def test_context_presets(): + assert context_slots(5) == 2 + assert context_slots(22) == 7 + assert context_slots(39) == 12 + assert latent_slot_offsets(7) == (0, 1, 5, 9, 13, 17, 18) + + +def test_valid_h3_counts(): + assert video_latent_t(124) == 37 + assert pixel_frames_for_latent_t(37) == 124 + assert audio_latent_t(124) == 207 + + +def test_balanced_five_second_extension(): + shape = make_extension_shape(22, 5.0) + assert shape.total_frames == 141 + assert shape.net_new_frames == 119 + assert shape.video_latent_t == 42 + assert shape.audio_latent_t == 235 + + +def test_nearest_grid_choice(): + assert align_frame_count_nearest(13) == 5 + assert align_frame_count_nearest(14) == 22 + + +def test_audio_grid_offset_is_signed_and_bounded_on_valid_h3_lengths(): + offsets = [] + for k in range(0, 32): + frames = 5 + 17 * k + offsets.append(audio_grid_offset(frames, audio_latent_t(frames))) + assert all(-0.500001 <= value <= 0.500001 for value in offsets) + assert any(value < 0 for value in offsets) + assert any(value > 0 for value in offsets) + + +def test_native_context_window_uses_source_av_grid_and_cycle_phase(): + balanced = make_av_context_window(141, audio_latent_t(141), 22) + assert (balanced.video_start_slot, balanced.video_stop_slot) == (35, 42) + assert (balanced.audio_start_tick, balanced.audio_stop_tick) == (198, 235) + assert balanced.audio_steps == 37 + + shifted = make_av_context_window(158, audio_latent_t(158), 22) + assert (shifted.video_start_slot, shifted.video_stop_slot) == (40, 47) + assert (shifted.audio_start_tick, shifted.audio_stop_tick) == (226, 263) + assert shifted.audio_grid_offset < 0 + + +def test_native_context_window_rejects_unknown_audio_grid_phase(): + with pytest.raises(ValueError, match="audio-grid offset"): + make_av_context_window(141, audio_latent_t(141) + 2, 22) + + +def test_dynamic_state_capacity_for_short_extensions(): + assert largest_context_capacity(119) == 39 + assert largest_context_capacity(34) == 22 + assert largest_context_capacity(17) == 5 + + +def test_extension_shapes_remain_valid_across_ui_range(): + for context in (5, 22, 39): + for tenths in range(5, 151): + shape = make_extension_shape(context, tenths / 10.0) + assert shape.total_frames % 17 == 5 + assert shape.net_new_frames >= 5 + assert pixel_frames_for_latent_t(shape.video_latent_t) == shape.total_frames diff --git a/tests/test_v214_ui_simplification.py b/tests/test_v214_ui_simplification.py new file mode 100644 index 0000000..5cec01e --- /dev/null +++ b/tests/test_v214_ui_simplification.py @@ -0,0 +1,32 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_v214_frontend_collapses_advanced_controls_without_reordering_outputs(): + source = (ROOT / "web" / "h3_continuum_v214.js").read_text(encoding="utf-8") + + assert 'const ADVANCED_PROPERTY = "h3_show_advanced"' in source + assert '["last_frame", "IMAGE"]' in source + assert '["prompt_plan", "H3_CONTINUUM_PROMPT_PLAN"]' in source + assert '["last_state", "H3_CONTINUUM_STATE"]' in source + assert "hasAdvancedConnection(node)" in source + assert "removeUnlinkedAdvancedSlots(node)" in source + assert "Advanced Settings ▸" in source + assert "placeAdvancedButton(node, button, expanded)" in source + + +def test_all_non_primary_nodes_use_native_comfyui_deprecation(): + source = (ROOT / "nodes.py").read_text(encoding="utf-8") + + assert 'if _node_id == "H3ContinuumSamplerV2"' in source + assert "_node_class.DEPRECATED = True" in source + assert 'f"{CATEGORY}/Legacy"' in source + assert 'f"[Legacy] {_display_name}"' in source + + +def test_v214_version_metadata_matches(): + assert 'PACKAGE_VERSION = "2.1.4"' in (ROOT / "version.py").read_text(encoding="utf-8") + assert "version = 2.1.4" in (ROOT / "metadata.ini").read_text(encoding="utf-8") + assert 'version = "2.1.4"' in (ROOT / "pyproject.toml").read_text(encoding="utf-8") diff --git a/tests/test_v2_decoder.py b/tests/test_v2_decoder.py new file mode 100644 index 0000000..7148a11 --- /dev/null +++ b/tests/test_v2_decoder.py @@ -0,0 +1,59 @@ +import torch + +from ComfyUI_H3_Continuum_Join.v2.decoder import ( + _slice_audio_for_timeline, + enforce_total_frames, +) + + +def _audio(samples=200000, rate=32000): + waveform = torch.arange(samples, dtype=torch.float32).reshape(1, 1, -1) + return {"waveform": waveform, "sample_rate": rate} + + +def test_cumulative_audio_boundaries_do_not_accumulate_rounding_error(): + audio = _audio() + cursor = 124 + pieces = [] + for _ in range(2): + stop = cursor + 119 + piece = _slice_audio_for_timeline( + audio, + trim_frames=22, + frame_start=cursor, + frame_stop=stop, + ) + pieces.append(piece) + assert piece.shape[-1] == round(stop / 24 * 32000) - round(cursor / 24 * 32000) + cursor = stop + assert sum(piece.shape[-1] for piece in pieces) == round(362 / 24 * 32000) - round(124 / 24 * 32000) + + +def test_exact_duration_aligns_video_and_audio(): + images = torch.zeros(362, 4, 6, 3) + audio = {"waveform": torch.zeros(1, 2, round(362 / 24 * 32000)), "sample_rate": 32000} + images, audio, report = enforce_total_frames(images, audio, target_frames=360) + assert images.shape[0] == 360 + assert audio["waveform"].shape[-1] == 480000 + assert "362->360" in report + + +def test_exact_duration_preserves_user_final_anchor_when_trimming(): + images = torch.arange(362, dtype=torch.float32).reshape(362, 1, 1, 1) + rate = 32000 + samples = round(362 / 24 * rate) + waveform = torch.linspace(-1.0, 1.0, samples).reshape(1, 1, -1) + audio = {"waveform": waveform, "sample_rate": rate} + + result_images, result_audio, report = enforce_total_frames( + images, + audio, + target_frames=360, + preserve_final_frame=True, + ) + + assert result_images.shape[0] == 360 + assert float(result_images[-1]) == 361.0 + assert float(result_images[-2]) == 358.0 + assert result_audio["waveform"].shape[-1] == 480000 + assert "final anchor preserved" in report diff --git a/tests/test_v2_prompts.py b/tests/test_v2_prompts.py new file mode 100644 index 0000000..0254b67 --- /dev/null +++ b/tests/test_v2_prompts.py @@ -0,0 +1,169 @@ +import pytest + +from ComfyUI_H3_Continuum_Join.constants import ( + PROMPT_FORMAT_AUTO, + PROMPT_FORMAT_FIXED, + PROMPT_MODE_FIXED, + PROMPT_MODE_LIST, + PROMPT_MODE_TIMELINE, +) +from ComfyUI_H3_Continuum_Join.v2.prompts import ( + PromptPlanError, + apply_prompt_overrides, + build_sampler_prompt_plan, + detect_prompt_mode, + make_prompt_plan, + validate_prompt_plan, +) + + +def test_fixed_prompt_repeats_without_mutation(): + plan = make_prompt_plan(mode=PROMPT_MODE_FIXED, script="hello", chunks=3, chunk_seconds=5) + assert plan["prompts"] == ["hello", "hello", "hello"] + assert len(set(plan["hashes"])) == 1 + assert validate_prompt_plan(plan) is plan + + +@pytest.mark.parametrize( + ("script", "expected"), + [ + ("one continuous prompt", PROMPT_MODE_FIXED), + ("first\n---\nsecond", PROMPT_MODE_LIST), + ("[0-5s]\nfirst", PROMPT_MODE_TIMELINE), + ("[Chunk 1]\nfirst", PROMPT_MODE_TIMELINE), + ], +) +def test_auto_detects_prompt_format(script, expected): + assert detect_prompt_mode(script) == expected + + +def test_auto_plan_records_detected_format_without_changing_schema_mode(): + plan = make_prompt_plan( + mode=PROMPT_FORMAT_AUTO, + script="first\n---\nsecond", + chunks=2, + chunk_seconds=5, + ) + assert plan["mode"] == PROMPT_MODE_LIST + assert plan["prompts"] == ["first", "second"] + assert plan["notes"][0] == "Auto detected List" + + +def test_simple_fixed_format_maps_to_stable_fixed_plan_mode(): + plan = make_prompt_plan( + mode=PROMPT_FORMAT_FIXED, + script="same", + chunks=2, + chunk_seconds=5, + ) + assert plan["mode"] == PROMPT_MODE_FIXED + + +def test_list_prompt_uses_separator_and_repeats_last(): + plan = make_prompt_plan( + mode=PROMPT_MODE_LIST, + script="first\n---\nsecond", + chunks=3, + chunk_seconds=5, + ) + assert plan["prompts"] == ["first", "second", "second"] + assert "repeated" in plan["notes"][0] + + +def test_timeline_prompt_maps_ranges_and_chunk_headers(): + plan = make_prompt_plan( + mode=PROMPT_MODE_TIMELINE, + script="[0-5s]\none\n[5-10s]\ntwo\n[Chunk 3]\nthree", + chunks=3, + chunk_seconds=5, + ) + assert plan["prompts"] == ["one", "two", "three"] + + +def test_timeline_requires_coverage(): + with pytest.raises(PromptPlanError): + make_prompt_plan( + mode=PROMPT_MODE_TIMELINE, + script="[0-5s]\none", + chunks=2, + chunk_seconds=5, + ) + + +def test_native_h3_chunk_duration_rejects_sub_four_seconds(): + with pytest.raises(PromptPlanError): + make_prompt_plan(mode=PROMPT_MODE_FIXED, script="x", chunks=1, chunk_seconds=3.9) + + +def test_external_prompt_overrides_only_the_connected_clip_and_rehashes_it(): + plan = make_prompt_plan( + mode=PROMPT_MODE_LIST, + script="first\n---\nsecond\n---\nthird", + chunks=3, + chunk_seconds=5, + ) + updated = apply_prompt_overrides(plan, [None, "external second", None]) + + assert updated["prompts"] == ["first", "external second", "third"] + assert updated["hashes"][0] == plan["hashes"][0] + assert updated["hashes"][1] != plan["hashes"][1] + assert updated["hashes"][2] == plan["hashes"][2] + assert updated["notes"][-1] == "external Clip Prompt input(s): 2" + + +def test_external_prompt_with_same_text_keeps_the_same_hash(): + plan = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="same", + chunks=2, + chunk_seconds=5, + ) + updated = apply_prompt_overrides(plan, [None, "same"]) + assert updated["hashes"] == plan["hashes"] + + +def test_external_prompt_rejects_empty_connected_text(): + plan = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="fallback", + chunks=1, + chunk_seconds=5, + ) + with pytest.raises(PromptPlanError): + apply_prompt_overrides(plan, [" "]) + + +def test_sequence_prompt_precedes_connected_plan_and_legacy_script(): + connected = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="connected plan", + chunks=2, + chunk_seconds=5, + ) + resolved = build_sampler_prompt_plan( + prompt_mode=PROMPT_FORMAT_AUTO, + prompt_script="legacy", + sequence_prompt="first\n---\nsecond", + prompt_plan=connected, + chunks=2, + chunk_seconds=5, + ) + assert resolved["prompts"] == ["first", "second"] + + +def test_connected_plan_precedes_legacy_script_without_sequence_input(): + connected = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="connected plan", + chunks=2, + chunk_seconds=5, + ) + resolved = build_sampler_prompt_plan( + prompt_mode=PROMPT_FORMAT_AUTO, + prompt_script="legacy", + sequence_prompt=None, + prompt_plan=connected, + chunks=2, + chunk_seconds=5, + ) + assert resolved is connected diff --git a/tests/test_v2_registration.py b/tests/test_v2_registration.py new file mode 100644 index 0000000..049f52c --- /dev/null +++ b/tests/test_v2_registration.py @@ -0,0 +1,56 @@ +from pathlib import Path + +from ComfyUI_H3_Continuum_Join.nodes import NODE_CLASS_MAPPINGS +from ComfyUI_H3_Continuum_Join.constants import ( + PROMPT_FORMAT_OPTIONS, + SEAM_CORRECTION_OFF, +) + + +def test_v2_nodes_are_registered_without_removing_v1(): + expected = { + "H3ContinuumJoin", + "H3ContinuumFinish", + "H3ContinuumAssemble", + "H3ContinuumSaveState", + "H3ContinuumLoadState", + "H3ContinuumSamplerV2", + "H3ContinuumPromptPlanPreview", + "H3ContinuumSaveSession", + "H3ContinuumLoadSession", + "H3ContinuumSessionInfo", + } + assert expected.issubset(NODE_CLASS_MAPPINGS) + + +def test_v2_sampler_accepts_connectable_prompt_plan(): + schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() + assert "prompt_plan" in schema["optional"] + + +def test_v2_sampler_registers_preview_setting_bridge_and_sixteen_prompt_inputs(): + schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() + optional = schema["optional"] + + assert schema["required"]["prompt_mode"][0] == PROMPT_FORMAT_OPTIONS + assert schema["required"]["prompt_mode"][1]["default"] == "Auto" + assert optional["sequence_prompt"][0] == "STRING" + assert optional["sequence_prompt"][1]["forceInput"] is True + assert optional["show_preview"][1]["default"] is True + for index in range(1, 17): + prompt_input = optional[f"clip_{index}_prompt"] + assert prompt_input[0] == "STRING" + assert prompt_input[1]["forceInput"] is True + + +def test_v2_sampler_defaults_to_the_legacy_seam_path(): + schema = NODE_CLASS_MAPPINGS["H3ContinuumSamplerV2"].INPUT_TYPES() + assert schema["required"]["seam_correction"][1]["default"] == SEAM_CORRECTION_OFF + + +def test_v2_interface_keeps_sequence_socket_and_collapsible_overrides(): + root = Path(__file__).resolve().parents[1] + source = (root / "web" / "h3_continuum_v2.js").read_text(encoding="utf-8") + assert 'const SEQUENCE_PROMPT_INPUT = "sequence_prompt"' in source + assert "Individual Clip Overrides" in source + assert "!inputIsConnected(input)" in source diff --git a/tests/test_v2_seam.py b/tests/test_v2_seam.py new file mode 100644 index 0000000..0d66b67 --- /dev/null +++ b/tests/test_v2_seam.py @@ -0,0 +1,86 @@ +import torch + +from ComfyUI_H3_Continuum_Join.v2.seam_guard import ( + apply_color_guard, + choose_adaptive_cut, + choose_video_blend, + correct_audio_seam, + correct_seam, +) +from ComfyUI_H3_Continuum_Join.v2.seam_types import VideoSeamMetrics + + +def _frames(values): + return torch.tensor(values, dtype=torch.float32).reshape(-1, 1, 1, 1).expand(-1, 12, 12, 3).clone() + + +def test_adaptive_cut_detects_a_two_frame_earlier_clean_boundary(): + previous = _frames([0.0, 0.0, 0.0, 0.05, 0.10, 0.20, 0.30, 0.40]) + current = _frames([0.05, 0.10, 0.95, 0.205, 0.21, 0.90, 0.90, 0.90]) + + rewind, legacy, selected = choose_adaptive_cut( + previous, + current, + context_frames=5, + ) + + assert rewind == 2 + assert selected.score < legacy.score + + +def test_corrected_cut_preserves_the_total_frame_count(): + previous = _frames([0.0, 0.0, 0.0, 0.05, 0.10, 0.20, 0.30, 0.40]) + current = _frames([0.05, 0.10, 0.95, 0.205, 0.21, 0.90, 0.90, 0.90]) + corrected, decision, _patch = correct_seam( + previous, + current, + context_frames=5, + ) + legacy_total = previous.shape[0] + current.shape[0] - 5 + corrected_total = ( + previous.shape[0] - decision.cut_rewind_frames + corrected.shape[0] + ) + assert corrected_total == legacy_total + + +def test_high_motion_disables_video_blend(): + metrics = VideoSeamMetrics( + pixel_mae_norm=0.1, + luma_norm=0.1, + chroma_norm=0.1, + edge_norm=0.1, + temporal_norm=0.1, + motion=0.2, + score=0.1, + ) + assert choose_video_blend(metrics) == 0 + + +def test_excessive_color_request_is_not_applied(): + previous = _frames([0.01, 0.01]) + current = _frames([1.0, 1.0, 1.0, 1.0]) + corrected, gain, bias, _metrics = apply_color_guard(previous, current) + assert gain == 1.0 + assert bias == (0.0, 0.0, 0.0) + assert torch.equal(corrected, current) + + +def test_audio_seam_uses_one_offset_for_all_stereo_channels_and_keeps_a_local_patch(): + sample_rate = 1000 + time = torch.arange(800, dtype=torch.float32) / sample_rate + signal = torch.sin(2.0 * torch.pi * 20.0 * time) + previous = torch.stack((signal[:500], signal[:500]), dim=0).unsqueeze(0) + current = torch.stack((signal, signal), dim=0).unsqueeze(0) + + patch, metrics, fade_samples, _gain, _dc = correct_audio_seam( + previous, + current, + sample_rate=sample_rate, + cut_sample=500, + ) + + assert isinstance(metrics.offset_samples, int) + assert fade_samples <= round(sample_rate * 0.060) + if patch is not None: + assert patch.shape[1] == 2 + assert patch.shape[-1] == fade_samples diff --git a/tests/test_v2_seeds_motion.py b/tests/test_v2_seeds_motion.py new file mode 100644 index 0000000..e9efc88 --- /dev/null +++ b/tests/test_v2_seeds_motion.py @@ -0,0 +1,46 @@ +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + STATE_MAGIC, + V2_CONTINUITY_AUTO, +) +from ComfyUI_H3_Continuum_Join.v2.motion import choose_context_frames, latent_motion_score +from ComfyUI_H3_Continuum_Join.v2.seeds import derive_chunk_seed + + +def _state(video): + return { + "magic": STATE_MAGIC, + "schema_version": 1, + "clip_index": 1, + "source_frame_count": 124, + "capacity_frames": 39, + "width": 96, + "height": 64, + "video_tail": video, + "audio_tail": torch.zeros(1, 32, 2, 65), + "audio_overhang": 1.0 / 3.0, + "source_mode": "latent_direct", + } + + +def test_chunk_seeds_are_stable_and_independent(): + values = [derive_chunk_seed(123, i, 0) for i in range(4)] + assert values == [derive_chunk_seed(123, i, 0) for i in range(4)] + assert len(set(values)) == 4 + rerolled = [derive_chunk_seed(123, i, 1) for i in range(4)] + assert values != rerolled + + +def test_auto_context_selects_static_and_dynamic_profiles(): + static_state = _state(torch.ones(1, 24, 12, 4, 6)) + assert latent_motion_score(static_state) == 0.0 + frames, _score, _reason = choose_context_frames(V2_CONTINUITY_AUTO, static_state) + assert frames == 5 + + dynamic = torch.zeros(1, 24, 12, 4, 6) + dynamic[:, :, 1::2] = 10 + dynamic_state = _state(dynamic) + frames, score, _reason = choose_context_frames(V2_CONTINUITY_AUTO, dynamic_state) + assert score > 0.12 + assert frames == 39 diff --git a/tests/test_v2_sequence_orchestration.py b/tests/test_v2_sequence_orchestration.py new file mode 100644 index 0000000..4e37467 --- /dev/null +++ b/tests/test_v2_sequence_orchestration.py @@ -0,0 +1,159 @@ +from types import SimpleNamespace + +import torch + +from ComfyUI_H3_Continuum_Join.constants import ( + CONTINUUM_INTEROP_KEY, + DIAGNOSTICS_BASIC, + PROMPT_MODE_FIXED, + V2_CONTINUITY_OPTIONS, +) +from ComfyUI_H3_Continuum_Join.temporal import audio_latent_t, video_latent_t +from ComfyUI_H3_Continuum_Join.v2.h3_builder import IdentityAssets +from ComfyUI_H3_Continuum_Join.v2.prompts import make_prompt_plan +from ComfyUI_H3_Continuum_Join.v2 import sequence + + +class Nested: + def __init__(self, parts): + self.parts = list(parts) + def unbind(self): + return self.parts + + +def _latent(width, height, frames): + return { + "samples": Nested( + ( + torch.zeros(1, 24, video_latent_t(frames), height // 16, width // 16), + torch.zeros(1, 32, 2, audio_latent_t(frames)), + ) + ) + } + + +class FakeClip: + def __init__(self, events): + self.events = events + def tokenize(self, prompt, images): + self.events.append(("tokenize", prompt, len(images))) + return prompt + def encode_from_tokens_scheduled(self, tokens): + self.events.append(("encode", tokens)) + return [[torch.zeros(1, 1, 2), {}]] + + +def test_v2_samples_every_chunk_before_any_decode(monkeypatch): + events = [] + width, height = 96, 64 + fake_model = SimpleNamespace( + model=SimpleNamespace(diffusion_model=SimpleNamespace()), + model_options={}, + wrappers={}, + model_dtype=lambda: torch.bfloat16, + model_size=lambda: 0, + ) + assets = IdentityAssets(None, None, None, None, "none") + + monkeypatch.setattr(sequence, "check_comfy_h3_runtime", lambda: []) + monkeypatch.setattr(sequence, "patch_model", lambda model, strict, debug: model) + def clone_model_for_chunk( + model, + *, + strict, + debug, + chunk_index, + context_frames, + ): + options = dict(model.model_options) + transformer_options = dict(options.get("transformer_options") or {}) + if context_frames is not None: + transformer_options[CONTINUUM_INTEROP_KEY] = { + "api": 1, + "active": True, + "min_actual_prefix_steps": 2, + } + options["transformer_options"] = transformer_options + events.append(("clone", chunk_index, context_frames)) + return SimpleNamespace( + model=model.model, + model_options=options, + wrappers=model.wrappers, + model_dtype=model.model_dtype, + model_size=model.model_size, + ) + + monkeypatch.setattr(sequence, "clone_model_for_chunk", clone_model_for_chunk) + monkeypatch.setattr(sequence, "prepare_identity_assets", lambda *a, **k: assets) + monkeypatch.setattr(sequence, "encode_identity_latents", lambda *a, **k: assets) + monkeypatch.setattr(sequence, "empty_h3_latent", _latent) + monkeypatch.setattr(sequence, "accelerator_summary", lambda model: "accelerators") + + def sample_chunk(**kwargs): + frames = kwargs["latent"]["samples"].unbind()[0].shape[2] + hint = kwargs["model"].model_options["transformer_options"].get( + CONTINUUM_INTEROP_KEY + ) + events.append(("sample", int(frames), kwargs["seed"], hint)) + return kwargs["latent"] + + def decode_sequence(**kwargs): + events.append(("decode", len(kwargs["entries"]))) + images = torch.zeros(362, 2, 2, 3) + audio = {"waveform": torch.zeros(1, 2, round(362 / 24 * 32000)), "sample_rate": 32000} + return images, audio, [] + + monkeypatch.setattr(sequence, "sample_chunk", sample_chunk) + monkeypatch.setattr(sequence, "decode_sequence", decode_sequence) + + def unexpected_seam_decode(**kwargs): + raise AssertionError("Off must not enter the V2.1 seam decoder") + + monkeypatch.setattr(sequence, "decode_sequence_with_seam", unexpected_seam_decode) + + plan = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="continuous shot", + chunks=3, + chunk_seconds=5.0, + ) + images, audio, last_state, session, report = sequence.run_sequence( + model=fake_model, + clip=FakeClip(events), + video_vae=object(), + audio_vae=object(), + sampler=object(), + sigmas=torch.tensor([1.0, 0.0]), + first_frame=None, + last_frame=None, + prompt_plan=plan, + width=width, + height=height, + continuity=V2_CONTINUITY_OPTIONS[1], + base_seed=42, + audio_continuity=True, + exact_total_duration=True, + diagnostics_mode=DIAGNOSTICS_BASIC, + reroll_from_chunk=0, + reroll_nonce=0, + strict_compatibility=True, + debug=False, + ) + + sample_positions = [i for i, event in enumerate(events) if event[0] == "sample"] + decode_position = next(i for i, event in enumerate(events) if event[0] == "decode") + assert len(sample_positions) == 3 + assert max(sample_positions) < decode_position + sample_events = [event for event in events if event[0] == "sample"] + assert sample_events[0][3] is None + assert sample_events[1][3]["min_actual_prefix_steps"] == 2 + assert sample_events[2][3]["min_actual_prefix_steps"] == 2 + clone_events = [event for event in events if event[0] == "clone"] + assert [event[2] for event in clone_events] == [None, 22, 22] + # Fixed mode encodes its shared text conditioning only once. + assert sum(event[0] == "encode" for event in events) == 1 + assert images.shape[0] == 360 + assert audio["waveform"].shape[-1] == 480000 + assert last_state["clip_index"] == 3 + assert len(session["chunks"]) == 3 + assert "call-local MODEL clone per chunk" in report diff --git a/tests/test_v2_session.py b/tests/test_v2_session.py new file mode 100644 index 0000000..75f4ff8 --- /dev/null +++ b/tests/test_v2_session.py @@ -0,0 +1,79 @@ +import torch + +from ComfyUI_H3_Continuum_Join.state import make_plan +from ComfyUI_H3_Continuum_Join.v2 import session_io +from ComfyUI_H3_Continuum_Join.v2.session import ( + entry_to_state, + make_chunk_entry, + make_session, + validate_session, +) + + +def _latent(frame_count=124): + import sys + from types import ModuleType + + # The session entry accepts a ComfyUI-like object exposing unbind(). + class Nested: + def __init__(self, parts): + self.parts = parts + def unbind(self): + return self.parts + + video_t = 37 if frame_count == 124 else 42 + audio_t = 207 if frame_count == 124 else 235 + return { + "samples": Nested( + ( + torch.randn(1, 24, video_t, 4, 6), + torch.randn(1, 32, 2, audio_t), + ) + ) + } + + +def test_session_roundtrip_and_last_state(tmp_path, monkeypatch): + plan = make_plan( + continuation=False, + clip_index=1, + total_frames=124, + trim_frames=0, + width=96, + height=64, + context_frames=22, + state_capacity_frames=39, + requested_extend_seconds=5, + debug=False, + ) + entry = make_chunk_entry( + latent=_latent(), + plan=plan, + prompt="p", + prompt_hash="0" * 64, + seed=1, + context_frames=0, + motion_score=0.0, + reused=False, + ) + session = make_session( + chunks=[entry], + width=96, + height=64, + chunk_seconds=5, + identity_hash="none", + model_fingerprint_value="f" * 64, + parent_session_id=None, + reroll_from_chunk=0, + settings={}, + ) + validate_session(session) + state = entry_to_state(session["chunks"][0]) + assert state["capacity_frames"] == 39 + + monkeypatch.setattr(session_io, "session_directory", lambda: tmp_path) + tensor_path, json_path = session_io.save_session(session, prefix="test", slot=1) + assert tensor_path.exists() and json_path.exists() + loaded = session_io.load_session(prefix="test", slot=1) + assert torch.equal(loaded["chunks"][0]["video"], session["chunks"][0]["video"]) + assert loaded["session_id"] == session["session_id"] diff --git a/tests/test_v2_widget_migration.py b/tests/test_v2_widget_migration.py new file mode 100644 index 0000000..0c3389c --- /dev/null +++ b/tests/test_v2_widget_migration.py @@ -0,0 +1,41 @@ +import json +from pathlib import Path + +from ComfyUI_H3_Continuum_Join.constants import DIAGNOSTICS_BASIC +from ComfyUI_H3_Continuum_Join.v2.nodes import _repair_v200_example_widget_shift + + +def test_malformed_v200_example_widget_shift_is_repaired(): + values = _repair_v200_example_widget_shift( + audio_continuity=True, + exact_total_duration=DIAGNOSTICS_BASIC, + diagnostics=0, + reroll_from_chunk=0, + reroll_nonce=True, + strict_compatibility=False, + debug=False, + ) + assert values == (True, True, DIAGNOSTICS_BASIC, 0, 0, True, False) + + +def test_valid_v2_widget_values_are_unchanged(): + values = _repair_v200_example_widget_shift( + audio_continuity=False, + exact_total_duration=False, + diagnostics=DIAGNOSTICS_BASIC, + reroll_from_chunk=2, + reroll_nonce=7, + strict_compatibility=True, + debug=True, + ) + assert values == (False, False, DIAGNOSTICS_BASIC, 2, 7, True, True) + + +def test_bundled_v2_example_serializes_control_after_generate_slot(): + root = Path(__file__).resolve().parents[1] + data = json.loads((root / "examples" / "H3_Continuum_V2_3x5s.json").read_text(encoding="utf-8")) + node = next(n for n in data["nodes"] if n["type"] == "H3ContinuumSamplerV2") + values = node["widgets_values"] + assert len(values) == 16 + assert values[8] in {"fixed", "increment", "decrement", "randomize"} + assert values[11] in {"Basic", "Full", "Off"} diff --git a/tools/verify_runtime.py b/tools/verify_runtime.py new file mode 100644 index 0000000..ce302bb --- /dev/null +++ b/tools/verify_runtime.py @@ -0,0 +1,95 @@ +"""Lightweight installation verifier for a local ComfyUI environment.""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +from pathlib import Path + + +def load_package(root: Path): + name = "h3_continuum_join_verify" + spec = importlib.util.spec_from_file_location( + name, + root / "__init__.py", + submodule_search_locations=[str(root)], + ) + if spec is None or spec.loader is None: + raise RuntimeError("cannot create package import specification") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--comfy-root", type=Path, required=True) + args = parser.parse_args() + comfy_root = args.comfy_root.resolve() + package_root = Path(__file__).resolve().parents[1] + sys.path.insert(0, str(comfy_root)) + + module = load_package(package_root) + from h3_continuum_join_verify.compatibility import ( + check_comfy_h3_runtime, + run_native_layout_self_test, + ) + from h3_continuum_join_verify.constants import PROMPT_MODE_FIXED + from h3_continuum_join_verify.v2.prompts import make_prompt_plan + from h3_continuum_join_verify.version import PACKAGE_VERSION + + issues = check_comfy_h3_runtime() + if issues: + print("H3 Continuum Join runtime verification FAILED:") + for issue in issues: + print(f" - {issue}") + return 1 + try: + layout_report = run_native_layout_self_test() + except Exception as exc: + print(f"Native PackedLayout self-test FAILED: {exc}") + return 1 + + try: + prompt_plan = make_prompt_plan( + mode=PROMPT_MODE_FIXED, + script="Continuum runtime verification prompt", + chunks=3, + chunk_seconds=5.0, + ) + if len(prompt_plan["prompts"]) != 3 or len(set(prompt_plan["hashes"])) != 1: + raise RuntimeError("unexpected Fixed prompt-plan result") + except Exception as exc: + print(f"V2 prompt/session core self-test FAILED: {exc}") + return 1 + + expected = { + "H3ContinuumJoin", + "H3ContinuumFinish", + "H3ContinuumAssemble", + "H3ContinuumSaveState", + "H3ContinuumLoadState", + "H3ContinuumSamplerV2", + "H3ContinuumPromptPlanPreview", + "H3ContinuumSaveSession", + "H3ContinuumLoadSession", + "H3ContinuumSessionInfo", + } + actual = set(module.NODE_CLASS_MAPPINGS) + if actual != expected: + print(f"Node registration mismatch: expected {sorted(expected)}, got {sorted(actual)}") + return 1 + print(f"H3 Continuum Join {PACKAGE_VERSION} runtime verification passed.") + print(layout_report) + print("V2 Fixed 3x5s prompt-plan self-test passed.") + print("Registered nodes:") + for key in sorted(actual): + print(f" - {module.NODE_DISPLAY_NAME_MAPPINGS[key]}") + print("GPU checkpoint generation was not run by this lightweight verifier.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/v2/__init__.py b/v2/__init__.py new file mode 100644 index 0000000..f743db0 --- /dev/null +++ b/v2/__init__.py @@ -0,0 +1,5 @@ +"""Integrated V2 runtime and nodes.""" + +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/v2/decoder.py b/v2/decoder.py new file mode 100644 index 0000000..fe8c99d --- /dev/null +++ b/v2/decoder.py @@ -0,0 +1,462 @@ +"""Deferred VAE decoding, overlap trimming, assembly, and exact-duration correction.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as F + +from ..constants import DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, FPS +from ..media import validate_audio +from ..state import validate_plan +from .diagnostics import format_join_metrics, format_seam_decision, overlap_diagnostics +from .seam_guard import correct_audio_seam, correct_seam +from .seam_types import SeamDecision +from .session import entry_to_latent, validate_chunk_entry + + +def decode_video(video_vae: Any, latent: dict[str, Any]) -> torch.Tensor: + samples = latent["samples"] + if hasattr(samples, "unbind"): + video = list(samples.unbind())[0] + else: + video = samples[0] + images = video_vae.decode(video) + if images.ndim == 5: + images = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1]) + if images.ndim != 4: + raise ValueError(f"video VAE returned unexpected shape {tuple(images.shape)}") + # Long sequences must not leave decoded frames resident on the GPU. ComfyUI + # IMAGE tensors are conventionally CPU float tensors, so transfer each chunk + # immediately and release its raw decode before moving to the next chunk. + return images.detach().to("cpu").contiguous() + + +def decode_audio(audio_vae: Any, latent: dict[str, Any]) -> dict[str, Any]: + samples = latent["samples"] + if hasattr(samples, "unbind"): + audio_latent = list(samples.unbind())[-1] + else: + audio_latent = samples[-1] + audio = audio_vae.decode(audio_latent).movedim(-1, 1) + std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 + std[std < 1.0] = 1.0 + audio = (audio / std).detach().to("cpu").contiguous() + sample_rate = getattr( + audio_vae, + "audio_sample_rate_output", + getattr(audio_vae, "audio_sample_rate", 32000), + ) + return {"waveform": audio, "sample_rate": int(sample_rate)} + + +def _trim_images(images: torch.Tensor, plan: dict[str, Any]) -> torch.Tensor: + plan = validate_plan(plan) + total_frames = int(plan["total_frames"]) + trim_frames = int(plan["trim_frames"]) + if images.shape[0] < total_frames: + raise ValueError( + f"decoded batch has {images.shape[0]} frames; expected at least {total_frames}" + ) + return images[:total_frames][trim_frames:].contiguous() + + +def _slice_audio_for_timeline( + audio: dict[str, Any], + *, + trim_frames: int, + frame_start: int, + frame_stop: int, +) -> torch.Tensor: + """Return one segment at its exact cumulative 24fps sample interval. + + Segment-local rounding drifts by one sample on common 119-frame Continuum + chunks. Deriving every boundary from cumulative frame counts guarantees that + the assembled audio length is always ``round(total_frames / 24 * rate)``. + """ + + waveform, sample_rate = validate_audio(audio) + trim_samples = int(round(float(trim_frames) / FPS * sample_rate)) + target_start = int(round(float(frame_start) / FPS * sample_rate)) + target_stop = int(round(float(frame_stop) / FPS * sample_rate)) + wanted = target_stop - target_start + if wanted < 0: + raise ValueError("audio timeline interval is negative") + result = waveform[..., trim_samples : trim_samples + wanted] + if result.shape[-1] < wanted: + missing = wanted - int(result.shape[-1]) + if result.shape[-1] > 0: + tail = result[..., -1:].expand(*result.shape[:-1], missing).clone() + result = torch.cat((result, tail), dim=-1) + else: + result = F.pad(result, (0, missing)) + return result.contiguous() + + +def decode_sequence( + *, + entries: list[dict[str, Any]], + video_vae: Any, + audio_vae: Any, + diagnostics_full: bool, +) -> tuple[torch.Tensor, dict[str, Any], list[str]]: + """Decode and assemble all chunks with bounded transient memory. + + The final IMAGE tensor is preallocated once. Each raw VAE decode is copied + into its destination range and released before the next chunk, avoiding the + repeated full-sequence copies produced by progressive ``torch.cat``. + """ + + if not entries: + raise ValueError("cannot decode an empty Continuum sequence") + checked = [validate_chunk_entry(entry) for entry in entries] + plans = [validate_plan(entry["plan"]) for entry in checked] + total_retained_frames = sum(int(plan["net_frames"]) for plan in plans) + if total_retained_frames < 1: + raise ValueError("Continuum sequence contains no retained frames") + + image_buffer: torch.Tensor | None = None + audio_buffer: torch.Tensor | None = None + audio_rate: int | None = None + frame_cursor = 0 + previous_segment_images: torch.Tensor | None = None + previous_segment_audio: dict[str, Any] | None = None + reports: list[str] = [] + + for index, (entry, plan) in enumerate(zip(checked, plans), start=1): + latent = entry_to_latent(entry) + raw_images = decode_video(video_vae, latent) + raw_audio = decode_audio(audio_vae, latent) + + if diagnostics_full and previous_segment_images is not None: + metrics = overlap_diagnostics( + previous_images=previous_segment_images, + previous_audio=previous_segment_audio, + next_raw_images=raw_images, + next_raw_audio=raw_audio, + context_frames=int(plan["trim_frames"]), + ) + reports.append(format_join_metrics(index, metrics)) + + images = _trim_images(raw_images, plan) + segment_frames = int(images.shape[0]) + if segment_frames != int(plan["net_frames"]): + raise ValueError( + f"chunk {index} retained {segment_frames} frames; plan expects {plan['net_frames']}" + ) + frame_stop = frame_cursor + segment_frames + + waveform, sample_rate = validate_audio(raw_audio) + if audio_rate is None: + audio_rate = sample_rate + elif sample_rate != audio_rate: + raise ValueError( + f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}" + ) + segment_waveform = _slice_audio_for_timeline( + raw_audio, + trim_frames=int(plan["trim_frames"]), + frame_start=frame_cursor, + frame_stop=frame_stop, + ) + + if image_buffer is None: + image_buffer = torch.empty( + (total_retained_frames, *images.shape[1:]), + dtype=images.dtype, + device="cpu", + ) + elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]): + raise ValueError( + f"decoded image geometry changed at chunk {index}: " + f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}" + ) + image_buffer[frame_cursor:frame_stop].copy_(images) + + if audio_buffer is None: + total_samples = int(round(total_retained_frames / FPS * sample_rate)) + audio_buffer = torch.empty( + (*waveform.shape[:-1], total_samples), + dtype=waveform.dtype, + device="cpu", + ) + elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]): + raise ValueError("decoded audio batch/channel structure changed between chunks") + sample_start = int(round(frame_cursor / FPS * sample_rate)) + sample_stop = int(round(frame_stop / FPS * sample_rate)) + audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform) + + # Retain only what Full diagnostics needs for the next seam. Audio is + # small, while video is bounded to the next overlap instead of a full clip. + next_context = int(plans[index]["trim_frames"]) if index < len(plans) else 0 + if next_context > 0: + previous_segment_images = images[-next_context:].contiguous() + previous_segment_audio = { + "waveform": segment_waveform, + "sample_rate": sample_rate, + } + else: + previous_segment_images = None + previous_segment_audio = None + + reports.append( + f"decoded chunk {index}: {segment_frames} retained frames, " + f"cumulative {frame_stop}/{total_retained_frames}" + ) + frame_cursor = frame_stop + del latent, raw_images, raw_audio, images, segment_waveform + + if image_buffer is None or audio_buffer is None or audio_rate is None: + raise RuntimeError("Continuum decoder failed to allocate output buffers") + if frame_cursor != total_retained_frames: + raise RuntimeError( + f"Continuum decode cursor mismatch: {frame_cursor} != {total_retained_frames}" + ) + audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate} + reports.append( + f"assembled {len(entries)} chunks into {total_retained_frames} frames with " + "cumulative sample-boundary alignment" + ) + return image_buffer.contiguous(), audio, reports + + +def decode_sequence_with_seam( + *, + entries: list[dict[str, Any]], + video_vae: Any, + audio_vae: Any, + diagnostics_mode: str, +) -> tuple[torch.Tensor, dict[str, Any], list[str]]: + """Decode with V2.1 Basic correction while preserving legacy output length.""" + + if not entries: + raise ValueError("cannot decode an empty Continuum sequence") + checked = [validate_chunk_entry(entry) for entry in entries] + plans = [validate_plan(entry["plan"]) for entry in checked] + total_retained_frames = sum(int(plan["net_frames"]) for plan in plans) + if total_retained_frames < 1: + raise ValueError("Continuum sequence contains no retained frames") + + image_buffer: torch.Tensor | None = None + audio_buffer: torch.Tensor | None = None + audio_rate: int | None = None + frame_cursor = 0 + reports: list[str] = [] + + for index, (entry, plan) in enumerate(zip(checked, plans), start=1): + latent = entry_to_latent(entry) + raw_images = decode_video(video_vae, latent) + raw_audio = decode_audio(audio_vae, latent) + waveform, sample_rate = validate_audio(raw_audio) + if audio_rate is None: + audio_rate = sample_rate + elif sample_rate != audio_rate: + raise ValueError( + f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}" + ) + + legacy_frames = int(plan["net_frames"]) + legacy_frame_stop = frame_cursor + legacy_frames + frame_start = frame_cursor + trim_frames = int(plan["trim_frames"]) + decision: SeamDecision | None = None + video_patch: torch.Tensor | None = None + audio_patch: torch.Tensor | None = None + + if index == 1: + images = _trim_images(raw_images, plan) + else: + try: + if image_buffer is None or audio_buffer is None: + raise RuntimeError("seam buffers are unavailable") + limited_raw_images = raw_images[: int(plan["total_frames"])] + images, decision, video_patch = correct_seam( + image_buffer[:frame_cursor], + limited_raw_images, + context_frames=trim_frames, + ) + frame_start = frame_cursor - int(decision.cut_rewind_frames) + trim_frames -= int(decision.cut_rewind_frames) + if frame_start < 0 or trim_frames < 0: + raise ValueError("adaptive cut produced a negative boundary") + if frame_start + int(images.shape[0]) != legacy_frame_stop: + raise ValueError("adaptive cut changed the cumulative frame boundary") + + sample_start = int(round(frame_start / FPS * sample_rate)) + cut_sample = int(round(trim_frames / FPS * sample_rate)) + ( + audio_patch, + audio_metrics, + crossfade_samples, + level_gain, + dc_bias, + ) = correct_audio_seam( + audio_buffer[..., :sample_start], + waveform, + sample_rate=sample_rate, + cut_sample=cut_sample, + ) + decision.audio_metrics = audio_metrics + decision.audio_crossfade_samples = crossfade_samples + decision.audio_level_gain = level_gain + decision.audio_dc_bias = dc_bias + except Exception as exc: + images = _trim_images(raw_images, plan) + frame_start = frame_cursor + trim_frames = int(plan["trim_frames"]) + video_patch = None + audio_patch = None + decision = SeamDecision( + fallback_reason=f"{type(exc).__name__}: {exc}", + ) + + segment_frames = int(images.shape[0]) + frame_stop = frame_start + segment_frames + if frame_stop != legacy_frame_stop: + raise ValueError( + f"chunk {index} corrected boundary ends at {frame_stop}; " + f"expected {legacy_frame_stop}" + ) + segment_waveform = _slice_audio_for_timeline( + raw_audio, + trim_frames=trim_frames, + frame_start=frame_start, + frame_stop=frame_stop, + ) + + if image_buffer is None: + image_buffer = torch.empty( + (total_retained_frames, *images.shape[1:]), + dtype=images.dtype, + device="cpu", + ) + elif tuple(images.shape[1:]) != tuple(image_buffer.shape[1:]): + raise ValueError( + f"decoded image geometry changed at chunk {index}: " + f"{tuple(images.shape[1:])} vs {tuple(image_buffer.shape[1:])}" + ) + if video_patch is not None and video_patch.shape[0] > 0: + patch_start = frame_start - int(video_patch.shape[0]) + if patch_start < 0: + raise ValueError("video seam patch starts before the output") + image_buffer[patch_start:frame_start].copy_(video_patch) + image_buffer[frame_start:frame_stop].copy_(images) + + if audio_buffer is None: + total_samples = int(round(total_retained_frames / FPS * sample_rate)) + audio_buffer = torch.empty( + (*waveform.shape[:-1], total_samples), + dtype=waveform.dtype, + device="cpu", + ) + elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]): + raise ValueError("decoded audio batch/channel structure changed between chunks") + sample_start = int(round(frame_start / FPS * sample_rate)) + sample_stop = int(round(frame_stop / FPS * sample_rate)) + if audio_patch is not None and audio_patch.shape[-1] > 0: + patch_start = sample_start - int(audio_patch.shape[-1]) + if patch_start < 0: + raise ValueError("audio seam patch starts before the output") + audio_buffer[..., patch_start:sample_start].copy_(audio_patch) + audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform) + + if decision is not None and diagnostics_mode != DIAGNOSTICS_OFF: + reports.append( + format_seam_decision( + index, + decision, + full=diagnostics_mode == DIAGNOSTICS_FULL, + ) + ) + reports.append( + f"decoded chunk {index}: {segment_frames} retained/corrected frames, " + f"cumulative {frame_stop}/{total_retained_frames}" + ) + frame_cursor = frame_stop + del latent, raw_images, raw_audio, images, segment_waveform + + if image_buffer is None or audio_buffer is None or audio_rate is None: + raise RuntimeError("Continuum seam decoder failed to allocate output buffers") + if frame_cursor != total_retained_frames: + raise RuntimeError( + f"Continuum seam decode cursor mismatch: {frame_cursor} != {total_retained_frames}" + ) + audio = {"waveform": audio_buffer.contiguous(), "sample_rate": audio_rate} + reports.append( + f"assembled {len(entries)} chunks into {total_retained_frames} frames with " + "V2.1 Basic seam correction and cumulative sample-boundary alignment" + ) + return image_buffer.contiguous(), audio, reports + + +def enforce_total_frames( + images: torch.Tensor, + audio: dict[str, Any], + *, + target_frames: int, + preserve_final_frame: bool = False, +) -> tuple[torch.Tensor, dict[str, Any], str]: + target_frames = int(target_frames) + if target_frames < 1: + raise ValueError("target_frames must be positive") + current = int(images.shape[0]) + adjustment = target_frames - current + anchor_preserved = bool(preserve_final_frame and adjustment < 0 and target_frames >= 2) + if adjustment < 0: + if anchor_preserved: + # H3's valid temporal grid rarely lands on an exact whole-second total. + # When a user supplied a final FL2VA frame, trimming only from the end + # would discard the very anchor they requested. Remove the small excess + # immediately before the final decoded frame instead. + images = torch.cat((images[: target_frames - 1], images[-1:]), dim=0).contiguous() + else: + images = images[:target_frames].contiguous() + elif adjustment > 0: + if current == 0: + raise ValueError("cannot pad an empty image sequence") + pad = images[-1:].expand(adjustment, *images.shape[1:]).clone() + images = torch.cat((images, pad), dim=0).contiguous() + + waveform = audio["waveform"] + sample_rate = int(audio["sample_rate"]) + original_samples = int(waveform.shape[-1]) + target_samples = int(round(target_frames / FPS * sample_rate)) + delta_samples = target_samples - original_samples + if delta_samples < 0: + if anchor_preserved and target_samples > 1: + # Preserve the final video-frame interval of audio as well. A short + # 5 ms interpolation at the deletion boundary suppresses a hard click + # without changing the exact output length. + tail_samples = min( + max(1, int(round(sample_rate / FPS))), + target_samples, + original_samples, + ) + head_samples = target_samples - tail_samples + head = waveform[..., :head_samples] + tail = waveform[..., -tail_samples:].clone() + fade = min(int(round(sample_rate * 0.005)), head_samples, tail_samples) + if fade > 0: + start = head[..., -1:].to(dtype=tail.dtype) + alpha = torch.linspace( + 0.0, 1.0, fade, dtype=tail.dtype, device=tail.device + ).reshape(*([1] * (tail.ndim - 1)), fade) + tail[..., :fade] = start * (1.0 - alpha) + tail[..., :fade] * alpha + waveform = torch.cat((head, tail), dim=-1).contiguous() + else: + waveform = waveform[..., :target_samples].contiguous() + elif delta_samples > 0: + # Repeat the final sample instead of introducing a zero click. + if waveform.shape[-1] == 0: + waveform = F.pad(waveform, (0, delta_samples)) + else: + tail = waveform[..., -1:].expand(*waveform.shape[:-1], delta_samples).clone() + waveform = torch.cat((waveform, tail), dim=-1).contiguous() + audio = {**audio, "waveform": waveform, "sample_rate": sample_rate} + mode = "final anchor preserved; pre-tail trim" if anchor_preserved else "tail trim/pad" + report = ( + f"Exact-duration adjustment: frames {current}->{target_frames} ({adjustment:+d}), " + f"audio samples correction {delta_samples:+d}; {mode}." + ) + return images, audio, report diff --git a/v2/diagnostics.py b/v2/diagnostics.py new file mode 100644 index 0000000..2aee5bf --- /dev/null +++ b/v2/diagnostics.py @@ -0,0 +1,131 @@ +"""Lightweight join diagnostics without external vision/audio models.""" + +from __future__ import annotations + +import math +from typing import Any + +import torch + +from ..constants import FPS +from .seam_types import SeamDecision + + +def _audio_waveform(audio: dict[str, Any]) -> tuple[torch.Tensor, int]: + waveform = audio.get("waveform") + sample_rate = int(audio.get("sample_rate", 0)) + if not torch.is_tensor(waveform) or waveform.ndim != 3: + raise ValueError("audio waveform must be [B,C,S]") + if sample_rate <= 0: + raise ValueError("audio sample_rate is invalid") + return waveform.to(dtype=torch.float32), sample_rate + + +def _normalized_correlation(a: torch.Tensor, b: torch.Tensor) -> float: + length = min(int(a.numel()), int(b.numel())) + if length < 8: + return 0.0 + a = a.reshape(-1)[:length].float() + b = b.reshape(-1)[:length].float() + a = a - a.mean() + b = b - b.mean() + denom = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12) + value = float((torch.sum(a * b) / denom).item()) + return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0 + + +def overlap_diagnostics( + *, + previous_images: torch.Tensor, + previous_audio: dict[str, Any], + next_raw_images: torch.Tensor, + next_raw_audio: dict[str, Any], + context_frames: int, +) -> dict[str, float]: + context_frames = int(context_frames) + frame_count = min( + context_frames, + int(previous_images.shape[0]), + int(next_raw_images.shape[0]), + ) + if frame_count <= 0: + return { + "video_overlap_mae": 0.0, + "video_overlap_psnr": 99.0, + "audio_overlap_correlation": 0.0, + "audio_boundary_jump": 0.0, + } + + prev = previous_images[-frame_count:].detach().to(dtype=torch.float32) + nxt = next_raw_images[:frame_count].detach().to(dtype=torch.float32) + mse = float(torch.mean((prev - nxt).square()).item()) + mae = float(torch.mean(torch.abs(prev - nxt)).item()) + psnr = 99.0 if mse <= 1e-12 else max(0.0, -10.0 * math.log10(mse)) + + prev_wave, prev_sr = _audio_waveform(previous_audio) + next_wave, next_sr = _audio_waveform(next_raw_audio) + if prev_sr != next_sr: + raise ValueError("diagnostic audio sample rates do not match") + samples = int(round(frame_count / FPS * prev_sr)) + samples = min(samples, int(prev_wave.shape[-1]), int(next_wave.shape[-1])) + if samples > 0: + correlation = _normalized_correlation(prev_wave[..., -samples:], next_wave[..., :samples]) + else: + correlation = 0.0 + if prev_wave.shape[-1] and next_wave.shape[-1]: + jump = float(torch.mean(torch.abs(prev_wave[..., -1] - next_wave[..., 0])).item()) + else: + jump = 0.0 + return { + "video_overlap_mae": mae, + "video_overlap_psnr": psnr, + "audio_overlap_correlation": correlation, + "audio_boundary_jump": jump, + } + + +def av_duration_error_samples(images: torch.Tensor, audio: dict[str, Any]) -> int: + waveform, sample_rate = _audio_waveform(audio) + expected = int(round(float(images.shape[0]) / FPS * sample_rate)) + return int(waveform.shape[-1]) - expected + + +def format_join_metrics(index: int, metrics: dict[str, float]) -> str: + return ( + f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, " + f"PSNR={metrics['video_overlap_psnr']:.2f}dB, " + f"audio corr={metrics['audio_overlap_correlation']:.4f}, " + f"boundary jump={metrics['audio_boundary_jump']:.6f}" + ) + + +def format_seam_decision(index: int, decision: SeamDecision, *, full: bool) -> str: + fallback = decision.fallback_reason or "none" + if not full: + return ( + f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, " + f"score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, " + f"video blend={decision.video_blend_frames}f, " + f"audio offset={decision.audio_metrics.offset_samples:+d} samples, " + f"fallback={fallback}" + ) + return ( + f"Seam {index-1}->{index}\n" + f" Legacy score : {decision.legacy_score:.6f}\n" + f" Selected cut : -{decision.cut_rewind_frames} frames\n" + f" Cut score : {decision.cut_score:.6f}\n" + f" Corrected score : {decision.corrected_score:.6f}\n" + f" Improvement : {decision.improvement * 100.0:.2f}%\n" + f" Luminance gain : {decision.luma_gain:.4f}\n" + f" Chroma bias : {decision.chroma_bias}\n" + f" Video blend : {decision.video_blend_frames} frames\n" + f" Audio correlation : " + f"{decision.audio_metrics.correlation_before:.4f}" + f"->{decision.audio_metrics.correlation_after:.4f}\n" + f" Audio jump : " + f"{decision.audio_metrics.boundary_jump_before:.6f}" + f"->{decision.audio_metrics.boundary_jump_after:.6f}\n" + f" Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n" + f" Audio crossfade : {decision.audio_crossfade_samples} samples\n" + f" Fallback : {fallback}" + ) diff --git a/v2/h3_builder.py b/v2/h3_builder.py new file mode 100644 index 0000000..c8c36ec --- /dev/null +++ b/v2/h3_builder.py @@ -0,0 +1,160 @@ +"""Native MiniMax H3 conditioning and latent construction for V2. + +All expensive image/VAE work is performed before the sampling phase. Prompt +conditioning is cached per unique prompt and image presentation. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Any + +import torch + +from ..continuation import clone_conditioning +from ..temporal import audio_latent_t, video_latent_t + + +@dataclass(slots=True) +class IdentityAssets: + first_image: torch.Tensor | None + first_latent: torch.Tensor | None + last_image: torch.Tensor | None + last_latent: torch.Tensor | None + identity_hash: str + + +def _tensor_fingerprint(tensor: torch.Tensor | None) -> str: + if tensor is None: + return "none" + sample = tensor.detach().to("cpu", dtype=torch.float32).contiguous() + # Hash a deterministic downsampled view rather than several gigabytes of bytes. + if sample.ndim == 4: + sample = sample[:1].movedim(-1, 1) + sample = torch.nn.functional.interpolate(sample, size=(64, 64), mode="area") + raw = sample.numpy().tobytes(order="C") + return hashlib.sha256(raw).hexdigest() + + +def resize_h3_image(image: torch.Tensor, width: int, height: int, crop: str) -> torch.Tensor: + try: + import comfy.utils + except Exception as exc: # pragma: no cover - ComfyUI runtime only + raise RuntimeError(f"ComfyUI resize helper unavailable: {exc}") from exc + if not torch.is_tensor(image) or image.ndim != 4 or image.shape[-1] < 3: + raise ValueError("H3 image input must be [B,H,W,C] with at least three channels") + samples = image[:1, ..., :3].movedim(-1, 1) + resized = comfy.utils.common_upscale(samples, int(width), int(height), "lanczos", crop) + return resized.movedim(1, -1).contiguous() + + +def prepare_identity_assets( + video_vae: Any, + *, + width: int, + height: int, + first_frame: torch.Tensor | None, + last_frame: torch.Tensor | None, + encode_latents: bool = True, +) -> IdentityAssets: + first_image = first_latent = last_image = last_latent = None + if first_frame is not None: + first_image = resize_h3_image(first_frame, width, height, "disabled") + if encode_latents: + first_latent = video_vae.encode(first_image) + if last_frame is not None: + last_image = resize_h3_image(last_frame, width, height, "center") + if encode_latents: + last_latent = video_vae.encode(last_image) + return IdentityAssets( + first_image=first_image, + first_latent=first_latent, + last_image=last_image, + last_latent=last_latent, + identity_hash=_tensor_fingerprint(first_image), + ) + + +def encode_prompt_conditioning( + clip: Any, + prompt: str, + *, + first_image: torch.Tensor | None, + last_image: torch.Tensor | None, +) -> list: + images = [] + if first_image is not None: + images.append(first_image) + if last_image is not None: + images.append(last_image) + tokens = clip.tokenize(str(prompt), images=images) + return clip.encode_from_tokens_scheduled(tokens) + + +def attach_keyframes( + conditioning: list, + *, + frame_count: int, + first_latent: torch.Tensor | None, + last_latent: torch.Tensor | None, +) -> list: + output = clone_conditioning(conditioning) + keyframes: list[dict[str, Any]] = [] + if first_latent is not None: + keyframes.append({"resolved_frame_index": 0, "latent": first_latent}) + if last_latent is not None: + keyframes.append( + {"resolved_frame_index": int(frame_count) - 1, "latent": last_latent} + ) + for _, metadata in output: + if keyframes: + metadata["minimax_keyframes"] = [dict(item) for item in keyframes] + metadata["minimax_frame_count"] = int(frame_count) + else: + metadata.pop("minimax_keyframes", None) + metadata["minimax_frame_count"] = int(frame_count) + return output + + +def empty_h3_latent(width: int, height: int, frame_count: int) -> dict[str, Any]: + try: + import comfy.model_management + import comfy.nested_tensor + except Exception as exc: # pragma: no cover - ComfyUI runtime only + raise RuntimeError(f"ComfyUI H3 latent helpers unavailable: {exc}") from exc + width = int(width) + height = int(height) + frame_count = int(frame_count) + if width <= 0 or height <= 0 or width % 32 or height % 32: + raise ValueError("H3 width and height must be positive multiples of 32") + device = comfy.model_management.intermediate_device() + video = torch.zeros( + (1, 24, video_latent_t(frame_count), height // 16, width // 16), + device=device, + dtype=torch.float32, + ) + audio = torch.zeros( + (1, 32, 2, audio_latent_t(frame_count)), + device=device, + dtype=torch.float32, + ) + return {"samples": comfy.nested_tensor.NestedTensor((video, audio))} + + +def encode_identity_latents(video_vae: Any, assets: IdentityAssets) -> IdentityAssets: + """Encode already-resized identity images without changing their fingerprint.""" + + first_latent = assets.first_latent + last_latent = assets.last_latent + if first_latent is None and assets.first_image is not None: + first_latent = video_vae.encode(assets.first_image) + if last_latent is None and assets.last_image is not None: + last_latent = video_vae.encode(assets.last_image) + return IdentityAssets( + first_image=assets.first_image, + first_latent=first_latent, + last_image=assets.last_image, + last_latent=last_latent, + identity_hash=assets.identity_hash, + ) diff --git a/v2/motion.py b/v2/motion.py new file mode 100644 index 0000000..39b6de9 --- /dev/null +++ b/v2/motion.py @@ -0,0 +1,53 @@ +"""Conservative latent-motion analysis for automatic context selection.""" + +from __future__ import annotations + +import math +from typing import Any + +import torch + +from ..constants import CONTINUITY_FRAMES, V2_CONTINUITY_AUTO +from ..state import validate_state + +# Conservative defaults. Borderline values intentionally fall back to 22 frames. +_AUTO_STATIC_MAX = 0.035 +_AUTO_STRONG_MIN = 0.120 + + +def latent_motion_score(state: dict[str, Any]) -> float: + state = validate_state(state) + video = state["video_tail"].detach().to(dtype=torch.float32) + if video.shape[2] < 2: + return 0.0 + temporal_delta = video[:, :, 1:] - video[:, :, :-1] + delta_rms = torch.sqrt(torch.mean(temporal_delta.square())) + signal_rms = torch.sqrt(torch.mean(video.square())).clamp_min(1e-6) + score = float((delta_rms / signal_rms).item()) + if not math.isfinite(score): + return _AUTO_STRONG_MIN + return max(0.0, min(score, 1000.0)) + + +def choose_context_frames(mode: str, state: dict[str, Any]) -> tuple[int, float, str]: + state = validate_state(state) + capacity = int(state["capacity_frames"]) + score = latent_motion_score(state) + + if mode != V2_CONTINUITY_AUTO: + if mode not in CONTINUITY_FRAMES: + raise ValueError(f"unknown continuity mode: {mode!r}") + requested = int(CONTINUITY_FRAMES[mode]) + if requested > capacity: + raise ValueError( + f"state retains {capacity} frames but continuity requests {requested}" + ) + return requested, score, "fixed profile" + + if capacity < 22: + return 5, score, "state capacity below 22 frames" + if score <= _AUTO_STATIC_MAX: + return 5, score, "low latent motion" + if score >= _AUTO_STRONG_MIN and capacity >= 39: + return 39, score, "high latent motion" + return 22, score, "conservative balanced fallback" diff --git a/v2/nodes.py b/v2/nodes.py new file mode 100644 index 0000000..9bc201e --- /dev/null +++ b/v2/nodes.py @@ -0,0 +1,473 @@ +"""ComfyUI node layer for H3 Continuum V2.""" + +from __future__ import annotations + +import logging + +from ..constants import ( + DIAGNOSTICS_OPTIONS, + PROMPT_FORMAT_OPTIONS, + PROMPT_MODE_OPTIONS, + SEAM_CORRECTION_OFF, + SEAM_CORRECTION_OPTIONS, + V2_CONTINUITY_OPTIONS, +) +from .prompts import ( + apply_prompt_overrides, + build_sampler_prompt_plan, + make_prompt_plan, + prompt_plan_report, +) +from .sequence import run_sequence +from .session import entry_to_state, session_summary, validate_session +from .session_io import ( + load_session, + save_session, + session_mtime, + session_paths, +) + +CATEGORY = "MiniMax H3/Continuum" +LOG = logging.getLogger("h3_continuum_join") + + +def _repair_v200_example_widget_shift( + *, + audio_continuity, + exact_total_duration, + diagnostics, + reroll_from_chunk, + reroll_nonce, + strict_compatibility, + debug, +): + """Repair the malformed bundled 2.0.0 example workflow in memory. + + ``base_seed`` uses ComfyUI's ``control_after_generate`` option, which owns an + additional serialized widget. The original hand-authored V2 example omitted + that widget, shifting every following widget by one slot. A distinctive + signature is therefore ``exact_total_duration == 'Basic'/'Full'/'Off'`` and + numeric ``diagnostics``. Valid workflows never have a diagnostics label in + the BOOLEAN exact-duration input, so this migration is narrow and safe. + + The missing exact-duration value cannot be recovered from the malformed JSON; + restore its documented V2 default (True). The remaining values can be shifted + back losslessly. + """ + + if isinstance(exact_total_duration, str) and exact_total_duration in DIAGNOSTICS_OPTIONS: + if isinstance(diagnostics, (int, float, bool)) and not isinstance(diagnostics, str): + LOG.warning( + "H3 Continuum V2 detected the malformed 2.0.0 bundled workflow widget " + "layout; repairing shifted settings in memory. exact_total_duration " + "is restored to its default True. Save the workflow again after this run." + ) + return ( + bool(audio_continuity), + True, + exact_total_duration, + int(diagnostics), + int(reroll_from_chunk), + bool(reroll_nonce), + bool(strict_compatibility), + ) + return ( + bool(audio_continuity), + bool(exact_total_duration), + diagnostics, + int(reroll_from_chunk), + int(reroll_nonce), + bool(strict_compatibility), + bool(debug), + ) + + +class H3ContinuumSamplerV2: + DESCRIPTION = ( + "Integrated N-chunk MiniMax H3 continuation sampler. Precomputes prompt/image " + "conditioning, isolates accelerator runtime state per chunk, captures latent " + "state before decoding, then decodes and assembles the final AV stream." + ) + SEARCH_ALIASES = [ + "H3 long video sampler", + "MiniMax H3 integrated continuation", + "H3 Continuum V2", + ] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": ("MODEL",), + "clip": ("CLIP",), + "video_vae": ("VAE",), + "audio_vae": ("VAE",), + "sampler": ("SAMPLER",), + "sigmas": ("SIGMAS",), + "prompt_mode": ( + PROMPT_FORMAT_OPTIONS, + { + "default": PROMPT_FORMAT_OPTIONS[0], + "tooltip": ( + "Auto detects Fixed text, --- separated List text, or " + "[0-5s] / [Chunk 1] Timeline sections." + ), + }, + ), + "prompt_script": ( + "STRING", + { + "default": "Describe one continuous MiniMax H3 shot with audio.", + "multiline": True, + "dynamicPrompts": True, + "tooltip": ( + "Fixed: one prompt. List: separate chunks with a line containing ---. " + "Timeline: use [0-5s], [5-10s], ... or [Chunk 1] headers." + ), + }, + ), + "chunks": ( + "INT", + {"default": 3, "min": 1, "max": 16, "step": 1}, + ), + "chunk_seconds": ( + "FLOAT", + {"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1, "tooltip": "Native H3 is trained for roughly 4–15 second outputs."}, + ), + "width": ( + "INT", + {"default": 1344, "min": 32, "max": 16384, "step": 32}, + ), + "height": ( + "INT", + {"default": 768, "min": 32, "max": 16384, "step": 32}, + ), + "continuity": ( + V2_CONTINUITY_OPTIONS, + { + "default": V2_CONTINUITY_OPTIONS[1], + "tooltip": ( + "Balanced 22f reproduces the V1 default. Auto uses a conservative " + "latent-motion score and falls back to 22f near thresholds." + ), + }, + ), + "base_seed": ( + "INT", + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFFFFFFFFFF, + "control_after_generate": True, + }, + ), + "audio_continuity": ("BOOLEAN", {"default": True}), + "exact_total_duration": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Trim or minimally pad the assembled result to chunks × seconds × 24fps.", + }, + ), + "diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}), + "reroll_from_chunk": ( + "INT", + { + "default": 0, + "min": 0, + "max": 16, + "step": 1, + "tooltip": ( + "0 resumes/reuses every matching accepted chunk from the optional session. " + "N preserves chunks before N and regenerates N onward." + ), + }, + ), + "reroll_nonce": ( + "INT", + { + "default": 0, + "min": 0, + "max": 0xFFFFFFFF, + "step": 1, + "tooltip": "Change this to derive new seeds for regenerated chunks only.", + }, + ), + "strict_compatibility": ("BOOLEAN", {"default": True}), + "debug": ("BOOLEAN", {"default": False}), + "seam_correction": ( + SEAM_CORRECTION_OPTIONS, + { + "default": SEAM_CORRECTION_OFF, + "tooltip": ( + "Off preserves the V2.0.2 decode path. Basic applies " + "conservative decoded A/V boundary correction." + ), + }, + ), + }, + "optional": { + "sequence_prompt": ( + "STRING", + { + "forceInput": True, + "tooltip": ( + "Connect one Text (Multiline). Prompt Format parses it for " + "the complete sequence without changing this socket." + ), + }, + ), + "first_frame": ("IMAGE",), + "last_frame": ("IMAGE",), + "session": ("H3_CONTINUUM_SESSION",), + "initial_state": ("H3_CONTINUUM_STATE",), + "prompt_plan": ("H3_CONTINUUM_PROMPT_PLAN",), + "show_preview": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Controlled by Settings > H3 Continuum > Show latent preview.", + }, + ), + **{ + f"clip_{index}_prompt": ( + "STRING", + { + "forceInput": True, + "tooltip": f"Connect Text (Multiline) for Clip {index}.", + }, + ) + for index in range(1, 17) + }, + }, + } + + RETURN_TYPES = ( + "IMAGE", + "AUDIO", + "H3_CONTINUUM_STATE", + "H3_CONTINUUM_SESSION", + "STRING", + ) + RETURN_NAMES = ("images", "audio", "last_state", "session", "report") + FUNCTION = "run" + CATEGORY = CATEGORY + + def run( + self, + model, + clip, + video_vae, + audio_vae, + sampler, + sigmas, + prompt_mode, + prompt_script, + chunks, + chunk_seconds, + width, + height, + continuity, + base_seed, + audio_continuity, + exact_total_duration, + diagnostics, + reroll_from_chunk, + reroll_nonce, + strict_compatibility, + debug, + seam_correction=SEAM_CORRECTION_OFF, + first_frame=None, + last_frame=None, + session=None, + initial_state=None, + prompt_plan=None, + sequence_prompt=None, + show_preview=True, + **clip_prompt_inputs, + ): + ( + audio_continuity, + exact_total_duration, + diagnostics, + reroll_from_chunk, + reroll_nonce, + strict_compatibility, + debug, + ) = _repair_v200_example_widget_shift( + audio_continuity=audio_continuity, + exact_total_duration=exact_total_duration, + diagnostics=diagnostics, + reroll_from_chunk=reroll_from_chunk, + reroll_nonce=reroll_nonce, + strict_compatibility=strict_compatibility, + debug=debug, + ) + + # V2.0.2 workflows stored show_preview in this newly inserted widget slot. + if isinstance(seam_correction, bool): + seam_correction = SEAM_CORRECTION_OFF + + prompt_plan = build_sampler_prompt_plan( + prompt_mode=prompt_mode, + prompt_script=prompt_script, + sequence_prompt=sequence_prompt, + prompt_plan=prompt_plan, + chunks=int(chunks), + chunk_seconds=float(chunk_seconds), + ) + prompt_plan = apply_prompt_overrides( + prompt_plan, + [ + clip_prompt_inputs.get(f"clip_{index}_prompt") + for index in range(1, int(chunks) + 1) + ], + ) + return run_sequence( + model=model, + clip=clip, + video_vae=video_vae, + audio_vae=audio_vae, + sampler=sampler, + sigmas=sigmas, + first_frame=first_frame, + last_frame=last_frame, + prompt_plan=prompt_plan, + width=int(width), + height=int(height), + continuity=continuity, + base_seed=int(base_seed), + audio_continuity=bool(audio_continuity), + exact_total_duration=bool(exact_total_duration), + diagnostics_mode=diagnostics, + reroll_from_chunk=int(reroll_from_chunk), + reroll_nonce=int(reroll_nonce), + strict_compatibility=bool(strict_compatibility), + debug=bool(debug), + seam_correction=seam_correction, + enable_preview=bool(show_preview), + session=session, + initial_state=initial_state, + ) + + +class H3ContinuumPromptPlanPreview: + DESCRIPTION = "Validate and preview Fixed/List/Timeline prompt parsing without sampling." + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prompt_mode": (PROMPT_MODE_OPTIONS, {"default": PROMPT_MODE_OPTIONS[0]}), + "prompt_script": ("STRING", {"default": "Prompt", "multiline": True}), + "chunks": ("INT", {"default": 3, "min": 1, "max": 16, "step": 1}), + "chunk_seconds": ( + "FLOAT", + {"default": 5.0, "min": 4.0, "max": 15.0, "step": 0.1, "tooltip": "Native H3 is trained for roughly 4–15 second outputs."}, + ), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_PROMPT_PLAN", "STRING") + RETURN_NAMES = ("prompt_plan", "report") + FUNCTION = "build" + CATEGORY = CATEGORY + + def build(self, prompt_mode, prompt_script, chunks, chunk_seconds): + plan = make_prompt_plan( + mode=prompt_mode, + script=prompt_script, + chunks=int(chunks), + chunk_seconds=float(chunk_seconds), + ) + report = prompt_plan_report(plan) + "\n" + "\n\n".join( + f"Chunk {index + 1}:\n{prompt}" for index, prompt in enumerate(plan["prompts"]) + ) + return plan, report + + +class H3ContinuumSaveSession: + DESCRIPTION = "Atomically save every accepted V2 chunk latent plus session metadata." + OUTPUT_NODE = True + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("H3_CONTINUUM_SESSION",), + "prefix": ("STRING", {"default": "h3_continuum_session"}), + "slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_SESSION", "STRING") + RETURN_NAMES = ("session", "path") + FUNCTION = "save" + CATEGORY = CATEGORY + + def save(self, session, prefix, slot): + tensor_path, _json_path = save_session(session, prefix=prefix, slot=int(slot)) + return session, str(tensor_path) + + +class H3ContinuumLoadSession: + DESCRIPTION = "Load an explicitly numbered V2 session for resume or reroll." + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "prefix": ("STRING", {"default": "h3_continuum_session"}), + "slot": ("INT", {"default": 1, "min": 1, "max": 9999, "step": 1}), + } + } + + RETURN_TYPES = ("H3_CONTINUUM_SESSION", "STRING") + RETURN_NAMES = ("session", "path") + FUNCTION = "load" + CATEGORY = CATEGORY + + @classmethod + def IS_CHANGED(cls, prefix, slot): + return session_mtime(prefix=prefix, slot=int(slot)) + + def load(self, prefix, slot): + session = load_session(prefix=prefix, slot=int(slot)) + tensor_path, _json_path = session_paths(prefix, int(slot)) + return session, str(tensor_path) + + +class H3ContinuumSessionInfo: + DESCRIPTION = "Inspect a V2 session and expose its final V1-compatible continuation state." + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"session": ("H3_CONTINUUM_SESSION",)}} + + RETURN_TYPES = ("H3_CONTINUUM_SESSION", "H3_CONTINUUM_STATE", "STRING") + RETURN_NAMES = ("session", "last_state", "report") + FUNCTION = "inspect" + CATEGORY = CATEGORY + + def inspect(self, session): + session = validate_session(session) + state = entry_to_state(session["chunks"][-1]) + report = session_summary(session) + return session, state, report + + +NODE_CLASS_MAPPINGS = { + "H3ContinuumSamplerV2": H3ContinuumSamplerV2, + "H3ContinuumPromptPlanPreview": H3ContinuumPromptPlanPreview, + "H3ContinuumSaveSession": H3ContinuumSaveSession, + "H3ContinuumLoadSession": H3ContinuumLoadSession, + "H3ContinuumSessionInfo": H3ContinuumSessionInfo, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "H3ContinuumSamplerV2": "H3 Continuum Sampler V2", + "H3ContinuumPromptPlanPreview": "H3 Continuum Prompt Plan", + "H3ContinuumSaveSession": "H3 Continuum Save Session", + "H3ContinuumLoadSession": "H3 Continuum Load Session", + "H3ContinuumSessionInfo": "H3 Continuum Session Info", +} diff --git a/v2/prompts.py b/v2/prompts.py new file mode 100644 index 0000000..7abf4b8 --- /dev/null +++ b/v2/prompts.py @@ -0,0 +1,331 @@ +"""Prompt planning for the integrated H3 Continuum sampler. + +The parser is intentionally deterministic and side-effect free so saved workflows +can be reproduced without depending on UI state. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any + +from ..constants import ( + PROMPT_FORMAT_AUTO, + PROMPT_FORMAT_FIXED, + PROMPT_FORMAT_LIST, + PROMPT_FORMAT_TIMELINE, + PROMPT_MODE_FIXED, + PROMPT_MODE_LIST, + PROMPT_MODE_TIMELINE, + PROMPT_PLAN_MAGIC, +) +from ..version import PROMPT_PLAN_SCHEMA_VERSION + +_TIMELINE_HEADER = re.compile( + r"^\s*\[\s*(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*[-–—]\s*" + r"(?P\d+(?:\.\d+)?)\s*(?:s|sec|seconds)?\s*\]\s*$", + re.IGNORECASE, +) +_CHUNK_HEADER = re.compile( + r"^\s*\[\s*(?:chunk|clip)\s*(?P\d+)\s*\]\s*$", + re.IGNORECASE, +) +_LIST_SEPARATOR = re.compile(r"^\s*---+\s*$", re.MULTILINE) + + +class PromptPlanError(ValueError): + pass + + +def prompt_hash(prompt: str) -> str: + return hashlib.sha256(prompt.encode("utf-8")).hexdigest() + + +def _normalize_prompt(value: str, *, label: str) -> str: + text = str(value).strip() + if not text: + raise PromptPlanError(f"{label} is empty") + return text + + +def _parse_json_list(script: str) -> list[str] | None: + stripped = script.lstrip() + if not stripped.startswith("["): + return None + try: + value = json.loads(script) + except json.JSONDecodeError: + return None + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise PromptPlanError("JSON prompt list must be an array of strings") + return [_normalize_prompt(item, label=f"prompt {index + 1}") for index, item in enumerate(value)] + + +def _parse_list(script: str, chunks: int) -> tuple[list[str], list[str]]: + values = _parse_json_list(script) + if values is None: + values = [ + _normalize_prompt(part, label=f"prompt {index + 1}") + for index, part in enumerate(_LIST_SEPARATOR.split(script)) + if part.strip() + ] + if not values: + raise PromptPlanError("prompt list contains no prompts") + notes: list[str] = [] + if len(values) < chunks: + notes.append(f"repeated the last prompt for {chunks - len(values)} chunk(s)") + values.extend([values[-1]] * (chunks - len(values))) + elif len(values) > chunks: + notes.append(f"ignored {len(values) - chunks} extra prompt(s)") + values = values[:chunks] + return values, notes + + +def _parse_timeline_sections(script: str) -> list[dict[str, Any]]: + sections: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + body: list[str] = [] + + def finish() -> None: + nonlocal current, body + if current is None: + if any(line.strip() for line in body): + raise PromptPlanError( + "timeline text before the first [0-5s] or [Chunk 1] header is not allowed" + ) + body = [] + return + current["prompt"] = _normalize_prompt("\n".join(body), label="timeline section") + sections.append(current) + current = None + body = [] + + for line in str(script).splitlines(): + time_match = _TIMELINE_HEADER.match(line) + chunk_match = _CHUNK_HEADER.match(line) + if time_match or chunk_match: + finish() + if time_match: + start = float(time_match.group("start")) + end = float(time_match.group("end")) + if not end > start: + raise PromptPlanError(f"timeline range must increase: {line.strip()}") + current = {"kind": "time", "start": start, "end": end} + else: + index = int(chunk_match.group("index")) + if index < 1: + raise PromptPlanError("chunk numbers are one-based") + current = {"kind": "chunk", "index": index} + continue + body.append(line) + finish() + if not sections: + raise PromptPlanError("timeline contains no sections") + return sections + + +def _parse_timeline(script: str, chunks: int, chunk_seconds: float) -> tuple[list[str], list[str]]: + sections = _parse_timeline_sections(script) + prompts: list[str] = [] + notes: list[str] = [] + for chunk_index in range(1, chunks + 1): + chunk_sections = [ + item for item in sections if item["kind"] == "chunk" and item["index"] == chunk_index + ] + if len(chunk_sections) > 1: + raise PromptPlanError(f"timeline defines Chunk {chunk_index} more than once") + if chunk_sections: + prompts.append(chunk_sections[0]["prompt"]) + continue + + start = (chunk_index - 1) * chunk_seconds + end = chunk_index * chunk_seconds + scored: list[tuple[float, int, dict[str, Any]]] = [] + for order, item in enumerate(sections): + if item["kind"] != "time": + continue + overlap = max(0.0, min(end, item["end"]) - max(start, item["start"])) + if overlap > 0: + scored.append((overlap, -order, item)) + if not scored: + raise PromptPlanError( + f"timeline does not cover chunk {chunk_index} ({start:.3f}-{end:.3f}s)" + ) + scored.sort(reverse=True, key=lambda row: (row[0], row[1])) + prompts.append(scored[0][2]["prompt"]) + if len(scored) > 1 and abs(scored[0][0] - scored[1][0]) < 1e-9: + notes.append(f"chunk {chunk_index} had an equal-overlap timeline tie; used the earlier section") + return prompts, notes + + +def detect_prompt_mode(script: str) -> str: + """Detect the stable Prompt Plan mode without changing the input text.""" + + text = str(script) + for line in text.splitlines(): + if _TIMELINE_HEADER.match(line) or _CHUNK_HEADER.match(line): + return PROMPT_MODE_TIMELINE + if _parse_json_list(text) is not None or _LIST_SEPARATOR.search(text): + return PROMPT_MODE_LIST + return PROMPT_MODE_FIXED + + +def resolve_prompt_mode(mode: str, script: str) -> str: + explicit = { + PROMPT_FORMAT_FIXED: PROMPT_MODE_FIXED, + PROMPT_FORMAT_LIST: PROMPT_MODE_LIST, + PROMPT_FORMAT_TIMELINE: PROMPT_MODE_TIMELINE, + } + if mode == PROMPT_FORMAT_AUTO: + return detect_prompt_mode(script) + if mode in explicit: + return explicit[mode] + if mode in (PROMPT_MODE_FIXED, PROMPT_MODE_LIST, PROMPT_MODE_TIMELINE): + return mode + raise PromptPlanError(f"unknown prompt format: {mode!r}") + + +def make_prompt_plan( + *, mode: str, script: str, chunks: int, chunk_seconds: float, +) -> dict[str, Any]: + chunks = int(chunks) + chunk_seconds = float(chunk_seconds) + if not (1 <= chunks <= 16): + raise PromptPlanError("chunks must be between 1 and 16") + if not (4.0 <= chunk_seconds <= 15.0): + raise PromptPlanError("chunk_seconds must be between 4.0 and 15.0 for native H3") + + resolved_mode = resolve_prompt_mode(mode, script) + notes: list[str] = [] + if mode == PROMPT_FORMAT_AUTO: + detected = { + PROMPT_MODE_FIXED: PROMPT_FORMAT_FIXED, + PROMPT_MODE_LIST: PROMPT_FORMAT_LIST, + PROMPT_MODE_TIMELINE: PROMPT_FORMAT_TIMELINE, + }[resolved_mode] + notes.append(f"Auto detected {detected}") + + if resolved_mode == PROMPT_MODE_FIXED: + prompt = _normalize_prompt(script, label="fixed prompt") + prompts = [prompt] * chunks + elif resolved_mode == PROMPT_MODE_LIST: + prompts, parse_notes = _parse_list(script, chunks) + notes.extend(parse_notes) + elif resolved_mode == PROMPT_MODE_TIMELINE: + prompts, parse_notes = _parse_timeline(script, chunks, chunk_seconds) + notes.extend(parse_notes) + else: + raise PromptPlanError(f"unknown prompt mode: {resolved_mode!r}") + + return { + "magic": PROMPT_PLAN_MAGIC, + "schema_version": PROMPT_PLAN_SCHEMA_VERSION, + "mode": resolved_mode, + "chunks": chunks, + "chunk_seconds": chunk_seconds, + "prompts": prompts, + "hashes": [prompt_hash(prompt) for prompt in prompts], + "notes": notes, + } + + +def validate_prompt_plan(plan: dict[str, Any]) -> dict[str, Any]: + if not isinstance(plan, dict) or plan.get("magic") != PROMPT_PLAN_MAGIC: + raise PromptPlanError("invalid H3 Continuum prompt plan") + if int(plan.get("schema_version", -1)) != PROMPT_PLAN_SCHEMA_VERSION: + raise PromptPlanError( + f"unsupported prompt-plan schema {plan.get('schema_version')}; " + f"expected {PROMPT_PLAN_SCHEMA_VERSION}" + ) + chunks = int(plan.get("chunks", 0)) + prompts = plan.get("prompts") + hashes = plan.get("hashes") + if not (1 <= chunks <= 16): + raise PromptPlanError("prompt-plan chunks are invalid") + if not isinstance(prompts, list) or len(prompts) != chunks: + raise PromptPlanError("prompt-plan prompt count does not match chunks") + if not isinstance(hashes, list) or len(hashes) != chunks: + raise PromptPlanError("prompt-plan hash count does not match chunks") + for index, prompt in enumerate(prompts): + _normalize_prompt(prompt, label=f"prompt {index + 1}") + if hashes[index] != prompt_hash(prompt): + raise PromptPlanError(f"prompt-plan hash mismatch at chunk {index + 1}") + return plan + + +def build_sampler_prompt_plan( + *, + prompt_mode: str, + prompt_script: str, + sequence_prompt: str | None, + prompt_plan: dict[str, Any] | None, + chunks: int, + chunk_seconds: float, +) -> dict[str, Any]: + """Resolve the fixed Sequence socket while preserving legacy fallbacks.""" + + chunks = int(chunks) + chunk_seconds = float(chunk_seconds) + if sequence_prompt is not None: + return make_prompt_plan( + mode=prompt_mode, + script=sequence_prompt, + chunks=chunks, + chunk_seconds=chunk_seconds, + ) + if prompt_plan is not None: + plan = validate_prompt_plan(prompt_plan) + if int(plan["chunks"]) != chunks: + raise PromptPlanError( + "connected prompt_plan chunks do not match the Sampler chunks widget" + ) + if abs(float(plan["chunk_seconds"]) - chunk_seconds) > 1e-6: + raise PromptPlanError( + "connected prompt_plan chunk_seconds do not match the Sampler widget" + ) + return plan + return make_prompt_plan( + mode=prompt_mode, + script=prompt_script, + chunks=chunks, + chunk_seconds=chunk_seconds, + ) + + +def apply_prompt_overrides( + plan: dict[str, Any], overrides: list[str | None] +) -> dict[str, Any]: + """Replace selected chunk prompts without changing the prompt-plan schema.""" + + plan = validate_prompt_plan(plan) + prompts = list(plan["prompts"]) + replaced: list[int] = [] + for index, value in enumerate(overrides[: int(plan["chunks"])]): + if value is None: + continue + prompts[index] = _normalize_prompt(value, label=f"Clip {index + 1} Prompt") + replaced.append(index + 1) + if not replaced: + return plan + + result = dict(plan) + result["mode"] = PROMPT_MODE_LIST + result["prompts"] = prompts + result["hashes"] = [prompt_hash(prompt) for prompt in prompts] + result["notes"] = list(plan.get("notes") or []) + [ + "external Clip Prompt input(s): " + ", ".join(map(str, replaced)) + ] + return validate_prompt_plan(result) + + +def prompt_plan_report(plan: dict[str, Any]) -> str: + plan = validate_prompt_plan(plan) + unique = len(set(plan["hashes"])) + note = "; ".join(plan.get("notes") or []) + result = ( + f"Prompt plan: {plan['mode']}; {plan['chunks']} chunks; " + f"{unique} unique prompt(s); {plan['chunk_seconds']:.3f}s per chunk." + ) + return result + (" " + note + "." if note else "") diff --git a/v2/sampling.py b/v2/sampling.py new file mode 100644 index 0000000..5b49409 --- /dev/null +++ b/v2/sampling.py @@ -0,0 +1,124 @@ +"""ComfyUI-native chunk sampling helpers.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from ..state import extract_av_streams + + +class SamplingRuntimeError(RuntimeError): + pass + + +def _make_basic_guider(model: Any, conditioning: list): + try: + import comfy.samplers + except Exception as exc: # pragma: no cover + raise SamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc + + class _BasicGuider(comfy.samplers.CFGGuider): + def set_positive(self, positive): + self.inner_set_conds({"positive": positive}) + + guider = _BasicGuider(model) + guider.set_positive(conditioning) + return guider + + +def _prepare_noise(latent: dict[str, Any], seed: int): + try: + import comfy.sample + except Exception as exc: # pragma: no cover + raise SamplingRuntimeError(f"ComfyUI noise API unavailable: {exc}") from exc + batch_inds = latent.get("batch_index") + return comfy.sample.prepare_noise(latent["samples"], int(seed), batch_inds) + + +def sample_chunk( + *, + model: Any, + conditioning: list, + latent: dict[str, Any], + sampler: Any, + sigmas: torch.Tensor, + seed: int, + enable_preview: bool = True, +) -> dict[str, Any]: + """Run one H3 chunk using the same path as SamplerCustomAdvanced.""" + + try: + import comfy.model_management + import comfy.sample + import comfy.utils + import latent_preview + except Exception as exc: # pragma: no cover + raise SamplingRuntimeError(f"ComfyUI sampling runtime unavailable: {exc}") from exc + + if not isinstance(latent, dict) or "samples" not in latent: + raise SamplingRuntimeError("latent must be a ComfyUI LATENT dictionary") + if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2: + raise SamplingRuntimeError("sigmas must contain at least two values") + + working = latent.copy() + latent_image = working["samples"] + latent_image = comfy.sample.fix_empty_latent_channels( + model, + latent_image, + working.get("downscale_ratio_spacial"), + working.get("downscale_ratio_temporal"), + ) + working["samples"] = latent_image + noise_mask = working.get("noise_mask") + guider = _make_basic_guider(model, conditioning) + noise = _prepare_noise(working, seed) + + x0_output: dict[str, Any] = {} + callback = None + if enable_preview: + callback = latent_preview.prepare_callback(model, int(sigmas.shape[-1]) - 1, x0_output) + disable_pbar = not comfy.utils.PROGRESS_BAR_ENABLED + samples = guider.sample( + noise, + latent_image, + sampler, + sigmas, + denoise_mask=noise_mask, + callback=callback, + disable_pbar=disable_pbar, + seed=int(seed), + ) + samples = samples.to(comfy.model_management.intermediate_device()) + output = working.copy() + output.pop("downscale_ratio_spacial", None) + output.pop("downscale_ratio_temporal", None) + output["samples"] = samples + # Validate the nested AV shape immediately so a bad sampler output cannot be + # committed into a session and fail much later during continuation. + extract_av_streams(output) + return output + + +def latent_to_cpu(latent: dict[str, Any]) -> tuple[torch.Tensor, torch.Tensor]: + video, audio = extract_av_streams(latent) + video_cpu = video.detach().to("cpu").contiguous().clone() + audio_cpu = audio.detach().to("cpu").contiguous().clone() + if not bool(torch.isfinite(video_cpu.float()).all().item()): + raise SamplingRuntimeError("sampled video latent contains NaN or Inf") + if not bool(torch.isfinite(audio_cpu.float()).all().item()): + raise SamplingRuntimeError("sampled audio latent contains NaN or Inf") + return video_cpu, audio_cpu + + +def latent_from_cpu(video: torch.Tensor, audio: torch.Tensor) -> dict[str, Any]: + try: + import comfy.nested_tensor + except Exception as exc: # pragma: no cover + raise SamplingRuntimeError(f"ComfyUI NestedTensor API unavailable: {exc}") from exc + return { + "samples": comfy.nested_tensor.NestedTensor( + (video.contiguous(), audio.contiguous()) + ) + } diff --git a/v2/seam_guard.py b/v2/seam_guard.py new file mode 100644 index 0000000..c8a50e6 --- /dev/null +++ b/v2/seam_guard.py @@ -0,0 +1,533 @@ +"""Conservative CPU-only decoded seam correction for H3 Continuum V2.1.""" + +from __future__ import annotations + +import math + +import torch +import torch.nn.functional as F + +from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics + +COMPARE_SIZE = 96 +MAX_CUT_REWIND = 3 +MIN_RELATIVE_IMPROVEMENT = 0.10 +MIN_ABSOLUTE_IMPROVEMENT = 0.003 +SAFE_SCORE_LIMIT = 0.35 + +WEIGHT_PIXEL = 0.35 +WEIGHT_LUMA = 0.20 +WEIGHT_CHROMA = 0.10 +WEIGHT_EDGE = 0.15 +WEIGHT_TEMPORAL = 0.20 + + +def _video_shape(frames: torch.Tensor, label: str) -> None: + if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3: + raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]") + + +def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor: + _video_shape(frames, "video seam frames") + rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32) + rgb = rgb.permute(0, 3, 1, 2) + if tuple(rgb.shape[-2:]) != (COMPARE_SIZE, COMPARE_SIZE): + rgb = F.interpolate( + rgb, + size=(COMPARE_SIZE, COMPARE_SIZE), + mode="bilinear", + align_corners=False, + ) + rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0) + if not bool(torch.isfinite(rgb).all().item()): + raise ValueError("video seam comparison contains NaN or Inf") + return rgb + + +def _luma(frame: torch.Tensor) -> torch.Tensor: + weights = frame.new_tensor((0.2126, 0.7152, 0.0722)) + return torch.sum(frame * weights, dim=-1) + + +def _boundary_metrics( + previous_images: torch.Tensor, + next_images: torch.Tensor, +) -> VideoSeamMetrics: + if previous_images.shape[0] < 2 or next_images.shape[0] < 2: + raise ValueError("video seam scoring needs two frames on each side") + prev = _downsample_rgb(previous_images[-2:]) + nxt = _downsample_rgb(next_images[:2]) + prev_before, prev_edge = prev[0], prev[1] + next_edge, next_after = nxt[0], nxt[1] + + pixel = torch.mean(torch.abs(prev_edge - next_edge)).clamp(0.0, 1.0) + prev_y = _luma(prev_edge) + next_y = _luma(next_edge) + luma = torch.mean(torch.abs(prev_y - next_y)).clamp(0.0, 1.0) + prev_chroma = prev_edge - prev_y.unsqueeze(-1) + next_chroma = next_edge - next_y.unsqueeze(-1) + chroma = ( + torch.mean(torch.abs(prev_chroma - next_chroma)) / 2.0 + ).clamp(0.0, 1.0) + + prev_gx = prev_y[:, 1:] - prev_y[:, :-1] + next_gx = next_y[:, 1:] - next_y[:, :-1] + prev_gy = prev_y[1:, :] - prev_y[:-1, :] + next_gy = next_y[1:, :] - next_y[:-1, :] + edge = ( + ( + torch.mean(torch.abs(prev_gx - next_gx)) + + torch.mean(torch.abs(prev_gy - next_gy)) + ) + / 4.0 + ).clamp(0.0, 1.0) + + prev_delta = prev_edge - prev_before + next_delta = next_after - next_edge + temporal = ( + torch.mean(torch.abs(prev_delta - next_delta)) / 2.0 + ).clamp(0.0, 1.0) + motion = max( + float(torch.mean(torch.abs(prev_delta)).item()), + float(torch.mean(torch.abs(next_delta)).item()), + ) + score = ( + WEIGHT_PIXEL * float(pixel.item()) + + WEIGHT_LUMA * float(luma.item()) + + WEIGHT_CHROMA * float(chroma.item()) + + WEIGHT_EDGE * float(edge.item()) + + WEIGHT_TEMPORAL * float(temporal.item()) + ) + if not math.isfinite(score): + raise ValueError("video seam score is not finite") + return VideoSeamMetrics( + pixel_mae_norm=float(pixel.item()), + luma_norm=float(luma.item()), + chroma_norm=float(chroma.item()), + edge_norm=float(edge.item()), + temporal_norm=float(temporal.item()), + motion=motion, + score=max(0.0, min(1.0, score)), + ) + + +def analyze_video_seam( + previous_images: torch.Tensor, + next_raw_images: torch.Tensor, + *, + context_frames: int, + rewind_frames: int = 0, +) -> VideoSeamMetrics: + context_frames = int(context_frames) + rewind_frames = int(rewind_frames) + cut_index = context_frames - rewind_frames + previous_stop = int(previous_images.shape[0]) - rewind_frames + if rewind_frames < 0 or rewind_frames > MAX_CUT_REWIND: + raise ValueError("rewind_frames is outside the V2.1 safe range") + if cut_index < 0 or previous_stop < 2 or cut_index + 2 > next_raw_images.shape[0]: + raise ValueError("insufficient frames for the requested adaptive cut") + return _boundary_metrics( + previous_images[previous_stop - 2 : previous_stop], + next_raw_images[cut_index : cut_index + 2], + ) + + +def choose_adaptive_cut( + previous_images: torch.Tensor, + next_raw_images: torch.Tensor, + *, + context_frames: int, +) -> tuple[int, VideoSeamMetrics, VideoSeamMetrics]: + legacy = analyze_video_seam( + previous_images, + next_raw_images, + context_frames=context_frames, + rewind_frames=0, + ) + best_rewind = 0 + best = legacy + maximum = min( + MAX_CUT_REWIND, + max(0, int(context_frames)), + max(0, int(previous_images.shape[0]) - 2), + ) + for rewind in range(1, maximum + 1): + try: + candidate = analyze_video_seam( + previous_images, + next_raw_images, + context_frames=context_frames, + rewind_frames=rewind, + ) + except ValueError: + continue + if candidate.score < best.score: + best_rewind, best = rewind, candidate + + absolute = legacy.score - best.score + relative = absolute / max(legacy.score, 1e-12) + accepted = ( + best_rewind > 0 + and relative >= MIN_RELATIVE_IMPROVEMENT + and absolute >= MIN_ABSOLUTE_IMPROVEMENT + and best.score <= SAFE_SCORE_LIMIT + ) + return (best_rewind, legacy, best) if accepted else (0, legacy, legacy) + + +def apply_color_guard( + previous_images: torch.Tensor, + current_images: torch.Tensor, +) -> tuple[ + torch.Tensor, + float, + tuple[float, float, float], + VideoSeamMetrics, +]: + before = _boundary_metrics(previous_images[-2:], current_images[:2]) + count = min(4, int(current_images.shape[0])) + if count < 2: + return current_images, 1.0, (0.0, 0.0, 0.0), before + + prev_rgb = _downsample_rgb(previous_images[-2:]) + current_rgb = _downsample_rgb(current_images[:2]) + prev_luma = float(_luma(prev_rgb).mean().item()) + current_luma = float(_luma(current_rgb).mean().item()) + requested_gain = prev_luma / max(current_luma, 1e-6) + if not 0.85 <= requested_gain <= 1.15: + return current_images, 1.0, (0.0, 0.0, 0.0), before + gain = max(0.94, min(1.06, requested_gain)) + + prev_y = _luma(prev_rgb).unsqueeze(-1) + current_y = _luma(current_rgb).unsqueeze(-1) + requested_bias = ( + (prev_rgb - prev_y).mean(dim=(0, 1, 2)) + - (current_rgb - current_y).mean(dim=(0, 1, 2)) + ) + if float(torch.max(torch.abs(requested_bias)).item()) > 0.08: + return current_images, 1.0, (0.0, 0.0, 0.0), before + bias = requested_bias.clamp(-0.02, 0.02) + + result = current_images.clone().contiguous() + original = result[:count].clone() + fade = result.new_tensor((1.0, 0.65, 0.30, 0.0))[:count] + luma_weights = result.new_tensor((0.2126, 0.7152, 0.0722)) + for index in range(count): + weight = fade[index] + rgb = result[index, ..., :3] + y = torch.sum(rgb * luma_weights, dim=-1, keepdim=True) + chroma = rgb - y + frame_gain = 1.0 + (gain - 1.0) * weight + frame_bias = bias.to(dtype=rgb.dtype) * weight + result[index, ..., :3] = ( + y * frame_gain + chroma + frame_bias + ).clamp(0.0, 1.0) + + after = _boundary_metrics(previous_images[-2:], result[:2]) + if after.score > before.score * 1.01: + result[:count].copy_(original) + return result, 1.0, (0.0, 0.0, 0.0), before + return ( + result, + gain, + tuple(float(value) for value in bias.tolist()), + after, + ) + + +def choose_video_blend(metrics: VideoSeamMetrics) -> int: + if metrics.motion <= 0.015: + return 2 + if metrics.motion <= 0.040: + return 1 + return 0 + + +def apply_video_blend( + previous_frames: torch.Tensor, + aligned_next_frames: torch.Tensor, + *, + blend_frames: int, +) -> torch.Tensor: + blend_frames = min( + int(blend_frames), + int(previous_frames.shape[0]), + int(aligned_next_frames.shape[0]), + 2, + ) + if blend_frames <= 0: + return previous_frames[:0].clone() + previous = previous_frames[-blend_frames:].clone() + current = aligned_next_frames[-blend_frames:].to(dtype=previous.dtype) + for index in range(blend_frames): + progress = float(index + 1) / float(blend_frames + 1) + alpha = 0.5 - 0.5 * math.cos(math.pi * progress) + previous[index].mul_(1.0 - alpha).add_(current[index], alpha=alpha) + return previous + + +def correct_seam( + previous_images: torch.Tensor, + next_raw_images: torch.Tensor, + *, + context_frames: int, +) -> tuple[torch.Tensor, SeamDecision, torch.Tensor | None]: + rewind, legacy, selected = choose_adaptive_cut( + previous_images, + next_raw_images, + context_frames=context_frames, + ) + cut_index = int(context_frames) - rewind + previous_stop = int(previous_images.shape[0]) - rewind + if previous_stop < 2 or cut_index < 0: + raise ValueError("adaptive cut would exhaust a seam side") + + current = next_raw_images[cut_index:].clone().contiguous() + kept_previous = previous_images[:previous_stop] + current, gain, bias, corrected = apply_color_guard(kept_previous, current) + + blend_frames = min( + choose_video_blend(selected), + cut_index, + previous_stop, + ) + video_patch: torch.Tensor | None = None + if blend_frames > 0: + video_patch = apply_video_blend( + kept_previous[-blend_frames:], + next_raw_images[cut_index - blend_frames : cut_index], + blend_frames=blend_frames, + ) + score_previous = kept_previous[-2:].clone() + replaced = min(blend_frames, 2) + score_previous[-replaced:].copy_(video_patch[-replaced:]) + corrected = _boundary_metrics(score_previous, current[:2]) + + improvement = (legacy.score - corrected.score) / max(legacy.score, 1e-12) + if corrected.score > legacy.score * 1.01: + fallback = next_raw_images[int(context_frames) :].clone().contiguous() + return ( + fallback, + SeamDecision( + legacy_score=legacy.score, + cut_score=legacy.score, + corrected_score=legacy.score, + fallback_reason="corrected score was worse than legacy", + ), + None, + ) + + decision = SeamDecision( + cut_rewind_frames=rewind, + legacy_score=legacy.score, + cut_score=selected.score, + corrected_score=corrected.score, + improvement=improvement, + video_blend_frames=blend_frames, + luma_gain=gain, + chroma_bias=bias, + ) + return current, decision, video_patch + + +def _audio_mid(waveform: torch.Tensor) -> torch.Tensor: + if not torch.is_tensor(waveform) or waveform.ndim != 3: + raise ValueError("audio seam waveform must be [B,C,S]") + value = waveform.detach().to(device="cpu", dtype=torch.float32) + if not bool(torch.isfinite(value).all().item()): + raise ValueError("audio seam waveform contains NaN or Inf") + return value.mean(dim=(0, 1)) + + +def _correlation(a: torch.Tensor, b: torch.Tensor) -> float: + length = min(int(a.numel()), int(b.numel())) + if length < 8: + return 0.0 + a = a[:length] - a[:length].mean() + b = b[:length] - b[:length].mean() + denominator = torch.sqrt(torch.sum(a.square()) * torch.sum(b.square())).clamp_min(1e-12) + value = float((torch.sum(a * b) / denominator).item()) + return max(-1.0, min(1.0, value)) if math.isfinite(value) else 0.0 + + +def analyze_audio_seam( + previous_waveform: torch.Tensor, + next_raw_waveform: torch.Tensor, + *, + sample_rate: int, + cut_sample: int, +) -> AudioSeamMetrics: + sample_rate = int(sample_rate) + cut_sample = int(cut_sample) + previous = _audio_mid(previous_waveform) + current = _audio_mid(next_raw_waveform) + max_lag = max(1, int(round(sample_rate * 0.020))) + window = min( + max(8, int(round(sample_rate * 0.080))), + int(previous.shape[-1]), + max(0, cut_sample - max_lag), + ) + if sample_rate <= 0 or window < 8: + raise ValueError("insufficient audio for seam alignment") + previous_tail = previous[-window:] + + def correlation_at(lag: int) -> float | None: + stop = cut_sample + int(lag) + start = stop - window + if start < 0 or stop > current.shape[-1]: + return None + return _correlation(previous_tail, current[start:stop]) + + baseline = correlation_at(0) + if baseline is None: + raise ValueError("audio seam cut is outside the decoded waveform") + best_lag, best_correlation = 0, baseline + for lag in range(-max_lag, max_lag + 1): + correlation = correlation_at(lag) + if correlation is not None and correlation > best_correlation: + best_lag, best_correlation = lag, correlation + + if best_correlation < 0.20 or best_correlation - baseline < 0.02: + best_lag, best_correlation = 0, baseline + else: + center = max(0, min(int(current.shape[-1]) - 1, cut_sample + best_lag)) + radius = max(1, int(round(sample_rate * 0.001))) + start = max(0, center - radius) + stop = min(int(current.shape[-1]), center + radius + 1) + if stop > start: + zero = int(torch.argmin(torch.abs(current[start:stop])).item()) + start + best_lag = zero - cut_sample + + before_jump = float( + torch.mean( + torch.abs(previous_waveform[..., -1] - next_raw_waveform[..., cut_sample]) + ).item() + ) + return AudioSeamMetrics( + correlation_before=baseline, + correlation_after=best_correlation, + boundary_jump_before=before_jump, + boundary_jump_after=before_jump, + offset_samples=int(best_lag), + ) + + +def _audio_crossfade_ms( + previous_waveform: torch.Tensor, + next_raw_waveform: torch.Tensor, + *, + cut_sample: int, + sample_rate: int, +) -> int: + probe = max(8, int(round(sample_rate * 0.020))) + if previous_waveform.shape[-1] < probe or cut_sample < probe: + return 10 + previous = _audio_mid(previous_waveform)[-probe:] + current = _audio_mid(next_raw_waveform)[cut_sample - probe : cut_sample] + rms = torch.sqrt(torch.mean(torch.cat((previous, current)).square())).clamp_min(1e-6) + transient = max( + float(torch.mean(torch.abs(torch.diff(previous))).item()), + float(torch.mean(torch.abs(torch.diff(current))).item()), + ) / float(rms.item()) + if transient > 0.80: + return 10 + if transient > 0.35: + return 20 + if transient > 0.15: + return 40 + return 60 + + +def apply_audio_seam( + previous_waveform: torch.Tensor, + next_raw_waveform: torch.Tensor, + metrics: AudioSeamMetrics, + *, + sample_rate: int, + cut_sample: int, +) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: + if metrics.correlation_after < 0.20: + return None, metrics, 0, 1.0, 0.0 + fade_ms = _audio_crossfade_ms( + previous_waveform, + next_raw_waveform, + cut_sample=cut_sample, + sample_rate=sample_rate, + ) + fade_samples = min( + int(round(sample_rate * fade_ms / 1000.0)), + int(previous_waveform.shape[-1]), + int(cut_sample), + ) + if fade_samples < 2: + return None, metrics, 0, 1.0, 0.0 + + progress = torch.linspace( + 0.0, + 1.0, + fade_samples, + dtype=torch.float32, + device="cpu", + ) + base_indices = torch.arange(cut_sample - fade_samples, cut_sample, dtype=torch.long) + offsets = torch.round( + float(metrics.offset_samples) * (1.0 - progress) + ).to(dtype=torch.long) + indices = (base_indices + offsets).clamp(0, int(next_raw_waveform.shape[-1]) - 1) + current = next_raw_waveform.detach().to("cpu").index_select(-1, indices) + previous = previous_waveform[..., -fade_samples:].detach().to("cpu") + + previous_rms = torch.sqrt(torch.mean(previous.float().square())).clamp_min(1e-6) + current_rms = torch.sqrt(torch.mean(current.float().square())).clamp_min(1e-6) + requested_db = 20.0 * math.log10(float((previous_rms / current_rms).item())) + level_db = max(-1.5, min(1.5, requested_db)) if abs(requested_db) <= 6.0 else 0.0 + level_gain = 10.0 ** (level_db / 20.0) + requested_dc = float((previous.float().mean() - current.float().mean()).item()) + dc_bias = max(-0.02, min(0.02, requested_dc)) if abs(requested_dc) <= 0.10 else 0.0 + current = current * level_gain + dc_bias + + shape = [1] * (previous.ndim - 1) + [fade_samples] + fade_in = torch.sin(progress * (math.pi / 2.0)).reshape(shape) + fade_out = torch.cos(progress * (math.pi / 2.0)).reshape(shape) + patch = previous * fade_out + current.to(dtype=previous.dtype) * fade_in + input_peak = max( + float(previous.abs().max().item()), + float(current.abs().max().item()), + 1e-6, + ) + patch_peak = float(patch.abs().max().item()) + if patch_peak > input_peak * 1.05: + patch = patch * ((input_peak * 1.05) / patch_peak) + after_jump = float( + torch.mean(torch.abs(patch[..., -1] - next_raw_waveform[..., cut_sample])).item() + ) + updated = AudioSeamMetrics( + correlation_before=metrics.correlation_before, + correlation_after=metrics.correlation_after, + boundary_jump_before=metrics.boundary_jump_before, + boundary_jump_after=after_jump, + offset_samples=metrics.offset_samples, + ) + return patch.contiguous(), updated, fade_samples, level_gain, dc_bias + + +def correct_audio_seam( + previous_waveform: torch.Tensor, + next_raw_waveform: torch.Tensor, + *, + sample_rate: int, + cut_sample: int, +) -> tuple[torch.Tensor | None, AudioSeamMetrics, int, float, float]: + metrics = analyze_audio_seam( + previous_waveform, + next_raw_waveform, + sample_rate=sample_rate, + cut_sample=cut_sample, + ) + return apply_audio_seam( + previous_waveform, + next_raw_waveform, + metrics, + sample_rate=sample_rate, + cut_sample=cut_sample, + ) diff --git a/v2/seam_types.py b/v2/seam_types.py new file mode 100644 index 0000000..ead805d --- /dev/null +++ b/v2/seam_types.py @@ -0,0 +1,42 @@ +"""Transient types used by V2.1 decoded seam correction.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class VideoSeamMetrics: + pixel_mae_norm: float + luma_norm: float + chroma_norm: float + edge_norm: float + temporal_norm: float + motion: float + score: float + + +@dataclass(frozen=True) +class AudioSeamMetrics: + correlation_before: float = 0.0 + correlation_after: float = 0.0 + boundary_jump_before: float = 0.0 + boundary_jump_after: float = 0.0 + offset_samples: int = 0 + + +@dataclass +class SeamDecision: + cut_rewind_frames: int = 0 + legacy_score: float = 0.0 + cut_score: float = 0.0 + corrected_score: float = 0.0 + improvement: float = 0.0 + video_blend_frames: int = 0 + luma_gain: float = 1.0 + chroma_bias: tuple[float, float, float] = (0.0, 0.0, 0.0) + audio_metrics: AudioSeamMetrics = AudioSeamMetrics() + audio_crossfade_samples: int = 0 + audio_level_gain: float = 1.0 + audio_dc_bias: float = 0.0 + fallback_reason: str | None = None diff --git a/v2/seeds.py b/v2/seeds.py new file mode 100644 index 0000000..7cec883 --- /dev/null +++ b/v2/seeds.py @@ -0,0 +1,36 @@ +"""Deterministic per-chunk seed derivation.""" + +from __future__ import annotations + +_MASK64 = (1 << 64) - 1 +_GOLDEN = 0x9E3779B97F4A7C15 +_MIX_A = 0xBF58476D1CE4E5B9 +_MIX_B = 0x94D049BB133111EB + + +def _splitmix64(value: int) -> int: + value = (value + _GOLDEN) & _MASK64 + value = ((value ^ (value >> 30)) * _MIX_A) & _MASK64 + value = ((value ^ (value >> 27)) * _MIX_B) & _MASK64 + return (value ^ (value >> 31)) & _MASK64 + + +def derive_chunk_seed(base_seed: int, chunk_index: int, reroll_nonce: int = 0) -> int: + """Return a stable 64-bit seed for a zero-based chunk index. + + The index and reroll nonce are mixed independently so rerolling chunk N does + not alter accepted chunks before N. + """ + + if isinstance(base_seed, bool) or isinstance(chunk_index, bool) or isinstance(reroll_nonce, bool): + raise TypeError("seed inputs must be integers, not booleans") + chunk_index = int(chunk_index) + reroll_nonce = int(reroll_nonce) + if chunk_index < 0: + raise ValueError("chunk_index must be non-negative") + if reroll_nonce < 0: + raise ValueError("reroll_nonce must be non-negative") + value = int(base_seed) & _MASK64 + value ^= _splitmix64(chunk_index + 1) + value ^= _splitmix64((reroll_nonce + 1) * 0xD6E8FEB86659FD93) + return _splitmix64(value) diff --git a/v2/sequence.py b/v2/sequence.py new file mode 100644 index 0000000..bb587c6 --- /dev/null +++ b/v2/sequence.py @@ -0,0 +1,540 @@ +"""Integrated N-chunk Continuum runtime. + +V2 deliberately reuses the proven V1 continuation core. It changes execution +orchestration only: all text/image conditioning is prepared first, H3 sampling +runs for every chunk without intermediate VAE decoding, and decoding/assembly +happens after the sampling phase. +""" + +from __future__ import annotations + +import copy +import logging +from typing import Any + +import torch + +from ..compatibility import accelerator_summary, check_comfy_h3_runtime +from ..constants import ( + DIAGNOSTICS_FULL, + DIAGNOSTICS_OFF, + DIAGNOSTICS_OPTIONS, + FPS, + CONTINUUM_ACTUAL_PREFIX_STEPS, + SEAM_CORRECTION_OFF, + SEAM_CORRECTION_OPTIONS, + V2_CONTINUITY_AUTO, +) +from ..continuation import POLICY_REPLACE, prepare_conditioning +from ..model_patch import clone_model_for_chunk, patch_model +from ..state import ( + assert_context_unchanged, + context_fingerprint, + make_plan, + select_context, + validate_state, +) +from ..temporal import ( + align_frame_count_up, + largest_context_capacity, + make_extension_shape, +) +from ..version import PACKAGE_VERSION +from .decoder import decode_sequence, decode_sequence_with_seam, enforce_total_frames +from .h3_builder import ( + attach_keyframes, + empty_h3_latent, + encode_identity_latents, + encode_prompt_conditioning, + prepare_identity_assets, +) +from .motion import choose_context_frames +from .prompts import prompt_plan_report, validate_prompt_plan +from .sampling import sample_chunk +from .seeds import derive_chunk_seed +from .session import ( + SessionValidationError, + entry_to_state, + make_chunk_entry, + make_session, + model_fingerprint, + session_summary, + validate_chunk_entry, + validate_session, +) + +LOG = logging.getLogger("h3_continuum_join") + + +class SequenceRuntimeError(RuntimeError): + pass + + +def _check_decode_memory_budget( + *, width: int, height: int, chunks: int, chunk_seconds: float +) -> tuple[float, float | None]: + """Fail before sampling when the eventual CPU IMAGE output is unsafe. + + ComfyUI IMAGE tensors are normally float32. V2 preallocates the final output + once, but one raw VAE chunk and normal process headroom still coexist with it. + The estimate is conservative and intentionally protects long/high-resolution + jobs from failing only after every expensive H3 chunk has completed. + """ + + target_frames = max(1, int(round(chunks * chunk_seconds * FPS))) + max_chunk_frames = max(5, int(round(chunk_seconds * FPS)) + 39) + image_bytes = target_frames * width * height * 3 * 4 + transient_bytes = max_chunk_frames * width * height * 3 * 4 + required_bytes = image_bytes + int(transient_bytes * 1.5) + 1 * 1024**3 + estimate_gib = required_bytes / 1024**3 + available_gib: float | None = None + try: + import psutil + + available = int(psutil.virtual_memory().available) + available_gib = available / 1024**3 + if required_bytes > int(available * 0.75): + raise SequenceRuntimeError( + "estimated decoded output requires about " + f"{estimate_gib:.1f} GiB including one-chunk headroom, but only " + f"{available_gib:.1f} GiB RAM is currently available. Reduce chunks, " + "chunk_seconds, or resolution before sampling." + ) + except ImportError: + pass + return estimate_gib, available_gib + + +def _clone_entry_for_reuse(entry: dict[str, Any]) -> dict[str, Any]: + entry = validate_chunk_entry(entry) + result = dict(entry) + result["plan"] = copy.deepcopy(entry["plan"]) + result["reused"] = True + return result + + +def _preserved_prefix( + *, + session: dict[str, Any] | None, + prompt_hashes: list[str], + chunks: int, + reroll_from_chunk: int, + width: int, + height: int, + chunk_seconds: float, + identity_hash: str, +) -> tuple[list[dict[str, Any]], list[str]]: + if session is None: + return [], [] + session = validate_session(session) + notes: list[str] = [] + if int(session["width"]) != int(width) or int(session["height"]) != int(height): + raise SessionValidationError( + f"session is {session['width']}x{session['height']} but V2 is configured for {width}x{height}" + ) + if abs(float(session["chunk_seconds"]) - float(chunk_seconds)) > 1e-6: + raise SessionValidationError( + f"session chunk_seconds={session['chunk_seconds']} does not match {chunk_seconds}" + ) + if str(session.get("identity_hash", "none")) != str(identity_hash): + raise SessionValidationError( + "the current first-frame identity image does not match the saved session" + ) + + if reroll_from_chunk < 0 or reroll_from_chunk > chunks: + raise SessionValidationError("reroll_from_chunk must be 0 or a valid one-based chunk index") + limit = min(len(session["chunks"]), chunks) + if reroll_from_chunk > 0: + limit = min(limit, reroll_from_chunk - 1) + + preserved: list[dict[str, Any]] = [] + for index in range(limit): + entry = validate_chunk_entry(session["chunks"][index]) + if entry["prompt_hash"] != prompt_hashes[index]: + notes.append( + f"session reuse stopped before chunk {index + 1}: prompt changed" + ) + break + preserved.append(_clone_entry_for_reuse(entry)) + if reroll_from_chunk == 0 and len(preserved) == min(len(session["chunks"]), chunks): + if len(session["chunks"]) < chunks: + notes.append(f"resuming after accepted chunk {len(session['chunks'])}") + else: + notes.append("all requested chunks reused from the session") + elif reroll_from_chunk > 0: + notes.append( + f"preserved chunks 1-{len(preserved)}; regenerated from chunk {reroll_from_chunk}" + ) + return preserved, notes + + +def _conditioning_cache( + *, + clip: Any, + prompts: list[str], + assets: Any, + final_has_last_frame: bool, +) -> dict[tuple[str, bool], list]: + cache: dict[tuple[str, bool], list] = {} + final_index = len(prompts) - 1 + for index, prompt in enumerate(prompts): + include_last = bool(final_has_last_frame and index == final_index) + key = (prompt, include_last) + if key in cache: + continue + cache[key] = encode_prompt_conditioning( + clip, + prompt, + first_image=assets.first_image, + last_image=assets.last_image if include_last else None, + ) + return cache + + +def run_sequence( + *, + model: Any, + clip: Any, + video_vae: Any, + audio_vae: Any, + sampler: Any, + sigmas: torch.Tensor, + first_frame: torch.Tensor | None, + last_frame: torch.Tensor | None, + prompt_plan: dict[str, Any], + width: int, + height: int, + continuity: str, + base_seed: int, + audio_continuity: bool, + exact_total_duration: bool, + diagnostics_mode: str, + reroll_from_chunk: int, + reroll_nonce: int, + strict_compatibility: bool, + debug: bool, + seam_correction: str = SEAM_CORRECTION_OFF, + enable_preview: bool = True, + session: dict[str, Any] | None = None, + initial_state: dict[str, Any] | None = None, +) -> tuple[torch.Tensor, dict[str, Any], dict[str, Any], dict[str, Any], str]: + """Generate or resume a complete Continuum sequence.""" + + plan = validate_prompt_plan(prompt_plan) + chunks = int(plan["chunks"]) + chunk_seconds = float(plan["chunk_seconds"]) + prompts = list(plan["prompts"]) + prompt_hashes = list(plan["hashes"]) + width, height = int(width), int(height) + if width <= 0 or height <= 0 or width % 32 or height % 32: + raise SequenceRuntimeError("width and height must be positive multiples of 32") + if session is not None and initial_state is not None: + raise SequenceRuntimeError("connect either a session or an initial_state, not both") + if diagnostics_mode not in DIAGNOSTICS_OPTIONS: + raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}") + if seam_correction not in SEAM_CORRECTION_OPTIONS: + raise SequenceRuntimeError(f"unknown seam correction mode: {seam_correction!r}") + if not (0 <= int(reroll_from_chunk) <= chunks): + raise SequenceRuntimeError( + "reroll_from_chunk must be 0 or a valid one-based chunk index" + ) + if initial_state is not None and int(reroll_from_chunk) not in (0, 1): + raise SequenceRuntimeError( + "with initial_state, reroll_from_chunk can only be 0 or 1" + ) + decode_estimate_gib, available_ram_gib = _check_decode_memory_budget( + width=width, + height=height, + chunks=chunks, + chunk_seconds=chunk_seconds, + ) + + issues = check_comfy_h3_runtime() + if issues and strict_compatibility: + raise SequenceRuntimeError("H3 runtime is incompatible: " + "; ".join(issues)) + + # Phase 1: all CLIP and VAE encoding happens before H3 sampling. This is the + # key residency optimization: no text encoder or VAE is requested between + # H3 chunks. + assets = prepare_identity_assets( + video_vae, + width=width, + height=height, + first_frame=first_frame, + last_frame=last_frame, + encode_latents=False, + ) + + patched_model = patch_model( + model, + strict=bool(strict_compatibility), + debug=bool(debug), + ) + current_model_fingerprint = model_fingerprint(patched_model) + preserved, reuse_notes = _preserved_prefix( + session=session, + prompt_hashes=prompt_hashes, + chunks=chunks, + reroll_from_chunk=int(reroll_from_chunk), + width=width, + height=height, + chunk_seconds=chunk_seconds, + identity_hash=assets.identity_hash, + ) + if session is not None: + old_fingerprint = str(session.get("model_fingerprint", "")) + if old_fingerprint and old_fingerprint != current_model_fingerprint: + reuse_notes.append( + "model/accelerator fingerprint differs from the saved session; accepted chunks were kept" + ) + + cache: dict[tuple[str, bool], list] = {} + if len(preserved) < chunks: + assets = encode_identity_latents(video_vae, assets) + cache = _conditioning_cache( + clip=clip, + prompts=prompts, + assets=assets, + final_has_last_frame=last_frame is not None, + ) + + entries: list[dict[str, Any]] = preserved[:] + previous_state = None + if entries: + previous_state = entry_to_state(entries[-1]) + elif initial_state is not None: + previous_state = validate_state(initial_state) + if int(previous_state["width"]) != width or int(previous_state["height"]) != height: + raise SequenceRuntimeError( + f"initial_state is {previous_state['width']}x{previous_state['height']} " + f"but V2 is configured for {width}x{height}" + ) + + initial_frame_count = align_frame_count_up(int(round(chunk_seconds * FPS))) + retained_frames = sum(int(entry["plan"]["net_frames"]) for entry in entries) + sampling_reports: list[str] = [] + + # Phase 2: H3-only loop. Each chunk gets a MODEL clone so accelerator runtime + # history cannot leak across chunks. Sampler/sigmas remain shared, and decoded + # media is deliberately not requested until every chunk is done. + for sequence_index in range(len(entries), chunks): + prompt = prompts[sequence_index] + prompt_hash_value = prompt_hashes[sequence_index] + is_final = sequence_index == chunks - 1 + seed = derive_chunk_seed(base_seed, sequence_index, reroll_nonce) + motion_score = 0.0 + video_context = None + audio_context = None + context_before = None + + if previous_state is None: + total_frames = initial_frame_count + latent = empty_h3_latent(width, height, total_frames) + conditioning = attach_keyframes( + cache[(prompt, bool(last_frame is not None and is_final))], + frame_count=total_frames, + first_latent=assets.first_latent, + last_latent=assets.last_latent if is_final else None, + ) + clip_index = 1 + context_frames = 0 + chunk_plan = make_plan( + continuation=False, + clip_index=clip_index, + total_frames=total_frames, + trim_frames=0, + width=width, + height=height, + context_frames=5, + state_capacity_frames=largest_context_capacity(total_frames), + requested_extend_seconds=chunk_seconds, + debug=debug, + ) + reason = "initial clip" + else: + context_frames, motion_score, reason = choose_context_frames( + continuity, previous_state + ) + desired_cumulative = int(round((sequence_index + 1) * chunk_seconds * FPS)) + requested_new_frames = max(1, desired_cumulative - retained_frames) + shape = make_extension_shape( + context_frames, requested_new_frames / FPS + ) + latent = empty_h3_latent(width, height, shape.total_frames) + base_conditioning = attach_keyframes( + cache[(prompt, bool(last_frame is not None and is_final))], + frame_count=shape.total_frames, + first_latent=assets.first_latent, + last_latent=assets.last_latent if is_final else None, + ) + video_context, audio_context, grid_offset = select_context( + previous_state, + context_frames, + include_audio=bool(audio_continuity), + ) + context_before = context_fingerprint(video_context, audio_context) + conditioning = prepare_conditioning( + base_conditioning, + video_context=video_context, + audio_context=audio_context, + audio_grid_offset=grid_offset, + context_frames=context_frames, + new_frame_count=shape.total_frames, + first_frame_policy=POLICY_REPLACE, + preserve_last_frame=True, + ) + clip_index = int(previous_state["clip_index"]) + 1 + chunk_plan = make_plan( + continuation=True, + clip_index=clip_index, + total_frames=shape.total_frames, + trim_frames=context_frames, + width=width, + height=height, + context_frames=context_frames, + state_capacity_frames=largest_context_capacity(shape.net_new_frames), + requested_extend_seconds=chunk_seconds, + debug=debug, + ) + + chunk_model = clone_model_for_chunk( + patched_model, + strict=bool(strict_compatibility), + debug=bool(debug), + chunk_index=clip_index, + context_frames=context_frames if previous_state is not None else None, + ) + sampled = sample_chunk( + model=chunk_model, + conditioning=conditioning, + latent=latent, + sampler=sampler, + sigmas=sigmas, + seed=seed, + enable_preview=bool(enable_preview), + ) + if context_before is not None and video_context is not None: + assert_context_unchanged( + video_context, + audio_context, + context_before, + ) + # Commit the full chunk to CPU once, then derive the small next-state tail + # from that CPU entry. This avoids a second device-to-host transfer. + entry = make_chunk_entry( + latent=sampled, + plan=chunk_plan, + prompt=prompt, + prompt_hash=prompt_hash_value, + seed=seed, + context_frames=context_frames, + motion_score=motion_score, + reused=False, + ) + previous_state = entry_to_state(entry) + entries.append(entry) + retained_frames += int(chunk_plan["net_frames"]) + sampling_reports.append( + f"chunk {sequence_index + 1}/{chunks}: seed={seed}, " + f"frames={chunk_plan['total_frames']}, trim={chunk_plan['trim_frames']}, " + f"retained_total={retained_frames}, context={context_frames} " + f"({reason}), motion={motion_score:.6f}, " + + ( + f"interop=emitted actual_prefix={CONTINUUM_ACTUAL_PREFIX_STEPS} " + "consumer=not_observable" + if context_before is not None + else "interop=not_emitted" + ) + ) + del sampled, latent, conditioning, chunk_model + + if len(entries) != chunks: + raise SequenceRuntimeError( + f"internal sequence length mismatch: expected {chunks}, got {len(entries)}" + ) + + # Phase 3: VAE decode only after all H3 sampling is complete. + if seam_correction == SEAM_CORRECTION_OFF: + images, audio, decode_reports = decode_sequence( + entries=entries, + video_vae=video_vae, + audio_vae=audio_vae, + diagnostics_full=diagnostics_mode == DIAGNOSTICS_FULL, + ) + else: + images, audio, decode_reports = decode_sequence_with_seam( + entries=entries, + video_vae=video_vae, + audio_vae=audio_vae, + diagnostics_mode=diagnostics_mode, + ) + duration_report = "" + if exact_total_duration: + target_frames = int(round(chunks * chunk_seconds * FPS)) + images, audio, duration_report = enforce_total_frames( + images, + audio, + target_frames=target_frames, + preserve_final_frame=last_frame is not None, + ) + + last_state = entry_to_state(entries[-1]) + parent_id = session.get("session_id") if session is not None else None + settings = { + "continuity": continuity, + "audio_continuity": bool(audio_continuity), + "exact_total_duration": bool(exact_total_duration), + "prompt_mode": plan["mode"], + "base_seed": int(base_seed), + "reroll_nonce": int(reroll_nonce), + "diagnostics_mode": diagnostics_mode, + "initial_state_source": initial_state is not None, + } + new_session = make_session( + chunks=entries, + width=width, + height=height, + chunk_seconds=chunk_seconds, + identity_hash=assets.identity_hash, + model_fingerprint_value=current_model_fingerprint, + parent_session_id=parent_id, + reroll_from_chunk=int(reroll_from_chunk), + settings=settings, + ) + + decoded_gib = ( + float(images.shape[0]) * float(width) * float(height) * 3.0 * 4.0 / (1024.0 ** 3) + ) + report_lines = [ + f"H3 Continuum V2 {PACKAGE_VERSION}", + prompt_plan_report(plan), + f"Seam correction: {seam_correction}.", + accelerator_summary(patched_model), + "Execution: conditioning precomputed; call-local MODEL clone per chunk; decode deferred until sampling completed.", + *reuse_notes, + ] + if diagnostics_mode != DIAGNOSTICS_OFF: + report_lines.extend( + [ + ( + f"Decode RAM estimate: {decode_estimate_gib:.2f} GiB including transient headroom" + + ( + f"; available at start {available_ram_gib:.2f} GiB." + if available_ram_gib is not None + else "." + ) + ), + *sampling_reports, + *decode_reports, + ] + ) + if duration_report: + report_lines.append(duration_report) + report_lines.extend( + [ + session_summary(new_session), + f"Output: {images.shape[0]} frames ({images.shape[0]/FPS:.3f}s), " + f"audio samples={audio['waveform'].shape[-1]}, decoded tensor≈{decoded_gib:.2f} GiB.", + ] + ) + return images, audio, last_state, new_session, "\n".join(report_lines) diff --git a/v2/session.py b/v2/session.py new file mode 100644 index 0000000..e7747f1 --- /dev/null +++ b/v2/session.py @@ -0,0 +1,261 @@ +"""In-memory session model for accepted chunks, resume, and branch-safe rerolls.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import uuid +from datetime import datetime, timezone +from typing import Any + +import torch + +from ..constants import SESSION_MAGIC, STATE_MAGIC +from ..state import validate_plan +from ..temporal import ( + audio_grid_offset, + audio_latent_t, + context_slots, + is_valid_frame_count, + pixel_frames_for_latent_t, + video_latent_t, +) +from ..version import PACKAGE_VERSION, SESSION_SCHEMA_VERSION, STATE_SCHEMA_VERSION +from .sampling import latent_from_cpu, latent_to_cpu + + +class SessionValidationError(ValueError): + pass + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def model_fingerprint(model: Any) -> str: + base = getattr(model, "model", None) + inner = getattr(base, "diffusion_model", None) + options = getattr(model, "model_options", {}) or {} + transformer = options.get("transformer_options", {}) or {} + wrappers = getattr(model, "wrappers", {}) or {} + wrapper_keys: list[str] = [] + if isinstance(wrappers, dict): + for group in wrappers.values(): + if isinstance(group, dict): + wrapper_keys.extend(str(key) for key in group) + descriptor = { + "base": f"{type(base).__module__}.{type(base).__name__}" if base is not None else "missing", + "inner": f"{type(inner).__module__}.{type(inner).__name__}" if inner is not None else "missing", + "dtype": str(getattr(model, "model_dtype", lambda: "unknown")()), + "model_size": int(getattr(model, "model_size", lambda: 0)() or 0), + "wrappers": sorted(wrapper_keys), + "has_attention_override": transformer.get("optimized_attention_override") is not None, + "has_sol_compose": transformer.get("sol_compose") is not None, + } + raw = json.dumps(descriptor, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def make_chunk_entry( + *, + latent: dict[str, Any], + plan: dict[str, Any], + prompt: str, + prompt_hash: str, + seed: int, + context_frames: int, + motion_score: float, + reused: bool, +) -> dict[str, Any]: + plan = validate_plan(plan) + video, audio = latent_to_cpu(latent) + actual_frames = pixel_frames_for_latent_t(int(video.shape[2])) + if actual_frames != int(plan["total_frames"]): + raise SessionValidationError( + f"chunk latent represents {actual_frames} frames but plan declares {plan['total_frames']}" + ) + return { + "sequence_index": 0, # populated by make_session + "clip_index": int(plan["clip_index"]), + "prompt": str(prompt), + "prompt_hash": str(prompt_hash), + "seed": int(seed), + "context_frames": int(context_frames), + "motion_score": float(motion_score), + "reused": bool(reused), + "plan": copy.deepcopy(plan), + "video": video, + "audio": audio, + } + + +def validate_chunk_entry(entry: dict[str, Any]) -> dict[str, Any]: + if not isinstance(entry, dict): + raise SessionValidationError("session chunk must be a dictionary") + video, audio = entry.get("video"), entry.get("audio") + if not torch.is_tensor(video) or video.ndim != 5 or tuple(video.shape[:2]) != (1, 24): + raise SessionValidationError("session chunk video must be [1,24,T,H,W]") + if not torch.is_tensor(audio) or audio.ndim != 4 or tuple(audio.shape[:3]) != (1, 32, 2): + raise SessionValidationError("session chunk audio must be [1,32,2,T]") + if video.device.type != "cpu" or audio.device.type != "cpu": + raise SessionValidationError("session tensors must remain on CPU") + if not bool(torch.isfinite(video.float()).all().item()): + raise SessionValidationError("session video contains NaN or Inf") + if not bool(torch.isfinite(audio.float()).all().item()): + raise SessionValidationError("session audio contains NaN or Inf") + plan = validate_plan(entry.get("plan")) + actual_frames = pixel_frames_for_latent_t(int(video.shape[2])) + if actual_frames != int(plan["total_frames"]): + raise SessionValidationError("session chunk video length does not match its plan") + if not isinstance(entry.get("prompt"), str) or not entry["prompt"].strip(): + raise SessionValidationError("session chunk prompt is empty") + if not isinstance(entry.get("prompt_hash"), str) or len(entry["prompt_hash"]) != 64: + raise SessionValidationError("session chunk prompt hash is invalid") + if int(entry.get("seed", -1)) < 0: + raise SessionValidationError("session chunk seed is invalid") + if int(entry.get("context_frames", 0)) not in (0, 5, 22, 39): + raise SessionValidationError("session chunk context_frames is invalid") + return entry + + +def make_session( + *, + chunks: list[dict[str, Any]], + width: int, + height: int, + chunk_seconds: float, + identity_hash: str, + model_fingerprint_value: str, + parent_session_id: str | None, + reroll_from_chunk: int, + settings: dict[str, Any], +) -> dict[str, Any]: + session_id = uuid.uuid4().hex + normalized: list[dict[str, Any]] = [] + for index, entry in enumerate(chunks, start=1): + item = dict(entry) + item["plan"] = copy.deepcopy(entry["plan"]) + item["sequence_index"] = index + validate_chunk_entry(item) + normalized.append(item) + return { + "magic": SESSION_MAGIC, + "schema_version": SESSION_SCHEMA_VERSION, + "package_version": PACKAGE_VERSION, + "session_id": session_id, + "parent_session_id": parent_session_id, + "created_utc": _now_iso(), + "reroll_from_chunk": int(reroll_from_chunk), + "width": int(width), + "height": int(height), + "chunk_seconds": float(chunk_seconds), + "identity_hash": str(identity_hash), + "model_fingerprint": str(model_fingerprint_value), + "settings": copy.deepcopy(settings), + "chunks": normalized, + } + + +def validate_session(session: dict[str, Any]) -> dict[str, Any]: + if not isinstance(session, dict) or session.get("magic") != SESSION_MAGIC: + raise SessionValidationError("invalid H3 Continuum session") + if int(session.get("schema_version", -1)) != SESSION_SCHEMA_VERSION: + raise SessionValidationError( + f"unsupported session schema {session.get('schema_version')}; " + f"expected {SESSION_SCHEMA_VERSION}" + ) + if int(session.get("width", 0)) <= 0 or int(session.get("height", 0)) <= 0: + raise SessionValidationError("session dimensions are invalid") + if float(session.get("chunk_seconds", 0.0)) <= 0: + raise SessionValidationError("session chunk_seconds is invalid") + chunks = session.get("chunks") + if not isinstance(chunks, list) or not chunks: + raise SessionValidationError("session contains no chunks") + previous_clip_index = None + for index, entry in enumerate(chunks, start=1): + validate_chunk_entry(entry) + if int(entry.get("sequence_index", index)) != index: + raise SessionValidationError("session chunk sequence indices are not contiguous") + clip_index = int(entry["plan"]["clip_index"]) + if previous_clip_index is not None and clip_index != previous_clip_index + 1: + raise SessionValidationError("session clip indices are not contiguous") + previous_clip_index = clip_index + video = entry["video"] + if int(video.shape[-1]) * 16 != int(session["width"]): + raise SessionValidationError("session width does not match chunk latent") + if int(video.shape[-2]) * 16 != int(session["height"]): + raise SessionValidationError("session height does not match chunk latent") + return session + + +def entry_to_latent(entry: dict[str, Any]) -> dict[str, Any]: + entry = validate_chunk_entry(entry) + return latent_from_cpu(entry["video"], entry["audio"]) + + +def entry_to_state(entry: dict[str, Any], *, capacity_frames: int | None = None) -> dict[str, Any]: + """Build a V1-compatible continuation state directly from a CPU chunk. + + This deliberately avoids reconstructing a ComfyUI ``NestedTensor``. Session + inspection, persistence tests, and branch selection therefore remain usable + outside a live ComfyUI process, while the returned state is byte-for-byte + compatible with the V1 state contract. + """ + + entry = validate_chunk_entry(entry) + plan = entry["plan"] + video = entry["video"] + audio = entry["audio"] + source_frames = int(plan["total_frames"]) + clip_index = int(plan["clip_index"]) + capacity = int(capacity_frames or plan["state_capacity_frames"]) + + actual_video_t = int(video.shape[2]) + actual_frames = pixel_frames_for_latent_t(actual_video_t) + if actual_frames != source_frames: + raise SessionValidationError( + f"chunk plan says {source_frames} frames but latent represents {actual_frames}" + ) + if not is_valid_frame_count(actual_frames) or video_latent_t(actual_frames) != actual_video_t: + raise SessionValidationError( + f"chunk video latent T={actual_video_t} is not on the native H3 temporal grid" + ) + + slots = context_slots(capacity) + audio_steps = audio_latent_t(capacity) + if int(video.shape[2]) < slots or int(audio.shape[-1]) < audio_steps: + raise SessionValidationError( + f"chunk is too short for a {capacity}-frame continuation state" + ) + grid_offset = audio_grid_offset(source_frames, int(audio.shape[-1])) + if not (-0.500001 <= grid_offset <= 0.500001): + raise SessionValidationError( + f"unexpected signed audio-grid offset {grid_offset:.6f}" + ) + + video_tail = video[:, :, -slots:].contiguous().clone() + audio_tail = audio[..., -audio_steps:].contiguous().clone() + return { + "magic": STATE_MAGIC, + "schema_version": STATE_SCHEMA_VERSION, + "clip_index": clip_index, + "source_frame_count": source_frames, + "capacity_frames": capacity, + "width": int(video.shape[-1]) * 16, + "height": int(video.shape[-2]) * 16, + "video_tail": video_tail, + "audio_tail": audio_tail, + "audio_overhang": float(grid_offset), + "source_mode": "latent_direct", + } + + +def session_summary(session: dict[str, Any]) -> str: + session = validate_session(session) + reused = sum(1 for item in session["chunks"] if item.get("reused")) + frames = sum(int(item["plan"]["net_frames"]) for item in session["chunks"]) + return ( + f"Session {session['session_id'][:8]}: {len(session['chunks'])} chunks, " + f"{frames} retained frames, {reused} reused, {session['width']}x{session['height']}." + ) diff --git a/v2/session_io.py b/v2/session_io.py new file mode 100644 index 0000000..c949f36 --- /dev/null +++ b/v2/session_io.py @@ -0,0 +1,142 @@ +"""Atomic safetensors + JSON persistence for V2 sessions.""" + +from __future__ import annotations + +import json +import logging +import os +import re +import uuid +from pathlib import Path +from typing import Any + +from safetensors import safe_open +from safetensors.torch import save_file + +from .session import validate_session + +LOG = logging.getLogger("h3_continuum_join") +_SAFE_PREFIX = re.compile(r"[^A-Za-z0-9._-]+") +_HEADER_KEY = "h3_continuum_session_json" + + +def sanitize_prefix(prefix: str) -> str: + value = _SAFE_PREFIX.sub("_", str(prefix).strip()).strip("._") + return value or "h3_continuum_session" + + +def session_directory() -> Path: + try: + import folder_paths + root = Path(folder_paths.get_output_directory()) + except Exception: + root = Path.cwd() + path = root / "h3_continuum_sessions" + path.mkdir(parents=True, exist_ok=True) + return path + + +def session_paths(prefix: str, slot: int) -> tuple[Path, Path]: + slot = int(slot) + if not (1 <= slot <= 9999): + raise ValueError("session slot must be between 1 and 9999") + name = f"{sanitize_prefix(prefix)}_slot{slot:04d}" + root = session_directory() + return root / f"{name}.safetensors", root / f"{name}.json" + + +def _fsync_file(path: Path) -> None: + try: + with path.open("rb") as handle: + os.fsync(handle.fileno()) + except OSError: + pass + + +def _fsync_directory(path: Path) -> None: + if os.name == "nt": + return + try: + descriptor = os.open(str(path), os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError: + pass + + +def _metadata_without_tensors(session: dict[str, Any]) -> dict[str, Any]: + metadata = {key: value for key, value in session.items() if key != "chunks"} + chunks = [] + for entry in session["chunks"]: + chunks.append({key: value for key, value in entry.items() if key not in ("video", "audio")}) + metadata["chunks"] = chunks + return metadata + + +def save_session(session: dict[str, Any], *, prefix: str, slot: int) -> tuple[Path, Path]: + session = validate_session(session) + tensor_path, json_path = session_paths(prefix, slot) + token = uuid.uuid4().hex + tensor_tmp = tensor_path.with_name(f".{tensor_path.name}.{token}.tmp") + json_tmp = json_path.with_name(f".{json_path.name}.{token}.tmp") + + tensors = {} + for index, entry in enumerate(session["chunks"], start=1): + tensors[f"chunk_{index:04d}_video"] = entry["video"].detach().to("cpu").contiguous() + tensors[f"chunk_{index:04d}_audio"] = entry["audio"].detach().to("cpu").contiguous() + metadata = _metadata_without_tensors(session) + metadata["_session_commit_id"] = token + serialized = json.dumps(metadata, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + try: + save_file(tensors, str(tensor_tmp), metadata={_HEADER_KEY: serialized}) + _fsync_file(tensor_tmp) + with json_tmp.open("w", encoding="utf-8", newline="\n") as handle: + json.dump(metadata, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(json_tmp, json_path) + os.replace(tensor_tmp, tensor_path) + _fsync_directory(tensor_path.parent) + finally: + for path in (tensor_tmp, json_tmp): + try: + path.unlink(missing_ok=True) + except OSError: + pass + return tensor_path, json_path + + +def load_session(*, prefix: str, slot: int) -> dict[str, Any]: + tensor_path, json_path = session_paths(prefix, slot) + if not tensor_path.is_file(): + raise FileNotFoundError(f"session slot does not exist: {tensor_path.name}") + with safe_open(str(tensor_path), framework="pt", device="cpu") as handle: + serialized = (handle.metadata() or {}).get(_HEADER_KEY) + if serialized is None: + raise ValueError("session safetensors has no embedded Continuum metadata") + session = json.loads(serialized) + for index, entry in enumerate(session.get("chunks") or [], start=1): + entry["video"] = handle.get_tensor(f"chunk_{index:04d}_video") + entry["audio"] = handle.get_tensor(f"chunk_{index:04d}_audio") + + if json_path.is_file(): + try: + with json_path.open("r", encoding="utf-8") as handle: + mirror = json.load(handle) + if mirror.get("_session_commit_id") != session.get("_session_commit_id"): + LOG.warning("Ignoring stale Continuum session JSON mirror: %s", json_path.name) + except (OSError, ValueError, TypeError) as exc: + LOG.warning("Ignoring unreadable session JSON mirror %s: %s", json_path.name, exc) + return validate_session(session) + + +def session_mtime(*, prefix: str, slot: int) -> tuple[float, float]: + tensor_path, json_path = session_paths(prefix, slot) + return ( + tensor_path.stat().st_mtime if tensor_path.exists() else -1.0, + json_path.stat().st_mtime if json_path.exists() else -1.0, + ) diff --git a/version.py b/version.py new file mode 100644 index 0000000..bf4f5f6 --- /dev/null +++ b/version.py @@ -0,0 +1,8 @@ +"""Package version and public schema identifiers.""" + +PACKAGE_VERSION = "2.1.4" +STATE_SCHEMA_VERSION = 1 +PLAN_SCHEMA_VERSION = 1 +PUBLIC_API_VERSION = 2 +SESSION_SCHEMA_VERSION = 1 +PROMPT_PLAN_SCHEMA_VERSION = 1 diff --git a/web/h3_continuum_v2.js b/web/h3_continuum_v2.js new file mode 100644 index 0000000..8082b66 --- /dev/null +++ b/web/h3_continuum_v2.js @@ -0,0 +1,211 @@ +import { app } from "../../../scripts/app.js"; + +const NODE_NAME = "H3ContinuumSamplerV2"; +const PREVIEW_SETTING = "H3Continuum.ShowLatentPreview"; +const PROMPT_INPUT = /^clip_(\d+)_prompt$/; +const SEQUENCE_PROMPT_INPUT = "sequence_prompt"; +const OVERRIDE_PROPERTY = "h3_show_clip_overrides"; +const MAX_CHUNKS = 16; +const LEGACY_PROMPT_FORMATS = new Map([ + ["Fixed — one prompt", "Fixed"], + ["List — split with ---", "List"], + ["Timeline — [0-5s] sections", "Timeline"], +]); + +let showLatentPreview = true; + +function hideWidget(widget) { + if (!widget) return; + if (!widget.__h3ContinuumHidden) { + widget.__h3ContinuumHidden = true; + widget.type = "converted-widget"; + widget.computeSize = () => [0, -4]; + } + widget.hidden = true; + for (const element of [widget.element, widget.inputEl, widget.el]) { + if (element?.style) element.style.display = "none"; + } +} + +function promptInputName(index) { + return `clip_${index}_prompt`; +} + +function inputIsConnected(input) { + return input?.link != null || (Array.isArray(input?.links) && input.links.length > 0); +} + +function overridesExpanded(node) { + return node.properties?.[OVERRIDE_PROPERTY] === true; +} + +function syncPromptFormat(node) { + const widget = node.widgets?.find((candidate) => candidate.name === "prompt_mode"); + if (!widget) return; + if (LEGACY_PROMPT_FORMATS.has(widget.value)) { + widget.value = LEGACY_PROMPT_FORMATS.get(widget.value); + } + widget.label = "Prompt Format"; +} + +function syncSequencePrompt(node) { + const input = node.inputs?.find((candidate) => candidate.name === SEQUENCE_PROMPT_INPUT); + if (input) input.label = "Sequence Prompt"; +} + +function syncPromptInputs(node) { + const chunksWidget = node.widgets?.find((widget) => widget.name === "chunks"); + const chunks = Math.max(1, Math.min(MAX_CHUNKS, Number(chunksWidget?.value) || 1)); + const expanded = overridesExpanded(node); + + for (let index = node.inputs.length - 1; index >= 0; index -= 1) { + const input = node.inputs[index]; + const match = PROMPT_INPUT.exec(input.name); + if (!match) continue; + const clipIndex = Number(match[1]); + const active = clipIndex <= chunks; + if ((!expanded || !active) && !inputIsConnected(input)) { + node.removeInput(index); + } else { + input.label = active + ? "Clip " + clipIndex + " Prompt Override" + : "Clip " + clipIndex + " Override (inactive)"; + } + } + + if (!expanded) return; + for (let index = 1; index <= chunks; index += 1) { + const name = promptInputName(index); + if (!node.inputs.some((input) => input.name === name)) { + node.addInput(name, "STRING"); + } + const input = node.inputs.find((candidate) => candidate.name === name); + if (input) input.label = "Clip " + index + " Prompt Override"; + } +} + +function ensureOverrideToggle(node) { + node.properties ||= {}; + if (typeof node.properties[OVERRIDE_PROPERTY] !== "boolean") { + node.properties[OVERRIDE_PROPERTY] = false; + } + let widget = node.widgets?.find((candidate) => candidate.__h3OverrideToggle); + if (!widget) { + widget = node.addWidget( + "button", + "Individual Clip Overrides ▸", + null, + () => { + node.properties[OVERRIDE_PROPERTY] = !overridesExpanded(node); + syncNode(node); + }, + { serialize: false }, + ); + widget.__h3OverrideToggle = true; + widget.serialize = false; + const widgets = node.widgets || []; + const currentIndex = widgets.indexOf(widget); + const formatIndex = widgets.findIndex((candidate) => candidate.name === "prompt_mode"); + if (currentIndex >= 0 && formatIndex >= 0) { + widgets.splice(currentIndex, 1); + widgets.splice(formatIndex + 1, 0, widget); + } + } + widget.name = overridesExpanded(node) + ? "Individual Clip Overrides ▾" + : "Individual Clip Overrides ▸"; +} + +function syncPreviewSetting(node) { + const widget = node.widgets?.find((candidate) => candidate.name === "show_preview"); + if (!widget) return; + hideWidget(widget); + widget.value = showLatentPreview; +} + +function repairLegacySeamCorrection(node) { + const widget = node.widgets?.find((candidate) => candidate.name === "seam_correction"); + if (widget && typeof widget.value === "boolean") widget.value = "Off"; +} + +function resizeNode(node) { + const size = node.computeSize(); + node.setSize([Math.max(node.size[0], size[0]), size[1]]); + node.setDirtyCanvas(true, true); +} + +function syncNode(node) { + hideWidget(node.widgets?.find((widget) => widget.name === "prompt_script")); + syncPromptFormat(node); + syncSequencePrompt(node); + ensureOverrideToggle(node); + repairLegacySeamCorrection(node); + syncPromptInputs(node); + syncPreviewSetting(node); + resizeNode(node); +} + +function wrapChunksWidget(node) { + const widget = node.widgets?.find((candidate) => candidate.name === "chunks"); + if (!widget || widget.__h3ContinuumWrapped) return; + widget.__h3ContinuumWrapped = true; + const originalCallback = widget.callback; + widget.callback = function (...args) { + const result = originalCallback?.apply(this, args); + queueMicrotask(() => syncNode(node)); + return result; + }; +} + +function syncAllSamplerNodes() { + for (const node of app.graph?._nodes || []) { + if (node.comfyClass === NODE_NAME || node.type === NODE_NAME) { + syncPreviewSetting(node); + node.setDirtyCanvas(true, true); + } + } +} + +app.registerExtension({ + name: "H3Continuum.V2Interface", + + async setup() { + showLatentPreview = app.ui.settings.getSettingValue(PREVIEW_SETTING) ?? true; + app.ui.settings.addSetting({ + id: PREVIEW_SETTING, + name: "Show latent preview", + category: ["H3 Continuum", "Preview", "Show latent preview"], + tooltip: "Show the in-progress latent thumbnail inside H3 Continuum Sampler V2 nodes.", + type: "boolean", + defaultValue: true, + onChange(value) { + showLatentPreview = value !== false; + syncAllSamplerNodes(); + }, + }); + }, + + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== NODE_NAME) return; + + const originalCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + const result = originalCreated?.apply(this, arguments); + setTimeout(() => { + wrapChunksWidget(this); + syncNode(this); + }, 0); + return result; + }; + + const originalConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function () { + const result = originalConfigure?.apply(this, arguments); + setTimeout(() => { + wrapChunksWidget(this); + syncNode(this); + }, 0); + return result; + }; + }, +}); diff --git a/web/h3_continuum_v214.js b/web/h3_continuum_v214.js new file mode 100644 index 0000000..93b48fb --- /dev/null +++ b/web/h3_continuum_v214.js @@ -0,0 +1,204 @@ +import { app } from "../../scripts/app.js"; + +const SAMPLER_NODE = "H3ContinuumSamplerV2"; +const ADVANCED_PROPERTY = "h3_show_advanced"; + +const ADVANCED_INPUTS = [ + ["last_frame", "IMAGE"], + ["session", "H3_CONTINUUM_SESSION"], + ["initial_state", "H3_CONTINUUM_STATE"], + ["prompt_plan", "H3_CONTINUUM_PROMPT_PLAN"], +]; + +const ADVANCED_OUTPUTS = [ + ["last_state", "H3_CONTINUUM_STATE"], + ["session", "H3_CONTINUUM_SESSION"], + ["report", "STRING"], +]; + +const ADVANCED_WIDGETS = new Set([ + "control_after_generate", + "audio_continuity", + "exact_total_duration", + "diagnostics", + "reroll_from_chunk", + "reroll_nonce", + "strict_compatibility", + "debug", + "show_preview", +]); + +function inputByName(node, name) { + return node.inputs?.find((input) => input.name === name); +} + +function outputByName(node, name) { + return node.outputs?.find((output) => output.name === name); +} + +function hasAdvancedConnection(node) { + const inputLinked = ADVANCED_INPUTS.some(([name]) => inputByName(node, name)?.link != null); + const outputLinked = ADVANCED_OUTPUTS.some(([name]) => { + const links = outputByName(node, name)?.links; + return Array.isArray(links) && links.length > 0; + }); + return inputLinked || outputLinked; +} + +function ensureAdvancedSlots(node) { + for (const [name, type] of ADVANCED_INPUTS) { + if (!inputByName(node, name)) { + node.addInput(name, type, { shape: 7 }); + } + } + for (const [name, type] of ADVANCED_OUTPUTS) { + if (!outputByName(node, name)) { + node.addOutput(name, type); + } + } +} + +function removeUnlinkedAdvancedSlots(node) { + if (hasAdvancedConnection(node)) { + return; + } + for (let index = (node.inputs?.length ?? 0) - 1; index >= 0; index -= 1) { + if (ADVANCED_INPUTS.some(([name]) => node.inputs[index].name === name)) { + node.removeInput(index); + } + } + for (let index = (node.outputs?.length ?? 0) - 1; index >= 0; index -= 1) { + if (ADVANCED_OUTPUTS.some(([name]) => node.outputs[index].name === name)) { + node.removeOutput(index); + } + } +} + +function setWidgetVisible(widget, visible) { + if (!widget.__h3V214Original) { + widget.__h3V214Original = { + type: widget.type, + computeSize: widget.computeSize, + hidden: widget.hidden, + }; + } + if (visible) { + widget.type = widget.__h3V214Original.type; + widget.computeSize = widget.__h3V214Original.computeSize; + widget.hidden = widget.__h3V214Original.hidden; + return; + } + widget.type = "hidden"; + widget.computeSize = () => [0, -4]; + widget.hidden = true; +} + +function ensureAdvancedButton(node) { + if (node.__h3V214AdvancedButton) { + return node.__h3V214AdvancedButton; + } + const button = node.addWidget( + "button", + "Advanced Settings ▸", + null, + () => { + node.properties ||= {}; + node.properties[ADVANCED_PROPERTY] = !node.properties[ADVANCED_PROPERTY]; + syncAdvancedUi(node); + }, + { serialize: false }, + ); + button.serialize = false; + node.__h3V214AdvancedButton = button; + return button; +} + +function placeAdvancedButton(node, button, expanded) { + const widgets = node.widgets ?? []; + const currentIndex = widgets.indexOf(button); + if (currentIndex >= 0) { + widgets.splice(currentIndex, 1); + } + if (!expanded) { + widgets.push(button); + return; + } + const firstAdvancedIndex = widgets.findIndex((widget) => ADVANCED_WIDGETS.has(widget.name)); + widgets.splice(firstAdvancedIndex >= 0 ? firstAdvancedIndex : widgets.length, 0, button); +} + +function resizeNode(node) { + const computed = node.computeSize(); + const width = Math.max(node.size?.[0] ?? 0, computed[0], 320); + node.setSize([width, computed[1]]); + node.setDirtyCanvas(true, true); +} + +function syncAdvancedUi(node) { + node.properties ||= {}; + if (typeof node.properties[ADVANCED_PROPERTY] !== "boolean") { + node.properties[ADVANCED_PROPERTY] = false; + } + if (hasAdvancedConnection(node)) { + node.properties[ADVANCED_PROPERTY] = true; + } + + const expanded = node.properties[ADVANCED_PROPERTY]; + if (expanded) { + ensureAdvancedSlots(node); + } else { + removeUnlinkedAdvancedSlots(node); + } + + for (const widget of node.widgets ?? []) { + if (ADVANCED_WIDGETS.has(widget.name)) { + setWidgetVisible(widget, expanded); + } + } + const button = ensureAdvancedButton(node); + button.name = expanded ? "Advanced Settings ▾" : "Advanced Settings ▸"; + placeAdvancedButton(node, button, expanded); + resizeNode(node); +} + +function scheduleSync(node) { + clearTimeout(node.__h3V214SyncTimer); + node.__h3V214SyncTimer = setTimeout(() => syncAdvancedUi(node), 0); +} + +app.registerExtension({ + name: "H3Continuum.V214SimplifiedUi", + beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== SAMPLER_NODE) { + return; + } + + const onNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + const result = onNodeCreated?.apply(this, arguments); + this.properties ||= {}; + if (typeof this.properties[ADVANCED_PROPERTY] !== "boolean") { + this.properties[ADVANCED_PROPERTY] = false; + } + ensureAdvancedButton(this); + scheduleSync(this); + return result; + }; + + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function () { + ensureAdvancedSlots(this); + const result = onConfigure?.apply(this, arguments); + ensureAdvancedButton(this); + scheduleSync(this); + return result; + }; + + const onConnectionsChange = nodeType.prototype.onConnectionsChange; + nodeType.prototype.onConnectionsChange = function () { + const result = onConnectionsChange?.apply(this, arguments); + scheduleSync(this); + return result; + }; + }, +});