diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index 7a88229..13a6d2f 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -106,12 +106,12 @@ bf0556b076dbaca67c3405a7d86c333ca7fdf3d4381529b0a9edc906519943e9 guide_timeline de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt 23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat 12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1 -6aa3f5780baa7bb0c0672b5f01a2f77e5f209da93f6373468cb274ea2f643d4b layout_adapter.py +83b9d5cda57079fbf8e4691893c1733e5afc08b13a70f12762aef1b912dd309a layout_adapter.py c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py 1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini fd23c7f32810788aaaf1709c4959c40ec72303aecc6f6afa5fa68d824cca97a8 minimax-h3-continuum-dance-director.zip -2d611f7201889851eac6459f8aa2b37baf342b345241979ca6042b61b252dff9 model_patch.py +d7e613e2cc24110213b477bdd1d390234a41f2af3b1ed006a809066d71d658eb model_patch.py e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py 2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt 4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt @@ -123,7 +123,7 @@ b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py 44127adcfa30ee84bff9cd4b2d0c695355209550e76982d374122008728aa5bc reference_audio.py 5cb9bfd6810d3a93a637451980a4cc5c1436e5b84cfd1c7a111cd192e876351b reference_video.py c7b14ed6f8cfc2a1f0a21db4fe56fba75a93ffcb0be0c22bbd2e691c45cd0e2d reference.py -ed892d90b644050bdf68f860c3b5bcdef6c59b565257275bbe7b07238ab1f630 REGISTRY_MANIFEST.sha256 +96ea9b3dcef1fec7bf05795d0857a18579e9b4b3dc482cc77d13b65b4ca49b35 REGISTRY_MANIFEST.sha256 467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt 8bfa113a2ec11325619be300b4d2b53a157b1d2c1ef7855057a4b3bbd05aeef3 run_storage.py 6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py @@ -154,9 +154,11 @@ aaed0385281e1782edc96f1d084fa84bc4e7ec9e0055ec35aaf9a9f8b6e5bfce tests/test_iss ff2e865fb49f0c9f73f3f82eff78ff15373ce8e843514d610b73231dd6217a93 tests/test_issue13_r28_tools.py c058f189d8e58fd48a80c20c0a857969737a8b13960eee60e5c17db2b9288bc1 tests/test_layout_adapter.py ab1ea95e363ad0effb9459bcbe713838f12e3501515871203672ea91e7fd4c32 tests/test_m3_conditioning_display.py -2dddf05bbb91b7aa803959a7a8ed78489792ea9c35c5634bfb7d5b1a6c81880f tests/test_model_patch.py +a5aa420008daaa84ffb998715f52fc8c13911a6dadc43c61a860ecc455dfaee6 tests/test_model_patch.py 45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py b635d404ca67df69894c4fcf80d8782cc5699e946bfb55ae4c44fff9d930b018 tests/test_p20_file_backed_tensor_poc.py + +1dfe5af2b83ce69efbc509f4367f24aed2b32fc7d64b2925b95a23204f7b40c8 tests/test_packed_layout_preflight.py 0f18b8810b2899f87ce6d6bfeee46fcd90ba787fa09665bddd8fc128f1952c65 tests/test_p21_memory_attribution.py 2b097971258dd10f0df33e0f8cb5843ee6cd3e9ef9e7abb80b890c66baf22908 tests/test_p22_file_backed_buffer.py d09d2bae2fac04a058ebce22fa0e761d90b267107923ab8cd01a99cc547624e6 tests/test_p23_assembly_v35.py diff --git a/REGISTRY_MANIFEST.sha256 b/REGISTRY_MANIFEST.sha256 index be12c93..8bc112a 100644 --- a/REGISTRY_MANIFEST.sha256 +++ b/REGISTRY_MANIFEST.sha256 @@ -25,11 +25,11 @@ bf0556b076dbaca67c3405a7d86c333ca7fdf3d4381529b0a9edc906519943e9 guide_timeline de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt 23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat 12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1 -6aa3f5780baa7bb0c0672b5f01a2f77e5f209da93f6373468cb274ea2f643d4b layout_adapter.py +83b9d5cda57079fbf8e4691893c1733e5afc08b13a70f12762aef1b912dd309a layout_adapter.py c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py 1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini -2d611f7201889851eac6459f8aa2b37baf342b345241979ca6042b61b252dff9 model_patch.py +d7e613e2cc24110213b477bdd1d390234a41f2af3b1ed006a809066d71d658eb model_patch.py e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py 2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt 4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt diff --git a/layout_adapter.py b/layout_adapter.py index 47a7fa9..d6eb690 100644 --- a/layout_adapter.py +++ b/layout_adapter.py @@ -22,6 +22,11 @@ from .constants import ( LOG = logging.getLogger("h3_continuum_join") class LayoutCompatibilityError(RuntimeError): pass +_VIDEO_CHANNELS = 24 +_AUDIO_CHANNELS = 32 +_VIDEO_PATCH = (1, 2, 2) +_AUDIO_STREAMS = 2 + def _reference_metadata(ref:dict[str,Any])->dict[str,Any]: value=ref.get(CONTINUUM_REFERENCE_METADATA_KEY); return value if isinstance(value,dict) else {} def _is_video_context(ref): @@ -40,6 +45,396 @@ def normalize_condition_latents(payload): audio_latents.extend(item["audio_latent"] for item in refs if item.get("audio_latent") is not None) payload["cond_video_latents"]=video_latents payload["cond_audio_latents"]=audio_latents + + +def _layout_condition_rows(layout): + try: + segments=tuple(layout.segments) + except Exception as exc: + raise LayoutCompatibilityError( + "Core PackedLayout does not expose segments" + ) from exc + visual=audio=0 + for item in segments: + if not isinstance(item,(tuple,list)) or len(item)!=3: + raise LayoutCompatibilityError( + "Core PackedLayout segment is not (start, stop, kind)" + ) + start,stop,kind=int(item[0]),int(item[1]),str(item[2]) + if stop0 or audio_latent is not None: + audio_index+=1 + audio_blocks.append(_audio_block( + index=audio_index, + kind=f"ref_{kind}", + latent=audio_latent, + expected_rows=ref_audio_t*_AUDIO_STREAMS, + metadata={"ref_audio_t":ref_audio_t}, + errors=geometry_errors, + )) + block=audio_blocks[-1] + if block["shape"] is not None: + actual_t=block["shape"][-1] + if actual_t!=ref_audio_t: + message=( + f"reference metadata audio_t {ref_audio_t} != " + f"latent audio_t {actual_t}" + ) + geometry_errors.append(message) + block["errors"]=tuple((*block["errors"],message)) + layout_visual_rows,layout_audio_rows=_layout_condition_rows(layout) + actual_visual_rows=sum( + int(block["actual_rows"] or 0) for block in visual_blocks + ) + actual_audio_rows=sum( + int(block["actual_rows"] or 0) for block in audio_blocks + ) + report={ + "status":"matched", + "repaired":False, + "visual_blocks":tuple(visual_blocks), + "audio_blocks":tuple(audio_blocks), + "layout_visual_rows":int(layout_visual_rows), + "actual_visual_rows":int(actual_visual_rows), + "layout_audio_rows":int(layout_audio_rows), + "actual_audio_rows":int(actual_audio_rows), + } + if geometry_errors: + raise LayoutCompatibilityError(_format_preflight_failure( + report,"Unsafe geometry mismatch." + )) + rows_match=( + layout_visual_rows==actual_visual_rows + and layout_audio_rows==actual_audio_rows + ) + if rows_match: + return report + if not repair_stale: + raise LayoutCompatibilityError(_format_preflight_failure( + report,"PackedLayout rows do not match the current payload." + )) + try: + rebuilt=type(layout)( + *signature, + keyframes=keyframes or None, + refs=refs or None, + ) + except Exception as exc: + raise LayoutCompatibilityError(_format_preflight_failure( + report, + "PackedLayout is stale, but a fresh layout could not be constructed: " + f"{type(exc).__name__}: {exc}", + )) from exc + rebuilt_visual_rows,rebuilt_audio_rows=_layout_condition_rows(rebuilt) + if ( + rebuilt_visual_rows!=actual_visual_rows + or rebuilt_audio_rows!=actual_audio_rows + ): + raise LayoutCompatibilityError(_format_preflight_failure( + report, + "A fresh PackedLayout still does not match the current payload; " + "automatic repair is unsafe.", + )) + payload["layout"]=rebuilt + report.update({ + "status":"layout_rebuilt", + "repaired":True, + "layout_visual_rows":int(rebuilt_visual_rows), + "layout_audio_rows":int(rebuilt_audio_rows), + }) + return report def _input_device(x): device=getattr(x,"device",None) if isinstance(device,torch.device): return device diff --git a/model_patch.py b/model_patch.py index c6982ce..67bc994 100644 --- a/model_patch.py +++ b/model_patch.py @@ -19,6 +19,7 @@ from .layout_adapter import ( materialize_continuum_latents, normalize_condition_latents, patch_layout_in_place, + preflight_packed_layout, payload_has_continuum, validate_native_continuity_layout, ) @@ -165,16 +166,15 @@ def _wrapper_factory( def apply_model_wrapper(executor, *args, **kwargs): payload = kwargs.get("minimax_payload") - if packed_row_planner is not None: - from .v3.packed_row_planner import observe_layout_fail_soft - observe_layout_fail_soft(packed_row_planner, payload) has_continuum = payload_has_continuum(payload) - has_mixed_keyframe_refs = ( + has_condition_blocks = ( isinstance(payload, dict) - and bool(payload.get("keyframes")) - and payload.get("refs") is not None + and (bool(payload.get("keyframes")) or bool(payload.get("refs"))) ) - if not has_continuum and not has_mixed_keyframe_refs: + if not has_continuum and not has_condition_blocks: + if packed_row_planner is not None: + from .v3.packed_row_planner import observe_layout_fail_soft + observe_layout_fail_soft(packed_row_planner, payload) return executor(*args, **kwargs) try: try: @@ -186,6 +186,21 @@ def _wrapper_factory( exc, ) patched_payload = dict(payload) + # Rebuild Core's flattened condition lists from the authoritative + # keyframe/reference blocks before checking their row contract. + normalize_condition_latents(patched_payload) + preflight_result = preflight_packed_layout( + patched_payload, + repair_stale=True, + ) + if preflight_result.get("repaired"): + # The wrapper is called for every model evaluation. Persist a + # proven-safe replacement on the queue-local payload so later + # steps preserve one layout and one position_ids identity. + payload["layout"] = patched_payload["layout"] + if packed_row_planner is not None: + from .v3.packed_row_planner import observe_layout_fail_soft + observe_layout_fail_soft(packed_row_planner, patched_payload) if has_continuum: patch_layout_in_place(patched_payload, strict=True, debug=debug) transformer_options = kwargs.get("transformer_options") @@ -219,10 +234,11 @@ def _wrapper_factory( ) input_x = args[0] if args else None materialize_continuum_latents(patched_payload, input_x, debug=debug) - # ComfyUI Core 0.33.1 replaces keyframe condition latents with - # reference latents when both contracts are present. Rebuild the - # combined list for First/Last Frame + standalone Reference Audio. + # Materialization may replace CPU block tensors with device-local + # tensors. Rebuild the flattened lists once more, then prove the + # final payload still matches the repaired/preserved layout. normalize_condition_latents(patched_payload) + preflight_packed_layout(patched_payload, repair_stale=False) kwargs["minimax_payload"] = patched_payload except (CompatibilityError, LayoutCompatibilityError, KeyError, TypeError, ValueError) as exc: raise RuntimeError(f"H3 Continuum Join compatibility failure: {exc}") from exc diff --git a/tests/test_model_patch.py b/tests/test_model_patch.py index 6988c3c..256559a 100644 --- a/tests/test_model_patch.py +++ b/tests/test_model_patch.py @@ -54,17 +54,17 @@ def _layout(): pos[3, 0] = 1 pos[4:6, 0] = 2 pos[6, 0] = 2 - return SimpleNamespace(segments=segments, position_ids=pos, signature=(1, 1, 1, 1, 1)) + return SimpleNamespace(segments=segments, position_ids=pos, signature=(1, 1, 2, 2, 1)) def test_apply_model_wrapper_repairs_payload_before_executor(): - video = torch.zeros(1, 24, 1, 1, 1) + video = torch.zeros(1, 24, 1, 2, 2) audio = torch.zeros(1, 32, 2, 1) ref = { "kind": "video_audio", "latent_t": 1, - "latent_h": 1, - "latent_w": 1, + "latent_h": 2, + "latent_w": 2, "ref_audio_t": 1, "latent": video, "audio_latent": audio, @@ -106,14 +106,47 @@ def test_apply_model_wrapper_repairs_keyframes_with_audio_reference_without_cont assert result["cond_audio_latents"] == [audio] +def test_apply_model_wrapper_persists_a_proven_stale_layout_repair(monkeypatch): + import ComfyUI_H3_Continuum_Join.model_patch as model_patch + + audio = torch.zeros(1, 32, 2, 1) + old_layout = object() + replacement = object() + payload = { + "layout": old_layout, + "keyframes": [ + {"resolved_frame_index": 0, "audio_latent": audio} + ], + "refs": [], + } + calls = [] + + def fake_preflight(value, *, repair_stale): + calls.append(bool(repair_stale)) + if repair_stale: + value["layout"] = replacement + return {"repaired": True} + assert value["layout"] is replacement + return {"repaired": False} + + monkeypatch.setattr(model_patch, "preflight_packed_layout", fake_preflight) + wrapper = _wrapper_factory(strict=True, debug=False) + + result = wrapper(Executor(), torch.zeros(1), minimax_payload=payload) + + assert calls == [True, False] + assert payload["layout"] is replacement + assert result["layout"] is replacement + + def test_apply_model_wrapper_fails_when_actual_topology_changes(): - video = torch.zeros(1, 24, 1, 1, 1) + video = torch.zeros(1, 24, 1, 2, 2) audio = torch.zeros(1, 32, 2, 1) ref = { "kind": "video_audio", "latent_t": 1, - "latent_h": 1, - "latent_w": 1, + "latent_h": 2, + "latent_w": 2, "ref_audio_t": 1, "latent": video, "audio_latent": audio, @@ -168,13 +201,13 @@ def test_layout_validation_profiler_records_full_calls_groups_and_step_contribut def test_apply_model_wrapper_reports_validator_status_without_changing_payload(): - video = torch.zeros(1, 24, 1, 1, 1) + video = torch.zeros(1, 24, 1, 2, 2) audio = torch.zeros(1, 32, 2, 1) ref = { "kind": "video_audio", "latent_t": 1, - "latent_h": 1, - "latent_w": 1, + "latent_h": 2, + "latent_w": 2, "ref_audio_t": 1, "latent": video, "audio_latent": audio, diff --git a/tests/test_packed_layout_preflight.py b/tests/test_packed_layout_preflight.py new file mode 100644 index 0000000..7892e76 --- /dev/null +++ b/tests/test_packed_layout_preflight.py @@ -0,0 +1,250 @@ +import pytest +import torch + +from ComfyUI_H3_Continuum_Join.layout_adapter import ( + LayoutCompatibilityError, + normalize_condition_latents, + preflight_packed_layout, +) + + +class FakePackedLayout: + """Small Core-compatible row topology used by contract tests.""" + + def __init__( + self, + text_len, + latent_t, + latent_h, + latent_w, + audio_t, + keyframes=None, + refs=None, + ): + self.signature = ( + int(text_len), + int(latent_t), + int(latent_h), + int(latent_w), + int(audio_t), + ) + frame_rows = (int(latent_h) // 2) * (int(latent_w) // 2) + parts = [("text", int(text_len))] + for keyframe in keyframes or (): + latent = keyframe.get("latent") + if latent is not None: + parts.append(("cond", int(latent.shape[2]) * frame_rows)) + audio = keyframe.get("audio_latent") + if audio is not None: + parts.append(("cond_audio", int(audio.shape[-1]) * 2)) + for ref in refs or (): + kind = ref["kind"] + if kind == "image": + parts.append( + ( + "ref_img", + (int(ref["latent_h"]) // 2) + * (int(ref["latent_w"]) // 2), + ) + ) + elif kind == "audio": + if int(ref["ref_audio_t"]) > 0: + parts.append(("ref_audio", int(ref["ref_audio_t"]) * 2)) + elif kind in ("video", "video_audio"): + if int(ref["ref_audio_t"]) > 0: + parts.append(("ref_audio", int(ref["ref_audio_t"]) * 2)) + parts.append( + ( + "ref_img", + int(ref["latent_t"]) + * (int(ref["latent_h"]) // 2) + * (int(ref["latent_w"]) // 2), + ) + ) + else: + raise ValueError(f"unsupported ref kind: {kind}") + parts.extend( + ( + ("audio", int(audio_t) * 2), + ("video", int(latent_t) * frame_rows), + ) + ) + row = 0 + self.segments = [] + for kind, count in parts: + self.segments.append((row, row + count, kind)) + row += count + self.seq_len = row + self.position_ids = torch.zeros(row, 3, dtype=torch.float64) + + +def _layout(*, keyframes=None, refs=None): + return FakePackedLayout( + 3, + 2, + 4, + 6, + 5, + keyframes=keyframes, + refs=refs, + ) + + +def test_preflight_accepts_self_consistent_visual_audio_and_multiple_blocks(): + first = torch.zeros(1, 24, 1, 4, 6) + guide_audio = torch.zeros(1, 32, 2, 5) + image = torch.zeros(1, 24, 1, 2, 4) + video = torch.zeros(1, 24, 2, 2, 2) + ref_audio = torch.zeros(1, 32, 2, 3) + keyframes = [ + {"resolved_frame_index": 0, "latent": first}, + {"resolved_frame_index": 0, "audio_latent": guide_audio}, + ] + refs = [ + { + "kind": "image", + "latent_h": 2, + "latent_w": 4, + "latent": image, + }, + { + "kind": "video_audio", + "latent_t": 2, + "latent_h": 2, + "latent_w": 2, + "ref_audio_t": 3, + "latent": video, + "audio_latent": ref_audio, + }, + ] + payload = { + "layout": _layout(keyframes=keyframes, refs=refs), + "keyframes": keyframes, + "refs": refs, + } + normalize_condition_latents(payload) + + result = preflight_packed_layout(payload, repair_stale=False) + + assert result["status"] == "matched" + assert result["layout_visual_rows"] == result["actual_visual_rows"] == 10 + assert result["layout_audio_rows"] == result["actual_audio_rows"] == 16 + assert len(result["visual_blocks"]) == 3 + assert len(result["audio_blocks"]) == 2 + + +def test_preflight_repairs_only_a_stale_layout_from_current_payload(): + first = torch.zeros(1, 24, 1, 4, 6) + guide_audio = torch.zeros(1, 32, 2, 5) + keyframes = [ + {"resolved_frame_index": 0, "latent": first}, + {"resolved_frame_index": 0, "audio_latent": guide_audio}, + ] + stale = _layout(keyframes=None, refs=None) + payload = {"layout": stale, "keyframes": keyframes, "refs": []} + normalize_condition_latents(payload) + + result = preflight_packed_layout(payload, repair_stale=True) + + assert result["status"] == "layout_rebuilt" + assert result["repaired"] is True + assert payload["layout"] is not stale + assert result["layout_visual_rows"] == result["actual_visual_rows"] == 6 + assert result["layout_audio_rows"] == result["actual_audio_rows"] == 10 + final = preflight_packed_layout(payload, repair_stale=False) + assert final["status"] == "matched" + + +def test_preflight_rejects_keyframe_target_geometry_mismatch_without_mutation(): + first = torch.zeros(1, 24, 1, 4, 4) + keyframes = [{"resolved_frame_index": 0, "latent": first}] + payload = { + "layout": _layout(keyframes=keyframes), + "keyframes": keyframes, + "refs": [], + } + original = first.clone() + normalize_condition_latents(payload) + + with pytest.raises(LayoutCompatibilityError) as caught: + preflight_packed_layout(payload, repair_stale=True) + + message = str(caught.value) + assert "Visual block #1" in message + assert "type: keyframe" in message + assert "resolved_frame_index: 0" in message + assert "latent shape: B=1 C=24 T=1 H=4 W=4" in message + assert "expected rows: 6" in message + assert "actual rows: 4" in message + assert "delta: -2" in message + assert "Unsafe geometry mismatch" in message + assert "Sampling was not started" in message + assert torch.equal(first, original) + + +def test_preflight_rejects_reference_metadata_geometry_mismatch(): + image = torch.zeros(1, 24, 1, 4, 4) + refs = [ + { + "kind": "image", + "latent_h": 4, + "latent_w": 6, + "latent": image, + } + ] + payload = {"layout": _layout(refs=refs), "keyframes": [], "refs": refs} + normalize_condition_latents(payload) + + with pytest.raises(LayoutCompatibilityError) as caught: + preflight_packed_layout(payload, repair_stale=True) + + message = str(caught.value) + assert "type: ref_image" in message + assert "reference metadata H/W (4, 6) != latent H/W (4, 4)" in message + assert "expected rows: 6" in message + assert "actual rows: 4" in message + + +def test_preflight_rejects_reference_audio_metadata_mismatch(): + audio = torch.zeros(1, 32, 2, 3) + refs = [ + { + "kind": "audio", + "ref_audio_t": 4, + "audio_latent": audio, + } + ] + payload = {"layout": _layout(refs=refs), "keyframes": [], "refs": refs} + normalize_condition_latents(payload) + + with pytest.raises(LayoutCompatibilityError) as caught: + preflight_packed_layout(payload, repair_stale=True) + + message = str(caught.value) + assert "Audio block #1" in message + assert "type: ref_audio" in message + assert "expected rows: 8" in message + assert "actual rows: 6" in message + assert "reference metadata audio_t 4 != latent audio_t 3" in message + + +def test_preflight_does_not_invent_a_layout_when_core_has_not_built_one(): + payload = { + "keyframes": [ + { + "resolved_frame_index": 0, + "audio_latent": torch.zeros(1, 32, 2, 3), + } + ], + "refs": [], + } + normalize_condition_latents(payload) + + result = preflight_packed_layout(payload, repair_stale=True) + + assert result == { + "status": "layout_unavailable", + "repaired": False, + "visual_blocks": (), + "audio_blocks": (), + }