Validate and repair H3 PackedLayout conditioning rows
This commit is contained in:
+6
-4
@@ -106,12 +106,12 @@ bf0556b076dbaca67c3405a7d86c333ca7fdf3d4381529b0a9edc906519943e9 guide_timeline
|
|||||||
de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt
|
de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt
|
||||||
23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat
|
23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat
|
||||||
12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1
|
12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1
|
||||||
6aa3f5780baa7bb0c0672b5f01a2f77e5f209da93f6373468cb274ea2f643d4b layout_adapter.py
|
83b9d5cda57079fbf8e4691893c1733e5afc08b13a70f12762aef1b912dd309a layout_adapter.py
|
||||||
c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE
|
c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE
|
||||||
d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py
|
d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py
|
||||||
1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini
|
1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini
|
||||||
fd23c7f32810788aaaf1709c4959c40ec72303aecc6f6afa5fa68d824cca97a8 minimax-h3-continuum-dance-director.zip
|
fd23c7f32810788aaaf1709c4959c40ec72303aecc6f6afa5fa68d824cca97a8 minimax-h3-continuum-dance-director.zip
|
||||||
2d611f7201889851eac6459f8aa2b37baf342b345241979ca6042b61b252dff9 model_patch.py
|
d7e613e2cc24110213b477bdd1d390234a41f2af3b1ed006a809066d71d658eb model_patch.py
|
||||||
e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py
|
e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py
|
||||||
2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt
|
2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt
|
||||||
4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt
|
4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt
|
||||||
@@ -123,7 +123,7 @@ b470819f16fc179654dc1600721a63f0a5a14e777f4c004d574b499da6886ba5 public_api.py
|
|||||||
44127adcfa30ee84bff9cd4b2d0c695355209550e76982d374122008728aa5bc reference_audio.py
|
44127adcfa30ee84bff9cd4b2d0c695355209550e76982d374122008728aa5bc reference_audio.py
|
||||||
5cb9bfd6810d3a93a637451980a4cc5c1436e5b84cfd1c7a111cd192e876351b reference_video.py
|
5cb9bfd6810d3a93a637451980a4cc5c1436e5b84cfd1c7a111cd192e876351b reference_video.py
|
||||||
c7b14ed6f8cfc2a1f0a21db4fe56fba75a93ffcb0be0c22bbd2e691c45cd0e2d reference.py
|
c7b14ed6f8cfc2a1f0a21db4fe56fba75a93ffcb0be0c22bbd2e691c45cd0e2d reference.py
|
||||||
ed892d90b644050bdf68f860c3b5bcdef6c59b565257275bbe7b07238ab1f630 REGISTRY_MANIFEST.sha256
|
96ea9b3dcef1fec7bf05795d0857a18579e9b4b3dc482cc77d13b65b4ca49b35 REGISTRY_MANIFEST.sha256
|
||||||
467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt
|
467edb2c188d024c2392aafe7423b3f835e16b10f570da76e2f7dda8a962c1c5 requirements.txt
|
||||||
8bfa113a2ec11325619be300b4d2b53a157b1d2c1ef7855057a4b3bbd05aeef3 run_storage.py
|
8bfa113a2ec11325619be300b4d2b53a157b1d2c1ef7855057a4b3bbd05aeef3 run_storage.py
|
||||||
6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py
|
6056de6c0a5322a8a51ba5a114538dca2ef67788cd1afba10fedd483b95f6afa state_io.py
|
||||||
@@ -154,9 +154,11 @@ aaed0385281e1782edc96f1d084fa84bc4e7ec9e0055ec35aaf9a9f8b6e5bfce tests/test_iss
|
|||||||
ff2e865fb49f0c9f73f3f82eff78ff15373ce8e843514d610b73231dd6217a93 tests/test_issue13_r28_tools.py
|
ff2e865fb49f0c9f73f3f82eff78ff15373ce8e843514d610b73231dd6217a93 tests/test_issue13_r28_tools.py
|
||||||
c058f189d8e58fd48a80c20c0a857969737a8b13960eee60e5c17db2b9288bc1 tests/test_layout_adapter.py
|
c058f189d8e58fd48a80c20c0a857969737a8b13960eee60e5c17db2b9288bc1 tests/test_layout_adapter.py
|
||||||
ab1ea95e363ad0effb9459bcbe713838f12e3501515871203672ea91e7fd4c32 tests/test_m3_conditioning_display.py
|
ab1ea95e363ad0effb9459bcbe713838f12e3501515871203672ea91e7fd4c32 tests/test_m3_conditioning_display.py
|
||||||
2dddf05bbb91b7aa803959a7a8ed78489792ea9c35c5634bfb7d5b1a6c81880f tests/test_model_patch.py
|
a5aa420008daaa84ffb998715f52fc8c13911a6dadc43c61a860ecc455dfaee6 tests/test_model_patch.py
|
||||||
45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py
|
45e8839cd82fd955df344b3bc6efa20092b8870ea378b0c898b073665d3e47d7 tests/test_nodes_logic.py
|
||||||
b635d404ca67df69894c4fcf80d8782cc5699e946bfb55ae4c44fff9d930b018 tests/test_p20_file_backed_tensor_poc.py
|
b635d404ca67df69894c4fcf80d8782cc5699e946bfb55ae4c44fff9d930b018 tests/test_p20_file_backed_tensor_poc.py
|
||||||
|
|
||||||
|
1dfe5af2b83ce69efbc509f4367f24aed2b32fc7d64b2925b95a23204f7b40c8 tests/test_packed_layout_preflight.py
|
||||||
0f18b8810b2899f87ce6d6bfeee46fcd90ba787fa09665bddd8fc128f1952c65 tests/test_p21_memory_attribution.py
|
0f18b8810b2899f87ce6d6bfeee46fcd90ba787fa09665bddd8fc128f1952c65 tests/test_p21_memory_attribution.py
|
||||||
2b097971258dd10f0df33e0f8cb5843ee6cd3e9ef9e7abb80b890c66baf22908 tests/test_p22_file_backed_buffer.py
|
2b097971258dd10f0df33e0f8cb5843ee6cd3e9ef9e7abb80b890c66baf22908 tests/test_p22_file_backed_buffer.py
|
||||||
d09d2bae2fac04a058ebce22fa0e761d90b267107923ab8cd01a99cc547624e6 tests/test_p23_assembly_v35.py
|
d09d2bae2fac04a058ebce22fa0e761d90b267107923ab8cd01a99cc547624e6 tests/test_p23_assembly_v35.py
|
||||||
|
|||||||
@@ -25,11 +25,11 @@ bf0556b076dbaca67c3405a7d86c333ca7fdf3d4381529b0a9edc906519943e9 guide_timeline
|
|||||||
de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt
|
de0da5590691554cbf7f6318ac539a160e639bc3e20ee0752ae189dc31e979ce INSTALL_JA.txt
|
||||||
23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat
|
23bddc5a41d2e997f4fbf789c7128e32f4f197f1405b2aef1c76794f57482446 install_windows.bat
|
||||||
12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1
|
12bf572a65d57bbc9a72b459f29691c2057768f4fb7a74f9db87f4775e995bda install_windows.ps1
|
||||||
6aa3f5780baa7bb0c0672b5f01a2f77e5f209da93f6373468cb274ea2f643d4b layout_adapter.py
|
83b9d5cda57079fbf8e4691893c1733e5afc08b13a70f12762aef1b912dd309a layout_adapter.py
|
||||||
c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE
|
c44abbd789091bbead01250dbd5453e60539d2665c69f162335fdbd2936019c2 LICENSE
|
||||||
d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py
|
d3e213005670e94dee33a240349a4d245cf9a3b6602099f11ed30cf5a32e2c21 media.py
|
||||||
1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini
|
1970342c93f0189ade437cbfc278900713f043882933789cb8c54f35cf2ec96b metadata.ini
|
||||||
2d611f7201889851eac6459f8aa2b37baf342b345241979ca6042b61b252dff9 model_patch.py
|
d7e613e2cc24110213b477bdd1d390234a41f2af3b1ed006a809066d71d658eb model_patch.py
|
||||||
e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py
|
e2b72149b0433e95c13659cc9e742fe38523e67d4f4a2fdb08aed1943fecf653 nodes.py
|
||||||
2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt
|
2b8c19f9474beabfebe1802cdf4a89780dce3c351c48d65e1ce8bb77d2fb9ba9 PACKAGE_INFO_JA.txt
|
||||||
4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt
|
4852ad216f85c22f008abe18464eab2c913bd88baa7db35764df2de9cf401d6d PACKAGE_VALIDATION.txt
|
||||||
|
|||||||
@@ -22,6 +22,11 @@ from .constants import (
|
|||||||
LOG = logging.getLogger("h3_continuum_join")
|
LOG = logging.getLogger("h3_continuum_join")
|
||||||
class LayoutCompatibilityError(RuntimeError): pass
|
class LayoutCompatibilityError(RuntimeError): pass
|
||||||
|
|
||||||
|
_VIDEO_CHANNELS = 24
|
||||||
|
_AUDIO_CHANNELS = 32
|
||||||
|
_VIDEO_PATCH = (1, 2, 2)
|
||||||
|
_AUDIO_STREAMS = 2
|
||||||
|
|
||||||
def _reference_metadata(ref:dict[str,Any])->dict[str,Any]:
|
def _reference_metadata(ref:dict[str,Any])->dict[str,Any]:
|
||||||
value=ref.get(CONTINUUM_REFERENCE_METADATA_KEY); return value if isinstance(value,dict) else {}
|
value=ref.get(CONTINUUM_REFERENCE_METADATA_KEY); return value if isinstance(value,dict) else {}
|
||||||
def _is_video_context(ref):
|
def _is_video_context(ref):
|
||||||
@@ -40,6 +45,396 @@ def normalize_condition_latents(payload):
|
|||||||
audio_latents.extend(item["audio_latent"] for item in refs if item.get("audio_latent") is not None)
|
audio_latents.extend(item["audio_latent"] for item in refs if item.get("audio_latent") is not None)
|
||||||
payload["cond_video_latents"]=video_latents
|
payload["cond_video_latents"]=video_latents
|
||||||
payload["cond_audio_latents"]=audio_latents
|
payload["cond_audio_latents"]=audio_latents
|
||||||
|
|
||||||
|
|
||||||
|
def _layout_condition_rows(layout):
|
||||||
|
try:
|
||||||
|
segments=tuple(layout.segments)
|
||||||
|
except Exception as exc:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout does not expose segments"
|
||||||
|
) from exc
|
||||||
|
visual=audio=0
|
||||||
|
for item in segments:
|
||||||
|
if not isinstance(item,(tuple,list)) or len(item)!=3:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout segment is not (start, stop, kind)"
|
||||||
|
)
|
||||||
|
start,stop,kind=int(item[0]),int(item[1]),str(item[2])
|
||||||
|
if stop<start:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout segment has a negative row span"
|
||||||
|
)
|
||||||
|
rows=stop-start
|
||||||
|
if kind in ("cond","ref_img"):
|
||||||
|
visual+=rows
|
||||||
|
elif kind in ("cond_audio","ref_audio"):
|
||||||
|
audio+=rows
|
||||||
|
return int(visual),int(audio)
|
||||||
|
|
||||||
|
|
||||||
|
def _layout_signature(layout):
|
||||||
|
try:
|
||||||
|
signature=tuple(int(value) for value in layout.signature)
|
||||||
|
except Exception as exc:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout does not expose a five-value signature"
|
||||||
|
) from exc
|
||||||
|
if len(signature)!=5:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout signature must be "
|
||||||
|
"(text_len, latent_t, latent_h, latent_w, audio_t)"
|
||||||
|
)
|
||||||
|
text_len,latent_t,latent_h,latent_w,audio_t=signature
|
||||||
|
if min(text_len,latent_t,latent_h,latent_w,audio_t)<0:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout signature contains a negative value"
|
||||||
|
)
|
||||||
|
if latent_h%_VIDEO_PATCH[1] or latent_w%_VIDEO_PATCH[2]:
|
||||||
|
raise LayoutCompatibilityError(
|
||||||
|
"Core PackedLayout target latent height/width is not divisible by 2"
|
||||||
|
)
|
||||||
|
return signature
|
||||||
|
|
||||||
|
|
||||||
|
def _tensor_shape(value,dimensions,label,errors):
|
||||||
|
if not torch.is_tensor(value):
|
||||||
|
errors.append(f"{label} is missing or is not a Tensor")
|
||||||
|
return None
|
||||||
|
shape=tuple(int(item) for item in value.shape)
|
||||||
|
if len(shape)!=dimensions:
|
||||||
|
errors.append(
|
||||||
|
f"{label} must have {dimensions} dimensions, got shape={shape}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
return shape
|
||||||
|
|
||||||
|
|
||||||
|
def _video_block(*,index,kind,latent,expected_rows,metadata,errors):
|
||||||
|
label=f"Visual block #{index} ({kind})"
|
||||||
|
block_errors=[]
|
||||||
|
shape=_tensor_shape(latent,5,label,block_errors)
|
||||||
|
actual_rows=None
|
||||||
|
if shape is not None:
|
||||||
|
batch,channels,latent_t,latent_h,latent_w=shape
|
||||||
|
if batch!=1:
|
||||||
|
block_errors.append(f"batch must be 1, got {batch}")
|
||||||
|
if channels!=_VIDEO_CHANNELS:
|
||||||
|
block_errors.append(
|
||||||
|
f"video latent channels must be {_VIDEO_CHANNELS}, got {channels}"
|
||||||
|
)
|
||||||
|
pt,ph,pw=_VIDEO_PATCH
|
||||||
|
if latent_t%pt or latent_h%ph or latent_w%pw:
|
||||||
|
block_errors.append(
|
||||||
|
"video latent T/H/W is not divisible by the H3 patch size "
|
||||||
|
f"{_VIDEO_PATCH}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
actual_rows=batch*(latent_t//pt)*(latent_h//ph)*(latent_w//pw)
|
||||||
|
if actual_rows is not None and int(expected_rows)!=int(actual_rows):
|
||||||
|
block_errors.append(
|
||||||
|
f"expected rows {int(expected_rows)} != actual rows {int(actual_rows)}"
|
||||||
|
)
|
||||||
|
errors.extend(block_errors)
|
||||||
|
return {
|
||||||
|
"index":int(index),
|
||||||
|
"type":str(kind),
|
||||||
|
"shape":shape,
|
||||||
|
"metadata":dict(metadata),
|
||||||
|
"expected_rows":int(expected_rows),
|
||||||
|
"actual_rows":None if actual_rows is None else int(actual_rows),
|
||||||
|
"errors":tuple(block_errors),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _audio_block(*,index,kind,latent,expected_rows,metadata,errors):
|
||||||
|
label=f"Audio block #{index} ({kind})"
|
||||||
|
block_errors=[]
|
||||||
|
shape=_tensor_shape(latent,4,label,block_errors)
|
||||||
|
actual_rows=None
|
||||||
|
if shape is not None:
|
||||||
|
batch,channels,streams,latent_t=shape
|
||||||
|
if batch!=1:
|
||||||
|
block_errors.append(f"batch must be 1, got {batch}")
|
||||||
|
if channels!=_AUDIO_CHANNELS:
|
||||||
|
block_errors.append(
|
||||||
|
f"audio latent channels must be {_AUDIO_CHANNELS}, got {channels}"
|
||||||
|
)
|
||||||
|
if streams!=_AUDIO_STREAMS:
|
||||||
|
block_errors.append(
|
||||||
|
f"audio latent streams must be {_AUDIO_STREAMS}, got {streams}"
|
||||||
|
)
|
||||||
|
actual_rows=streams*latent_t
|
||||||
|
if actual_rows is not None and int(expected_rows)!=int(actual_rows):
|
||||||
|
block_errors.append(
|
||||||
|
f"expected rows {int(expected_rows)} != actual rows {int(actual_rows)}"
|
||||||
|
)
|
||||||
|
errors.extend(block_errors)
|
||||||
|
return {
|
||||||
|
"index":int(index),
|
||||||
|
"type":str(kind),
|
||||||
|
"shape":shape,
|
||||||
|
"metadata":dict(metadata),
|
||||||
|
"expected_rows":int(expected_rows),
|
||||||
|
"actual_rows":None if actual_rows is None else int(actual_rows),
|
||||||
|
"errors":tuple(block_errors),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _format_preflight_failure(report,reason):
|
||||||
|
lines=["H3 Continuum PackedLayout preflight failed",""]
|
||||||
|
for group,label in (
|
||||||
|
(report["visual_blocks"],"Visual"),
|
||||||
|
(report["audio_blocks"],"Audio"),
|
||||||
|
):
|
||||||
|
for block in group:
|
||||||
|
if not block["errors"]:
|
||||||
|
continue
|
||||||
|
lines.extend((
|
||||||
|
f"{label} block #{block['index']}",
|
||||||
|
f"type: {block['type']}",
|
||||||
|
))
|
||||||
|
metadata=block["metadata"]
|
||||||
|
if "resolved_frame_index" in metadata:
|
||||||
|
lines.append(
|
||||||
|
f"resolved_frame_index: {metadata['resolved_frame_index']}"
|
||||||
|
)
|
||||||
|
shape=block["shape"]
|
||||||
|
if shape is not None:
|
||||||
|
if label=="Visual":
|
||||||
|
b,c,t,h,w=shape
|
||||||
|
lines.append(
|
||||||
|
f"latent shape: B={b} C={c} T={t} H={h} W={w}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
b,c,ch,t=shape
|
||||||
|
lines.append(
|
||||||
|
f"latent shape: B={b} C={c} streams={ch} T={t}"
|
||||||
|
)
|
||||||
|
lines.extend((
|
||||||
|
f"expected rows: {block['expected_rows']}",
|
||||||
|
f"actual rows: {block['actual_rows']}",
|
||||||
|
))
|
||||||
|
if block["actual_rows"] is not None:
|
||||||
|
lines.append(
|
||||||
|
"delta: "
|
||||||
|
f"{block['actual_rows']-block['expected_rows']:+d}"
|
||||||
|
)
|
||||||
|
lines.extend(f"detail: {value}" for value in block["errors"])
|
||||||
|
lines.append("")
|
||||||
|
lines.extend((
|
||||||
|
f"Layout expected visual rows: {report['layout_visual_rows']}",
|
||||||
|
f"Actual visual rows: {report['actual_visual_rows']}",
|
||||||
|
f"Layout expected audio rows: {report['layout_audio_rows']}",
|
||||||
|
f"Actual audio rows: {report['actual_audio_rows']}",
|
||||||
|
"",
|
||||||
|
str(reason),
|
||||||
|
"No tensors were resized, truncated, padded, or replaced.",
|
||||||
|
"Sampling was not started.",
|
||||||
|
))
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def preflight_packed_layout(payload,*,repair_stale=False):
|
||||||
|
"""Validate Core H3 condition rows and narrowly repair a stale layout.
|
||||||
|
|
||||||
|
A repair is allowed only when every current keyframe/reference tensor is
|
||||||
|
self-consistent with the geometry Core will use to construct a fresh
|
||||||
|
PackedLayout. Tensor geometry is never changed here.
|
||||||
|
"""
|
||||||
|
if not isinstance(payload,dict):
|
||||||
|
raise LayoutCompatibilityError("MiniMax H3 payload must be a dict")
|
||||||
|
layout=payload.get("layout")
|
||||||
|
if layout is None:
|
||||||
|
return {
|
||||||
|
"status":"layout_unavailable",
|
||||||
|
"repaired":False,
|
||||||
|
"visual_blocks":(),
|
||||||
|
"audio_blocks":(),
|
||||||
|
}
|
||||||
|
signature=_layout_signature(layout)
|
||||||
|
_text_len,_target_t,target_h,target_w,_target_audio_t=signature
|
||||||
|
target_frame_rows=(target_h//_VIDEO_PATCH[1])*(target_w//_VIDEO_PATCH[2])
|
||||||
|
keyframes=list(payload.get("keyframes") or ())
|
||||||
|
refs=list(payload.get("refs") or ())
|
||||||
|
visual_blocks=[]; audio_blocks=[]; geometry_errors=[]
|
||||||
|
visual_index=audio_index=0
|
||||||
|
for keyframe in keyframes:
|
||||||
|
if not isinstance(keyframe,dict):
|
||||||
|
geometry_errors.append("keyframe entry is not a dict")
|
||||||
|
continue
|
||||||
|
latent=keyframe.get("latent")
|
||||||
|
if latent is not None:
|
||||||
|
visual_index+=1
|
||||||
|
shape=tuple(int(value) for value in getattr(latent,"shape",()))
|
||||||
|
latent_t=shape[2] if len(shape)==5 else 0
|
||||||
|
visual_blocks.append(_video_block(
|
||||||
|
index=visual_index,
|
||||||
|
kind="keyframe",
|
||||||
|
latent=latent,
|
||||||
|
expected_rows=latent_t*target_frame_rows,
|
||||||
|
metadata={
|
||||||
|
"resolved_frame_index":keyframe.get("resolved_frame_index")
|
||||||
|
},
|
||||||
|
errors=geometry_errors,
|
||||||
|
))
|
||||||
|
audio_latent=keyframe.get("audio_latent")
|
||||||
|
if audio_latent is not None:
|
||||||
|
audio_index+=1
|
||||||
|
shape=tuple(int(value) for value in getattr(audio_latent,"shape",()))
|
||||||
|
latent_t=shape[-1] if len(shape)==4 else 0
|
||||||
|
audio_blocks.append(_audio_block(
|
||||||
|
index=audio_index,
|
||||||
|
kind="keyframe",
|
||||||
|
latent=audio_latent,
|
||||||
|
expected_rows=latent_t*_AUDIO_STREAMS,
|
||||||
|
metadata={
|
||||||
|
"resolved_frame_index":keyframe.get("resolved_frame_index")
|
||||||
|
},
|
||||||
|
errors=geometry_errors,
|
||||||
|
))
|
||||||
|
for ref in refs:
|
||||||
|
if not isinstance(ref,dict):
|
||||||
|
geometry_errors.append("reference entry is not a dict")
|
||||||
|
continue
|
||||||
|
kind=str(ref.get("kind",""))
|
||||||
|
if kind=="image":
|
||||||
|
visual_index+=1
|
||||||
|
latent_h=int(ref.get("latent_h",0)); latent_w=int(ref.get("latent_w",0))
|
||||||
|
visual_blocks.append(_video_block(
|
||||||
|
index=visual_index,
|
||||||
|
kind="ref_image",
|
||||||
|
latent=ref.get("latent"),
|
||||||
|
expected_rows=(latent_h//2)*(latent_w//2),
|
||||||
|
metadata={"latent_t":1,"latent_h":latent_h,"latent_w":latent_w},
|
||||||
|
errors=geometry_errors,
|
||||||
|
))
|
||||||
|
block=visual_blocks[-1]
|
||||||
|
if block["shape"] is not None:
|
||||||
|
_b,_c,actual_t,actual_h,actual_w=block["shape"]
|
||||||
|
metadata_errors=[]
|
||||||
|
if actual_t!=1:
|
||||||
|
metadata_errors.append(
|
||||||
|
f"image reference latent_t must be 1, got {actual_t}"
|
||||||
|
)
|
||||||
|
if (actual_h,actual_w)!=(latent_h,latent_w):
|
||||||
|
metadata_errors.append(
|
||||||
|
"reference metadata H/W "
|
||||||
|
f"{(latent_h,latent_w)} != latent H/W {(actual_h,actual_w)}"
|
||||||
|
)
|
||||||
|
if metadata_errors:
|
||||||
|
geometry_errors.extend(metadata_errors)
|
||||||
|
block["errors"]=tuple((*block["errors"],*metadata_errors))
|
||||||
|
elif kind in ("video","video_audio"):
|
||||||
|
visual_index+=1
|
||||||
|
latent_t=int(ref.get("latent_t",0)); latent_h=int(ref.get("latent_h",0)); latent_w=int(ref.get("latent_w",0))
|
||||||
|
visual_blocks.append(_video_block(
|
||||||
|
index=visual_index,
|
||||||
|
kind=f"ref_{kind}",
|
||||||
|
latent=ref.get("latent"),
|
||||||
|
expected_rows=latent_t*(latent_h//2)*(latent_w//2),
|
||||||
|
metadata={"latent_t":latent_t,"latent_h":latent_h,"latent_w":latent_w},
|
||||||
|
errors=geometry_errors,
|
||||||
|
))
|
||||||
|
block=visual_blocks[-1]
|
||||||
|
if block["shape"] is not None:
|
||||||
|
_b,_c,actual_t,actual_h,actual_w=block["shape"]
|
||||||
|
metadata_errors=[]
|
||||||
|
if (actual_t,actual_h,actual_w)!=(latent_t,latent_h,latent_w):
|
||||||
|
metadata_errors.append(
|
||||||
|
"reference metadata T/H/W "
|
||||||
|
f"{(latent_t,latent_h,latent_w)} != latent T/H/W "
|
||||||
|
f"{(actual_t,actual_h,actual_w)}"
|
||||||
|
)
|
||||||
|
if metadata_errors:
|
||||||
|
geometry_errors.extend(metadata_errors)
|
||||||
|
block["errors"]=tuple((*block["errors"],*metadata_errors))
|
||||||
|
elif kind not in ("audio",):
|
||||||
|
geometry_errors.append(f"unsupported reference kind: {kind!r}")
|
||||||
|
if kind in ("audio","video","video_audio"):
|
||||||
|
ref_audio_t=int(ref.get("ref_audio_t",0))
|
||||||
|
audio_latent=ref.get("audio_latent")
|
||||||
|
if ref_audio_t>0 or audio_latent is not None:
|
||||||
|
audio_index+=1
|
||||||
|
audio_blocks.append(_audio_block(
|
||||||
|
index=audio_index,
|
||||||
|
kind=f"ref_{kind}",
|
||||||
|
latent=audio_latent,
|
||||||
|
expected_rows=ref_audio_t*_AUDIO_STREAMS,
|
||||||
|
metadata={"ref_audio_t":ref_audio_t},
|
||||||
|
errors=geometry_errors,
|
||||||
|
))
|
||||||
|
block=audio_blocks[-1]
|
||||||
|
if block["shape"] is not None:
|
||||||
|
actual_t=block["shape"][-1]
|
||||||
|
if actual_t!=ref_audio_t:
|
||||||
|
message=(
|
||||||
|
f"reference metadata audio_t {ref_audio_t} != "
|
||||||
|
f"latent audio_t {actual_t}"
|
||||||
|
)
|
||||||
|
geometry_errors.append(message)
|
||||||
|
block["errors"]=tuple((*block["errors"],message))
|
||||||
|
layout_visual_rows,layout_audio_rows=_layout_condition_rows(layout)
|
||||||
|
actual_visual_rows=sum(
|
||||||
|
int(block["actual_rows"] or 0) for block in visual_blocks
|
||||||
|
)
|
||||||
|
actual_audio_rows=sum(
|
||||||
|
int(block["actual_rows"] or 0) for block in audio_blocks
|
||||||
|
)
|
||||||
|
report={
|
||||||
|
"status":"matched",
|
||||||
|
"repaired":False,
|
||||||
|
"visual_blocks":tuple(visual_blocks),
|
||||||
|
"audio_blocks":tuple(audio_blocks),
|
||||||
|
"layout_visual_rows":int(layout_visual_rows),
|
||||||
|
"actual_visual_rows":int(actual_visual_rows),
|
||||||
|
"layout_audio_rows":int(layout_audio_rows),
|
||||||
|
"actual_audio_rows":int(actual_audio_rows),
|
||||||
|
}
|
||||||
|
if geometry_errors:
|
||||||
|
raise LayoutCompatibilityError(_format_preflight_failure(
|
||||||
|
report,"Unsafe geometry mismatch."
|
||||||
|
))
|
||||||
|
rows_match=(
|
||||||
|
layout_visual_rows==actual_visual_rows
|
||||||
|
and layout_audio_rows==actual_audio_rows
|
||||||
|
)
|
||||||
|
if rows_match:
|
||||||
|
return report
|
||||||
|
if not repair_stale:
|
||||||
|
raise LayoutCompatibilityError(_format_preflight_failure(
|
||||||
|
report,"PackedLayout rows do not match the current payload."
|
||||||
|
))
|
||||||
|
try:
|
||||||
|
rebuilt=type(layout)(
|
||||||
|
*signature,
|
||||||
|
keyframes=keyframes or None,
|
||||||
|
refs=refs or None,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise LayoutCompatibilityError(_format_preflight_failure(
|
||||||
|
report,
|
||||||
|
"PackedLayout is stale, but a fresh layout could not be constructed: "
|
||||||
|
f"{type(exc).__name__}: {exc}",
|
||||||
|
)) from exc
|
||||||
|
rebuilt_visual_rows,rebuilt_audio_rows=_layout_condition_rows(rebuilt)
|
||||||
|
if (
|
||||||
|
rebuilt_visual_rows!=actual_visual_rows
|
||||||
|
or rebuilt_audio_rows!=actual_audio_rows
|
||||||
|
):
|
||||||
|
raise LayoutCompatibilityError(_format_preflight_failure(
|
||||||
|
report,
|
||||||
|
"A fresh PackedLayout still does not match the current payload; "
|
||||||
|
"automatic repair is unsafe.",
|
||||||
|
))
|
||||||
|
payload["layout"]=rebuilt
|
||||||
|
report.update({
|
||||||
|
"status":"layout_rebuilt",
|
||||||
|
"repaired":True,
|
||||||
|
"layout_visual_rows":int(rebuilt_visual_rows),
|
||||||
|
"layout_audio_rows":int(rebuilt_audio_rows),
|
||||||
|
})
|
||||||
|
return report
|
||||||
def _input_device(x):
|
def _input_device(x):
|
||||||
device=getattr(x,"device",None)
|
device=getattr(x,"device",None)
|
||||||
if isinstance(device,torch.device): return device
|
if isinstance(device,torch.device): return device
|
||||||
|
|||||||
+26
-10
@@ -19,6 +19,7 @@ from .layout_adapter import (
|
|||||||
materialize_continuum_latents,
|
materialize_continuum_latents,
|
||||||
normalize_condition_latents,
|
normalize_condition_latents,
|
||||||
patch_layout_in_place,
|
patch_layout_in_place,
|
||||||
|
preflight_packed_layout,
|
||||||
payload_has_continuum,
|
payload_has_continuum,
|
||||||
validate_native_continuity_layout,
|
validate_native_continuity_layout,
|
||||||
)
|
)
|
||||||
@@ -165,16 +166,15 @@ def _wrapper_factory(
|
|||||||
|
|
||||||
def apply_model_wrapper(executor, *args, **kwargs):
|
def apply_model_wrapper(executor, *args, **kwargs):
|
||||||
payload = kwargs.get("minimax_payload")
|
payload = kwargs.get("minimax_payload")
|
||||||
if packed_row_planner is not None:
|
|
||||||
from .v3.packed_row_planner import observe_layout_fail_soft
|
|
||||||
observe_layout_fail_soft(packed_row_planner, payload)
|
|
||||||
has_continuum = payload_has_continuum(payload)
|
has_continuum = payload_has_continuum(payload)
|
||||||
has_mixed_keyframe_refs = (
|
has_condition_blocks = (
|
||||||
isinstance(payload, dict)
|
isinstance(payload, dict)
|
||||||
and bool(payload.get("keyframes"))
|
and (bool(payload.get("keyframes")) or bool(payload.get("refs")))
|
||||||
and payload.get("refs") is not None
|
|
||||||
)
|
)
|
||||||
if not has_continuum and not has_mixed_keyframe_refs:
|
if not has_continuum and not has_condition_blocks:
|
||||||
|
if packed_row_planner is not None:
|
||||||
|
from .v3.packed_row_planner import observe_layout_fail_soft
|
||||||
|
observe_layout_fail_soft(packed_row_planner, payload)
|
||||||
return executor(*args, **kwargs)
|
return executor(*args, **kwargs)
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
@@ -186,6 +186,21 @@ def _wrapper_factory(
|
|||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
patched_payload = dict(payload)
|
patched_payload = dict(payload)
|
||||||
|
# Rebuild Core's flattened condition lists from the authoritative
|
||||||
|
# keyframe/reference blocks before checking their row contract.
|
||||||
|
normalize_condition_latents(patched_payload)
|
||||||
|
preflight_result = preflight_packed_layout(
|
||||||
|
patched_payload,
|
||||||
|
repair_stale=True,
|
||||||
|
)
|
||||||
|
if preflight_result.get("repaired"):
|
||||||
|
# The wrapper is called for every model evaluation. Persist a
|
||||||
|
# proven-safe replacement on the queue-local payload so later
|
||||||
|
# steps preserve one layout and one position_ids identity.
|
||||||
|
payload["layout"] = patched_payload["layout"]
|
||||||
|
if packed_row_planner is not None:
|
||||||
|
from .v3.packed_row_planner import observe_layout_fail_soft
|
||||||
|
observe_layout_fail_soft(packed_row_planner, patched_payload)
|
||||||
if has_continuum:
|
if has_continuum:
|
||||||
patch_layout_in_place(patched_payload, strict=True, debug=debug)
|
patch_layout_in_place(patched_payload, strict=True, debug=debug)
|
||||||
transformer_options = kwargs.get("transformer_options")
|
transformer_options = kwargs.get("transformer_options")
|
||||||
@@ -219,10 +234,11 @@ def _wrapper_factory(
|
|||||||
)
|
)
|
||||||
input_x = args[0] if args else None
|
input_x = args[0] if args else None
|
||||||
materialize_continuum_latents(patched_payload, input_x, debug=debug)
|
materialize_continuum_latents(patched_payload, input_x, debug=debug)
|
||||||
# ComfyUI Core 0.33.1 replaces keyframe condition latents with
|
# Materialization may replace CPU block tensors with device-local
|
||||||
# reference latents when both contracts are present. Rebuild the
|
# tensors. Rebuild the flattened lists once more, then prove the
|
||||||
# combined list for First/Last Frame + standalone Reference Audio.
|
# final payload still matches the repaired/preserved layout.
|
||||||
normalize_condition_latents(patched_payload)
|
normalize_condition_latents(patched_payload)
|
||||||
|
preflight_packed_layout(patched_payload, repair_stale=False)
|
||||||
kwargs["minimax_payload"] = patched_payload
|
kwargs["minimax_payload"] = patched_payload
|
||||||
except (CompatibilityError, LayoutCompatibilityError, KeyError, TypeError, ValueError) as exc:
|
except (CompatibilityError, LayoutCompatibilityError, KeyError, TypeError, ValueError) as exc:
|
||||||
raise RuntimeError(f"H3 Continuum Join compatibility failure: {exc}") from exc
|
raise RuntimeError(f"H3 Continuum Join compatibility failure: {exc}") from exc
|
||||||
|
|||||||
+43
-10
@@ -54,17 +54,17 @@ def _layout():
|
|||||||
pos[3, 0] = 1
|
pos[3, 0] = 1
|
||||||
pos[4:6, 0] = 2
|
pos[4:6, 0] = 2
|
||||||
pos[6, 0] = 2
|
pos[6, 0] = 2
|
||||||
return SimpleNamespace(segments=segments, position_ids=pos, signature=(1, 1, 1, 1, 1))
|
return SimpleNamespace(segments=segments, position_ids=pos, signature=(1, 1, 2, 2, 1))
|
||||||
|
|
||||||
|
|
||||||
def test_apply_model_wrapper_repairs_payload_before_executor():
|
def test_apply_model_wrapper_repairs_payload_before_executor():
|
||||||
video = torch.zeros(1, 24, 1, 1, 1)
|
video = torch.zeros(1, 24, 1, 2, 2)
|
||||||
audio = torch.zeros(1, 32, 2, 1)
|
audio = torch.zeros(1, 32, 2, 1)
|
||||||
ref = {
|
ref = {
|
||||||
"kind": "video_audio",
|
"kind": "video_audio",
|
||||||
"latent_t": 1,
|
"latent_t": 1,
|
||||||
"latent_h": 1,
|
"latent_h": 2,
|
||||||
"latent_w": 1,
|
"latent_w": 2,
|
||||||
"ref_audio_t": 1,
|
"ref_audio_t": 1,
|
||||||
"latent": video,
|
"latent": video,
|
||||||
"audio_latent": audio,
|
"audio_latent": audio,
|
||||||
@@ -106,14 +106,47 @@ def test_apply_model_wrapper_repairs_keyframes_with_audio_reference_without_cont
|
|||||||
assert result["cond_audio_latents"] == [audio]
|
assert result["cond_audio_latents"] == [audio]
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_model_wrapper_persists_a_proven_stale_layout_repair(monkeypatch):
|
||||||
|
import ComfyUI_H3_Continuum_Join.model_patch as model_patch
|
||||||
|
|
||||||
|
audio = torch.zeros(1, 32, 2, 1)
|
||||||
|
old_layout = object()
|
||||||
|
replacement = object()
|
||||||
|
payload = {
|
||||||
|
"layout": old_layout,
|
||||||
|
"keyframes": [
|
||||||
|
{"resolved_frame_index": 0, "audio_latent": audio}
|
||||||
|
],
|
||||||
|
"refs": [],
|
||||||
|
}
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake_preflight(value, *, repair_stale):
|
||||||
|
calls.append(bool(repair_stale))
|
||||||
|
if repair_stale:
|
||||||
|
value["layout"] = replacement
|
||||||
|
return {"repaired": True}
|
||||||
|
assert value["layout"] is replacement
|
||||||
|
return {"repaired": False}
|
||||||
|
|
||||||
|
monkeypatch.setattr(model_patch, "preflight_packed_layout", fake_preflight)
|
||||||
|
wrapper = _wrapper_factory(strict=True, debug=False)
|
||||||
|
|
||||||
|
result = wrapper(Executor(), torch.zeros(1), minimax_payload=payload)
|
||||||
|
|
||||||
|
assert calls == [True, False]
|
||||||
|
assert payload["layout"] is replacement
|
||||||
|
assert result["layout"] is replacement
|
||||||
|
|
||||||
|
|
||||||
def test_apply_model_wrapper_fails_when_actual_topology_changes():
|
def test_apply_model_wrapper_fails_when_actual_topology_changes():
|
||||||
video = torch.zeros(1, 24, 1, 1, 1)
|
video = torch.zeros(1, 24, 1, 2, 2)
|
||||||
audio = torch.zeros(1, 32, 2, 1)
|
audio = torch.zeros(1, 32, 2, 1)
|
||||||
ref = {
|
ref = {
|
||||||
"kind": "video_audio",
|
"kind": "video_audio",
|
||||||
"latent_t": 1,
|
"latent_t": 1,
|
||||||
"latent_h": 1,
|
"latent_h": 2,
|
||||||
"latent_w": 1,
|
"latent_w": 2,
|
||||||
"ref_audio_t": 1,
|
"ref_audio_t": 1,
|
||||||
"latent": video,
|
"latent": video,
|
||||||
"audio_latent": audio,
|
"audio_latent": audio,
|
||||||
@@ -168,13 +201,13 @@ def test_layout_validation_profiler_records_full_calls_groups_and_step_contribut
|
|||||||
|
|
||||||
|
|
||||||
def test_apply_model_wrapper_reports_validator_status_without_changing_payload():
|
def test_apply_model_wrapper_reports_validator_status_without_changing_payload():
|
||||||
video = torch.zeros(1, 24, 1, 1, 1)
|
video = torch.zeros(1, 24, 1, 2, 2)
|
||||||
audio = torch.zeros(1, 32, 2, 1)
|
audio = torch.zeros(1, 32, 2, 1)
|
||||||
ref = {
|
ref = {
|
||||||
"kind": "video_audio",
|
"kind": "video_audio",
|
||||||
"latent_t": 1,
|
"latent_t": 1,
|
||||||
"latent_h": 1,
|
"latent_h": 2,
|
||||||
"latent_w": 1,
|
"latent_w": 2,
|
||||||
"ref_audio_t": 1,
|
"ref_audio_t": 1,
|
||||||
"latent": video,
|
"latent": video,
|
||||||
"audio_latent": audio,
|
"audio_latent": audio,
|
||||||
|
|||||||
@@ -0,0 +1,250 @@
|
|||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from ComfyUI_H3_Continuum_Join.layout_adapter import (
|
||||||
|
LayoutCompatibilityError,
|
||||||
|
normalize_condition_latents,
|
||||||
|
preflight_packed_layout,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FakePackedLayout:
|
||||||
|
"""Small Core-compatible row topology used by contract tests."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
text_len,
|
||||||
|
latent_t,
|
||||||
|
latent_h,
|
||||||
|
latent_w,
|
||||||
|
audio_t,
|
||||||
|
keyframes=None,
|
||||||
|
refs=None,
|
||||||
|
):
|
||||||
|
self.signature = (
|
||||||
|
int(text_len),
|
||||||
|
int(latent_t),
|
||||||
|
int(latent_h),
|
||||||
|
int(latent_w),
|
||||||
|
int(audio_t),
|
||||||
|
)
|
||||||
|
frame_rows = (int(latent_h) // 2) * (int(latent_w) // 2)
|
||||||
|
parts = [("text", int(text_len))]
|
||||||
|
for keyframe in keyframes or ():
|
||||||
|
latent = keyframe.get("latent")
|
||||||
|
if latent is not None:
|
||||||
|
parts.append(("cond", int(latent.shape[2]) * frame_rows))
|
||||||
|
audio = keyframe.get("audio_latent")
|
||||||
|
if audio is not None:
|
||||||
|
parts.append(("cond_audio", int(audio.shape[-1]) * 2))
|
||||||
|
for ref in refs or ():
|
||||||
|
kind = ref["kind"]
|
||||||
|
if kind == "image":
|
||||||
|
parts.append(
|
||||||
|
(
|
||||||
|
"ref_img",
|
||||||
|
(int(ref["latent_h"]) // 2)
|
||||||
|
* (int(ref["latent_w"]) // 2),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
elif kind == "audio":
|
||||||
|
if int(ref["ref_audio_t"]) > 0:
|
||||||
|
parts.append(("ref_audio", int(ref["ref_audio_t"]) * 2))
|
||||||
|
elif kind in ("video", "video_audio"):
|
||||||
|
if int(ref["ref_audio_t"]) > 0:
|
||||||
|
parts.append(("ref_audio", int(ref["ref_audio_t"]) * 2))
|
||||||
|
parts.append(
|
||||||
|
(
|
||||||
|
"ref_img",
|
||||||
|
int(ref["latent_t"])
|
||||||
|
* (int(ref["latent_h"]) // 2)
|
||||||
|
* (int(ref["latent_w"]) // 2),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"unsupported ref kind: {kind}")
|
||||||
|
parts.extend(
|
||||||
|
(
|
||||||
|
("audio", int(audio_t) * 2),
|
||||||
|
("video", int(latent_t) * frame_rows),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
row = 0
|
||||||
|
self.segments = []
|
||||||
|
for kind, count in parts:
|
||||||
|
self.segments.append((row, row + count, kind))
|
||||||
|
row += count
|
||||||
|
self.seq_len = row
|
||||||
|
self.position_ids = torch.zeros(row, 3, dtype=torch.float64)
|
||||||
|
|
||||||
|
|
||||||
|
def _layout(*, keyframes=None, refs=None):
|
||||||
|
return FakePackedLayout(
|
||||||
|
3,
|
||||||
|
2,
|
||||||
|
4,
|
||||||
|
6,
|
||||||
|
5,
|
||||||
|
keyframes=keyframes,
|
||||||
|
refs=refs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_accepts_self_consistent_visual_audio_and_multiple_blocks():
|
||||||
|
first = torch.zeros(1, 24, 1, 4, 6)
|
||||||
|
guide_audio = torch.zeros(1, 32, 2, 5)
|
||||||
|
image = torch.zeros(1, 24, 1, 2, 4)
|
||||||
|
video = torch.zeros(1, 24, 2, 2, 2)
|
||||||
|
ref_audio = torch.zeros(1, 32, 2, 3)
|
||||||
|
keyframes = [
|
||||||
|
{"resolved_frame_index": 0, "latent": first},
|
||||||
|
{"resolved_frame_index": 0, "audio_latent": guide_audio},
|
||||||
|
]
|
||||||
|
refs = [
|
||||||
|
{
|
||||||
|
"kind": "image",
|
||||||
|
"latent_h": 2,
|
||||||
|
"latent_w": 4,
|
||||||
|
"latent": image,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"kind": "video_audio",
|
||||||
|
"latent_t": 2,
|
||||||
|
"latent_h": 2,
|
||||||
|
"latent_w": 2,
|
||||||
|
"ref_audio_t": 3,
|
||||||
|
"latent": video,
|
||||||
|
"audio_latent": ref_audio,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
payload = {
|
||||||
|
"layout": _layout(keyframes=keyframes, refs=refs),
|
||||||
|
"keyframes": keyframes,
|
||||||
|
"refs": refs,
|
||||||
|
}
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
result = preflight_packed_layout(payload, repair_stale=False)
|
||||||
|
|
||||||
|
assert result["status"] == "matched"
|
||||||
|
assert result["layout_visual_rows"] == result["actual_visual_rows"] == 10
|
||||||
|
assert result["layout_audio_rows"] == result["actual_audio_rows"] == 16
|
||||||
|
assert len(result["visual_blocks"]) == 3
|
||||||
|
assert len(result["audio_blocks"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_repairs_only_a_stale_layout_from_current_payload():
|
||||||
|
first = torch.zeros(1, 24, 1, 4, 6)
|
||||||
|
guide_audio = torch.zeros(1, 32, 2, 5)
|
||||||
|
keyframes = [
|
||||||
|
{"resolved_frame_index": 0, "latent": first},
|
||||||
|
{"resolved_frame_index": 0, "audio_latent": guide_audio},
|
||||||
|
]
|
||||||
|
stale = _layout(keyframes=None, refs=None)
|
||||||
|
payload = {"layout": stale, "keyframes": keyframes, "refs": []}
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
result = preflight_packed_layout(payload, repair_stale=True)
|
||||||
|
|
||||||
|
assert result["status"] == "layout_rebuilt"
|
||||||
|
assert result["repaired"] is True
|
||||||
|
assert payload["layout"] is not stale
|
||||||
|
assert result["layout_visual_rows"] == result["actual_visual_rows"] == 6
|
||||||
|
assert result["layout_audio_rows"] == result["actual_audio_rows"] == 10
|
||||||
|
final = preflight_packed_layout(payload, repair_stale=False)
|
||||||
|
assert final["status"] == "matched"
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_rejects_keyframe_target_geometry_mismatch_without_mutation():
|
||||||
|
first = torch.zeros(1, 24, 1, 4, 4)
|
||||||
|
keyframes = [{"resolved_frame_index": 0, "latent": first}]
|
||||||
|
payload = {
|
||||||
|
"layout": _layout(keyframes=keyframes),
|
||||||
|
"keyframes": keyframes,
|
||||||
|
"refs": [],
|
||||||
|
}
|
||||||
|
original = first.clone()
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
with pytest.raises(LayoutCompatibilityError) as caught:
|
||||||
|
preflight_packed_layout(payload, repair_stale=True)
|
||||||
|
|
||||||
|
message = str(caught.value)
|
||||||
|
assert "Visual block #1" in message
|
||||||
|
assert "type: keyframe" in message
|
||||||
|
assert "resolved_frame_index: 0" in message
|
||||||
|
assert "latent shape: B=1 C=24 T=1 H=4 W=4" in message
|
||||||
|
assert "expected rows: 6" in message
|
||||||
|
assert "actual rows: 4" in message
|
||||||
|
assert "delta: -2" in message
|
||||||
|
assert "Unsafe geometry mismatch" in message
|
||||||
|
assert "Sampling was not started" in message
|
||||||
|
assert torch.equal(first, original)
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_rejects_reference_metadata_geometry_mismatch():
|
||||||
|
image = torch.zeros(1, 24, 1, 4, 4)
|
||||||
|
refs = [
|
||||||
|
{
|
||||||
|
"kind": "image",
|
||||||
|
"latent_h": 4,
|
||||||
|
"latent_w": 6,
|
||||||
|
"latent": image,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
payload = {"layout": _layout(refs=refs), "keyframes": [], "refs": refs}
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
with pytest.raises(LayoutCompatibilityError) as caught:
|
||||||
|
preflight_packed_layout(payload, repair_stale=True)
|
||||||
|
|
||||||
|
message = str(caught.value)
|
||||||
|
assert "type: ref_image" in message
|
||||||
|
assert "reference metadata H/W (4, 6) != latent H/W (4, 4)" in message
|
||||||
|
assert "expected rows: 6" in message
|
||||||
|
assert "actual rows: 4" in message
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_rejects_reference_audio_metadata_mismatch():
|
||||||
|
audio = torch.zeros(1, 32, 2, 3)
|
||||||
|
refs = [
|
||||||
|
{
|
||||||
|
"kind": "audio",
|
||||||
|
"ref_audio_t": 4,
|
||||||
|
"audio_latent": audio,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
payload = {"layout": _layout(refs=refs), "keyframes": [], "refs": refs}
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
with pytest.raises(LayoutCompatibilityError) as caught:
|
||||||
|
preflight_packed_layout(payload, repair_stale=True)
|
||||||
|
|
||||||
|
message = str(caught.value)
|
||||||
|
assert "Audio block #1" in message
|
||||||
|
assert "type: ref_audio" in message
|
||||||
|
assert "expected rows: 8" in message
|
||||||
|
assert "actual rows: 6" in message
|
||||||
|
assert "reference metadata audio_t 4 != latent audio_t 3" in message
|
||||||
|
|
||||||
|
|
||||||
|
def test_preflight_does_not_invent_a_layout_when_core_has_not_built_one():
|
||||||
|
payload = {
|
||||||
|
"keyframes": [
|
||||||
|
{
|
||||||
|
"resolved_frame_index": 0,
|
||||||
|
"audio_latent": torch.zeros(1, 32, 2, 3),
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refs": [],
|
||||||
|
}
|
||||||
|
normalize_condition_latents(payload)
|
||||||
|
|
||||||
|
result = preflight_packed_layout(payload, repair_stale=True)
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"status": "layout_unavailable",
|
||||||
|
"repaired": False,
|
||||||
|
"visual_blocks": (),
|
||||||
|
"audio_blocks": (),
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user