Files
ComfyUI-H3-Continuum-Tiled/layout_adapter.py
T

567 lines
22 KiB
Python

"""Per-run H3 payload normalization and MM-RoPE timeline adaptation.
No ComfyUI class is monkey-patched. ComfyUI builds a normal H3 PackedLayout;
this module then changes only its coordinates in place. Keeping the same
``position_ids`` Tensor identity preserves Sol-Attn's native-H3 span registry.
The preferred Continuum representation is one native H3 ``video`` or
``video_audio`` reference block. Its latent rows are moved onto the beginning
of the target timeline. The older marked-keyframe form remains supported as an
internal compatibility path, but the UI does not emit it.
"""
from __future__ import annotations
import logging
from typing import Any
import torch
from .constants import (
CONTINUUM_INTEROP_API,
CONTINUUM_REFERENCE_METADATA_KEY,
FRAME_RESCALE,
LAYOUT_DEVICE_CACHE_ATTR,
LAYOUT_ORIGINAL_TIME_ATTR,
LAYOUT_SIGNATURE_ATTR,
MARK_AUDIO_CONTEXT,
MARK_AUDIO_END_FRAME,
MARK_AUDIO_OVERHANG,
MARK_CONTEXT_FRAMES,
MARK_VIDEO_CONTEXT,
MARK_VIDEO_SLOT,
)
LOG = logging.getLogger("h3_continuum_join")
class LayoutCompatibilityError(RuntimeError):
pass
def _reference_metadata(ref: dict[str, Any]) -> dict[str, Any]:
value = ref.get(CONTINUUM_REFERENCE_METADATA_KEY)
return value if isinstance(value, dict) else {}
def _is_video_context(ref: dict[str, Any]) -> bool:
return bool(
ref.get(MARK_VIDEO_CONTEXT)
or _reference_metadata(ref).get("role") == "video_context"
)
def _is_audio_context(ref: dict[str, Any]) -> bool:
metadata = _reference_metadata(ref)
return bool(
ref.get(MARK_AUDIO_CONTEXT)
or metadata.get("role") == "audio_context"
or metadata.get("audio_role") == "audio_context"
)
def payload_has_continuum(payload: Any) -> bool:
if not isinstance(payload, dict):
return False
keyframes = payload.get("keyframes") or ()
refs = payload.get("refs") or ()
return any(MARK_VIDEO_SLOT in item for item in keyframes) or any(
_is_video_context(item) or _is_audio_context(item) for item in refs
)
def normalize_condition_latents(payload: dict[str, Any]) -> None:
"""Keep keyframe rows first and reference rows second, matching PackedLayout."""
keyframes = list(payload.get("keyframes") or ())
refs = list(payload.get("refs") or ())
video_latents = [item["latent"] for item in keyframes if item.get("latent") is not None]
video_latents.extend(item["latent"] for item in refs if item.get("latent") is not None)
audio_latents = [
item["audio_latent"] for item in refs if item.get("audio_latent") is not None
]
payload["cond_video_latents"] = video_latents
payload["cond_audio_latents"] = audio_latents
def _input_device(x: Any) -> torch.device | None:
device = getattr(x, "device", None)
if isinstance(device, torch.device):
return device
if hasattr(x, "unbind"):
try:
parts = x.unbind()
if parts:
part_device = getattr(parts[0], "device", None)
if isinstance(part_device, torch.device):
return part_device
except Exception:
return None
return None
def materialize_continuum_latents(
payload: dict[str, Any], x: Any, *, debug: bool = False,
) -> None:
"""Move only Continuum context tensors to the active device once per run.
Native H3 intentionally keeps payload dictionaries outside BaseModel's
generic dtype/device conversion. Without this cache, a CPU continuation
state would be transferred again on every solver step. Existing user refs
are left untouched to avoid unexpectedly pinning large Ref2VA inputs in
VRAM.
"""
layout = payload.get("layout")
device = _input_device(x)
if layout is None or device is None or device.type == "cpu":
return
keyframes = [dict(item) for item in (payload.get("keyframes") or ())]
refs = [dict(item) for item in (payload.get("refs") or ())]
marked_tensors: list[tuple[str, int, torch.Tensor]] = []
for index, item in enumerate(keyframes):
if MARK_VIDEO_SLOT in item and torch.is_tensor(item.get("latent")):
marked_tensors.append(("keyframe", index, item["latent"]))
for index, item in enumerate(refs):
if _is_video_context(item) and torch.is_tensor(item.get("latent")):
marked_tensors.append(("ref_video", index, item["latent"]))
if _is_audio_context(item) and torch.is_tensor(item.get("audio_latent")):
marked_tensors.append(("ref_audio", index, item["audio_latent"]))
if not marked_tensors:
return
cache_key = (
str(device),
tuple(
(kind, index, id(tensor), tuple(tensor.shape), str(tensor.dtype))
for kind, index, tensor in marked_tensors
),
)
cached = getattr(layout, LAYOUT_DEVICE_CACHE_ATTR, None)
if not isinstance(cached, dict) or cached.get("key") != cache_key:
values: dict[tuple[str, int], torch.Tensor] = {}
for kind, index, tensor in marked_tensors:
values[(kind, index)] = tensor.to(
device=device, dtype=torch.float32, non_blocking=True
).contiguous()
cached = {"key": cache_key, "values": values}
setattr(layout, LAYOUT_DEVICE_CACHE_ATTR, cached)
if debug:
total_bytes = sum(t.numel() * t.element_size() for t in values.values())
LOG.info(
"Continuum context cached on %s: %d tensors, %.2f MiB",
device,
len(values),
total_bytes / (1024 * 1024),
)
values = cached["values"]
for index, item in enumerate(keyframes):
replacement = values.get(("keyframe", index))
if replacement is not None:
item["latent"] = replacement
for index, item in enumerate(refs):
replacement = values.get(("ref_video", index))
if replacement is not None:
item["latent"] = replacement
replacement = values.get(("ref_audio", index))
if replacement is not None:
item["audio_latent"] = replacement
payload["keyframes"] = keyframes
payload["refs"] = refs
def _single_segment(layout: Any, kind: str) -> tuple[int, int]:
matches = [(int(a), int(b)) for a, b, k in layout.segments if k == kind]
if len(matches) != 1:
raise LayoutCompatibilityError(f"expected one H3 '{kind}' segment, found {len(matches)}")
return matches[0]
def _map_refs_to_segments(
layout: Any, refs: list[dict[str, Any]],
) -> list[dict[str, tuple[int, int] | None]]:
available = [
(int(a), int(b), str(kind))
for a, b, kind in layout.segments
if kind in ("ref_img", "ref_audio")
]
cursor = 0
result: list[dict[str, tuple[int, int] | None]] = []
def consume(expected: str) -> tuple[int, int]:
nonlocal cursor
if cursor >= len(available):
raise LayoutCompatibilityError(f"layout ended while mapping '{expected}'")
a, b, kind = available[cursor]
cursor += 1
if kind != expected:
raise LayoutCompatibilityError(
f"reference layout mismatch: expected '{expected}', found '{kind}'"
)
return a, b
for ref in refs:
kind = ref.get("kind")
mapping: dict[str, tuple[int, int] | None] = {"audio": None, "video": None}
if kind == "image":
mapping["video"] = consume("ref_img")
elif kind == "audio":
if int(ref.get("ref_audio_t", 0)) > 0:
mapping["audio"] = consume("ref_audio")
elif kind in ("video", "video_audio"):
if int(ref.get("ref_audio_t", 0)) > 0:
mapping["audio"] = consume("ref_audio")
mapping["video"] = consume("ref_img")
else:
raise LayoutCompatibilityError(f"unsupported H3 reference kind: {kind!r}")
result.append(mapping)
if cursor != len(available):
raise LayoutCompatibilityError(
f"layout contains {len(available)-cursor} unmapped reference segments"
)
return result
def _patch_signature(payload: dict[str, Any], layout: Any) -> tuple[Any, ...]:
return (
tuple(
(
int(kf.get("resolved_frame_index", -1)),
int(kf.get(MARK_VIDEO_SLOT, -1)),
)
for kf in (payload.get("keyframes") or ())
),
tuple(
(
bool(ref.get(MARK_VIDEO_CONTEXT)),
tuple(sorted(_reference_metadata(ref).items())),
int(ref.get("latent_t", 0)),
int(ref.get(MARK_CONTEXT_FRAMES, 0)),
bool(ref.get(MARK_AUDIO_CONTEXT)),
int(ref.get("ref_audio_t", 0)),
float(ref.get(MARK_AUDIO_END_FRAME, 0.0)),
float(ref.get(MARK_AUDIO_OVERHANG, 0.0)),
)
for ref in (payload.get("refs") or ())
),
tuple(getattr(layout, "signature", ())),
tuple((int(a), int(b), str(k)) for a, b, k in layout.segments),
)
def _branch_key(transformer_options: dict[str, Any]) -> tuple[Any, ...]:
conds = transformer_options.get("cond_or_uncond")
uuids = transformer_options.get("uuids")
if conds is None or uuids is None:
return ("default",)
try:
cond_values = tuple(int(value) for value in conds)
uuid_values = tuple(str(value) for value in uuids)
except (TypeError, ValueError):
return ("default",)
if not cond_values or len(cond_values) != len(uuid_values):
return ("default",)
return tuple(
(cond_values[index], uuid_values[index])
for index in range(len(cond_values))
)
def _sampled_tensor_signature(tensor: torch.Tensor | None) -> tuple[Any, ...] | None:
if tensor is None:
return None
value = tensor.detach().to(dtype=torch.float32)
flat = value.reshape(-1)
if flat.numel() <= 64:
sample = flat
else:
indices = torch.linspace(
0,
flat.numel() - 1,
64,
dtype=torch.long,
device=flat.device,
)
sample = flat.index_select(0, indices)
sample = sample.to("cpu")
finite = bool(torch.isfinite(sample).all().item())
if not finite:
raise LayoutCompatibilityError("Continuum layout/context contains NaN or Inf")
weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32)
return (
tuple(int(part) for part in tensor.shape),
str(tensor.dtype),
finite,
float(sample.mean().item()),
float(sample.std(unbiased=False).item()),
float((sample * weights).sum().item()),
)
def _ranges_overlap(left: tuple[int, int], right: tuple[int, int]) -> bool:
return max(left[0], right[0]) < min(left[1], right[1])
def validate_native_continuity_layout(
payload: dict[str, Any],
*,
transformer_options: dict[str, Any] | None,
branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]],
) -> dict[str, Any]:
"""Validate immutable native-H3 Context/Target topology per logical branch."""
layout = payload.get("layout")
if layout is None:
raise LayoutCompatibilityError("Native Continuity payload has no PackedLayout")
position_ids = getattr(layout, "position_ids", None)
segments = tuple(
(int(start), int(stop), str(kind))
for start, stop, kind in getattr(layout, "segments", ())
)
if not torch.is_tensor(position_ids) or position_ids.ndim != 2:
raise LayoutCompatibilityError("Native Continuity position_ids are unavailable")
seq_len = int(getattr(layout, "seq_len", position_ids.shape[0]))
if seq_len != int(position_ids.shape[0]):
raise LayoutCompatibilityError("PackedLayout seq_len does not match position_ids")
previous_stop = 0
for start, stop, kind in segments:
if start < previous_stop or stop <= start or stop > seq_len:
raise LayoutCompatibilityError(
f"PackedLayout segment {kind!r} has an overlapping or invalid row range"
)
previous_stop = stop
target_audio = [(a, b) for a, b, kind in segments if kind == "audio"]
target_video = [(a, b) for a, b, kind in segments if kind == "video"]
if len(target_audio) != 1 or len(target_video) != 1:
raise LayoutCompatibilityError(
"Native Continuity requires one target audio and one target video segment"
)
audio_range, video_range = target_audio[0], target_video[0]
if audio_range[1] != video_range[0] or video_range[1] != seq_len:
raise LayoutCompatibilityError(
"target segments must be the contiguous packed tail [audio | video]"
)
refs = list(payload.get("refs") or ())
mappings = _map_refs_to_segments(layout, refs)
context_ranges: list[tuple[int, int, str]] = []
context_descriptors: list[tuple[Any, ...]] = []
for ref, mapping in zip(refs, mappings):
metadata = _reference_metadata(ref)
if metadata:
if (
type(metadata.get("api")) is not int
or int(metadata["api"]) != CONTINUUM_INTEROP_API
):
raise LayoutCompatibilityError("unsupported Continuum reference metadata")
if _is_video_context(ref) and metadata.get("role") != "video_context":
raise LayoutCompatibilityError("Continuum video reference role is invalid")
if _is_audio_context(ref) and metadata.get("audio_role") not in (
None,
"audio_context",
):
raise LayoutCompatibilityError("Continuum audio reference role is invalid")
if _is_video_context(ref):
row_range = mapping.get("video")
if row_range is None:
raise LayoutCompatibilityError("Continuum video context has no reference rows")
context_ranges.append((*row_range, "video_context"))
if _is_audio_context(ref):
row_range = mapping.get("audio")
if row_range is None:
raise LayoutCompatibilityError("Continuum audio context has no reference rows")
context_ranges.append((*row_range, "audio_context"))
if _is_video_context(ref) or _is_audio_context(ref):
context_descriptors.append(
(
ref.get("kind"),
int(ref.get("latent_t", 0)),
int(ref.get("latent_h", 0)),
int(ref.get("latent_w", 0)),
int(ref.get("ref_audio_t", 0)),
tuple(sorted(metadata.items())),
_sampled_tensor_signature(ref.get("latent")),
_sampled_tensor_signature(ref.get("audio_latent")),
)
)
if not context_ranges:
raise LayoutCompatibilityError("Native Continuity payload has no context rows")
for start, stop, role in context_ranges:
row_range = (start, stop)
if _ranges_overlap(row_range, audio_range) or _ranges_overlap(row_range, video_range):
raise LayoutCompatibilityError(f"{role} rows overlap target rows")
topology = (
tuple(getattr(layout, "signature", ())),
segments,
audio_range,
video_range,
tuple(context_ranges),
tuple(context_descriptors),
int(payload.get("frame_count", 0) or 0),
_sampled_tensor_signature(position_ids),
)
options = transformer_options if isinstance(transformer_options, dict) else {}
if options.get("spectrum_h3_actual") is False:
return {"status": "forecast_validated", "branch": _branch_key(options)}
branch = _branch_key(options)
baseline = branch_baselines.get(branch)
if baseline is None:
branch_baselines[branch] = topology
return {"status": "baseline_created", "branch": branch}
if baseline != topology:
raise LayoutCompatibilityError(
f"Native Continuity topology changed during sampling for branch {branch!r}"
)
return {"status": "baseline_matched", "branch": branch}
def patch_layout_in_place(
payload: dict[str, Any], *, strict: bool = True, debug: bool = False,
) -> dict[str, Any]:
layout = payload.get("layout")
if layout is None:
raise LayoutCompatibilityError(
"H3 payload has no prebuilt PackedLayout; use ComfyUI v0.31.0 or newer"
)
missing = [name for name in ("position_ids", "segments", "signature") if not hasattr(layout, name)]
if missing:
raise LayoutCompatibilityError("PackedLayout is missing: " + ", ".join(missing))
position_ids = layout.position_ids
if not torch.is_tensor(position_ids) or position_ids.ndim != 2 or position_ids.shape[1] < 3:
raise LayoutCompatibilityError("position_ids must be an [S,3] Tensor")
signature = _patch_signature(payload, layout)
if getattr(layout, LAYOUT_SIGNATURE_ATTR, None) == signature:
return {"status": "already_patched", "position_ids_id": id(position_ids)}
if not hasattr(layout, LAYOUT_ORIGINAL_TIME_ATTR):
setattr(layout, LAYOUT_ORIGINAL_TIME_ATTR, position_ids[:, 0].clone())
original_time = getattr(layout, LAYOUT_ORIGINAL_TIME_ATTR)
if not torch.is_tensor(original_time) or original_time.shape != position_ids[:, 0].shape:
raise LayoutCompatibilityError("stored layout baseline is incompatible")
position_ids[:, 0].copy_(original_time)
keyframes = list(payload.get("keyframes") or ())
refs = list(payload.get("refs") or ())
cond_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "cond"]
if len(cond_segments) != len(keyframes):
raise LayoutCompatibilityError(
f"layout has {len(cond_segments)} cond segments for {len(keyframes)} keyframes"
)
video_start, video_stop = _single_segment(layout, "video")
_single_segment(layout, "audio")
text_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "text"]
if len(text_segments) != 1 or text_segments[0][0] != 0:
raise LayoutCompatibilityError("unexpected H3 text segment")
text_len = text_segments[0][1]
latent_t = int(layout.signature[1])
video_rows = video_stop - video_start
if latent_t <= 0 or video_rows % latent_t:
raise LayoutCompatibilityError(
f"video rows {video_rows} are incompatible with latent T={latent_t}"
)
frame_rows = video_rows // latent_t
target_origin = float(position_ids[video_start, 0])
reference_shift = target_origin - float(text_len)
# Existing first/last-frame keyframes are text-relative in stock H3. Once
# refs are present, move them by the target-origin shift. The legacy marked
# slot path is retained for compatibility with early development states.
patched_video_slots = 0
for (start, stop), keyframe in zip(cond_segments, keyframes):
if stop - start != frame_rows:
raise LayoutCompatibilityError("keyframe rows no longer equal one target slot")
if MARK_VIDEO_SLOT in keyframe:
slot = int(keyframe[MARK_VIDEO_SLOT])
if not (0 <= slot < latent_t):
raise LayoutCompatibilityError(
f"context slot {slot} is outside target latent T={latent_t}"
)
target_row = video_start + slot * frame_rows
position_ids[start:stop].copy_(
position_ids[target_row : target_row + frame_rows]
)
patched_video_slots += 1
elif reference_shift:
position_ids[start:stop, 0].add_(reference_shift)
ref_mappings = _map_refs_to_segments(layout, refs)
patched_video_refs = 0
patched_audio = 0
for ref, mapping in zip(refs, ref_mappings):
if _is_video_context(ref):
segment = mapping.get("video")
if segment is None:
raise LayoutCompatibilityError("Continuum video ref has no video segment")
start, stop = segment
context_t = int(ref.get("latent_t", 0))
expected_rows = context_t * frame_rows
if context_t <= 0 or stop - start != expected_rows:
raise LayoutCompatibilityError(
f"video-context rows {stop-start} do not match latent T={context_t}"
)
if context_t > latent_t:
raise LayoutCompatibilityError(
f"video context T={context_t} exceeds target latent T={latent_t}"
)
target_stop = video_start + expected_rows
# Copy all (t,h,w) coordinates. Resolution equality is already
# enforced by state validation; this also fails safely if row geometry changes.
position_ids[start:stop].copy_(position_ids[video_start:target_stop])
patched_video_refs += 1
if _is_audio_context(ref):
segment = mapping.get("audio")
if segment is None:
raise LayoutCompatibilityError("Continuum audio ref has no audio segment")
start, stop = segment
rt = int(ref.get("ref_audio_t", 0))
if rt <= 0 or stop - start != rt * 2:
raise LayoutCompatibilityError(
f"audio rows {stop-start} do not match {rt} stereo steps"
)
end_frame = float(ref.get(MARK_AUDIO_END_FRAME, 0.0))
grid_offset = float(ref.get(MARK_AUDIO_OVERHANG, 0.0))
desired_end = target_origin + FRAME_RESCALE * end_frame + grid_offset
desired_start = desired_end - float(rt)
old_start = float(position_ids[start, 0])
position_ids[start:stop, 0].add_(desired_start - old_start)
patched_audio += 1
patched_video = patched_video_slots + patched_video_refs
if patched_video == 0:
message = "no marked Continuum video context was found"
if strict:
raise LayoutCompatibilityError(message)
LOG.warning(message)
setattr(layout, LAYOUT_SIGNATURE_ATTR, signature)
if debug:
LOG.info(
"Continuum layout: video_refs=%d legacy_slots=%d audio_windows=%d "
"origin=%.6f rows=%d pos_id=%s",
patched_video_refs,
patched_video_slots,
patched_audio,
target_origin,
position_ids.shape[0],
id(position_ids),
)
return {
"status": "patched",
"video_contexts": patched_video,
"video_refs": patched_video_refs,
"legacy_video_slots": patched_video_slots,
"audio_windows": patched_audio,
"target_origin": target_origin,
"position_ids_id": id(position_ids),
}