Release V2.1.4 UI simplification
This commit is contained in:
@@ -0,0 +1,566 @@
|
||||
"""Per-run H3 payload normalization and MM-RoPE timeline adaptation.
|
||||
|
||||
No ComfyUI class is monkey-patched. ComfyUI builds a normal H3 PackedLayout;
|
||||
this module then changes only its coordinates in place. Keeping the same
|
||||
``position_ids`` Tensor identity preserves Sol-Attn's native-H3 span registry.
|
||||
|
||||
The preferred Continuum representation is one native H3 ``video`` or
|
||||
``video_audio`` reference block. Its latent rows are moved onto the beginning
|
||||
of the target timeline. The older marked-keyframe form remains supported as an
|
||||
internal compatibility path, but the UI does not emit it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .constants import (
|
||||
CONTINUUM_INTEROP_API,
|
||||
CONTINUUM_REFERENCE_METADATA_KEY,
|
||||
FRAME_RESCALE,
|
||||
LAYOUT_DEVICE_CACHE_ATTR,
|
||||
LAYOUT_ORIGINAL_TIME_ATTR,
|
||||
LAYOUT_SIGNATURE_ATTR,
|
||||
MARK_AUDIO_CONTEXT,
|
||||
MARK_AUDIO_END_FRAME,
|
||||
MARK_AUDIO_OVERHANG,
|
||||
MARK_CONTEXT_FRAMES,
|
||||
MARK_VIDEO_CONTEXT,
|
||||
MARK_VIDEO_SLOT,
|
||||
)
|
||||
|
||||
LOG = logging.getLogger("h3_continuum_join")
|
||||
|
||||
|
||||
class LayoutCompatibilityError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _reference_metadata(ref: dict[str, Any]) -> dict[str, Any]:
|
||||
value = ref.get(CONTINUUM_REFERENCE_METADATA_KEY)
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _is_video_context(ref: dict[str, Any]) -> bool:
|
||||
return bool(
|
||||
ref.get(MARK_VIDEO_CONTEXT)
|
||||
or _reference_metadata(ref).get("role") == "video_context"
|
||||
)
|
||||
|
||||
|
||||
def _is_audio_context(ref: dict[str, Any]) -> bool:
|
||||
metadata = _reference_metadata(ref)
|
||||
return bool(
|
||||
ref.get(MARK_AUDIO_CONTEXT)
|
||||
or metadata.get("role") == "audio_context"
|
||||
or metadata.get("audio_role") == "audio_context"
|
||||
)
|
||||
|
||||
|
||||
def payload_has_continuum(payload: Any) -> bool:
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
keyframes = payload.get("keyframes") or ()
|
||||
refs = payload.get("refs") or ()
|
||||
return any(MARK_VIDEO_SLOT in item for item in keyframes) or any(
|
||||
_is_video_context(item) or _is_audio_context(item) for item in refs
|
||||
)
|
||||
|
||||
|
||||
def normalize_condition_latents(payload: dict[str, Any]) -> None:
|
||||
"""Keep keyframe rows first and reference rows second, matching PackedLayout."""
|
||||
keyframes = list(payload.get("keyframes") or ())
|
||||
refs = list(payload.get("refs") or ())
|
||||
video_latents = [item["latent"] for item in keyframes if item.get("latent") is not None]
|
||||
video_latents.extend(item["latent"] for item in refs if item.get("latent") is not None)
|
||||
audio_latents = [
|
||||
item["audio_latent"] for item in refs if item.get("audio_latent") is not None
|
||||
]
|
||||
payload["cond_video_latents"] = video_latents
|
||||
payload["cond_audio_latents"] = audio_latents
|
||||
|
||||
|
||||
def _input_device(x: Any) -> torch.device | None:
|
||||
device = getattr(x, "device", None)
|
||||
if isinstance(device, torch.device):
|
||||
return device
|
||||
if hasattr(x, "unbind"):
|
||||
try:
|
||||
parts = x.unbind()
|
||||
if parts:
|
||||
part_device = getattr(parts[0], "device", None)
|
||||
if isinstance(part_device, torch.device):
|
||||
return part_device
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def materialize_continuum_latents(
|
||||
payload: dict[str, Any], x: Any, *, debug: bool = False,
|
||||
) -> None:
|
||||
"""Move only Continuum context tensors to the active device once per run.
|
||||
|
||||
Native H3 intentionally keeps payload dictionaries outside BaseModel's
|
||||
generic dtype/device conversion. Without this cache, a CPU continuation
|
||||
state would be transferred again on every solver step. Existing user refs
|
||||
are left untouched to avoid unexpectedly pinning large Ref2VA inputs in
|
||||
VRAM.
|
||||
"""
|
||||
layout = payload.get("layout")
|
||||
device = _input_device(x)
|
||||
if layout is None or device is None or device.type == "cpu":
|
||||
return
|
||||
|
||||
keyframes = [dict(item) for item in (payload.get("keyframes") or ())]
|
||||
refs = [dict(item) for item in (payload.get("refs") or ())]
|
||||
marked_tensors: list[tuple[str, int, torch.Tensor]] = []
|
||||
for index, item in enumerate(keyframes):
|
||||
if MARK_VIDEO_SLOT in item and torch.is_tensor(item.get("latent")):
|
||||
marked_tensors.append(("keyframe", index, item["latent"]))
|
||||
for index, item in enumerate(refs):
|
||||
if _is_video_context(item) and torch.is_tensor(item.get("latent")):
|
||||
marked_tensors.append(("ref_video", index, item["latent"]))
|
||||
if _is_audio_context(item) and torch.is_tensor(item.get("audio_latent")):
|
||||
marked_tensors.append(("ref_audio", index, item["audio_latent"]))
|
||||
if not marked_tensors:
|
||||
return
|
||||
|
||||
cache_key = (
|
||||
str(device),
|
||||
tuple(
|
||||
(kind, index, id(tensor), tuple(tensor.shape), str(tensor.dtype))
|
||||
for kind, index, tensor in marked_tensors
|
||||
),
|
||||
)
|
||||
cached = getattr(layout, LAYOUT_DEVICE_CACHE_ATTR, None)
|
||||
if not isinstance(cached, dict) or cached.get("key") != cache_key:
|
||||
values: dict[tuple[str, int], torch.Tensor] = {}
|
||||
for kind, index, tensor in marked_tensors:
|
||||
values[(kind, index)] = tensor.to(
|
||||
device=device, dtype=torch.float32, non_blocking=True
|
||||
).contiguous()
|
||||
cached = {"key": cache_key, "values": values}
|
||||
setattr(layout, LAYOUT_DEVICE_CACHE_ATTR, cached)
|
||||
if debug:
|
||||
total_bytes = sum(t.numel() * t.element_size() for t in values.values())
|
||||
LOG.info(
|
||||
"Continuum context cached on %s: %d tensors, %.2f MiB",
|
||||
device,
|
||||
len(values),
|
||||
total_bytes / (1024 * 1024),
|
||||
)
|
||||
|
||||
values = cached["values"]
|
||||
for index, item in enumerate(keyframes):
|
||||
replacement = values.get(("keyframe", index))
|
||||
if replacement is not None:
|
||||
item["latent"] = replacement
|
||||
for index, item in enumerate(refs):
|
||||
replacement = values.get(("ref_video", index))
|
||||
if replacement is not None:
|
||||
item["latent"] = replacement
|
||||
replacement = values.get(("ref_audio", index))
|
||||
if replacement is not None:
|
||||
item["audio_latent"] = replacement
|
||||
payload["keyframes"] = keyframes
|
||||
payload["refs"] = refs
|
||||
|
||||
|
||||
def _single_segment(layout: Any, kind: str) -> tuple[int, int]:
|
||||
matches = [(int(a), int(b)) for a, b, k in layout.segments if k == kind]
|
||||
if len(matches) != 1:
|
||||
raise LayoutCompatibilityError(f"expected one H3 '{kind}' segment, found {len(matches)}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _map_refs_to_segments(
|
||||
layout: Any, refs: list[dict[str, Any]],
|
||||
) -> list[dict[str, tuple[int, int] | None]]:
|
||||
available = [
|
||||
(int(a), int(b), str(kind))
|
||||
for a, b, kind in layout.segments
|
||||
if kind in ("ref_img", "ref_audio")
|
||||
]
|
||||
cursor = 0
|
||||
result: list[dict[str, tuple[int, int] | None]] = []
|
||||
|
||||
def consume(expected: str) -> tuple[int, int]:
|
||||
nonlocal cursor
|
||||
if cursor >= len(available):
|
||||
raise LayoutCompatibilityError(f"layout ended while mapping '{expected}'")
|
||||
a, b, kind = available[cursor]
|
||||
cursor += 1
|
||||
if kind != expected:
|
||||
raise LayoutCompatibilityError(
|
||||
f"reference layout mismatch: expected '{expected}', found '{kind}'"
|
||||
)
|
||||
return a, b
|
||||
|
||||
for ref in refs:
|
||||
kind = ref.get("kind")
|
||||
mapping: dict[str, tuple[int, int] | None] = {"audio": None, "video": None}
|
||||
if kind == "image":
|
||||
mapping["video"] = consume("ref_img")
|
||||
elif kind == "audio":
|
||||
if int(ref.get("ref_audio_t", 0)) > 0:
|
||||
mapping["audio"] = consume("ref_audio")
|
||||
elif kind in ("video", "video_audio"):
|
||||
if int(ref.get("ref_audio_t", 0)) > 0:
|
||||
mapping["audio"] = consume("ref_audio")
|
||||
mapping["video"] = consume("ref_img")
|
||||
else:
|
||||
raise LayoutCompatibilityError(f"unsupported H3 reference kind: {kind!r}")
|
||||
result.append(mapping)
|
||||
if cursor != len(available):
|
||||
raise LayoutCompatibilityError(
|
||||
f"layout contains {len(available)-cursor} unmapped reference segments"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _patch_signature(payload: dict[str, Any], layout: Any) -> tuple[Any, ...]:
|
||||
return (
|
||||
tuple(
|
||||
(
|
||||
int(kf.get("resolved_frame_index", -1)),
|
||||
int(kf.get(MARK_VIDEO_SLOT, -1)),
|
||||
)
|
||||
for kf in (payload.get("keyframes") or ())
|
||||
),
|
||||
tuple(
|
||||
(
|
||||
bool(ref.get(MARK_VIDEO_CONTEXT)),
|
||||
tuple(sorted(_reference_metadata(ref).items())),
|
||||
int(ref.get("latent_t", 0)),
|
||||
int(ref.get(MARK_CONTEXT_FRAMES, 0)),
|
||||
bool(ref.get(MARK_AUDIO_CONTEXT)),
|
||||
int(ref.get("ref_audio_t", 0)),
|
||||
float(ref.get(MARK_AUDIO_END_FRAME, 0.0)),
|
||||
float(ref.get(MARK_AUDIO_OVERHANG, 0.0)),
|
||||
)
|
||||
for ref in (payload.get("refs") or ())
|
||||
),
|
||||
tuple(getattr(layout, "signature", ())),
|
||||
tuple((int(a), int(b), str(k)) for a, b, k in layout.segments),
|
||||
)
|
||||
|
||||
|
||||
def _branch_key(transformer_options: dict[str, Any]) -> tuple[Any, ...]:
|
||||
conds = transformer_options.get("cond_or_uncond")
|
||||
uuids = transformer_options.get("uuids")
|
||||
if conds is None or uuids is None:
|
||||
return ("default",)
|
||||
try:
|
||||
cond_values = tuple(int(value) for value in conds)
|
||||
uuid_values = tuple(str(value) for value in uuids)
|
||||
except (TypeError, ValueError):
|
||||
return ("default",)
|
||||
if not cond_values or len(cond_values) != len(uuid_values):
|
||||
return ("default",)
|
||||
return tuple(
|
||||
(cond_values[index], uuid_values[index])
|
||||
for index in range(len(cond_values))
|
||||
)
|
||||
|
||||
|
||||
def _sampled_tensor_signature(tensor: torch.Tensor | None) -> tuple[Any, ...] | None:
|
||||
if tensor is None:
|
||||
return None
|
||||
value = tensor.detach().to(dtype=torch.float32)
|
||||
flat = value.reshape(-1)
|
||||
if flat.numel() <= 64:
|
||||
sample = flat
|
||||
else:
|
||||
indices = torch.linspace(
|
||||
0,
|
||||
flat.numel() - 1,
|
||||
64,
|
||||
dtype=torch.long,
|
||||
device=flat.device,
|
||||
)
|
||||
sample = flat.index_select(0, indices)
|
||||
sample = sample.to("cpu")
|
||||
finite = bool(torch.isfinite(sample).all().item())
|
||||
if not finite:
|
||||
raise LayoutCompatibilityError("Continuum layout/context contains NaN or Inf")
|
||||
weights = torch.arange(1, sample.numel() + 1, dtype=torch.float32)
|
||||
return (
|
||||
tuple(int(part) for part in tensor.shape),
|
||||
str(tensor.dtype),
|
||||
finite,
|
||||
float(sample.mean().item()),
|
||||
float(sample.std(unbiased=False).item()),
|
||||
float((sample * weights).sum().item()),
|
||||
)
|
||||
|
||||
|
||||
def _ranges_overlap(left: tuple[int, int], right: tuple[int, int]) -> bool:
|
||||
return max(left[0], right[0]) < min(left[1], right[1])
|
||||
|
||||
|
||||
def validate_native_continuity_layout(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
transformer_options: dict[str, Any] | None,
|
||||
branch_baselines: dict[tuple[Any, ...], tuple[Any, ...]],
|
||||
) -> dict[str, Any]:
|
||||
"""Validate immutable native-H3 Context/Target topology per logical branch."""
|
||||
|
||||
layout = payload.get("layout")
|
||||
if layout is None:
|
||||
raise LayoutCompatibilityError("Native Continuity payload has no PackedLayout")
|
||||
position_ids = getattr(layout, "position_ids", None)
|
||||
segments = tuple(
|
||||
(int(start), int(stop), str(kind))
|
||||
for start, stop, kind in getattr(layout, "segments", ())
|
||||
)
|
||||
if not torch.is_tensor(position_ids) or position_ids.ndim != 2:
|
||||
raise LayoutCompatibilityError("Native Continuity position_ids are unavailable")
|
||||
seq_len = int(getattr(layout, "seq_len", position_ids.shape[0]))
|
||||
if seq_len != int(position_ids.shape[0]):
|
||||
raise LayoutCompatibilityError("PackedLayout seq_len does not match position_ids")
|
||||
|
||||
previous_stop = 0
|
||||
for start, stop, kind in segments:
|
||||
if start < previous_stop or stop <= start or stop > seq_len:
|
||||
raise LayoutCompatibilityError(
|
||||
f"PackedLayout segment {kind!r} has an overlapping or invalid row range"
|
||||
)
|
||||
previous_stop = stop
|
||||
|
||||
target_audio = [(a, b) for a, b, kind in segments if kind == "audio"]
|
||||
target_video = [(a, b) for a, b, kind in segments if kind == "video"]
|
||||
if len(target_audio) != 1 or len(target_video) != 1:
|
||||
raise LayoutCompatibilityError(
|
||||
"Native Continuity requires one target audio and one target video segment"
|
||||
)
|
||||
audio_range, video_range = target_audio[0], target_video[0]
|
||||
if audio_range[1] != video_range[0] or video_range[1] != seq_len:
|
||||
raise LayoutCompatibilityError(
|
||||
"target segments must be the contiguous packed tail [audio | video]"
|
||||
)
|
||||
|
||||
refs = list(payload.get("refs") or ())
|
||||
mappings = _map_refs_to_segments(layout, refs)
|
||||
context_ranges: list[tuple[int, int, str]] = []
|
||||
context_descriptors: list[tuple[Any, ...]] = []
|
||||
for ref, mapping in zip(refs, mappings):
|
||||
metadata = _reference_metadata(ref)
|
||||
if metadata:
|
||||
if (
|
||||
type(metadata.get("api")) is not int
|
||||
or int(metadata["api"]) != CONTINUUM_INTEROP_API
|
||||
):
|
||||
raise LayoutCompatibilityError("unsupported Continuum reference metadata")
|
||||
if _is_video_context(ref) and metadata.get("role") != "video_context":
|
||||
raise LayoutCompatibilityError("Continuum video reference role is invalid")
|
||||
if _is_audio_context(ref) and metadata.get("audio_role") not in (
|
||||
None,
|
||||
"audio_context",
|
||||
):
|
||||
raise LayoutCompatibilityError("Continuum audio reference role is invalid")
|
||||
|
||||
if _is_video_context(ref):
|
||||
row_range = mapping.get("video")
|
||||
if row_range is None:
|
||||
raise LayoutCompatibilityError("Continuum video context has no reference rows")
|
||||
context_ranges.append((*row_range, "video_context"))
|
||||
if _is_audio_context(ref):
|
||||
row_range = mapping.get("audio")
|
||||
if row_range is None:
|
||||
raise LayoutCompatibilityError("Continuum audio context has no reference rows")
|
||||
context_ranges.append((*row_range, "audio_context"))
|
||||
if _is_video_context(ref) or _is_audio_context(ref):
|
||||
context_descriptors.append(
|
||||
(
|
||||
ref.get("kind"),
|
||||
int(ref.get("latent_t", 0)),
|
||||
int(ref.get("latent_h", 0)),
|
||||
int(ref.get("latent_w", 0)),
|
||||
int(ref.get("ref_audio_t", 0)),
|
||||
tuple(sorted(metadata.items())),
|
||||
_sampled_tensor_signature(ref.get("latent")),
|
||||
_sampled_tensor_signature(ref.get("audio_latent")),
|
||||
)
|
||||
)
|
||||
|
||||
if not context_ranges:
|
||||
raise LayoutCompatibilityError("Native Continuity payload has no context rows")
|
||||
for start, stop, role in context_ranges:
|
||||
row_range = (start, stop)
|
||||
if _ranges_overlap(row_range, audio_range) or _ranges_overlap(row_range, video_range):
|
||||
raise LayoutCompatibilityError(f"{role} rows overlap target rows")
|
||||
|
||||
topology = (
|
||||
tuple(getattr(layout, "signature", ())),
|
||||
segments,
|
||||
audio_range,
|
||||
video_range,
|
||||
tuple(context_ranges),
|
||||
tuple(context_descriptors),
|
||||
int(payload.get("frame_count", 0) or 0),
|
||||
_sampled_tensor_signature(position_ids),
|
||||
)
|
||||
options = transformer_options if isinstance(transformer_options, dict) else {}
|
||||
if options.get("spectrum_h3_actual") is False:
|
||||
return {"status": "forecast_validated", "branch": _branch_key(options)}
|
||||
|
||||
branch = _branch_key(options)
|
||||
baseline = branch_baselines.get(branch)
|
||||
if baseline is None:
|
||||
branch_baselines[branch] = topology
|
||||
return {"status": "baseline_created", "branch": branch}
|
||||
if baseline != topology:
|
||||
raise LayoutCompatibilityError(
|
||||
f"Native Continuity topology changed during sampling for branch {branch!r}"
|
||||
)
|
||||
return {"status": "baseline_matched", "branch": branch}
|
||||
|
||||
|
||||
def patch_layout_in_place(
|
||||
payload: dict[str, Any], *, strict: bool = True, debug: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
layout = payload.get("layout")
|
||||
if layout is None:
|
||||
raise LayoutCompatibilityError(
|
||||
"H3 payload has no prebuilt PackedLayout; use ComfyUI v0.31.0 or newer"
|
||||
)
|
||||
missing = [name for name in ("position_ids", "segments", "signature") if not hasattr(layout, name)]
|
||||
if missing:
|
||||
raise LayoutCompatibilityError("PackedLayout is missing: " + ", ".join(missing))
|
||||
position_ids = layout.position_ids
|
||||
if not torch.is_tensor(position_ids) or position_ids.ndim != 2 or position_ids.shape[1] < 3:
|
||||
raise LayoutCompatibilityError("position_ids must be an [S,3] Tensor")
|
||||
|
||||
signature = _patch_signature(payload, layout)
|
||||
if getattr(layout, LAYOUT_SIGNATURE_ATTR, None) == signature:
|
||||
return {"status": "already_patched", "position_ids_id": id(position_ids)}
|
||||
|
||||
if not hasattr(layout, LAYOUT_ORIGINAL_TIME_ATTR):
|
||||
setattr(layout, LAYOUT_ORIGINAL_TIME_ATTR, position_ids[:, 0].clone())
|
||||
original_time = getattr(layout, LAYOUT_ORIGINAL_TIME_ATTR)
|
||||
if not torch.is_tensor(original_time) or original_time.shape != position_ids[:, 0].shape:
|
||||
raise LayoutCompatibilityError("stored layout baseline is incompatible")
|
||||
position_ids[:, 0].copy_(original_time)
|
||||
|
||||
keyframes = list(payload.get("keyframes") or ())
|
||||
refs = list(payload.get("refs") or ())
|
||||
cond_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "cond"]
|
||||
if len(cond_segments) != len(keyframes):
|
||||
raise LayoutCompatibilityError(
|
||||
f"layout has {len(cond_segments)} cond segments for {len(keyframes)} keyframes"
|
||||
)
|
||||
|
||||
video_start, video_stop = _single_segment(layout, "video")
|
||||
_single_segment(layout, "audio")
|
||||
text_segments = [(int(a), int(b)) for a, b, kind in layout.segments if kind == "text"]
|
||||
if len(text_segments) != 1 or text_segments[0][0] != 0:
|
||||
raise LayoutCompatibilityError("unexpected H3 text segment")
|
||||
text_len = text_segments[0][1]
|
||||
|
||||
latent_t = int(layout.signature[1])
|
||||
video_rows = video_stop - video_start
|
||||
if latent_t <= 0 or video_rows % latent_t:
|
||||
raise LayoutCompatibilityError(
|
||||
f"video rows {video_rows} are incompatible with latent T={latent_t}"
|
||||
)
|
||||
frame_rows = video_rows // latent_t
|
||||
target_origin = float(position_ids[video_start, 0])
|
||||
reference_shift = target_origin - float(text_len)
|
||||
|
||||
# Existing first/last-frame keyframes are text-relative in stock H3. Once
|
||||
# refs are present, move them by the target-origin shift. The legacy marked
|
||||
# slot path is retained for compatibility with early development states.
|
||||
patched_video_slots = 0
|
||||
for (start, stop), keyframe in zip(cond_segments, keyframes):
|
||||
if stop - start != frame_rows:
|
||||
raise LayoutCompatibilityError("keyframe rows no longer equal one target slot")
|
||||
if MARK_VIDEO_SLOT in keyframe:
|
||||
slot = int(keyframe[MARK_VIDEO_SLOT])
|
||||
if not (0 <= slot < latent_t):
|
||||
raise LayoutCompatibilityError(
|
||||
f"context slot {slot} is outside target latent T={latent_t}"
|
||||
)
|
||||
target_row = video_start + slot * frame_rows
|
||||
position_ids[start:stop].copy_(
|
||||
position_ids[target_row : target_row + frame_rows]
|
||||
)
|
||||
patched_video_slots += 1
|
||||
elif reference_shift:
|
||||
position_ids[start:stop, 0].add_(reference_shift)
|
||||
|
||||
ref_mappings = _map_refs_to_segments(layout, refs)
|
||||
patched_video_refs = 0
|
||||
patched_audio = 0
|
||||
for ref, mapping in zip(refs, ref_mappings):
|
||||
if _is_video_context(ref):
|
||||
segment = mapping.get("video")
|
||||
if segment is None:
|
||||
raise LayoutCompatibilityError("Continuum video ref has no video segment")
|
||||
start, stop = segment
|
||||
context_t = int(ref.get("latent_t", 0))
|
||||
expected_rows = context_t * frame_rows
|
||||
if context_t <= 0 or stop - start != expected_rows:
|
||||
raise LayoutCompatibilityError(
|
||||
f"video-context rows {stop-start} do not match latent T={context_t}"
|
||||
)
|
||||
if context_t > latent_t:
|
||||
raise LayoutCompatibilityError(
|
||||
f"video context T={context_t} exceeds target latent T={latent_t}"
|
||||
)
|
||||
target_stop = video_start + expected_rows
|
||||
# Copy all (t,h,w) coordinates. Resolution equality is already
|
||||
# enforced by state validation; this also fails safely if row geometry changes.
|
||||
position_ids[start:stop].copy_(position_ids[video_start:target_stop])
|
||||
patched_video_refs += 1
|
||||
|
||||
if _is_audio_context(ref):
|
||||
segment = mapping.get("audio")
|
||||
if segment is None:
|
||||
raise LayoutCompatibilityError("Continuum audio ref has no audio segment")
|
||||
start, stop = segment
|
||||
rt = int(ref.get("ref_audio_t", 0))
|
||||
if rt <= 0 or stop - start != rt * 2:
|
||||
raise LayoutCompatibilityError(
|
||||
f"audio rows {stop-start} do not match {rt} stereo steps"
|
||||
)
|
||||
end_frame = float(ref.get(MARK_AUDIO_END_FRAME, 0.0))
|
||||
grid_offset = float(ref.get(MARK_AUDIO_OVERHANG, 0.0))
|
||||
desired_end = target_origin + FRAME_RESCALE * end_frame + grid_offset
|
||||
desired_start = desired_end - float(rt)
|
||||
old_start = float(position_ids[start, 0])
|
||||
position_ids[start:stop, 0].add_(desired_start - old_start)
|
||||
patched_audio += 1
|
||||
|
||||
patched_video = patched_video_slots + patched_video_refs
|
||||
if patched_video == 0:
|
||||
message = "no marked Continuum video context was found"
|
||||
if strict:
|
||||
raise LayoutCompatibilityError(message)
|
||||
LOG.warning(message)
|
||||
|
||||
setattr(layout, LAYOUT_SIGNATURE_ATTR, signature)
|
||||
if debug:
|
||||
LOG.info(
|
||||
"Continuum layout: video_refs=%d legacy_slots=%d audio_windows=%d "
|
||||
"origin=%.6f rows=%d pos_id=%s",
|
||||
patched_video_refs,
|
||||
patched_video_slots,
|
||||
patched_audio,
|
||||
target_origin,
|
||||
position_ids.shape[0],
|
||||
id(position_ids),
|
||||
)
|
||||
return {
|
||||
"status": "patched",
|
||||
"video_contexts": patched_video,
|
||||
"video_refs": patched_video_refs,
|
||||
"legacy_video_slots": patched_video_slots,
|
||||
"audio_windows": patched_audio,
|
||||
"target_origin": target_origin,
|
||||
"position_ids_id": id(position_ids),
|
||||
}
|
||||
Reference in New Issue
Block a user