git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
700 lines
31 KiB
Python
700 lines
31 KiB
Python
"""Runtime-only First Pass conditioning for Continuum-aware refinement.
|
|
|
|
The context deliberately remains outside the persisted Assembly Plan contract. It
|
|
owns CPU copies of the First Pass conditioning tensors and records them in physical
|
|
sampling order. Second Pass callers may then derive a target-canvas conditioning
|
|
without mutating the captured source context.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
import copy
|
|
from types import MappingProxyType
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..constants import MARK_VIDEO_CONTEXT
|
|
from ..guide_timeline import MARK_STILL_IMAGE_GUIDE
|
|
|
|
|
|
MAGIC = "H3_CONTINUUM_REFINE_CONTEXT"
|
|
VERSION = 1
|
|
OUTPUT_SCOPE_KEY = "output_scope_v1"
|
|
|
|
|
|
class RefineContextError(ValueError):
|
|
"""Raised when the runtime-only refine context is structurally invalid."""
|
|
|
|
|
|
class RefineConditioningAdaptationError(RefineContextError):
|
|
"""Raised when a valid captured conditioning cannot be adapted safely."""
|
|
|
|
|
|
def _cpu_tensor(value: torch.Tensor) -> torch.Tensor:
|
|
return value.detach().to(device="cpu").clone().contiguous()
|
|
|
|
|
|
def _freeze(value: Any) -> Any:
|
|
if torch.is_tensor(value):
|
|
return _cpu_tensor(value)
|
|
if isinstance(value, Mapping):
|
|
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
|
if isinstance(value, (list, tuple)):
|
|
return tuple(_freeze(item) for item in value)
|
|
if isinstance(value, (set, frozenset)):
|
|
return frozenset(_freeze(item) for item in value)
|
|
return value
|
|
|
|
|
|
def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]:
|
|
"""Return a CPU-owned conditioning clone with immutable containers.
|
|
|
|
Tensors are detached and cloned, so later mutation or device movement of the
|
|
original First Pass conditioning cannot affect the refine context. Container
|
|
immutability makes the result safe to share until target adaptation performs
|
|
copy-on-write replacement of visual fields.
|
|
"""
|
|
|
|
if not isinstance(conditioning, Sequence) or isinstance(
|
|
conditioning, (str, bytes, bytearray)
|
|
):
|
|
raise RefineContextError("conditioning must be a non-empty sequence")
|
|
output: list[tuple[torch.Tensor, Mapping]] = []
|
|
for index, entry in enumerate(conditioning):
|
|
if not isinstance(entry, Sequence) or isinstance(
|
|
entry, (str, bytes, bytearray)
|
|
) or len(entry) != 2:
|
|
raise RefineContextError(
|
|
f"conditioning entry {index} is not [tensor, metadata]"
|
|
)
|
|
tensor, metadata = entry
|
|
if not torch.is_tensor(tensor):
|
|
raise RefineContextError(
|
|
f"conditioning entry {index} text value is not a tensor"
|
|
)
|
|
if not isinstance(metadata, Mapping):
|
|
raise RefineContextError(
|
|
f"conditioning entry {index} metadata is not a mapping"
|
|
)
|
|
output.append((_cpu_tensor(tensor), _freeze(metadata)))
|
|
if not output:
|
|
raise RefineContextError("conditioning is empty")
|
|
return tuple(output)
|
|
|
|
|
|
def _optional_image(value: Any, name: str) -> torch.Tensor | None:
|
|
if value is None:
|
|
return None
|
|
if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3:
|
|
raise RefineContextError(
|
|
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
|
)
|
|
return _cpu_tensor(value)
|
|
|
|
|
|
def _validate_optional_image(value: Any, name: str) -> None:
|
|
if value is not None and (
|
|
not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3
|
|
):
|
|
raise RefineContextError(
|
|
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
|
)
|
|
|
|
|
|
def _guide_image_map(value: Any) -> Mapping[int, torch.Tensor]:
|
|
if value is None:
|
|
return MappingProxyType({})
|
|
if not isinstance(value, Mapping):
|
|
raise RefineContextError("guide_images must be a frame-to-IMAGE mapping")
|
|
output: dict[int, torch.Tensor] = {}
|
|
for frame_index, image in value.items():
|
|
if type(frame_index) is not int or frame_index < 0:
|
|
raise RefineContextError("guide_images frame indices must be non-negative integers")
|
|
prepared = _optional_image(image, f"guide_images[{frame_index}]")
|
|
assert prepared is not None
|
|
output[frame_index] = prepared
|
|
return MappingProxyType(output)
|
|
|
|
|
|
def _validate_guide_images(value: Any, *, physical_frames: int) -> None:
|
|
if value is None:
|
|
return
|
|
if not isinstance(value, Mapping):
|
|
raise RefineContextError("refine group guide_images is invalid")
|
|
for frame_index, image in value.items():
|
|
if type(frame_index) is not int or not 0 <= frame_index < physical_frames:
|
|
raise RefineContextError("refine group guide image frame is outside the physical group")
|
|
_validate_optional_image(image, f"guide_images[{frame_index}]")
|
|
|
|
|
|
def make_refine_group(
|
|
*,
|
|
conditioning: Any,
|
|
group_id: int,
|
|
logical_chunks: Sequence[int],
|
|
physical_frames: int,
|
|
prompt_policy: str,
|
|
physical_prompt: str,
|
|
source_video_shape: Sequence[int],
|
|
physical_clip_index: int,
|
|
context_frames: int,
|
|
first_image: torch.Tensor | None = None,
|
|
last_image: torch.Tensor | None = None,
|
|
guide_images: Mapping[int, torch.Tensor] | None = None,
|
|
reference_routing_group_contract: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Capture one physical First Pass sampling group's effective conditioning."""
|
|
|
|
group_id = int(group_id)
|
|
physical_frames = int(physical_frames)
|
|
physical_clip_index = int(physical_clip_index)
|
|
context_frames = int(context_frames)
|
|
chunks = tuple(int(value) for value in logical_chunks)
|
|
shape = tuple(int(value) for value in source_video_shape)
|
|
if group_id < 0:
|
|
raise RefineContextError("refine group id must be non-negative")
|
|
if not chunks or any(value <= 0 for value in chunks):
|
|
raise RefineContextError("refine group logical chunks are invalid")
|
|
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
|
|
raise RefineContextError("refine group logical chunks must be ordered and unique")
|
|
if physical_frames <= 0:
|
|
raise RefineContextError("refine group physical frame count is invalid")
|
|
if not str(prompt_policy):
|
|
raise RefineContextError("refine group prompt policy is empty")
|
|
if len(shape) != 5 or any(value <= 0 for value in shape):
|
|
raise RefineContextError("refine group source video shape must be [B,C,T,H,W]")
|
|
if physical_clip_index <= 0:
|
|
raise RefineContextError("refine group physical clip index is invalid")
|
|
if context_frames < 0:
|
|
raise RefineContextError("refine group context frame count is invalid")
|
|
output = {
|
|
"group_id": group_id,
|
|
"logical_chunks": chunks,
|
|
"physical_frames": physical_frames,
|
|
"prompt_policy": str(prompt_policy),
|
|
"physical_prompt": str(physical_prompt),
|
|
"source_video_shape": shape,
|
|
"physical_clip_index": physical_clip_index,
|
|
"context_frames": context_frames,
|
|
"conditioning": freeze_conditioning(conditioning),
|
|
"first_image": _optional_image(first_image, "first_image"),
|
|
"last_image": _optional_image(last_image, "last_image"),
|
|
}
|
|
frozen_guides = _guide_image_map(guide_images)
|
|
if frozen_guides:
|
|
output["guide_images"] = frozen_guides
|
|
if reference_routing_group_contract is not None:
|
|
from .reference_storage_contract import canonical_sha256
|
|
descriptor = reference_routing_group_contract.get("descriptor")
|
|
digest = reference_routing_group_contract.get("sha256")
|
|
if not isinstance(descriptor, dict) or canonical_sha256(descriptor) != digest:
|
|
raise RefineContextError("refine group Reference contract SHA-256 is invalid")
|
|
if descriptor.get("logical_chunks") != list(chunks):
|
|
raise RefineContextError("refine group Reference contract has different logical chunks")
|
|
output["reference_routing_v1"] = copy.deepcopy(dict(reference_routing_group_contract))
|
|
return output
|
|
|
|
|
|
def make_refine_context(
|
|
groups: Sequence[Mapping[str, Any]],
|
|
*,
|
|
source_width: int,
|
|
source_height: int,
|
|
conditioning_mode: str,
|
|
complete: bool = True,
|
|
notes: Sequence[str] = (),
|
|
) -> dict[str, Any]:
|
|
"""Build and validate a physical-order runtime refine context."""
|
|
|
|
context = {
|
|
"magic": MAGIC,
|
|
"version": VERSION,
|
|
"source_width": int(source_width),
|
|
"source_height": int(source_height),
|
|
"conditioning_mode": str(conditioning_mode),
|
|
"complete": bool(complete),
|
|
"groups": tuple(dict(group) for group in groups),
|
|
"notes": tuple(str(note) for note in notes),
|
|
}
|
|
return validate_refine_context(context)
|
|
|
|
|
|
def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None:
|
|
if not isinstance(value, tuple) or not value:
|
|
raise RefineContextError(f"refine group {group_id} conditioning is invalid")
|
|
for index, entry in enumerate(value):
|
|
if not isinstance(entry, tuple) or len(entry) != 2:
|
|
raise RefineContextError(
|
|
f"refine group {group_id} conditioning entry {index} is invalid"
|
|
)
|
|
tensor, metadata = entry
|
|
if not torch.is_tensor(tensor) or tensor.device.type != "cpu":
|
|
raise RefineContextError(
|
|
f"refine group {group_id} conditioning tensor {index} is not CPU-owned"
|
|
)
|
|
if not isinstance(metadata, Mapping):
|
|
raise RefineContextError(
|
|
f"refine group {group_id} conditioning metadata {index} is invalid"
|
|
)
|
|
|
|
|
|
def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]:
|
|
value = group.get(name)
|
|
if not isinstance(value, (tuple, list)):
|
|
raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid")
|
|
try:
|
|
return tuple(int(item) for item in value)
|
|
except (TypeError, ValueError) as exc:
|
|
raise RefineContextError(
|
|
f"refine group {group.get('group_id')} {name} is invalid"
|
|
) from exc
|
|
|
|
|
|
def project_refine_context(context: dict[str, Any], assembly_plan: Mapping[str, Any]):
|
|
"""Bind a V3.9 runtime-only view to complete physical output groups.
|
|
|
|
Sequence completion and saved history are unchanged. A partial capture may
|
|
serve a smaller decode view only after the ordinary complete-group checks
|
|
pass, including the frozen Reference route. Missing or mismatched partial
|
|
evidence retains the legacy diagnostic fallback, never guessed conditions.
|
|
"""
|
|
validate_refine_context(context)
|
|
contract = assembly_plan.get("second_pass_contract")
|
|
planned = contract.get("physical_groups") if isinstance(contract, Mapping) else None
|
|
if not isinstance(planned, list) or not planned:
|
|
return context
|
|
source = context["groups"]
|
|
keys = [tuple(group["logical_chunks"]) for group in source]
|
|
targets = [tuple(group.get("logical_chunks", ())) for group in planned]
|
|
if context["complete"] and keys == targets:
|
|
return validate_refine_context(context, assembly_plan=assembly_plan)
|
|
if len(set(keys)) != len(keys) or len(set(targets)) != len(targets):
|
|
return context
|
|
lookup = dict(zip(keys, source))
|
|
if any(key not in lookup for key in targets):
|
|
return context
|
|
selected = [lookup[key] for key in targets]
|
|
view = dict(context)
|
|
view["groups"] = tuple(
|
|
{**group, "group_id": index} for index, group in enumerate(selected)
|
|
)
|
|
view[OUTPUT_SCOPE_KEY] = MappingProxyType({
|
|
"version": 1,
|
|
"source_group_ids": tuple(group["group_id"] for group in selected),
|
|
})
|
|
try:
|
|
validate_refine_context(view, assembly_plan=assembly_plan)
|
|
except RefineContextError as exc:
|
|
# These groups were previously optional/incomplete. Do not add a new
|
|
# execution stop for unavailable output-scope evidence.
|
|
fallback = dict(context)
|
|
fallback["notes"] = (*context["notes"], f"Output scope unavailable: {exc}")
|
|
return fallback
|
|
return view
|
|
|
|
|
|
def validate_refine_context(
|
|
context: Any, assembly_plan: Mapping[str, Any] | None = None
|
|
) -> dict[str, Any]:
|
|
"""Validate a runtime context and, when supplied, its Assembly Plan pairing."""
|
|
|
|
if not isinstance(context, dict) or context.get("magic") != MAGIC:
|
|
raise RefineContextError("invalid H3 Continuum refine context")
|
|
if type(context.get("version")) is not int or context["version"] != VERSION:
|
|
raise RefineContextError(
|
|
f"unsupported refine context version {context.get('version')!r}"
|
|
)
|
|
if int(context.get("source_width", 0)) <= 0 or int(
|
|
context.get("source_height", 0)
|
|
) <= 0:
|
|
raise RefineContextError("refine context source dimensions are invalid")
|
|
if not str(context.get("conditioning_mode", "")):
|
|
raise RefineContextError("refine context conditioning mode is empty")
|
|
if type(context.get("complete")) is not bool:
|
|
raise RefineContextError("refine context complete flag is invalid")
|
|
groups = context.get("groups")
|
|
if not isinstance(groups, tuple):
|
|
raise RefineContextError("refine context groups must be an immutable tuple")
|
|
if context["complete"] and not groups:
|
|
raise RefineContextError("complete refine context contains no physical groups")
|
|
notes = context.get("notes")
|
|
if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes):
|
|
raise RefineContextError("refine context notes must be an immutable string tuple")
|
|
scope = context.get(OUTPUT_SCOPE_KEY)
|
|
if scope is not None:
|
|
ids = scope.get("source_group_ids") if isinstance(scope, Mapping) else None
|
|
if (not isinstance(scope, Mapping) or scope.get("version") != 1
|
|
or not isinstance(ids, tuple) or len(ids) != len(groups)
|
|
or any(type(value) is not int or value < 0 for value in ids)
|
|
or tuple(sorted(set(ids))) != ids or not groups):
|
|
raise RefineContextError("refine context output scope is invalid")
|
|
|
|
previous_clip_index = 0
|
|
for expected_id, group in enumerate(groups):
|
|
if not isinstance(group, Mapping):
|
|
raise RefineContextError(f"refine group {expected_id} is not a mapping")
|
|
if type(group.get("group_id")) is not int or group["group_id"] != expected_id:
|
|
raise RefineContextError("refine context physical group order is not contiguous")
|
|
logical_chunks = _group_field_tuple(group, "logical_chunks")
|
|
if not logical_chunks or any(value <= 0 for value in logical_chunks):
|
|
raise RefineContextError(f"refine group {expected_id} logical chunks are invalid")
|
|
if tuple(sorted(logical_chunks)) != logical_chunks or len(
|
|
set(logical_chunks)
|
|
) != len(logical_chunks):
|
|
raise RefineContextError(
|
|
f"refine group {expected_id} logical chunks are not ordered and unique"
|
|
)
|
|
if int(group.get("physical_frames", 0)) <= 0:
|
|
raise RefineContextError(f"refine group {expected_id} physical frames are invalid")
|
|
if not str(group.get("prompt_policy", "")):
|
|
raise RefineContextError(f"refine group {expected_id} prompt policy is empty")
|
|
if "physical_prompt" not in group:
|
|
raise RefineContextError(f"refine group {expected_id} physical prompt is absent")
|
|
shape = _group_field_tuple(group, "source_video_shape")
|
|
if len(shape) != 5 or any(value <= 0 for value in shape):
|
|
raise RefineContextError(f"refine group {expected_id} source shape is invalid")
|
|
clip_index = int(group.get("physical_clip_index", 0))
|
|
if clip_index <= previous_clip_index:
|
|
raise RefineContextError(
|
|
"refine context physical clip indices are not strictly increasing"
|
|
)
|
|
previous_clip_index = clip_index
|
|
if int(group.get("context_frames", -1)) < 0:
|
|
raise RefineContextError(f"refine group {expected_id} context frames are invalid")
|
|
_validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id)
|
|
_validate_optional_image(group.get("first_image"), "first_image")
|
|
_validate_optional_image(group.get("last_image"), "last_image")
|
|
_validate_guide_images(
|
|
group.get("guide_images"),
|
|
physical_frames=int(group["physical_frames"]),
|
|
)
|
|
route_contract = group.get("reference_routing_v1")
|
|
if route_contract is not None:
|
|
from .reference_storage_contract import canonical_sha256
|
|
if not isinstance(route_contract, Mapping):
|
|
raise RefineContextError("refine group Reference contract is invalid")
|
|
descriptor = route_contract.get("descriptor")
|
|
if (not isinstance(descriptor, dict)
|
|
or canonical_sha256(descriptor) != route_contract.get("sha256")
|
|
or descriptor.get("logical_chunks") != list(logical_chunks)):
|
|
raise RefineContextError("refine group Reference contract differs from its content")
|
|
|
|
if assembly_plan is not None:
|
|
if not isinstance(assembly_plan, Mapping):
|
|
raise RefineContextError("assembly plan paired with refine context is invalid")
|
|
if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int(
|
|
assembly_plan.get("height", 0)
|
|
) != int(context["source_height"]):
|
|
raise RefineContextError("refine context source dimensions differ from assembly plan")
|
|
contract = assembly_plan.get("second_pass_contract")
|
|
plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None
|
|
if not isinstance(plan_groups, Sequence):
|
|
raise RefineContextError("assembly plan has no Second Pass physical-group contract")
|
|
# An incomplete context is diagnostic-only (for example, Run Storage may
|
|
# have reused an earlier group that was never sampled in this execution).
|
|
# Its local group ids describe capture order, not Assembly Plan position;
|
|
# validating those partial entries by index would incorrectly turn the
|
|
# intended prompt-only fallback into a broken typed-contract hard error.
|
|
if not context["complete"] and scope is None:
|
|
return context
|
|
if len(plan_groups) != len(groups):
|
|
raise RefineContextError(
|
|
"complete refine context physical-group count differs from assembly plan"
|
|
)
|
|
for group in groups:
|
|
group_id = int(group["group_id"])
|
|
if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping):
|
|
raise RefineContextError(
|
|
f"refine group {group_id} has no matching assembly-plan group"
|
|
)
|
|
planned = plan_groups[group_id]
|
|
route_view = assembly_plan.get("reference_routing_v1")
|
|
if route_view is not None:
|
|
view_groups = route_view.get("groups") if isinstance(route_view, Mapping) else None
|
|
expected_route = (
|
|
view_groups[group_id] if isinstance(view_groups, list)
|
|
and group_id < len(view_groups) else None
|
|
)
|
|
if scope is not None:
|
|
# Inspector retains the full run, including pending groups.
|
|
# A Chunk-2 view must not compare against Inspector group 1.
|
|
matches = [item for item in view_groups
|
|
if isinstance(item, Mapping)
|
|
and isinstance(item.get("descriptor"), Mapping)
|
|
and tuple(item["descriptor"].get("logical_chunks", ()))
|
|
== tuple(group["logical_chunks"])] if isinstance(view_groups, list) else []
|
|
expected_route = matches[0] if len(matches) == 1 else None
|
|
captured_route = group.get("reference_routing_v1")
|
|
if (not isinstance(expected_route, Mapping)
|
|
or not isinstance(captured_route, Mapping)
|
|
or captured_route.get("sha256") != expected_route.get("contract_sha256")
|
|
or captured_route.get("descriptor") != expected_route.get("descriptor")):
|
|
raise RefineContextError(
|
|
f"refine group {group_id} Reference route differs from First Pass Plan"
|
|
)
|
|
comparisons = (
|
|
("logical_chunks", tuple(int(v) for v in group["logical_chunks"]),
|
|
tuple(int(v) for v in planned.get("logical_chunks", ()))),
|
|
("physical_frames", int(group["physical_frames"]),
|
|
int(planned.get("physical_frames", 0))),
|
|
("prompt_policy", str(group["prompt_policy"]),
|
|
str(planned.get("prompt_policy", ""))),
|
|
("physical_prompt", str(group["physical_prompt"]),
|
|
str(planned.get("physical_prompt", ""))),
|
|
("context_frames", int(group["context_frames"]),
|
|
int(planned.get("trim_prefix_frames", -1))),
|
|
)
|
|
for name, actual, expected in comparisons:
|
|
if actual != expected:
|
|
raise RefineContextError(
|
|
f"refine group {group_id} {name} differs from assembly plan"
|
|
)
|
|
expected_shape = (
|
|
int(planned.get("source_batch", 0)),
|
|
int(planned.get("latent_channels", 0)),
|
|
int(planned.get("source_latent_t", 0)),
|
|
int(planned.get("source_latent_h", 0)),
|
|
int(planned.get("source_latent_w", 0)),
|
|
)
|
|
if tuple(group["source_video_shape"]) != expected_shape:
|
|
raise RefineContextError(
|
|
f"refine group {group_id} source shape differs from assembly plan"
|
|
)
|
|
return context
|
|
|
|
|
|
def _common_upscale(
|
|
samples: torch.Tensor,
|
|
width: int,
|
|
height: int,
|
|
method: str,
|
|
crop: str,
|
|
) -> torch.Tensor:
|
|
try:
|
|
import comfy.utils
|
|
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
|
raise RefineConditioningAdaptationError(
|
|
f"ComfyUI Core resize support is unavailable: {exc}"
|
|
) from exc
|
|
return comfy.utils.common_upscale(samples, width, height, method, crop)
|
|
|
|
|
|
def _resize_latent(
|
|
latent: Any,
|
|
*,
|
|
target_h: int,
|
|
target_w: int,
|
|
method: str,
|
|
) -> torch.Tensor:
|
|
if not torch.is_tensor(latent) or latent.ndim != 5:
|
|
raise RefineConditioningAdaptationError(
|
|
"H3 visual conditioning latent must be [B,C,T,H,W]"
|
|
)
|
|
batch, channels, temporal, _, _ = (int(value) for value in latent.shape)
|
|
flat = latent.permute(0, 2, 1, 3, 4).reshape(
|
|
batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1])
|
|
)
|
|
try:
|
|
resized = _common_upscale(flat, target_w, target_h, method, "disabled")
|
|
except RefineConditioningAdaptationError:
|
|
raise
|
|
except Exception as exc:
|
|
raise RefineConditioningAdaptationError(
|
|
f"visual conditioning latent resize failed: {exc}"
|
|
) from exc
|
|
if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != (
|
|
target_h,
|
|
target_w,
|
|
):
|
|
raise RefineConditioningAdaptationError(
|
|
"Core resize returned an incompatible visual conditioning shape"
|
|
)
|
|
return (
|
|
resized.reshape(batch, temporal, channels, target_h, target_w)
|
|
.permute(0, 2, 1, 3, 4)
|
|
.contiguous()
|
|
)
|
|
|
|
|
|
def _encode_keyframe_image(
|
|
image: torch.Tensor,
|
|
*,
|
|
target_h: int,
|
|
target_w: int,
|
|
crop: str,
|
|
video_vae: Any,
|
|
) -> torch.Tensor:
|
|
pixels_h = target_h * 16
|
|
pixels_w = target_w * 16
|
|
samples = image[:1, ..., :3].movedim(-1, 1)
|
|
try:
|
|
resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop)
|
|
latent = video_vae.encode(resized.movedim(1, -1).contiguous())
|
|
except RefineConditioningAdaptationError:
|
|
raise
|
|
except Exception as exc:
|
|
raise RefineConditioningAdaptationError(
|
|
f"target keyframe VAE encode failed: {exc}"
|
|
) from exc
|
|
if not torch.is_tensor(latent) or latent.ndim != 5 or tuple(
|
|
int(value) for value in latent.shape[-2:]
|
|
) != (target_h, target_w):
|
|
raise RefineConditioningAdaptationError(
|
|
"target keyframe VAE encode returned an incompatible latent shape"
|
|
)
|
|
return latent.contiguous()
|
|
|
|
|
|
def adapt_group_conditioning(
|
|
group: Mapping[str, Any],
|
|
target_latent_h: int,
|
|
target_latent_w: int,
|
|
video_vae: Any = None,
|
|
upscale_method: str = "bilinear",
|
|
) -> tuple[list[list[Any]], dict[str, Any]]:
|
|
"""Copy-on-write target adaptation for keyframes and Continuum video context.
|
|
|
|
Ordinary MiniMax references, all audio latents, and text conditioning tensors
|
|
remain shared with the frozen source. Only keyframe video latents and
|
|
``MARK_VIDEO_CONTEXT`` reference video latents are target-sized.
|
|
"""
|
|
|
|
if not isinstance(group, Mapping):
|
|
raise RefineConditioningAdaptationError("refine group is not a mapping")
|
|
group_id = int(group.get("group_id", -1))
|
|
_validate_frozen_conditioning(group.get("conditioning"), group_id=group_id)
|
|
target_h = int(target_latent_h)
|
|
target_w = int(target_latent_w)
|
|
if target_h <= 0 or target_w <= 0:
|
|
raise RefineConditioningAdaptationError("target latent dimensions are invalid")
|
|
physical_frames = int(group.get("physical_frames", 0))
|
|
if physical_frames <= 0:
|
|
raise RefineConditioningAdaptationError("refine group physical frames are invalid")
|
|
|
|
warnings: list[str] = []
|
|
stats: dict[str, Any] = {
|
|
"group_id": group_id,
|
|
"target_latent_h": target_h,
|
|
"target_latent_w": target_w,
|
|
"keyframes_resized": 0,
|
|
"keyframes_reencoded": 0,
|
|
"keyframes_latent_fallback": 0,
|
|
"guide_keyframes_reencoded": 0,
|
|
"context_refs_resized": 0,
|
|
}
|
|
output: list[list[Any]] = []
|
|
for text_tensor, frozen_metadata in group["conditioning"]:
|
|
metadata = dict(frozen_metadata)
|
|
keyframes = frozen_metadata.get("minimax_keyframes")
|
|
if keyframes is not None:
|
|
adapted_keyframes = []
|
|
for frozen_keyframe in keyframes:
|
|
if not isinstance(frozen_keyframe, Mapping):
|
|
raise RefineConditioningAdaptationError(
|
|
f"refine group {group_id} contains an invalid keyframe"
|
|
)
|
|
keyframe = dict(frozen_keyframe)
|
|
latent = keyframe.get("latent")
|
|
audio_latent = keyframe.get("audio_latent")
|
|
if audio_latent is not None and (
|
|
not torch.is_tensor(audio_latent) or audio_latent.ndim != 4
|
|
):
|
|
raise RefineConditioningAdaptationError(
|
|
f"refine group {group_id} keyframe audio latent is invalid"
|
|
)
|
|
# Core keyframes may carry audio without a spatial video latent.
|
|
# Keep its frozen audio and timing; resize only video payloads.
|
|
if latent is None and audio_latent is not None:
|
|
adapted_keyframes.append(keyframe)
|
|
continue
|
|
if not torch.is_tensor(latent) or latent.ndim != 5:
|
|
raise RefineConditioningAdaptationError(
|
|
f"refine group {group_id} keyframe latent is invalid"
|
|
)
|
|
if tuple(int(value) for value in latent.shape[-2:]) != (
|
|
target_h,
|
|
target_w,
|
|
):
|
|
frame_index = int(keyframe.get("resolved_frame_index", -1))
|
|
image = None
|
|
crop = "disabled"
|
|
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
|
|
guide_images = group.get("guide_images")
|
|
if isinstance(guide_images, Mapping):
|
|
image = guide_images.get(frame_index)
|
|
crop = "center"
|
|
elif frame_index == 0:
|
|
image = group.get("first_image")
|
|
elif frame_index == physical_frames - 1:
|
|
image = group.get("last_image")
|
|
crop = "center"
|
|
if video_vae is not None and torch.is_tensor(image):
|
|
keyframe["latent"] = _encode_keyframe_image(
|
|
image,
|
|
target_h=target_h,
|
|
target_w=target_w,
|
|
crop=crop,
|
|
video_vae=video_vae,
|
|
)
|
|
stats["keyframes_reencoded"] += 1
|
|
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
|
|
stats["guide_keyframes_reencoded"] += 1
|
|
else:
|
|
keyframe["latent"] = _resize_latent(
|
|
latent,
|
|
target_h=target_h,
|
|
target_w=target_w,
|
|
method=str(upscale_method),
|
|
)
|
|
stats["keyframes_latent_fallback"] += 1
|
|
warning = (
|
|
f"group {group_id} keyframe {frame_index} used latent resize "
|
|
"because its source image and video VAE were not both available"
|
|
)
|
|
warnings.append(warning)
|
|
stats["keyframes_resized"] += 1
|
|
adapted_keyframes.append(keyframe)
|
|
metadata["minimax_keyframes"] = adapted_keyframes
|
|
|
|
refs = frozen_metadata.get("minimax_refs")
|
|
if refs is not None:
|
|
adapted_refs = []
|
|
refs_changed = False
|
|
for frozen_ref in refs:
|
|
if not isinstance(frozen_ref, Mapping):
|
|
raise RefineConditioningAdaptationError(
|
|
f"refine group {group_id} contains an invalid MiniMax reference"
|
|
)
|
|
if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)):
|
|
latent = frozen_ref.get("latent")
|
|
if not torch.is_tensor(latent) or latent.ndim != 5:
|
|
raise RefineConditioningAdaptationError(
|
|
f"refine group {group_id} Continuum video context is invalid"
|
|
)
|
|
if tuple(int(value) for value in latent.shape[-2:]) != (
|
|
target_h,
|
|
target_w,
|
|
):
|
|
adapted_ref = dict(frozen_ref)
|
|
adapted_ref["latent"] = _resize_latent(
|
|
latent,
|
|
target_h=target_h,
|
|
target_w=target_w,
|
|
method=str(upscale_method),
|
|
)
|
|
adapted_ref["latent_h"] = target_h
|
|
adapted_ref["latent_w"] = target_w
|
|
adapted_refs.append(adapted_ref)
|
|
refs_changed = True
|
|
stats["context_refs_resized"] += 1
|
|
continue
|
|
adapted_refs.append(frozen_ref)
|
|
if refs_changed:
|
|
metadata["minimax_refs"] = adapted_refs
|
|
|
|
output.append([text_tensor, metadata])
|
|
stats["warnings"] = tuple(warnings)
|
|
return output, stats
|