Preserve Second Pass audio and scoped Review conditioning, strengthen Take compatibility, and align Review Driving Audio. Keep v5 history readable, document v6 restart requirements, and describe the fixes at the top of both READMEs. Remove workstation-specific paths and private operational records from public documentation.
467 lines
19 KiB
Python
467 lines
19 KiB
Python
"""Assembly-plan contract for latent-first H3 Continuum V3."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..state import validate_plan
|
|
|
|
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
|
|
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
|
|
FPS = 24
|
|
SECOND_PASS_CONTRACT_VERSION = 1
|
|
REVIEW_AUDIO_PROJECTION_KEY = "review_audio_projection_v1"
|
|
|
|
|
|
class AssemblyPlanError(ValueError):
|
|
pass
|
|
|
|
|
|
def make_assembly_plan(
|
|
entries: list[dict[str, Any]],
|
|
*,
|
|
chunk_seconds: float,
|
|
preserve_final_frame: bool,
|
|
) -> dict[str, Any]:
|
|
if not entries:
|
|
raise AssemblyPlanError("cannot create an assembly plan for an empty sequence")
|
|
|
|
chunks: list[dict[str, Any]] = []
|
|
width = height = None
|
|
for sequence_index, entry in enumerate(entries, start=1):
|
|
if not isinstance(entry, dict):
|
|
raise AssemblyPlanError(f"chunk {sequence_index} entry is not a dictionary")
|
|
plan = validate_plan(entry.get("plan"))
|
|
video = entry.get("video")
|
|
audio = entry.get("audio")
|
|
if not torch.is_tensor(video) or video.ndim < 3:
|
|
raise AssemblyPlanError(f"chunk {sequence_index} video latent is invalid")
|
|
if not torch.is_tensor(audio) or audio.ndim < 3:
|
|
raise AssemblyPlanError(f"chunk {sequence_index} audio latent is invalid")
|
|
|
|
plan_width = int(plan["width"])
|
|
plan_height = int(plan["height"])
|
|
if width is None:
|
|
width, height = plan_width, plan_height
|
|
elif (plan_width, plan_height) != (width, height):
|
|
raise AssemblyPlanError("chunk dimensions changed within the sequence")
|
|
|
|
chunks.append(
|
|
{
|
|
"sequence_index": sequence_index,
|
|
"chunk_index": int(plan["clip_index"]),
|
|
"total_frames": int(plan["total_frames"]),
|
|
"trim_frames": int(plan["trim_frames"]),
|
|
"net_frames": int(plan["net_frames"]),
|
|
"context_frames": int(plan["context_frames"]),
|
|
"expected_video_latent_t": int(video.shape[2]),
|
|
"expected_audio_latent_t": int(audio.shape[-1]),
|
|
}
|
|
)
|
|
|
|
result = {
|
|
"magic": ASSEMBLY_PLAN_MAGIC,
|
|
"schema_version": ASSEMBLY_PLAN_SCHEMA_VERSION,
|
|
"fps": FPS,
|
|
"width": int(width),
|
|
"height": int(height),
|
|
"chunk_seconds": float(chunk_seconds),
|
|
"target_frames": int(round(len(chunks) * float(chunk_seconds) * FPS)),
|
|
"preserve_final_frame": bool(preserve_final_frame),
|
|
"chunks": chunks,
|
|
}
|
|
return validate_assembly_plan(result)
|
|
|
|
|
|
def validate_assembly_plan(plan: Any) -> dict[str, Any]:
|
|
if not isinstance(plan, dict) or plan.get("magic") != ASSEMBLY_PLAN_MAGIC:
|
|
raise AssemblyPlanError("invalid H3 Continuum V3 assembly plan")
|
|
if (
|
|
type(plan.get("schema_version")) is not int
|
|
or plan["schema_version"] != ASSEMBLY_PLAN_SCHEMA_VERSION
|
|
):
|
|
raise AssemblyPlanError(
|
|
f"unsupported assembly plan schema {plan.get('schema_version')!r}"
|
|
)
|
|
if int(plan.get("fps", 0)) != FPS:
|
|
raise AssemblyPlanError("assembly plan FPS must be 24")
|
|
if int(plan.get("width", 0)) <= 0 or int(plan.get("height", 0)) <= 0:
|
|
raise AssemblyPlanError("assembly plan dimensions are invalid")
|
|
if float(plan.get("chunk_seconds", 0.0)) <= 0:
|
|
raise AssemblyPlanError("assembly plan chunk duration is invalid")
|
|
if int(plan.get("target_frames", 0)) <= 0:
|
|
raise AssemblyPlanError("assembly plan target frame count is invalid")
|
|
|
|
chunks = plan.get("chunks")
|
|
if not isinstance(chunks, list) or not chunks:
|
|
raise AssemblyPlanError("assembly plan contains no chunks")
|
|
for expected_index, chunk in enumerate(chunks, start=1):
|
|
if not isinstance(chunk, dict):
|
|
raise AssemblyPlanError(f"assembly chunk {expected_index} is invalid")
|
|
if int(chunk.get("sequence_index", 0)) != expected_index:
|
|
raise AssemblyPlanError("assembly chunk order is not contiguous")
|
|
total = int(chunk.get("total_frames", 0))
|
|
trim = int(chunk.get("trim_frames", -1))
|
|
net = int(chunk.get("net_frames", 0))
|
|
if total <= 0 or trim < 0 or net <= 0 or total - trim != net:
|
|
raise AssemblyPlanError(
|
|
f"assembly chunk {expected_index} frame plan is invalid"
|
|
)
|
|
if int(chunk.get("context_frames", -1)) < 0:
|
|
raise AssemblyPlanError(
|
|
f"assembly chunk {expected_index} context is invalid"
|
|
)
|
|
if int(chunk.get("expected_video_latent_t", 0)) <= 0:
|
|
raise AssemblyPlanError(
|
|
f"assembly chunk {expected_index} video latent length is invalid"
|
|
)
|
|
if int(chunk.get("expected_audio_latent_t", 0)) <= 0:
|
|
raise AssemblyPlanError(
|
|
f"assembly chunk {expected_index} audio latent length is invalid"
|
|
)
|
|
return plan
|
|
# V3.0.1 hardening integration: redundant metadata plus fail-closed validation.
|
|
from ..hardening import (
|
|
enrich_assembly_plan as _enrich_assembly_plan,
|
|
validate_assembly_plan_contract as _validate_assembly_plan_contract,
|
|
)
|
|
|
|
_make_assembly_plan_v300 = make_assembly_plan
|
|
_validate_assembly_plan_v300 = validate_assembly_plan
|
|
|
|
|
|
def validate_assembly_plan(plan):
|
|
result = _validate_assembly_plan_v300(plan)
|
|
_validate_assembly_plan_contract(plan)
|
|
validate_review_audio_projection(plan)
|
|
return result
|
|
|
|
|
|
def validate_review_audio_projection(plan):
|
|
"""Check a scoped PCM coordinate without recomputing its absolute origin."""
|
|
if REVIEW_AUDIO_PROJECTION_KEY not in plan:
|
|
return None
|
|
metadata = plan[REVIEW_AUDIO_PROJECTION_KEY]
|
|
if not isinstance(metadata, dict):
|
|
raise AssemblyPlanError("Review audio projection must be a dictionary")
|
|
if (type(metadata.get("version")) is not int or metadata.get("version") != 1 or metadata.get("kind") != "current_review_unit"
|
|
or metadata.get("coordinate_space") != "full_source_pcm"):
|
|
raise AssemblyPlanError("Review audio projection version or coordinate space is invalid")
|
|
for field in ("fps", "start_chunk", "end_chunk", "source_start_frame", "source_stop_frame"):
|
|
if type(metadata.get(field)) is not int:
|
|
raise AssemblyPlanError(f"Review audio projection {field} must be an integer")
|
|
if metadata["fps"] != FPS or int(plan.get("fps", 0)) != FPS:
|
|
raise AssemblyPlanError("Review audio projection FPS must be 24")
|
|
start, end = metadata["start_chunk"], metadata["end_chunk"]
|
|
first, stop = metadata["source_start_frame"], metadata["source_stop_frame"]
|
|
chunks = [int(chunk["chunk_index"]) for chunk in plan["chunks"]]
|
|
units = plan.get("decode_groups") or plan["chunks"]
|
|
if (start < 1 or end < start or len(chunks) != end - start + 1
|
|
or any(index != start + position for position, index in enumerate(chunks))
|
|
or first < 0 or stop <= first
|
|
or stop - first != sum(int(unit["net_frames"]) for unit in units)):
|
|
raise AssemblyPlanError("Review audio projection physical range is inconsistent")
|
|
return metadata
|
|
|
|
|
|
def make_review_audio_projection(plan, *, start_chunk: int, end_chunk: int):
|
|
"""Read natural retained ranges before narrowing the accepted decode view."""
|
|
validate_assembly_plan(plan)
|
|
if REVIEW_AUDIO_PROJECTION_KEY in plan:
|
|
raise AssemblyPlanError("Review output cannot be projected twice")
|
|
start, end = int(start_chunk), int(end_chunk)
|
|
selected = []
|
|
covered = []
|
|
for unit in plan.get("decode_groups") or plan["chunks"]:
|
|
indices = unit.get("logical_chunk_indices") or [unit["chunk_index"]]
|
|
if not any(start <= int(index) <= end for index in indices):
|
|
continue
|
|
if not all(start <= int(index) <= end for index in indices):
|
|
raise AssemblyPlanError("Review audio projection cannot split a physical group")
|
|
first, stop = unit.get("frame_start"), unit.get("frame_stop")
|
|
if type(first) is not int or type(stop) is not int or stop - first != int(unit["net_frames"]):
|
|
raise AssemblyPlanError("Review audio projection needs retained natural frame ranges")
|
|
if selected and selected[-1][1] != first:
|
|
raise AssemblyPlanError("Review audio projection ranges are not contiguous")
|
|
selected.append((first, stop))
|
|
covered.extend(int(index) for index in indices)
|
|
if not selected or covered != list(range(start, end + 1)):
|
|
raise AssemblyPlanError("Review audio projection is outside the accepted sequence")
|
|
return dict(version=1, kind="current_review_unit", coordinate_space="full_source_pcm",
|
|
fps=FPS, start_chunk=start, end_chunk=end,
|
|
source_start_frame=selected[0][0], source_stop_frame=selected[-1][1])
|
|
|
|
|
|
def make_assembly_plan(*args, **kwargs):
|
|
plan = _make_assembly_plan_v300(*args, **kwargs)
|
|
_enrich_assembly_plan(plan)
|
|
validate_assembly_plan(plan)
|
|
return plan
|
|
|
|
|
|
def _recombine_terminal_tensor(
|
|
first,
|
|
second,
|
|
*,
|
|
slices,
|
|
time_dim: int,
|
|
name: str,
|
|
):
|
|
"""Invert the terminal logical split without changing any tensor value."""
|
|
|
|
import torch
|
|
|
|
if not torch.is_tensor(first) or not torch.is_tensor(second):
|
|
raise ValueError(f"terminal {name} entries must be tensors")
|
|
if first.ndim != second.ndim:
|
|
raise ValueError(f"terminal {name} tensor ranks differ")
|
|
dim = int(time_dim)
|
|
if dim < 0:
|
|
dim += first.ndim
|
|
if dim < 0 or dim >= first.ndim:
|
|
raise ValueError(f"terminal {name} time dimension is invalid")
|
|
for axis, (left, right) in enumerate(zip(first.shape, second.shape)):
|
|
if axis != dim and int(left) != int(right):
|
|
raise ValueError(f"terminal {name} tensor geometry differs")
|
|
|
|
(first_start, first_stop), (second_start, second_stop) = slices
|
|
first_start = int(first_start)
|
|
first_stop = int(first_stop)
|
|
second_start = int(second_start)
|
|
second_stop = int(second_stop)
|
|
if first_start != 0 or second_start > first_stop or second_stop <= first_stop:
|
|
raise ValueError(f"terminal {name} split ranges do not form one physical unit")
|
|
if int(first.shape[dim]) != first_stop - first_start:
|
|
raise ValueError(f"terminal {name} first slice length is inconsistent")
|
|
if int(second.shape[dim]) != second_stop - second_start:
|
|
raise ValueError(f"terminal {name} second slice length is inconsistent")
|
|
|
|
overlap_start = max(first_start, second_start)
|
|
overlap_stop = min(first_stop, second_stop)
|
|
overlap_length = max(0, overlap_stop - overlap_start)
|
|
if overlap_length:
|
|
first_overlap = first.narrow(dim, overlap_start - first_start, overlap_length)
|
|
second_overlap = second.narrow(dim, overlap_start - second_start, overlap_length)
|
|
if not torch.equal(first_overlap, second_overlap):
|
|
raise ValueError(
|
|
f"terminal {name} overlap differs; refusing lossy physical reconstruction"
|
|
)
|
|
|
|
output_shape = list(first.shape)
|
|
output_shape[dim] = second_stop
|
|
output = first.new_empty(output_shape)
|
|
output.narrow(dim, first_start, first_stop - first_start).copy_(first)
|
|
output.narrow(dim, second_start, second_stop - second_start).copy_(second)
|
|
return output.contiguous()
|
|
|
|
|
|
def _terminal_physical_prompt(
|
|
prompts: list[str],
|
|
*,
|
|
chunk_seconds: float,
|
|
) -> tuple[str, str]:
|
|
"""Reproduce the established terminal-pair prompt without reparsing it later."""
|
|
first, second = prompts
|
|
if first == second:
|
|
return first, "shared_prompt_v1"
|
|
split = f"{float(chunk_seconds):g}"
|
|
stop = f"{float(chunk_seconds) * 2:g}"
|
|
return (
|
|
f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}",
|
|
"paired_timeline_v1",
|
|
)
|
|
|
|
|
|
def _attach_second_pass_contract(
|
|
plan: dict[str, Any],
|
|
*,
|
|
logical_entries: list[dict[str, Any]],
|
|
physical_entries: list[dict[str, Any]],
|
|
chunk_seconds: float,
|
|
logical_chunk_indices: list[int] | None = None,
|
|
) -> None:
|
|
"""Attach additive V3.5 metadata without changing V3.4 generation semantics."""
|
|
chunk_indices = (
|
|
list(range(1, len(logical_entries) + 1))
|
|
if logical_chunk_indices is None
|
|
else [int(value) for value in logical_chunk_indices]
|
|
)
|
|
if len(chunk_indices) != len(logical_entries):
|
|
raise ValueError("logical chunk identity count does not match entries")
|
|
decode_groups = plan.get("decode_groups")
|
|
if not isinstance(decode_groups, list):
|
|
decode_groups = [
|
|
{
|
|
"logical_chunk_indices": [chunk_indices[index]],
|
|
"terminal_merged": False,
|
|
"total_frames": plan["chunks"][index]["total_frames"],
|
|
"trim_frames": plan["chunks"][index]["trim_frames"],
|
|
}
|
|
for index in range(len(physical_entries))
|
|
]
|
|
|
|
prompt_by_chunk = {
|
|
chunk_indices[index]: str(entry.get("prompt", ""))
|
|
for index, entry in enumerate(logical_entries)
|
|
}
|
|
groups: list[dict[str, Any]] = []
|
|
for group_id, (entry, decode_group) in enumerate(
|
|
zip(physical_entries, decode_groups, strict=True)
|
|
):
|
|
video = entry["video"]
|
|
audio = entry["audio"]
|
|
logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]]
|
|
prompts = [prompt_by_chunk[index] for index in logical_chunks]
|
|
if len(prompts) == 1:
|
|
physical_prompt = prompts[0]
|
|
prompt_policy = "single"
|
|
elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")):
|
|
physical_prompt, prompt_policy = _terminal_physical_prompt(
|
|
prompts,
|
|
chunk_seconds=chunk_seconds,
|
|
)
|
|
else:
|
|
physical_prompt = "\n\n".join(prompts)
|
|
prompt_policy = "physical_group_v1"
|
|
|
|
groups.append(
|
|
{
|
|
"group_id": group_id,
|
|
"logical_chunks": logical_chunks,
|
|
"physical_prompt": physical_prompt,
|
|
"prompt_policy": prompt_policy,
|
|
"physical_frames": int(decode_group["total_frames"]),
|
|
"trim_prefix_frames": int(decode_group["trim_frames"]),
|
|
"terminal_merged": bool(decode_group.get("terminal_merged", False)),
|
|
"source_width": int(plan["width"]),
|
|
"source_height": int(plan["height"]),
|
|
"source_batch": int(video.shape[0]),
|
|
"latent_channels": int(video.shape[1]),
|
|
"source_latent_t": int(video.shape[2]),
|
|
"source_latent_h": int(video.shape[3]),
|
|
"source_latent_w": int(video.shape[4]),
|
|
"source_audio_shape": [int(value) for value in audio.shape],
|
|
}
|
|
)
|
|
|
|
plan["second_pass_contract"] = {
|
|
"version": SECOND_PASS_CONTRACT_VERSION,
|
|
"physical_groups": groups,
|
|
}
|
|
|
|
|
|
def prepare_physical_decode_entries(
|
|
entries,
|
|
*,
|
|
chunk_seconds: float,
|
|
preserve_final_frame: bool,
|
|
terminal_merged: bool,
|
|
terminal_initial_pair: bool | None = None,
|
|
):
|
|
"""Return external-VAE decode units while preserving logical plan metadata."""
|
|
|
|
logical_entries = list(entries)
|
|
plan = make_assembly_plan(
|
|
logical_entries,
|
|
chunk_seconds=float(chunk_seconds),
|
|
preserve_final_frame=bool(preserve_final_frame),
|
|
)
|
|
logical_chunk_indices = [
|
|
int(chunk["chunk_index"]) for chunk in plan["chunks"]
|
|
]
|
|
if not terminal_merged:
|
|
_attach_second_pass_contract(
|
|
plan,
|
|
logical_entries=logical_entries,
|
|
physical_entries=logical_entries,
|
|
chunk_seconds=float(chunk_seconds),
|
|
logical_chunk_indices=logical_chunk_indices,
|
|
)
|
|
return logical_entries, plan
|
|
if len(logical_entries) < 2:
|
|
raise ValueError("terminal physical decode requires two logical entries")
|
|
|
|
from ..v2.sequence import _terminal_pair_contract
|
|
|
|
initial_pair = (
|
|
len(logical_entries) == 2
|
|
if terminal_initial_pair is None
|
|
else bool(terminal_initial_pair)
|
|
)
|
|
contract = _terminal_pair_contract(
|
|
initial_pair=initial_pair,
|
|
chunk_seconds=float(chunk_seconds),
|
|
)
|
|
first_entry = logical_entries[-2]
|
|
second_entry = logical_entries[-1]
|
|
physical_video = _recombine_terminal_tensor(
|
|
first_entry["video"],
|
|
second_entry["video"],
|
|
slices=contract["video_slices"],
|
|
time_dim=2,
|
|
name="video",
|
|
)
|
|
physical_audio = _recombine_terminal_tensor(
|
|
first_entry["audio"],
|
|
second_entry["audio"],
|
|
slices=contract["audio_slices"],
|
|
time_dim=-1,
|
|
name="audio",
|
|
)
|
|
|
|
physical_entry = dict(first_entry)
|
|
physical_entry["video"] = physical_video
|
|
physical_entry["audio"] = physical_audio
|
|
decode_entries = [*logical_entries[:-2], physical_entry]
|
|
|
|
logical_chunks = list(plan["chunks"])
|
|
decode_groups = []
|
|
for chunk in logical_chunks[:-2]:
|
|
group = dict(chunk)
|
|
group["logical_chunk_indices"] = [int(chunk["chunk_index"])]
|
|
group["terminal_merged"] = False
|
|
decode_groups.append(group)
|
|
|
|
first_chunk = logical_chunks[-2]
|
|
second_chunk = logical_chunks[-1]
|
|
frame_start = int(first_chunk.get("frame_start", 0))
|
|
net_frames = int(contract["physical_frames"]) - int(
|
|
contract["physical_context_frames"]
|
|
)
|
|
terminal_group = dict(first_chunk)
|
|
terminal_group.update(
|
|
{
|
|
"total_frames": int(contract["physical_frames"]),
|
|
"trim_frames": int(contract["physical_context_frames"]),
|
|
"net_frames": net_frames,
|
|
"context_frames": int(contract["physical_context_frames"]),
|
|
"expected_video_latent_t": int(physical_video.shape[2]),
|
|
"expected_audio_latent_t": int(physical_audio.shape[-1]),
|
|
"frame_start": frame_start,
|
|
"frame_stop": frame_start + net_frames,
|
|
"logical_chunk_indices": [
|
|
int(first_chunk["chunk_index"]),
|
|
int(second_chunk["chunk_index"]),
|
|
],
|
|
"terminal_merged": True,
|
|
}
|
|
)
|
|
decode_groups.append(terminal_group)
|
|
|
|
plan = dict(plan)
|
|
plan["decode_group_version"] = 1
|
|
plan["logical_chunk_count"] = len(logical_chunks)
|
|
plan["physical_decode_group_count"] = len(decode_groups)
|
|
plan["decode_groups"] = decode_groups
|
|
_attach_second_pass_contract(
|
|
plan,
|
|
logical_entries=logical_entries,
|
|
physical_entries=decode_entries,
|
|
chunk_seconds=float(chunk_seconds),
|
|
logical_chunk_indices=logical_chunk_indices,
|
|
)
|
|
validate_assembly_plan(plan)
|
|
return decode_entries, plan
|