Files
ComfyUI-H3-Continuum-Tiled/v3/reference_storage_contract.py
T

380 lines
19 KiB
Python

"""Private RR-R5 physical-group Reference persistence contract.
The stored descriptor records *effective* conditioning, not selector spelling.
No VAE, CLIP, Sampling, or saved-run operation is performed here.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import hashlib
import json
from typing import Any
from ..branch_provenance import physical_groups
from ..reference import HYBRID_PRESENTATION_VERSION, REFERENCE_PREPROCESS_VERSION
from .reference_effective_plan import PictureMap, compile_group_picture_map
from .reference_routing import REFERENCE_SLOT_IDS
REFERENCE_ROUTING_CONTRACT_VERSION = 1
SESSION_ROUTING_IDENTITY_VERSION = 1
_HEX = frozenset("0123456789abcdef")
class ReferenceStorageContractError(ValueError):
"""A routed producer or accepted-prefix contract cannot be trusted."""
def canonical_json(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
def canonical_sha256(value: Any) -> str:
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
def routed_session_identity(base_identity_hash: str) -> str:
return canonical_sha256({
"rr_r5_session_identity_version": SESSION_ROUTING_IDENTITY_VERSION,
"base_identity_hash": str(base_identity_hash),
})
def _valid_sha(value: Any) -> bool:
return isinstance(value, str) and len(value) == 64 and all(c in _HEX for c in value)
def _anchor(value: str, present: bool) -> str | None:
if not present:
return None
if not _valid_sha(value):
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
return value
def _present(value: str) -> bool:
if str(value).lower() in {"", "none", "null"}:
return False
if not _valid_sha(value):
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
return True
def descriptor_from_facts(
*, physical_group: int, logical_chunks: tuple[int, ...],
selected_source_slot_ids: tuple[str, ...],
prepared_image_hashes: tuple[str, ...], picture_map: PictureMap,
first_frame_hash: str, last_frame_hash: str,
effective_prompt: str, size_mode: str,
terminal_prompt_policy: str | None,
) -> dict[str, Any]:
if len(selected_source_slot_ids) != len(prepared_image_hashes):
raise ReferenceStorageContractError("selected Reference slots and image hashes differ")
if picture_map.source_slot_ids != selected_source_slot_ids:
raise ReferenceStorageContractError("Reference Picture map differs from selected source slots")
if any(slot not in REFERENCE_SLOT_IDS for slot in selected_source_slot_ids):
raise ReferenceStorageContractError("selected Reference slot is unknown")
if tuple(sorted(selected_source_slot_ids, key=REFERENCE_SLOT_IDS.index)) != selected_source_slot_ids:
raise ReferenceStorageContractError("selected Reference slots are not in fixed R1-R9 order")
if len(set(selected_source_slot_ids)) != len(selected_source_slot_ids):
raise ReferenceStorageContractError("selected Reference slot is duplicated")
if any(not _valid_sha(value) for value in prepared_image_hashes):
raise ReferenceStorageContractError("prepared Reference image hash is invalid")
if any(item.image_sha256 != prepared_image_hashes[index]
for index, item in enumerate(picture_map.references)):
raise ReferenceStorageContractError("Reference Picture map image hash is inconsistent")
if not logical_chunks or tuple(range(logical_chunks[0], logical_chunks[-1] + 1)) != logical_chunks:
raise ReferenceStorageContractError("physical group logical chunks are not contiguous")
if int(physical_group) != logical_chunks[0]:
raise ReferenceStorageContractError("physical group number differs from its first logical chunk")
if (len(logical_chunks) > 1) != (terminal_prompt_policy is not None):
raise ReferenceStorageContractError("terminal prompt policy does not match physical group")
first_present = picture_map.first_picture_number is not None
last_present = picture_map.last_picture_number is not None
order = []
if first_present:
order.append("first_frame")
if last_present:
order.append("last_frame")
order.extend(selected_source_slot_ids)
descriptor: dict[str, Any] = {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"physical_group": int(physical_group),
"logical_chunks": list(logical_chunks),
"terminal_atomic": len(logical_chunks) > 1,
"references": [
{
"source_slot_id": slot_id,
"prepared_sha256": prepared_image_hashes[index],
"picture_number": picture_map.references[index].picture_number,
}
for index, slot_id in enumerate(selected_source_slot_ids)
],
"first_anchor": _anchor(first_frame_hash, first_present),
"last_anchor": _anchor(last_frame_hash, last_present),
"picture_order": order,
"picture_presentation_version": HYBRID_PRESENTATION_VERSION,
"effective_prompt_sha256": hashlib.sha256(str(effective_prompt).encode("utf-8")).hexdigest(),
"terminal_prompt_policy": str(terminal_prompt_policy) if len(logical_chunks) > 1 else None,
}
if selected_source_slot_ids:
descriptor["reference_size_mode"] = str(size_mode)
descriptor["reference_preprocess_version"] = REFERENCE_PREPROCESS_VERSION
return descriptor
@dataclass(frozen=True, slots=True)
class ReferenceStoragePlan:
"""Canonical JSON is immutable even if the caller's source dicts are not."""
descriptor_jsons: tuple[str, ...]
routing_config_sha256: str
requested_routes: tuple[tuple[str, ...], ...]
_group_index: dict[int, int] = field(init=False, repr=False, compare=False, hash=False)
def __post_init__(self) -> None:
index: dict[int, int] = {}
for position, payload in enumerate(self.descriptor_jsons):
descriptor = json.loads(payload)
# Preserve the former first-match lookup for duplicate group IDs.
index.setdefault(int(descriptor["physical_group"]), position)
object.__setattr__(self, "_group_index", index)
def _record_at(self, position: int) -> dict[str, Any]:
payload = self.descriptor_jsons[position]
return {
"descriptor": json.loads(payload),
"sha256": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
}
@property
def group_contracts(self) -> list[dict[str, Any]]:
return [self._record_at(position) for position in range(len(self.descriptor_jsons))]
def group_for(self, physical_group: int) -> dict[str, Any]:
position = self._group_index.get(int(physical_group))
if position is None:
raise ReferenceStorageContractError(
f"physical group {physical_group} has no frozen Reference contract"
)
return self._record_at(position)
def audit(self) -> dict[str, Any]:
return {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"routing_config_sha256": self.routing_config_sha256,
"requested_routes": [list(route) for route in self.requested_routes],
}
def build_reference_storage_plan(
*, runtime: Any, prompts: list[str], first_frame_hash: str,
last_frame_hash: str, terminal_prompt: str | None = None,
terminal_prompt_policy: str | None = None,
through_chunk: int | None = None,
) -> ReferenceStoragePlan:
"""Prehash only connected R slots selected by the requested group prefix."""
from .reference_runtime import ReferenceRoutingRuntime, resolve_reference_group_prompt
if not isinstance(runtime, ReferenceRoutingRuntime):
raise ReferenceStorageContractError("routed storage requires ReferenceRoutingRuntime")
schedule = runtime.schedule
total = int(schedule.total_chunks or 0)
if total < 1 or len(prompts) != total:
raise ReferenceStorageContractError("Reference schedule and prompt count differ")
runtime.check_physical_contract(
chunks=total, terminal_merge_enabled=bool(schedule.terminal_merge_enabled),
)
limit = total if through_chunk is None else int(through_chunk)
if limit < 1 or limit > total:
raise ReferenceStorageContractError("Reference plan prefix is outside configured chunks")
routes = [route for route in schedule.physical_routes if route.logical_chunks[-1] <= limit]
if not routes or routes[-1].logical_chunks[-1] != limit:
raise ReferenceStorageContractError("Reference plan prefix ends inside a physical group")
connected = set(runtime.inputs.connected_source_slot_ids)
selected = {slot for route in routes for slot in (route.reference_slot_ids or ()) if slot in connected}
image_hashes: dict[str, str] = {}
for slot in REFERENCE_SLOT_IDS:
if slot not in selected:
continue
assets = runtime.inputs.prepare_selected((slot,))
if assets is None or assets.source_slot_ids != (slot,) or len(assets.image_hashes) != 1:
raise ReferenceStorageContractError(f"selected {slot} cannot be prepared")
image_hashes[slot] = assets.image_hashes[0]
del assets
descriptor_jsons: list[str] = []
requested_routes: list[tuple[str, ...]] = []
pending_group_warnings: dict[int, tuple[str, ...]] = {}
for route in routes:
requested = route.reference_slot_ids
if requested is None:
raise ReferenceStorageContractError("conflicting terminal Reference route")
requested_routes.append(requested)
source_slots = tuple(slot for slot in requested if slot in connected)
hashes = tuple(image_hashes[slot] for slot in source_slots)
final = route.logical_chunks[-1] == total
has_first = _present(first_frame_hash)
has_last = _present(last_frame_hash) and (bool(source_slots) or final)
picture_map = compile_group_picture_map(
source_slot_ids=source_slots,
reference_image_hashes=hashes,
has_first_image=has_first,
has_last_image=has_last,
)
prompt = (
terminal_prompt if len(route.logical_chunks) > 1
else prompts[route.logical_chunks[0] - 1]
)
if prompt is None:
raise ReferenceStorageContractError("terminal physical prompt is unavailable")
effective_prompt, warnings = resolve_reference_group_prompt(
physical_group=int(route.physical_group), prompt=prompt,
requested_source_slot_ids=requested,
connected_source_slot_ids=runtime.inputs.connected_source_slot_ids,
picture_map=picture_map,
)
pending_group_warnings[int(route.physical_group)] = warnings
descriptor = descriptor_from_facts(
physical_group=route.physical_group,
logical_chunks=route.logical_chunks,
selected_source_slot_ids=source_slots,
prepared_image_hashes=hashes,
picture_map=picture_map,
first_frame_hash=first_frame_hash,
last_frame_hash=last_frame_hash,
effective_prompt=effective_prompt,
size_mode=runtime.inputs.size_mode,
terminal_prompt_policy=terminal_prompt_policy if len(route.logical_chunks) > 1 else None,
)
descriptor_jsons.append(canonical_json(descriptor))
routing_config_sha256 = canonical_sha256({
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"mode": schedule.mode,
"logical_routes": [
[route.logical_chunk, list(route.reference_slot_ids)]
for route in schedule.logical_routes
],
})
plan = ReferenceStoragePlan(
tuple(descriptor_jsons), routing_config_sha256, tuple(requested_routes)
)
runtime.group_warnings.update({
group: warnings for group, warnings in pending_group_warnings.items()
if group not in runtime.observed_group_contracts
})
return plan
def validate_reference_group_contracts(
records: Any, *, chunk_count: int, terminal_merge_enabled: bool,
) -> list[dict[str, Any]]:
"""Validate the complete routed stored plan, not just its declared hashes."""
expected = physical_groups(chunks=chunk_count, terminal_merge_enabled=terminal_merge_enabled)
if not isinstance(records, list) or len(records) != len(expected):
raise ReferenceStorageContractError("stored Reference group count does not match physical topology")
clean: list[dict[str, Any]] = []
for record, group in zip(records, expected):
if not isinstance(record, dict) or not isinstance(record.get("descriptor"), dict):
raise ReferenceStorageContractError("stored Reference group descriptor is missing")
descriptor = record["descriptor"]
if descriptor.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
raise ReferenceStorageContractError("stored Reference group version is unsupported")
if (descriptor.get("physical_group") != group.physical_group
or descriptor.get("logical_chunks") != list(range(group.start, group.end + 1))
or descriptor.get("terminal_atomic") is not (group.start != group.end)):
raise ReferenceStorageContractError("stored Reference group boundary is invalid")
if not _valid_sha(record.get("sha256")) or canonical_sha256(descriptor) != record["sha256"]:
raise ReferenceStorageContractError("stored Reference group SHA-256 is invalid")
refs = descriptor.get("references")
if not isinstance(refs, list):
raise ReferenceStorageContractError("stored Reference group source list is invalid")
slots = [ref.get("source_slot_id") for ref in refs if isinstance(ref, dict)]
if (len(slots) != len(refs) or any(slot not in REFERENCE_SLOT_IDS for slot in slots)
or slots != sorted(set(slots), key=REFERENCE_SLOT_IDS.index)
or any(not _valid_sha(ref.get("prepared_sha256")) for ref in refs)):
raise ReferenceStorageContractError("stored Reference group source identity is invalid")
expected_order = (["first_frame"] if descriptor.get("first_anchor") else [])
expected_order += (["last_frame"] if descriptor.get("last_anchor") else [])
expected_order += slots
if descriptor.get("picture_order") != expected_order:
raise ReferenceStorageContractError("stored Reference Picture order is invalid")
if descriptor.get("picture_presentation_version") != HYBRID_PRESENTATION_VERSION:
raise ReferenceStorageContractError("stored Reference Picture presentation version is invalid")
offset = int(descriptor.get("first_anchor") is not None) + int(
descriptor.get("last_anchor") is not None
)
if any(ref.get("picture_number") != offset + index + 1
for index, ref in enumerate(refs)):
raise ReferenceStorageContractError("stored Reference Picture numbers are invalid")
if (group.start != group.end) != (descriptor.get("terminal_prompt_policy") is not None):
raise ReferenceStorageContractError("stored terminal prompt policy is invalid")
if any(not _valid_sha(descriptor.get(name)) for name in ("effective_prompt_sha256",)):
raise ReferenceStorageContractError("stored Reference effective Prompt hash is invalid")
if any(descriptor.get(name) is not None and not _valid_sha(descriptor[name])
for name in ("first_anchor", "last_anchor")):
raise ReferenceStorageContractError("stored Reference anchor hash is invalid")
if refs and (descriptor.get("reference_preprocess_version") != REFERENCE_PREPROCESS_VERSION
or not isinstance(descriptor.get("reference_size_mode"), str)):
raise ReferenceStorageContractError("stored Reference preprocessing contract is invalid")
if not refs and ("reference_preprocess_version" in descriptor or "reference_size_mode" in descriptor):
raise ReferenceStorageContractError("empty Reference group has preprocessing metadata")
clean.append(record)
return clean
def session_routing_settings(plan: ReferenceStoragePlan, accepted_chunks: int) -> dict[str, Any]:
accepted: list[dict[str, Any]] = []
for record in plan.group_contracts:
descriptor = record["descriptor"]
end = int(descriptor["logical_chunks"][-1])
if end > accepted_chunks:
break
accepted.append({
"physical_group": int(descriptor["physical_group"]),
"logical_chunks": list(descriptor["logical_chunks"]),
"sha256": record["sha256"],
})
if not accepted or accepted[-1]["logical_chunks"][-1] != accepted_chunks:
raise ReferenceStorageContractError("accepted Session ends inside a physical group")
return {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"accepted_groups": accepted,
"routing_config_sha256": plan.routing_config_sha256,
}
def compatible_session_prefix(
evidence: Any, plan: ReferenceStoragePlan, *, saved_chunks: int,
) -> int:
"""Return a complete-group accepted prefix; malformed evidence is rejected."""
if not isinstance(evidence, dict) or evidence.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
raise ReferenceStorageContractError("saved Session has no RR-R5 group evidence")
groups = evidence.get("accepted_groups")
if not isinstance(groups, list) or not groups:
raise ReferenceStorageContractError("saved Session group evidence is missing")
last_end = 0
for position, item in enumerate(groups):
if not isinstance(item, dict) or not _valid_sha(item.get("sha256")):
raise ReferenceStorageContractError("saved Session group hash is invalid")
chunks = item.get("logical_chunks")
if (not isinstance(chunks, list) or not chunks or chunks[0] != last_end + 1
or chunks != list(range(chunks[0], chunks[-1] + 1))
or item.get("physical_group") != chunks[0]):
raise ReferenceStorageContractError("saved Session group boundaries are invalid")
last_end = chunks[-1]
if last_end != saved_chunks:
raise ReferenceStorageContractError("saved Session group evidence does not cover its chunks")
preserved = 0
current = plan.group_contracts
for position, item in enumerate(groups):
if position >= len(current):
break
now = current[position]
if (item["physical_group"] != now["descriptor"]["physical_group"]
or item["logical_chunks"] != now["descriptor"]["logical_chunks"]
or item["sha256"] != now["sha256"]):
break
preserved = item["logical_chunks"][-1]
return preserved