git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
3752 lines
168 KiB
Python
3752 lines
168 KiB
Python
"""Crash-safe V3.1B raw-latent storage and deterministic auto-resume."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextvars
|
|
import copy
|
|
import hashlib
|
|
import functools
|
|
import inspect
|
|
import json
|
|
import marshal
|
|
import os
|
|
import re
|
|
import shutil
|
|
import time
|
|
import socket
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from types import MappingProxyType
|
|
from typing import Any, Mapping
|
|
|
|
import torch
|
|
from safetensors import safe_open
|
|
from safetensors.torch import save_file
|
|
|
|
from .branch_provenance import (
|
|
BRANCH_PROVENANCE_VERSION,
|
|
TAKE_ACTION_AUTOMATIC,
|
|
TAKE_ACTION_CONTINUE,
|
|
TAKE_ACTION_OPTIONS,
|
|
TAKE_ACTION_USE,
|
|
BranchProvenanceError,
|
|
active_revision_map,
|
|
make_group_revision,
|
|
physical_groups,
|
|
resolve_chain,
|
|
)
|
|
from .constants import CONTINUUM_ACTUAL_PREFIX_STEPS, continuity_storage_mode
|
|
from .spectrum_signature import spectrum_binding_signature
|
|
from .preview_override_signature import (
|
|
PREVIEW_REUSE_REASONS, observe_model_wrappers, preview_override_reuse_reasons,
|
|
)
|
|
from .v2.session import make_session, validate_chunk_entry
|
|
from .v3.review_control import (
|
|
GENERATION_MODE_REVIEW,
|
|
REVIEW_ACTION_CONTINUE,
|
|
REVIEW_ACTION_REGENERATE_CURRENT,
|
|
REVISION_STATUS_COMPLETE,
|
|
REVISION_STATUS_IN_PROGRESS,
|
|
REVISION_STATUS_INTERRUPTED,
|
|
REVISION_STATUS_REVIEW_READY,
|
|
RUN_STORAGE_SAVE_AUTO_RESUME,
|
|
ReviewControlError,
|
|
ReviewUnit,
|
|
make_review_pause_metadata,
|
|
resolve_review_execution,
|
|
resolve_take_execution,
|
|
validate_review_prefix_metadata,
|
|
)
|
|
from .v3.planning_types import PrefixFacts
|
|
from .v3.reference_storage_contract import (
|
|
REFERENCE_ROUTING_CONTRACT_VERSION,
|
|
ReferenceStoragePlan,
|
|
ReferenceStorageContractError,
|
|
session_routing_settings,
|
|
validate_reference_group_contracts,
|
|
)
|
|
|
|
|
|
RUN_STORAGE_SCHEMA_VERSION = 3
|
|
RUN_STORAGE_READABLE_SCHEMA_VERSIONS = (2, RUN_STORAGE_SCHEMA_VERSION)
|
|
from .conditioning import (
|
|
CONDITIONING_MODES,
|
|
conditioning_mode_from_presence,
|
|
conditioning_mode_uses_video_vae,
|
|
)
|
|
|
|
|
|
SAMPLING_CONTRACT_VERSION = 6
|
|
SAMPLING_READABLE_CONTRACT_VERSIONS = {5, 6}
|
|
RUNTIME_SIGNATURE_VERSION = 1
|
|
_OBSERVATION_MAX_ITEMS = 4096
|
|
REVIEW_CONTROL_VERSION = 1
|
|
RUN_STORAGE_OFF = "Off"
|
|
RUN_STORAGE_AUTO = "Save + Auto Resume"
|
|
RUN_STORAGE_OPTIONS = (RUN_STORAGE_OFF, RUN_STORAGE_AUTO)
|
|
|
|
_ACTIVE: contextvars.ContextVar["RunStorageController | None"] = contextvars.ContextVar(
|
|
"h3_continuum_run_storage", default=None
|
|
)
|
|
_RESERVED = {
|
|
"CON", "PRN", "AUX", "NUL",
|
|
*(f"COM{i}" for i in range(1, 10)),
|
|
*(f"LPT{i}" for i in range(1, 10)),
|
|
}
|
|
_ADDRESS = re.compile(r"0x[0-9a-fA-F]+")
|
|
|
|
|
|
class RunStorageError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ValidatedPrefix:
|
|
"""One immutable Run Storage prefix after manifest/raw/group validation."""
|
|
|
|
entries: tuple[dict[str, Any], ...]
|
|
records: tuple[dict[str, Any], ...]
|
|
manifest: Mapping[str, Any] | None
|
|
revision_id: str | None
|
|
status: str | None
|
|
review_unit: ReviewUnit | None
|
|
effective_nonce: int
|
|
branch_regenerate_from: int
|
|
physical_group_boundaries: tuple[tuple[int, int], ...]
|
|
configured_chunks: int
|
|
|
|
def to_planning_facts(self) -> PrefixFacts:
|
|
return PrefixFacts(
|
|
accepted_chunks=len(self.entries),
|
|
configured_chunks=int(self.configured_chunks),
|
|
revision_id=self.revision_id,
|
|
status=self.status,
|
|
review_unit=self.review_unit,
|
|
effective_nonce=int(self.effective_nonce),
|
|
branch_regenerate_from=int(self.branch_regenerate_from),
|
|
physical_group_boundaries=self.physical_group_boundaries,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class EmptyValidatedPrefix:
|
|
"""The absence of a Run Storage prefix, not of other continuation sources."""
|
|
|
|
configured_chunks: int
|
|
entries: tuple = ()
|
|
records: tuple = ()
|
|
manifest: None = None
|
|
revision_id: None = None
|
|
status: None = None
|
|
review_unit: None = None
|
|
effective_nonce: int = 0
|
|
branch_regenerate_from: int = 0
|
|
physical_group_boundaries: tuple = ()
|
|
|
|
def to_planning_facts(self) -> PrefixFacts:
|
|
return PrefixFacts(
|
|
accepted_chunks=0,
|
|
configured_chunks=int(self.configured_chunks),
|
|
revision_id=None,
|
|
status=None,
|
|
review_unit=None,
|
|
effective_nonce=0,
|
|
branch_regenerate_from=0,
|
|
physical_group_boundaries=(),
|
|
)
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
|
|
|
|
def _canonical(value: Any) -> str:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
|
|
|
|
|
def _hash(value: Any) -> str:
|
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _review_contract_mismatches(stored: dict, current: dict) -> list[str]:
|
|
"""Diagnostic hashes only; never relax identity or print payload values."""
|
|
result: list[str] = []
|
|
|
|
def compare(left: Any, right: Any, path: str) -> None:
|
|
if _hash(left) == _hash(right):
|
|
return
|
|
# Split known contract containers, but not arbitrary wrapper/prompt data.
|
|
containers = {
|
|
"global", "global.model", "global.model.runtime",
|
|
"global.clip", "global.clip.runtime", "global.video_vae",
|
|
"global.video_vae.runtime",
|
|
}
|
|
if path in containers and isinstance(left, dict) and isinstance(right, dict):
|
|
for key in sorted(left.keys() | right.keys()):
|
|
if key not in left or key not in right:
|
|
result.append(f"{path}.{key}: field missing on one side")
|
|
else:
|
|
compare(left[key], right[key], f"{path}.{key}")
|
|
return
|
|
result.append(f"{path}: saved={_hash(left)[:16]} current={_hash(right)[:16]}")
|
|
|
|
# Match the inputs to _nonce_lineage_hash, not derived nonce/branch fields.
|
|
for key in (
|
|
"global", "chunk_count", "prompt_mode", "prompt_hashes",
|
|
"last_frame_hash", "timeline_video_chunk_contracts",
|
|
):
|
|
compare(stored.get(key), current.get(key), key)
|
|
return result
|
|
|
|
|
|
def _canonical_optional_hash(value: Any) -> str:
|
|
"""Normalize legacy absent-value spellings without changing real hashes."""
|
|
|
|
if value is None:
|
|
return ""
|
|
text = str(value).strip()
|
|
return "" if text.lower() in {"", "none", "null"} else text
|
|
|
|
|
|
def sanitize_run_name(value: str) -> str:
|
|
name = str(value)
|
|
if not name or name != name.strip() or len(name) > 96:
|
|
raise RunStorageError("Run Name must be 1-96 characters without surrounding spaces")
|
|
if any(ord(char) < 32 or char in '<>:"/\\|?*' for char in name):
|
|
raise RunStorageError("Run Name contains a reserved path character")
|
|
if name.endswith((".", " ")) or name in {".", ".."}:
|
|
raise RunStorageError("Run Name must not be a relative path or end with dot/space")
|
|
if name.split(".", 1)[0].upper() in _RESERVED:
|
|
raise RunStorageError(f"Run Name {name!r} is reserved by Windows")
|
|
return name
|
|
|
|
|
|
def resolve_run_storage_name(
|
|
*, project_id: str, legacy_run_name: str = "", automatic_key: str = ""
|
|
) -> str:
|
|
"""Resolve a stable storage folder while preserving old Run Name workflows."""
|
|
legacy = str(legacy_run_name)
|
|
if legacy:
|
|
return sanitize_run_name(legacy)
|
|
raw_project_id = str(project_id).strip()
|
|
if not raw_project_id:
|
|
key = str(automatic_key).strip()
|
|
if not key:
|
|
raise RunStorageError(
|
|
"Automatic Resume ID is unavailable; enter a Run Name for this workflow"
|
|
)
|
|
digest = hashlib.sha256(
|
|
f"H3ContinuumSamplerProduction:{key}".encode("utf-8")
|
|
).hexdigest()[:24]
|
|
return f"run_auto_{digest}"
|
|
try:
|
|
parsed = uuid.UUID(raw_project_id)
|
|
except (ValueError, AttributeError, TypeError) as exc:
|
|
raise RunStorageError(
|
|
"Automatic Project ID is missing or invalid; reload ComfyUI and the workflow"
|
|
) from exc
|
|
if parsed.int == 0:
|
|
raise RunStorageError("Automatic Project ID must not be the nil UUID")
|
|
return f"run_{parsed.hex}"
|
|
|
|
|
|
def automatic_project_key(prompt: Any, unique_id: Any) -> str:
|
|
"""Build a stable key from sampler identity and graph topology only."""
|
|
node_id = str(unique_id).strip()
|
|
if not node_id:
|
|
raise RunStorageError("Automatic Resume ID requires the sampler node ID")
|
|
if not isinstance(prompt, dict):
|
|
raise RunStorageError(
|
|
"Automatic Resume ID requires the ComfyUI prompt graph; enter a Run Name"
|
|
)
|
|
def visit(current_id: str, stack: set[str]) -> dict[str, Any]:
|
|
if current_id in stack:
|
|
return {"node_id": current_id, "cycle": True}
|
|
raw = prompt.get(current_id)
|
|
if not isinstance(raw, dict):
|
|
return {"node_id": current_id, "missing": True}
|
|
links = []
|
|
inputs = raw.get("inputs") or {}
|
|
next_stack = set(stack)
|
|
next_stack.add(current_id)
|
|
if isinstance(inputs, dict):
|
|
for name in sorted(inputs):
|
|
if current_id == node_id and name == "reference_audio_vae":
|
|
audio_link = inputs.get("reference_audio_1")
|
|
if not (
|
|
isinstance(audio_link, (list, tuple))
|
|
and len(audio_link) == 2
|
|
and isinstance(audio_link[0], (str, int))
|
|
and isinstance(audio_link[1], int)
|
|
and str(audio_link[0]) in prompt
|
|
):
|
|
continue
|
|
if current_id == node_id and name == "audio_vae":
|
|
audio_link = inputs.get("driving_audio")
|
|
if not (
|
|
isinstance(audio_link, (list, tuple))
|
|
and len(audio_link) == 2
|
|
and isinstance(audio_link[0], (str, int))
|
|
and isinstance(audio_link[1], int)
|
|
and str(audio_link[0]) in prompt
|
|
):
|
|
continue
|
|
value = inputs[name]
|
|
if (
|
|
isinstance(value, (list, tuple))
|
|
and len(value) == 2
|
|
and isinstance(value[0], (str, int))
|
|
and isinstance(value[1], int)
|
|
and str(value[0]) in prompt
|
|
):
|
|
child_id = str(value[0])
|
|
links.append(
|
|
{
|
|
"input": str(name),
|
|
"output": int(value[1]),
|
|
"node": visit(child_id, next_stack),
|
|
}
|
|
)
|
|
return {
|
|
"node_id": current_id,
|
|
"class_type": str(raw.get("class_type", "")),
|
|
"links": links,
|
|
}
|
|
return _hash({"sampler_node_id": node_id, "topology": visit(node_id, set())})
|
|
|
|
|
|
def _output_root() -> Path:
|
|
try:
|
|
import folder_paths
|
|
|
|
return Path(folder_paths.get_output_directory())
|
|
except Exception:
|
|
return Path.cwd() / "output"
|
|
|
|
|
|
def _fsync_file(path: Path) -> None:
|
|
# Windows requires a writable descriptor for fsync(). The file contents
|
|
# are already complete at this point; opening r+b does not modify them.
|
|
with path.open("r+b") as handle:
|
|
os.fsync(handle.fileno())
|
|
|
|
|
|
def _fsync_dir(path: Path) -> None:
|
|
if os.name == "nt":
|
|
return
|
|
descriptor = os.open(str(path), os.O_RDONLY)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
_WINDOWS_REPLACE_RETRY_SECONDS = (0.02, 0.05, 0.10, 0.20, 0.40)
|
|
_WINDOWS_REPLACE_RETRY_WINERRORS = frozenset({5, 32})
|
|
|
|
|
|
def _replace_json_atomically(temporary: Path, destination: Path) -> None:
|
|
"""Publish a complete JSON file without weakening replace atomicity.
|
|
|
|
Windows virus scanners and indexers can briefly hold either side of an
|
|
otherwise complete replacement. Retry only those transient Windows errors;
|
|
all other failures retain the old destination and fail with diagnostics.
|
|
"""
|
|
|
|
attempts = 0
|
|
for delay in (*_WINDOWS_REPLACE_RETRY_SECONDS, None):
|
|
try:
|
|
os.replace(temporary, destination)
|
|
return
|
|
except PermissionError as exc:
|
|
winerror = getattr(exc, "winerror", None)
|
|
error_code = winerror if winerror is not None else exc.errno
|
|
transient = (
|
|
error_code in _WINDOWS_REPLACE_RETRY_WINERRORS
|
|
and delay is not None
|
|
)
|
|
if transient:
|
|
attempts += 1
|
|
time.sleep(delay)
|
|
continue
|
|
source_exists = temporary.exists()
|
|
destination_exists = destination.exists()
|
|
readonly = False
|
|
if destination_exists:
|
|
try:
|
|
readonly = not bool(destination.stat().st_mode & 0o200)
|
|
except OSError:
|
|
readonly = False
|
|
raise RunStorageError(
|
|
"Run Storage atomic manifest replace failed "
|
|
f"after {attempts + 1} attempt(s): src={temporary}; "
|
|
f"dst={destination}; src_exists={source_exists}; "
|
|
f"dst_exists={destination_exists}; dst_readonly={readonly}; "
|
|
f"parent={destination.parent}; winerror={winerror}; "
|
|
f"errno={exc.errno}"
|
|
) from exc
|
|
|
|
|
|
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
try:
|
|
with temporary.open("w", encoding="utf-8", newline="\n") as handle:
|
|
json.dump(payload, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
_replace_json_atomically(temporary, path)
|
|
_fsync_dir(path.parent)
|
|
finally:
|
|
if temporary.exists():
|
|
temporary.unlink()
|
|
|
|
|
|
def _read_json(path: Path) -> dict[str, Any]:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise RunStorageError(f"{path.name} must contain a JSON object")
|
|
return value
|
|
|
|
|
|
def _file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _qualified(value: Any) -> str:
|
|
if value is None:
|
|
return "none"
|
|
target = value if isinstance(value, type) or callable(value) else type(value)
|
|
module = getattr(target, "__module__", "")
|
|
name = getattr(target, "__qualname__", getattr(target, "__name__", type(target).__name__))
|
|
return f"{module}.{name}"
|
|
|
|
|
|
def _tensor_probe(tensor: torch.Tensor, *, dtype: torch.dtype | None = None) -> dict[str, Any]:
|
|
flat = tensor.detach().reshape(-1)
|
|
count = int(flat.numel())
|
|
positions = sorted({0, count // 4, count // 2, 3 * count // 4, count - 1}) if count else []
|
|
if positions:
|
|
index = torch.tensor(positions, device=flat.device)
|
|
values = flat[index]
|
|
if dtype is not None and dtype != tensor.dtype:
|
|
values = values.to(dtype=dtype)
|
|
sample = values.contiguous().cpu().view(torch.uint8).numpy().tobytes()
|
|
else:
|
|
sample = b""
|
|
return {
|
|
"shape": list(tensor.shape),
|
|
"dtype": str(dtype if dtype is not None else tensor.dtype),
|
|
"sample_sha256": hashlib.sha256(sample).hexdigest(),
|
|
}
|
|
|
|
|
|
def _tensor_exact(tensor: torch.Tensor) -> dict[str, Any]:
|
|
value = tensor.detach().contiguous().cpu()
|
|
raw = value.view(torch.uint8).numpy().tobytes()
|
|
return {
|
|
"shape": list(value.shape),
|
|
"dtype": str(value.dtype),
|
|
"exact_sha256": hashlib.sha256(raw).hexdigest(),
|
|
}
|
|
|
|
|
|
def _observe(value: Any, *, depth: int = 0) -> tuple[Any, bool]:
|
|
try:
|
|
return _observe_value(value, depth=depth)
|
|
except Exception as exc:
|
|
return {"type": _qualified(value), "observation_error": type(exc).__name__}, False
|
|
|
|
|
|
def _observe_value(value: Any, *, depth: int = 0) -> tuple[Any, bool]:
|
|
if depth > 10:
|
|
return {"type": _qualified(value), "truncated": True}, False
|
|
if value is None or isinstance(value, (bool, int, float, str)):
|
|
return value, True
|
|
if isinstance(value, Path):
|
|
return str(value), True
|
|
if torch.is_tensor(value):
|
|
try:
|
|
return {"tensor": _tensor_probe(value)}, True
|
|
except Exception:
|
|
return {"tensor_type": _qualified(value)}, False
|
|
if isinstance(value, dict):
|
|
if len(value) > _OBSERVATION_MAX_ITEMS:
|
|
return {"type": _qualified(value), "items": len(value), "truncated": True}, False
|
|
result: dict[str, Any] = {}
|
|
safe = True
|
|
for key in sorted(value, key=lambda item: str(item)):
|
|
observed, item_safe = _observe(value[key], depth=depth + 1)
|
|
result[str(key)] = observed
|
|
safe = safe and item_safe
|
|
return result, safe
|
|
if isinstance(value, (list, tuple)):
|
|
if len(value) > _OBSERVATION_MAX_ITEMS:
|
|
return {"type": _qualified(value), "items": len(value), "truncated": True}, False
|
|
result = []
|
|
safe = True
|
|
for item in value:
|
|
observed, item_safe = _observe(item, depth=depth + 1)
|
|
result.append(observed)
|
|
safe = safe and item_safe
|
|
return result, safe
|
|
if isinstance(value, (set, frozenset)):
|
|
if len(value) > _OBSERVATION_MAX_ITEMS:
|
|
return {"type": _qualified(value), "items": len(value), "truncated": True}, False
|
|
# Set repr order varies with PYTHONHASHSEED; keep every item and its type.
|
|
items = [_observe(item, depth=depth + 1) for item in value]
|
|
return {
|
|
"type": _qualified(value),
|
|
"items": sorted((item for item, _ in items), key=_canonical),
|
|
}, all(safe for _, safe in items)
|
|
if isinstance(value, torch.nn.Module):
|
|
# Never traverse every parameter through a wrapper's bound owner.
|
|
return {"type": _qualified(value), "module_state_unobserved": True}, False
|
|
if callable(value):
|
|
if isinstance(value, functools.partial):
|
|
observed, safe = _observe({
|
|
"function": value.func, "args": value.args,
|
|
"keywords": value.keywords, "attributes": vars(value),
|
|
}, depth=depth + 1)
|
|
return {"partial": observed}, safe
|
|
code = getattr(value, "__code__", None)
|
|
descriptor: dict[str, Any] = {"callable": _qualified(value)}
|
|
safe = True
|
|
if code is not None:
|
|
descriptor["code_sha256"] = hashlib.sha256(
|
|
marshal.dumps(code)
|
|
).hexdigest()
|
|
defaults, defaults_safe = _observe(getattr(value, "__defaults__", None), depth=depth + 1)
|
|
kwdefaults, kw_safe = _observe(getattr(value, "__kwdefaults__", None), depth=depth + 1)
|
|
closure_values = tuple(cell.cell_contents for cell in (getattr(value, "__closure__", None) or ()))
|
|
closure, closure_safe = _observe(closure_values, depth=depth + 1)
|
|
descriptor.update(defaults=defaults, kwdefaults=kwdefaults, closure=closure)
|
|
safe = defaults_safe and kw_safe and closure_safe
|
|
if inspect.ismethod(value):
|
|
owner, owner_safe = _observe(value.__self__, depth=depth + 1)
|
|
descriptor["bound_self"] = owner
|
|
safe = safe and owner_safe
|
|
elif hasattr(value, "__dict__"):
|
|
attributes, attributes_safe = _observe(vars(value), depth=depth + 1)
|
|
descriptor["attributes"] = attributes
|
|
# Callable instances need both state and executable implementation.
|
|
# An empty __dict__ is not evidence for an opaque extension wrapper.
|
|
implementation = getattr(type(value), "__call__", None)
|
|
if inspect.isfunction(implementation):
|
|
call, call_safe = _observe(implementation, depth=depth + 1)
|
|
descriptor["implementation"] = call
|
|
slots = any(getattr(cls, "__slots__", ()) for cls in type(value).__mro__)
|
|
safe = attributes_safe and call_safe and bool(vars(value)) and not slots
|
|
else:
|
|
safe = False
|
|
else:
|
|
safe = False
|
|
return descriptor, safe
|
|
if hasattr(value, "__dict__"):
|
|
attributes, safe = _observe(vars(value), depth=depth + 1)
|
|
return {"type": _qualified(value), "attributes": attributes}, safe
|
|
rendered = _ADDRESS.sub("0x...", repr(value))
|
|
return {"type": _qualified(value), "repr": rendered[:256]}, False
|
|
|
|
|
|
# The generic observer remains byte-for-byte compatible with unaffected v6 Runs.
|
|
# Only the Core setter / registered PyTorch and Kitchen override gets this
|
|
# finite descriptor. Never import Core/Kitchen or execute an attention kernel.
|
|
def _attention_code(code):
|
|
constants = tuple(
|
|
_attention_code(item) if isinstance(item, type(code)) else item
|
|
for item in code.co_consts
|
|
)
|
|
return code.replace(co_consts=constants, co_filename="", co_firstlineno=0)
|
|
|
|
|
|
def _attention_class(value):
|
|
import dataclasses
|
|
methods = {}
|
|
safe = True
|
|
for name, method in vars(value).items():
|
|
if isinstance(method, property):
|
|
methods[name] = {"property_unobserved": True}
|
|
safe = False
|
|
continue
|
|
if not inspect.isfunction(method):
|
|
continue
|
|
references = {}
|
|
for cell_name, cell in zip(method.__code__.co_freevars, method.__closure__ or ()):
|
|
target = cell.cell_contents
|
|
origin = (isinstance(target, type) and dataclasses.is_dataclass(value)
|
|
and target is not value and _qualified(target) == _qualified(value)
|
|
and getattr(target, "__dataclass_fields__", None) is value.__dataclass_fields__
|
|
and getattr(target, "__init__", None) is value.__init__)
|
|
if target is value or origin or target is object or (
|
|
isinstance(target, type) and target.__module__ in {"builtins", "dataclasses"}):
|
|
references[cell_name] = target
|
|
descriptor, complete = _attention_function(method, references=references)
|
|
descriptor["reference_targets"] = {
|
|
k: {"type": _qualified(v), "owner": v is value,
|
|
"dataclass_origin": isinstance(v, type) and v is not value and _qualified(v) == _qualified(value)}
|
|
for k, v in references.items()
|
|
}
|
|
methods[name] = descriptor
|
|
safe = safe and complete
|
|
return {"class": _qualified(value), "slots": list(getattr(value, "__slots__", ())),
|
|
"methods": methods}, safe
|
|
|
|
def _attention_globals(function):
|
|
import builtins
|
|
import dis
|
|
import sys
|
|
from types import ModuleType
|
|
names = set()
|
|
def visit(code):
|
|
names.update(i.argval for i in dis.get_instructions(code) if i.opname == "LOAD_GLOBAL")
|
|
for child in code.co_consts:
|
|
if isinstance(child, type(code)):
|
|
visit(child)
|
|
visit(function.__code__)
|
|
result = {}
|
|
safe = True
|
|
for name in sorted(names):
|
|
if name not in function.__globals__:
|
|
if not hasattr(builtins, name):
|
|
safe = False
|
|
continue
|
|
value = function.__globals__[name]
|
|
if isinstance(value, ModuleType):
|
|
if sys.modules.get(value.__name__) is not value or not value.__name__.startswith(
|
|
("torch", "comfy", "math", "functools", "typing")):
|
|
return result, False
|
|
path = getattr(value, "__file__", None)
|
|
result[name] = {"module": value.__name__,
|
|
"file_sha256": _attention_source_file(path) if path else None}
|
|
elif inspect.isbuiltin(value):
|
|
owner = sys.modules.get(value.__module__)
|
|
if (owner is None or getattr(owner, value.__name__, None) is not value or
|
|
not value.__module__.startswith(("torch.", "comfy_kitchen.", "_thread", "math", "builtins"))):
|
|
return result, False
|
|
path = getattr(owner, "__file__", None)
|
|
result[name] = {"builtin": _qualified(value), "python_version": list(sys.version_info[:3]),
|
|
"file_sha256": _attention_source_file(path) if path else None}
|
|
elif inspect.isfunction(value):
|
|
observed, complete = _observe(value)
|
|
if not complete:
|
|
# The only global cycle admitted is another registered Core
|
|
# backend, used by Kitchen's low-precision fallback.
|
|
attention = sys.modules.get("comfy.ldm.modules.attention")
|
|
registry = getattr(attention, "REGISTERED_ATTENTION_FUNCTIONS", {})
|
|
backend = next((n for n in ("pytorch", "comfy_kitchen_int8")
|
|
if registry.get(n) is value), None)
|
|
cells = dict(zip(value.__code__.co_freevars,
|
|
(c.cell_contents for c in value.__closure__ or ())))
|
|
if (backend is None or set(cells) != {"func", "wrapper"} or
|
|
cells["wrapper"] is not value or value.__globals__ is not vars(attention)):
|
|
return result, False
|
|
wrapper, wrapper_safe = _attention_function(value, references=cells)
|
|
raw, raw_safe = _observe(cells["func"])
|
|
observed = {"backend": backend, "wrapper": wrapper, "raw": raw}
|
|
complete = wrapper_safe and raw_safe and value.container_function is None
|
|
attributes, attributes_safe = _observe(vars(value)) if complete and not (
|
|
isinstance(observed, dict) and "backend" in observed) else ({}, complete)
|
|
result[name] = {"function": observed, "attributes": attributes}
|
|
safe = safe and complete and attributes_safe
|
|
elif isinstance(value, type):
|
|
if not value.__module__.startswith(("torch", "comfy")):
|
|
return result, False
|
|
result[name], complete = _attention_class(value)
|
|
safe = safe and complete
|
|
else:
|
|
try:
|
|
result[name] = _attention_provider_state(value)
|
|
except Exception:
|
|
result[name] = {"type": _qualified(value), "unobservable": True}
|
|
safe = False
|
|
return result, safe
|
|
|
|
def _attention_function(function, *, references=None):
|
|
if not inspect.isfunction(function):
|
|
return _observe(function)[0], False
|
|
references = references or {}
|
|
closure = []
|
|
safe = True
|
|
for name, cell in zip(function.__code__.co_freevars, function.__closure__ or ()):
|
|
value = cell.cell_contents
|
|
if name in references and value is references[name]:
|
|
observed, complete = {"reference": name}, True
|
|
else:
|
|
observed, complete = _observe(value)
|
|
closure.append({"name": name, "value": observed})
|
|
safe = safe and complete
|
|
attributes = dict(vars(function))
|
|
for name, value in references.items():
|
|
if name == "func" and attributes.get("__wrapped__") is value:
|
|
attributes.pop("__wrapped__")
|
|
# Both container paths are recorded separately by the adapter.
|
|
attributes.pop("container_function", None)
|
|
defaults, defaults_safe = _observe(function.__defaults__)
|
|
kwdefaults, kw_safe = _observe(function.__kwdefaults__)
|
|
attributes, attributes_safe = _observe(attributes)
|
|
globals_value, globals_safe = _attention_globals(function)
|
|
return {
|
|
"callable": _qualified(function),
|
|
"code_sha256": hashlib.sha256(marshal.dumps(_attention_code(function.__code__))).hexdigest(),
|
|
"defaults": defaults, "kwdefaults": kwdefaults,
|
|
"closure": closure, "attributes": attributes, "globals": globals_value,
|
|
}, safe and defaults_safe and kw_safe and attributes_safe and globals_safe
|
|
|
|
|
|
@functools.lru_cache(maxsize=512)
|
|
def _attention_file_hash(path, size, mtime_ns, ctime_ns):
|
|
return _file_sha256(Path(path))
|
|
|
|
|
|
def _attention_source_file(path):
|
|
path = Path(path)
|
|
stat = path.stat()
|
|
return _attention_file_hash(str(path), stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)
|
|
|
|
|
|
def _attention_provider_state(value, *, depth=0):
|
|
# Provider constraints contain torch dtype / SDP enums, not MODEL tensors.
|
|
if depth > 10:
|
|
raise ValueError("provider state exceeds observation depth")
|
|
import dataclasses
|
|
for name in ("MISSING", "_FIELD_CLASSVAR", "_FIELD_INITVAR", "_HAS_DEFAULT_FACTORY"):
|
|
if value is getattr(dataclasses, name):
|
|
return {"dataclass_sentinel": name}
|
|
if isinstance(value, torch.dtype):
|
|
return {"torch_dtype": str(value)}
|
|
if value is None or isinstance(value, (bool, int, float, str)):
|
|
return value
|
|
if isinstance(value, (dict, tuple, list, set, frozenset)) and len(value) > _OBSERVATION_MAX_ITEMS:
|
|
raise ValueError("provider state exceeds item bound")
|
|
if isinstance(value, (tuple, list)):
|
|
return [_attention_provider_state(v, depth=depth + 1) for v in value]
|
|
if isinstance(value, (set, frozenset)):
|
|
return sorted((_attention_provider_state(v, depth=depth + 1) for v in value), key=_canonical)
|
|
if isinstance(value, dict):
|
|
return sorted([
|
|
[_attention_provider_state(k, depth=depth + 1),
|
|
_attention_provider_state(v, depth=depth + 1)]
|
|
for k, v in value.items()
|
|
], key=_canonical)
|
|
import dataclasses
|
|
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
return {"type": _qualified(value), "fields": {
|
|
field.name: _attention_provider_state(getattr(value, field.name), depth=depth + 1)
|
|
for field in dataclasses.fields(value)
|
|
}}
|
|
sdp = getattr(getattr(torch.nn, "attention", None), "SDPBackend", None)
|
|
if sdp is not None and isinstance(value, sdp):
|
|
return {"torch_sdp_backend": int(value.value)}
|
|
if inspect.isfunction(value):
|
|
result, safe = _attention_function(value)
|
|
if safe:
|
|
return result
|
|
raise ValueError("unsupported attention provider state")
|
|
|
|
|
|
def _attention_providers(attention, backend):
|
|
import sys
|
|
import importlib.metadata
|
|
ops = sys.modules.get("comfy.ops")
|
|
if ops is None:
|
|
raise ValueError("Core attention providers are not loaded")
|
|
functions = {
|
|
"reshape": attention._reshape_qkv_to_heads,
|
|
"heads": attention._heads_from_dim,
|
|
"sdpa": ops.scaled_dot_product_attention,
|
|
"repeat_kv": ops.repeat_kv_for_gqa,
|
|
}
|
|
state = {"batch_limit": attention.SDP_BATCH_LIMIT}
|
|
for name in ("SDPA_BACKEND_PRIORITY", "SDPA_CONTIGUOUS_KV_L2_SIZE"):
|
|
if hasattr(ops, name):
|
|
state[name] = getattr(ops, name)
|
|
container = attention.AttentionTensorContainer
|
|
functions.update({
|
|
"container_init": container.__init__,
|
|
"container_peek": container.peek, "container_take": container.take,
|
|
})
|
|
# The provider's native binding must still be the loaded PyTorch binding.
|
|
if torch.nn.functional.scaled_dot_product_attention is not torch._C._nn.scaled_dot_product_attention:
|
|
raise ValueError("unsupported PyTorch SDPA replacement")
|
|
cuda = torch.backends.cuda
|
|
state["torch_sdpa"] = {
|
|
name: getattr(cuda, name)() for name in
|
|
("flash_sdp_enabled", "mem_efficient_sdp_enabled", "math_sdp_enabled",
|
|
"cudnn_sdp_enabled", "fp16_bf16_reduction_math_sdp_allowed") if hasattr(cuda, name)
|
|
}
|
|
state["torch_matmul"] = {
|
|
"allow_tf32": cuda.matmul.allow_tf32,
|
|
"float32_precision": torch.get_float32_matmul_precision(),
|
|
}
|
|
files = {
|
|
"core_attention": _attention_source_file(attention.__file__),
|
|
"core_ops": _attention_source_file(ops.__file__),
|
|
"torch_functional": _attention_source_file(torch.nn.functional.__file__),
|
|
"torch_native": _attention_source_file(torch._C.__file__),
|
|
}
|
|
versions = {"torch": str(torch.__version__), "torch_git": str(torch.version.git_version)}
|
|
if backend == "comfy_kitchen_int8":
|
|
kitchen = sys.modules.get("comfy_kitchen")
|
|
if kitchen is None or getattr(attention, "comfy_kitchen", None) is not kitchen:
|
|
raise ValueError("Kitchen provider is not loaded")
|
|
functions["int8_inputs"] = attention._comfy_kitchen_int8_inputs
|
|
for name in ("int8_attention", "prequantize_int8_attention", "int8_attention_from_prequantized"):
|
|
functions[name] = getattr(kitchen, name)
|
|
versions["comfy_kitchen"] = importlib.metadata.version("comfy-kitchen")
|
|
root = Path(kitchen.__file__).parent
|
|
paths = sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in {".py", ".pyd", ".so"})
|
|
if len(paths) > _OBSERVATION_MAX_ITEMS:
|
|
raise ValueError("Kitchen provider exceeds file observation bound")
|
|
files["comfy_kitchen"] = {p.relative_to(root).as_posix(): _attention_source_file(p) for p in paths}
|
|
# Capture runtime replacement of Python numerical helpers too.
|
|
module = sys.modules.get("comfy_kitchen.sage_attention")
|
|
if module is None:
|
|
raise ValueError("Kitchen implementation is not loaded")
|
|
functions.update({
|
|
"kitchen_helper." + name: value for name, value in vars(module).items()
|
|
if inspect.isfunction(value) and value.__module__ == module.__name__
|
|
})
|
|
# Kitchen INT8 dispatches directly to its CUDA/HIP extension, not the
|
|
# general op registry. Record only the dispatch actually used here.
|
|
state["kitchen_constants"] = {
|
|
name: value for name, value in vars(module).items()
|
|
if name.isupper() and not name.startswith("__")
|
|
}
|
|
for slot in ("_cuda_backend", "_hip_backend"):
|
|
provider = getattr(module, slot)
|
|
if provider is None:
|
|
state[slot] = None
|
|
continue
|
|
if sys.modules.get(provider.__name__) is not provider:
|
|
raise ValueError("unregistered Kitchen native provider")
|
|
native = getattr(provider, "_C", None)
|
|
if native is not None and sys.modules.get(native.__name__) is not native:
|
|
raise ValueError("unsupported Kitchen native binding")
|
|
state[slot] = {
|
|
"module": provider.__name__,
|
|
"extension_available": getattr(provider, "_EXT_AVAILABLE", None),
|
|
"native_sha256": _attention_source_file(native.__file__) if native is not None else None,
|
|
}
|
|
functions.update({
|
|
slot + "." + name: function for name, function in vars(provider).items()
|
|
if inspect.isfunction(function) and
|
|
(name.startswith(("sage_", "_sage_")) or name in {"_dl", "_wrap_for_dlpack"})
|
|
})
|
|
observed_functions = {}
|
|
for name, function in functions.items():
|
|
owner = sys.modules.get(getattr(function, "__module__", ""))
|
|
if (not inspect.isfunction(function) or owner is None or
|
|
not function.__module__.startswith(("comfy.", "comfy_kitchen.")) or
|
|
function.__globals__ is not vars(owner)):
|
|
raise ValueError("unsupported attention helper binding: " + name)
|
|
descriptor, safe = _attention_function(function)
|
|
if not safe:
|
|
raise ValueError("unobservable attention helper: " + name)
|
|
observed_functions[name] = descriptor
|
|
return {"versions": versions, "files": files, "functions": observed_functions,
|
|
"state": _attention_provider_state(state)}
|
|
|
|
|
|
def _core_attention_override(value):
|
|
import sys
|
|
attention = sys.modules.get("comfy.ldm.modules.attention")
|
|
patcher = sys.modules.get("comfy.model_patcher")
|
|
if attention is None or patcher is None or not inspect.isfunction(value):
|
|
return None
|
|
setter = patcher.ModelPatcher.set_model_optimized_attention
|
|
override_codes = [c for c in setter.__code__.co_consts
|
|
if isinstance(c, type(setter.__code__)) and c.co_name == "optimized_attention_override"]
|
|
wrapper_codes = [c for c in attention.wrap_attn.__code__.co_consts
|
|
if isinstance(c, type(setter.__code__)) and c.co_name == "wrapper"]
|
|
if len(override_codes) != 1 or len(wrapper_codes) != 1 or value.__code__ != override_codes[0]:
|
|
return None
|
|
cells = dict(zip(value.__code__.co_freevars, (c.cell_contents for c in value.__closure__ or ())))
|
|
if set(cells) != {"optimized_attention"}:
|
|
return None
|
|
wrapped = cells["optimized_attention"]
|
|
registry = attention.REGISTERED_ATTENTION_FUNCTIONS
|
|
backend = next((name for name in ("pytorch", "comfy_kitchen_int8")
|
|
if registry.get(name) is wrapped and
|
|
getattr(attention, "attention_" + name, None) is wrapped), None)
|
|
if backend is None or not inspect.isfunction(wrapped) or wrapped.__code__ != wrapper_codes[0]:
|
|
return None
|
|
cells = dict(zip(wrapped.__code__.co_freevars, (c.cell_contents for c in wrapped.__closure__ or ())))
|
|
if set(cells) != {"func", "wrapper"} or cells["wrapper"] is not wrapped or getattr(wrapped, "__wrapped__", None) is not cells["func"]:
|
|
return None
|
|
raw = cells["func"]
|
|
if (value.__globals__ is not vars(patcher) or wrapped.__globals__ is not vars(attention)
|
|
or not inspect.isfunction(raw) or raw.__globals__ is not vars(attention)):
|
|
return None
|
|
expected_container = (getattr(attention, "_attention_comfy_kitchen_int8_containers", None)
|
|
if backend == "comfy_kitchen_int8" else None)
|
|
inner = getattr(wrapped, "container_function", None)
|
|
outer = getattr(value, "container_function", None)
|
|
components = {}
|
|
safe = inner is expected_container and outer is expected_container
|
|
if expected_container is not None:
|
|
safe = safe and inspect.isfunction(expected_container) and expected_container.__globals__ is vars(attention)
|
|
for name, function, refs in (
|
|
("override", value, {"optimized_attention": wrapped}),
|
|
("wrapper", wrapped, {"wrapper": wrapped, "func": raw}),
|
|
("raw", raw, {}),
|
|
("inner_container", inner, {}),
|
|
("outer_container", outer, {}),
|
|
):
|
|
if function is None:
|
|
descriptor, complete = None, True
|
|
else:
|
|
descriptor, complete = _attention_function(function, references=refs)
|
|
components[name] = descriptor
|
|
safe = safe and complete
|
|
result = {"core_attention_signature_version": 1, "backend": backend, **components,
|
|
"outer_container_present": hasattr(value, "container_function")}
|
|
try:
|
|
result["providers"] = _attention_providers(attention, backend)
|
|
except Exception as exc:
|
|
result["observation_error"] = type(exc).__name__
|
|
result["reuse_disabled_reason"] = "Core attention provider implementation/state is unobservable"
|
|
safe = False
|
|
return result, safe
|
|
|
|
|
|
def _observe_model_options(options):
|
|
spectrum = None
|
|
if "spectrum_h3_binding" in options:
|
|
spectrum = spectrum_binding_signature(options["spectrum_h3_binding"])
|
|
options = dict(options)
|
|
options["spectrum_h3_binding"] = None
|
|
observed, safe = _observe_attention_model_options(options)
|
|
if spectrum is not None:
|
|
observed["spectrum_h3_binding"] = spectrum[0]
|
|
safe = safe and spectrum[1]
|
|
return observed, safe
|
|
|
|
|
|
def _observe_attention_model_options(options):
|
|
transformer = options.get("transformer_options")
|
|
if not isinstance(transformer, dict) or "optimized_attention_override" not in transformer:
|
|
return _observe(options)
|
|
try:
|
|
adapted = _core_attention_override(transformer["optimized_attention_override"])
|
|
except Exception:
|
|
adapted = None
|
|
if adapted is None:
|
|
# Unknown overrides keep their prior descriptor and safe-side policy.
|
|
return _observe(options)
|
|
copied = dict(options)
|
|
copied["transformer_options"] = dict(transformer)
|
|
copied["transformer_options"]["optimized_attention_override"] = None
|
|
observed, safe = _observe(copied)
|
|
if not isinstance(observed.get("transformer_options"), dict) or "truncated" in observed["transformer_options"]:
|
|
return observed, False
|
|
observed["transformer_options"]["optimized_attention_override"] = adapted[0]
|
|
return observed, safe and adapted[1]
|
|
|
|
def _model_signature(model: Any, legacy_fingerprint: str) -> tuple[dict[str, Any], bool]:
|
|
base = getattr(model, "model", None)
|
|
inner = getattr(base, "diffusion_model", None)
|
|
wrappers, wrappers_safe = observe_model_wrappers(getattr(model, "wrappers", {}) or {}, _observe)
|
|
patches, patches_safe = _observe(getattr(model, "patches", {}) or {})
|
|
options = getattr(model, "model_options", {}) or {}
|
|
if not isinstance(options, dict):
|
|
return {"model_patcher": _qualified(model), "error": "model_options is not a mapping"}, False
|
|
observed_options = dict(options)
|
|
transformer = dict(observed_options.get("transformer_options", {}) or {})
|
|
transformer.pop("h3_continuum_join_context", None)
|
|
if "transformer_options" in observed_options:
|
|
observed_options["transformer_options"] = transformer
|
|
observed_options, options_safe = _observe_model_options(observed_options)
|
|
weights: list[dict[str, Any]] = []
|
|
weights_safe = True
|
|
try:
|
|
parameters = list(inner.named_parameters()) if inner is not None else []
|
|
for position in sorted({0, len(parameters) // 2, len(parameters) - 1}) if parameters else []:
|
|
name, parameter = parameters[position]
|
|
weights.append({"name": str(name), **_tensor_probe(parameter)})
|
|
except Exception:
|
|
weights_safe = False
|
|
descriptor = {
|
|
"legacy_fingerprint": str(legacy_fingerprint),
|
|
"model_patcher": _qualified(model),
|
|
"base": _qualified(base),
|
|
"inner": _qualified(inner),
|
|
"dtype": str(getattr(model, "model_dtype", lambda: "unknown")()),
|
|
"model_size": int(getattr(model, "model_size", lambda: 0)() or 0),
|
|
"wrappers": wrappers,
|
|
"model_options": observed_options,
|
|
"patches": patches,
|
|
"weight_probe": weights,
|
|
"runtime_observation": {
|
|
"wrappers_complete": bool(wrappers_safe),
|
|
"patches_complete": bool(patches_safe),
|
|
"model_options_complete": bool(options_safe),
|
|
},
|
|
}
|
|
return descriptor, weights_safe and bool(weights) and wrappers_safe and patches_safe and options_safe
|
|
|
|
|
|
def _sampler_signature(sampler: Any) -> tuple[dict[str, Any], bool]:
|
|
function = getattr(sampler, "sampler_function", None)
|
|
observed, safe = _observe({
|
|
"function": function,
|
|
"extra_options": getattr(sampler, "extra_options", {}) or {},
|
|
"inpaint_options": getattr(sampler, "inpaint_options", {}) or {},
|
|
})
|
|
return {
|
|
"type": _qualified(sampler),
|
|
"options": observed,
|
|
}, function is not None and safe
|
|
|
|
|
|
def _signature_observation(observer, *args, **kwargs):
|
|
try:
|
|
return observer(*args, **kwargs)
|
|
except Exception as exc:
|
|
return {"observation_error": type(exc).__name__}, False
|
|
|
|
|
|
def _reusable_sampling_contract(contract: dict[str, Any], *, resume_safe: bool = True) -> bool:
|
|
global_contract = contract.get("global") or {}
|
|
return bool(
|
|
resume_safe
|
|
and global_contract.get("sampling_contract_version") == SAMPLING_CONTRACT_VERSION
|
|
and global_contract.get("runtime_signature_version") == RUNTIME_SIGNATURE_VERSION
|
|
and contract.get("reuse_policy") != "disabled_unobservable_contract"
|
|
)
|
|
|
|
|
|
def _reusable_manifest(manifest: dict[str, Any]) -> bool:
|
|
return bool(
|
|
manifest.get("sampling_contract_version") == SAMPLING_CONTRACT_VERSION
|
|
and manifest.get("resume_safe") is True
|
|
and _reusable_sampling_contract(manifest.get("contract") or {})
|
|
)
|
|
|
|
|
|
_UNKNOWN_REUSE_REASON = "Saved generation conditions could not be fully verified; see Detailed Report"
|
|
_PUBLIC_REUSE_REASONS = PREVIEW_REUSE_REASONS | frozenset((
|
|
"MODEL wrapper/patch contract is not completely observable",
|
|
"SAMPLER function/options are not completely observable",
|
|
"CLIP/Qwen encoder contract is not completely observable",
|
|
"Video VAE contract is not completely observable",
|
|
"Reference Audio VAE contract is not completely observable",
|
|
"Driving Audio VAE contract is not completely observable",
|
|
_UNKNOWN_REUSE_REASON,
|
|
))
|
|
|
|
|
|
def _project_resume_diagnostics(manifest: dict[str, Any]) -> dict[str, Any]:
|
|
"""Expose bounded saved reasons without changing contracts or old manifests."""
|
|
source = manifest.get("resume_disabled_reasons")
|
|
if _reusable_manifest(manifest) or type(source) is not list:
|
|
return {}
|
|
reasons = []
|
|
for value in source[:8]:
|
|
reason = value if type(value) is str and value in _PUBLIC_REUSE_REASONS else _UNKNOWN_REUSE_REASON
|
|
if reason not in reasons:
|
|
reasons.append(reason[:256])
|
|
return {"resume_disabled_reasons": reasons} if reasons else {}
|
|
|
|
|
|
def _module_parameter_dtype(module: Any, name: str, parameter: torch.Tensor) -> torch.dtype:
|
|
"""Observe ordinary weights at Core's declared load dtype without casting them."""
|
|
# Quantized/custom Tensor representations retain their original fingerprint.
|
|
if type(parameter) not in (torch.Tensor, torch.nn.Parameter):
|
|
return parameter.dtype
|
|
owner_name, _, local_name = name.rpartition(".")
|
|
if owner_name and not hasattr(module, "get_submodule"):
|
|
return parameter.dtype
|
|
owner = module.get_submodule(owner_name) if owner_name else module
|
|
dtype = getattr(owner, local_name + "_comfy_model_dtype", None)
|
|
if isinstance(dtype, torch.dtype) and dtype.is_floating_point and parameter.is_floating_point():
|
|
return dtype
|
|
return parameter.dtype
|
|
|
|
|
|
def _module_signature(module: Any) -> tuple[dict[str, Any], bool]:
|
|
descriptor: dict[str, Any] = {"type": _qualified(module)}
|
|
if module is None or not hasattr(module, "named_parameters"):
|
|
descriptor["error"] = "named_parameters unavailable"
|
|
return descriptor, False
|
|
try:
|
|
parameters = list(module.named_parameters())
|
|
dtypes = {
|
|
name: _module_parameter_dtype(module, name, parameter)
|
|
for name, parameter in parameters
|
|
}
|
|
except Exception as exc:
|
|
descriptor["error"] = f"named_parameters failed: {type(exc).__name__}"
|
|
return descriptor, False
|
|
layout = [
|
|
{
|
|
"name": str(name),
|
|
"shape": list(parameter.shape),
|
|
"dtype": str(dtypes[name]),
|
|
}
|
|
for name, parameter in parameters
|
|
]
|
|
descriptor.update(
|
|
tensor_count=len(parameters),
|
|
parameter_count=sum(int(parameter.numel()) for _, parameter in parameters),
|
|
parameter_layout_sha256=_hash(layout),
|
|
)
|
|
probes = []
|
|
probes_safe = bool(parameters)
|
|
for position in sorted({0, len(parameters) // 2, len(parameters) - 1}) if parameters else []:
|
|
name, parameter = parameters[position]
|
|
try:
|
|
probes.append({"name": str(name), **_tensor_probe(parameter, dtype=dtypes[name])})
|
|
except Exception as exc:
|
|
probes.append({"name": str(name), "error": type(exc).__name__})
|
|
probes_safe = False
|
|
descriptor["weight_probe"] = probes
|
|
return descriptor, probes_safe
|
|
|
|
|
|
def _patcher_signature(patcher: Any) -> tuple[dict[str, Any], bool]:
|
|
if patcher is None:
|
|
return {"type": "missing"}, False
|
|
state, safe = _observe({
|
|
"wrappers": getattr(patcher, "wrappers", {}) or {},
|
|
"patches": getattr(patcher, "patches", {}) or {},
|
|
"object_patches": getattr(patcher, "object_patches", {}) or {},
|
|
"model_options": getattr(patcher, "model_options", {}) or {},
|
|
})
|
|
return {
|
|
"type": _qualified(patcher),
|
|
"model_type": _qualified(getattr(patcher, "model", None)),
|
|
"state": state,
|
|
}, safe
|
|
|
|
|
|
def _clip_signature(clip: Any) -> tuple[dict[str, Any], bool]:
|
|
cond_stage = getattr(clip, "cond_stage_model", None)
|
|
tokenizer = getattr(clip, "tokenizer", None)
|
|
tokenizer_wrapper = getattr(tokenizer, "qwen3vl_32b", None)
|
|
tokenizer_core = getattr(tokenizer_wrapper, "tokenizer", None)
|
|
module_value, module_safe = _module_signature(cond_stage)
|
|
patcher_value, patcher_safe = _patcher_signature(getattr(clip, "patcher", None))
|
|
options, options_safe = _observe({
|
|
"tokenizer_options": getattr(clip, "tokenizer_options", {}) or {},
|
|
"layer_idx": getattr(clip, "layer_idx", None),
|
|
"use_clip_schedule": bool(getattr(clip, "use_clip_schedule", False)),
|
|
})
|
|
tokenizer_config = {
|
|
"wrapper_type": _qualified(tokenizer),
|
|
"model_wrapper_type": _qualified(tokenizer_wrapper),
|
|
"core_type": _qualified(tokenizer_core),
|
|
}
|
|
for name in (
|
|
"name_or_path", "vocab_size", "model_max_length", "padding_side",
|
|
"truncation_side", "bos_token_id", "eos_token_id", "pad_token_id",
|
|
):
|
|
value = getattr(tokenizer_core, name, None)
|
|
if value is None or isinstance(value, (bool, int, float, str)):
|
|
tokenizer_config[name] = value
|
|
tokenizer_safe = tokenizer is not None and tokenizer_wrapper is not None and tokenizer_core is not None
|
|
descriptor = {
|
|
"wrapper_type": _qualified(clip),
|
|
"conditioner": module_value,
|
|
"patcher": patcher_value,
|
|
"tokenizer": tokenizer_config,
|
|
"options": options,
|
|
"runtime_observation": {
|
|
"patcher_complete": bool(patcher_safe),
|
|
"options_complete": bool(options_safe),
|
|
},
|
|
}
|
|
return descriptor, module_safe and tokenizer_safe
|
|
|
|
|
|
def _video_vae_signature(video_vae: Any, *, required: bool) -> tuple[dict[str, Any], bool]:
|
|
if not required:
|
|
return {"required": False}, True
|
|
first_stage = getattr(video_vae, "first_stage_model", None)
|
|
module_value, module_safe = _module_signature(first_stage)
|
|
patcher_value, patcher_safe = _patcher_signature(getattr(video_vae, "patcher", None))
|
|
config = {
|
|
"required": True,
|
|
"wrapper_type": _qualified(video_vae),
|
|
"first_stage": module_value,
|
|
"patcher": patcher_value,
|
|
"vae_dtype": str(getattr(video_vae, "vae_dtype", "unknown")),
|
|
"latent_channels": getattr(video_vae, "latent_channels", None),
|
|
"downscale_ratio": getattr(video_vae, "downscale_ratio", None),
|
|
"upscale_ratio": getattr(video_vae, "upscale_ratio", None),
|
|
"chunked_io": bool(getattr(first_stage, "comfy_has_chunked_io", False)),
|
|
}
|
|
observed, config_safe = _observe(config)
|
|
observed["runtime_observation"] = {
|
|
"patcher_complete": bool(patcher_safe),
|
|
"config_complete": bool(config_safe),
|
|
}
|
|
return observed, module_safe
|
|
|
|
|
|
def _audio_vae_signature(audio_vae: Any) -> tuple[dict[str, Any], bool]:
|
|
first_stage = getattr(audio_vae, "first_stage_model", None)
|
|
module_value, module_safe = _module_signature(first_stage)
|
|
patcher_value, patcher_safe = _patcher_signature(getattr(audio_vae, "patcher", None))
|
|
config = {
|
|
"wrapper_type": _qualified(audio_vae),
|
|
"first_stage": module_value,
|
|
"patcher": patcher_value,
|
|
"vae_dtype": str(getattr(audio_vae, "vae_dtype", "unknown")),
|
|
"audio_sample_rate": getattr(audio_vae, "audio_sample_rate", 32000),
|
|
"latent_channels": getattr(audio_vae, "latent_channels", None),
|
|
}
|
|
observed, config_safe = _observe(config)
|
|
observed["runtime_observation"] = {
|
|
"patcher_complete": bool(patcher_safe),
|
|
"config_complete": bool(config_safe),
|
|
}
|
|
return observed, module_safe
|
|
|
|
|
|
def _apply_nonce_contract(
|
|
contract: dict[str, Any], *, requested_nonce: int, effective_nonce: int,
|
|
) -> dict[str, Any]:
|
|
result = dict(contract)
|
|
result["last_frame_hash"] = _canonical_optional_hash(
|
|
result.get("last_frame_hash")
|
|
)
|
|
boundary = int(result["reroll_from_chunk"])
|
|
if boundary <= 0:
|
|
mode = "inactive"
|
|
requested_nonce = 0
|
|
effective_nonce = 0
|
|
else:
|
|
mode = "explicit" if int(requested_nonce) >= 1 else "auto"
|
|
global_hash = _hash(result["global"])
|
|
chunk_contracts = []
|
|
chunk_hashes = []
|
|
routed_groups = result.get("reference_group_contracts")
|
|
routed_by_chunk = {}
|
|
if routed_groups is not None:
|
|
for group_record in routed_groups:
|
|
for chunk in group_record["descriptor"]["logical_chunks"]:
|
|
routed_by_chunk[int(chunk)] = group_record["sha256"]
|
|
for position, prompt_hash in enumerate(result["prompt_hashes"]):
|
|
number = position + 1
|
|
affected = boundary > 0 and number >= boundary
|
|
chunk_contract = {
|
|
"global_hash": global_hash,
|
|
"chunk_number": number,
|
|
"prompt_hash": str(prompt_hash),
|
|
"reroll_boundary": boundary if affected else 0,
|
|
"effective_reroll_nonce": int(effective_nonce) if affected else 0,
|
|
"last_frame_hash": str(result["last_frame_hash"]) if number == int(result["chunk_count"]) else "",
|
|
}
|
|
if routed_groups is not None:
|
|
chunk_contract["physical_group_reference_sha256"] = routed_by_chunk[number]
|
|
timeline_chunks = result.get("timeline_video_chunk_contracts") or []
|
|
if position < len(timeline_chunks):
|
|
chunk_contract["timeline_video"] = dict(timeline_chunks[position])
|
|
chunk_contracts.append(chunk_contract)
|
|
chunk_hashes.append(_hash(chunk_contract))
|
|
result.update(
|
|
effective_reroll_nonce=int(effective_nonce),
|
|
chunk_contracts=chunk_contracts,
|
|
chunk_contract_hashes=chunk_hashes,
|
|
nonce_lifecycle={
|
|
"mode": mode,
|
|
"requested_nonce": int(requested_nonce),
|
|
"effective_nonce": int(effective_nonce),
|
|
"lineage_sha256": str(result["nonce_lineage_sha256"]),
|
|
"request_sha256": str(result["nonce_request_sha256"]),
|
|
},
|
|
)
|
|
return result
|
|
|
|
|
|
def _nonce_lineage_hash(
|
|
contract: dict[str, Any],
|
|
*,
|
|
timeline_video_chunk_contracts: list[dict[str, Any]] | None = None,
|
|
) -> str:
|
|
"""Recompute the nonce-independent compatible sampling lineage."""
|
|
|
|
identity = {
|
|
"global_hash": _hash(contract["global"]),
|
|
"chunk_count": int(contract["chunk_count"]),
|
|
"prompt_mode": str(contract["prompt_mode"]),
|
|
"prompt_hashes": [str(value) for value in contract["prompt_hashes"]],
|
|
"last_frame_hash": str(contract.get("last_frame_hash", "")),
|
|
"timeline_video_chunk_contracts": list(
|
|
timeline_video_chunk_contracts
|
|
if timeline_video_chunk_contracts is not None
|
|
else contract.get("timeline_video_chunk_contracts") or []
|
|
),
|
|
}
|
|
if "reference_group_contracts" in contract:
|
|
identity["reference_group_hashes"] = [
|
|
record["sha256"] for record in contract["reference_group_contracts"]
|
|
]
|
|
return _hash(identity)
|
|
|
|
|
|
def _apply_reroll_branch_contract(
|
|
contract: dict[str, Any],
|
|
*,
|
|
boundary: int,
|
|
requested_nonce: int,
|
|
effective_nonce: int,
|
|
) -> dict[str, Any]:
|
|
"""Apply the existing reroll/seed contract to a resolved review branch."""
|
|
|
|
result = dict(contract)
|
|
result["reroll_from_chunk"] = int(boundary)
|
|
result["nonce_request_sha256"] = _hash({
|
|
"lineage_sha256": str(result["nonce_lineage_sha256"]),
|
|
"reroll_from_chunk": int(boundary),
|
|
})
|
|
return _apply_nonce_contract(
|
|
result,
|
|
requested_nonce=int(requested_nonce),
|
|
effective_nonce=int(effective_nonce),
|
|
)
|
|
|
|
|
|
def _legacy_absent_last_frame_compatible(
|
|
*,
|
|
stored_contract: dict[str, Any],
|
|
current_contract: dict[str, Any],
|
|
position: int,
|
|
stored_hash: str,
|
|
current_hash: str,
|
|
) -> bool:
|
|
"""Accept only the V3.6.1 ``none`` versus ``""`` Last Frame mismatch."""
|
|
|
|
if not isinstance(stored_contract, dict) or not isinstance(current_contract, dict):
|
|
return False
|
|
if _canonical_optional_hash(stored_contract.get("last_frame_hash")):
|
|
return False
|
|
if _canonical_optional_hash(current_contract.get("last_frame_hash")):
|
|
return False
|
|
stored_chunks = stored_contract.get("chunk_contracts") or []
|
|
current_chunks = current_contract.get("chunk_contracts") or []
|
|
if position >= len(stored_chunks) or position >= len(current_chunks):
|
|
return False
|
|
stored_chunk = stored_chunks[position]
|
|
current_chunk = current_chunks[position]
|
|
if not isinstance(stored_chunk, dict) or not isinstance(current_chunk, dict):
|
|
return False
|
|
if _hash(stored_chunk) != str(stored_hash):
|
|
return False
|
|
if _hash(current_chunk) != str(current_hash):
|
|
return False
|
|
stored_normalized = dict(stored_chunk)
|
|
current_normalized = dict(current_chunk)
|
|
stored_normalized["last_frame_hash"] = _canonical_optional_hash(
|
|
stored_normalized.get("last_frame_hash")
|
|
)
|
|
current_normalized["last_frame_hash"] = _canonical_optional_hash(
|
|
current_normalized.get("last_frame_hash")
|
|
)
|
|
return stored_normalized == current_normalized
|
|
|
|
|
|
def build_sampling_contract(
|
|
*, model: Any, model_fingerprint_value: str, clip: Any, video_vae: Any,
|
|
sampler: Any,
|
|
sigmas: torch.Tensor, prompt_plan: dict[str, Any], width: int,
|
|
height: int, chunk_seconds: float, continuity: str,
|
|
audio_continuity: bool, base_seed: int, reroll_from_chunk: int,
|
|
reroll_nonce: int, first_frame_hash: str, last_frame_hash: str,
|
|
strict_compatibility: bool, reference_contract: dict[str, Any] | None = None,
|
|
conditioning_mode: str | None = None,
|
|
upstream_graph_contract: dict[str, Any] | None = None,
|
|
upstream_graph_safe: bool | None = None,
|
|
upstream_graph_reasons: list[str] | None = None,
|
|
reference_audio_contract: dict[str, Any] | None = None,
|
|
reference_audio_vae: Any = None,
|
|
driving_audio_contract: dict[str, Any] | None = None,
|
|
driving_audio_vae: Any = None,
|
|
reference_video_contract: dict[str, Any] | None = None,
|
|
timeline_video_contract: dict[str, Any] | None = None,
|
|
guide_contract: dict[str, Any] | None = None,
|
|
execution_semantics: dict[str, Any] | None = None,
|
|
reference_storage_plan: ReferenceStoragePlan | None = None,
|
|
) -> tuple[dict[str, Any], bool, list[str]]:
|
|
if reference_storage_plan is not None and reference_contract is not None:
|
|
raise RunStorageError("routed Reference plan cannot use legacy Reference contract")
|
|
last_frame_hash = _canonical_optional_hash(last_frame_hash)
|
|
model_value, model_safe = _signature_observation(_model_signature, model, model_fingerprint_value)
|
|
preview_reasons = preview_override_reuse_reasons(model_value.get("wrappers"))
|
|
clip_value, clip_safe = _clip_signature(clip)
|
|
has_first = str(first_frame_hash).lower() not in {"", "none", "null"}
|
|
has_last = str(last_frame_hash).lower() not in {"", "none", "null"}
|
|
inferred_mode = conditioning_mode_from_presence(
|
|
has_first=has_first,
|
|
has_last=has_last,
|
|
has_reference=reference_contract is not None,
|
|
)
|
|
conditioning_mode = inferred_mode if conditioning_mode is None else str(conditioning_mode)
|
|
if conditioning_mode not in CONDITIONING_MODES:
|
|
raise RunStorageError(f"unknown conditioning mode: {conditioning_mode!r}")
|
|
if conditioning_mode != inferred_mode:
|
|
raise RunStorageError(
|
|
"conditioning mode does not match the connected image inputs: "
|
|
f"declared={conditioning_mode}, inferred={inferred_mode}"
|
|
)
|
|
uses_video_vae = (
|
|
reference_storage_plan is not None
|
|
or
|
|
conditioning_mode_uses_video_vae(conditioning_mode)
|
|
or reference_video_contract is not None
|
|
or timeline_video_contract is not None
|
|
or guide_contract is not None
|
|
)
|
|
if uses_video_vae:
|
|
video_vae_value, video_vae_safe = _video_vae_signature(
|
|
video_vae, required=True
|
|
)
|
|
else:
|
|
video_vae_value, video_vae_safe = None, True
|
|
sampler_value, sampler_safe = _signature_observation(_sampler_signature, sampler)
|
|
if reference_audio_contract is not None:
|
|
audio_vae_value, audio_vae_safe = _audio_vae_signature(reference_audio_vae)
|
|
else:
|
|
audio_vae_value, audio_vae_safe = None, True
|
|
if driving_audio_contract is not None:
|
|
driving_audio_vae_value, driving_audio_vae_safe = _audio_vae_signature(
|
|
driving_audio_vae
|
|
)
|
|
else:
|
|
driving_audio_vae_value, driving_audio_vae_safe = None, True
|
|
if not torch.is_tensor(sigmas) or sigmas.ndim != 1:
|
|
raise RunStorageError("Run Storage requires a one-dimensional SIGMAS tensor")
|
|
chunks = int(prompt_plan["chunks"])
|
|
prompt_hashes = [str(value) for value in prompt_plan["hashes"]]
|
|
boundary = int(reroll_from_chunk)
|
|
graph_authoritative = upstream_graph_contract is not None
|
|
if graph_authoritative:
|
|
routes = upstream_graph_contract.get("routes", {})
|
|
model_value = {
|
|
"runtime": model_value,
|
|
"graph_route": routes.get("model"),
|
|
}
|
|
clip_value = {
|
|
"runtime": clip_value,
|
|
"graph_route": routes.get("clip"),
|
|
}
|
|
if uses_video_vae:
|
|
video_vae_value = {
|
|
"runtime": video_vae_value,
|
|
"graph_route": routes.get("video_vae"),
|
|
}
|
|
if reference_audio_contract is not None:
|
|
audio_vae_value = {
|
|
"runtime": audio_vae_value,
|
|
"graph_route": routes.get("reference_audio_vae"),
|
|
}
|
|
if driving_audio_contract is not None:
|
|
driving_audio_vae_value = {
|
|
"runtime": driving_audio_vae_value,
|
|
"graph_route": routes.get("audio_vae"),
|
|
}
|
|
global_contract = {
|
|
"sampling_contract_version": SAMPLING_CONTRACT_VERSION,
|
|
"runtime_signature_version": RUNTIME_SIGNATURE_VERSION,
|
|
"conditioning_mode": conditioning_mode,
|
|
"model": model_value,
|
|
"clip": clip_value,
|
|
"sampler": sampler_value,
|
|
"sigmas": _tensor_exact(sigmas),
|
|
"width": int(width), "height": int(height),
|
|
"chunk_seconds": float(chunk_seconds),
|
|
"continuity": continuity_storage_mode(continuity),
|
|
"audio_continuity": bool(audio_continuity),
|
|
"base_seed": int(base_seed),
|
|
"strict_compatibility": bool(strict_compatibility),
|
|
"continuum_interop_api": 1,
|
|
"actual_prefix_steps": CONTINUUM_ACTUAL_PREFIX_STEPS,
|
|
}
|
|
if graph_authoritative:
|
|
global_contract["upstream_graph"] = dict(upstream_graph_contract)
|
|
if execution_semantics is not None:
|
|
semantics = dict(execution_semantics)
|
|
if reference_storage_plan is not None:
|
|
semantics.pop("terminal_prompt_policy", None)
|
|
global_contract["execution_semantics"] = semantics
|
|
if reference_storage_plan is not None:
|
|
global_contract["reference_routing_contract_version"] = REFERENCE_ROUTING_CONTRACT_VERSION
|
|
if uses_video_vae:
|
|
global_contract["video_vae"] = video_vae_value
|
|
if has_first:
|
|
global_contract["first_frame_hash"] = str(first_frame_hash)
|
|
if reference_contract is not None:
|
|
global_contract["reference"] = dict(reference_contract)
|
|
if has_first or has_last:
|
|
from .reference import HYBRID_PRESENTATION_VERSION
|
|
|
|
picture_order: list[str] = []
|
|
if has_first:
|
|
picture_order.append("first_frame")
|
|
if has_last:
|
|
picture_order.append("last_frame")
|
|
picture_order.extend(
|
|
f"reference_image_{index}"
|
|
for index in range(1, int(reference_contract.get("count", 0)) + 1)
|
|
)
|
|
global_contract["hybrid_qwen_presentation"] = {
|
|
"version": HYBRID_PRESENTATION_VERSION,
|
|
"picture_order": picture_order,
|
|
}
|
|
if reference_audio_contract is not None:
|
|
global_contract["reference_audio"] = dict(reference_audio_contract)
|
|
global_contract["reference_audio_vae"] = audio_vae_value
|
|
if driving_audio_contract is not None:
|
|
global_contract["driving_audio"] = dict(driving_audio_contract)
|
|
global_contract["driving_audio_vae"] = driving_audio_vae_value
|
|
if reference_video_contract is not None:
|
|
global_contract["reference_video"] = dict(reference_video_contract)
|
|
if guide_contract is not None:
|
|
global_contract["guide"] = dict(guide_contract)
|
|
timeline_chunk_contracts = []
|
|
if timeline_video_contract is not None:
|
|
timeline_value = dict(timeline_video_contract)
|
|
timeline_chunk_contracts = list(timeline_value.pop("chunk_slices", []))
|
|
if len(timeline_chunk_contracts) != chunks:
|
|
raise RunStorageError(
|
|
"Timeline Video contract does not match the configured chunk count"
|
|
)
|
|
global_contract["timeline_video"] = timeline_value
|
|
contract = {
|
|
"global": global_contract,
|
|
"chunk_count": chunks,
|
|
"prompt_mode": str(prompt_plan["mode"]),
|
|
"prompt_hashes": prompt_hashes,
|
|
"reroll_from_chunk": boundary,
|
|
"last_frame_hash": str(last_frame_hash),
|
|
}
|
|
if reference_storage_plan is not None:
|
|
routed_groups = reference_storage_plan.group_contracts
|
|
validate_reference_group_contracts(
|
|
routed_groups, chunk_count=chunks,
|
|
terminal_merge_enabled=(
|
|
(execution_semantics or {}).get("flf_execution") == "terminal_merged_10s_seed_v2"
|
|
),
|
|
)
|
|
contract["reference_group_contracts"] = routed_groups
|
|
lineage_sha256 = _nonce_lineage_hash(
|
|
contract,
|
|
timeline_video_chunk_contracts=timeline_chunk_contracts,
|
|
)
|
|
contract["nonce_lineage_sha256"] = lineage_sha256
|
|
contract = _apply_reroll_branch_contract(
|
|
contract,
|
|
boundary=boundary,
|
|
requested_nonce=int(reroll_nonce),
|
|
effective_nonce=int(reroll_nonce) if boundary > 0 else 0,
|
|
)
|
|
reasons = list(upstream_graph_reasons or []) if graph_authoritative else []
|
|
if not model_safe:
|
|
reasons.append("MODEL wrapper/patch contract is not completely observable")
|
|
reasons.extend(preview_reasons)
|
|
if not sampler_safe:
|
|
reasons.append("SAMPLER function/options are not completely observable")
|
|
if not clip_safe:
|
|
reasons.append("CLIP/Qwen encoder contract is not completely observable")
|
|
if not video_vae_safe:
|
|
reasons.append("Video VAE contract is not completely observable")
|
|
if not audio_vae_safe:
|
|
reasons.append("Reference Audio VAE contract is not completely observable")
|
|
if not driving_audio_vae_safe:
|
|
reasons.append("Driving Audio VAE contract is not completely observable")
|
|
graph_safe = bool(upstream_graph_safe) if graph_authoritative else True
|
|
return contract, model_safe and clip_safe and video_vae_safe and audio_vae_safe and driving_audio_vae_safe and sampler_safe and graph_safe, reasons
|
|
|
|
|
|
def revision_identity(contract: dict[str, Any]) -> tuple[str, str]:
|
|
full = _hash(contract)
|
|
return full[:16], full
|
|
|
|
|
|
def _pid_exists_windows(pid: int) -> bool:
|
|
"""Probe a Windows PID without sending a console event or terminating it."""
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
|
|
process_query_limited_information = 0x1000
|
|
error_access_denied = 5
|
|
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
kernel32.OpenProcess.argtypes = (
|
|
wintypes.DWORD,
|
|
wintypes.BOOL,
|
|
wintypes.DWORD,
|
|
)
|
|
kernel32.OpenProcess.restype = wintypes.HANDLE
|
|
kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
|
|
kernel32.CloseHandle.restype = wintypes.BOOL
|
|
ctypes.set_last_error(0)
|
|
handle = kernel32.OpenProcess(
|
|
process_query_limited_information,
|
|
False,
|
|
int(pid),
|
|
)
|
|
if handle:
|
|
kernel32.CloseHandle(handle)
|
|
return True
|
|
return ctypes.get_last_error() == error_access_denied
|
|
|
|
|
|
def _pid_exists(pid: int) -> bool:
|
|
if pid <= 0:
|
|
return False
|
|
if pid == os.getpid():
|
|
return True
|
|
if os.name == "nt":
|
|
return _pid_exists_windows(pid)
|
|
try:
|
|
os.kill(pid, 0)
|
|
except ProcessLookupError:
|
|
return False
|
|
except PermissionError:
|
|
return True
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
class _RunLock:
|
|
def __init__(self, path: Path):
|
|
self.path = path
|
|
self.token = uuid.uuid4().hex
|
|
self.acquired = False
|
|
|
|
def acquire(self) -> None:
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
payload = {
|
|
"pid": os.getpid(), "hostname": socket.gethostname(),
|
|
"started_utc": _now(), "token": self.token,
|
|
}
|
|
encoded = (_canonical(payload) + "\n").encode("utf-8")
|
|
for _ in range(2):
|
|
try:
|
|
descriptor = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
except FileExistsError:
|
|
try:
|
|
existing = _read_json(self.path)
|
|
except Exception as exc:
|
|
raise RunStorageError(f"Run Storage lock exists and is unreadable: {self.path}") from exc
|
|
if str(existing.get("hostname", "")) == socket.gethostname() and not _pid_exists(int(existing.get("pid", -1))):
|
|
self.path.unlink()
|
|
continue
|
|
raise RunStorageError(
|
|
f"Run Name is locked by pid={existing.get('pid')} host={existing.get('hostname')}"
|
|
)
|
|
else:
|
|
try:
|
|
os.write(descriptor, encoded)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
self.acquired = True
|
|
return
|
|
raise RunStorageError("failed to recover stale Run Storage lock")
|
|
|
|
def release(self) -> None:
|
|
if not self.acquired:
|
|
return
|
|
try:
|
|
existing = _read_json(self.path)
|
|
if existing.get("token") == self.token:
|
|
self.path.unlink()
|
|
finally:
|
|
self.acquired = False
|
|
|
|
|
|
def _entry_metadata(entry: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"sequence_index": int(entry["sequence_index"]),
|
|
"clip_index": int(entry["clip_index"]),
|
|
"prompt_hash": str(entry["prompt_hash"]),
|
|
"seed": int(entry["seed"]),
|
|
"context_frames": int(entry["context_frames"]),
|
|
"motion_score": float(entry["motion_score"]),
|
|
"plan": dict(entry["plan"]),
|
|
}
|
|
|
|
|
|
def _resume_session_settings(
|
|
*,
|
|
contract: dict[str, Any],
|
|
revision_id: str,
|
|
first_frame_hash: str,
|
|
last_frame_hash: str,
|
|
reference_storage_plan: ReferenceStoragePlan | None = None,
|
|
accepted_chunks: int = 0,
|
|
) -> dict[str, Any]:
|
|
settings = {
|
|
"run_storage_validated_prefix": True,
|
|
"revision_id": str(revision_id),
|
|
"first_frame_hash": str(first_frame_hash),
|
|
"last_frame_hash": str(last_frame_hash),
|
|
}
|
|
settings.update(
|
|
dict((contract.get("global") or {}).get("execution_semantics") or {})
|
|
)
|
|
if reference_storage_plan is not None:
|
|
settings["reference_routing_v1"] = session_routing_settings(
|
|
reference_storage_plan, accepted_chunks,
|
|
)
|
|
return settings
|
|
|
|
|
|
def _terminal_merge_enabled(contract: dict[str, Any]) -> bool:
|
|
semantics = (contract.get("global") or {}).get("execution_semantics") or {}
|
|
return semantics.get("flf_execution") == "terminal_merged_10s_seed_v2"
|
|
|
|
|
|
def _manifest_sampling_identity(manifest: dict[str, Any]) -> tuple[str, str]:
|
|
contract = manifest.get("contract")
|
|
if not isinstance(contract, dict):
|
|
raise RunStorageError("stored sampling contract is missing")
|
|
routed_version = (contract.get("global") or {}).get("reference_routing_contract_version")
|
|
routed_groups = contract.get("reference_group_contracts")
|
|
if routed_version is not None or routed_groups is not None:
|
|
if routed_version != REFERENCE_ROUTING_CONTRACT_VERSION:
|
|
raise RunStorageError("stored Reference routing contract version is incompatible")
|
|
try:
|
|
groups = validate_reference_group_contracts(
|
|
routed_groups, chunk_count=int(contract["chunk_count"]),
|
|
terminal_merge_enabled=_terminal_merge_enabled(contract),
|
|
)
|
|
except (ReferenceStorageContractError, KeyError, TypeError, ValueError) as exc:
|
|
raise RunStorageError(f"stored Reference group contract is invalid: {exc}") from exc
|
|
expected = {
|
|
int(chunk): record["sha256"]
|
|
for record in groups for chunk in record["descriptor"]["logical_chunks"]
|
|
}
|
|
chunks = contract.get("chunk_contracts")
|
|
if (not isinstance(chunks, list) or len(chunks) != len(expected)
|
|
or any(not isinstance(item, dict) or
|
|
item.get("physical_group_reference_sha256") != expected[index + 1]
|
|
for index, item in enumerate(chunks))):
|
|
raise RunStorageError("stored Reference group/chunk contracts disagree")
|
|
contract_revision_id, contract_sha256 = revision_identity(contract)
|
|
if str(manifest.get("contract_sha256", "")) != contract_sha256:
|
|
raise RunStorageError("stored sampling contract SHA-256 is invalid")
|
|
schema = int(manifest.get("run_storage_schema_version", -1))
|
|
if schema not in RUN_STORAGE_READABLE_SCHEMA_VERSIONS:
|
|
raise RunStorageError("stored Run Storage schema is incompatible")
|
|
stored_sampling_id = str(
|
|
manifest.get("sampling_revision_id", manifest.get("revision_id", ""))
|
|
)
|
|
if stored_sampling_id != contract_revision_id:
|
|
raise RunStorageError("stored sampling revision identity is invalid")
|
|
if schema == 2 and str(manifest.get("revision_id", "")) != contract_revision_id:
|
|
raise RunStorageError("legacy storage revision identity is invalid")
|
|
return contract_revision_id, contract_sha256
|
|
|
|
|
|
def _storage_revision_identity(
|
|
*,
|
|
contract_sha256: str,
|
|
take_action: str,
|
|
selected_revision_id: str,
|
|
effective_nonce: int,
|
|
branch_boundary: int,
|
|
) -> str:
|
|
return _hash(
|
|
{
|
|
"run_storage_schema_version": RUN_STORAGE_SCHEMA_VERSION,
|
|
"contract_sha256": str(contract_sha256),
|
|
"take_action": str(take_action),
|
|
"selected_revision_id": str(selected_revision_id),
|
|
"effective_nonce": int(effective_nonce),
|
|
"branch_boundary": int(branch_boundary),
|
|
}
|
|
)[:16]
|
|
|
|
|
|
class RunStorageController:
|
|
def __init__(self, run_name: str):
|
|
self.run_name = sanitize_run_name(run_name)
|
|
self.run_root = _output_root() / "h3_continuum" / "runs" / self.run_name
|
|
self.revisions_root = self.run_root / "revisions"
|
|
self.lock = _RunLock(self.run_root / ".lock")
|
|
self.context_token = None
|
|
self.contract: dict[str, Any] | None = None
|
|
self.contract_sha256 = ""
|
|
self.revision_id = ""
|
|
self.revision_root: Path | None = None
|
|
self.manifest: dict[str, Any] | None = None
|
|
self.prompts: list[str] = []
|
|
self.reused_count = 0
|
|
self.generated_count = 0
|
|
self.disabled_reasons: list[str] = []
|
|
self.notes: list[str] = []
|
|
self.resume_safe = False
|
|
self.effective_reroll_nonce = 0
|
|
self.nonce_decision = "inactive"
|
|
self.prompt_graph: dict[str, Any] | None = None
|
|
self.sampler_node_id: str | None = None
|
|
self.review_generation_mode: str | None = None
|
|
self.review_action: str | None = None
|
|
self.review_manual_regenerate_from: int = 0
|
|
self.review_execution = None
|
|
self.review_head: dict[str, Any] | None = None
|
|
self.validated_prefix: ValidatedPrefix | EmptyValidatedPrefix | None = None
|
|
self.review_runtime_metrics = {
|
|
"review_decision_created": 0,
|
|
"review_reconcile": 0,
|
|
}
|
|
self.inherited_review_unit: dict[str, int] | None = None
|
|
self.pending_review_pause_metadata: dict[str, Any] | None = None
|
|
self.take_action = TAKE_ACTION_AUTOMATIC
|
|
self.take_group = 0
|
|
self.take_revision_id = ""
|
|
self.selected_take_chain: list[dict[str, Any]] = []
|
|
self.selected_take_records: list[dict[str, Any]] = []
|
|
self.selected_take_revision: dict[str, Any] | None = None
|
|
self.pending_branch_cut: dict[str, Any] | None = None
|
|
self.storage_revision_id_override = ""
|
|
# Lock-scoped metadata only. Never cache tensors or skip raw integrity checks.
|
|
self._manifest_snapshot = None
|
|
self._plan_import = None
|
|
self.io_metrics = {"manifest_scans": 0, "manifest_reads": 0,
|
|
"raw_loads": 0, "raw_bytes": 0,
|
|
"manifest_seconds": 0.0, "raw_seconds": 0.0}
|
|
|
|
def set_prompt_graph(self, prompt: Any, unique_id: Any) -> None:
|
|
self.prompt_graph = prompt if isinstance(prompt, dict) else None
|
|
self.sampler_node_id = None if unique_id is None else str(unique_id)
|
|
|
|
def configure_review(
|
|
self,
|
|
*,
|
|
generation_mode: str,
|
|
review_action: str,
|
|
manual_regenerate_from: int = 0,
|
|
take_action: str = TAKE_ACTION_AUTOMATIC,
|
|
take_group: int = 0,
|
|
take_revision_id: str = "",
|
|
) -> None:
|
|
"""Attach a non-identity review intent before Production preparation."""
|
|
|
|
self.review_generation_mode = str(generation_mode)
|
|
self.review_action = str(review_action)
|
|
self.review_manual_regenerate_from = int(manual_regenerate_from)
|
|
if str(take_action) not in TAKE_ACTION_OPTIONS:
|
|
raise RunStorageError(f"unknown Take action: {take_action!r}")
|
|
self.take_action = str(take_action)
|
|
self.take_group = int(take_group)
|
|
self.take_revision_id = str(take_revision_id).strip()
|
|
if self.take_action != TAKE_ACTION_AUTOMATIC and (
|
|
self.take_group < 1 or not self.take_revision_id
|
|
):
|
|
raise RunStorageError(
|
|
"Use This Take / Continue From Here requires a group and revision ID"
|
|
)
|
|
|
|
def __enter__(self) -> "RunStorageController":
|
|
self.lock.acquire()
|
|
self._manifest_snapshot = None
|
|
self.context_token = _ACTIVE.set(self)
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, traceback) -> None:
|
|
try:
|
|
if exc is not None and self.manifest is not None:
|
|
self.manifest.pop("review_unit", None)
|
|
self.manifest.pop("review_pause_reason", None)
|
|
self.manifest.update(status="interrupted", updated_utc=_now(), last_error=f"{type(exc).__name__}: {exc}"[:2048])
|
|
self._write_manifest()
|
|
self._write_project()
|
|
finally:
|
|
if self.context_token is not None:
|
|
_ACTIVE.reset(self.context_token)
|
|
self.context_token = None
|
|
self._manifest_snapshot = None
|
|
self._plan_import = None
|
|
self.lock.release()
|
|
|
|
def _manifest_path(self) -> Path:
|
|
if self.revision_root is None:
|
|
raise RunStorageError("Run Storage revision is not prepared")
|
|
return self.revision_root / "manifest.json"
|
|
|
|
def _write_manifest(self) -> None:
|
|
if self.manifest is not None:
|
|
_write_json(self._manifest_path(), self.manifest)
|
|
self._manifest_snapshot = None
|
|
|
|
def _read_manifests(self) -> dict[str, dict[str, Any]]:
|
|
if self.context_token is not None and self._manifest_snapshot is not None:
|
|
return self._manifest_snapshot
|
|
started = time.perf_counter()
|
|
manifests: dict[str, dict[str, Any]] = {}
|
|
self.io_metrics["manifest_scans"] += 1
|
|
for path in sorted(self.revisions_root.glob("*/manifest.json")):
|
|
try:
|
|
self.io_metrics["manifest_reads"] += 1
|
|
value = _read_json(path)
|
|
_manifest_sampling_identity(value)
|
|
revision_id = str(value.get("revision_id", ""))
|
|
if not revision_id or revision_id != path.parent.name:
|
|
raise RunStorageError("storage revision path identity is invalid")
|
|
manifests[revision_id] = value
|
|
except Exception as exc:
|
|
self.notes.append(f"provenance manifest {path.parent.name} rejected: {exc}")
|
|
self.io_metrics["manifest_seconds"] += time.perf_counter() - started
|
|
if self.context_token is not None:
|
|
self._manifest_snapshot = manifests
|
|
return manifests
|
|
|
|
def _manifest_provenance_chain(
|
|
self,
|
|
manifest: dict[str, Any],
|
|
manifests: dict[str, dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
contract = manifest.get("contract") or {}
|
|
records = list(manifest.get("chunks") or [])
|
|
groups = physical_groups(
|
|
chunks=int(contract.get("chunk_count", 0)),
|
|
terminal_merge_enabled=_terminal_merge_enabled(contract),
|
|
)
|
|
chain: list[dict[str, Any]] = []
|
|
parent_revision_id: str | None = None
|
|
for group in groups:
|
|
if group.end > len(records):
|
|
break
|
|
group_records = records[group.start - 1 : group.end]
|
|
sources = {str(record.get("storage_revision_id", "")) for record in group_records}
|
|
if len(sources) != 1 or "" in sources:
|
|
raise RunStorageError("physical group has ambiguous storage sources")
|
|
source_id = next(iter(sources))
|
|
source_manifest = manifests.get(source_id)
|
|
if source_manifest is None:
|
|
raise RunStorageError(
|
|
f"physical group source manifest is missing: {source_id}"
|
|
)
|
|
_, source_contract_sha256 = _manifest_sampling_identity(source_manifest)
|
|
source_contract = source_manifest.get("contract") or {}
|
|
source_lineage = str(source_contract.get("nonce_lineage_sha256", ""))
|
|
if source_lineage != str(contract.get("nonce_lineage_sha256", "")):
|
|
raise RunStorageError("physical group source crosses sampling lineages")
|
|
source_boundary = int(source_contract.get("reroll_from_chunk", 0))
|
|
source_nonce = int(source_contract.get("effective_reroll_nonce", 0))
|
|
variation_nonce = (
|
|
source_nonce if source_boundary > 0 and group.start >= source_boundary else 0
|
|
)
|
|
branch_cut = None
|
|
if source_nonce > 0 and source_boundary == group.start:
|
|
branch_cut = {
|
|
"selected_revision_id": parent_revision_id or "",
|
|
"after_physical_group": group.start - 1,
|
|
}
|
|
revision = make_group_revision(
|
|
parent_revision_id=parent_revision_id,
|
|
group=group,
|
|
variation_nonce=variation_nonce,
|
|
created_utc=str(
|
|
source_manifest.get("created_utc")
|
|
or source_manifest.get("updated_utc")
|
|
or ""
|
|
),
|
|
storage_revision_id=source_id,
|
|
generation_contract_sha256=source_contract_sha256,
|
|
lineage_sha256=source_lineage,
|
|
records=group_records,
|
|
branch_cut=branch_cut,
|
|
)
|
|
chain.append(revision)
|
|
parent_revision_id = revision["revision_id"]
|
|
persisted = manifest.get("branch_provenance")
|
|
if persisted is not None:
|
|
if not isinstance(persisted, dict):
|
|
raise RunStorageError("branch provenance metadata must be an object")
|
|
if int(persisted.get("version", -1)) != BRANCH_PROVENANCE_VERSION:
|
|
raise RunStorageError("branch provenance version is unsupported")
|
|
expected = [revision["revision_id"] for revision in chain]
|
|
if list(persisted.get("active_chain") or []) != expected:
|
|
raise RunStorageError("persisted active chain does not match stored records")
|
|
return chain
|
|
|
|
def _provenance_catalog(
|
|
self,
|
|
*,
|
|
lineage_sha256: str | None = None,
|
|
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]], dict[str, list[str]]]:
|
|
manifests = self._read_manifests()
|
|
catalog: dict[str, dict[str, Any]] = {}
|
|
chains: dict[str, list[str]] = {}
|
|
for revision_id, manifest in manifests.items():
|
|
if str(manifest.get("status", "")) not in {
|
|
REVISION_STATUS_REVIEW_READY,
|
|
REVISION_STATUS_COMPLETE,
|
|
}:
|
|
continue
|
|
contract = manifest.get("contract") or {}
|
|
if lineage_sha256 is not None and str(
|
|
contract.get("nonce_lineage_sha256", "")
|
|
) != str(lineage_sha256):
|
|
continue
|
|
try:
|
|
chain = self._manifest_provenance_chain(manifest, manifests)
|
|
except Exception as exc:
|
|
self.notes.append(f"provenance chain {revision_id} rejected: {exc}")
|
|
continue
|
|
if not chain:
|
|
continue
|
|
chains[revision_id] = [item["revision_id"] for item in chain]
|
|
for item in chain:
|
|
existing = catalog.get(item["revision_id"])
|
|
if existing is not None and existing != item:
|
|
raise RunStorageError("immutable group revision collision")
|
|
catalog[item["revision_id"]] = item
|
|
return manifests, catalog, chains
|
|
|
|
def _canonical_project_selection(
|
|
self,
|
|
*,
|
|
catalog: dict[str, dict[str, Any]],
|
|
chains: dict[str, list[str]],
|
|
) -> tuple[str | None, list[str]]:
|
|
path = self.run_root / "project.json"
|
|
if path.exists():
|
|
try:
|
|
project = _read_json(path)
|
|
if int(project.get("run_storage_schema_version", -1)) == RUN_STORAGE_SCHEMA_VERSION:
|
|
storage_revision_id = str(project.get("canonical_storage_revision_id", ""))
|
|
chain = list(project.get("canonical_chain") or [])
|
|
if storage_revision_id in chains and chain == chains[storage_revision_id]:
|
|
if chain and chain[-1] in catalog:
|
|
return storage_revision_id, chain
|
|
except Exception as exc:
|
|
self.notes.append(f"canonical project head rejected: {exc}")
|
|
candidates: list[tuple[int, str, str, str, list[str]]] = []
|
|
legacy_candidates: list[tuple[int, str, str, list[str]]] = []
|
|
manifests = self._read_manifests()
|
|
for storage_revision_id, chain in chains.items():
|
|
manifest = manifests[storage_revision_id]
|
|
provenance = manifest.get("branch_provenance") or {}
|
|
committed = str(provenance.get("canonical_committed_utc", ""))
|
|
if committed:
|
|
candidates.append(
|
|
(
|
|
int(provenance.get("canonical_sequence", 0)),
|
|
committed,
|
|
storage_revision_id,
|
|
chain[-1],
|
|
chain,
|
|
)
|
|
)
|
|
else:
|
|
contract = manifest.get("contract") or {}
|
|
legacy_candidates.append(
|
|
(
|
|
int(contract.get("effective_reroll_nonce", 0)),
|
|
str(manifest.get("updated_utc", "")),
|
|
storage_revision_id,
|
|
chain,
|
|
)
|
|
)
|
|
if candidates:
|
|
_, _, storage_revision_id, _, chain = max(candidates)
|
|
return storage_revision_id, chain
|
|
if legacy_candidates:
|
|
_, _, storage_revision_id, chain = max(legacy_candidates)
|
|
return storage_revision_id, chain
|
|
return None, []
|
|
|
|
def _write_project(self, *, set_canonical: bool = False) -> None:
|
|
manifests, catalog, chains = self._provenance_catalog()
|
|
revisions = []
|
|
for revision_id, value in sorted(manifests.items()):
|
|
lifecycle = value.get("nonce_lifecycle") or {}
|
|
revision = {
|
|
key: value.get(key)
|
|
for key in (
|
|
"revision_id",
|
|
"sampling_revision_id",
|
|
"contract_sha256",
|
|
"status",
|
|
"updated_utc",
|
|
"resume_safe",
|
|
)
|
|
}
|
|
revision.update(
|
|
sampling_contract_version=value.get("sampling_contract_version"),
|
|
generation_reusable=_reusable_manifest(value),
|
|
nonce_mode=lifecycle.get("mode"),
|
|
effective_reroll_nonce=lifecycle.get("effective_nonce"),
|
|
reroll_from_chunk=(value.get("contract") or {}).get("reroll_from_chunk"),
|
|
review_control_version=value.get("review_control_version"),
|
|
review_unit=value.get("review_unit"),
|
|
review_pause_reason=value.get("review_pause_reason"),
|
|
branch_regenerate_from=value.get("branch_regenerate_from"),
|
|
active_chain=chains.get(revision_id, []),
|
|
)
|
|
revision.update(_project_resume_diagnostics(value))
|
|
revisions.append(revision)
|
|
canonical_storage_revision_id, canonical_chain = self._canonical_project_selection(
|
|
catalog=catalog,
|
|
chains=chains,
|
|
)
|
|
if set_canonical and self.revision_id in chains:
|
|
canonical_storage_revision_id = self.revision_id
|
|
canonical_chain = chains[self.revision_id]
|
|
canonical_head = canonical_chain[-1] if canonical_chain else None
|
|
_write_json(
|
|
self.run_root / "project.json",
|
|
{
|
|
"run_storage_schema_version": RUN_STORAGE_SCHEMA_VERSION,
|
|
"branch_provenance_version": BRANCH_PROVENANCE_VERSION,
|
|
"run_name": self.run_name,
|
|
"updated_utc": _now(),
|
|
"canonical_storage_revision_id": canonical_storage_revision_id,
|
|
"canonical_head_revision_id": canonical_head,
|
|
"canonical_chain": canonical_chain,
|
|
"active_revisions": active_revision_map(
|
|
[catalog[revision_id] for revision_id in canonical_chain]
|
|
) if canonical_chain else {},
|
|
"group_revisions": sorted(
|
|
[dict(item,
|
|
sampling_contract_version=(manifests.get(str(item.get("storage_revision_id"))) or {}).get("sampling_contract_version"),
|
|
generation_reusable=_reusable_manifest(manifests.get(str(item.get("storage_revision_id"))) or {}),
|
|
**_project_resume_diagnostics(manifests.get(str(item.get("storage_revision_id"))) or {}))
|
|
for item in catalog.values()],
|
|
key=lambda item: (
|
|
int(item["group"]["physical_group"]),
|
|
str(item["revision_order"]),
|
|
str(item["revision_id"]),
|
|
),
|
|
),
|
|
"revisions": revisions,
|
|
},
|
|
)
|
|
|
|
def _load_entry(self, record: dict[str, Any], prompt: str) -> dict[str, Any]:
|
|
source = str(record["storage_revision_id"])
|
|
filename = str(record["filename"])
|
|
if Path(filename).name != filename or any(c in filename for c in "/\\") or filename in {"", ".", ".."}:
|
|
raise RunStorageError("stored chunk filename is invalid")
|
|
if source in {"", ".", ".."} or any(c in source for c in "/\\"):
|
|
raise RunStorageError("stored revision directory is invalid")
|
|
path = self.revisions_root / source / "chunks" / filename
|
|
if not path.resolve().is_relative_to(self.revisions_root.resolve()):
|
|
raise RunStorageError("stored chunk escapes the Run Storage directory")
|
|
started = time.perf_counter()
|
|
if path.stat().st_size != int(record["file_size"]):
|
|
raise RunStorageError(f"stored chunk size mismatch: {filename}")
|
|
expected_sha256 = str(record.get("file_sha256", ""))
|
|
if not expected_sha256:
|
|
raise RunStorageError(f"stored chunk SHA-256 is missing: {filename}")
|
|
if _file_sha256(path) != expected_sha256:
|
|
raise RunStorageError(f"stored chunk SHA-256 mismatch: {filename}")
|
|
self.io_metrics["raw_bytes"] += int(record["file_size"])
|
|
self.io_metrics["raw_loads"] += 1
|
|
with safe_open(str(path), framework="pt", device="cpu") as handle:
|
|
if set(handle.keys()) != {"audio", "video"}:
|
|
raise RunStorageError(f"stored chunk tensors are invalid: {filename}")
|
|
video, audio = handle.get_tensor("video"), handle.get_tensor("audio")
|
|
entry = dict(record["entry"])
|
|
entry.update(prompt=str(prompt), video=video, audio=audio, reused=False)
|
|
result = validate_chunk_entry(entry)
|
|
self.io_metrics["raw_seconds"] += time.perf_counter() - started
|
|
return result
|
|
|
|
def _valid_prefix(
|
|
self,
|
|
manifest: dict[str, Any],
|
|
hashes: list[str],
|
|
*,
|
|
current_contract: dict[str, Any] | None = None,
|
|
prompts: list[str] | None = None,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
stored_contract = manifest.get("contract") or {}
|
|
prompts = self.prompts if prompts is None else prompts
|
|
active_contract = self.contract if current_contract is None else current_contract
|
|
stored = list(stored_contract.get("chunk_contract_hashes") or [])
|
|
stored_prompt_hashes = list(stored_contract.get("prompt_hashes") or [])
|
|
records = list(manifest.get("chunks") or [])
|
|
entries, accepted = [], []
|
|
for position, current in enumerate(hashes):
|
|
if position >= len(stored):
|
|
break
|
|
compatible_absent_last_frame = False
|
|
if stored[position] != current:
|
|
compatible_absent_last_frame = _legacy_absent_last_frame_compatible(
|
|
stored_contract=stored_contract,
|
|
current_contract=active_contract or {},
|
|
position=position,
|
|
stored_hash=stored[position],
|
|
current_hash=current,
|
|
)
|
|
if stored[position] != current and not compatible_absent_last_frame:
|
|
break
|
|
if position >= len(records) or int(records[position].get("sequence_index", -1)) != position:
|
|
break
|
|
try:
|
|
entry = self._load_entry(records[position], prompts[position])
|
|
except Exception as exc:
|
|
self.notes.append(f"stored chunk {position + 1} rejected: {exc}")
|
|
break
|
|
if (
|
|
position < len(stored_prompt_hashes)
|
|
and str(entry.get("prompt_hash", ""))
|
|
!= str(stored_prompt_hashes[position])
|
|
):
|
|
self.notes.append(
|
|
f"stored chunk {position + 1} rejected: prompt lineage mismatch"
|
|
)
|
|
break
|
|
entries.append(entry)
|
|
accepted.append(records[position])
|
|
if compatible_absent_last_frame:
|
|
self.notes.append(
|
|
f"stored chunk {position + 1} reused through V3.6.1 absent "
|
|
"Last Frame compatibility"
|
|
)
|
|
return entries, accepted
|
|
|
|
def _valid_provenance_prefix(
|
|
self,
|
|
manifest: dict[str, Any],
|
|
*,
|
|
current_contract: dict[str, Any],
|
|
enforce_sampling_contract: bool = False,
|
|
prompts: list[str] | None = None,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
prompts = self.prompts if prompts is None else prompts
|
|
manifests = self._read_manifests()
|
|
storage_revision_id = str(manifest.get("revision_id", ""))
|
|
if storage_revision_id not in manifests:
|
|
return [], []
|
|
chain = self._manifest_provenance_chain(manifest, manifests)
|
|
if chain and str(chain[-1].get("lineage_sha256", "")) != str(
|
|
current_contract.get("nonce_lineage_sha256", "")
|
|
):
|
|
raise RunStorageError("provenance prefix belongs to another sampling lineage")
|
|
records = list(manifest.get("chunks") or [])
|
|
if sum(
|
|
int(item["group"]["end"]) - int(item["group"]["start"]) + 1
|
|
for item in chain
|
|
) != len(records):
|
|
raise RunStorageError("provenance chain length does not match stored records")
|
|
expected_prompts = list(current_contract.get("prompt_hashes") or [])
|
|
entries: list[dict[str, Any]] = []
|
|
for position, record in enumerate(records):
|
|
if position >= len(prompts) or position >= len(expected_prompts):
|
|
raise RunStorageError("provenance prefix exceeds the current prompt plan")
|
|
if enforce_sampling_contract:
|
|
boundary = int(current_contract.get("reroll_from_chunk", 0))
|
|
# Ancestry proves where a Take came from, not that it satisfies a
|
|
# new reroll. Preserve the accepted prefix before the branch cut;
|
|
# after it, compare the actual producing revision's contract.
|
|
# Same-nonce interrupted work remains reusable.
|
|
if boundary <= 0 or position + 1 >= boundary:
|
|
producer_id = str(record.get("storage_revision_id", storage_revision_id))
|
|
producer = (manifests.get(producer_id) or {}).get("contract") or {}
|
|
produced_hashes = list(producer.get("chunk_contract_hashes") or [])
|
|
requested_hashes = list(current_contract.get("chunk_contract_hashes") or [])
|
|
if (
|
|
position >= len(produced_hashes)
|
|
or position >= len(requested_hashes)
|
|
or produced_hashes[position] != requested_hashes[position]
|
|
):
|
|
break
|
|
entry = self._load_entry(record, prompts[position])
|
|
if str(entry.get("prompt_hash", "")) != str(expected_prompts[position]):
|
|
raise RunStorageError("provenance prefix prompt lineage is incompatible")
|
|
entries.append(entry)
|
|
return entries, records[:len(entries)]
|
|
|
|
def _compatible_plan_prefix(
|
|
self, manifest: dict[str, Any], current_contract: dict[str, Any],
|
|
*, stop_before: int = 0, resume_safe: bool = True,
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
"""Certify a prefix for import, not for cross-lineage Take selection.
|
|
|
|
Original provenance is validated first. Every reused physical group must
|
|
retain its topology and actual producing per-chunk contract. New plans
|
|
get their own raw copies below; no foreign lineage is spliced into a DAG.
|
|
"""
|
|
if not _reusable_sampling_contract(current_contract, resume_safe=resume_safe) or not _reusable_manifest(manifest):
|
|
return [], []
|
|
manifests = self._read_manifests()
|
|
source_contract = manifest.get("contract") or {}
|
|
_manifest_sampling_identity(manifest)
|
|
if _hash(source_contract.get("global")) != _hash(current_contract.get("global")):
|
|
return [], []
|
|
chain = self._manifest_provenance_chain(manifest, manifests)
|
|
records = list(manifest.get("chunks") or [])
|
|
allowed = {g.start: g for g in physical_groups(
|
|
chunks=int(current_contract["chunk_count"]),
|
|
terminal_merge_enabled=_terminal_merge_enabled(current_contract),
|
|
)}
|
|
accepted = []
|
|
requested_by_producer = {}
|
|
for revision in chain:
|
|
group = revision["group"]
|
|
expected = allowed.get(int(group["start"]))
|
|
if (expected is None or expected.as_metadata() != group
|
|
or (stop_before > 0 and expected.end >= stop_before)):
|
|
break
|
|
group_records = records[expected.start - 1:expected.end]
|
|
if len(group_records) != expected.end - expected.start + 1:
|
|
break
|
|
compatible = True
|
|
for position, record in zip(range(expected.start - 1, expected.end), group_records):
|
|
producer = manifests.get(str(record.get("storage_revision_id", "")))
|
|
if producer is None or not _reusable_manifest(producer):
|
|
compatible = False
|
|
break
|
|
if "reference_group_contracts" in (producer.get("contract") or {}):
|
|
_manifest_sampling_identity(producer)
|
|
producing = producer.get("contract") or {}
|
|
hashes = producing.get("chunk_contract_hashes") or []
|
|
# Accepted earlier Takes can come from several nonce branches.
|
|
# Compare each producing contract against the new plan while
|
|
# preserving that producer's verified variation, not the latest
|
|
# branch's nonce for the entire ancestry.
|
|
producer_id = str(record["storage_revision_id"])
|
|
if producer_id not in requested_by_producer:
|
|
requested_by_producer[producer_id] = _apply_reroll_branch_contract(
|
|
current_contract,
|
|
boundary=int(producing.get("reroll_from_chunk", 0)),
|
|
requested_nonce=int(producing.get("effective_reroll_nonce", 0)),
|
|
effective_nonce=int(producing.get("effective_reroll_nonce", 0)),
|
|
)["chunk_contract_hashes"]
|
|
requested_hashes = requested_by_producer[producer_id]
|
|
producer_records = producer.get("chunks") or []
|
|
if (position >= len(hashes) or position >= len(requested_hashes)
|
|
or hashes[position] != requested_hashes[position]
|
|
or position >= len(producer_records)
|
|
or producer_records[position] != record):
|
|
compatible = False
|
|
break
|
|
if not compatible:
|
|
break
|
|
accepted.extend(group_records)
|
|
entries = []
|
|
# Only now load the certified prefix. A corrupt group never becomes reusable.
|
|
for position, record in enumerate(accepted):
|
|
item = self._load_entry(record, self.prompts[position])
|
|
if item["prompt_hash"] != current_contract["prompt_hashes"][position]:
|
|
raise RunStorageError("imported prefix prompt does not match its producing contract")
|
|
entries.append(item)
|
|
return entries, accepted
|
|
|
|
def _find_plan_import(
|
|
self, contract: dict[str, Any], *, stop_before: int = 0, resume_safe: bool = True,
|
|
) -> dict[str, Any] | None:
|
|
"""Find a verified plan prefix, stopping before explicit regeneration."""
|
|
if stop_before == 1 or not _reusable_sampling_contract(contract, resume_safe=resume_safe):
|
|
return None
|
|
manifests = self._read_manifests()
|
|
# A matching plan already owns its history. Do not reload an older plan
|
|
# merely because that older plan also supplies a compatible prefix.
|
|
if any((m.get("contract") or {}).get("nonce_lineage_sha256")
|
|
== contract.get("nonce_lineage_sha256") for m in manifests.values()):
|
|
return None
|
|
canonical_id = ""
|
|
try:
|
|
project = _read_json(self.run_root / "project.json")
|
|
canonical_id = str(project.get("canonical_storage_revision_id", ""))
|
|
selected = manifests.get(canonical_id) or {}
|
|
expected_chain = (selected.get("branch_provenance") or {}).get("active_chain")
|
|
if expected_chain is not None and project.get("canonical_chain") != expected_chain:
|
|
canonical_id = ""
|
|
except (OSError, ValueError, RunStorageError):
|
|
pass
|
|
ordered = sorted(manifests.values(), key=lambda m: (
|
|
str(m.get("revision_id", "")) == canonical_id,
|
|
int((m.get("branch_provenance") or {}).get("canonical_sequence", 0)),
|
|
str(m.get("updated_utc", "")), str(m.get("revision_id", "")),
|
|
), reverse=True)
|
|
for manifest in ordered:
|
|
source = manifest.get("contract") or {}
|
|
if (not _reusable_manifest(manifest)
|
|
or manifest.get("status") not in {"complete", "review_ready", "interrupted"}
|
|
or source.get("nonce_lineage_sha256") == contract.get("nonce_lineage_sha256")
|
|
or _hash(source.get("global")) != _hash(contract.get("global"))):
|
|
continue
|
|
# Preserve accepted variations when extending/editing a plan. Explicit
|
|
# regeneration still uses the existing nonce resolver, not this path.
|
|
boundary = int(source.get("reroll_from_chunk", 0))
|
|
nonce = int(source.get("effective_reroll_nonce", 0))
|
|
if boundary > int(contract["chunk_count"]):
|
|
boundary = nonce = 0
|
|
requested = _apply_reroll_branch_contract(
|
|
contract, boundary=boundary, requested_nonce=nonce, effective_nonce=nonce,
|
|
)
|
|
try:
|
|
entries, records = self._compatible_plan_prefix(
|
|
manifest, requested, stop_before=stop_before,
|
|
)
|
|
except (RunStorageError, ValueError, OSError) as exc:
|
|
self.notes.append(f"plan prefix {manifest.get('revision_id', '')} rejected: {exc}")
|
|
continue
|
|
if not entries:
|
|
continue
|
|
if boundary > len(entries):
|
|
boundary = nonce = 0
|
|
requested = _apply_reroll_branch_contract(
|
|
contract, boundary=0, requested_nonce=0, effective_nonce=0,
|
|
)
|
|
groups = physical_groups(chunks=int(contract["chunk_count"]),
|
|
terminal_merge_enabled=_terminal_merge_enabled(contract))
|
|
unit = next(g for g in groups if g.end == len(entries))
|
|
return {"revision_id": str(manifest["revision_id"]),
|
|
"status": "complete" if len(entries) == int(contract["chunk_count"]) else "review_ready",
|
|
"validated_prefix_count": len(entries), "review_unit": unit.as_metadata(),
|
|
"branch_regenerate_from": boundary, "effective_reroll_nonce": nonce,
|
|
"manifest": manifest, "entries": entries, "records": records,
|
|
"contract": requested, "updated_utc": str(manifest.get("updated_utc", ""))}
|
|
return None
|
|
|
|
def _adopt_plan_prefix(self, imported: dict[str, Any]) -> list[dict[str, Any]]:
|
|
"""Copy verified tensors into the new plan; keep every old revision intact."""
|
|
if not _reusable_sampling_contract(self.contract, resume_safe=self.resume_safe) or not _reusable_manifest(imported.get("manifest") or {}):
|
|
raise RunStorageError("plan prefix has no reusable Sampling v6 identity")
|
|
self.manifest["prefix_import"] = {
|
|
"version": 1, "source_revision_id": imported["revision_id"],
|
|
"source_contract_sha256": imported["manifest"]["contract_sha256"],
|
|
"records": [
|
|
{"sequence_index": position,
|
|
"storage_revision_id": original["storage_revision_id"],
|
|
"filename": original["filename"], "file_sha256": original["file_sha256"]}
|
|
for position, original in enumerate(imported["records"])
|
|
],
|
|
}
|
|
if imported.get("selected_take_import") is not None:
|
|
self.manifest["prefix_import"]["selected_take"] = dict(
|
|
imported["selected_take_import"]
|
|
)
|
|
self._write_manifest()
|
|
records = []
|
|
imported_entries = list(imported["entries"])
|
|
groups = physical_groups(
|
|
chunks=int((self.contract or {})["chunk_count"]),
|
|
terminal_merge_enabled=_terminal_merge_enabled(self.contract or {}),
|
|
)
|
|
for group in groups:
|
|
if group.end > len(imported_entries):
|
|
break
|
|
positions = tuple(range(group.start - 1, group.end))
|
|
self.commit_group(
|
|
tuple(imported_entries[position] for position in positions),
|
|
positions=positions,
|
|
)
|
|
records.extend(
|
|
dict(record)
|
|
for record in self.manifest["chunks"][-len(positions):]
|
|
)
|
|
self.generated_count = 0 # Copies are reused work, never new Sampling.
|
|
if imported.get("selected_take_import") is not None:
|
|
local_chain = self._manifest_provenance_chain(
|
|
self.manifest, self._read_manifests(),
|
|
)
|
|
if not local_chain:
|
|
raise RunStorageError("imported Take has no new-plan provenance")
|
|
self.pending_branch_cut = {
|
|
"selected_revision_id": local_chain[-1]["revision_id"],
|
|
"after_physical_group": int(local_chain[-1]["group"]["physical_group"]),
|
|
}
|
|
self.notes.append(f"Imported {len(records)} verified prefix chunks into the new plan; originals preserved")
|
|
return records
|
|
|
|
def _highest_lineage_nonce(self, lineage_sha256: str) -> int:
|
|
highest = 0
|
|
for manifest in self._read_manifests().values():
|
|
contract = manifest.get("contract") or {}
|
|
if str(contract.get("nonce_lineage_sha256", "")) != str(lineage_sha256):
|
|
continue
|
|
highest = max(highest, int(contract.get("effective_reroll_nonce", 0)))
|
|
return highest
|
|
|
|
def _validated_take_selection(
|
|
self,
|
|
contract: dict[str, Any],
|
|
*, source_lineage: str | None = None, resume_safe: bool = True,
|
|
) -> dict[str, Any]:
|
|
if not _reusable_sampling_contract(contract, resume_safe=resume_safe):
|
|
raise RunStorageError("selected Take cannot be reused: current sampling identity is unobservable or unsupported")
|
|
lineage = str(source_lineage or contract.get("nonce_lineage_sha256", ""))
|
|
manifests, catalog, _ = self._provenance_catalog(
|
|
lineage_sha256=lineage,
|
|
)
|
|
try:
|
|
chain = resolve_chain(catalog, self.take_revision_id)
|
|
except BranchProvenanceError as exc:
|
|
raise RunStorageError(f"selected Take provenance is invalid: {exc}") from exc
|
|
selected = chain[-1]
|
|
group = selected["group"]
|
|
if int(group["physical_group"]) != self.take_group:
|
|
raise RunStorageError(
|
|
"selected Take group does not match its immutable revision"
|
|
)
|
|
allowed = {
|
|
item.physical_group: item
|
|
for item in physical_groups(
|
|
chunks=int(contract["chunk_count"]),
|
|
terminal_merge_enabled=_terminal_merge_enabled(contract),
|
|
)
|
|
}
|
|
records: list[dict[str, Any]] = []
|
|
entries: list[dict[str, Any]] = []
|
|
for revision in chain:
|
|
revision_group = revision["group"]
|
|
physical_group = int(revision_group["physical_group"])
|
|
expected_group = allowed.get(physical_group)
|
|
if expected_group is None or (
|
|
expected_group.start != int(revision_group["start"])
|
|
or expected_group.end != int(revision_group["end"])
|
|
):
|
|
raise RunStorageError(
|
|
"selected Take splits or changes a physical group"
|
|
)
|
|
if str(revision.get("lineage_sha256", "")) != lineage:
|
|
raise RunStorageError("selected Take belongs to another sampling lineage")
|
|
source_id = str(revision["storage_revision_id"])
|
|
source_manifest = manifests.get(source_id)
|
|
if source_manifest is None:
|
|
raise RunStorageError("selected Take source manifest is unavailable")
|
|
if not _reusable_manifest(source_manifest):
|
|
raise RunStorageError("selected Take is read-only: Sampling v6 with an observable identity is required")
|
|
source_records = list(source_manifest.get("chunks") or [])
|
|
for position in range(expected_group.start - 1, expected_group.end):
|
|
if position >= len(source_records):
|
|
raise RunStorageError("selected Take source record is incomplete")
|
|
record = source_records[position]
|
|
if int(record.get("sequence_index", -1)) != position:
|
|
raise RunStorageError("selected Take source record position is invalid")
|
|
try:
|
|
entry = self._load_entry(record, self.prompts[position])
|
|
except Exception as exc:
|
|
raise RunStorageError(
|
|
f"selected Take chunk {position + 1} failed integrity validation: {exc}"
|
|
) from exc
|
|
expected_prompt = str(contract["prompt_hashes"][position])
|
|
if str(entry.get("prompt_hash", "")) != expected_prompt:
|
|
raise RunStorageError("selected Take prompt lineage is incompatible")
|
|
records.append(record)
|
|
entries.append(entry)
|
|
if len(records) != int(group["end"]):
|
|
raise RunStorageError("selected Take does not resolve to a complete prefix")
|
|
branch_start = 0
|
|
for revision in chain:
|
|
cut = revision.get("branch_cut")
|
|
if isinstance(cut, dict):
|
|
candidate = int(cut.get("after_physical_group", 0)) + 1
|
|
if candidate > 0:
|
|
branch_start = candidate
|
|
selected_nonce = int(selected.get("variation_nonce", 0))
|
|
if selected_nonce == 0:
|
|
branch_start = 0
|
|
elif branch_start == 0:
|
|
raise RunStorageError("selected Take nonce has no branch cut provenance")
|
|
return {
|
|
"revision_id": selected["revision_id"],
|
|
"status": (
|
|
REVISION_STATUS_COMPLETE
|
|
if len(records) == int(contract["chunk_count"])
|
|
else REVISION_STATUS_REVIEW_READY
|
|
),
|
|
"validated_prefix_count": len(records),
|
|
"review_unit": dict(group),
|
|
"branch_regenerate_from": branch_start,
|
|
"effective_reroll_nonce": selected_nonce,
|
|
"updated_utc": str(selected["created_utc"]),
|
|
"records": records,
|
|
"entries": entries,
|
|
"chain": chain,
|
|
"manifest": manifests.get(str(selected["storage_revision_id"])),
|
|
}
|
|
|
|
def _validated_review_head(
|
|
self,
|
|
manifest: dict[str, Any],
|
|
*,
|
|
current_contract: dict[str, Any] | None = None,
|
|
for_reuse: bool = True,
|
|
resume_safe: bool = True,
|
|
) -> dict[str, Any]:
|
|
"""Validate stored integrity first; optionally certify current reuse."""
|
|
|
|
schema = int(manifest.get("run_storage_schema_version", -1))
|
|
if schema not in RUN_STORAGE_READABLE_SCHEMA_VERSIONS:
|
|
raise RunStorageError("review head uses an incompatible Run Storage schema")
|
|
stored_contract = manifest.get("contract")
|
|
if not isinstance(stored_contract, dict):
|
|
raise RunStorageError("review head sampling contract is missing")
|
|
if for_reuse and (
|
|
current_contract is None
|
|
or not _reusable_sampling_contract(current_contract, resume_safe=resume_safe)
|
|
or not _reusable_manifest(manifest)
|
|
):
|
|
raise RunStorageError("review head is read-only or its sampling identity is unobservable")
|
|
if for_reuse and str(stored_contract.get("nonce_lineage_sha256", "")) != str(
|
|
current_contract.get("nonce_lineage_sha256", "")
|
|
):
|
|
raise RunStorageError("review head belongs to a different sampling lineage")
|
|
_, contract_sha256 = _manifest_sampling_identity(manifest)
|
|
revision_id = str(manifest.get("revision_id", ""))
|
|
if int(manifest.get("sampling_contract_version", -1)) not in SAMPLING_READABLE_CONTRACT_VERSIONS:
|
|
raise RunStorageError("review head sampling contract version is incompatible")
|
|
|
|
status = str(manifest.get("status", ""))
|
|
if status not in {
|
|
REVISION_STATUS_IN_PROGRESS,
|
|
REVISION_STATUS_INTERRUPTED,
|
|
REVISION_STATUS_REVIEW_READY,
|
|
REVISION_STATUS_COMPLETE,
|
|
}:
|
|
raise RunStorageError(f"review head status is invalid: {status!r}")
|
|
chunks = int(stored_contract.get("chunk_count", 0))
|
|
if chunks <= 0:
|
|
raise RunStorageError("review head configured chunk count is invalid")
|
|
hashes = list(stored_contract.get("chunk_contract_hashes") or [])
|
|
if len(hashes) != chunks:
|
|
raise RunStorageError("review head chunk contract is incomplete")
|
|
records = list(manifest.get("chunks") or [])
|
|
stored_prompts = None if for_reuse else [""] * chunks
|
|
if schema == RUN_STORAGE_SCHEMA_VERSION and manifest.get("branch_provenance"):
|
|
entries, accepted = self._valid_provenance_prefix(
|
|
manifest,
|
|
current_contract=stored_contract,
|
|
prompts=stored_prompts,
|
|
)
|
|
else:
|
|
entries, accepted = self._valid_prefix(
|
|
manifest,
|
|
hashes,
|
|
current_contract=stored_contract,
|
|
prompts=stored_prompts,
|
|
)
|
|
if len(accepted) != len(records):
|
|
raise RunStorageError("review head stored prefix failed validation")
|
|
prefix = len(entries)
|
|
if prefix > chunks:
|
|
raise RunStorageError("review head stored prefix exceeds configured chunks")
|
|
if status == REVISION_STATUS_COMPLETE and prefix != chunks:
|
|
raise RunStorageError("complete review head does not contain every chunk")
|
|
if status != REVISION_STATUS_COMPLETE and prefix == chunks:
|
|
raise RunStorageError("incomplete review head status contains every chunk")
|
|
|
|
lifecycle = manifest.get("nonce_lifecycle") or stored_contract.get(
|
|
"nonce_lifecycle"
|
|
) or {}
|
|
contract_boundary = int(stored_contract.get("reroll_from_chunk", 0))
|
|
contract_nonce = int(stored_contract.get("effective_reroll_nonce", 0))
|
|
if int(lifecycle.get("effective_nonce", contract_nonce)) != contract_nonce:
|
|
raise RunStorageError("review head nonce lifecycle is inconsistent")
|
|
branch_boundary = int(
|
|
manifest.get("branch_regenerate_from", contract_boundary)
|
|
)
|
|
effective_nonce = int(
|
|
manifest.get("effective_reroll_nonce", contract_nonce)
|
|
)
|
|
if branch_boundary != contract_boundary or effective_nonce != contract_nonce:
|
|
raise RunStorageError("review head branch metadata is inconsistent")
|
|
|
|
review_version = manifest.get("review_control_version")
|
|
if review_version is not None and int(review_version) != REVIEW_CONTROL_VERSION:
|
|
raise RunStorageError("review head control version is unsupported")
|
|
review_unit = manifest.get("review_unit")
|
|
pause_reason = manifest.get("review_pause_reason")
|
|
if review_unit is not None and pause_reason != "review_each_chunk":
|
|
raise RunStorageError("review head pause reason is inconsistent")
|
|
if review_unit is None and pause_reason is not None:
|
|
raise RunStorageError("review head pause reason has no review unit")
|
|
if status == REVISION_STATUS_REVIEW_READY and review_unit is None:
|
|
raise RunStorageError("review_ready head is missing review_unit metadata")
|
|
if status in {REVISION_STATUS_IN_PROGRESS, REVISION_STATUS_INTERRUPTED}:
|
|
if review_unit is not None:
|
|
raise RunStorageError("unfinished review execution has stale review_unit metadata")
|
|
|
|
semantics = (stored_contract.get("global") or {}).get(
|
|
"execution_semantics"
|
|
) or {}
|
|
terminal_merge_enabled = (
|
|
semantics.get("flf_execution") == "terminal_merged_10s_seed_v2"
|
|
)
|
|
terminal_pair_start = chunks - 1 if terminal_merge_enabled else None
|
|
try:
|
|
normalized_review_unit = validate_review_prefix_metadata(
|
|
configured_chunks=chunks,
|
|
validated_prefix_count=prefix,
|
|
terminal_merge_enabled=terminal_merge_enabled,
|
|
terminal_pair_start=terminal_pair_start,
|
|
latest_review_unit=review_unit,
|
|
latest_revision_status=status,
|
|
latest_effective_nonce=effective_nonce,
|
|
latest_branch_regenerate_from=branch_boundary,
|
|
)
|
|
except ReviewControlError as exc:
|
|
raise RunStorageError(f"review head metadata is invalid: {exc}") from exc
|
|
|
|
updated_utc = str(
|
|
manifest.get("updated_utc") or manifest.get("created_utc") or ""
|
|
)
|
|
try:
|
|
datetime.fromisoformat(updated_utc)
|
|
except ValueError as exc:
|
|
raise RunStorageError("review head update timestamp is invalid") from exc
|
|
return {
|
|
"revision_id": revision_id,
|
|
"status": status,
|
|
"validated_prefix_count": prefix,
|
|
"review_unit": review_unit,
|
|
"branch_regenerate_from": branch_boundary,
|
|
"effective_reroll_nonce": effective_nonce,
|
|
"updated_utc": updated_utc,
|
|
"manifest": manifest,
|
|
"entries": entries,
|
|
"records": accepted,
|
|
"normalized_review_unit": normalized_review_unit,
|
|
}
|
|
|
|
def find_latest_review_head(
|
|
self,
|
|
contract: dict[str, Any],
|
|
*,
|
|
smart_regenerate_only: bool = False,
|
|
resume_safe: bool = True,
|
|
) -> dict[str, Any] | None:
|
|
"""Return the latest validated compatible disk head deterministically."""
|
|
|
|
if not _reusable_sampling_contract(contract, resume_safe=resume_safe):
|
|
self.notes.append("auto-resume disabled: current sampling identity is unobservable or unsupported")
|
|
return None
|
|
if not self.revisions_root.exists():
|
|
return None
|
|
compatible_lineage = str(contract.get("nonce_lineage_sha256", ""))
|
|
mismatched_contracts: list[tuple[str, dict]] = []
|
|
manifests = self._read_manifests()
|
|
canonical_id = ""
|
|
try:
|
|
project = _read_json(self.run_root / "project.json")
|
|
canonical_id = str(project.get("canonical_storage_revision_id", ""))
|
|
selected = manifests.get(canonical_id) or {}
|
|
expected_chain = (selected.get("branch_provenance") or {}).get("active_chain")
|
|
if expected_chain is not None and project.get("canonical_chain") != expected_chain:
|
|
canonical_id = ""
|
|
except (OSError, ValueError, RunStorageError):
|
|
pass
|
|
ordered = sorted(manifests.items(), key=lambda item: (
|
|
item[0] == canonical_id,
|
|
int((item[1].get("contract") or {}).get("effective_reroll_nonce", 0)),
|
|
str(item[1].get("updated_utc", "")), item[0],
|
|
), reverse=True)
|
|
for revision_id, manifest in ordered:
|
|
stored_contract = manifest.get("contract") or {}
|
|
if not _reusable_manifest(manifest):
|
|
self.notes.append(f"review head {revision_id} is read-only: observable Sampling v6 required")
|
|
continue
|
|
if str(stored_contract.get("nonce_lineage_sha256", "")) != compatible_lineage:
|
|
mismatched_contracts.append((revision_id, stored_contract))
|
|
continue
|
|
try:
|
|
head = self._validated_review_head(
|
|
manifest,
|
|
current_contract=contract,
|
|
)
|
|
except Exception as exc:
|
|
self.notes.append(f"review head {revision_id} rejected: {exc}")
|
|
continue
|
|
if smart_regenerate_only and not (
|
|
head["status"] == REVISION_STATUS_REVIEW_READY
|
|
or (
|
|
head["status"] == REVISION_STATUS_COMPLETE
|
|
and head["review_unit"] is not None
|
|
)
|
|
):
|
|
continue
|
|
return head
|
|
# Explain a missing head without changing which heads are reusable.
|
|
# Evaluate only after lookup fails; successful runs are unaffected.
|
|
candidates = []
|
|
for revision_id, stored_contract in mismatched_contracts:
|
|
try:
|
|
delta = _review_contract_mismatches(stored_contract, contract)
|
|
candidates.append((len(delta), revision_id, delta))
|
|
except Exception as exc:
|
|
self.notes.append(
|
|
f"review head {revision_id} diagnostic unavailable: {type(exc).__name__}"
|
|
)
|
|
for _, revision_id, delta in sorted(candidates)[:3]:
|
|
detail = "; ".join(delta[:12]) or "lineage digest differs; inspect stored integrity"
|
|
if len(delta) > 12:
|
|
detail += f"; {len(delta) - 12} more differing sections"
|
|
self.notes.append(f"review head {revision_id} incompatible: {detail}")
|
|
return None
|
|
|
|
def _freeze_validated_prefix(
|
|
self,
|
|
candidate: dict[str, Any] | None,
|
|
*,
|
|
contract: dict[str, Any],
|
|
) -> ValidatedPrefix | EmptyValidatedPrefix:
|
|
"""Freeze the one repository result that may drive this Queue."""
|
|
|
|
configured_chunks = int(contract["chunk_count"])
|
|
if candidate is None:
|
|
return EmptyValidatedPrefix(configured_chunks=configured_chunks)
|
|
entries = tuple(dict(entry) for entry in candidate.get("entries") or ())
|
|
records = tuple(dict(record) for record in candidate.get("records") or ())
|
|
if len(entries) != len(records):
|
|
raise RunStorageError("validated prefix entry/record counts differ")
|
|
if int(candidate.get("validated_prefix_count", -1)) != len(entries):
|
|
raise RunStorageError("validated prefix count does not match loaded raw entries")
|
|
status = str(candidate.get("status", "")) or None
|
|
terminal_merge_enabled = _terminal_merge_enabled(contract)
|
|
review_unit = validate_review_prefix_metadata(
|
|
configured_chunks=configured_chunks,
|
|
validated_prefix_count=len(entries),
|
|
terminal_merge_enabled=terminal_merge_enabled,
|
|
terminal_pair_start=(
|
|
configured_chunks - 1 if terminal_merge_enabled else None
|
|
),
|
|
latest_review_unit=candidate.get("review_unit"),
|
|
latest_revision_status=status,
|
|
latest_effective_nonce=int(candidate.get("effective_reroll_nonce", 0)),
|
|
latest_branch_regenerate_from=int(
|
|
candidate.get("branch_regenerate_from", 0)
|
|
),
|
|
)
|
|
boundaries = tuple(
|
|
(group.start, group.end)
|
|
for group in physical_groups(
|
|
chunks=configured_chunks,
|
|
terminal_merge_enabled=terminal_merge_enabled,
|
|
)
|
|
if group.end <= len(entries)
|
|
)
|
|
if boundaries and boundaries[-1][1] != len(entries):
|
|
raise RunStorageError("validated prefix ends inside a physical group")
|
|
if entries and not boundaries:
|
|
raise RunStorageError("validated prefix has no complete physical group")
|
|
source_manifest = candidate.get("manifest")
|
|
frozen_manifest = (
|
|
None
|
|
if source_manifest is None
|
|
else MappingProxyType(copy.deepcopy(dict(source_manifest)))
|
|
)
|
|
revision_id = str(candidate.get("revision_id", "")) or None
|
|
return ValidatedPrefix(
|
|
entries=entries,
|
|
records=records,
|
|
manifest=frozen_manifest,
|
|
revision_id=revision_id,
|
|
status=status,
|
|
review_unit=review_unit,
|
|
effective_nonce=int(candidate.get("effective_reroll_nonce", 0)),
|
|
branch_regenerate_from=int(candidate.get("branch_regenerate_from", 0)),
|
|
physical_group_boundaries=boundaries,
|
|
configured_chunks=configured_chunks,
|
|
)
|
|
|
|
def _load_validated_review_prefix(
|
|
self,
|
|
contract: dict[str, Any],
|
|
*,
|
|
resume_safe: bool,
|
|
) -> ValidatedPrefix | EmptyValidatedPrefix:
|
|
"""Select, fully validate, load, and freeze one branch-neutral prefix."""
|
|
|
|
if self.review_generation_mode is None or self.review_action is None:
|
|
raise RunStorageError("review execution intent is incomplete")
|
|
candidate = None
|
|
if self.take_action != TAKE_ACTION_AUTOMATIC:
|
|
if self.review_generation_mode != GENERATION_MODE_REVIEW:
|
|
raise RunStorageError("Take selection requires Review Each Chunk")
|
|
if self.review_manual_regenerate_from:
|
|
raise RunStorageError(
|
|
"Take selection cannot be combined with manual Regenerate From"
|
|
)
|
|
source_lineage = str(contract.get("nonce_lineage_sha256", ""))
|
|
if "reference_group_contracts" in contract:
|
|
_, all_takes, _ = self._provenance_catalog()
|
|
selected_take = all_takes.get(self.take_revision_id)
|
|
if selected_take is not None:
|
|
source_lineage = str(selected_take.get("lineage_sha256", ""))
|
|
candidate = self._validated_take_selection(
|
|
contract, source_lineage=source_lineage, resume_safe=resume_safe,
|
|
)
|
|
if source_lineage != str(contract.get("nonce_lineage_sha256", "")):
|
|
source_manifest = candidate.get("manifest")
|
|
if not isinstance(source_manifest, dict):
|
|
raise RunStorageError("selected Take source manifest is unavailable")
|
|
source_contract = source_manifest.get("contract") or {}
|
|
if _hash(source_contract.get("global")) != _hash(contract.get("global")):
|
|
raise RunStorageError("selected Take has a different common execution contract")
|
|
manifests = self._read_manifests()
|
|
source_chain = self._manifest_provenance_chain(source_manifest, manifests)
|
|
selected_chain = candidate["chain"]
|
|
if ([item["revision_id"] for item in source_chain[:len(selected_chain)]]
|
|
!= [item["revision_id"] for item in selected_chain]):
|
|
raise RunStorageError("selected Take is not the source manifest's validated prefix")
|
|
selected_end = int(candidate["validated_prefix_count"])
|
|
try:
|
|
compatible_entries, compatible_records = self._compatible_plan_prefix(
|
|
source_manifest, contract, stop_before=selected_end + 1,
|
|
)
|
|
except (OSError, ValueError) as exc:
|
|
raise RunStorageError(f"selected Take prefix cannot be imported: {exc}") from exc
|
|
if (len(compatible_entries) != selected_end
|
|
or compatible_records != candidate["records"]):
|
|
failed_group = len(compatible_entries) + 1
|
|
raise RunStorageError(
|
|
f"selected Take is incompatible with the new Reference plan "
|
|
f"at physical group starting with chunk {failed_group}; "
|
|
"choose an earlier compatible Take or deselect Take and Regenerate From"
|
|
)
|
|
candidate["entries"] = compatible_entries
|
|
candidate["records"] = compatible_records
|
|
candidate["selected_take_import"] = {
|
|
"revision_id": self.take_revision_id,
|
|
"physical_group": self.take_group,
|
|
}
|
|
self._plan_import = candidate
|
|
else:
|
|
self.selected_take_chain = list(candidate["chain"])
|
|
self.selected_take_records = list(candidate["records"])
|
|
self.selected_take_revision = dict(candidate)
|
|
else:
|
|
smart_only = self.review_action == REVIEW_ACTION_REGENERATE_CURRENT
|
|
candidate = self.find_latest_review_head(
|
|
contract,
|
|
smart_regenerate_only=smart_only,
|
|
resume_safe=resume_safe,
|
|
)
|
|
if candidate is None and not smart_only and resume_safe:
|
|
self._plan_import = self._find_plan_import(
|
|
contract,
|
|
stop_before=self.review_manual_regenerate_from,
|
|
resume_safe=resume_safe,
|
|
)
|
|
candidate = self._plan_import
|
|
if candidate is None and smart_only:
|
|
details = "\n".join(self.notes[-6:]) or (
|
|
"No compatible saved review head was found."
|
|
)
|
|
raise ReviewControlError(
|
|
"Regenerate Current requires an existing reviewed unit. "
|
|
"The saved review could not be matched to this execution.\n"
|
|
+ details
|
|
)
|
|
self.review_head = candidate
|
|
self.inherited_review_unit = (
|
|
None
|
|
if candidate is None or candidate.get("review_unit") is None
|
|
else dict(candidate["review_unit"])
|
|
)
|
|
frozen = self._freeze_validated_prefix(candidate, contract=contract)
|
|
self.validated_prefix = frozen
|
|
return frozen
|
|
|
|
def _record_review_decision_created(self) -> None:
|
|
created = int(self.review_runtime_metrics["review_decision_created"])
|
|
if created != 0:
|
|
raise RunStorageError("Review Decision was already created for this Queue")
|
|
self.review_runtime_metrics["review_decision_created"] = 1
|
|
|
|
def _resolve_review_contract(
|
|
self,
|
|
contract: dict[str, Any],
|
|
*,
|
|
requested_nonce: int,
|
|
resume_safe: bool,
|
|
validated_prefix: ValidatedPrefix | EmptyValidatedPrefix | None = None,
|
|
) -> tuple[dict[str, Any], int, str]:
|
|
"""Create the single Review Decision from an immutable validated prefix."""
|
|
|
|
if self.review_generation_mode is None or self.review_action is None:
|
|
raise RunStorageError("review execution intent is incomplete")
|
|
if validated_prefix is None:
|
|
validated_prefix = self._load_validated_review_prefix(
|
|
contract,
|
|
resume_safe=resume_safe,
|
|
)
|
|
if self.validated_prefix is None:
|
|
self.validated_prefix = validated_prefix
|
|
elif self.validated_prefix is not validated_prefix:
|
|
raise RunStorageError(
|
|
"Review Decision cannot replace the fixed ValidatedPrefix"
|
|
)
|
|
facts = validated_prefix.to_planning_facts()
|
|
if self.take_action != TAKE_ACTION_AUTOMATIC:
|
|
next_nonce = self._highest_lineage_nonce(
|
|
str(contract["nonce_lineage_sha256"])
|
|
) + 1
|
|
self._record_review_decision_created()
|
|
execution = resolve_take_execution(
|
|
take_action=self.take_action,
|
|
configured_chunks=int(contract["chunk_count"]),
|
|
selected_prefix_count=facts.accepted_chunks,
|
|
terminal_merge_enabled=_terminal_merge_enabled(contract),
|
|
terminal_pair_start=(
|
|
int(contract["chunk_count"]) - 1
|
|
if _terminal_merge_enabled(contract)
|
|
else None
|
|
),
|
|
selected_review_unit=facts.review_unit,
|
|
selected_effective_nonce=facts.effective_nonce,
|
|
selected_branch_regenerate_from=facts.branch_regenerate_from,
|
|
next_effective_nonce=next_nonce,
|
|
)
|
|
boundary = int(execution.effective_regenerate_from)
|
|
effective_nonce = int(execution.requested_effective_nonce or 0)
|
|
resolved = _apply_reroll_branch_contract(
|
|
contract,
|
|
boundary=boundary,
|
|
requested_nonce=effective_nonce,
|
|
effective_nonce=effective_nonce,
|
|
)
|
|
self.review_execution = execution
|
|
self.pending_branch_cut = {
|
|
"selected_revision_id": str(facts.revision_id),
|
|
"after_physical_group": int(facts.review_unit.physical_group),
|
|
}
|
|
self.storage_revision_id_override = _storage_revision_identity(
|
|
contract_sha256=revision_identity(resolved)[1],
|
|
take_action=self.take_action,
|
|
selected_revision_id=str(facts.revision_id),
|
|
effective_nonce=effective_nonce,
|
|
branch_boundary=boundary,
|
|
)
|
|
return resolved, effective_nonce, (
|
|
"select_take" if self.take_action == TAKE_ACTION_USE else "branch_from_take"
|
|
)
|
|
semantics = (contract.get("global") or {}).get("execution_semantics") or {}
|
|
terminal_merge_enabled = (
|
|
semantics.get("flf_execution") == "terminal_merged_10s_seed_v2"
|
|
)
|
|
configured_chunks = int(contract["chunk_count"])
|
|
self._record_review_decision_created()
|
|
execution = resolve_review_execution(
|
|
generation_mode=self.review_generation_mode,
|
|
review_action=self.review_action,
|
|
configured_chunks=configured_chunks,
|
|
validated_prefix_count=facts.accepted_chunks,
|
|
terminal_merge_enabled=terminal_merge_enabled,
|
|
terminal_pair_start=(
|
|
configured_chunks - 1 if terminal_merge_enabled else None
|
|
),
|
|
manual_regenerate_from=self.review_manual_regenerate_from,
|
|
run_storage_mode=RUN_STORAGE_SAVE_AUTO_RESUME,
|
|
latest_review_unit=facts.review_unit,
|
|
latest_revision_status=facts.status,
|
|
latest_effective_nonce=facts.effective_nonce,
|
|
latest_branch_regenerate_from=facts.branch_regenerate_from,
|
|
)
|
|
boundary = int(execution.effective_regenerate_from)
|
|
if execution.requested_effective_nonce is None:
|
|
draft = _apply_reroll_branch_contract(
|
|
contract,
|
|
boundary=boundary,
|
|
requested_nonce=int(requested_nonce),
|
|
effective_nonce=(int(requested_nonce) if boundary > 0 else 0),
|
|
)
|
|
effective_nonce, decision = self._resolve_effective_nonce(
|
|
draft,
|
|
requested_nonce=int(requested_nonce),
|
|
resume_safe=resume_safe,
|
|
)
|
|
applied_requested_nonce = int(requested_nonce)
|
|
else:
|
|
effective_nonce = int(execution.requested_effective_nonce)
|
|
applied_requested_nonce = effective_nonce
|
|
decision = str(execution.nonce_policy)
|
|
resolved = _apply_reroll_branch_contract(
|
|
contract,
|
|
boundary=boundary,
|
|
requested_nonce=applied_requested_nonce,
|
|
effective_nonce=effective_nonce,
|
|
)
|
|
self.review_execution = execution
|
|
return resolved, effective_nonce, decision
|
|
|
|
def _resolve_effective_nonce(
|
|
self, contract: dict[str, Any], *, requested_nonce: int, resume_safe: bool,
|
|
) -> tuple[int, str]:
|
|
boundary = int(contract["reroll_from_chunk"])
|
|
requested = int(requested_nonce)
|
|
if boundary <= 0:
|
|
return 0, "inactive"
|
|
if requested >= 1:
|
|
return requested, "explicit"
|
|
lineage = str(contract["nonce_lineage_sha256"])
|
|
request = str(contract["nonce_request_sha256"])
|
|
highest = 0
|
|
matching: dict[int, dict[str, Any]] = {}
|
|
if self.revisions_root.exists():
|
|
for path in self.revisions_root.glob("*/manifest.json"):
|
|
try:
|
|
manifest = _read_json(path)
|
|
lifecycle = manifest.get("nonce_lifecycle") or (manifest.get("contract") or {}).get("nonce_lifecycle") or {}
|
|
if str(lifecycle.get("lineage_sha256", "")) != lineage:
|
|
continue
|
|
nonce = int(lifecycle.get("effective_nonce", 0))
|
|
highest = max(highest, nonce)
|
|
if str(lifecycle.get("request_sha256", "")) == request:
|
|
current = matching.get(nonce)
|
|
if current is None or str(manifest.get("updated_utc", "")) >= str(current.get("updated_utc", "")):
|
|
matching[nonce] = manifest
|
|
except Exception:
|
|
continue
|
|
if resume_safe and matching:
|
|
latest_nonce = max(matching)
|
|
latest = matching[latest_nonce]
|
|
if latest_nonce == highest and str(latest.get("status", "")) != "complete":
|
|
return latest_nonce, "resume_interrupted"
|
|
return highest + 1, "new_revision"
|
|
|
|
def prepare(
|
|
self, *, model: Any, model_fingerprint_value: str, clip: Any,
|
|
video_vae: Any, sampler: Any,
|
|
sigmas: torch.Tensor, prompt_plan: dict[str, Any], width: int,
|
|
height: int, chunk_seconds: float, continuity: str,
|
|
audio_continuity: bool, base_seed: int, reroll_from_chunk: int,
|
|
reroll_nonce: int, first_frame_hash: str, last_frame_hash: str,
|
|
identity_hash: str, strict_compatibility: bool,
|
|
existing_session: dict[str, Any] | None,
|
|
reference_contract: dict[str, Any] | None = None,
|
|
conditioning_mode: str | None = None,
|
|
reference_audio_contract: dict[str, Any] | None = None,
|
|
reference_audio_vae: Any = None,
|
|
driving_audio_contract: dict[str, Any] | None = None,
|
|
driving_audio_vae: Any = None,
|
|
reference_video_contract: dict[str, Any] | None = None,
|
|
timeline_video_contract: dict[str, Any] | None = None,
|
|
guide_contract: dict[str, Any] | None = None,
|
|
execution_semantics: dict[str, Any] | None = None,
|
|
reference_storage_plan: ReferenceStoragePlan | None = None,
|
|
) -> dict[str, Any] | None:
|
|
if existing_session is not None:
|
|
raise RunStorageError("Run Storage cannot be combined with an explicit Session")
|
|
self.prompts = [str(value) for value in prompt_plan["prompts"]]
|
|
from .graph_contract import build_upstream_graph_contract
|
|
graph_contract, graph_safe, graph_reasons = build_upstream_graph_contract(
|
|
self.prompt_graph,
|
|
self.sampler_node_id,
|
|
require_video_vae=(
|
|
reference_storage_plan is not None
|
|
or
|
|
conditioning_mode_uses_video_vae(str(conditioning_mode))
|
|
or reference_video_contract is not None
|
|
or timeline_video_contract is not None
|
|
or guide_contract is not None
|
|
),
|
|
require_reference_audio_vae=reference_audio_contract is not None,
|
|
reference_audio_route=(
|
|
"audio_references"
|
|
if "reference_audio_bundle_contract_version" in (reference_audio_contract or {})
|
|
else "reference_audio_vae"
|
|
),
|
|
require_audio_vae=driving_audio_contract is not None,
|
|
)
|
|
contract, safe, reasons = build_sampling_contract(
|
|
model=model, model_fingerprint_value=model_fingerprint_value,
|
|
clip=clip, video_vae=video_vae,
|
|
sampler=sampler, sigmas=sigmas, prompt_plan=prompt_plan,
|
|
width=width, height=height, chunk_seconds=chunk_seconds,
|
|
continuity=continuity, audio_continuity=audio_continuity,
|
|
base_seed=base_seed, reroll_from_chunk=reroll_from_chunk,
|
|
reroll_nonce=reroll_nonce, first_frame_hash=first_frame_hash,
|
|
last_frame_hash=last_frame_hash,
|
|
strict_compatibility=strict_compatibility,
|
|
reference_contract=reference_contract,
|
|
conditioning_mode=conditioning_mode,
|
|
upstream_graph_contract=graph_contract,
|
|
upstream_graph_safe=graph_safe,
|
|
upstream_graph_reasons=graph_reasons,
|
|
reference_audio_contract=reference_audio_contract,
|
|
reference_audio_vae=reference_audio_vae,
|
|
driving_audio_contract=driving_audio_contract,
|
|
driving_audio_vae=driving_audio_vae,
|
|
reference_video_contract=reference_video_contract,
|
|
timeline_video_contract=timeline_video_contract,
|
|
guide_contract=guide_contract,
|
|
execution_semantics=execution_semantics,
|
|
reference_storage_plan=reference_storage_plan,
|
|
)
|
|
self.contract = contract
|
|
if self.review_generation_mode is not None or self.review_action is not None:
|
|
validated_prefix = self._load_validated_review_prefix(
|
|
contract,
|
|
resume_safe=safe,
|
|
)
|
|
contract, effective_nonce, nonce_decision = self._resolve_review_contract(
|
|
contract,
|
|
requested_nonce=int(reroll_nonce),
|
|
resume_safe=safe,
|
|
validated_prefix=validated_prefix,
|
|
)
|
|
else:
|
|
if safe:
|
|
self._plan_import = self._find_plan_import(
|
|
contract, stop_before=int(reroll_from_chunk),
|
|
)
|
|
inherited_import = self._plan_import is not None and int(reroll_from_chunk) == 0
|
|
requested = (self._plan_import["effective_reroll_nonce"]
|
|
if inherited_import else int(reroll_nonce))
|
|
if inherited_import:
|
|
contract = self._plan_import["contract"]
|
|
effective_nonce, nonce_decision = self._resolve_effective_nonce(
|
|
contract, requested_nonce=requested, resume_safe=safe,
|
|
)
|
|
contract = _apply_nonce_contract(
|
|
contract, requested_nonce=requested, effective_nonce=effective_nonce,
|
|
)
|
|
self.resume_safe = bool(safe)
|
|
self.effective_reroll_nonce = int(effective_nonce)
|
|
self.nonce_decision = nonce_decision
|
|
if not safe:
|
|
contract = dict(contract)
|
|
contract["reuse_policy"] = "disabled_unobservable_contract"
|
|
contract["execution_nonce"] = uuid.uuid4().hex
|
|
self.disabled_reasons = reasons
|
|
self.contract = contract
|
|
sampling_revision_id, self.contract_sha256 = revision_identity(contract)
|
|
# A selected, shorter canonical Take is not interchangeable with an old
|
|
# longer revision having the same sampling inputs. Continue in its own
|
|
# storage revision when the contract is unchanged; preserve old Takes.
|
|
head_manifest = (self.review_head or {}).get("manifest")
|
|
if (not self.storage_revision_id_override and self._plan_import is None
|
|
and isinstance(head_manifest, dict)
|
|
and head_manifest.get("contract_sha256") == self.contract_sha256):
|
|
self.storage_revision_id_override = str(head_manifest["revision_id"])
|
|
self.revision_id = self.storage_revision_id_override or _storage_revision_identity(
|
|
contract_sha256=self.contract_sha256,
|
|
take_action=TAKE_ACTION_AUTOMATIC,
|
|
selected_revision_id="",
|
|
effective_nonce=int(effective_nonce),
|
|
branch_boundary=int(contract.get("reroll_from_chunk", 0)),
|
|
)
|
|
|
|
self.revision_root = self.revisions_root / self.revision_id
|
|
self.revision_root.mkdir(parents=True, exist_ok=True)
|
|
exact = None
|
|
if self._manifest_path().exists():
|
|
try:
|
|
exact = _read_json(self._manifest_path())
|
|
except Exception as exc:
|
|
self.notes.append(f"exact manifest rejected: {exc}")
|
|
if exact is not None and exact.get("contract_sha256") != self.contract_sha256:
|
|
raise RunStorageError(f"short revision id collision: {self.revision_id}")
|
|
|
|
hashes = list(contract["chunk_contract_hashes"])
|
|
best_entries: list[dict[str, Any]] = []
|
|
best_records: list[dict[str, Any]] = []
|
|
review_queue = self.review_generation_mode is not None or self.review_action is not None
|
|
if exact is not None and not review_queue:
|
|
self._plan_import = None
|
|
if review_queue:
|
|
if self.validated_prefix is None:
|
|
raise RunStorageError("Review Decision has no fixed ValidatedPrefix")
|
|
if self.validated_prefix.entries and not _reusable_sampling_contract(contract, resume_safe=safe):
|
|
raise RunStorageError("Review prefix has no reusable Sampling v6 identity")
|
|
best_entries = list(self.validated_prefix.entries)
|
|
best_records = list(self.validated_prefix.records)
|
|
if self._plan_import is not None:
|
|
# Imported raw tensors become records owned by the new plan only
|
|
# after the initial manifest has been created.
|
|
best_records = []
|
|
execution = self.review_execution
|
|
if execution is None:
|
|
raise RunStorageError("Review Decision was not created")
|
|
if bool(execution.smart_regenerate) or self.review_manual_regenerate_from:
|
|
reusable = max(0, int(execution.effective_regenerate_from) - 1)
|
|
best_entries = best_entries[:reusable]
|
|
if self._plan_import is None:
|
|
best_records = best_records[:reusable]
|
|
elif self._plan_import is not None:
|
|
best_entries = list(self._plan_import["entries"])
|
|
best_records = [] # Populated with new-plan-owned copies after manifest creation.
|
|
else:
|
|
candidates = [exact] if exact is not None and safe else []
|
|
if safe and exact is None and self.revisions_root.exists():
|
|
if isinstance(head_manifest, dict):
|
|
candidates.append(head_manifest)
|
|
else:
|
|
candidates.extend(self._read_manifests().values())
|
|
candidates.sort(key=lambda m: (len(m.get("chunks") or []),
|
|
int((m.get("branch_provenance") or {}).get("canonical_sequence", 0))), reverse=True)
|
|
for candidate in candidates:
|
|
if not _reusable_sampling_contract(contract, resume_safe=safe) or not _reusable_manifest(candidate):
|
|
self.notes.append(f"stored revision {candidate.get('revision_id', '')} is read-only or unobservable")
|
|
continue
|
|
if len(candidate.get("chunks") or []) <= len(best_entries):
|
|
continue
|
|
if int(candidate.get("run_storage_schema_version", -1)) not in RUN_STORAGE_READABLE_SCHEMA_VERSIONS:
|
|
continue
|
|
if (
|
|
int(candidate.get("run_storage_schema_version", -1))
|
|
== RUN_STORAGE_SCHEMA_VERSION
|
|
and candidate.get("branch_provenance")
|
|
):
|
|
try:
|
|
entries, records = self._valid_provenance_prefix(
|
|
candidate,
|
|
current_contract=contract,
|
|
enforce_sampling_contract=True,
|
|
)
|
|
except Exception as exc:
|
|
self.notes.append(
|
|
f"provenance prefix {candidate.get('revision_id', '')} rejected: {exc}"
|
|
)
|
|
continue
|
|
else:
|
|
entries, records = self._valid_prefix(candidate, hashes)
|
|
if len(entries) > len(best_entries):
|
|
best_entries, best_records = entries, records
|
|
|
|
self.reused_count = len(best_entries)
|
|
now = _now()
|
|
self.manifest = {
|
|
"run_storage_schema_version": RUN_STORAGE_SCHEMA_VERSION,
|
|
"sampling_contract_version": SAMPLING_CONTRACT_VERSION,
|
|
"run_name": self.run_name,
|
|
"revision_id": self.revision_id,
|
|
"sampling_revision_id": sampling_revision_id,
|
|
"contract_sha256": self.contract_sha256,
|
|
"contract": contract,
|
|
"resume_safe": bool(safe),
|
|
"resume_disabled_reasons": list(reasons),
|
|
"nonce_lifecycle": dict(contract["nonce_lifecycle"]),
|
|
"status": "in_progress",
|
|
"created_utc": (exact or {}).get("created_utc", now),
|
|
"updated_utc": now,
|
|
"chunks": best_records,
|
|
"report_summary": (exact or {}).get("report_summary", ""),
|
|
}
|
|
if reference_storage_plan is not None:
|
|
self.manifest["reference_routing_audit"] = (
|
|
(exact or {}).get("reference_routing_audit")
|
|
or reference_storage_plan.audit()
|
|
)
|
|
if exact is not None and isinstance(exact.get("prefix_import"), dict):
|
|
self.manifest["prefix_import"] = dict(exact["prefix_import"])
|
|
if self.selected_take_chain:
|
|
self.manifest["branch_provenance"] = {
|
|
"version": BRANCH_PROVENANCE_VERSION,
|
|
"active_chain": [
|
|
item["revision_id"] for item in self.selected_take_chain
|
|
],
|
|
"selection": {
|
|
"action": self.take_action,
|
|
"revision_id": self.take_revision_id,
|
|
"physical_group": self.take_group,
|
|
},
|
|
"branch_cut": dict(self.pending_branch_cut or {}),
|
|
}
|
|
if self.review_execution is not None:
|
|
self.manifest.update(
|
|
review_control_version=REVIEW_CONTROL_VERSION,
|
|
branch_regenerate_from=int(contract["reroll_from_chunk"]),
|
|
effective_reroll_nonce=int(effective_nonce),
|
|
)
|
|
self._write_manifest()
|
|
if self._plan_import is not None:
|
|
best_records = self._adopt_plan_prefix(self._plan_import)
|
|
self._plan_import = None
|
|
self._write_project()
|
|
if not best_entries:
|
|
return None
|
|
session_settings = _resume_session_settings(
|
|
contract=contract,
|
|
revision_id=self.revision_id,
|
|
first_frame_hash=first_frame_hash,
|
|
last_frame_hash=last_frame_hash,
|
|
reference_storage_plan=reference_storage_plan,
|
|
accepted_chunks=len(best_entries),
|
|
)
|
|
return make_session(
|
|
chunks=best_entries, width=int(width), height=int(height),
|
|
chunk_seconds=float(chunk_seconds), identity_hash=str(identity_hash),
|
|
model_fingerprint_value=str(model_fingerprint_value),
|
|
parent_session_id=None, reroll_from_chunk=0,
|
|
settings=session_settings,
|
|
)
|
|
|
|
def mark_review_group(
|
|
self,
|
|
*,
|
|
start: int,
|
|
end: int,
|
|
physical_group: int,
|
|
) -> None:
|
|
"""Record an atomic group only after every logical commit has succeeded."""
|
|
|
|
execution = self.review_execution
|
|
if execution is None or self.review_generation_mode != GENERATION_MODE_REVIEW:
|
|
return
|
|
actual = (int(start), int(end), int(physical_group))
|
|
if execution.max_new_physical_groups is not None:
|
|
expected = (
|
|
execution.next_review_unit_start,
|
|
execution.next_review_unit_end,
|
|
execution.next_review_physical_group,
|
|
)
|
|
if expected != actual:
|
|
raise RunStorageError(
|
|
f"completed review group {actual} does not match resolved unit {expected}"
|
|
)
|
|
if self.pending_review_pause_metadata is not None:
|
|
raise RunStorageError(
|
|
"more than one review group completed in one execution"
|
|
)
|
|
self.pending_review_pause_metadata = make_review_pause_metadata(
|
|
ReviewUnit(
|
|
start=actual[0],
|
|
end=actual[1],
|
|
physical_group=actual[2],
|
|
)
|
|
)
|
|
|
|
def review_pause_metadata(self) -> dict[str, Any] | None:
|
|
if self.review_generation_mode != GENERATION_MODE_REVIEW:
|
|
return None
|
|
if self.pending_review_pause_metadata is not None:
|
|
return dict(self.pending_review_pause_metadata)
|
|
if self.review_execution is None or self.inherited_review_unit is None:
|
|
return None
|
|
unit = self.inherited_review_unit
|
|
return make_review_pause_metadata(
|
|
ReviewUnit(
|
|
start=int(unit["start"]),
|
|
end=int(unit["end"]),
|
|
physical_group=int(unit["physical_group"]),
|
|
)
|
|
)
|
|
|
|
def commit_chunk(self, entry: dict[str, Any], *, position: int) -> None:
|
|
"""Compatibility entry point for one-logical-chunk physical groups."""
|
|
|
|
self.commit_group((entry,), positions=(position,))
|
|
|
|
def commit_group(
|
|
self,
|
|
entries: tuple[dict[str, Any], ...] | list[dict[str, Any]],
|
|
*,
|
|
positions: tuple[int, ...] | list[int],
|
|
) -> None:
|
|
"""Durably publish one complete physical group with one manifest switch."""
|
|
|
|
if self.manifest is None or self.revision_root is None:
|
|
return
|
|
entry_values = tuple(entries)
|
|
position_values = tuple(positions)
|
|
if not entry_values or len(entry_values) != len(position_values):
|
|
raise RunStorageError(
|
|
"Run Storage physical group entries and positions must be non-empty and equal"
|
|
)
|
|
if any(
|
|
isinstance(position, bool) or not isinstance(position, int)
|
|
for position in position_values
|
|
):
|
|
raise RunStorageError("Run Storage chunk positions must be integers")
|
|
expected = int((self.contract or {}).get("chunk_count", 0))
|
|
if any(position < 0 or position >= expected for position in position_values):
|
|
raise RunStorageError(
|
|
"Run Storage physical group position is outside "
|
|
f"0..{expected - 1}: {position_values}"
|
|
)
|
|
groups = physical_groups(
|
|
chunks=expected,
|
|
terminal_merge_enabled=_terminal_merge_enabled(self.contract or {}),
|
|
)
|
|
matched_group = next(
|
|
(
|
|
group
|
|
for group in groups
|
|
if position_values == tuple(range(group.start - 1, group.end))
|
|
),
|
|
None,
|
|
)
|
|
if matched_group is None:
|
|
raise RunStorageError(
|
|
"Run Storage physical group is incomplete or crosses a group boundary: "
|
|
f"{position_values}"
|
|
)
|
|
validated_entries = tuple(
|
|
validate_chunk_entry(entry) for entry in entry_values
|
|
)
|
|
existing_records = list(self.manifest.get("chunks") or [])
|
|
first_position = position_values[0]
|
|
if len(existing_records) < first_position:
|
|
raise RunStorageError(
|
|
"Run Storage cannot commit physical group starting at chunk "
|
|
f"{first_position + 1} before chunk {first_position}"
|
|
)
|
|
chunks_root = self.revision_root / "chunks"
|
|
chunks_root.mkdir(parents=True, exist_ok=True)
|
|
revision_id = str(self.revision_id or "")
|
|
if not revision_id or not re.fullmatch(r"[A-Za-z0-9._-]+", revision_id):
|
|
raise RunStorageError("Run Storage revision id is invalid for raw storage")
|
|
transaction_id = uuid.uuid4().hex
|
|
new_records: list[dict[str, Any]] = []
|
|
for validated, position in zip(
|
|
validated_entries, position_values, strict=True
|
|
):
|
|
filename = (
|
|
f"rev-{revision_id}-txn-{transaction_id}-"
|
|
f"chunk-{position + 1:04d}.safetensors"
|
|
)
|
|
target = chunks_root / filename
|
|
temporary = chunks_root / f".{filename}.{uuid.uuid4().hex}.tmp"
|
|
if target.exists():
|
|
raise RunStorageError(
|
|
f"immutable Run Storage raw already exists: {filename}"
|
|
)
|
|
tensors = {
|
|
"video": validated["video"].detach().to("cpu").contiguous(),
|
|
"audio": validated["audio"].detach().to("cpu").contiguous(),
|
|
}
|
|
try:
|
|
save_file(tensors, str(temporary), metadata={
|
|
"h3_continuum_run_storage": str(RUN_STORAGE_SCHEMA_VERSION),
|
|
"revision_id": revision_id,
|
|
"chunk_number": str(position + 1),
|
|
})
|
|
_fsync_file(temporary)
|
|
os.replace(temporary, target)
|
|
finally:
|
|
if temporary.exists():
|
|
temporary.unlink()
|
|
file_size = target.stat().st_size
|
|
file_sha256 = _file_sha256(target)
|
|
with safe_open(str(target), framework="pt", device="cpu") as handle:
|
|
if set(handle.keys()) != {"audio", "video"}:
|
|
raise RunStorageError(
|
|
f"persisted physical group tensors are invalid: {filename}"
|
|
)
|
|
metadata = handle.metadata() or {}
|
|
if (
|
|
metadata.get("h3_continuum_run_storage")
|
|
!= str(RUN_STORAGE_SCHEMA_VERSION)
|
|
or metadata.get("revision_id") != revision_id
|
|
or metadata.get("chunk_number") != str(position + 1)
|
|
):
|
|
raise RunStorageError(
|
|
f"persisted physical group metadata is invalid: {filename}"
|
|
)
|
|
if (
|
|
target.stat().st_size != file_size
|
|
or _file_sha256(target) != file_sha256
|
|
):
|
|
raise RunStorageError(
|
|
f"persisted physical group SHA-256 verification failed: {filename}"
|
|
)
|
|
storage_entry = dict(validated)
|
|
storage_entry["sequence_index"] = position
|
|
new_records.append({
|
|
"sequence_index": position,
|
|
"storage_revision_id": revision_id,
|
|
"filename": filename,
|
|
"file_size": file_size,
|
|
"file_sha256": file_sha256,
|
|
"entry": _entry_metadata(storage_entry),
|
|
})
|
|
_fsync_dir(chunks_root)
|
|
records = existing_records[:first_position]
|
|
records.extend(new_records)
|
|
candidate_manifest = dict(self.manifest)
|
|
candidate_manifest.update(
|
|
chunks=records,
|
|
updated_utc=_now(),
|
|
status="in_progress",
|
|
)
|
|
# Assign only after every raw in the physical group is complete and
|
|
# verified. __exit__ may then recover this complete candidate after a
|
|
# catchable manifest-write exception, preserving the R0 contract.
|
|
self.manifest = candidate_manifest
|
|
self._write_manifest()
|
|
self.generated_count += len(new_records)
|
|
|
|
def summary(self, *, detailed: bool = False) -> str:
|
|
total = int((self.contract or {}).get("chunk_count", 0))
|
|
resume = self.reused_count + 1 if self.reused_count < total else "complete"
|
|
policy = (" auto-resume disabled: " + "; ".join(self.disabled_reasons) + ".") if self.disabled_reasons else ""
|
|
shown_notes = self.notes if detailed else list(dict.fromkeys(self.notes))[:3]
|
|
note = (" " + " ".join(shown_notes)) if shown_notes else ""
|
|
if not detailed and len(self.notes) > len(shown_notes):
|
|
note += f" ({len(self.notes) - len(shown_notes)} additional diagnostics; enable Detailed Report.)"
|
|
basic = (
|
|
f"Run Storage: {self.run_name} / revision {self.revision_id}; "
|
|
f"{self.reused_count} reused, {self.generated_count} generated, "
|
|
f"{total} total; resume={resume}; nonce={self.effective_reroll_nonce} "
|
|
f"({self.nonce_decision}).{policy}{note}"
|
|
)
|
|
if not detailed or self.manifest is None:
|
|
return basic
|
|
records = list(self.manifest.get("chunks") or [])
|
|
lines = [basic, f"Run Storage path: {self.revision_root}",
|
|
"Run Storage I/O: " + _canonical(self.io_metrics)]
|
|
for record in records:
|
|
lines.append(
|
|
f"stored chunk {int(record['sequence_index']) + 1}: "
|
|
f"{record['storage_revision_id']}/chunks/{record['filename']} "
|
|
f"({int(record['file_size']) / (1024.0 ** 2):.1f} MiB)"
|
|
)
|
|
try:
|
|
free_gib = shutil.disk_usage(self.run_root).free / (1024.0 ** 3)
|
|
lines.append(f"Run Storage free disk: {free_gib:.2f} GiB")
|
|
except OSError:
|
|
lines.append("Run Storage free disk: unknown")
|
|
first_invalid = self.reused_count + 1 if self.reused_count < total else "none"
|
|
lines.append(f"Run Storage first regenerated chunk: {first_invalid}")
|
|
return "\n".join(lines)
|
|
|
|
def finalize(
|
|
self,
|
|
*,
|
|
session: dict[str, Any],
|
|
report: str,
|
|
review_pause_metadata: dict[str, Any] | None = None,
|
|
) -> None:
|
|
if self.manifest is None:
|
|
return
|
|
expected = int((self.contract or {}).get("chunk_count", 0))
|
|
records = list(self.manifest.get("chunks") or [])
|
|
entries, accepted = self._valid_prefix(
|
|
self.manifest,
|
|
list((self.contract or {}).get("chunk_contract_hashes") or []),
|
|
current_contract=self.contract,
|
|
)
|
|
if len(accepted) != len(records):
|
|
raise RunStorageError("Run Storage finalize rejected the committed prefix")
|
|
completed = len(entries)
|
|
if completed > expected:
|
|
raise RunStorageError("Run Storage finalize found too many committed chunks")
|
|
session_chunks = list(session.get("chunks") or [])
|
|
if len(session_chunks) != completed:
|
|
raise RunStorageError(
|
|
"Run Storage finalize session does not match the committed prefix"
|
|
)
|
|
|
|
canonical_pause = None
|
|
if review_pause_metadata is not None:
|
|
value = review_pause_metadata.get("review_unit")
|
|
if not isinstance(value, dict):
|
|
raise RunStorageError("review pause metadata is missing review_unit")
|
|
canonical_pause = make_review_pause_metadata(
|
|
ReviewUnit(
|
|
start=int(value.get("start", 0)),
|
|
end=int(value.get("end", 0)),
|
|
physical_group=int(value.get("physical_group", 0)),
|
|
)
|
|
)
|
|
if review_pause_metadata.get("review_pause_reason") != canonical_pause[
|
|
"review_pause_reason"
|
|
]:
|
|
raise RunStorageError("review pause reason is invalid")
|
|
semantics = (self.contract or {}).get("global", {}).get(
|
|
"execution_semantics"
|
|
) or {}
|
|
terminal_merge_enabled = (
|
|
semantics.get("flf_execution") == "terminal_merged_10s_seed_v2"
|
|
)
|
|
candidate_status = (
|
|
REVISION_STATUS_COMPLETE
|
|
if completed == expected
|
|
else REVISION_STATUS_REVIEW_READY
|
|
)
|
|
try:
|
|
validate_review_prefix_metadata(
|
|
configured_chunks=expected,
|
|
validated_prefix_count=completed,
|
|
terminal_merge_enabled=terminal_merge_enabled,
|
|
terminal_pair_start=(
|
|
expected - 1 if terminal_merge_enabled else None
|
|
),
|
|
latest_review_unit=canonical_pause["review_unit"],
|
|
latest_revision_status=candidate_status,
|
|
latest_effective_nonce=int(self.effective_reroll_nonce),
|
|
latest_branch_regenerate_from=int(
|
|
(self.contract or {}).get("reroll_from_chunk", 0)
|
|
),
|
|
)
|
|
except ReviewControlError as exc:
|
|
raise RunStorageError(f"review pause metadata is invalid: {exc}") from exc
|
|
|
|
status = REVISION_STATUS_COMPLETE if completed == expected else REVISION_STATUS_INTERRUPTED
|
|
intentional_review_pause = (
|
|
completed < expected
|
|
and canonical_pause is not None
|
|
and self.review_execution is not None
|
|
and self.review_execution.max_new_physical_groups is not None
|
|
and (
|
|
self.pending_review_pause_metadata is not None
|
|
or self.review_execution.max_new_physical_groups == 0
|
|
)
|
|
)
|
|
if intentional_review_pause:
|
|
status = REVISION_STATUS_REVIEW_READY
|
|
|
|
for key in ("review_unit", "review_pause_reason"):
|
|
self.manifest.pop(key, None)
|
|
if canonical_pause is not None and status in {
|
|
REVISION_STATUS_REVIEW_READY,
|
|
REVISION_STATUS_COMPLETE,
|
|
}:
|
|
self.manifest.update(canonical_pause)
|
|
if self.review_execution is not None:
|
|
self.manifest.update(
|
|
review_control_version=REVIEW_CONTROL_VERSION,
|
|
branch_regenerate_from=int(
|
|
(self.contract or {}).get("reroll_from_chunk", 0)
|
|
),
|
|
effective_reroll_nonce=int(self.effective_reroll_nonce),
|
|
)
|
|
self.manifest.pop("last_error", None)
|
|
pending_provenance = dict(self.manifest.pop("branch_provenance", {}) or {})
|
|
self.manifest.update(
|
|
status=status, updated_utc=_now(),
|
|
session_id=str(session.get("session_id", "")),
|
|
report_summary=str(report)[-8192:],
|
|
)
|
|
self._write_manifest()
|
|
manifests = self._read_manifests()
|
|
try:
|
|
active_chain = self._manifest_provenance_chain(self.manifest, manifests)
|
|
except (BranchProvenanceError, RunStorageError) as exc:
|
|
raise RunStorageError(
|
|
f"finalized Branch Provenance Contract is invalid: {exc}"
|
|
) from exc
|
|
canonical_sequence = 1 + max(
|
|
(
|
|
int((value.get("branch_provenance") or {}).get("canonical_sequence", 0))
|
|
for value in manifests.values()
|
|
),
|
|
default=0,
|
|
)
|
|
provenance = pending_provenance
|
|
provenance.update(
|
|
version=BRANCH_PROVENANCE_VERSION,
|
|
active_chain=[item["revision_id"] for item in active_chain],
|
|
canonical_committed_utc=_now(),
|
|
canonical_sequence=canonical_sequence,
|
|
)
|
|
if self.pending_branch_cut is not None:
|
|
provenance["branch_cut"] = dict(self.pending_branch_cut)
|
|
self.manifest["branch_provenance"] = provenance
|
|
self._write_manifest()
|
|
self._write_project(set_canonical=True)
|
|
|
|
|
|
def get_active_run_storage() -> RunStorageController | None:
|
|
return _ACTIVE.get()
|
|
|
|
|
|
def run_storage_scope(
|
|
run_name: str, *, prompt: Any = None, unique_id: Any = None
|
|
) -> RunStorageController:
|
|
controller = RunStorageController(run_name)
|
|
controller.set_prompt_graph(prompt, unique_id)
|
|
return controller
|