Publish V3.9 source with per-chunk references and retained V3.8X2 workflows

This commit is contained in:
ukr8b3g-cmyk
2026-09-29 01:53:22 +09:00
parent 55fc2d545f
commit 0582db62ef
67 changed files with 16505 additions and 1916 deletions
+146 -7
View File
@@ -54,6 +54,13 @@ from .v3.review_control import (
validate_review_prefix_metadata,
)
from .v3.planning_types import PrefixFacts
from .v3.reference_storage_contract import (
REFERENCE_ROUTING_CONTRACT_VERSION,
ReferenceStoragePlan,
ReferenceStorageContractError,
session_routing_settings,
validate_reference_group_contracts,
)
RUN_STORAGE_SCHEMA_VERSION = 3
@@ -731,6 +738,12 @@ def _apply_nonce_contract(
global_hash = _hash(result["global"])
chunk_contracts = []
chunk_hashes = []
routed_groups = result.get("reference_group_contracts")
routed_by_chunk = {}
if routed_groups is not None:
for group_record in routed_groups:
for chunk in group_record["descriptor"]["logical_chunks"]:
routed_by_chunk[int(chunk)] = group_record["sha256"]
for position, prompt_hash in enumerate(result["prompt_hashes"]):
number = position + 1
affected = boundary > 0 and number >= boundary
@@ -742,6 +755,8 @@ def _apply_nonce_contract(
"effective_reroll_nonce": int(effective_nonce) if affected else 0,
"last_frame_hash": str(result["last_frame_hash"]) if number == int(result["chunk_count"]) else "",
}
if routed_groups is not None:
chunk_contract["physical_group_reference_sha256"] = routed_by_chunk[number]
timeline_chunks = result.get("timeline_video_chunk_contracts") or []
if position < len(timeline_chunks):
chunk_contract["timeline_video"] = dict(timeline_chunks[position])
@@ -769,7 +784,7 @@ def _nonce_lineage_hash(
) -> str:
"""Recompute the nonce-independent compatible sampling lineage."""
return _hash({
identity = {
"global_hash": _hash(contract["global"]),
"chunk_count": int(contract["chunk_count"]),
"prompt_mode": str(contract["prompt_mode"]),
@@ -780,7 +795,12 @@ def _nonce_lineage_hash(
if timeline_video_chunk_contracts is not None
else contract.get("timeline_video_chunk_contracts") or []
),
})
}
if "reference_group_contracts" in contract:
identity["reference_group_hashes"] = [
record["sha256"] for record in contract["reference_group_contracts"]
]
return _hash(identity)
def _apply_reroll_branch_contract(
@@ -864,7 +884,10 @@ def build_sampling_contract(
timeline_video_contract: dict[str, Any] | None = None,
guide_contract: dict[str, Any] | None = None,
execution_semantics: dict[str, Any] | None = None,
reference_storage_plan: ReferenceStoragePlan | None = None,
) -> tuple[dict[str, Any], bool, list[str]]:
if reference_storage_plan is not None and reference_contract is not None:
raise RunStorageError("routed Reference plan cannot use legacy Reference contract")
last_frame_hash = _canonical_optional_hash(last_frame_hash)
model_value, model_safe = _model_signature(model, model_fingerprint_value)
clip_value, clip_safe = _clip_signature(clip)
@@ -884,6 +907,8 @@ def build_sampling_contract(
f"declared={conditioning_mode}, inferred={inferred_mode}"
)
uses_video_vae = (
reference_storage_plan is not None
or
conditioning_mode_uses_video_vae(conditioning_mode)
or reference_video_contract is not None
or timeline_video_contract is not None
@@ -956,7 +981,12 @@ def build_sampling_contract(
if graph_authoritative:
global_contract["upstream_graph"] = dict(upstream_graph_contract)
if execution_semantics is not None:
global_contract["execution_semantics"] = dict(execution_semantics)
semantics = dict(execution_semantics)
if reference_storage_plan is not None:
semantics.pop("terminal_prompt_policy", None)
global_contract["execution_semantics"] = semantics
if reference_storage_plan is not None:
global_contract["reference_routing_contract_version"] = REFERENCE_ROUTING_CONTRACT_VERSION
if uses_video_vae:
global_contract["video_vae"] = video_vae_value
if has_first:
@@ -1006,6 +1036,15 @@ def build_sampling_contract(
"reroll_from_chunk": boundary,
"last_frame_hash": str(last_frame_hash),
}
if reference_storage_plan is not None:
routed_groups = reference_storage_plan.group_contracts
validate_reference_group_contracts(
routed_groups, chunk_count=chunks,
terminal_merge_enabled=(
(execution_semantics or {}).get("flf_execution") == "terminal_merged_10s_seed_v2"
),
)
contract["reference_group_contracts"] = routed_groups
lineage_sha256 = _nonce_lineage_hash(
contract,
timeline_video_chunk_contracts=timeline_chunk_contracts,
@@ -1151,6 +1190,8 @@ def _resume_session_settings(
revision_id: str,
first_frame_hash: str,
last_frame_hash: str,
reference_storage_plan: ReferenceStoragePlan | None = None,
accepted_chunks: int = 0,
) -> dict[str, Any]:
settings = {
"run_storage_validated_prefix": True,
@@ -1161,6 +1202,10 @@ def _resume_session_settings(
settings.update(
dict((contract.get("global") or {}).get("execution_semantics") or {})
)
if reference_storage_plan is not None:
settings["reference_routing_v1"] = session_routing_settings(
reference_storage_plan, accepted_chunks,
)
return settings
@@ -1173,6 +1218,28 @@ def _manifest_sampling_identity(manifest: dict[str, Any]) -> tuple[str, str]:
contract = manifest.get("contract")
if not isinstance(contract, dict):
raise RunStorageError("stored sampling contract is missing")
routed_version = (contract.get("global") or {}).get("reference_routing_contract_version")
routed_groups = contract.get("reference_group_contracts")
if routed_version is not None or routed_groups is not None:
if routed_version != REFERENCE_ROUTING_CONTRACT_VERSION:
raise RunStorageError("stored Reference routing contract version is incompatible")
try:
groups = validate_reference_group_contracts(
routed_groups, chunk_count=int(contract["chunk_count"]),
terminal_merge_enabled=_terminal_merge_enabled(contract),
)
except (ReferenceStorageContractError, KeyError, TypeError, ValueError) as exc:
raise RunStorageError(f"stored Reference group contract is invalid: {exc}") from exc
expected = {
int(chunk): record["sha256"]
for record in groups for chunk in record["descriptor"]["logical_chunks"]
}
chunks = contract.get("chunk_contracts")
if (not isinstance(chunks, list) or len(chunks) != len(expected)
or any(not isinstance(item, dict) or
item.get("physical_group_reference_sha256") != expected[index + 1]
for index, item in enumerate(chunks))):
raise RunStorageError("stored Reference group/chunk contracts disagree")
contract_revision_id, contract_sha256 = revision_identity(contract)
if str(manifest.get("contract_sha256", "")) != contract_sha256:
raise RunStorageError("stored sampling contract SHA-256 is invalid")
@@ -1730,6 +1797,8 @@ class RunStorageController:
if producer is None:
compatible = False
break
if "reference_group_contracts" in (producer.get("contract") or {}):
_manifest_sampling_identity(producer)
producing = producer.get("contract") or {}
hashes = producing.get("chunk_contract_hashes") or []
# Accepted earlier Takes can come from several nonce branches.
@@ -1843,6 +1912,10 @@ class RunStorageController:
for position, original in enumerate(imported["records"])
],
}
if imported.get("selected_take_import") is not None:
self.manifest["prefix_import"]["selected_take"] = dict(
imported["selected_take_import"]
)
self._write_manifest()
records = []
imported_entries = list(imported["entries"])
@@ -1863,6 +1936,16 @@ class RunStorageController:
for record in self.manifest["chunks"][-len(positions):]
)
self.generated_count = 0 # Copies are reused work, never new Sampling.
if imported.get("selected_take_import") is not None:
local_chain = self._manifest_provenance_chain(
self.manifest, self._read_manifests(),
)
if not local_chain:
raise RunStorageError("imported Take has no new-plan provenance")
self.pending_branch_cut = {
"selected_revision_id": local_chain[-1]["revision_id"],
"after_physical_group": int(local_chain[-1]["group"]["physical_group"]),
}
self.notes.append(f"Imported {len(records)} verified prefix chunks into the new plan; originals preserved")
return records
@@ -1878,8 +1961,9 @@ class RunStorageController:
def _validated_take_selection(
self,
contract: dict[str, Any],
*, source_lineage: str | None = None,
) -> dict[str, Any]:
lineage = str(contract.get("nonce_lineage_sha256", ""))
lineage = str(source_lineage or contract.get("nonce_lineage_sha256", ""))
manifests, catalog, _ = self._provenance_catalog(
lineage_sha256=lineage,
)
@@ -2252,9 +2336,53 @@ class RunStorageController:
raise RunStorageError(
"Take selection cannot be combined with manual Regenerate From"
)
candidate = self._validated_take_selection(contract)
self.selected_take_chain = list(candidate["chain"])
self.selected_take_records = list(candidate["records"])
source_lineage = str(contract.get("nonce_lineage_sha256", ""))
if "reference_group_contracts" in contract:
_, all_takes, _ = self._provenance_catalog()
selected_take = all_takes.get(self.take_revision_id)
if selected_take is not None:
source_lineage = str(selected_take.get("lineage_sha256", ""))
candidate = self._validated_take_selection(
contract, source_lineage=source_lineage,
)
if source_lineage != str(contract.get("nonce_lineage_sha256", "")):
source_manifest = candidate.get("manifest")
if not isinstance(source_manifest, dict):
raise RunStorageError("selected Take source manifest is unavailable")
source_contract = source_manifest.get("contract") or {}
if _hash(source_contract.get("global")) != _hash(contract.get("global")):
raise RunStorageError("selected Take has a different common execution contract")
manifests = self._read_manifests()
source_chain = self._manifest_provenance_chain(source_manifest, manifests)
selected_chain = candidate["chain"]
if ([item["revision_id"] for item in source_chain[:len(selected_chain)]]
!= [item["revision_id"] for item in selected_chain]):
raise RunStorageError("selected Take is not the source manifest's validated prefix")
selected_end = int(candidate["validated_prefix_count"])
try:
compatible_entries, compatible_records = self._compatible_plan_prefix(
source_manifest, contract, stop_before=selected_end + 1,
)
except (OSError, ValueError) as exc:
raise RunStorageError(f"selected Take prefix cannot be imported: {exc}") from exc
if (len(compatible_entries) != selected_end
or compatible_records != candidate["records"]):
failed_group = len(compatible_entries) + 1
raise RunStorageError(
f"selected Take is incompatible with the new Reference plan "
f"at physical group starting with chunk {failed_group}; "
"choose an earlier compatible Take or deselect Take and Regenerate From"
)
candidate["entries"] = compatible_entries
candidate["records"] = compatible_records
candidate["selected_take_import"] = {
"revision_id": self.take_revision_id,
"physical_group": self.take_group,
}
self._plan_import = candidate
else:
self.selected_take_chain = list(candidate["chain"])
self.selected_take_records = list(candidate["records"])
self.selected_take_revision = dict(candidate)
else:
smart_only = self.review_action == REVIEW_ACTION_REGENERATE_CURRENT
@@ -2463,6 +2591,7 @@ class RunStorageController:
timeline_video_contract: dict[str, Any] | None = None,
guide_contract: dict[str, Any] | None = None,
execution_semantics: dict[str, Any] | None = None,
reference_storage_plan: ReferenceStoragePlan | None = None,
) -> dict[str, Any] | None:
if existing_session is not None:
raise RunStorageError("Run Storage cannot be combined with an explicit Session")
@@ -2472,6 +2601,8 @@ class RunStorageController:
self.prompt_graph,
self.sampler_node_id,
require_video_vae=(
reference_storage_plan is not None
or
conditioning_mode_uses_video_vae(str(conditioning_mode))
or reference_video_contract is not None
or timeline_video_contract is not None
@@ -2508,6 +2639,7 @@ class RunStorageController:
timeline_video_contract=timeline_video_contract,
guide_contract=guide_contract,
execution_semantics=execution_semantics,
reference_storage_plan=reference_storage_plan,
)
self.contract = contract
if self.review_generation_mode is not None or self.review_action is not None:
@@ -2654,6 +2786,11 @@ class RunStorageController:
"chunks": best_records,
"report_summary": (exact or {}).get("report_summary", ""),
}
if reference_storage_plan is not None:
self.manifest["reference_routing_audit"] = (
(exact or {}).get("reference_routing_audit")
or reference_storage_plan.audit()
)
if exact is not None and isinstance(exact.get("prefix_import"), dict):
self.manifest["prefix_import"] = dict(exact["prefix_import"])
if self.selected_take_chain:
@@ -2687,6 +2824,8 @@ class RunStorageController:
revision_id=self.revision_id,
first_frame_hash=first_frame_hash,
last_frame_hash=last_frame_hash,
reference_storage_plan=reference_storage_plan,
accepted_chunks=len(best_entries),
)
return make_session(
chunks=best_entries, width=int(width), height=int(height),