"""Crash-safe V3.1B raw-latent storage and deterministic auto-resume.""" from __future__ import annotations import contextvars import hashlib import json import marshal import os import re import shutil import socket import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any import torch from safetensors import safe_open from safetensors.torch import save_file from .constants import CONTINUUM_ACTUAL_PREFIX_STEPS from .v2.session import make_session, validate_chunk_entry RUN_STORAGE_SCHEMA_VERSION = 2 from .conditioning import ( CONDITIONING_MODES, conditioning_mode_from_presence, conditioning_mode_uses_video_vae, ) SAMPLING_CONTRACT_VERSION = 5 RUN_STORAGE_OFF = "Off" RUN_STORAGE_AUTO = "Save + Auto Resume" RUN_STORAGE_OPTIONS = (RUN_STORAGE_OFF, RUN_STORAGE_AUTO) _ACTIVE: contextvars.ContextVar["RunStorageController | None"] = contextvars.ContextVar( "h3_continuum_run_storage", default=None ) _RESERVED = { "CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)), *(f"LPT{i}" for i in range(1, 10)), } _ADDRESS = re.compile(r"0x[0-9a-fA-F]+") class RunStorageError(RuntimeError): pass def _now() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat() def _canonical(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def _hash(value: Any) -> str: return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() def sanitize_run_name(value: str) -> str: name = str(value) if not name or name != name.strip() or len(name) > 96: raise RunStorageError("Run Name must be 1-96 characters without surrounding spaces") if any(ord(char) < 32 or char in '<>:"/\\|?*' for char in name): raise RunStorageError("Run Name contains a reserved path character") if name.endswith((".", " ")) or name in {".", ".."}: raise RunStorageError("Run Name must not be a relative path or end with dot/space") if name.split(".", 1)[0].upper() in _RESERVED: raise RunStorageError(f"Run Name {name!r} is reserved by Windows") return name def resolve_run_storage_name( *, project_id: str, legacy_run_name: str = "", automatic_key: str = "" ) -> str: """Resolve a stable storage folder while preserving old Run Name workflows.""" legacy = str(legacy_run_name) if legacy: return sanitize_run_name(legacy) raw_project_id = str(project_id).strip() if not raw_project_id: key = str(automatic_key).strip() if not key: raise RunStorageError( "Automatic Resume ID is unavailable; enter a Run Name for this workflow" ) digest = hashlib.sha256( f"H3ContinuumSamplerProduction:{key}".encode("utf-8") ).hexdigest()[:24] return f"run_auto_{digest}" try: parsed = uuid.UUID(raw_project_id) except (ValueError, AttributeError, TypeError) as exc: raise RunStorageError( "Automatic Project ID is missing or invalid; reload ComfyUI and the workflow" ) from exc if parsed.int == 0: raise RunStorageError("Automatic Project ID must not be the nil UUID") return f"run_{parsed.hex}" def automatic_project_key(prompt: Any, unique_id: Any) -> str: """Build a stable key from sampler identity and graph topology only.""" node_id = str(unique_id).strip() if not node_id: raise RunStorageError("Automatic Resume ID requires the sampler node ID") if not isinstance(prompt, dict): raise RunStorageError( "Automatic Resume ID requires the ComfyUI prompt graph; enter a Run Name" ) def visit(current_id: str, stack: set[str]) -> dict[str, Any]: if current_id in stack: return {"node_id": current_id, "cycle": True} raw = prompt.get(current_id) if not isinstance(raw, dict): return {"node_id": current_id, "missing": True} links = [] inputs = raw.get("inputs") or {} next_stack = set(stack) next_stack.add(current_id) if isinstance(inputs, dict): for name in sorted(inputs): value = inputs[name] if ( isinstance(value, (list, tuple)) and len(value) == 2 and isinstance(value[0], (str, int)) and isinstance(value[1], int) and str(value[0]) in prompt ): child_id = str(value[0]) links.append( { "input": str(name), "output": int(value[1]), "node": visit(child_id, next_stack), } ) return { "node_id": current_id, "class_type": str(raw.get("class_type", "")), "links": links, } return _hash({"sampler_node_id": node_id, "topology": visit(node_id, set())}) def _output_root() -> Path: try: import folder_paths return Path(folder_paths.get_output_directory()) except Exception: return Path.cwd() / "output" def _fsync_file(path: Path) -> None: # Windows requires a writable descriptor for fsync(). The file contents # are already complete at this point; opening r+b does not modify them. with path.open("r+b") as handle: os.fsync(handle.fileno()) def _fsync_dir(path: Path) -> None: if os.name == "nt": return descriptor = os.open(str(path), os.O_RDONLY) try: os.fsync(descriptor) finally: os.close(descriptor) def _write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") try: with temporary.open("w", encoding="utf-8", newline="\n") as handle: json.dump(payload, handle, ensure_ascii=False, sort_keys=True, indent=2) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) _fsync_dir(path.parent) finally: if temporary.exists(): temporary.unlink() def _read_json(path: Path) -> dict[str, Any]: value = json.loads(path.read_text(encoding="utf-8")) if not isinstance(value, dict): raise RunStorageError(f"{path.name} must contain a JSON object") return value def _file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _qualified(value: Any) -> str: if value is None: return "none" target = value if isinstance(value, type) or callable(value) else type(value) module = getattr(target, "__module__", "") name = getattr(target, "__qualname__", getattr(target, "__name__", type(target).__name__)) return f"{module}.{name}" def _tensor_probe(tensor: torch.Tensor) -> dict[str, Any]: flat = tensor.detach().reshape(-1) count = int(flat.numel()) positions = sorted({0, count // 4, count // 2, 3 * count // 4, count - 1}) if count else [] if positions: index = torch.tensor(positions, device=flat.device) sample = flat[index].contiguous().cpu().view(torch.uint8).numpy().tobytes() else: sample = b"" return { "shape": list(tensor.shape), "dtype": str(tensor.dtype), "sample_sha256": hashlib.sha256(sample).hexdigest(), } def _tensor_exact(tensor: torch.Tensor) -> dict[str, Any]: value = tensor.detach().contiguous().cpu() raw = value.view(torch.uint8).numpy().tobytes() return { "shape": list(value.shape), "dtype": str(value.dtype), "exact_sha256": hashlib.sha256(raw).hexdigest(), } def _observe(value: Any, *, depth: int = 0) -> tuple[Any, bool]: if depth > 10: return {"type": _qualified(value), "truncated": True}, False if value is None or isinstance(value, (bool, int, float, str)): return value, True if isinstance(value, Path): return str(value), True if torch.is_tensor(value): try: return {"tensor": _tensor_probe(value)}, True except Exception: return {"tensor_type": _qualified(value)}, False if isinstance(value, dict): result: dict[str, Any] = {} safe = True for key in sorted(value, key=lambda item: str(item)): observed, item_safe = _observe(value[key], depth=depth + 1) result[str(key)] = observed safe = safe and item_safe return result, safe if isinstance(value, (list, tuple)): result = [] safe = True for item in value: observed, item_safe = _observe(item, depth=depth + 1) result.append(observed) safe = safe and item_safe return result, safe if callable(value): code = getattr(value, "__code__", None) descriptor: dict[str, Any] = {"callable": _qualified(value)} safe = True if code is not None: descriptor["code_sha256"] = hashlib.sha256( marshal.dumps(code) ).hexdigest() defaults, defaults_safe = _observe(getattr(value, "__defaults__", None), depth=depth + 1) kwdefaults, kw_safe = _observe(getattr(value, "__kwdefaults__", None), depth=depth + 1) closure_values = tuple(cell.cell_contents for cell in (getattr(value, "__closure__", None) or ())) closure, closure_safe = _observe(closure_values, depth=depth + 1) descriptor.update(defaults=defaults, kwdefaults=kwdefaults, closure=closure) safe = defaults_safe and kw_safe and closure_safe elif hasattr(value, "__dict__"): attributes, safe = _observe(vars(value), depth=depth + 1) descriptor["attributes"] = attributes else: safe = False return descriptor, safe if hasattr(value, "__dict__"): attributes, safe = _observe(vars(value), depth=depth + 1) return {"type": _qualified(value), "attributes": attributes}, safe rendered = _ADDRESS.sub("0x...", repr(value)) return {"type": _qualified(value), "repr": rendered[:256]}, False def _model_signature(model: Any, legacy_fingerprint: str) -> tuple[dict[str, Any], bool]: base = getattr(model, "model", None) inner = getattr(base, "diffusion_model", None) wrappers, wrappers_safe = _observe(getattr(model, "wrappers", {}) or {}) patches, patches_safe = _observe(getattr(model, "patches", {}) or {}) options = getattr(model, "model_options", {}) or {} transformer = options.get("transformer_options", {}) or {} transformer_values = { str(key): transformer[key] for key in transformer if key not in {"h3_continuum_join_context"} } observed_transformer, transformer_safe = _observe(transformer_values) weights: list[dict[str, Any]] = [] weights_safe = True try: parameters = list(inner.named_parameters()) if inner is not None else [] for position in sorted({0, len(parameters) // 2, len(parameters) - 1}) if parameters else []: name, parameter = parameters[position] weights.append({"name": str(name), **_tensor_probe(parameter)}) except Exception: weights_safe = False descriptor = { "legacy_fingerprint": str(legacy_fingerprint), "model_patcher": _qualified(model), "base": _qualified(base), "inner": _qualified(inner), "dtype": str(getattr(model, "model_dtype", lambda: "unknown")()), "model_size": int(getattr(model, "model_size", lambda: 0)() or 0), "wrappers": wrappers, "transformer_options": observed_transformer, "patches": patches, "weight_probe": weights, "runtime_observation": { "wrappers_complete": bool(wrappers_safe), "patches_complete": bool(patches_safe), "transformer_options_complete": bool(transformer_safe), }, } return descriptor, weights_safe and bool(weights) def _sampler_signature(sampler: Any) -> tuple[dict[str, Any], bool]: function = getattr(sampler, "sampler_function", None) observed, safe = _observe({ "extra_options": getattr(sampler, "extra_options", {}) or {}, "inpaint_options": getattr(sampler, "inpaint_options", {}) or {}, }) return { "type": _qualified(sampler), "function": _qualified(function) if function is not None else "unknown", "options": observed, }, function is not None and safe def _module_signature(module: Any) -> tuple[dict[str, Any], bool]: descriptor: dict[str, Any] = {"type": _qualified(module)} if module is None or not hasattr(module, "named_parameters"): descriptor["error"] = "named_parameters unavailable" return descriptor, False try: parameters = list(module.named_parameters()) except Exception as exc: descriptor["error"] = f"named_parameters failed: {type(exc).__name__}" return descriptor, False layout = [ { "name": str(name), "shape": list(parameter.shape), "dtype": str(parameter.dtype), } for name, parameter in parameters ] descriptor.update( tensor_count=len(parameters), parameter_count=sum(int(parameter.numel()) for _, parameter in parameters), parameter_layout_sha256=_hash(layout), ) probes = [] probes_safe = bool(parameters) for position in sorted({0, len(parameters) // 2, len(parameters) - 1}) if parameters else []: name, parameter = parameters[position] try: probes.append({"name": str(name), **_tensor_probe(parameter)}) except Exception as exc: probes.append({"name": str(name), "error": type(exc).__name__}) probes_safe = False descriptor["weight_probe"] = probes return descriptor, probes_safe def _patcher_signature(patcher: Any) -> tuple[dict[str, Any], bool]: if patcher is None: return {"type": "missing"}, False state, safe = _observe({ "wrappers": getattr(patcher, "wrappers", {}) or {}, "patches": getattr(patcher, "patches", {}) or {}, "object_patches": getattr(patcher, "object_patches", {}) or {}, "model_options": getattr(patcher, "model_options", {}) or {}, }) return { "type": _qualified(patcher), "model_type": _qualified(getattr(patcher, "model", None)), "state": state, }, safe def _clip_signature(clip: Any) -> tuple[dict[str, Any], bool]: cond_stage = getattr(clip, "cond_stage_model", None) tokenizer = getattr(clip, "tokenizer", None) tokenizer_wrapper = getattr(tokenizer, "qwen3vl_32b", None) tokenizer_core = getattr(tokenizer_wrapper, "tokenizer", None) module_value, module_safe = _module_signature(cond_stage) patcher_value, patcher_safe = _patcher_signature(getattr(clip, "patcher", None)) options, options_safe = _observe({ "tokenizer_options": getattr(clip, "tokenizer_options", {}) or {}, "layer_idx": getattr(clip, "layer_idx", None), "use_clip_schedule": bool(getattr(clip, "use_clip_schedule", False)), }) tokenizer_config = { "wrapper_type": _qualified(tokenizer), "model_wrapper_type": _qualified(tokenizer_wrapper), "core_type": _qualified(tokenizer_core), } for name in ( "name_or_path", "vocab_size", "model_max_length", "padding_side", "truncation_side", "bos_token_id", "eos_token_id", "pad_token_id", ): value = getattr(tokenizer_core, name, None) if value is None or isinstance(value, (bool, int, float, str)): tokenizer_config[name] = value tokenizer_safe = tokenizer is not None and tokenizer_wrapper is not None and tokenizer_core is not None descriptor = { "wrapper_type": _qualified(clip), "conditioner": module_value, "patcher": patcher_value, "tokenizer": tokenizer_config, "options": options, "runtime_observation": { "patcher_complete": bool(patcher_safe), "options_complete": bool(options_safe), }, } return descriptor, module_safe and tokenizer_safe def _video_vae_signature(video_vae: Any, *, required: bool) -> tuple[dict[str, Any], bool]: if not required: return {"required": False}, True first_stage = getattr(video_vae, "first_stage_model", None) module_value, module_safe = _module_signature(first_stage) patcher_value, patcher_safe = _patcher_signature(getattr(video_vae, "patcher", None)) config = { "required": True, "wrapper_type": _qualified(video_vae), "first_stage": module_value, "patcher": patcher_value, "vae_dtype": str(getattr(video_vae, "vae_dtype", "unknown")), "latent_channels": getattr(video_vae, "latent_channels", None), "downscale_ratio": getattr(video_vae, "downscale_ratio", None), "upscale_ratio": getattr(video_vae, "upscale_ratio", None), "chunked_io": bool(getattr(first_stage, "comfy_has_chunked_io", False)), } observed, config_safe = _observe(config) observed["runtime_observation"] = { "patcher_complete": bool(patcher_safe), "config_complete": bool(config_safe), } return observed, module_safe def _apply_nonce_contract( contract: dict[str, Any], *, requested_nonce: int, effective_nonce: int, ) -> dict[str, Any]: result = dict(contract) boundary = int(result["reroll_from_chunk"]) if boundary <= 0: mode = "inactive" requested_nonce = 0 effective_nonce = 0 else: mode = "explicit" if int(requested_nonce) >= 1 else "auto" global_hash = _hash(result["global"]) chunk_contracts = [] chunk_hashes = [] for position, prompt_hash in enumerate(result["prompt_hashes"]): number = position + 1 affected = boundary > 0 and number >= boundary chunk_contract = { "global_hash": global_hash, "chunk_number": number, "prompt_hash": str(prompt_hash), "reroll_boundary": boundary if affected else 0, "effective_reroll_nonce": int(effective_nonce) if affected else 0, "last_frame_hash": str(result["last_frame_hash"]) if number == int(result["chunk_count"]) else "", } chunk_contracts.append(chunk_contract) chunk_hashes.append(_hash(chunk_contract)) result.update( effective_reroll_nonce=int(effective_nonce), chunk_contracts=chunk_contracts, chunk_contract_hashes=chunk_hashes, nonce_lifecycle={ "mode": mode, "requested_nonce": int(requested_nonce), "effective_nonce": int(effective_nonce), "lineage_sha256": str(result["nonce_lineage_sha256"]), "request_sha256": str(result["nonce_request_sha256"]), }, ) return result def build_sampling_contract( *, model: Any, model_fingerprint_value: str, clip: Any, video_vae: Any, sampler: Any, sigmas: torch.Tensor, prompt_plan: dict[str, Any], width: int, height: int, chunk_seconds: float, continuity: str, audio_continuity: bool, base_seed: int, reroll_from_chunk: int, reroll_nonce: int, first_frame_hash: str, last_frame_hash: str, strict_compatibility: bool, reference_contract: dict[str, Any] | None = None, conditioning_mode: str | None = None, upstream_graph_contract: dict[str, Any] | None = None, upstream_graph_safe: bool | None = None, upstream_graph_reasons: list[str] | None = None, ) -> tuple[dict[str, Any], bool, list[str]]: model_value, model_safe = _model_signature(model, model_fingerprint_value) clip_value, clip_safe = _clip_signature(clip) inferred_mode = conditioning_mode_from_presence( has_first=str(first_frame_hash).lower() not in {"", "none", "null"}, has_last=str(last_frame_hash).lower() not in {"", "none", "null"}, has_reference=reference_contract is not None, ) conditioning_mode = inferred_mode if conditioning_mode is None else str(conditioning_mode) if conditioning_mode not in CONDITIONING_MODES: raise RunStorageError(f"unknown conditioning mode: {conditioning_mode!r}") if conditioning_mode != inferred_mode: raise RunStorageError( "conditioning mode does not match the connected image inputs: " f"declared={conditioning_mode}, inferred={inferred_mode}" ) uses_video_vae = conditioning_mode_uses_video_vae(conditioning_mode) if uses_video_vae: video_vae_value, video_vae_safe = _video_vae_signature( video_vae, required=True ) else: video_vae_value, video_vae_safe = None, True sampler_value, sampler_safe = _sampler_signature(sampler) if not torch.is_tensor(sigmas) or sigmas.ndim != 1: raise RunStorageError("Run Storage requires a one-dimensional SIGMAS tensor") chunks = int(prompt_plan["chunks"]) prompt_hashes = [str(value) for value in prompt_plan["hashes"]] boundary = int(reroll_from_chunk) graph_authoritative = upstream_graph_contract is not None if graph_authoritative: routes = upstream_graph_contract.get("routes", {}) model_value = { "runtime": model_value, "graph_route": routes.get("model"), } clip_value = { "runtime": clip_value, "graph_route": routes.get("clip"), } if uses_video_vae: video_vae_value = { "runtime": video_vae_value, "graph_route": routes.get("video_vae"), } global_contract = { "sampling_contract_version": SAMPLING_CONTRACT_VERSION, "conditioning_mode": conditioning_mode, "model": model_value, "clip": clip_value, "sampler": sampler_value, "sigmas": _tensor_exact(sigmas), "width": int(width), "height": int(height), "chunk_seconds": float(chunk_seconds), "continuity": str(continuity), "audio_continuity": bool(audio_continuity), "base_seed": int(base_seed), "strict_compatibility": bool(strict_compatibility), "continuum_interop_api": 1, "actual_prefix_steps": CONTINUUM_ACTUAL_PREFIX_STEPS, } if graph_authoritative: global_contract["upstream_graph"] = dict(upstream_graph_contract) if uses_video_vae: global_contract["video_vae"] = video_vae_value if conditioning_mode in {"i2va", "fl2va"}: global_contract["first_frame_hash"] = str(first_frame_hash) if reference_contract is not None: global_contract["reference"] = dict(reference_contract) lineage_sha256 = _hash({ "global_hash": _hash(global_contract), "chunk_count": chunks, "prompt_mode": str(prompt_plan["mode"]), "prompt_hashes": prompt_hashes, "last_frame_hash": str(last_frame_hash), }) contract = { "global": global_contract, "chunk_count": chunks, "prompt_mode": str(prompt_plan["mode"]), "prompt_hashes": prompt_hashes, "reroll_from_chunk": boundary, "last_frame_hash": str(last_frame_hash), "nonce_lineage_sha256": lineage_sha256, "nonce_request_sha256": _hash({ "lineage_sha256": lineage_sha256, "reroll_from_chunk": boundary, }), } contract = _apply_nonce_contract( contract, requested_nonce=int(reroll_nonce), effective_nonce=int(reroll_nonce) if boundary > 0 else 0, ) reasons = list(upstream_graph_reasons or []) if graph_authoritative else [] if not model_safe: reasons.append("MODEL wrapper/patch contract is not completely observable") if not sampler_safe: reasons.append("SAMPLER function/options are not completely observable") if not clip_safe: reasons.append("CLIP/Qwen encoder contract is not completely observable") if not video_vae_safe: reasons.append("Video VAE contract is not completely observable") graph_safe = bool(upstream_graph_safe) if graph_authoritative else True return contract, model_safe and clip_safe and video_vae_safe and sampler_safe and graph_safe, reasons def revision_identity(contract: dict[str, Any]) -> tuple[str, str]: full = _hash(contract) return full[:16], full def _pid_exists_windows(pid: int) -> bool: """Probe a Windows PID without sending a console event or terminating it.""" import ctypes from ctypes import wintypes process_query_limited_information = 0x1000 error_access_denied = 5 kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) kernel32.OpenProcess.argtypes = ( wintypes.DWORD, wintypes.BOOL, wintypes.DWORD, ) kernel32.OpenProcess.restype = wintypes.HANDLE kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) kernel32.CloseHandle.restype = wintypes.BOOL ctypes.set_last_error(0) handle = kernel32.OpenProcess( process_query_limited_information, False, int(pid), ) if handle: kernel32.CloseHandle(handle) return True return ctypes.get_last_error() == error_access_denied def _pid_exists(pid: int) -> bool: if pid <= 0: return False if pid == os.getpid(): return True if os.name == "nt": return _pid_exists_windows(pid) try: os.kill(pid, 0) except ProcessLookupError: return False except PermissionError: return True except OSError: return False return True class _RunLock: def __init__(self, path: Path): self.path = path self.token = uuid.uuid4().hex self.acquired = False def acquire(self) -> None: self.path.parent.mkdir(parents=True, exist_ok=True) payload = { "pid": os.getpid(), "hostname": socket.gethostname(), "started_utc": _now(), "token": self.token, } encoded = (_canonical(payload) + "\n").encode("utf-8") for _ in range(2): try: descriptor = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) except FileExistsError: try: existing = _read_json(self.path) except Exception as exc: raise RunStorageError(f"Run Storage lock exists and is unreadable: {self.path}") from exc if str(existing.get("hostname", "")) == socket.gethostname() and not _pid_exists(int(existing.get("pid", -1))): self.path.unlink() continue raise RunStorageError( f"Run Name is locked by pid={existing.get('pid')} host={existing.get('hostname')}" ) else: try: os.write(descriptor, encoded) os.fsync(descriptor) finally: os.close(descriptor) self.acquired = True return raise RunStorageError("failed to recover stale Run Storage lock") def release(self) -> None: if not self.acquired: return try: existing = _read_json(self.path) if existing.get("token") == self.token: self.path.unlink() finally: self.acquired = False def _entry_metadata(entry: dict[str, Any]) -> dict[str, Any]: return { "sequence_index": int(entry["sequence_index"]), "clip_index": int(entry["clip_index"]), "prompt_hash": str(entry["prompt_hash"]), "seed": int(entry["seed"]), "context_frames": int(entry["context_frames"]), "motion_score": float(entry["motion_score"]), "plan": dict(entry["plan"]), } class RunStorageController: def __init__(self, run_name: str): self.run_name = sanitize_run_name(run_name) self.run_root = _output_root() / "h3_continuum" / "runs" / self.run_name self.revisions_root = self.run_root / "revisions" self.lock = _RunLock(self.run_root / ".lock") self.context_token = None self.contract: dict[str, Any] | None = None self.contract_sha256 = "" self.revision_id = "" self.revision_root: Path | None = None self.manifest: dict[str, Any] | None = None self.prompts: list[str] = [] self.reused_count = 0 self.generated_count = 0 self.disabled_reasons: list[str] = [] self.notes: list[str] = [] self.resume_safe = False self.effective_reroll_nonce = 0 self.nonce_decision = "inactive" self.prompt_graph: dict[str, Any] | None = None self.sampler_node_id: str | None = None def set_prompt_graph(self, prompt: Any, unique_id: Any) -> None: self.prompt_graph = prompt if isinstance(prompt, dict) else None self.sampler_node_id = None if unique_id is None else str(unique_id) def __enter__(self) -> "RunStorageController": self.lock.acquire() self.context_token = _ACTIVE.set(self) return self def __exit__(self, exc_type, exc, traceback) -> None: try: if exc is not None and self.manifest is not None: self.manifest.update(status="interrupted", updated_utc=_now(), last_error=f"{type(exc).__name__}: {exc}"[:2048]) self._write_manifest() finally: if self.context_token is not None: _ACTIVE.reset(self.context_token) self.context_token = None self.lock.release() def _manifest_path(self) -> Path: if self.revision_root is None: raise RunStorageError("Run Storage revision is not prepared") return self.revision_root / "manifest.json" def _write_manifest(self) -> None: if self.manifest is not None: _write_json(self._manifest_path(), self.manifest) def _write_project(self) -> None: revisions = [] if self.revisions_root.exists(): for path in sorted(self.revisions_root.glob("*/manifest.json")): try: value = _read_json(path) lifecycle = value.get("nonce_lifecycle") or {} revision = {key: value.get(key) for key in ("revision_id", "contract_sha256", "status", "updated_utc", "resume_safe")} revision.update( nonce_mode=lifecycle.get("mode"), effective_reroll_nonce=lifecycle.get("effective_nonce"), reroll_from_chunk=(value.get("contract") or {}).get("reroll_from_chunk"), ) revisions.append(revision) except Exception: continue _write_json(self.run_root / "project.json", { "run_storage_schema_version": RUN_STORAGE_SCHEMA_VERSION, "run_name": self.run_name, "updated_utc": _now(), "revisions": revisions, }) def _load_entry(self, record: dict[str, Any], prompt: str) -> dict[str, Any]: source = str(record["storage_revision_id"]) filename = str(record["filename"]) if Path(filename).name != filename: raise RunStorageError("stored chunk filename is invalid") path = self.revisions_root / source / "chunks" / filename if path.stat().st_size != int(record["file_size"]): raise RunStorageError(f"stored chunk size mismatch: {filename}") expected_sha256 = str(record.get("file_sha256", "")) if not expected_sha256: raise RunStorageError(f"stored chunk SHA-256 is missing: {filename}") if _file_sha256(path) != expected_sha256: raise RunStorageError(f"stored chunk SHA-256 mismatch: {filename}") with safe_open(str(path), framework="pt", device="cpu") as handle: if set(handle.keys()) != {"audio", "video"}: raise RunStorageError(f"stored chunk tensors are invalid: {filename}") video, audio = handle.get_tensor("video"), handle.get_tensor("audio") entry = dict(record["entry"]) entry.update(prompt=str(prompt), video=video, audio=audio, reused=False) return validate_chunk_entry(entry) def _valid_prefix(self, manifest: dict[str, Any], hashes: list[str]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: stored = list((manifest.get("contract") or {}).get("chunk_contract_hashes") or []) records = list(manifest.get("chunks") or []) entries, accepted = [], [] for position, current in enumerate(hashes): if position >= len(stored) or stored[position] != current: break if position >= len(records) or int(records[position].get("sequence_index", -1)) != position: break try: entry = self._load_entry(records[position], self.prompts[position]) except Exception as exc: self.notes.append(f"stored chunk {position + 1} rejected: {exc}") break entries.append(entry) accepted.append(records[position]) return entries, accepted def _resolve_effective_nonce( self, contract: dict[str, Any], *, requested_nonce: int, resume_safe: bool, ) -> tuple[int, str]: boundary = int(contract["reroll_from_chunk"]) requested = int(requested_nonce) if boundary <= 0: return 0, "inactive" if requested >= 1: return requested, "explicit" lineage = str(contract["nonce_lineage_sha256"]) request = str(contract["nonce_request_sha256"]) highest = 0 matching: dict[int, dict[str, Any]] = {} if self.revisions_root.exists(): for path in self.revisions_root.glob("*/manifest.json"): try: manifest = _read_json(path) lifecycle = manifest.get("nonce_lifecycle") or (manifest.get("contract") or {}).get("nonce_lifecycle") or {} if str(lifecycle.get("lineage_sha256", "")) != lineage: continue nonce = int(lifecycle.get("effective_nonce", 0)) highest = max(highest, nonce) if str(lifecycle.get("request_sha256", "")) == request: current = matching.get(nonce) if current is None or str(manifest.get("updated_utc", "")) >= str(current.get("updated_utc", "")): matching[nonce] = manifest except Exception: continue if resume_safe and matching: latest_nonce = max(matching) latest = matching[latest_nonce] if latest_nonce == highest and str(latest.get("status", "")) != "complete": return latest_nonce, "resume_interrupted" return highest + 1, "new_revision" def prepare( self, *, model: Any, model_fingerprint_value: str, clip: Any, video_vae: Any, sampler: Any, sigmas: torch.Tensor, prompt_plan: dict[str, Any], width: int, height: int, chunk_seconds: float, continuity: str, audio_continuity: bool, base_seed: int, reroll_from_chunk: int, reroll_nonce: int, first_frame_hash: str, last_frame_hash: str, identity_hash: str, strict_compatibility: bool, existing_session: dict[str, Any] | None, reference_contract: dict[str, Any] | None = None, conditioning_mode: str | None = None, ) -> dict[str, Any] | None: if existing_session is not None: raise RunStorageError("Run Storage cannot be combined with an explicit Session") self.prompts = [str(value) for value in prompt_plan["prompts"]] from .graph_contract import build_upstream_graph_contract graph_contract, graph_safe, graph_reasons = build_upstream_graph_contract( self.prompt_graph, self.sampler_node_id, require_video_vae=conditioning_mode_uses_video_vae( str(conditioning_mode) ), ) contract, safe, reasons = build_sampling_contract( model=model, model_fingerprint_value=model_fingerprint_value, clip=clip, video_vae=video_vae, sampler=sampler, sigmas=sigmas, prompt_plan=prompt_plan, width=width, height=height, chunk_seconds=chunk_seconds, continuity=continuity, audio_continuity=audio_continuity, base_seed=base_seed, reroll_from_chunk=reroll_from_chunk, reroll_nonce=reroll_nonce, first_frame_hash=first_frame_hash, last_frame_hash=last_frame_hash, strict_compatibility=strict_compatibility, reference_contract=reference_contract, conditioning_mode=conditioning_mode, upstream_graph_contract=graph_contract, upstream_graph_safe=graph_safe, upstream_graph_reasons=graph_reasons, ) effective_nonce, nonce_decision = self._resolve_effective_nonce( contract, requested_nonce=int(reroll_nonce), resume_safe=safe ) contract = _apply_nonce_contract( contract, requested_nonce=int(reroll_nonce), effective_nonce=effective_nonce, ) self.resume_safe = bool(safe) self.effective_reroll_nonce = int(effective_nonce) self.nonce_decision = nonce_decision if not safe: contract = dict(contract) contract["reuse_policy"] = "disabled_unobservable_contract" contract["execution_nonce"] = uuid.uuid4().hex self.disabled_reasons = reasons self.contract = contract self.revision_id, self.contract_sha256 = revision_identity(contract) self.revision_root = self.revisions_root / self.revision_id self.revision_root.mkdir(parents=True, exist_ok=True) exact = None if self._manifest_path().exists(): try: exact = _read_json(self._manifest_path()) except Exception as exc: self.notes.append(f"exact manifest rejected: {exc}") if exact is not None and exact.get("contract_sha256") != self.contract_sha256: raise RunStorageError(f"short revision id collision: {self.revision_id}") hashes = list(contract["chunk_contract_hashes"]) best_entries: list[dict[str, Any]] = [] best_records: list[dict[str, Any]] = [] candidates = [exact] if exact is not None and safe else [] if safe and exact is None and self.revisions_root.exists(): for path in self.revisions_root.glob("*/manifest.json"): try: candidates.append(_read_json(path)) except Exception: continue for candidate in candidates: if int(candidate.get("run_storage_schema_version", -1)) != RUN_STORAGE_SCHEMA_VERSION: continue entries, records = self._valid_prefix(candidate, hashes) if len(entries) > len(best_entries): best_entries, best_records = entries, records self.reused_count = len(best_entries) now = _now() self.manifest = { "run_storage_schema_version": RUN_STORAGE_SCHEMA_VERSION, "sampling_contract_version": SAMPLING_CONTRACT_VERSION, "run_name": self.run_name, "revision_id": self.revision_id, "contract_sha256": self.contract_sha256, "contract": contract, "resume_safe": bool(safe), "resume_disabled_reasons": list(reasons), "nonce_lifecycle": dict(contract["nonce_lifecycle"]), "status": "in_progress", "created_utc": (exact or {}).get("created_utc", now), "updated_utc": now, "chunks": best_records, "report_summary": (exact or {}).get("report_summary", ""), } self._write_manifest() self._write_project() if not best_entries: return None return make_session( chunks=best_entries, width=int(width), height=int(height), chunk_seconds=float(chunk_seconds), identity_hash=str(identity_hash), model_fingerprint_value=str(model_fingerprint_value), parent_session_id=None, reroll_from_chunk=0, settings={ "run_storage_validated_prefix": True, "revision_id": self.revision_id, "first_frame_hash": str(first_frame_hash), "last_frame_hash": str(last_frame_hash), }, ) def commit_chunk(self, entry: dict[str, Any], *, position: int) -> None: if self.manifest is None or self.revision_root is None: return validated = validate_chunk_entry(entry) if isinstance(position, bool) or not isinstance(position, int): raise RunStorageError("Run Storage chunk position must be an integer") expected = int((self.contract or {}).get("chunk_count", 0)) if position < 0 or position >= expected: raise RunStorageError( f"Run Storage chunk position {position} is outside 0..{expected - 1}" ) existing_records = list(self.manifest.get("chunks") or []) if len(existing_records) < position: raise RunStorageError( f"Run Storage cannot commit chunk {position + 1} before chunk {position}" ) chunks_root = self.revision_root / "chunks" chunks_root.mkdir(parents=True, exist_ok=True) filename = f"chunk_{position + 1:04d}.safetensors" target = chunks_root / filename temporary = chunks_root / f".{filename}.{uuid.uuid4().hex}.tmp" tensors = { "video": validated["video"].detach().to("cpu").contiguous(), "audio": validated["audio"].detach().to("cpu").contiguous(), } try: save_file(tensors, str(temporary), metadata={ "h3_continuum_run_storage": str(RUN_STORAGE_SCHEMA_VERSION), "revision_id": self.revision_id, "chunk_number": str(position + 1), }) _fsync_file(temporary) os.replace(temporary, target) _fsync_dir(chunks_root) finally: if temporary.exists(): temporary.unlink() storage_entry = dict(validated) storage_entry["sequence_index"] = position record = { "sequence_index": position, "storage_revision_id": self.revision_id, "filename": filename, "file_size": target.stat().st_size, "file_sha256": _file_sha256(target), "entry": _entry_metadata(storage_entry), } records = existing_records[:position] records.append(record) self.manifest.update(chunks=records, updated_utc=_now(), status="in_progress") self._write_manifest() self.generated_count += 1 def summary(self, *, detailed: bool = False) -> str: total = int((self.contract or {}).get("chunk_count", 0)) resume = self.reused_count + 1 if self.reused_count < total else "complete" policy = (" auto-resume disabled: " + "; ".join(self.disabled_reasons) + ".") if self.disabled_reasons else "" note = (" " + " ".join(self.notes)) if self.notes else "" basic = ( f"Run Storage: {self.run_name} / revision {self.revision_id}; " f"{self.reused_count} reused, {self.generated_count} generated, " f"{total} total; resume={resume}; nonce={self.effective_reroll_nonce} " f"({self.nonce_decision}).{policy}{note}" ) if not detailed or self.manifest is None: return basic records = list(self.manifest.get("chunks") or []) lines = [basic, f"Run Storage path: {self.revision_root}"] for record in records: lines.append( f"stored chunk {int(record['sequence_index']) + 1}: " f"{record['storage_revision_id']}/chunks/{record['filename']} " f"({int(record['file_size']) / (1024.0 ** 2):.1f} MiB)" ) try: free_gib = shutil.disk_usage(self.run_root).free / (1024.0 ** 3) lines.append(f"Run Storage free disk: {free_gib:.2f} GiB") except OSError: lines.append("Run Storage free disk: unknown") first_invalid = self.reused_count + 1 if self.reused_count < total else "none" lines.append(f"Run Storage first regenerated chunk: {first_invalid}") return "\n".join(lines) def finalize(self, *, session: dict[str, Any], report: str) -> None: if self.manifest is None: return expected = int((self.contract or {}).get("chunk_count", 0)) status = "complete" if len(self.manifest.get("chunks") or []) == expected else "interrupted" self.manifest.update( status=status, updated_utc=_now(), session_id=str(session.get("session_id", "")), report_summary=str(report)[-8192:], ) self._write_manifest() self._write_project() def get_active_run_storage() -> RunStorageController | None: return _ACTIVE.get() def run_storage_scope( run_name: str, *, prompt: Any = None, unique_id: Any = None ) -> RunStorageController: controller = RunStorageController(run_name) controller.set_prompt_graph(prompt, unique_id) return controller