"""Server-owned scratch settings; never part of a workflow or generation identity.""" from __future__ import annotations from contextlib import contextmanager, nullcontext from functools import wraps import json import mmap import os from pathlib import Path import re import shutil import tempfile import threading import time import uuid import weakref from . import storage_fs as fs from .storage_memory import system_memory_bytes _MANAGER = None _CACHE_NAME = re.compile(r"^[0-9a-f]{32}\.(npy|tmp)$") _REMOTE_FS = {"nfs", "nfs4", "cifs", "smbfs", "smb3", "sshfs", "fuse.sshfs", "9p", "afs", "davfs"} _OWNER_FILE = ".h3-owner.json" class StorageError(RuntimeError): status = 400 class StorageBusy(StorageError): status = 409 def install_manager(manager): global _MANAGER _MANAGER = manager def get_manager(): return _MANAGER def storage_operation(function): @wraps(function) def wrapped(*args, **kwargs): manager = get_manager() with manager.operation() if manager else nullcontext(): return function(*args, **kwargs) return wrapped def _anchor(path): path = Path(path) while not path.exists() and path != path.parent: path = path.parent return path def volumes(): result = [] try: import psutil for part in psutil.disk_partitions(all=False): if part.fstype.lower() in _REMOTE_FS or not Path(part.mountpoint).is_dir(): continue try: result.append({"path": part.mountpoint, "free_bytes": shutil.disk_usage(part.mountpoint).free, "device": str(Path(part.mountpoint).stat().st_dev)}) except OSError: pass except (ImportError, OSError): pass return result def _local_path(value): if not isinstance(value, str) or len(value) > 4096 or "\0" in value: raise StorageError("Enter an absolute folder path on the ComfyUI server") value = value.strip() if not value: return None if value.startswith(("\\\\", "//")): raise StorageError("Network shares are not supported for scratch storage") path = Path(value).expanduser() if not path.is_absolute(): raise StorageError("The scratch folder must be an absolute server path") path = path.resolve() if os.name == "nt": import ctypes if ctypes.windll.kernel32.GetDriveTypeW(str(path.anchor)) == 4: raise StorageError("Network drives are not supported for scratch storage") try: import psutil matches = [p for p in psutil.disk_partitions(all=True) if path == Path(p.mountpoint) or Path(p.mountpoint) in path.parents] if matches and max(matches, key=lambda p: len(p.mountpoint)).fstype.lower() in _REMOTE_FS: raise StorageError("Network mounts are not supported for scratch storage") except ImportError: pass return path def _probe(directory): descriptor, name = tempfile.mkstemp(prefix=".h3-probe-", dir=directory) try: with os.fdopen(descriptor, "w+b") as handle: handle.truncate(4096) with mmap.mmap(handle.fileno(), 4096, access=mmap.ACCESS_WRITE) as mapping: mapping[:4] = b"H3OK" mapping.flush() if mapping[:4] != b"H3OK": raise StorageError("Scratch mmap verification failed") os.fsync(handle.fileno()) return shutil.disk_usage(directory).free finally: Path(name).unlink(missing_ok=True) def _prepare(value): parent = _local_path(value) if parent is None: return {"path": "", "roots": []} scratch = parent / "h3-continuum-scratch" scratch.mkdir(parents=True, exist_ok=True) if scratch.resolve() != scratch: raise StorageError("The dedicated scratch directory must not be a link") marker = scratch / ".h3-scratch.json" with fs.file_lock(scratch / ".settings.lock"): if marker.exists(): payload = fs.read_json(marker) if payload.get("schema") != "h3-scratch-root-v1": raise StorageError("This scratch directory has unrecognized ownership metadata") fs.check_identity(scratch, payload["identity"]) else: if any(p.name != ".settings.lock" for p in scratch.iterdir()): raise StorageError("Choose an empty scratch folder; existing unknown files are protected") payload = {"schema": "h3-scratch-root-v1", "id": uuid.uuid4().hex, "identity": fs.identity(scratch)} fs.atomic_json(marker, payload) roots = [] for kind in ("finalize", "decode"): directory = scratch / kind directory.mkdir(exist_ok=True) if directory.resolve() != directory: raise StorageError("A scratch subdirectory must not be a link") free = _probe(directory) roots.append({"kind": kind, "path": str(directory), "identity": fs.identity(directory), "scratch": str(scratch), "marker": payload["id"], "free_bytes": free}) return {"path": str(parent), "roots": roots} def _check_root(record): fs.check_identity(record["path"], record["identity"]) if record.get("scratch"): marker = fs.read_json(Path(record["scratch"]) / ".h3-scratch.json") if marker.get("schema") != "h3-scratch-root-v1" or marker.get("id") != record["marker"]: raise StorageError("Scratch ownership changed") fs.check_identity(record["scratch"], marker["identity"]) class StorageSettings: def __init__(self, config_file, *, default_finalize, default_decode, busy=lambda: False, queue_guard=nullcontext, read_only=False): self.config_file = Path(config_file) self.default_finalize = default_finalize self.default_decode = default_decode self.busy = busy self.queue_guard = queue_guard self.read_only = read_only self._gate = threading.RLock() self._active = 0 self._local = threading.local() self._stores = weakref.WeakSet() # Disk entries contain only metadata, weak guards/leases, and paths. # Retain this ledger after a helper is GC'd, without retaining its tensors. self._cache_states = {} self._seen = {} self._last_backend = None def _read(self): if not self.config_file.exists(): return {"schema": 1, "revision": 0, "path": "", "roots": []} value = fs.read_json(self.config_file) if (value.get("schema") != 1 or type(value.get("revision")) is not int or value["revision"] < 0 or not isinstance(value.get("path"), str) or not isinstance(value.get("roots"), list)): raise StorageError("Invalid scratch settings; restore the server settings file") return value def _config(self): return getattr(self._local, "config", None) or self._read() @contextmanager def operation(self): if getattr(self._local, "config", None) is not None: yield return with self._gate: self._local.config = self._read() self._active += 1 try: yield finally: with self._gate: self._active -= 1 del self._local.config @contextmanager def _maintenance(self): if self.read_only: error = StorageError("Temporary storage is read-only in multi-user mode") error.status = 403 raise error with self._gate, self.queue_guard(): if self._active or self.busy(): raise StorageBusy("Wait until the running and pending queues are empty") self.config_file.parent.mkdir(parents=True, exist_ok=True) with fs.file_lock(self.config_file.with_suffix(".lock")): yield def validate(self, path): with self._maintenance(): return _prepare(path) def save(self, path, revision): with self._maintenance(): old = self._read() if type(revision) is not int or revision != old["revision"]: raise StorageBusy("Settings changed; refresh before saving") prepared = _prepare(path) records = {(r["kind"], r["path"]): r for r in self._records(old)} records.update({(r["kind"], r["path"]): r for r in prepared["roots"]}) value = {"schema": 1, "revision": revision + 1, "path": prepared["path"], "roots": list(records.values())} fs.atomic_json(self.config_file, value) return value def resolve(self, kind, default, *, verify=True): config = self._config() if not config["path"]: return Path(default).expanduser().resolve() target = str(Path(config["path"]) / "h3-continuum-scratch" / kind) record = next((r for r in config["roots"] if r["kind"] == kind and r["path"] == target), None) if record is None: raise StorageError("Configured scratch storage has no validated directory") if verify: _check_root(record) return Path(target) def verify_final_root(self, path): config = self._config() if config["path"] and str(path) == str(Path(config["path"]) / "h3-continuum-scratch" / "finalize"): self.resolve("finalize", path) def record_root(self, kind, path): path = Path(path).resolve() with self._gate: key = kind, str(path) if key not in self._seen: self._seen[key] = {"kind": kind, "path": str(path), "identity": fs.identity(path)} def register_store(self, store): with self._gate: self._stores.add(store) self._cache_states[store._scratch_token] = store._cleanup_state def record_backend(self, backend, required_bytes, root=None): with self._gate: self._last_backend = {"backend": backend, "image_bytes": required_bytes, "path": str(root) if root is not None else None, "at": time.time()} def _records(self, config): records = {(r["kind"], r["path"]): r for r in config["roots"]} for kind, default in (("finalize", self.default_finalize()), ("decode", self.default_decode())): path = Path(default).expanduser().resolve() key = kind, str(path) if key not in records: records[key] = {"kind": kind, "path": str(path), "identity": fs.identity(path) if path.exists() else None} for key, record in self._seen.items(): if key not in records or records[key].get("identity") is None: records[key] = record return list(records.values()) def _scan(self, cleanup=False): from .tools.p20_file_backed_tensor_poc import inspect_managed_files config = self._read() with self._gate: records = self._records(config) stores = list(self._stores) live_ids = {store._scratch_token for store in stores} detached = [state for key, state in self._cache_states.items() if key not in live_ids] results = [] for record in records: row = {"kind": record["kind"], "path": record["path"], "bytes": 0, "in_use_bytes": 0, "deletable_bytes": 0, "unknown_bytes": 0, "files": 0, "removed_files": 0, "removed_bytes": 0, "errors": []} path = Path(record["path"]) configured = (str(Path(config["path"]) / "h3-continuum-scratch" / record["kind"]) if config["path"] else str(Path(self.default_finalize() if record["kind"] == "finalize" else self.default_decode()).expanduser().resolve())) row["current"] = record["path"] == configured try: row["free_bytes"] = shutil.disk_usage(_anchor(path)).free row["device"] = str(_anchor(path).stat().st_dev) if record.get("identity") is not None: _check_root(record) if not path.exists(): row["exists"] = False results.append(row) continue row["exists"] = True cleanup_this = cleanup and record.get("identity") is not None if record["kind"] == "finalize": scanned = inspect_managed_files(path, cleanup=cleanup_this) else: scanned = inspect_decode_files(path, stores, detached=detached, cleanup=cleanup_this) for item in scanned["files"]: row["files"] += 1 row["bytes"] += item["bytes"] row[item["state"] + "_bytes"] += item["bytes"] row.update({k: scanned[k] for k in ("removed_files", "removed_bytes", "errors")}) except Exception as exc: row["free_bytes"] = None row["device"] = None row["errors"].append(str(exc)) results.append(row) return results def status(self): config = self._read() available, total = system_memory_bytes() return {"revision": config["revision"], "path": config["path"], "read_only": self.read_only, "busy": bool(self._active or self.busy()), "roots": self._scan(), "volumes": volumes(), "last_backend": self._last_backend, "memory": {"available_bytes": available, "total_bytes": total}, "size_kind": "logical_file_bytes", "at": time.time()} def cleanup(self): with self._maintenance(): return {"roots": self._scan(cleanup=True)} def create_cache_owner(root): owner = {"schema": "h3-decode-scratch-v1", "owner": fs.process_owner(), "identity": fs.identity(root), "files": []} fs.atomic_json(Path(root) / _OWNER_FILE, owner) return owner def register_cache_file(root, owner, name): fs.check_identity(root, owner["identity"]) if not re.fullmatch(r"[0-9a-f]{32}", name): raise StorageError("Invalid cache file identity") # Register before writing so an interrupted writer leaves attributable files. owner["files"] = [name for name in owner["files"] if fs.regular_file(Path(root) / name)] owner["files"].extend((name + ".tmp", name + ".npy")) fs.atomic_json(Path(root) / _OWNER_FILE, owner) def remove_empty_cache_root(root, expected): root = Path(root) try: fs.check_identity(root, expected) if all(p.name == _OWNER_FILE for p in root.iterdir()): (root / _OWNER_FILE).unlink(missing_ok=True) root.rmdir() except OSError: pass def inspect_decode_files(parent, stores, *, detached=(), cleanup=False): result = {"files": [], "removed_files": 0, "removed_bytes": 0, "errors": []} managed = {} for store in stores: managed.update(store.inspect_scratch(cleanup=cleanup, parent=parent)) for state in detached: with state["lock"]: for path, entry in list(state["entries"].items()): if path.parent.parent != Path(parent): continue try: fs.check_identity(path.parent, state["identities"][path.parent]) if not fs.regular_file(path): continue live = any(ref() is not None for ref in entry.leases) if cleanup and not live: size = path.stat().st_size path.unlink() state["entries"].pop(path, None) result["removed_files"] += 1 result["removed_bytes"] += size else: managed[str(path)] = "in_use" if live else "deletable" except (OSError, KeyError): managed[str(path)] = "unknown" if cleanup: for root, owner in state.get("owners", {}).items(): if root.parent != Path(parent): continue try: fs.check_identity(root, owner["identity"]) for name in owner["files"]: path = root / name if path not in state["entries"] and fs.regular_file(path): size = path.stat().st_size path.unlink() result["removed_files"] += 1 result["removed_bytes"] += size except OSError as exc: result["errors"].append(str(exc)) for root, expected in state["identities"].items(): if root.parent == Path(parent): remove_empty_cache_root(root, expected) for root in Path(parent).glob("h3-decode-cache-*"): if root.is_symlink() or root.resolve() != root or not root.is_dir(): continue known, dead = set(), False try: owner = fs.read_json(root / _OWNER_FILE) if owner.get("schema") != "h3-decode-scratch-v1": raise ValueError("Unknown cache owner") fs.check_identity(root, owner["identity"]) known = {n for n in owner["files"] if isinstance(n, str) and _CACHE_NAME.fullmatch(n)} dead = not fs.owner_alive(owner["owner"]) except (OSError, ValueError, KeyError, TypeError): pass try: children = list(root.iterdir()) except OSError as exc: result["errors"].append(str(exc)) continue for path in children: if not _CACHE_NAME.fullmatch(path.name) or not fs.regular_file(path): continue try: size = path.stat().st_size state = managed.get(str(path), "deletable" if dead and path.name in known else "unknown") if path.name in known and not dead and str(path) not in managed: state = "in_use" if cleanup and dead and path.name in known: fs.check_identity(root, owner["identity"]) path.unlink() result["removed_files"] += 1 result["removed_bytes"] += size else: result["files"].append({"bytes": size, "state": state}) except OSError as exc: result["errors"].append(str(exc)) if cleanup and dead: remove_empty_cache_root(root, owner["identity"]) # Current stores remove entries under their own locks; count those removals. for state in managed.values(): if isinstance(state, dict): result["removed_files"] += state["removed_files"] result["removed_bytes"] += state["removed_bytes"] result["errors"].extend(state.get("errors", [])) return result