Add server-side temporary storage controls for 3.9.3
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import vm from "node:vm";
|
||||
|
||||
class Element extends EventTarget {
|
||||
constructor(tag) { super(); this.tag = tag; this.children = []; this.style = {}; this.dataset = {}; this.attributes = {}; this.value = ""; this.disabled = false; this.textContent = ""; }
|
||||
append(child) { child.parentElement = this; this.children.push(child); }
|
||||
replaceChildren() { for (const child of this.children) child.parentElement = null; this.children = []; }
|
||||
setAttribute(name, value) { this.attributes[name] = value; }
|
||||
get isConnected() { return this === document.body || !!this.parentElement?.isConnected; }
|
||||
closest() { let node = this; while (node) { if (node.tag === "dialog" || node.attributes.role === "dialog") return node; node = node.parentElement; } return null; }
|
||||
remove() { this.parentElement.children = this.parentElement.children.filter(x => x !== this); this.parentElement = null; }
|
||||
showModal() { this.open = true; }
|
||||
close() { this.open = false; this.dispatchEvent(new Event("close")); }
|
||||
focus() {}
|
||||
click() { if (!this.disabled) this.dispatchEvent(new Event("click")); }
|
||||
}
|
||||
const all = [];
|
||||
globalThis.document = { createElement: tag => { const e = new Element(tag); all.push(e); return e; }, body: new Element("body") };
|
||||
Object.defineProperty(globalThis, "navigator", { value: { language: "en" }, configurable: true });
|
||||
let definition;
|
||||
globalThis.__app = { ui: { settings: { getSettingValue: () => "en", addSetting: d => { definition = d; } } }, registerExtension: d => d.setup() };
|
||||
const api = globalThis.__api = new EventTarget();
|
||||
const requests = [];
|
||||
let state = { path: "", revision: 0, busy: false, read_only: false, memory: { available_bytes: 8 * 1024 ** 3 }, roots: [], volumes: [], last_backend: null };
|
||||
api.fetchApi = async (path, options) => {
|
||||
const action = path.split("/").at(-1);
|
||||
const body = options.body ? JSON.parse(options.body) : undefined;
|
||||
requests.push({ action, body });
|
||||
let value = structuredClone(state), ok = true;
|
||||
if (action === "settings") { assert.equal(body.revision, state.revision); state = { ...state, path: body.path, revision: state.revision + 1 }; value = state; }
|
||||
if (action === "validate") {
|
||||
ok = !body.path || body.path.startsWith("/");
|
||||
value = ok ? { path: body.path, roots: [{ free_bytes: 10 * 1024 ** 3 }] } : { error: "absolute path required" };
|
||||
}
|
||||
if (action === "cleanup") { assert.deepEqual(body, {}); value = { roots: [] }; }
|
||||
return { ok, status: ok ? 200 : 400, json: async () => value };
|
||||
};
|
||||
const source = fs.readFileSync(new URL("../web/storage_settings.js", import.meta.url), "utf8")
|
||||
.replace('import { app } from "../../scripts/app.js";', "const app = globalThis.__app;")
|
||||
.replace('import { api } from "../../scripts/api.js";', "const api = globalThis.__api;")
|
||||
.replace("export function openStorageDialog", "function openStorageDialog");
|
||||
vm.runInThisContext(source);
|
||||
const settings = new Element("section"); settings.setAttribute("role", "dialog"); document.body.append(settings);
|
||||
const row = definition.type(); settings.append(row);
|
||||
const launch = row.children[0].children[0];
|
||||
const connected = tag => all.filter(x => x.tag === tag && x.isConnected);
|
||||
const dialog = () => connected("dialog")[0];
|
||||
const control = name => all.find(x => x.tag === "button" && x.isConnected && x.textContent === name);
|
||||
const settle = () => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
launch.click(); await settle();
|
||||
assert.equal(dialog().parentElement, settings, "preserve Core's settings dialog during native-modal interactions");
|
||||
assert.equal(connected("input")[0].value, "");
|
||||
assert.equal(requests.filter(r => r.action === "status").length, 1);
|
||||
launch.click(); assert.equal(connected("dialog").length, 1, "no duplicate modal");
|
||||
const input = connected("input")[0];
|
||||
input.value = "relative"; input.dispatchEvent(new Event("input"));
|
||||
control("Validate").click(); await settle();
|
||||
assert.ok(all.some(x => x.isConnected && x.textContent === "absolute path required"));
|
||||
input.value = "/scratch"; input.dispatchEvent(new Event("input"));
|
||||
control("Save").click(); control("Save").click(); await settle();
|
||||
assert.equal(requests.filter(r => r.action === "settings").length, 1, "one save while request is pending");
|
||||
assert.equal(state.path, "/scratch");
|
||||
const status = new Event("status"); status.detail = { exec_info: { queue_remaining: 1 } }; api.dispatchEvent(status);
|
||||
assert.equal(control("Save").disabled, true);
|
||||
assert.equal(control("Delete unused temporary files").disabled, true);
|
||||
status.detail.exec_info.queue_remaining = 0; api.dispatchEvent(status);
|
||||
control("Delete unused temporary files").click(); await settle();
|
||||
assert.equal(requests.filter(r => r.action === "cleanup").length, 1);
|
||||
control("Close").click(); assert.equal(connected("dialog").length, 0);
|
||||
const before = requests.length;
|
||||
for (let n = 0; n < 100; n++) {
|
||||
launch.click(); await settle();
|
||||
assert.equal(connected("input")[0].value, "/scratch");
|
||||
control("Close").click();
|
||||
}
|
||||
await settle();
|
||||
assert.equal(requests.length - before, 100, "one fetch per open and no periodic polling");
|
||||
assert.equal(connected("dialog").length, 0);
|
||||
// A disposed dialog's handler must not receive later status events.
|
||||
api.dispatchEvent(status);
|
||||
assert.equal(requests.length - before, 100);
|
||||
state.read_only = true;
|
||||
launch.click(); await settle();
|
||||
assert.equal(control("Save").disabled, true);
|
||||
assert.equal(control("Delete unused temporary files").disabled, true);
|
||||
assert.equal(connected("input")[0].disabled, true);
|
||||
control("Close").click();
|
||||
console.log("Temporary storage frontend: settings nesting, deduplication, errors, save, busy/read-only gates, cleanup, and 100 reopen cycles passed");
|
||||
@@ -6,7 +6,7 @@ import tomllib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EXPECTED_VERSION = "3.9.2"
|
||||
EXPECTED_VERSION = "3.9.3"
|
||||
|
||||
|
||||
def test_v35_release_metadata_is_consistent():
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Real HTTP requests to the production routes, using a model-free server fixture."""
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from ComfyUI_H3_Continuum_Join import storage_api, storage_settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_server(tmp_path, monkeypatch):
|
||||
user = tmp_path / "user"
|
||||
temp = tmp_path / "temp"
|
||||
temp.mkdir()
|
||||
def system_user(name):
|
||||
if not name.replace("_", "").isalnum():
|
||||
raise ValueError("Invalid system user name")
|
||||
return str(user / ("__" + name))
|
||||
monkeypatch.setitem(sys.modules, "folder_paths", SimpleNamespace(
|
||||
get_system_user_directory=system_user, get_temp_directory=lambda: str(temp)))
|
||||
monkeypatch.setitem(sys.modules, "comfy.cli_args", SimpleNamespace(args=SimpleNamespace(multi_user=False)))
|
||||
monkeypatch.setattr(storage_settings, "_MANAGER", None)
|
||||
queue = SimpleNamespace(mutex=threading.RLock(), running=[], pending=[])
|
||||
queue.get_current_queue = lambda: (queue.running, queue.pending)
|
||||
server = SimpleNamespace(routes=web.RouteTableDef(), prompt_queue=queue)
|
||||
manager = storage_api.register_routes(server)
|
||||
assert storage_api.register_routes(server) is None
|
||||
app = web.Application()
|
||||
app.add_routes(server.routes)
|
||||
return app, manager, queue
|
||||
|
||||
|
||||
def test_http_validation_save_status_and_revision_conflict(api_server, tmp_path):
|
||||
app, manager, queue = api_server
|
||||
async def scenario():
|
||||
async with TestClient(TestServer(app)) as client:
|
||||
status = await client.get("/h3-continuum/storage/status")
|
||||
assert status.status == 200
|
||||
state = await status.json()
|
||||
assert state["path"] == "" and not manager.config_file.exists()
|
||||
destination = str(tmp_path / "scratch disk")
|
||||
checked = await client.post("/h3-continuum/storage/validate", json={"path": destination})
|
||||
assert checked.status == 200 and not manager.config_file.exists()
|
||||
calls = [client.post("/h3-continuum/storage/settings", json={"path": destination, "revision": 0}) for _ in range(2)]
|
||||
responses = await asyncio.gather(*calls)
|
||||
assert sorted(r.status for r in responses) == [200, 409]
|
||||
status = await client.get("/h3-continuum/storage/status")
|
||||
state = await status.json()
|
||||
assert state["revision"] == 1 and state["path"] == destination
|
||||
assert len([r for r in state["roots"] if r["current"]]) == 2
|
||||
assert (await client.post("/h3-continuum/storage/cleanup", json={})).status == 200
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_http_rejects_busy_read_only_and_arbitrary_cleanup_paths(api_server, tmp_path):
|
||||
app, manager, queue = api_server
|
||||
protected = tmp_path / "accepted.safetensors"
|
||||
protected.write_bytes(b"keep")
|
||||
async def scenario():
|
||||
async with TestClient(TestServer(app)) as client:
|
||||
prefix = "/h3-continuum/storage/"
|
||||
assert (await client.post(prefix + "cleanup", json={"path": str(protected)})).status == 400
|
||||
assert (await client.post(prefix + "cleanup", data="{}")).status == 415
|
||||
queue.pending.append("queued")
|
||||
for action, data in (("settings", {"path": "", "revision": 0}), ("cleanup", {}), ("validate", {"path": ""})):
|
||||
assert (await client.post(prefix + action, json=data)).status == 409
|
||||
queue.pending.clear()
|
||||
with manager.operation():
|
||||
assert (await client.post(prefix + "cleanup", json={})).status == 409
|
||||
manager.read_only = True
|
||||
assert (await client.get(prefix + "status")).status == 200
|
||||
assert (await client.post(prefix + "settings", json={"path": "", "revision": 0})).status == 403
|
||||
assert (await client.post(prefix + "cleanup", json={})).status == 403
|
||||
assert protected.read_bytes() == b"keep"
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,13 @@
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
|
||||
def test_storage_settings_frontend_lifecycle():
|
||||
node = shutil.which("node")
|
||||
assert node, "Node.js is required for frontend regression checks"
|
||||
script = Path(__file__).with_name("frontend_storage_settings.mjs")
|
||||
result = subprocess.run([node, str(script)], text=True, encoding="utf-8",
|
||||
capture_output=True, timeout=30)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert "100 reopen cycles passed" in result.stdout
|
||||
@@ -0,0 +1,471 @@
|
||||
"""Scratch relocation and cleanup regressions with real small CPU mappings."""
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from ComfyUI_H3_Continuum_Join import storage_settings as settings
|
||||
from ComfyUI_H3_Continuum_Join import storage_fs as fs
|
||||
from ComfyUI_H3_Continuum_Join import storage_memory as memory
|
||||
from ComfyUI_H3_Continuum_Join.decode_cache.store import DecodeStore
|
||||
from ComfyUI_H3_Continuum_Join.decode_cache import store as store_module
|
||||
from ComfyUI_H3_Continuum_Join.tools import p20_file_backed_tensor_poc as backing
|
||||
from ComfyUI_H3_Continuum_Join.v3 import file_backed_buffer as buffer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager(tmp_path, monkeypatch):
|
||||
finalize, decode = tmp_path / "default-final", tmp_path / "default-decode"
|
||||
decode.mkdir()
|
||||
value = settings.StorageSettings(tmp_path / "user" / "settings.json",
|
||||
default_finalize=lambda: finalize, default_decode=lambda: decode)
|
||||
monkeypatch.setattr(settings, "_MANAGER", value)
|
||||
monkeypatch.setenv("H3_DECODE_CACHE_TEMP_DIR", str(decode))
|
||||
return value
|
||||
|
||||
|
||||
def cache():
|
||||
value = DecodeStore(memory_probe=lambda: 10**12, headroom_bytes=0)
|
||||
value.configure("Auto", 1024**2, 1024**2, 0)
|
||||
return value
|
||||
|
||||
|
||||
def cache_put(store, name="first"):
|
||||
x = torch.arange(24, dtype=torch.float32).reshape(2, 3, 4)
|
||||
result = store.put(name, x, "video", {}, (), store.epoch)
|
||||
assert result is not None
|
||||
return result[0]
|
||||
|
||||
|
||||
def total_removed(result):
|
||||
return sum(row["removed_files"] for row in result["roots"])
|
||||
|
||||
|
||||
def test_default_and_explicit_internal_paths_keep_their_precedence(manager, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(buffer.tempfile, "gettempdir", lambda: str(tmp_path))
|
||||
import sys
|
||||
monkeypatch.setitem(sys.modules, "folder_paths", SimpleNamespace(get_temp_directory=lambda: str(tmp_path)))
|
||||
assert buffer.resolve_backing_root() == tmp_path / "h3-continuum-v35-file-backed"
|
||||
manager.save(str(tmp_path / "日本語 scratch"), 0)
|
||||
assert buffer.resolve_backing_root() == tmp_path / "日本語 scratch/h3-continuum-scratch/finalize"
|
||||
assert buffer.resolve_backing_root(tmp_path / "injected") == tmp_path / "injected"
|
||||
store = cache()
|
||||
output = cache_put(store)
|
||||
assert store.root.parent == tmp_path / "日本語 scratch/h3-continuum-scratch/decode"
|
||||
assert not list(manager.default_decode().iterdir())
|
||||
del output
|
||||
store.close()
|
||||
|
||||
|
||||
def test_validation_does_not_activate_a_path_and_settings_survive_reload(manager, tmp_path):
|
||||
destination = str(tmp_path / "other disk")
|
||||
manager.validate(destination)
|
||||
assert manager.status()["path"] == "" and not manager.config_file.exists()
|
||||
manager.save(destination, 0)
|
||||
restored = settings.StorageSettings(manager.config_file, default_finalize=manager.default_finalize,
|
||||
default_decode=manager.default_decode)
|
||||
assert restored.status()["path"] == destination
|
||||
with pytest.raises(settings.StorageBusy, match="refresh"):
|
||||
manager.save("", 0)
|
||||
manager.save("", 1)
|
||||
assert manager.status()["path"] == ""
|
||||
|
||||
|
||||
def test_switch_keeps_old_finalize_alias_and_both_cache_mappings(manager, tmp_path):
|
||||
manager.save(str(tmp_path / "disk-a"), 0)
|
||||
with buffer.allocate_file_backed_image((2, 3, 4), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.copy_(torch.arange(24).reshape(2, 3, 4))
|
||||
output = allocation.publish()
|
||||
alias = output[1:]
|
||||
old_final = allocation.record.data_path
|
||||
del output, allocation
|
||||
store = cache()
|
||||
first = cache_put(store)
|
||||
old_cache = store.entries["first"].path
|
||||
manager.save(str(tmp_path / "disk-b"), 1)
|
||||
second = cache_put(store, "second")
|
||||
assert old_cache.parent != store.entries["second"].path.parent
|
||||
assert torch.equal(first, second)
|
||||
result = manager.cleanup()
|
||||
assert total_removed(result) == 0
|
||||
assert old_cache.exists() and old_final.exists()
|
||||
assert alias.sum() == sum(range(12, 24))
|
||||
del first, second, alias
|
||||
gc.collect()
|
||||
result = manager.cleanup()
|
||||
assert total_removed(result) == 4 # one bin/sidecar pair and two npy files
|
||||
assert not old_cache.exists() and not old_final.exists()
|
||||
assert not store.contains("first") and not store.contains("second")
|
||||
assert cache_put(store, "third").sum() == sum(range(24))
|
||||
store.close()
|
||||
|
||||
|
||||
def test_status_is_read_only_and_unknown_files_and_runs_are_protected(manager, tmp_path):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
root = buffer.resolve_backing_root()
|
||||
unknown = root / (backing.FILE_PREFIX + "a" * 32 + ".bin")
|
||||
unknown.write_bytes(b"orphan")
|
||||
run = tmp_path / "saved-run.safetensors"
|
||||
run.write_bytes(b"accepted take")
|
||||
store = cache()
|
||||
tensor = cache_put(store)
|
||||
path = store.entries["first"].path
|
||||
del tensor
|
||||
before = {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
|
||||
report = manager.status()
|
||||
assert any(row["deletable_bytes"] > 0 for row in report["roots"])
|
||||
assert before == {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
|
||||
assert store.contains("first") and path.exists()
|
||||
manager.cleanup()
|
||||
assert unknown.read_bytes() == b"orphan" and run.read_bytes() == b"accepted take"
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_changed_mount_is_not_recreated_or_replaced_by_default(manager, tmp_path):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
scratch = tmp_path / "disk/h3-continuum-scratch"
|
||||
scratch.rename(tmp_path / "detached")
|
||||
with pytest.raises(OSError):
|
||||
buffer.allocate_file_backed_image((1,), dtype=torch.float32)
|
||||
store = cache()
|
||||
with pytest.raises(OSError):
|
||||
cache_put(store)
|
||||
assert not scratch.exists()
|
||||
assert not list(manager.default_decode().iterdir())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["finalize", "decode"])
|
||||
def test_symlink_substitution_protects_the_target(manager, tmp_path, kind):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
target = tmp_path / "disk/h3-continuum-scratch" / kind
|
||||
moved = tmp_path / "kept"
|
||||
target.rename(moved)
|
||||
try:
|
||||
target.symlink_to(moved, target_is_directory=True)
|
||||
except OSError:
|
||||
pytest.skip("OS does not allow creating symbolic links")
|
||||
report = manager.cleanup()
|
||||
assert any(row["errors"] for row in report["roots"] if row["kind"] == kind and row["current"])
|
||||
assert moved.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("read_only,busy", [(True, False), (False, True)])
|
||||
def test_server_enforces_read_only_and_busy(manager, tmp_path, read_only, busy):
|
||||
manager.read_only = read_only
|
||||
manager.busy = lambda: busy
|
||||
for action in (lambda: manager.validate(str(tmp_path / "disk")),
|
||||
lambda: manager.save(str(tmp_path / "disk"), 0), manager.cleanup):
|
||||
with pytest.raises(settings.StorageError):
|
||||
action()
|
||||
assert not manager.config_file.exists()
|
||||
|
||||
|
||||
def test_active_operation_blocks_maintenance_and_freezes_configuration(manager, tmp_path):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
errors = []
|
||||
def work():
|
||||
try:
|
||||
with manager.operation():
|
||||
entered.set()
|
||||
assert release.wait(3)
|
||||
except Exception as error:
|
||||
errors.append(error)
|
||||
worker = threading.Thread(target=work)
|
||||
worker.start()
|
||||
assert entered.wait(3)
|
||||
try:
|
||||
with pytest.raises(settings.StorageBusy):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
with pytest.raises(settings.StorageBusy):
|
||||
manager.cleanup()
|
||||
finally:
|
||||
release.set()
|
||||
worker.join(3)
|
||||
assert not errors and not worker.is_alive()
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
|
||||
|
||||
def test_ownerless_legacy_cache_is_never_swept(manager):
|
||||
root = manager.default_decode() / "h3-decode-cache-old"
|
||||
root.mkdir()
|
||||
path = root / ("b" * 32 + ".npy")
|
||||
path.write_bytes(b"unattributed")
|
||||
manager.cleanup()
|
||||
assert path.read_bytes() == b"unattributed"
|
||||
|
||||
|
||||
def test_dead_cache_owner_only_authorizes_registered_files(manager, monkeypatch):
|
||||
store = cache()
|
||||
tensor = cache_put(store)
|
||||
path = store.entries["first"].path
|
||||
unknown = path.parent / ("c" * 32 + ".npy")
|
||||
unknown.write_bytes(b"not ours")
|
||||
second = cache_put(store, "second")
|
||||
# Isolate the crash-recovery path: the real mapping remains protected until
|
||||
# released; only then simulate a terminated writer with no live store.
|
||||
del tensor, second
|
||||
manager._stores.clear()
|
||||
manager._cache_states.clear()
|
||||
monkeypatch.setattr(fs, "owner_alive", lambda owner: False)
|
||||
result = manager.cleanup()
|
||||
assert total_removed(result) == 2
|
||||
assert not path.exists() and unknown.read_bytes() == b"not ours"
|
||||
|
||||
|
||||
def test_deleted_helper_keeps_mapping_until_last_alias_is_released(manager):
|
||||
store = cache()
|
||||
output = cache_put(store)
|
||||
view = output[1:]
|
||||
path = store.entries["first"].path
|
||||
ref = weakref.ref(store)
|
||||
del output, store
|
||||
gc.collect()
|
||||
assert ref() is None and path.exists()
|
||||
assert total_removed(manager.cleanup()) == 0
|
||||
assert view.sum() == sum(range(12, 24))
|
||||
del view
|
||||
gc.collect()
|
||||
assert total_removed(manager.cleanup()) == 1
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("owner", [{}, {"pid": 1, "created": float("nan")}, {"pid": -1, "created": 1}])
|
||||
def test_unknown_process_ownership_is_protected(owner):
|
||||
assert fs.owner_alive(owner)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["relative/folder", "\\\\server\\share", "//server/share", "bad\0path"])
|
||||
def test_invalid_destinations_do_not_change_settings(manager, path):
|
||||
with pytest.raises(settings.StorageError):
|
||||
manager.save(path, 0)
|
||||
assert manager.status()["path"] == ""
|
||||
|
||||
|
||||
def test_existing_unknown_scratch_folder_is_not_adopted(manager, tmp_path):
|
||||
root = tmp_path / "disk/h3-continuum-scratch"
|
||||
root.mkdir(parents=True)
|
||||
(root / "important").write_text("keep")
|
||||
with pytest.raises(settings.StorageError, match="unknown"):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
assert (root / "important").read_text() == "keep"
|
||||
|
||||
|
||||
def test_mmap_failure_does_not_activate_destination(manager, tmp_path, monkeypatch):
|
||||
def fail(*args, **kwargs):
|
||||
raise OSError("mapping unsupported")
|
||||
monkeypatch.setattr(settings.mmap, "mmap", fail)
|
||||
with pytest.raises(OSError):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
assert manager.status()["path"] == ""
|
||||
assert not list(tmp_path.rglob(".h3-probe-*"))
|
||||
|
||||
|
||||
def test_manual_ram_uses_real_bytes_and_never_falls_back(monkeypatch, tmp_path):
|
||||
gib = 1024**3
|
||||
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (16*gib, 32*gib))
|
||||
buffer.preflight_manual_buffer("RAM", 5*gib) # no Auto 4 GiB cap
|
||||
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (6*gib, 32*gib))
|
||||
with pytest.raises(RuntimeError, match="RAM buffer needs"):
|
||||
buffer.preflight_manual_buffer("RAM", 3*gib, tmp_path)
|
||||
assert not list(tmp_path.iterdir())
|
||||
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (None, None))
|
||||
with pytest.raises(RuntimeError, match="unavailable"):
|
||||
buffer.preflight_manual_buffer("RAM", 1)
|
||||
|
||||
|
||||
def test_manual_disk_and_auto_disk_shortage_are_explicit(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(buffer, "_disk_free_bytes", lambda path: 1024)
|
||||
with pytest.raises(RuntimeError, match="Disk-backed buffer needs"):
|
||||
buffer.preflight_manual_buffer("Disk-backed", 1024, tmp_path)
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
|
||||
def test_cgroup_v2_uses_most_restrictive_ancestor(tmp_path):
|
||||
mount = tmp_path / "cgroup"
|
||||
child = mount / "nested"
|
||||
child.mkdir(parents=True)
|
||||
(mount / "memory.max").write_text("1000")
|
||||
(mount / "memory.current").write_text("900")
|
||||
(child / "memory.max").write_text("500")
|
||||
(child / "memory.current").write_text("200")
|
||||
proc = tmp_path / "cgroups"; proc.write_text("0::/nested\n")
|
||||
mounts = tmp_path / "mounts"; mounts.write_text(f"1 0 0:1 / {mount} rw - cgroup2 cgroup rw\n")
|
||||
assert list(memory._cgroup_limits(proc, mounts)) == [(300, 500), (100, 1000)]
|
||||
|
||||
|
||||
def test_real_terminated_cache_writer_is_reclaimed(manager, tmp_path):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
script = '''
|
||||
import os,sys,types
|
||||
from pathlib import Path
|
||||
import torch
|
||||
package=types.ModuleType("storage_child")
|
||||
package.__path__=[sys.argv[1]]
|
||||
sys.modules[package.__name__]=package
|
||||
from storage_child import storage_settings as settings
|
||||
from storage_child.decode_cache.store import DecodeStore
|
||||
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
|
||||
settings.install_manager(manager)
|
||||
store=DecodeStore(memory_probe=lambda:10**12,headroom_bytes=0)
|
||||
store.configure("Auto",1024**2,1024**2,0)
|
||||
output=store.put("entry",torch.ones(2,3,4),"video",{},(),store.epoch)
|
||||
assert output is not None
|
||||
os._exit(0)
|
||||
'''
|
||||
subprocess.run([sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)],
|
||||
check=True, timeout=30)
|
||||
cache_root = tmp_path / "disk/h3-continuum-scratch/decode"
|
||||
assert len(list(cache_root.rglob("*.npy"))) == 1
|
||||
assert total_removed(manager.cleanup()) == 1
|
||||
assert not list(cache_root.iterdir())
|
||||
|
||||
|
||||
def test_existing_manager_refuses_replaced_root_before_deletion(tmp_path):
|
||||
root = tmp_path / "original"
|
||||
manager = backing.StorageLifetimeManager(root)
|
||||
with manager.allocate((2,), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(3)
|
||||
output = allocation.publish()
|
||||
record = allocation.record
|
||||
del allocation, output
|
||||
moved = tmp_path / "detached"
|
||||
root.rename(moved)
|
||||
root.mkdir()
|
||||
protected = root / record.data_path.name
|
||||
protected.write_bytes(b"replacement data")
|
||||
with pytest.raises(backing.FileBackedPoCError, match="changed"):
|
||||
manager.collect_ready()
|
||||
assert protected.read_bytes() == b"replacement data"
|
||||
|
||||
|
||||
def test_failed_unpublished_cache_write_remains_cleanable(manager, monkeypatch):
|
||||
store = cache()
|
||||
def broken_blocks(tensor):
|
||||
yield b"partial"
|
||||
raise OSError("writer interrupted")
|
||||
original_unlink = Path.unlink
|
||||
def busy_temp(path, *args, **kwargs):
|
||||
if path.suffix == ".tmp":
|
||||
raise PermissionError("temporarily locked")
|
||||
return original_unlink(path, *args, **kwargs)
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(store_module, "tensor_blocks", broken_blocks)
|
||||
patch.setattr(Path, "unlink", busy_temp)
|
||||
with pytest.raises(OSError, match="writer interrupted"):
|
||||
cache_put(store)
|
||||
assert not store.entries
|
||||
assert len(list(store.root.glob("*.tmp"))) == 1
|
||||
assert total_removed(manager.cleanup()) == 1
|
||||
assert not list(store.root.glob("*.tmp"))
|
||||
|
||||
|
||||
def test_two_processes_cannot_overwrite_the_same_settings_revision(manager, tmp_path):
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
script = '''
|
||||
import sys,types
|
||||
from pathlib import Path
|
||||
package=types.ModuleType("storage_child")
|
||||
package.__path__=[sys.argv[1]]
|
||||
sys.modules[package.__name__]=package
|
||||
from storage_child import storage_settings as settings
|
||||
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
|
||||
sys.stdin.read(1)
|
||||
try:
|
||||
manager.save("",0)
|
||||
print("saved")
|
||||
except settings.StorageBusy:
|
||||
print("conflict")
|
||||
'''
|
||||
command = [sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)]
|
||||
children = [subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True) for _ in range(2)]
|
||||
try:
|
||||
for child in children:
|
||||
child.stdin.write("x"); child.stdin.flush()
|
||||
outputs = [child.communicate(timeout=15) for child in children]
|
||||
assert all(child.returncode == 0 for child in children), outputs
|
||||
assert sorted(out.strip() for out, err in outputs) == ["conflict", "saved"]
|
||||
assert manager.status()["revision"] == 1
|
||||
finally:
|
||||
for child in children:
|
||||
if child.poll() is None:
|
||||
child.kill(); child.wait()
|
||||
|
||||
|
||||
def test_cleanup_reports_a_failed_unlink_and_can_retry(manager, monkeypatch):
|
||||
store = cache()
|
||||
output = cache_put(store)
|
||||
path = store.entries["first"].path
|
||||
del output
|
||||
original = Path.unlink
|
||||
def denied(value, *args, **kwargs):
|
||||
if value == path:
|
||||
raise PermissionError("file locked")
|
||||
return original(value, *args, **kwargs)
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(Path, "unlink", denied)
|
||||
result = manager.cleanup()
|
||||
assert total_removed(result) == 0 and path.exists()
|
||||
assert any(row["errors"] for row in result["roots"])
|
||||
assert total_removed(manager.cleanup()) == 1 and not path.exists()
|
||||
|
||||
|
||||
def test_finalize_startup_also_preserves_ownerless_files(manager, tmp_path):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
root = buffer.resolve_backing_root()
|
||||
orphan = root / (backing.FILE_PREFIX + "d" * 32 + ".bin")
|
||||
orphan.write_bytes(b"unknown ownership")
|
||||
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(1)
|
||||
output = allocation.publish()
|
||||
assert output.item() == 1
|
||||
assert orphan.read_bytes() == b"unknown ownership"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [".bin", ".json"])
|
||||
def test_finalize_cleanup_does_not_follow_file_links(manager, tmp_path, monkeypatch, suffix):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
||||
output = allocation.publish()
|
||||
record = allocation.record
|
||||
del allocation, output
|
||||
target = record.data_path if suffix == ".bin" else record.metadata_path
|
||||
outside = tmp_path / ("outside" + suffix)
|
||||
target.rename(outside)
|
||||
try:
|
||||
target.symlink_to(outside)
|
||||
except OSError:
|
||||
pytest.skip("File symlinks are unavailable")
|
||||
before = outside.read_bytes()
|
||||
assert total_removed(manager.cleanup()) == 0
|
||||
assert target.is_symlink() and outside.read_bytes() == before
|
||||
|
||||
|
||||
def test_finalize_failed_delete_keeps_ownership_for_retry(manager, tmp_path, monkeypatch):
|
||||
manager.save(str(tmp_path / "disk"), 0)
|
||||
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
||||
output = allocation.publish()
|
||||
record = allocation.record
|
||||
del allocation, output
|
||||
original = Path.unlink
|
||||
def denied(value, *args, **kwargs):
|
||||
if value == record.data_path:
|
||||
raise PermissionError("file locked")
|
||||
return original(value, *args, **kwargs)
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(Path, "unlink", denied)
|
||||
result = manager.cleanup()
|
||||
assert total_removed(result) == 0 and record.metadata_path.exists()
|
||||
assert any(row["errors"] for row in result["roots"])
|
||||
assert total_removed(manager.cleanup()) == 2
|
||||
@@ -145,4 +145,4 @@ def test_v217_result_pack_expands_legacy_outputs():
|
||||
def test_v217_has_no_frontend_display_controller():
|
||||
assert not (ROOT / "web" / "h3_continuum_v2.js").exists()
|
||||
version_source = (ROOT / "version.py").read_text(encoding="utf-8")
|
||||
assert 'PACKAGE_VERSION = "3.9.2"' in version_source
|
||||
assert 'PACKAGE_VERSION = "3.9.3"' in version_source
|
||||
|
||||
Reference in New Issue
Block a user