Files

472 lines
19 KiB
Python

"""Scratch relocation and cleanup regressions with real small CPU mappings."""
from __future__ import annotations
import gc
import json
import os
import subprocess
import sys
from pathlib import Path
import threading
import weakref
from types import SimpleNamespace
import pytest
import torch
from ComfyUI_H3_Continuum_Join import storage_settings as settings
from ComfyUI_H3_Continuum_Join import storage_fs as fs
from ComfyUI_H3_Continuum_Join import storage_memory as memory
from ComfyUI_H3_Continuum_Join.decode_cache.store import DecodeStore
from ComfyUI_H3_Continuum_Join.decode_cache import store as store_module
from ComfyUI_H3_Continuum_Join.tools import p20_file_backed_tensor_poc as backing
from ComfyUI_H3_Continuum_Join.v3 import file_backed_buffer as buffer
@pytest.fixture
def manager(tmp_path, monkeypatch):
finalize, decode = tmp_path / "default-final", tmp_path / "default-decode"
decode.mkdir()
value = settings.StorageSettings(tmp_path / "user" / "settings.json",
default_finalize=lambda: finalize, default_decode=lambda: decode)
monkeypatch.setattr(settings, "_MANAGER", value)
monkeypatch.setenv("H3_DECODE_CACHE_TEMP_DIR", str(decode))
return value
def cache():
value = DecodeStore(memory_probe=lambda: 10**12, headroom_bytes=0)
value.configure("Auto", 1024**2, 1024**2, 0)
return value
def cache_put(store, name="first"):
x = torch.arange(24, dtype=torch.float32).reshape(2, 3, 4)
result = store.put(name, x, "video", {}, (), store.epoch)
assert result is not None
return result[0]
def total_removed(result):
return sum(row["removed_files"] for row in result["roots"])
def test_default_and_explicit_internal_paths_keep_their_precedence(manager, tmp_path, monkeypatch):
monkeypatch.setattr(buffer.tempfile, "gettempdir", lambda: str(tmp_path))
import sys
monkeypatch.setitem(sys.modules, "folder_paths", SimpleNamespace(get_temp_directory=lambda: str(tmp_path)))
assert buffer.resolve_backing_root() == tmp_path / "h3-continuum-v35-file-backed"
manager.save(str(tmp_path / "日本語 scratch"), 0)
assert buffer.resolve_backing_root() == tmp_path / "日本語 scratch/h3-continuum-scratch/finalize"
assert buffer.resolve_backing_root(tmp_path / "injected") == tmp_path / "injected"
store = cache()
output = cache_put(store)
assert store.root.parent == tmp_path / "日本語 scratch/h3-continuum-scratch/decode"
assert not list(manager.default_decode().iterdir())
del output
store.close()
def test_validation_does_not_activate_a_path_and_settings_survive_reload(manager, tmp_path):
destination = str(tmp_path / "other disk")
manager.validate(destination)
assert manager.status()["path"] == "" and not manager.config_file.exists()
manager.save(destination, 0)
restored = settings.StorageSettings(manager.config_file, default_finalize=manager.default_finalize,
default_decode=manager.default_decode)
assert restored.status()["path"] == destination
with pytest.raises(settings.StorageBusy, match="refresh"):
manager.save("", 0)
manager.save("", 1)
assert manager.status()["path"] == ""
def test_switch_keeps_old_finalize_alias_and_both_cache_mappings(manager, tmp_path):
manager.save(str(tmp_path / "disk-a"), 0)
with buffer.allocate_file_backed_image((2, 3, 4), dtype=torch.float32) as allocation:
allocation.tensor.copy_(torch.arange(24).reshape(2, 3, 4))
output = allocation.publish()
alias = output[1:]
old_final = allocation.record.data_path
del output, allocation
store = cache()
first = cache_put(store)
old_cache = store.entries["first"].path
manager.save(str(tmp_path / "disk-b"), 1)
second = cache_put(store, "second")
assert old_cache.parent != store.entries["second"].path.parent
assert torch.equal(first, second)
result = manager.cleanup()
assert total_removed(result) == 0
assert old_cache.exists() and old_final.exists()
assert alias.sum() == sum(range(12, 24))
del first, second, alias
gc.collect()
result = manager.cleanup()
assert total_removed(result) == 4 # one bin/sidecar pair and two npy files
assert not old_cache.exists() and not old_final.exists()
assert not store.contains("first") and not store.contains("second")
assert cache_put(store, "third").sum() == sum(range(24))
store.close()
def test_status_is_read_only_and_unknown_files_and_runs_are_protected(manager, tmp_path):
manager.save(str(tmp_path / "disk"), 0)
root = buffer.resolve_backing_root()
unknown = root / (backing.FILE_PREFIX + "a" * 32 + ".bin")
unknown.write_bytes(b"orphan")
run = tmp_path / "saved-run.safetensors"
run.write_bytes(b"accepted take")
store = cache()
tensor = cache_put(store)
path = store.entries["first"].path
del tensor
before = {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
report = manager.status()
assert any(row["deletable_bytes"] > 0 for row in report["roots"])
assert before == {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
assert store.contains("first") and path.exists()
manager.cleanup()
assert unknown.read_bytes() == b"orphan" and run.read_bytes() == b"accepted take"
assert not path.exists()
def test_changed_mount_is_not_recreated_or_replaced_by_default(manager, tmp_path):
manager.save(str(tmp_path / "disk"), 0)
scratch = tmp_path / "disk/h3-continuum-scratch"
scratch.rename(tmp_path / "detached")
with pytest.raises(OSError):
buffer.allocate_file_backed_image((1,), dtype=torch.float32)
store = cache()
with pytest.raises(OSError):
cache_put(store)
assert not scratch.exists()
assert not list(manager.default_decode().iterdir())
@pytest.mark.parametrize("kind", ["finalize", "decode"])
def test_symlink_substitution_protects_the_target(manager, tmp_path, kind):
manager.save(str(tmp_path / "disk"), 0)
target = tmp_path / "disk/h3-continuum-scratch" / kind
moved = tmp_path / "kept"
target.rename(moved)
try:
target.symlink_to(moved, target_is_directory=True)
except OSError:
pytest.skip("OS does not allow creating symbolic links")
report = manager.cleanup()
assert any(row["errors"] for row in report["roots"] if row["kind"] == kind and row["current"])
assert moved.exists()
@pytest.mark.parametrize("read_only,busy", [(True, False), (False, True)])
def test_server_enforces_read_only_and_busy(manager, tmp_path, read_only, busy):
manager.read_only = read_only
manager.busy = lambda: busy
for action in (lambda: manager.validate(str(tmp_path / "disk")),
lambda: manager.save(str(tmp_path / "disk"), 0), manager.cleanup):
with pytest.raises(settings.StorageError):
action()
assert not manager.config_file.exists()
def test_active_operation_blocks_maintenance_and_freezes_configuration(manager, tmp_path):
entered, release = threading.Event(), threading.Event()
errors = []
def work():
try:
with manager.operation():
entered.set()
assert release.wait(3)
except Exception as error:
errors.append(error)
worker = threading.Thread(target=work)
worker.start()
assert entered.wait(3)
try:
with pytest.raises(settings.StorageBusy):
manager.save(str(tmp_path / "disk"), 0)
with pytest.raises(settings.StorageBusy):
manager.cleanup()
finally:
release.set()
worker.join(3)
assert not errors and not worker.is_alive()
manager.save(str(tmp_path / "disk"), 0)
def test_ownerless_legacy_cache_is_never_swept(manager):
root = manager.default_decode() / "h3-decode-cache-old"
root.mkdir()
path = root / ("b" * 32 + ".npy")
path.write_bytes(b"unattributed")
manager.cleanup()
assert path.read_bytes() == b"unattributed"
def test_dead_cache_owner_only_authorizes_registered_files(manager, monkeypatch):
store = cache()
tensor = cache_put(store)
path = store.entries["first"].path
unknown = path.parent / ("c" * 32 + ".npy")
unknown.write_bytes(b"not ours")
second = cache_put(store, "second")
# Isolate the crash-recovery path: the real mapping remains protected until
# released; only then simulate a terminated writer with no live store.
del tensor, second
manager._stores.clear()
manager._cache_states.clear()
monkeypatch.setattr(fs, "owner_alive", lambda owner: False)
result = manager.cleanup()
assert total_removed(result) == 2
assert not path.exists() and unknown.read_bytes() == b"not ours"
def test_deleted_helper_keeps_mapping_until_last_alias_is_released(manager):
store = cache()
output = cache_put(store)
view = output[1:]
path = store.entries["first"].path
ref = weakref.ref(store)
del output, store
gc.collect()
assert ref() is None and path.exists()
assert total_removed(manager.cleanup()) == 0
assert view.sum() == sum(range(12, 24))
del view
gc.collect()
assert total_removed(manager.cleanup()) == 1
assert not path.exists()
@pytest.mark.parametrize("owner", [{}, {"pid": 1, "created": float("nan")}, {"pid": -1, "created": 1}])
def test_unknown_process_ownership_is_protected(owner):
assert fs.owner_alive(owner)
@pytest.mark.parametrize("path", ["relative/folder", "\\\\server\\share", "//server/share", "bad\0path"])
def test_invalid_destinations_do_not_change_settings(manager, path):
with pytest.raises(settings.StorageError):
manager.save(path, 0)
assert manager.status()["path"] == ""
def test_existing_unknown_scratch_folder_is_not_adopted(manager, tmp_path):
root = tmp_path / "disk/h3-continuum-scratch"
root.mkdir(parents=True)
(root / "important").write_text("keep")
with pytest.raises(settings.StorageError, match="unknown"):
manager.save(str(tmp_path / "disk"), 0)
assert (root / "important").read_text() == "keep"
def test_mmap_failure_does_not_activate_destination(manager, tmp_path, monkeypatch):
def fail(*args, **kwargs):
raise OSError("mapping unsupported")
monkeypatch.setattr(settings.mmap, "mmap", fail)
with pytest.raises(OSError):
manager.save(str(tmp_path / "disk"), 0)
assert manager.status()["path"] == ""
assert not list(tmp_path.rglob(".h3-probe-*"))
def test_manual_ram_uses_real_bytes_and_never_falls_back(monkeypatch, tmp_path):
gib = 1024**3
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (16*gib, 32*gib))
buffer.preflight_manual_buffer("RAM", 5*gib) # no Auto 4 GiB cap
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (6*gib, 32*gib))
with pytest.raises(RuntimeError, match="RAM buffer needs"):
buffer.preflight_manual_buffer("RAM", 3*gib, tmp_path)
assert not list(tmp_path.iterdir())
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (None, None))
with pytest.raises(RuntimeError, match="unavailable"):
buffer.preflight_manual_buffer("RAM", 1)
def test_manual_disk_and_auto_disk_shortage_are_explicit(monkeypatch, tmp_path):
monkeypatch.setattr(buffer, "_disk_free_bytes", lambda path: 1024)
with pytest.raises(RuntimeError, match="Disk-backed buffer needs"):
buffer.preflight_manual_buffer("Disk-backed", 1024, tmp_path)
assert not list(tmp_path.iterdir())
def test_cgroup_v2_uses_most_restrictive_ancestor(tmp_path):
mount = tmp_path / "cgroup"
child = mount / "nested"
child.mkdir(parents=True)
(mount / "memory.max").write_text("1000")
(mount / "memory.current").write_text("900")
(child / "memory.max").write_text("500")
(child / "memory.current").write_text("200")
proc = tmp_path / "cgroups"; proc.write_text("0::/nested\n")
mounts = tmp_path / "mounts"; mounts.write_text(f"1 0 0:1 / {mount} rw - cgroup2 cgroup rw\n")
assert list(memory._cgroup_limits(proc, mounts)) == [(300, 500), (100, 1000)]
def test_real_terminated_cache_writer_is_reclaimed(manager, tmp_path):
manager.save(str(tmp_path / "disk"), 0)
repository = Path(__file__).resolve().parents[1]
script = '''
import os,sys,types
from pathlib import Path
import torch
package=types.ModuleType("storage_child")
package.__path__=[sys.argv[1]]
sys.modules[package.__name__]=package
from storage_child import storage_settings as settings
from storage_child.decode_cache.store import DecodeStore
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
settings.install_manager(manager)
store=DecodeStore(memory_probe=lambda:10**12,headroom_bytes=0)
store.configure("Auto",1024**2,1024**2,0)
output=store.put("entry",torch.ones(2,3,4),"video",{},(),store.epoch)
assert output is not None
os._exit(0)
'''
subprocess.run([sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)],
check=True, timeout=30)
cache_root = tmp_path / "disk/h3-continuum-scratch/decode"
assert len(list(cache_root.rglob("*.npy"))) == 1
assert total_removed(manager.cleanup()) == 1
assert not list(cache_root.iterdir())
def test_existing_manager_refuses_replaced_root_before_deletion(tmp_path):
root = tmp_path / "original"
manager = backing.StorageLifetimeManager(root)
with manager.allocate((2,), dtype=torch.float32) as allocation:
allocation.tensor.fill_(3)
output = allocation.publish()
record = allocation.record
del allocation, output
moved = tmp_path / "detached"
root.rename(moved)
root.mkdir()
protected = root / record.data_path.name
protected.write_bytes(b"replacement data")
with pytest.raises(backing.FileBackedPoCError, match="changed"):
manager.collect_ready()
assert protected.read_bytes() == b"replacement data"
def test_failed_unpublished_cache_write_remains_cleanable(manager, monkeypatch):
store = cache()
def broken_blocks(tensor):
yield b"partial"
raise OSError("writer interrupted")
original_unlink = Path.unlink
def busy_temp(path, *args, **kwargs):
if path.suffix == ".tmp":
raise PermissionError("temporarily locked")
return original_unlink(path, *args, **kwargs)
with monkeypatch.context() as patch:
patch.setattr(store_module, "tensor_blocks", broken_blocks)
patch.setattr(Path, "unlink", busy_temp)
with pytest.raises(OSError, match="writer interrupted"):
cache_put(store)
assert not store.entries
assert len(list(store.root.glob("*.tmp"))) == 1
assert total_removed(manager.cleanup()) == 1
assert not list(store.root.glob("*.tmp"))
def test_two_processes_cannot_overwrite_the_same_settings_revision(manager, tmp_path):
repository = Path(__file__).resolve().parents[1]
script = '''
import sys,types
from pathlib import Path
package=types.ModuleType("storage_child")
package.__path__=[sys.argv[1]]
sys.modules[package.__name__]=package
from storage_child import storage_settings as settings
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
sys.stdin.read(1)
try:
manager.save("",0)
print("saved")
except settings.StorageBusy:
print("conflict")
'''
command = [sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)]
children = [subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True) for _ in range(2)]
try:
for child in children:
child.stdin.write("x"); child.stdin.flush()
outputs = [child.communicate(timeout=15) for child in children]
assert all(child.returncode == 0 for child in children), outputs
assert sorted(out.strip() for out, err in outputs) == ["conflict", "saved"]
assert manager.status()["revision"] == 1
finally:
for child in children:
if child.poll() is None:
child.kill(); child.wait()
def test_cleanup_reports_a_failed_unlink_and_can_retry(manager, monkeypatch):
store = cache()
output = cache_put(store)
path = store.entries["first"].path
del output
original = Path.unlink
def denied(value, *args, **kwargs):
if value == path:
raise PermissionError("file locked")
return original(value, *args, **kwargs)
with monkeypatch.context() as patch:
patch.setattr(Path, "unlink", denied)
result = manager.cleanup()
assert total_removed(result) == 0 and path.exists()
assert any(row["errors"] for row in result["roots"])
assert total_removed(manager.cleanup()) == 1 and not path.exists()
def test_finalize_startup_also_preserves_ownerless_files(manager, tmp_path):
manager.save(str(tmp_path / "disk"), 0)
root = buffer.resolve_backing_root()
orphan = root / (backing.FILE_PREFIX + "d" * 32 + ".bin")
orphan.write_bytes(b"unknown ownership")
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
allocation.tensor.fill_(1)
output = allocation.publish()
assert output.item() == 1
assert orphan.read_bytes() == b"unknown ownership"
@pytest.mark.parametrize("suffix", [".bin", ".json"])
def test_finalize_cleanup_does_not_follow_file_links(manager, tmp_path, monkeypatch, suffix):
manager.save(str(tmp_path / "disk"), 0)
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
output = allocation.publish()
record = allocation.record
del allocation, output
target = record.data_path if suffix == ".bin" else record.metadata_path
outside = tmp_path / ("outside" + suffix)
target.rename(outside)
try:
target.symlink_to(outside)
except OSError:
pytest.skip("File symlinks are unavailable")
before = outside.read_bytes()
assert total_removed(manager.cleanup()) == 0
assert target.is_symlink() and outside.read_bytes() == before
def test_finalize_failed_delete_keeps_ownership_for_retry(manager, tmp_path, monkeypatch):
manager.save(str(tmp_path / "disk"), 0)
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
output = allocation.publish()
record = allocation.record
del allocation, output
original = Path.unlink
def denied(value, *args, **kwargs):
if value == record.data_path:
raise PermissionError("file locked")
return original(value, *args, **kwargs)
with monkeypatch.context() as patch:
patch.setattr(Path, "unlink", denied)
result = manager.cleanup()
assert total_removed(result) == 0 and record.metadata_path.exists()
assert any(row["errors"] for row in result["roots"])
assert total_removed(manager.cleanup()) == 2