git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
472 lines
19 KiB
Python
472 lines
19 KiB
Python
"""Scratch relocation and cleanup regressions with real small CPU mappings."""
|
|
from __future__ import annotations
|
|
|
|
import gc
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
import threading
|
|
import weakref
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from ComfyUI_H3_Continuum_Join import storage_settings as settings
|
|
from ComfyUI_H3_Continuum_Join import storage_fs as fs
|
|
from ComfyUI_H3_Continuum_Join import storage_memory as memory
|
|
from ComfyUI_H3_Continuum_Join.decode_cache.store import DecodeStore
|
|
from ComfyUI_H3_Continuum_Join.decode_cache import store as store_module
|
|
from ComfyUI_H3_Continuum_Join.tools import p20_file_backed_tensor_poc as backing
|
|
from ComfyUI_H3_Continuum_Join.v3 import file_backed_buffer as buffer
|
|
|
|
|
|
@pytest.fixture
|
|
def manager(tmp_path, monkeypatch):
|
|
finalize, decode = tmp_path / "default-final", tmp_path / "default-decode"
|
|
decode.mkdir()
|
|
value = settings.StorageSettings(tmp_path / "user" / "settings.json",
|
|
default_finalize=lambda: finalize, default_decode=lambda: decode)
|
|
monkeypatch.setattr(settings, "_MANAGER", value)
|
|
monkeypatch.setenv("H3_DECODE_CACHE_TEMP_DIR", str(decode))
|
|
return value
|
|
|
|
|
|
def cache():
|
|
value = DecodeStore(memory_probe=lambda: 10**12, headroom_bytes=0)
|
|
value.configure("Auto", 1024**2, 1024**2, 0)
|
|
return value
|
|
|
|
|
|
def cache_put(store, name="first"):
|
|
x = torch.arange(24, dtype=torch.float32).reshape(2, 3, 4)
|
|
result = store.put(name, x, "video", {}, (), store.epoch)
|
|
assert result is not None
|
|
return result[0]
|
|
|
|
|
|
def total_removed(result):
|
|
return sum(row["removed_files"] for row in result["roots"])
|
|
|
|
|
|
def test_default_and_explicit_internal_paths_keep_their_precedence(manager, tmp_path, monkeypatch):
|
|
monkeypatch.setattr(buffer.tempfile, "gettempdir", lambda: str(tmp_path))
|
|
import sys
|
|
monkeypatch.setitem(sys.modules, "folder_paths", SimpleNamespace(get_temp_directory=lambda: str(tmp_path)))
|
|
assert buffer.resolve_backing_root() == tmp_path / "h3-continuum-v35-file-backed"
|
|
manager.save(str(tmp_path / "日本語 scratch"), 0)
|
|
assert buffer.resolve_backing_root() == tmp_path / "日本語 scratch/h3-continuum-scratch/finalize"
|
|
assert buffer.resolve_backing_root(tmp_path / "injected") == tmp_path / "injected"
|
|
store = cache()
|
|
output = cache_put(store)
|
|
assert store.root.parent == tmp_path / "日本語 scratch/h3-continuum-scratch/decode"
|
|
assert not list(manager.default_decode().iterdir())
|
|
del output
|
|
store.close()
|
|
|
|
|
|
def test_validation_does_not_activate_a_path_and_settings_survive_reload(manager, tmp_path):
|
|
destination = str(tmp_path / "other disk")
|
|
manager.validate(destination)
|
|
assert manager.status()["path"] == "" and not manager.config_file.exists()
|
|
manager.save(destination, 0)
|
|
restored = settings.StorageSettings(manager.config_file, default_finalize=manager.default_finalize,
|
|
default_decode=manager.default_decode)
|
|
assert restored.status()["path"] == destination
|
|
with pytest.raises(settings.StorageBusy, match="refresh"):
|
|
manager.save("", 0)
|
|
manager.save("", 1)
|
|
assert manager.status()["path"] == ""
|
|
|
|
|
|
def test_switch_keeps_old_finalize_alias_and_both_cache_mappings(manager, tmp_path):
|
|
manager.save(str(tmp_path / "disk-a"), 0)
|
|
with buffer.allocate_file_backed_image((2, 3, 4), dtype=torch.float32) as allocation:
|
|
allocation.tensor.copy_(torch.arange(24).reshape(2, 3, 4))
|
|
output = allocation.publish()
|
|
alias = output[1:]
|
|
old_final = allocation.record.data_path
|
|
del output, allocation
|
|
store = cache()
|
|
first = cache_put(store)
|
|
old_cache = store.entries["first"].path
|
|
manager.save(str(tmp_path / "disk-b"), 1)
|
|
second = cache_put(store, "second")
|
|
assert old_cache.parent != store.entries["second"].path.parent
|
|
assert torch.equal(first, second)
|
|
result = manager.cleanup()
|
|
assert total_removed(result) == 0
|
|
assert old_cache.exists() and old_final.exists()
|
|
assert alias.sum() == sum(range(12, 24))
|
|
del first, second, alias
|
|
gc.collect()
|
|
result = manager.cleanup()
|
|
assert total_removed(result) == 4 # one bin/sidecar pair and two npy files
|
|
assert not old_cache.exists() and not old_final.exists()
|
|
assert not store.contains("first") and not store.contains("second")
|
|
assert cache_put(store, "third").sum() == sum(range(24))
|
|
store.close()
|
|
|
|
|
|
def test_status_is_read_only_and_unknown_files_and_runs_are_protected(manager, tmp_path):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
root = buffer.resolve_backing_root()
|
|
unknown = root / (backing.FILE_PREFIX + "a" * 32 + ".bin")
|
|
unknown.write_bytes(b"orphan")
|
|
run = tmp_path / "saved-run.safetensors"
|
|
run.write_bytes(b"accepted take")
|
|
store = cache()
|
|
tensor = cache_put(store)
|
|
path = store.entries["first"].path
|
|
del tensor
|
|
before = {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
|
|
report = manager.status()
|
|
assert any(row["deletable_bytes"] > 0 for row in report["roots"])
|
|
assert before == {p: p.read_bytes() for p in tmp_path.rglob("*") if p.is_file()}
|
|
assert store.contains("first") and path.exists()
|
|
manager.cleanup()
|
|
assert unknown.read_bytes() == b"orphan" and run.read_bytes() == b"accepted take"
|
|
assert not path.exists()
|
|
|
|
|
|
def test_changed_mount_is_not_recreated_or_replaced_by_default(manager, tmp_path):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
scratch = tmp_path / "disk/h3-continuum-scratch"
|
|
scratch.rename(tmp_path / "detached")
|
|
with pytest.raises(OSError):
|
|
buffer.allocate_file_backed_image((1,), dtype=torch.float32)
|
|
store = cache()
|
|
with pytest.raises(OSError):
|
|
cache_put(store)
|
|
assert not scratch.exists()
|
|
assert not list(manager.default_decode().iterdir())
|
|
|
|
|
|
@pytest.mark.parametrize("kind", ["finalize", "decode"])
|
|
def test_symlink_substitution_protects_the_target(manager, tmp_path, kind):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
target = tmp_path / "disk/h3-continuum-scratch" / kind
|
|
moved = tmp_path / "kept"
|
|
target.rename(moved)
|
|
try:
|
|
target.symlink_to(moved, target_is_directory=True)
|
|
except OSError:
|
|
pytest.skip("OS does not allow creating symbolic links")
|
|
report = manager.cleanup()
|
|
assert any(row["errors"] for row in report["roots"] if row["kind"] == kind and row["current"])
|
|
assert moved.exists()
|
|
|
|
|
|
@pytest.mark.parametrize("read_only,busy", [(True, False), (False, True)])
|
|
def test_server_enforces_read_only_and_busy(manager, tmp_path, read_only, busy):
|
|
manager.read_only = read_only
|
|
manager.busy = lambda: busy
|
|
for action in (lambda: manager.validate(str(tmp_path / "disk")),
|
|
lambda: manager.save(str(tmp_path / "disk"), 0), manager.cleanup):
|
|
with pytest.raises(settings.StorageError):
|
|
action()
|
|
assert not manager.config_file.exists()
|
|
|
|
|
|
def test_active_operation_blocks_maintenance_and_freezes_configuration(manager, tmp_path):
|
|
entered, release = threading.Event(), threading.Event()
|
|
errors = []
|
|
def work():
|
|
try:
|
|
with manager.operation():
|
|
entered.set()
|
|
assert release.wait(3)
|
|
except Exception as error:
|
|
errors.append(error)
|
|
worker = threading.Thread(target=work)
|
|
worker.start()
|
|
assert entered.wait(3)
|
|
try:
|
|
with pytest.raises(settings.StorageBusy):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
with pytest.raises(settings.StorageBusy):
|
|
manager.cleanup()
|
|
finally:
|
|
release.set()
|
|
worker.join(3)
|
|
assert not errors and not worker.is_alive()
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
|
|
|
|
def test_ownerless_legacy_cache_is_never_swept(manager):
|
|
root = manager.default_decode() / "h3-decode-cache-old"
|
|
root.mkdir()
|
|
path = root / ("b" * 32 + ".npy")
|
|
path.write_bytes(b"unattributed")
|
|
manager.cleanup()
|
|
assert path.read_bytes() == b"unattributed"
|
|
|
|
|
|
def test_dead_cache_owner_only_authorizes_registered_files(manager, monkeypatch):
|
|
store = cache()
|
|
tensor = cache_put(store)
|
|
path = store.entries["first"].path
|
|
unknown = path.parent / ("c" * 32 + ".npy")
|
|
unknown.write_bytes(b"not ours")
|
|
second = cache_put(store, "second")
|
|
# Isolate the crash-recovery path: the real mapping remains protected until
|
|
# released; only then simulate a terminated writer with no live store.
|
|
del tensor, second
|
|
manager._stores.clear()
|
|
manager._cache_states.clear()
|
|
monkeypatch.setattr(fs, "owner_alive", lambda owner: False)
|
|
result = manager.cleanup()
|
|
assert total_removed(result) == 2
|
|
assert not path.exists() and unknown.read_bytes() == b"not ours"
|
|
|
|
|
|
def test_deleted_helper_keeps_mapping_until_last_alias_is_released(manager):
|
|
store = cache()
|
|
output = cache_put(store)
|
|
view = output[1:]
|
|
path = store.entries["first"].path
|
|
ref = weakref.ref(store)
|
|
del output, store
|
|
gc.collect()
|
|
assert ref() is None and path.exists()
|
|
assert total_removed(manager.cleanup()) == 0
|
|
assert view.sum() == sum(range(12, 24))
|
|
del view
|
|
gc.collect()
|
|
assert total_removed(manager.cleanup()) == 1
|
|
assert not path.exists()
|
|
|
|
|
|
@pytest.mark.parametrize("owner", [{}, {"pid": 1, "created": float("nan")}, {"pid": -1, "created": 1}])
|
|
def test_unknown_process_ownership_is_protected(owner):
|
|
assert fs.owner_alive(owner)
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["relative/folder", "\\\\server\\share", "//server/share", "bad\0path"])
|
|
def test_invalid_destinations_do_not_change_settings(manager, path):
|
|
with pytest.raises(settings.StorageError):
|
|
manager.save(path, 0)
|
|
assert manager.status()["path"] == ""
|
|
|
|
|
|
def test_existing_unknown_scratch_folder_is_not_adopted(manager, tmp_path):
|
|
root = tmp_path / "disk/h3-continuum-scratch"
|
|
root.mkdir(parents=True)
|
|
(root / "important").write_text("keep")
|
|
with pytest.raises(settings.StorageError, match="unknown"):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
assert (root / "important").read_text() == "keep"
|
|
|
|
|
|
def test_mmap_failure_does_not_activate_destination(manager, tmp_path, monkeypatch):
|
|
def fail(*args, **kwargs):
|
|
raise OSError("mapping unsupported")
|
|
monkeypatch.setattr(settings.mmap, "mmap", fail)
|
|
with pytest.raises(OSError):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
assert manager.status()["path"] == ""
|
|
assert not list(tmp_path.rglob(".h3-probe-*"))
|
|
|
|
|
|
def test_manual_ram_uses_real_bytes_and_never_falls_back(monkeypatch, tmp_path):
|
|
gib = 1024**3
|
|
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (16*gib, 32*gib))
|
|
buffer.preflight_manual_buffer("RAM", 5*gib) # no Auto 4 GiB cap
|
|
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (6*gib, 32*gib))
|
|
with pytest.raises(RuntimeError, match="RAM buffer needs"):
|
|
buffer.preflight_manual_buffer("RAM", 3*gib, tmp_path)
|
|
assert not list(tmp_path.iterdir())
|
|
monkeypatch.setattr(buffer, "_system_memory_bytes", lambda: (None, None))
|
|
with pytest.raises(RuntimeError, match="unavailable"):
|
|
buffer.preflight_manual_buffer("RAM", 1)
|
|
|
|
|
|
def test_manual_disk_and_auto_disk_shortage_are_explicit(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(buffer, "_disk_free_bytes", lambda path: 1024)
|
|
with pytest.raises(RuntimeError, match="Disk-backed buffer needs"):
|
|
buffer.preflight_manual_buffer("Disk-backed", 1024, tmp_path)
|
|
assert not list(tmp_path.iterdir())
|
|
|
|
|
|
def test_cgroup_v2_uses_most_restrictive_ancestor(tmp_path):
|
|
mount = tmp_path / "cgroup"
|
|
child = mount / "nested"
|
|
child.mkdir(parents=True)
|
|
(mount / "memory.max").write_text("1000")
|
|
(mount / "memory.current").write_text("900")
|
|
(child / "memory.max").write_text("500")
|
|
(child / "memory.current").write_text("200")
|
|
proc = tmp_path / "cgroups"; proc.write_text("0::/nested\n")
|
|
mounts = tmp_path / "mounts"; mounts.write_text(f"1 0 0:1 / {mount} rw - cgroup2 cgroup rw\n")
|
|
assert list(memory._cgroup_limits(proc, mounts)) == [(300, 500), (100, 1000)]
|
|
|
|
|
|
def test_real_terminated_cache_writer_is_reclaimed(manager, tmp_path):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
repository = Path(__file__).resolve().parents[1]
|
|
script = '''
|
|
import os,sys,types
|
|
from pathlib import Path
|
|
import torch
|
|
package=types.ModuleType("storage_child")
|
|
package.__path__=[sys.argv[1]]
|
|
sys.modules[package.__name__]=package
|
|
from storage_child import storage_settings as settings
|
|
from storage_child.decode_cache.store import DecodeStore
|
|
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
|
|
settings.install_manager(manager)
|
|
store=DecodeStore(memory_probe=lambda:10**12,headroom_bytes=0)
|
|
store.configure("Auto",1024**2,1024**2,0)
|
|
output=store.put("entry",torch.ones(2,3,4),"video",{},(),store.epoch)
|
|
assert output is not None
|
|
os._exit(0)
|
|
'''
|
|
subprocess.run([sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)],
|
|
check=True, timeout=30)
|
|
cache_root = tmp_path / "disk/h3-continuum-scratch/decode"
|
|
assert len(list(cache_root.rglob("*.npy"))) == 1
|
|
assert total_removed(manager.cleanup()) == 1
|
|
assert not list(cache_root.iterdir())
|
|
|
|
|
|
def test_existing_manager_refuses_replaced_root_before_deletion(tmp_path):
|
|
root = tmp_path / "original"
|
|
manager = backing.StorageLifetimeManager(root)
|
|
with manager.allocate((2,), dtype=torch.float32) as allocation:
|
|
allocation.tensor.fill_(3)
|
|
output = allocation.publish()
|
|
record = allocation.record
|
|
del allocation, output
|
|
moved = tmp_path / "detached"
|
|
root.rename(moved)
|
|
root.mkdir()
|
|
protected = root / record.data_path.name
|
|
protected.write_bytes(b"replacement data")
|
|
with pytest.raises(backing.FileBackedPoCError, match="changed"):
|
|
manager.collect_ready()
|
|
assert protected.read_bytes() == b"replacement data"
|
|
|
|
|
|
def test_failed_unpublished_cache_write_remains_cleanable(manager, monkeypatch):
|
|
store = cache()
|
|
def broken_blocks(tensor):
|
|
yield b"partial"
|
|
raise OSError("writer interrupted")
|
|
original_unlink = Path.unlink
|
|
def busy_temp(path, *args, **kwargs):
|
|
if path.suffix == ".tmp":
|
|
raise PermissionError("temporarily locked")
|
|
return original_unlink(path, *args, **kwargs)
|
|
with monkeypatch.context() as patch:
|
|
patch.setattr(store_module, "tensor_blocks", broken_blocks)
|
|
patch.setattr(Path, "unlink", busy_temp)
|
|
with pytest.raises(OSError, match="writer interrupted"):
|
|
cache_put(store)
|
|
assert not store.entries
|
|
assert len(list(store.root.glob("*.tmp"))) == 1
|
|
assert total_removed(manager.cleanup()) == 1
|
|
assert not list(store.root.glob("*.tmp"))
|
|
|
|
|
|
def test_two_processes_cannot_overwrite_the_same_settings_revision(manager, tmp_path):
|
|
repository = Path(__file__).resolve().parents[1]
|
|
script = '''
|
|
import sys,types
|
|
from pathlib import Path
|
|
package=types.ModuleType("storage_child")
|
|
package.__path__=[sys.argv[1]]
|
|
sys.modules[package.__name__]=package
|
|
from storage_child import storage_settings as settings
|
|
manager=settings.StorageSettings(Path(sys.argv[2]),default_finalize=lambda:Path(sys.argv[3]),default_decode=lambda:Path(sys.argv[3]))
|
|
sys.stdin.read(1)
|
|
try:
|
|
manager.save("",0)
|
|
print("saved")
|
|
except settings.StorageBusy:
|
|
print("conflict")
|
|
'''
|
|
command = [sys.executable, "-c", script, str(repository), str(manager.config_file), str(tmp_path)]
|
|
children = [subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=True) for _ in range(2)]
|
|
try:
|
|
for child in children:
|
|
child.stdin.write("x"); child.stdin.flush()
|
|
outputs = [child.communicate(timeout=15) for child in children]
|
|
assert all(child.returncode == 0 for child in children), outputs
|
|
assert sorted(out.strip() for out, err in outputs) == ["conflict", "saved"]
|
|
assert manager.status()["revision"] == 1
|
|
finally:
|
|
for child in children:
|
|
if child.poll() is None:
|
|
child.kill(); child.wait()
|
|
|
|
|
|
def test_cleanup_reports_a_failed_unlink_and_can_retry(manager, monkeypatch):
|
|
store = cache()
|
|
output = cache_put(store)
|
|
path = store.entries["first"].path
|
|
del output
|
|
original = Path.unlink
|
|
def denied(value, *args, **kwargs):
|
|
if value == path:
|
|
raise PermissionError("file locked")
|
|
return original(value, *args, **kwargs)
|
|
with monkeypatch.context() as patch:
|
|
patch.setattr(Path, "unlink", denied)
|
|
result = manager.cleanup()
|
|
assert total_removed(result) == 0 and path.exists()
|
|
assert any(row["errors"] for row in result["roots"])
|
|
assert total_removed(manager.cleanup()) == 1 and not path.exists()
|
|
|
|
|
|
def test_finalize_startup_also_preserves_ownerless_files(manager, tmp_path):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
root = buffer.resolve_backing_root()
|
|
orphan = root / (backing.FILE_PREFIX + "d" * 32 + ".bin")
|
|
orphan.write_bytes(b"unknown ownership")
|
|
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
|
allocation.tensor.fill_(1)
|
|
output = allocation.publish()
|
|
assert output.item() == 1
|
|
assert orphan.read_bytes() == b"unknown ownership"
|
|
|
|
|
|
@pytest.mark.parametrize("suffix", [".bin", ".json"])
|
|
def test_finalize_cleanup_does_not_follow_file_links(manager, tmp_path, monkeypatch, suffix):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
|
output = allocation.publish()
|
|
record = allocation.record
|
|
del allocation, output
|
|
target = record.data_path if suffix == ".bin" else record.metadata_path
|
|
outside = tmp_path / ("outside" + suffix)
|
|
target.rename(outside)
|
|
try:
|
|
target.symlink_to(outside)
|
|
except OSError:
|
|
pytest.skip("File symlinks are unavailable")
|
|
before = outside.read_bytes()
|
|
assert total_removed(manager.cleanup()) == 0
|
|
assert target.is_symlink() and outside.read_bytes() == before
|
|
|
|
|
|
def test_finalize_failed_delete_keeps_ownership_for_retry(manager, tmp_path, monkeypatch):
|
|
manager.save(str(tmp_path / "disk"), 0)
|
|
with buffer.allocate_file_backed_image((1,), dtype=torch.float32) as allocation:
|
|
output = allocation.publish()
|
|
record = allocation.record
|
|
del allocation, output
|
|
original = Path.unlink
|
|
def denied(value, *args, **kwargs):
|
|
if value == record.data_path:
|
|
raise PermissionError("file locked")
|
|
return original(value, *args, **kwargs)
|
|
with monkeypatch.context() as patch:
|
|
patch.setattr(Path, "unlink", denied)
|
|
result = manager.cleanup()
|
|
assert total_removed(result) == 0 and record.metadata_path.exists()
|
|
assert any(row["errors"] for row in result["roots"])
|
|
assert total_removed(manager.cleanup()) == 2
|