Files
ComfyUI-H3-Continuum-Tiled/tests/test_p20_file_backed_tensor_poc.py
T
2026-08-24 15:00:45 +09:00

516 lines
17 KiB
Python

from __future__ import annotations
import gc
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import weakref
import pytest
import torch
MODULE_PATH = (
Path(__file__).resolve().parents[1] / "tools" / "p20_file_backed_tensor_poc.py"
)
SPEC = importlib.util.spec_from_file_location("h3c_p20_file_backed_tensor_poc", MODULE_PATH)
MODULE = importlib.util.module_from_spec(SPEC)
assert SPEC.loader is not None
sys.modules[SPEC.name] = MODULE
SPEC.loader.exec_module(MODULE)
def _known_files(root: Path) -> list[Path]:
return sorted(path for path in root.iterdir() if path.name.startswith(MODULE.FILE_PREFIX))
def test_shared_mapping_is_bit_exact_contiguous_and_write_through(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
reference = torch.arange(120, dtype=torch.float32).reshape(2, 3, 4, 5)
with manager.allocate(tuple(reference.shape), dtype=reference.dtype) as allocation:
allocation.tensor.copy_(reference)
record = allocation.record
output = allocation.publish()
assert output.shape == reference.shape
assert output.dtype == reference.dtype
assert output.device.type == "cpu"
assert output.is_contiguous()
assert output.contiguous().data_ptr() == output.data_ptr()
assert record.data_path.stat().st_size == reference.numel() * reference.element_size()
assert record.data_path.read_bytes() == reference.numpy().tobytes(order="C")
assert torch.equal(output, reference)
storage_pointer = output.untyped_storage().data_ptr()
assert output[1].untyped_storage().data_ptr() == storage_pointer
assert output.detach().untyped_storage().data_ptr() == storage_pointer
assert output.reshape(6, 4, 5).untyped_storage().data_ptr() == storage_pointer
assert output.clone().untyped_storage().data_ptr() != storage_pointer
del output
gc.collect()
result = manager.collect_ready()
assert result.reclaimed == 1
assert result.deferred == 0
assert not _known_files(tmp_path)
def test_lifetime_waits_for_last_storage_alias_and_not_root_tensor(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
with manager.allocate((4, 3, 2), dtype=torch.float32) as allocation:
allocation.tensor.copy_(torch.arange(24, dtype=torch.float32).reshape(4, 3, 2))
record = allocation.record
output = allocation.publish()
view = output[1:]
cache = {"images": view}
root_reference = weakref.ref(output)
del output
gc.collect()
assert root_reference() is None
assert float(cache["images"].sum()) == pytest.approx(261.0)
first = manager.collect_ready()
assert first.reclaimed == 0
assert first.deferred == 1
assert record.data_path.exists()
assert record.metadata_path.exists()
del view
cache.clear()
gc.collect()
second = manager.collect_ready()
assert second.reclaimed == 1
assert second.deferred == 0
assert not record.data_path.exists()
assert not record.metadata_path.exists()
@pytest.mark.parametrize("exception_type", [RuntimeError, MODULE.ProbeCancelled])
def test_unpublished_exception_and_cancel_remove_provisional_files(
tmp_path,
exception_type,
):
manager = MODULE.StorageLifetimeManager(tmp_path)
with pytest.raises(exception_type):
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
allocation.tensor.fill_(1.0)
raise exception_type("stop before publish")
assert manager.tracked_count() == 0
assert not _known_files(tmp_path)
def test_mapping_creation_failure_removes_reserved_file(tmp_path, monkeypatch):
manager = MODULE.StorageLifetimeManager(tmp_path)
def fail_from_file(*args, **kwargs):
raise RuntimeError("synthetic mapping failure")
monkeypatch.setattr(MODULE.torch, "from_file", fail_from_file)
with pytest.raises(RuntimeError, match="synthetic mapping failure"):
manager.allocate((2, 2), dtype=torch.float32)
assert not _known_files(tmp_path)
def test_temporary_unlink_failure_preserves_cancel_and_is_retried(
tmp_path,
monkeypatch,
):
manager = MODULE.StorageLifetimeManager(tmp_path)
original_unlink = MODULE._unlink_known
fail_unlink = {"enabled": True}
def sharing_violation(root, path):
if fail_unlink["enabled"]:
raise PermissionError(32, "synthetic sharing violation", str(path))
return original_unlink(root, path)
monkeypatch.setattr(MODULE, "_unlink_known", sharing_violation)
with pytest.raises(MODULE.ProbeCancelled, match="original cancel"):
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
allocation.tensor.fill_(1.0)
raise MODULE.ProbeCancelled("original cancel")
assert manager.pending_count() == 1
assert len(_known_files(tmp_path)) == 2
fail_unlink["enabled"] = False
result = manager.collect_ready()
assert result.reclaimed == 1
assert result.deferred == 0
assert manager.pending_count() == 0
assert not _known_files(tmp_path)
def test_cancel_after_backing_file_create_does_not_leave_orphan(
tmp_path,
monkeypatch,
):
manager = MODULE.StorageLifetimeManager(tmp_path)
original_close = MODULE.os.close
injected = False
sentinel_path = tmp_path / "unrelated-live-descriptor.bin"
sentinel_descriptor = None
def cancel_after_real_close(descriptor):
nonlocal injected, sentinel_descriptor
original_close(descriptor)
if not injected:
injected = True
opened = MODULE.os.open(
sentinel_path,
MODULE.os.O_CREAT | MODULE.os.O_EXCL | MODULE.os.O_RDWR,
0o600,
)
if opened != descriptor:
MODULE.os.dup2(opened, descriptor)
original_close(opened)
opened = descriptor
sentinel_descriptor = opened
raise MODULE.ProbeCancelled("cancel after backing file create")
try:
monkeypatch.setattr(MODULE.os, "close", cancel_after_real_close)
with pytest.raises(MODULE.ProbeCancelled, match="after backing file create"):
manager.allocate((2, 2), dtype=torch.float32)
finally:
monkeypatch.setattr(MODULE.os, "close", original_close)
assert sentinel_descriptor is not None
MODULE.os.write(sentinel_descriptor, b"still-open")
original_close(sentinel_descriptor)
assert sentinel_path.read_bytes() == b"still-open"
manager.collect_ready()
assert manager.pending_count() == 0
assert not _known_files(tmp_path)
def test_dropped_unpublished_allocation_is_finalized(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
allocation = manager.allocate((2, 2), dtype=torch.float32)
allocation.tensor.fill_(3.0)
allocation_reference = weakref.ref(allocation)
del allocation
gc.collect()
assert allocation_reference() is None
manager.collect_ready()
assert manager.pending_count() == 0
assert not _known_files(tmp_path)
def test_cache_requeue_keeps_backing_file_count_bounded(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
cache = {}
for index in range(20):
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
allocation.tensor.fill_(float(index))
output = allocation.publish()
cache["same_node_output"] = output
del output
gc.collect()
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) <= 2
same_requeue = cache["same_node_output"]
assert same_requeue is cache["same_node_output"]
manager.collect_ready()
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
cache.clear()
del same_requeue
gc.collect()
result = manager.collect_ready()
assert result.reclaimed == 1
assert not _known_files(tmp_path)
def test_published_downstream_error_keeps_cache_then_reclaims_after_eviction(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
cache = {}
try:
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
allocation.tensor.fill_(4.0)
output = allocation.publish()
cache["assembler_output"] = output
raise RuntimeError("downstream node failed")
except RuntimeError:
pass
del output
gc.collect()
held = manager.collect_ready()
assert held.reclaimed == 0
assert held.deferred == 1
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
cache.clear()
gc.collect()
released = manager.collect_ready()
assert released.reclaimed == 1
assert released.deferred == 0
assert not _known_files(tmp_path)
def test_process_global_manager_survives_node_local_scope(tmp_path):
first = MODULE.get_process_lifetime_manager(tmp_path)
second = MODULE.get_process_lifetime_manager(tmp_path)
assert first is second
with first.allocate((2, 2), dtype=torch.float32) as allocation:
output = allocation.publish()
manager_reference = weakref.ref(first)
del first
del second
gc.collect()
assert manager_reference() is not None
del output
gc.collect()
manager = MODULE.get_process_lifetime_manager(tmp_path)
assert manager.collect_ready().reclaimed == 1
assert not _known_files(tmp_path)
def test_stale_child_files_are_reclaimed_but_live_and_unknown_files_are_kept(tmp_path):
subprocess.run(
[
sys.executable,
str(MODULE_PATH),
"--root",
str(tmp_path),
"--crash-child",
],
check=True,
)
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.json"))) == 1
unknown = tmp_path / "unrelated.bin"
unknown.write_bytes(b"do not delete")
outside = tmp_path.parent / "outside.bin"
outside.write_bytes(b"outside root")
malicious_id = "f" * 32
malicious_metadata = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.json"
malicious_data = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.bin"
malicious_data.write_bytes(b"protected")
malicious_metadata.write_text(
json.dumps(
{
"schema": MODULE.SCHEMA,
"schema_version": MODULE.SCHEMA_VERSION,
"allocation_id": malicious_id,
"data_file": "..\\outside.bin",
"owner_pid": -1,
"created_at": 0,
}
),
encoding="utf-8",
)
result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
assert result.removed_pairs == 1
assert result.invalid_metadata == 1
assert unknown.read_bytes() == b"do not delete"
assert outside.read_bytes() == b"outside root"
assert malicious_data.read_bytes() == b"protected"
assert malicious_metadata.exists()
@pytest.mark.parametrize(
("field", "value", "remove"),
[
("owner_pid", None, True),
("owner_pid", -1, False),
("owner_create_time", float("nan"), False),
("created_at", float("nan"), False),
],
)
def test_invalid_ownership_metadata_never_deletes_live_mapping(
tmp_path,
field,
value,
remove,
):
manager = MODULE.StorageLifetimeManager(tmp_path)
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
record = allocation.record
output = allocation.publish()
payload = json.loads(record.metadata_path.read_text(encoding="utf-8"))
if remove:
payload.pop(field)
else:
payload[field] = value
record.metadata_path.write_text(json.dumps(payload), encoding="utf-8")
result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
assert result.invalid_metadata == 1
assert result.removed_pairs == 0
assert record.data_path.exists()
assert record.metadata_path.exists()
assert float(output.sum()) == pytest.approx(0.0)
del output
gc.collect()
assert manager.collect_ready().reclaimed == 1
assert not _known_files(tmp_path)
def test_cleanup_waits_for_in_progress_allocation_root_lock(tmp_path):
allocation_entered = threading.Event()
allow_allocation = threading.Event()
output_holder = []
cleanup_holder = []
def allocation_hook(_path):
allocation_entered.set()
assert allow_allocation.wait(timeout=5.0)
manager = MODULE.StorageLifetimeManager(
tmp_path,
allocation_hook=allocation_hook,
)
def allocate_output():
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
output_holder.append(allocation.publish())
def cleanup():
cleanup_holder.append(
MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
)
allocation_thread = threading.Thread(target=allocate_output)
allocation_thread.start()
assert allocation_entered.wait(timeout=5.0)
cleanup_thread = threading.Thread(target=cleanup)
cleanup_thread.start()
assert cleanup_thread.is_alive()
allow_allocation.set()
allocation_thread.join(timeout=5.0)
cleanup_thread.join(timeout=5.0)
assert not allocation_thread.is_alive()
assert not cleanup_thread.is_alive()
assert cleanup_holder[0].retained_live == 1
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
output_holder.clear()
gc.collect()
assert manager.collect_ready().reclaimed == 1
assert not _known_files(tmp_path)
def test_root_lock_is_reentrant_when_finalizer_runs_during_allocation(tmp_path):
first_manager = MODULE.StorageLifetimeManager(tmp_path)
abandoned = first_manager.allocate((2, 2), dtype=torch.float32)
cycle = [abandoned]
cycle.append(cycle)
del abandoned
del cycle
def collect_abandoned_allocation(_path):
gc.collect()
second_manager = MODULE.StorageLifetimeManager(
tmp_path,
allocation_hook=collect_abandoned_allocation,
)
started = time.perf_counter()
with second_manager.allocate((2, 2), dtype=torch.float32) as allocation:
output = allocation.publish()
elapsed = time.perf_counter() - started
assert elapsed < 2.0
first_manager.collect_ready()
del output
gc.collect()
second_manager.collect_ready()
assert not _known_files(tmp_path)
def test_cleanup_collect_and_gc_finalizer_have_no_lock_order_deadlock(tmp_path):
subprocess.run(
[
sys.executable,
str(MODULE_PATH),
"--root",
str(tmp_path),
"--lock-order-child",
],
check=True,
timeout=10.0,
)
assert not _known_files(tmp_path)
def test_live_sidecar_is_retained_and_cleanup_is_idempotent(tmp_path):
manager = MODULE.StorageLifetimeManager(tmp_path)
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
allocation.tensor.fill_(2.0)
record = allocation.record
output = allocation.publish()
live = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
assert live.retained_live == 1
assert record.data_path.exists()
assert record.metadata_path.exists()
del output
gc.collect()
assert manager.collect_ready().reclaimed == 1
empty = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
assert empty.removed_pairs == 0
assert empty.removed_orphans == 0
assert not _known_files(tmp_path)
def test_orphan_and_partial_cleanup_respects_grace_period(tmp_path):
allocation_id = "a" * 32
orphan = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.bin"
partial = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.json.partial"
orphan.write_bytes(b"orphan")
partial.write_text("partial", encoding="utf-8")
now = time.time()
young = MODULE.cleanup_stale_files(
tmp_path,
grace_seconds=60.0,
now=now,
)
assert young.retained_young == 2
assert orphan.exists()
assert partial.exists()
old = MODULE.cleanup_stale_files(
tmp_path,
grace_seconds=60.0,
now=now + 61.0,
)
assert old.removed_orphans == 2
assert not orphan.exists()
assert not partial.exists()
@pytest.mark.parametrize("shape", [(2.5, 2), (True, 2), ("2", 2), ()])
def test_shape_contract_rejects_lossy_or_empty_dimensions(tmp_path, shape):
manager = MODULE.StorageLifetimeManager(tmp_path)
with pytest.raises((TypeError, ValueError)):
manager.allocate(shape, dtype=torch.float32)
assert not _known_files(tmp_path)
def test_standalone_probe_passes(tmp_path):
result = MODULE.run_probe(tmp_path)
assert result.bit_exact
assert result.file_size == result.expected_bytes
assert result.root_released_while_alias_alive
assert result.deferred_while_alias_alive == 1
assert result.reclaimed_after_last_alias == 1
assert result.files_remaining == 0