Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
This commit is contained in:
@@ -0,0 +1,515 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gc
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1] / "tools" / "p20_file_backed_tensor_poc.py"
|
||||
)
|
||||
SPEC = importlib.util.spec_from_file_location("h3c_p20_file_backed_tensor_poc", MODULE_PATH)
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
assert SPEC.loader is not None
|
||||
sys.modules[SPEC.name] = MODULE
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def _known_files(root: Path) -> list[Path]:
|
||||
return sorted(path for path in root.iterdir() if path.name.startswith(MODULE.FILE_PREFIX))
|
||||
|
||||
|
||||
def test_shared_mapping_is_bit_exact_contiguous_and_write_through(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
reference = torch.arange(120, dtype=torch.float32).reshape(2, 3, 4, 5)
|
||||
|
||||
with manager.allocate(tuple(reference.shape), dtype=reference.dtype) as allocation:
|
||||
allocation.tensor.copy_(reference)
|
||||
record = allocation.record
|
||||
output = allocation.publish()
|
||||
|
||||
assert output.shape == reference.shape
|
||||
assert output.dtype == reference.dtype
|
||||
assert output.device.type == "cpu"
|
||||
assert output.is_contiguous()
|
||||
assert output.contiguous().data_ptr() == output.data_ptr()
|
||||
assert record.data_path.stat().st_size == reference.numel() * reference.element_size()
|
||||
assert record.data_path.read_bytes() == reference.numpy().tobytes(order="C")
|
||||
assert torch.equal(output, reference)
|
||||
|
||||
storage_pointer = output.untyped_storage().data_ptr()
|
||||
assert output[1].untyped_storage().data_ptr() == storage_pointer
|
||||
assert output.detach().untyped_storage().data_ptr() == storage_pointer
|
||||
assert output.reshape(6, 4, 5).untyped_storage().data_ptr() == storage_pointer
|
||||
assert output.clone().untyped_storage().data_ptr() != storage_pointer
|
||||
|
||||
del output
|
||||
gc.collect()
|
||||
result = manager.collect_ready()
|
||||
assert result.reclaimed == 1
|
||||
assert result.deferred == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_lifetime_waits_for_last_storage_alias_and_not_root_tensor(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
with manager.allocate((4, 3, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.copy_(torch.arange(24, dtype=torch.float32).reshape(4, 3, 2))
|
||||
record = allocation.record
|
||||
output = allocation.publish()
|
||||
|
||||
view = output[1:]
|
||||
cache = {"images": view}
|
||||
root_reference = weakref.ref(output)
|
||||
del output
|
||||
gc.collect()
|
||||
|
||||
assert root_reference() is None
|
||||
assert float(cache["images"].sum()) == pytest.approx(261.0)
|
||||
first = manager.collect_ready()
|
||||
assert first.reclaimed == 0
|
||||
assert first.deferred == 1
|
||||
assert record.data_path.exists()
|
||||
assert record.metadata_path.exists()
|
||||
|
||||
del view
|
||||
cache.clear()
|
||||
gc.collect()
|
||||
second = manager.collect_ready()
|
||||
assert second.reclaimed == 1
|
||||
assert second.deferred == 0
|
||||
assert not record.data_path.exists()
|
||||
assert not record.metadata_path.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exception_type", [RuntimeError, MODULE.ProbeCancelled])
|
||||
def test_unpublished_exception_and_cancel_remove_provisional_files(
|
||||
tmp_path,
|
||||
exception_type,
|
||||
):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
with pytest.raises(exception_type):
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(1.0)
|
||||
raise exception_type("stop before publish")
|
||||
|
||||
assert manager.tracked_count() == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_mapping_creation_failure_removes_reserved_file(tmp_path, monkeypatch):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
|
||||
def fail_from_file(*args, **kwargs):
|
||||
raise RuntimeError("synthetic mapping failure")
|
||||
|
||||
monkeypatch.setattr(MODULE.torch, "from_file", fail_from_file)
|
||||
with pytest.raises(RuntimeError, match="synthetic mapping failure"):
|
||||
manager.allocate((2, 2), dtype=torch.float32)
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_temporary_unlink_failure_preserves_cancel_and_is_retried(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
original_unlink = MODULE._unlink_known
|
||||
fail_unlink = {"enabled": True}
|
||||
|
||||
def sharing_violation(root, path):
|
||||
if fail_unlink["enabled"]:
|
||||
raise PermissionError(32, "synthetic sharing violation", str(path))
|
||||
return original_unlink(root, path)
|
||||
|
||||
monkeypatch.setattr(MODULE, "_unlink_known", sharing_violation)
|
||||
with pytest.raises(MODULE.ProbeCancelled, match="original cancel"):
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(1.0)
|
||||
raise MODULE.ProbeCancelled("original cancel")
|
||||
|
||||
assert manager.pending_count() == 1
|
||||
assert len(_known_files(tmp_path)) == 2
|
||||
fail_unlink["enabled"] = False
|
||||
result = manager.collect_ready()
|
||||
assert result.reclaimed == 1
|
||||
assert result.deferred == 0
|
||||
assert manager.pending_count() == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_cancel_after_backing_file_create_does_not_leave_orphan(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
original_close = MODULE.os.close
|
||||
injected = False
|
||||
sentinel_path = tmp_path / "unrelated-live-descriptor.bin"
|
||||
sentinel_descriptor = None
|
||||
|
||||
def cancel_after_real_close(descriptor):
|
||||
nonlocal injected, sentinel_descriptor
|
||||
original_close(descriptor)
|
||||
if not injected:
|
||||
injected = True
|
||||
opened = MODULE.os.open(
|
||||
sentinel_path,
|
||||
MODULE.os.O_CREAT | MODULE.os.O_EXCL | MODULE.os.O_RDWR,
|
||||
0o600,
|
||||
)
|
||||
if opened != descriptor:
|
||||
MODULE.os.dup2(opened, descriptor)
|
||||
original_close(opened)
|
||||
opened = descriptor
|
||||
sentinel_descriptor = opened
|
||||
raise MODULE.ProbeCancelled("cancel after backing file create")
|
||||
|
||||
try:
|
||||
monkeypatch.setattr(MODULE.os, "close", cancel_after_real_close)
|
||||
with pytest.raises(MODULE.ProbeCancelled, match="after backing file create"):
|
||||
manager.allocate((2, 2), dtype=torch.float32)
|
||||
finally:
|
||||
monkeypatch.setattr(MODULE.os, "close", original_close)
|
||||
|
||||
assert sentinel_descriptor is not None
|
||||
MODULE.os.write(sentinel_descriptor, b"still-open")
|
||||
original_close(sentinel_descriptor)
|
||||
assert sentinel_path.read_bytes() == b"still-open"
|
||||
manager.collect_ready()
|
||||
assert manager.pending_count() == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_dropped_unpublished_allocation_is_finalized(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
allocation = manager.allocate((2, 2), dtype=torch.float32)
|
||||
allocation.tensor.fill_(3.0)
|
||||
allocation_reference = weakref.ref(allocation)
|
||||
del allocation
|
||||
gc.collect()
|
||||
|
||||
assert allocation_reference() is None
|
||||
manager.collect_ready()
|
||||
assert manager.pending_count() == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_cache_requeue_keeps_backing_file_count_bounded(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
cache = {}
|
||||
|
||||
for index in range(20):
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(float(index))
|
||||
output = allocation.publish()
|
||||
cache["same_node_output"] = output
|
||||
del output
|
||||
gc.collect()
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) <= 2
|
||||
|
||||
same_requeue = cache["same_node_output"]
|
||||
assert same_requeue is cache["same_node_output"]
|
||||
manager.collect_ready()
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
|
||||
cache.clear()
|
||||
del same_requeue
|
||||
gc.collect()
|
||||
result = manager.collect_ready()
|
||||
assert result.reclaimed == 1
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_published_downstream_error_keeps_cache_then_reclaims_after_eviction(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
cache = {}
|
||||
try:
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(4.0)
|
||||
output = allocation.publish()
|
||||
cache["assembler_output"] = output
|
||||
raise RuntimeError("downstream node failed")
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
del output
|
||||
gc.collect()
|
||||
held = manager.collect_ready()
|
||||
assert held.reclaimed == 0
|
||||
assert held.deferred == 1
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
|
||||
|
||||
cache.clear()
|
||||
gc.collect()
|
||||
released = manager.collect_ready()
|
||||
assert released.reclaimed == 1
|
||||
assert released.deferred == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_process_global_manager_survives_node_local_scope(tmp_path):
|
||||
first = MODULE.get_process_lifetime_manager(tmp_path)
|
||||
second = MODULE.get_process_lifetime_manager(tmp_path)
|
||||
assert first is second
|
||||
|
||||
with first.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
output = allocation.publish()
|
||||
manager_reference = weakref.ref(first)
|
||||
del first
|
||||
del second
|
||||
gc.collect()
|
||||
assert manager_reference() is not None
|
||||
|
||||
del output
|
||||
gc.collect()
|
||||
manager = MODULE.get_process_lifetime_manager(tmp_path)
|
||||
assert manager.collect_ready().reclaimed == 1
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_stale_child_files_are_reclaimed_but_live_and_unknown_files_are_kept(tmp_path):
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(MODULE_PATH),
|
||||
"--root",
|
||||
str(tmp_path),
|
||||
"--crash-child",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.json"))) == 1
|
||||
|
||||
unknown = tmp_path / "unrelated.bin"
|
||||
unknown.write_bytes(b"do not delete")
|
||||
outside = tmp_path.parent / "outside.bin"
|
||||
outside.write_bytes(b"outside root")
|
||||
malicious_id = "f" * 32
|
||||
malicious_metadata = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.json"
|
||||
malicious_data = tmp_path / f"{MODULE.FILE_PREFIX}{malicious_id}.bin"
|
||||
malicious_data.write_bytes(b"protected")
|
||||
malicious_metadata.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": MODULE.SCHEMA,
|
||||
"schema_version": MODULE.SCHEMA_VERSION,
|
||||
"allocation_id": malicious_id,
|
||||
"data_file": "..\\outside.bin",
|
||||
"owner_pid": -1,
|
||||
"created_at": 0,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
|
||||
assert result.removed_pairs == 1
|
||||
assert result.invalid_metadata == 1
|
||||
assert unknown.read_bytes() == b"do not delete"
|
||||
assert outside.read_bytes() == b"outside root"
|
||||
assert malicious_data.read_bytes() == b"protected"
|
||||
assert malicious_metadata.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "remove"),
|
||||
[
|
||||
("owner_pid", None, True),
|
||||
("owner_pid", -1, False),
|
||||
("owner_create_time", float("nan"), False),
|
||||
("created_at", float("nan"), False),
|
||||
],
|
||||
)
|
||||
def test_invalid_ownership_metadata_never_deletes_live_mapping(
|
||||
tmp_path,
|
||||
field,
|
||||
value,
|
||||
remove,
|
||||
):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
record = allocation.record
|
||||
output = allocation.publish()
|
||||
|
||||
payload = json.loads(record.metadata_path.read_text(encoding="utf-8"))
|
||||
if remove:
|
||||
payload.pop(field)
|
||||
else:
|
||||
payload[field] = value
|
||||
record.metadata_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
result = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
|
||||
assert result.invalid_metadata == 1
|
||||
assert result.removed_pairs == 0
|
||||
assert record.data_path.exists()
|
||||
assert record.metadata_path.exists()
|
||||
assert float(output.sum()) == pytest.approx(0.0)
|
||||
|
||||
del output
|
||||
gc.collect()
|
||||
assert manager.collect_ready().reclaimed == 1
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_cleanup_waits_for_in_progress_allocation_root_lock(tmp_path):
|
||||
allocation_entered = threading.Event()
|
||||
allow_allocation = threading.Event()
|
||||
output_holder = []
|
||||
cleanup_holder = []
|
||||
|
||||
def allocation_hook(_path):
|
||||
allocation_entered.set()
|
||||
assert allow_allocation.wait(timeout=5.0)
|
||||
|
||||
manager = MODULE.StorageLifetimeManager(
|
||||
tmp_path,
|
||||
allocation_hook=allocation_hook,
|
||||
)
|
||||
|
||||
def allocate_output():
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
output_holder.append(allocation.publish())
|
||||
|
||||
def cleanup():
|
||||
cleanup_holder.append(
|
||||
MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
|
||||
)
|
||||
|
||||
allocation_thread = threading.Thread(target=allocate_output)
|
||||
allocation_thread.start()
|
||||
assert allocation_entered.wait(timeout=5.0)
|
||||
cleanup_thread = threading.Thread(target=cleanup)
|
||||
cleanup_thread.start()
|
||||
assert cleanup_thread.is_alive()
|
||||
allow_allocation.set()
|
||||
allocation_thread.join(timeout=5.0)
|
||||
cleanup_thread.join(timeout=5.0)
|
||||
|
||||
assert not allocation_thread.is_alive()
|
||||
assert not cleanup_thread.is_alive()
|
||||
assert cleanup_holder[0].retained_live == 1
|
||||
assert len(list(tmp_path.glob(f"{MODULE.FILE_PREFIX}*.bin"))) == 1
|
||||
|
||||
output_holder.clear()
|
||||
gc.collect()
|
||||
assert manager.collect_ready().reclaimed == 1
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_root_lock_is_reentrant_when_finalizer_runs_during_allocation(tmp_path):
|
||||
first_manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
abandoned = first_manager.allocate((2, 2), dtype=torch.float32)
|
||||
cycle = [abandoned]
|
||||
cycle.append(cycle)
|
||||
del abandoned
|
||||
del cycle
|
||||
|
||||
def collect_abandoned_allocation(_path):
|
||||
gc.collect()
|
||||
|
||||
second_manager = MODULE.StorageLifetimeManager(
|
||||
tmp_path,
|
||||
allocation_hook=collect_abandoned_allocation,
|
||||
)
|
||||
started = time.perf_counter()
|
||||
with second_manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
output = allocation.publish()
|
||||
elapsed = time.perf_counter() - started
|
||||
|
||||
assert elapsed < 2.0
|
||||
first_manager.collect_ready()
|
||||
del output
|
||||
gc.collect()
|
||||
second_manager.collect_ready()
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_cleanup_collect_and_gc_finalizer_have_no_lock_order_deadlock(tmp_path):
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(MODULE_PATH),
|
||||
"--root",
|
||||
str(tmp_path),
|
||||
"--lock-order-child",
|
||||
],
|
||||
check=True,
|
||||
timeout=10.0,
|
||||
)
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_live_sidecar_is_retained_and_cleanup_is_idempotent(tmp_path):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
with manager.allocate((2, 2), dtype=torch.float32) as allocation:
|
||||
allocation.tensor.fill_(2.0)
|
||||
record = allocation.record
|
||||
output = allocation.publish()
|
||||
|
||||
live = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
|
||||
assert live.retained_live == 1
|
||||
assert record.data_path.exists()
|
||||
assert record.metadata_path.exists()
|
||||
|
||||
del output
|
||||
gc.collect()
|
||||
assert manager.collect_ready().reclaimed == 1
|
||||
empty = MODULE.cleanup_stale_files(tmp_path, grace_seconds=0.0)
|
||||
assert empty.removed_pairs == 0
|
||||
assert empty.removed_orphans == 0
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_orphan_and_partial_cleanup_respects_grace_period(tmp_path):
|
||||
allocation_id = "a" * 32
|
||||
orphan = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.bin"
|
||||
partial = tmp_path / f"{MODULE.FILE_PREFIX}{allocation_id}.json.partial"
|
||||
orphan.write_bytes(b"orphan")
|
||||
partial.write_text("partial", encoding="utf-8")
|
||||
now = time.time()
|
||||
|
||||
young = MODULE.cleanup_stale_files(
|
||||
tmp_path,
|
||||
grace_seconds=60.0,
|
||||
now=now,
|
||||
)
|
||||
assert young.retained_young == 2
|
||||
assert orphan.exists()
|
||||
assert partial.exists()
|
||||
|
||||
old = MODULE.cleanup_stale_files(
|
||||
tmp_path,
|
||||
grace_seconds=60.0,
|
||||
now=now + 61.0,
|
||||
)
|
||||
assert old.removed_orphans == 2
|
||||
assert not orphan.exists()
|
||||
assert not partial.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(2.5, 2), (True, 2), ("2", 2), ()])
|
||||
def test_shape_contract_rejects_lossy_or_empty_dimensions(tmp_path, shape):
|
||||
manager = MODULE.StorageLifetimeManager(tmp_path)
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
manager.allocate(shape, dtype=torch.float32)
|
||||
assert not _known_files(tmp_path)
|
||||
|
||||
|
||||
def test_standalone_probe_passes(tmp_path):
|
||||
result = MODULE.run_probe(tmp_path)
|
||||
assert result.bit_exact
|
||||
assert result.file_size == result.expected_bytes
|
||||
assert result.root_released_while_alias_alive
|
||||
assert result.deferred_while_alias_alive == 1
|
||||
assert result.reclaimed_after_last_alias == 1
|
||||
assert result.files_remaining == 0
|
||||
Reference in New Issue
Block a user