Release V3.8X2 package 3.8.3
This commit is contained in:
@@ -0,0 +1,295 @@
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from decode_cache.identity import CacheBypass
|
||||
from decode_cache.store import DecodeStore, MiB
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
s = DecodeStore(temp_parent=tmp_path, memory_probe=lambda: 10**12,
|
||||
disk_probe=lambda p: SimpleNamespace(free=10**12), headroom_bytes=0)
|
||||
s.configure("Auto", 1024**2, 1024**2, 0)
|
||||
yield s
|
||||
s.close()
|
||||
|
||||
|
||||
def put(store, key, x, stream="video"):
|
||||
return store.put(key, x, stream, {}, (), store.epoch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["Auto", "RAM"])
|
||||
def test_hit_bit_exact_mutation_isolation(store, mode):
|
||||
store.configure(mode, MiB, MiB, 0)
|
||||
x = torch.arange(120).float().reshape(5, 3, 4, 2)
|
||||
replacement = put(store, "a", x)
|
||||
first = replacement[0] if replacement else x
|
||||
first.add_(7)
|
||||
hit, _ = store.get("a")
|
||||
assert torch.equal(hit, torch.arange(120).float().reshape(5, 3, 4, 2))
|
||||
hit.zero_()
|
||||
again, _ = store.get("a")
|
||||
assert again.sum().item() > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["Auto", "RAM"])
|
||||
def test_view_mutation_does_not_poison(store, mode):
|
||||
store.configure(mode, MiB, MiB, 0)
|
||||
put(store, "a", torch.ones(4, 3, 2, 1))
|
||||
hit, _ = store.get("a")
|
||||
hit[1:3].mul_(0)
|
||||
other, _ = store.get("a")
|
||||
assert other.sum() == 24
|
||||
|
||||
|
||||
def test_noncontiguous_serialization(store):
|
||||
x = torch.arange(120).float().reshape(5, 3, 4, 2).transpose(0, 2)
|
||||
put(store, "a", x)
|
||||
y, _ = store.get("a")
|
||||
assert y.dtype == x.dtype and torch.equal(x, y)
|
||||
|
||||
|
||||
def test_dtype_unchanged(store):
|
||||
for dtype in (torch.float16, torch.float32, torch.float64):
|
||||
x = torch.arange(12).to(dtype)
|
||||
put(store, str(dtype), x)
|
||||
y, _ = store.get(str(dtype))
|
||||
assert y.dtype == dtype and torch.equal(x, y)
|
||||
|
||||
|
||||
def test_ram_lru_and_budget(store):
|
||||
store.configure("RAM", 32, MiB, 0)
|
||||
put(store, "a", torch.ones(4))
|
||||
put(store, "b", torch.ones(4)*2)
|
||||
assert store.get("a") is not None
|
||||
put(store, "c", torch.ones(4)*3)
|
||||
assert store.get("b") is None
|
||||
assert store.ram_bytes == 32
|
||||
|
||||
|
||||
def test_ram_pressure_evicts(store):
|
||||
store.configure("RAM", MiB, MiB, 0)
|
||||
put(store, "a", torch.ones(4))
|
||||
store.headroom_bytes = 1024
|
||||
store.memory_probe = lambda: 0
|
||||
store.configure("RAM", MiB, MiB, 0)
|
||||
assert store.get("a") is None
|
||||
assert store.ram_bytes == 0
|
||||
|
||||
|
||||
def test_auto_under_ram_pressure_uses_disk(store):
|
||||
store.memory_probe = lambda: 0
|
||||
put(store, "a", torch.ones(24), "audio")
|
||||
assert store.entries["a"].path is not None
|
||||
|
||||
|
||||
def test_disk_full_skips_cache(store):
|
||||
store.disk_probe = lambda p: SimpleNamespace(free=0)
|
||||
assert put(store, "a", torch.ones(4)) is None
|
||||
assert store.get("a") is None
|
||||
|
||||
|
||||
def test_oversized_skips(store):
|
||||
store.configure("Auto", 0, 8, 0)
|
||||
assert put(store, "a", torch.ones(4)) is None
|
||||
assert store.root is None
|
||||
|
||||
|
||||
def test_living_mmap_survives_clear_and_accounts_bytes(store):
|
||||
hit = put(store, "a", torch.ones(4))[0]
|
||||
path = store.entries["a"].path
|
||||
store.clear()
|
||||
assert path.exists()
|
||||
assert store.disk_bytes > 0
|
||||
hit.add_(2)
|
||||
assert torch.equal(hit, torch.ones(4)*3)
|
||||
del hit
|
||||
gc.collect()
|
||||
store.stats()
|
||||
assert not path.exists()
|
||||
assert store.disk_bytes == 0
|
||||
|
||||
|
||||
def test_no_cross_process_store_reuse(store, tmp_path):
|
||||
put(store, "a", torch.ones(4))
|
||||
other = DecodeStore(temp_parent=tmp_path)
|
||||
assert other.get("a") is None
|
||||
other.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("damage", ["truncate", "header", "data", "delete"])
|
||||
def test_corrupt_disk_invalidates(store, damage):
|
||||
result = put(store, "a", torch.ones(20))
|
||||
del result
|
||||
gc.collect()
|
||||
path = store.entries["a"].path
|
||||
if damage == "delete":
|
||||
path.unlink()
|
||||
elif damage == "truncate":
|
||||
path.write_bytes(b"bad")
|
||||
else:
|
||||
with open(path, "r+b") as f:
|
||||
f.seek(-1 if damage == "data" else 0, 2 if damage == "data" else 0)
|
||||
b = f.read(1)
|
||||
f.seek(-1, 1)
|
||||
f.write(bytes([b[0] ^ 1]))
|
||||
with pytest.raises((CacheBypass, OSError, ValueError)):
|
||||
store.get("a")
|
||||
assert store.get("a") is None
|
||||
|
||||
|
||||
def test_corrupt_ram_invalidates(store):
|
||||
store.configure("RAM", MiB, MiB, 0)
|
||||
put(store, "a", torch.ones(4))
|
||||
store.entries["a"].tensor.zero_()
|
||||
with pytest.raises(CacheBypass):
|
||||
store.get("a")
|
||||
assert store.get("a") is None
|
||||
|
||||
|
||||
def test_dead_vae_eviction(store):
|
||||
class Owner:
|
||||
pass
|
||||
owner = Owner()
|
||||
store.put("a", torch.ones(4), "audio", {}, (weakref.ref(owner),), store.epoch)
|
||||
del owner
|
||||
gc.collect()
|
||||
assert store.get("a") is None
|
||||
|
||||
|
||||
def test_write_fault_no_published_entry(store, monkeypatch):
|
||||
def fail(*args):
|
||||
raise OSError("read-only test cache")
|
||||
monkeypatch.setattr(os, "replace", fail)
|
||||
with pytest.raises(OSError):
|
||||
put(store, "a", torch.ones(4))
|
||||
assert not store.entries
|
||||
assert not list(store.root.iterdir())
|
||||
|
||||
|
||||
def test_unknown_temp_is_never_loaded_or_removed(store):
|
||||
root = store._directory()
|
||||
stale = root / "foreign.tmp"
|
||||
stale.write_text("not a cache entry")
|
||||
store.close()
|
||||
assert stale.exists()
|
||||
|
||||
|
||||
def test_reset_and_disabled_no_storage(store):
|
||||
put(store, "a", torch.ones(4))
|
||||
store.configure("Off", MiB, MiB, 1)
|
||||
assert store.get("a") is None
|
||||
assert put(store, "b", torch.ones(4)) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["Auto", "RAM"])
|
||||
def test_token_change_clears_once_then_same_token_keeps_cache(store, mode):
|
||||
store.configure(mode, MiB, MiB, 0)
|
||||
put(store, "before", torch.ones(4))
|
||||
epoch = store.epoch
|
||||
store.configure(mode, MiB, MiB, 0)
|
||||
assert store.get("before") is not None
|
||||
assert store.epoch == epoch
|
||||
# The UI increments only the submitted INT; configure acts on next Queue.
|
||||
store.configure(mode, MiB, MiB, 1)
|
||||
assert store.get("before") is None
|
||||
assert store.epoch > epoch
|
||||
put(store, "after", torch.ones(4))
|
||||
epoch = store.epoch
|
||||
store.configure(mode, MiB, MiB, 1)
|
||||
assert store.get("after") is not None
|
||||
assert store.epoch == epoch
|
||||
|
||||
|
||||
def test_epoch_prevents_inflight_reinsertion(store):
|
||||
epoch = store.epoch
|
||||
store.clear()
|
||||
assert store.put("a", torch.ones(4), "video", {}, (), epoch) is None
|
||||
assert store.get("a") is None
|
||||
|
||||
|
||||
def test_pending_disk_files_are_in_quota(store):
|
||||
store.configure("Auto", 0, 6000, 0)
|
||||
x = torch.ones(1000) # 4000 bytes + header; next insert must not delete a live mmap
|
||||
first = put(store, "a", x)
|
||||
assert first is None # conservative header reservation is > 6000
|
||||
store.configure("Auto", 0, 10000, 0)
|
||||
first = put(store, "a", x)
|
||||
assert first is not None
|
||||
store.clear()
|
||||
# 4128 + conservative 8096 > quota; pinned pending file counts
|
||||
assert put(store, "b", x) is None
|
||||
del first
|
||||
gc.collect()
|
||||
assert put(store, "b", x) is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["Auto", "RAM"])
|
||||
def test_output_metadata_is_defensively_copied(store, mode):
|
||||
store.configure(mode, MiB, MiB, 0)
|
||||
extra = {"sample_rate": 32000, "future_metadata": {"labels": ["original"]}}
|
||||
store.put("a", torch.ones(12), "video", extra, (), store.epoch)
|
||||
extra["future_metadata"]["labels"][0] = "caller mutation"
|
||||
_, first = store.get("a")
|
||||
assert first["future_metadata"]["labels"] == ["original"]
|
||||
first["future_metadata"]["labels"][0] = "downstream mutation"
|
||||
_, second = store.get("a")
|
||||
assert second["future_metadata"]["labels"] == ["original"]
|
||||
|
||||
|
||||
def test_delete_failure_is_accounted_and_retried(store, monkeypatch):
|
||||
put(store, "a", torch.ones(12))
|
||||
path = store.entries["a"].path
|
||||
original_unlink = Path.unlink
|
||||
def denied(self, *args, **kwargs):
|
||||
if self == path:
|
||||
raise PermissionError("simulated Windows sharing violation")
|
||||
return original_unlink(self, *args, **kwargs)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(Path, "unlink", denied)
|
||||
store.clear()
|
||||
assert store.disk_bytes > 0 and len(store.retired) == 1
|
||||
store.stats()
|
||||
assert not path.exists() and store.disk_bytes == 0
|
||||
|
||||
|
||||
def test_symlink_cache_file_is_not_opened(store, tmp_path):
|
||||
put(store, "a", torch.ones(12))
|
||||
path = store.entries["a"].path
|
||||
outside = tmp_path / "outside-user-file.bin"
|
||||
outside.write_bytes(b"DO NOT TOUCH")
|
||||
path.unlink()
|
||||
try:
|
||||
path.symlink_to(outside)
|
||||
except OSError:
|
||||
pytest.skip("symbolic links not available")
|
||||
with pytest.raises(CacheBypass):
|
||||
store.get("a")
|
||||
assert outside.read_bytes() == b"DO NOT TOUCH"
|
||||
|
||||
|
||||
def test_store_garbage_collection_cleans_unleased_private_files(tmp_path):
|
||||
s = DecodeStore(temp_parent=tmp_path, memory_probe=lambda: 10**12, headroom_bytes=0)
|
||||
put(s, "a", torch.ones(12))
|
||||
root = s.root
|
||||
assert root.exists()
|
||||
ref = weakref.ref(s)
|
||||
del s
|
||||
gc.collect()
|
||||
assert ref() is None and not root.exists()
|
||||
|
||||
|
||||
def test_explicit_private_cache_parent_uses_only_child_directory(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("H3_DECODE_CACHE_TEMP_DIR", str(tmp_path))
|
||||
s = DecodeStore(memory_probe=lambda: 10**12, headroom_bytes=0)
|
||||
put(s, "a", torch.ones(12))
|
||||
assert s.root.parent == tmp_path and s.root.name.startswith("h3-decode-cache-")
|
||||
s.close()
|
||||
assert tmp_path.exists()
|
||||
Reference in New Issue
Block a user