417 lines
14 KiB
Python
417 lines
14 KiB
Python
"""V3.5-only, non-synchronizing memory attribution helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import math
|
|
import time
|
|
from typing import Any, Mapping
|
|
|
|
import torch
|
|
|
|
from ..constants import FPS
|
|
from ..hardening import tensor_storage_stats
|
|
from ..state import extract_av_streams
|
|
from .plan import validate_assembly_plan
|
|
|
|
|
|
_MIB = 1024**2
|
|
|
|
_CONDITIONING_SUBPHASES = (
|
|
"conditioning_identity_video_vae",
|
|
"conditioning_reference_image_vae",
|
|
"conditioning_reference_audio_vae",
|
|
"conditioning_driving_audio_vae",
|
|
"conditioning_reference_video_vae",
|
|
"conditioning_timeline_video_vae",
|
|
"conditioning_prompt_clip",
|
|
)
|
|
|
|
|
|
def _mib(value: int | None) -> str:
|
|
if value is None:
|
|
return "n/a"
|
|
return f"{int(value) / _MIB:.1f} MiB"
|
|
|
|
|
|
def _bytes_and_mib(value: int) -> str:
|
|
return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)"
|
|
|
|
|
|
def _process_counters() -> tuple[int | None, int | None]:
|
|
try:
|
|
import psutil
|
|
|
|
return (
|
|
int(psutil.Process().memory_info().rss),
|
|
int(psutil.virtual_memory().available),
|
|
)
|
|
except Exception:
|
|
return None, None
|
|
|
|
|
|
def _cuda_counters() -> tuple[int | None, int | None, int | None]:
|
|
"""Read counters only; never synchronize, clear caches, or reset peaks."""
|
|
|
|
try:
|
|
if torch.cuda.is_available():
|
|
return (
|
|
int(torch.cuda.memory_allocated()),
|
|
int(torch.cuda.memory_reserved()),
|
|
int(torch.cuda.max_memory_allocated()),
|
|
)
|
|
except Exception:
|
|
pass
|
|
return None, None, None
|
|
|
|
|
|
def _retained_stream_bytes(
|
|
entries: Any,
|
|
stream: str,
|
|
) -> int:
|
|
values: list[Any] = []
|
|
if isinstance(entries, (list, tuple)):
|
|
for entry in entries:
|
|
if isinstance(entry, Mapping):
|
|
values.append(entry.get(stream))
|
|
stats = tensor_storage_stats(values)
|
|
return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"])
|
|
|
|
|
|
def _observed_streams(latent: Any) -> tuple[Any, ...]:
|
|
if latent is None:
|
|
return ()
|
|
if isinstance(latent, Mapping) and "samples" in latent:
|
|
try:
|
|
return tuple(extract_av_streams(latent))
|
|
except Exception:
|
|
return ()
|
|
return (latent,)
|
|
|
|
|
|
def format_attribution_snapshot(
|
|
label: str,
|
|
*,
|
|
retained_entries: Any = None,
|
|
observed_latent: Any = None,
|
|
retained_refine_context: Any = None,
|
|
) -> str:
|
|
"""Format one fail-soft snapshot without changing allocator state."""
|
|
try:
|
|
rss, available = _process_counters()
|
|
cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters()
|
|
retained_video = _retained_stream_bytes(retained_entries, "video")
|
|
retained_audio = _retained_stream_bytes(retained_entries, "audio")
|
|
observed = tensor_storage_stats(_observed_streams(observed_latent))
|
|
refine_context = tensor_storage_stats(retained_refine_context)
|
|
return (
|
|
f"Memory [{label}]: RSS={_mib(rss)}, "
|
|
f"system_available={_mib(available)}, "
|
|
f"CUDA_allocated={_mib(cuda_allocated)}, "
|
|
f"CUDA_reserved={_mib(cuda_reserved)}, "
|
|
f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, "
|
|
f"retained_video={_bytes_and_mib(retained_video)}, "
|
|
f"retained_audio={_bytes_and_mib(retained_audio)}, "
|
|
"retained_refine_context_CPU="
|
|
f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, "
|
|
"retained_refine_context_GPU="
|
|
f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, "
|
|
f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, "
|
|
f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}"
|
|
)
|
|
except Exception as exc:
|
|
return (
|
|
f"Memory [{label}]: unavailable "
|
|
f"({type(exc).__name__}: {exc})"
|
|
)
|
|
|
|
|
|
class MemoryAttributionCollector:
|
|
"""Keep V3.5 memory events in chronological order until report assembly."""
|
|
|
|
def __init__(self):
|
|
self.lines: list[str] = []
|
|
self._sequence_started_at: float | None = None
|
|
self._phase_started_at: dict[tuple[str, int | None, tuple[int, ...]], float] = {}
|
|
self._phase_totals: dict[str, float] = {}
|
|
self._phase_counts: dict[str, int] = {}
|
|
self._group_totals: dict[
|
|
tuple[int, tuple[int, ...]], dict[str, float]
|
|
] = {}
|
|
|
|
def capture_sequence_start(self) -> None:
|
|
self._sequence_started_at = time.perf_counter()
|
|
self.lines.append(format_attribution_snapshot("V3.5 sequence start"))
|
|
self.start_phase(phase="preparation")
|
|
|
|
def start_phase(
|
|
self,
|
|
*,
|
|
phase: str,
|
|
physical_group: int | None = None,
|
|
logical_chunks: tuple[int, ...] = (),
|
|
) -> None:
|
|
key = (
|
|
str(phase),
|
|
None if physical_group is None else int(physical_group),
|
|
tuple(int(value) for value in logical_chunks),
|
|
)
|
|
self._phase_started_at[key] = time.perf_counter()
|
|
|
|
def finish_phase(
|
|
self,
|
|
*,
|
|
phase: str,
|
|
physical_group: int | None = None,
|
|
logical_chunks: tuple[int, ...] = (),
|
|
) -> None:
|
|
phase = str(phase)
|
|
group = None if physical_group is None else int(physical_group)
|
|
logical = tuple(int(value) for value in logical_chunks)
|
|
key = (phase, group, logical)
|
|
started = self._phase_started_at.pop(key, None)
|
|
if started is None:
|
|
return
|
|
elapsed = time.perf_counter() - started
|
|
if not math.isfinite(elapsed) or elapsed < 0.0:
|
|
return
|
|
self._phase_totals[phase] = self._phase_totals.get(phase, 0.0) + elapsed
|
|
self._phase_counts[phase] = self._phase_counts.get(phase, 0) + 1
|
|
if group is not None:
|
|
phases = self._group_totals.setdefault((group, logical), {})
|
|
phases[phase] = phases.get(phase, 0.0) + elapsed
|
|
|
|
@staticmethod
|
|
def _format_seconds(value: float | None) -> str:
|
|
return "n/a" if value is None else f"{float(value):.6f}s"
|
|
|
|
def _performance_lines(self) -> list[str]:
|
|
now = time.perf_counter()
|
|
host_wall = None
|
|
if self._sequence_started_at is not None:
|
|
candidate = now - self._sequence_started_at
|
|
if math.isfinite(candidate) and candidate >= 0.0:
|
|
host_wall = candidate
|
|
ordered = (
|
|
"preparation",
|
|
"conditioning",
|
|
*_CONDITIONING_SUBPHASES,
|
|
"group_prepare",
|
|
"sampling",
|
|
"cpu_commit_validation",
|
|
"refine_context",
|
|
"finalization",
|
|
)
|
|
lines = []
|
|
for (group, logical), phases in sorted(self._group_totals.items()):
|
|
logical_label = ",".join(str(value) for value in logical)
|
|
fields = ", ".join(
|
|
f"{name}={self._format_seconds(phases.get(name))}"
|
|
for name in (
|
|
"group_prepare",
|
|
"sampling",
|
|
"cpu_commit_validation",
|
|
"refine_context",
|
|
)
|
|
)
|
|
lines.append(
|
|
f"Performance [V3.5 physical group {group} "
|
|
f"logical=[{logical_label}]]: {fields}"
|
|
)
|
|
totals = ", ".join(
|
|
f"{name}={self._format_seconds(self._phase_totals.get(name))}"
|
|
for name in ordered
|
|
)
|
|
counts = ", ".join(
|
|
f"{name}={int(self._phase_counts.get(name, 0))}"
|
|
for name in (
|
|
"group_prepare",
|
|
"sampling",
|
|
"cpu_commit_validation",
|
|
"refine_context",
|
|
)
|
|
)
|
|
conditioning_counts = ", ".join(
|
|
f"{name}={int(self._phase_counts.get(name, 0))}"
|
|
for name in _CONDITIONING_SUBPHASES
|
|
)
|
|
lines.append(
|
|
"Performance [V3.5 summary]: "
|
|
f"host_wall_including_diagnostics={self._format_seconds(host_wall)}, "
|
|
f"{totals}; "
|
|
f"phase_counts({counts}); "
|
|
f"conditioning_counts({conditioning_counts}); "
|
|
"CUDA_synchronize=not_used"
|
|
)
|
|
return lines
|
|
|
|
def capture_group(
|
|
self,
|
|
*,
|
|
physical_group: int,
|
|
logical_chunks: tuple[int, ...],
|
|
stage: str,
|
|
retained_entries: Any,
|
|
observed_latent: Any = None,
|
|
retained_refine_context: Any = None,
|
|
) -> None:
|
|
logical = ",".join(str(int(value)) for value in logical_chunks)
|
|
self.lines.append(
|
|
format_attribution_snapshot(
|
|
f"V3.5 physical group {int(physical_group)} "
|
|
f"logical=[{logical}] {stage}",
|
|
retained_entries=retained_entries,
|
|
observed_latent=observed_latent,
|
|
retained_refine_context=retained_refine_context,
|
|
)
|
|
)
|
|
|
|
def capture_sequence_complete(
|
|
self,
|
|
entries: Any,
|
|
retained_refine_context: Any = None,
|
|
) -> None:
|
|
self.lines.extend(self._performance_lines())
|
|
self.lines.append(
|
|
format_attribution_snapshot(
|
|
"V3.5 sequence complete",
|
|
retained_entries=entries,
|
|
retained_refine_context=retained_refine_context,
|
|
)
|
|
)
|
|
|
|
|
|
def capture_attribution_fail_soft(
|
|
collector: Any,
|
|
method: str,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
"""A diagnostics failure must never replace a sampling result."""
|
|
|
|
try:
|
|
getattr(collector, method)(**kwargs)
|
|
except Exception as exc:
|
|
try:
|
|
collector.lines.append(
|
|
f"Memory attribution [{method}]: unavailable "
|
|
f"({type(exc).__name__}: {exc})"
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AssemblyBufferProjection:
|
|
natural_frames: int
|
|
target_frames: int
|
|
natural_image_bytes: int
|
|
natural_audio_bytes: int
|
|
final_image_bytes: int
|
|
final_audio_bytes: int
|
|
frame_count_adjustment_required: bool
|
|
|
|
|
|
def _as_list(value: Any) -> list[Any]:
|
|
if isinstance(value, list):
|
|
if len(value) == 1 and isinstance(value[0], list):
|
|
return list(value[0])
|
|
return list(value)
|
|
if isinstance(value, tuple):
|
|
return list(value)
|
|
return [value]
|
|
|
|
|
|
def _singleton(value: Any, name: str) -> Any:
|
|
while isinstance(value, list):
|
|
if len(value) != 1:
|
|
raise ValueError(f"{name} must contain exactly one value")
|
|
value = value[0]
|
|
return value
|
|
|
|
|
|
def project_assembly_buffers(
|
|
*,
|
|
images: Any,
|
|
audio: Any,
|
|
assembly_plan: Any,
|
|
exact_total_duration: Any,
|
|
) -> AssemblyBufferProjection:
|
|
"""Calculate natural and final buffer sizes without allocating a buffer."""
|
|
|
|
plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan"))
|
|
image_chunks = _as_list(images)
|
|
audio_chunks = _as_list(audio)
|
|
if not image_chunks or not audio_chunks:
|
|
raise ValueError("decoded image/audio chunks are required for projection")
|
|
|
|
first_image = _singleton(image_chunks[0], "decoded image chunk")
|
|
if not torch.is_tensor(first_image) or first_image.ndim != 4:
|
|
raise ValueError("decoded image chunk must be [frames,H,W,C]")
|
|
frame_shape = tuple(int(value) for value in first_image.shape[1:])
|
|
for index, value in enumerate(image_chunks, start=1):
|
|
tensor = _singleton(value, f"decoded image chunk {index}")
|
|
if (
|
|
not torch.is_tensor(tensor)
|
|
or tensor.ndim != 4
|
|
or tuple(int(item) for item in tensor.shape[1:]) != frame_shape
|
|
or tensor.dtype != first_image.dtype
|
|
):
|
|
raise ValueError("decoded image geometry or dtype changed between chunks")
|
|
|
|
first_audio = _singleton(audio_chunks[0], "decoded audio chunk")
|
|
if not isinstance(first_audio, Mapping):
|
|
raise ValueError("decoded audio chunk must be an AUDIO mapping")
|
|
waveform = first_audio.get("waveform")
|
|
sample_rate = first_audio.get("sample_rate")
|
|
if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None:
|
|
raise ValueError("decoded audio chunk has an invalid waveform or sample rate")
|
|
sample_rate = int(sample_rate)
|
|
if sample_rate <= 0:
|
|
raise ValueError("decoded audio sample rate must be positive")
|
|
|
|
decode_units = list(plan.get("decode_groups") or plan["chunks"])
|
|
natural_frames = sum(int(unit["net_frames"]) for unit in decode_units)
|
|
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
|
|
target_frames = int(plan["target_frames"]) if exact else natural_frames
|
|
if natural_frames <= 0 or target_frames <= 0:
|
|
raise ValueError("assembly frame counts must be positive")
|
|
|
|
frame_elements = math.prod(frame_shape)
|
|
audio_streams = math.prod(int(value) for value in waveform.shape[:-1])
|
|
natural_samples = int(round(natural_frames / FPS * sample_rate))
|
|
target_samples = int(round(target_frames / FPS * sample_rate))
|
|
return AssemblyBufferProjection(
|
|
natural_frames=natural_frames,
|
|
target_frames=target_frames,
|
|
natural_image_bytes=(
|
|
natural_frames * frame_elements * first_image.element_size()
|
|
),
|
|
natural_audio_bytes=(
|
|
natural_samples * audio_streams * waveform.element_size()
|
|
),
|
|
final_image_bytes=(
|
|
target_frames * frame_elements * first_image.element_size()
|
|
),
|
|
final_audio_bytes=(
|
|
target_samples * audio_streams * waveform.element_size()
|
|
),
|
|
frame_count_adjustment_required=bool(
|
|
exact and target_frames != natural_frames
|
|
),
|
|
)
|
|
|
|
|
|
def format_assembly_projection(projection: AssemblyBufferProjection) -> str:
|
|
return (
|
|
"Projected assembly buffers [V3.5]: "
|
|
f"natural_frames={projection.natural_frames}, "
|
|
f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, "
|
|
"projected_natural_generated_AUDIO="
|
|
f"{_bytes_and_mib(projection.natural_audio_bytes)}, "
|
|
f"target_frames={projection.target_frames}, "
|
|
f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, "
|
|
f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, "
|
|
"frame_count_adjustment_required="
|
|
f"{'yes' if projection.frame_count_adjustment_required else 'no'}"
|
|
)
|