Squashed 'vendor/dumas_h3_continuum/' content from commit d07ed31
git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
# H3 Core vs Continuum GPU Diagnostic
|
||||
|
||||
This is a standalone diagnostic node. It does not change or register through the
|
||||
production H3 Continuum package.
|
||||
|
||||
## Install for ComfyUI_W
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```powershell
|
||||
.\tools\install_gpu_diagnostic_node.ps1
|
||||
```
|
||||
|
||||
Restart ComfyUI and search for `H3 Core vs Continuum GPU Diagnostic`.
|
||||
|
||||
## Connections
|
||||
|
||||
- Connect the same direct MODEL used by both tests. Keep Spectrum off.
|
||||
- Connect the same CLIP, Video VAE, Audio VAE, sampler, and sigmas.
|
||||
- Connect the exact same First Frame and Last Frame.
|
||||
- Use 640x640, seed 42, 8 steps, Euler/simple for the baseline comparison.
|
||||
- Run Storage and Session are not used by this node.
|
||||
|
||||
One queue executes Core FL2VA at 243 frames, then Continuum Terminal Merge 2x5s.
|
||||
Results are written below:
|
||||
|
||||
```text
|
||||
ComfyUI/output/h3_continuum/diagnostics/<timestamp>/
|
||||
```
|
||||
|
||||
The report identifies the first difference across pre-sampler data, first H3
|
||||
MODEL call, sampled physical latent, split/recombine, and decode/assembly.
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Standalone GPU diagnostics for H3 Continuum.
|
||||
|
||||
This package is intentionally separate from the production node package. Copy the
|
||||
directory into ComfyUI/custom_nodes only while diagnostics are needed.
|
||||
"""
|
||||
|
||||
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
@@ -0,0 +1,302 @@
|
||||
"""Deterministic tensor capture and comparison helpers for the GPU runner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
STAGE_ORDER = (
|
||||
"A_pre_sampler",
|
||||
"B_first_model_call",
|
||||
"C_physical_sample",
|
||||
"D_split_recombine",
|
||||
"E1_decode_243",
|
||||
"E2_final_240",
|
||||
)
|
||||
|
||||
|
||||
def _dense_cpu(tensor: torch.Tensor) -> torch.Tensor:
|
||||
return tensor.detach().to("cpu").contiguous()
|
||||
|
||||
|
||||
def _raw_sha256(tensor: torch.Tensor) -> str:
|
||||
value = _dense_cpu(tensor)
|
||||
raw = value.view(torch.uint8).numpy().tobytes()
|
||||
return sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def tensor_fingerprint(tensor: torch.Tensor) -> dict[str, Any]:
|
||||
value = _dense_cpu(tensor)
|
||||
return {
|
||||
"shape": list(value.shape),
|
||||
"dtype": str(value.dtype),
|
||||
"sha256": _raw_sha256(value),
|
||||
"numel": int(value.numel()),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TensorDifference:
|
||||
comparable: bool
|
||||
bit_exact: bool
|
||||
allclose: bool
|
||||
max_abs_diff: float | None
|
||||
mean_abs_diff: float | None
|
||||
reason: str | None = None
|
||||
|
||||
@property
|
||||
def matches(self) -> bool:
|
||||
return self.comparable and (self.bit_exact or self.allclose)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
value = asdict(self)
|
||||
value["matches"] = self.matches
|
||||
return value
|
||||
|
||||
|
||||
def compare_tensors(
|
||||
left: torch.Tensor,
|
||||
right: torch.Tensor,
|
||||
*,
|
||||
rtol: float = 1.0e-4,
|
||||
atol: float = 1.0e-5,
|
||||
) -> TensorDifference:
|
||||
a = _dense_cpu(left)
|
||||
b = _dense_cpu(right)
|
||||
if tuple(a.shape) != tuple(b.shape):
|
||||
return TensorDifference(False, False, False, None, None, "shape mismatch")
|
||||
if a.dtype != b.dtype:
|
||||
return TensorDifference(False, False, False, None, None, "dtype mismatch")
|
||||
bit_exact = bool(torch.equal(a, b))
|
||||
if not (a.is_floating_point() or a.is_complex()):
|
||||
return TensorDifference(True, bit_exact, bit_exact, 0.0 if bit_exact else None, 0.0 if bit_exact else None)
|
||||
af = a.to(torch.float64)
|
||||
bf = b.to(torch.float64)
|
||||
delta = (af - bf).abs()
|
||||
max_abs = float(delta.max().item()) if delta.numel() else 0.0
|
||||
mean_abs = float(delta.mean().item()) if delta.numel() else 0.0
|
||||
close = bool(torch.allclose(af, bf, rtol=rtol, atol=atol, equal_nan=True))
|
||||
return TensorDifference(True, bit_exact, close, max_abs, mean_abs)
|
||||
|
||||
|
||||
def _safe_tensor_key(value: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("_") or "tensor"
|
||||
|
||||
|
||||
def _is_primitive(value: Any) -> bool:
|
||||
return value is None or isinstance(value, (bool, int, float, str))
|
||||
|
||||
|
||||
def _primitive_sequence(value: Any) -> bool:
|
||||
return isinstance(value, (list, tuple)) and all(_is_primitive(item) for item in value)
|
||||
|
||||
|
||||
class DiagnosticCapture:
|
||||
def __init__(self, side: str):
|
||||
self.side = side
|
||||
self.manifest: dict[str, Any] = {
|
||||
"format": "h3-core-continuum-gpu-diagnostic-v1",
|
||||
"side": side,
|
||||
"stages": {},
|
||||
}
|
||||
self.tensors: dict[str, torch.Tensor] = {}
|
||||
self.runtime: dict[str, Any] = {}
|
||||
|
||||
def _stage(self, stage: str) -> dict[str, Any]:
|
||||
return self.manifest["stages"].setdefault(stage, {"values": {}, "tensors": {}})
|
||||
|
||||
def add_value(self, stage: str, field: str, value: Any) -> None:
|
||||
if not (_is_primitive(value) or _primitive_sequence(value)):
|
||||
value = json.loads(json.dumps(value, default=str))
|
||||
self._stage(stage)["values"][field] = value
|
||||
|
||||
def add_tensor(self, stage: str, field: str, tensor: torch.Tensor, *, persist: bool = True) -> None:
|
||||
if not torch.is_tensor(tensor):
|
||||
raise TypeError(f"{field} is not a tensor")
|
||||
if getattr(tensor, "is_nested", False):
|
||||
parts = list(tensor.unbind())
|
||||
self.add_value(stage, f"{field}.components", len(parts))
|
||||
for index, part in enumerate(parts):
|
||||
self.add_tensor(stage, f"{field}.component_{index}", part, persist=persist)
|
||||
return
|
||||
value = _dense_cpu(tensor)
|
||||
key = _safe_tensor_key(f"{stage}.{field}")
|
||||
self._stage(stage)["tensors"][field] = {
|
||||
**tensor_fingerprint(value),
|
||||
"stored_as": key if persist else None,
|
||||
}
|
||||
if persist:
|
||||
self.tensors[key] = value.clone()
|
||||
|
||||
def capture_value(
|
||||
self,
|
||||
stage: str,
|
||||
field: str,
|
||||
value: Any,
|
||||
*,
|
||||
persist_tensors: bool = True,
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
if depth > 7:
|
||||
self.add_value(stage, f"{field}.truncated", True)
|
||||
return
|
||||
if torch.is_tensor(value):
|
||||
self.add_tensor(stage, field, value, persist=persist_tensors)
|
||||
return
|
||||
if _is_primitive(value):
|
||||
self.add_value(stage, field, value)
|
||||
return
|
||||
if _primitive_sequence(value):
|
||||
self.add_value(stage, field, list(value))
|
||||
return
|
||||
if isinstance(value, Mapping):
|
||||
keys = sorted(str(key) for key in value.keys())
|
||||
self.add_value(stage, f"{field}.keys", keys)
|
||||
for key in keys[:256]:
|
||||
try:
|
||||
child = value[key]
|
||||
except (KeyError, TypeError):
|
||||
child = next((item for raw, item in value.items() if str(raw) == key), None)
|
||||
self.capture_value(
|
||||
stage,
|
||||
f"{field}.{key}",
|
||||
child,
|
||||
persist_tensors=persist_tensors,
|
||||
depth=depth + 1,
|
||||
)
|
||||
return
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
self.add_value(stage, f"{field}.length", len(value))
|
||||
for index, child in enumerate(value[:256]):
|
||||
self.capture_value(
|
||||
stage,
|
||||
f"{field}.{index}",
|
||||
child,
|
||||
persist_tensors=persist_tensors,
|
||||
depth=depth + 1,
|
||||
)
|
||||
return
|
||||
if hasattr(value, "unbind") and value.__class__.__name__ == "NestedTensor":
|
||||
parts = list(value.unbind())
|
||||
self.add_value(stage, f"{field}.components", len(parts))
|
||||
for index, child in enumerate(parts):
|
||||
label = ("video", "audio")[index] if index < 2 else f"component_{index}"
|
||||
self.capture_value(
|
||||
stage,
|
||||
f"{field}.{label}",
|
||||
child,
|
||||
persist_tensors=persist_tensors,
|
||||
depth=depth + 1,
|
||||
)
|
||||
return
|
||||
if all(hasattr(value, name) for name in ("signature", "segments", "position_ids")):
|
||||
self.add_value(stage, f"{field}.type", value.__class__.__name__)
|
||||
self.add_value(stage, f"{field}.signature", list(value.signature))
|
||||
self.add_value(stage, f"{field}.segments", [list(item) for item in value.segments])
|
||||
for name in ("position_ids", "img_pos", "img_update", "audio_pos", "audio_update"):
|
||||
child = getattr(value, name, None)
|
||||
if torch.is_tensor(child):
|
||||
self.add_tensor(stage, f"{field}.{name}", child, persist=persist_tensors)
|
||||
return
|
||||
self.add_value(stage, f"{field}.type", value.__class__.__name__)
|
||||
|
||||
def save(self, directory: Path) -> None:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
from safetensors.torch import save_file
|
||||
|
||||
save_file(self.tensors, str(directory / "tensors.safetensors"))
|
||||
(directory / "manifest.json").write_text(
|
||||
json.dumps(self.manifest, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def compare_stage(
|
||||
left: DiagnosticCapture,
|
||||
right: DiagnosticCapture,
|
||||
stage: str,
|
||||
) -> dict[str, Any]:
|
||||
a = left.manifest["stages"].get(stage, {"values": {}, "tensors": {}})
|
||||
b = right.manifest["stages"].get(stage, {"values": {}, "tensors": {}})
|
||||
details: dict[str, Any] = {}
|
||||
first_field: str | None = None
|
||||
|
||||
value_fields = list(a["values"].keys()) + [key for key in b["values"] if key not in a["values"]]
|
||||
for field in value_fields:
|
||||
av = a["values"].get(field, {"missing": True})
|
||||
bv = b["values"].get(field, {"missing": True})
|
||||
matches = av == bv
|
||||
details[field] = {"kind": "value", "matches": matches, "left": av, "right": bv}
|
||||
if not matches and first_field is None:
|
||||
first_field = field
|
||||
|
||||
tensor_fields = list(a["tensors"].keys()) + [key for key in b["tensors"] if key not in a["tensors"]]
|
||||
for field in tensor_fields:
|
||||
a_meta = a["tensors"].get(field)
|
||||
b_meta = b["tensors"].get(field)
|
||||
if a_meta is None or b_meta is None:
|
||||
result = {"kind": "tensor", "matches": False, "reason": "missing tensor"}
|
||||
else:
|
||||
a_key = a_meta.get("stored_as")
|
||||
b_key = b_meta.get("stored_as")
|
||||
if a_key in left.tensors and b_key in right.tensors:
|
||||
result = {"kind": "tensor", **compare_tensors(left.tensors[a_key], right.tensors[b_key]).to_dict()}
|
||||
else:
|
||||
matches = a_meta == b_meta
|
||||
result = {"kind": "fingerprint", "matches": matches, "left": a_meta, "right": b_meta}
|
||||
details[field] = result
|
||||
if not result["matches"] and first_field is None:
|
||||
first_field = field
|
||||
|
||||
return {
|
||||
"status": "MATCH" if first_field is None else "DIFFER",
|
||||
"first_field": first_field,
|
||||
"fields": details,
|
||||
}
|
||||
|
||||
|
||||
def recombine_terminal_streams(
|
||||
first: torch.Tensor,
|
||||
second: torch.Tensor,
|
||||
*,
|
||||
first_range: tuple[int, int] | list[int],
|
||||
second_range: tuple[int, int] | list[int],
|
||||
dim: int,
|
||||
) -> torch.Tensor:
|
||||
first_start, first_stop = (int(value) for value in first_range)
|
||||
second_start, second_stop = (int(value) for value in second_range)
|
||||
overlap = first_stop - second_start
|
||||
if first_start != 0 or overlap < 0:
|
||||
raise ValueError("terminal split ranges do not form an overlapping sequence")
|
||||
if first.shape[dim] != first_stop - first_start:
|
||||
raise ValueError("first logical stream length does not match its range")
|
||||
if second.shape[dim] != second_stop - second_start:
|
||||
raise ValueError("second logical stream length does not match its range")
|
||||
if overlap > second.shape[dim]:
|
||||
raise ValueError("terminal overlap exceeds the second logical stream")
|
||||
slices = [slice(None)] * second.ndim
|
||||
slices[dim] = slice(overlap, None)
|
||||
return torch.cat((first, second[tuple(slices)]), dim=dim).contiguous()
|
||||
|
||||
|
||||
def format_comparison(comparison: Mapping[str, Any], output_directory: Path) -> str:
|
||||
lines = []
|
||||
first_divergence = None
|
||||
for stage in STAGE_ORDER:
|
||||
result = comparison.get(stage, {"status": "NOT_RUN", "first_field": None})
|
||||
lines.append(f"{stage}: {result['status']}")
|
||||
if first_divergence is None and result["status"] == "DIFFER":
|
||||
field = result.get("first_field")
|
||||
first_divergence = f"{stage}.{field}" if field else stage
|
||||
lines.append("")
|
||||
lines.append(f"First divergence: {first_divergence or 'none'}")
|
||||
lines.append(f"Output: {output_directory}")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Diagnostic-only continuation depth and mask-strength policies for Issue #13."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
MODE_BASELINE_22 = "Baseline 22f"
|
||||
MODE_DEPTH_13 = "Video Depth 13f"
|
||||
MODE_DEPTH_9 = "Video Depth 9f"
|
||||
MODE_STRENGTH_75 = "Video Strength 75%"
|
||||
MODE_DEPTH_13_STRENGTH_75 = "Video Depth 13f + Strength 75%"
|
||||
MODE_OPTIONS = (
|
||||
MODE_BASELINE_22,
|
||||
MODE_DEPTH_13,
|
||||
MODE_DEPTH_9,
|
||||
MODE_STRENGTH_75,
|
||||
MODE_DEPTH_13_STRENGTH_75,
|
||||
)
|
||||
|
||||
_PREFIX_SLOTS = 7
|
||||
_AUDIO_PREFIX_STEPS = 37
|
||||
_ACTIVE_SLOT_CONTRACT = {
|
||||
MODE_BASELINE_22: (_PREFIX_SLOTS, 22),
|
||||
MODE_DEPTH_13: (4, 13),
|
||||
MODE_DEPTH_9: (3, 9),
|
||||
MODE_STRENGTH_75: (_PREFIX_SLOTS, 22),
|
||||
MODE_DEPTH_13_STRENGTH_75: (4, 13),
|
||||
}
|
||||
|
||||
|
||||
def _mask_values(tensor: torch.Tensor) -> list[float]:
|
||||
return [float(value) for value in torch.unique(tensor.detach().float()).tolist()]
|
||||
|
||||
|
||||
class R28ContinuationPolicy:
|
||||
"""Scoped callback used only by the external R2.8 diagnostic node."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode: str,
|
||||
*,
|
||||
tensor_summary: Callable[[torch.Tensor], dict[str, Any]],
|
||||
) -> None:
|
||||
if mode not in MODE_OPTIONS:
|
||||
raise ValueError(f"unknown R2.8 continuation policy: {mode!r}")
|
||||
self.mode = str(mode)
|
||||
self.tensor_summary = tensor_summary
|
||||
self._prepared: dict[int, dict[str, Any]] = {}
|
||||
self.sample_calls: list[dict[str, Any]] = []
|
||||
|
||||
@property
|
||||
def active_video_slots(self) -> int:
|
||||
return int(_ACTIVE_SLOT_CONTRACT[self.mode][0])
|
||||
|
||||
@property
|
||||
def active_video_frames(self) -> int:
|
||||
return int(_ACTIVE_SLOT_CONTRACT[self.mode][1])
|
||||
|
||||
@property
|
||||
def prefix_denoise_mask_value(self) -> float:
|
||||
if self.mode in (MODE_STRENGTH_75, MODE_DEPTH_13_STRENGTH_75):
|
||||
return 0.25
|
||||
return 0.0
|
||||
|
||||
def prepare_masked_latent(
|
||||
self,
|
||||
*,
|
||||
latent: dict[str, Any],
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
context_frames: int,
|
||||
video_context: torch.Tensor,
|
||||
audio_context: torch.Tensor | None,
|
||||
) -> dict[str, Any]:
|
||||
if int(context_frames) != 22:
|
||||
raise ValueError("R2.8 policy requires the unchanged Balanced 22-frame transport")
|
||||
mask = latent.get("noise_mask")
|
||||
if mask is None or not hasattr(mask, "unbind"):
|
||||
raise ValueError("R2.8 policy requires the native nested AV noise mask")
|
||||
parts = list(mask.unbind())
|
||||
if len(parts) < 2:
|
||||
raise ValueError("R2.8 policy requires Video and Audio masks")
|
||||
video_mask, audio_mask = parts[0], parts[1]
|
||||
if int(video_mask.shape[2]) <= _PREFIX_SLOTS:
|
||||
raise ValueError("R2.8 Video target has no generated region after the prefix")
|
||||
if int(audio_mask.shape[-1]) <= _AUDIO_PREFIX_STEPS:
|
||||
raise ValueError("R2.8 Audio target has no generated region after the prefix")
|
||||
|
||||
original_video_mask = video_mask.detach().clone()
|
||||
original_audio_mask = audio_mask.detach().clone()
|
||||
transformed = latent
|
||||
if self.mode != MODE_BASELINE_22:
|
||||
video_mask = video_mask.clone()
|
||||
oldest_inactive = _PREFIX_SLOTS - self.active_video_slots
|
||||
if oldest_inactive:
|
||||
video_mask[:, :, :oldest_inactive] = 1.0
|
||||
video_mask[
|
||||
:, :, oldest_inactive:_PREFIX_SLOTS
|
||||
] = self.prefix_denoise_mask_value
|
||||
transformed = dict(latent)
|
||||
transformed["noise_mask"] = mask.__class__((video_mask, audio_mask.clone()))
|
||||
|
||||
effective_mask = list(transformed["noise_mask"].unbind())
|
||||
effective_video_mask, effective_audio_mask = effective_mask[0], effective_mask[1]
|
||||
self._prepared[int(physical_group)] = {
|
||||
"physical_group": int(physical_group),
|
||||
"logical_chunks": [int(value) for value in logical_chunks],
|
||||
"transport_context_frames": int(context_frames),
|
||||
"transport_video_prefix_slots": _PREFIX_SLOTS,
|
||||
"transport_audio_prefix_steps": _AUDIO_PREFIX_STEPS,
|
||||
"active_video_slots": self.active_video_slots,
|
||||
"active_video_frames": self.active_video_frames,
|
||||
"video_prefix_denoise_mask_value": self.prefix_denoise_mask_value,
|
||||
"video_context": self.tensor_summary(video_context),
|
||||
"audio_context": self.tensor_summary(audio_context)
|
||||
if audio_context is not None
|
||||
else None,
|
||||
"source_samples_object_passthrough": transformed.get("samples")
|
||||
is latent.get("samples"),
|
||||
"input_video_mask_unchanged": bool(
|
||||
torch.equal(original_video_mask, list(mask.unbind())[0])
|
||||
),
|
||||
"input_audio_mask_unchanged": bool(
|
||||
torch.equal(original_audio_mask, list(mask.unbind())[1])
|
||||
),
|
||||
"effective_video_prefix_mask_values": _mask_values(
|
||||
effective_video_mask[:, :, :_PREFIX_SLOTS]
|
||||
),
|
||||
"effective_video_generated_mask_values": _mask_values(
|
||||
effective_video_mask[:, :, _PREFIX_SLOTS:]
|
||||
),
|
||||
"effective_audio_prefix_mask_values": _mask_values(
|
||||
effective_audio_mask[..., :_AUDIO_PREFIX_STEPS]
|
||||
),
|
||||
"effective_audio_generated_mask_values": _mask_values(
|
||||
effective_audio_mask[..., _AUDIO_PREFIX_STEPS:]
|
||||
),
|
||||
}
|
||||
return transformed
|
||||
|
||||
def before_sampling(
|
||||
self,
|
||||
*,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
context_frames: int,
|
||||
latent: dict[str, Any],
|
||||
conditioning: Any,
|
||||
seed: int,
|
||||
sigmas: torch.Tensor,
|
||||
) -> dict[str, Any]:
|
||||
samples = list(latent["samples"].unbind())
|
||||
metadata = conditioning[0][1] if conditioning else {}
|
||||
record = {
|
||||
"physical_group": int(physical_group),
|
||||
"logical_chunks": [int(value) for value in logical_chunks],
|
||||
"context_frames": int(context_frames),
|
||||
"seed": int(seed),
|
||||
"sigmas": self.tensor_summary(sigmas),
|
||||
"input_video": self.tensor_summary(samples[0]),
|
||||
"input_audio": self.tensor_summary(samples[1]),
|
||||
"prompt_embedding": self.tensor_summary(conditioning[0][0])
|
||||
if conditioning and torch.is_tensor(conditioning[0][0])
|
||||
else None,
|
||||
"minimax_frame_count": metadata.get("minimax_frame_count"),
|
||||
"policy": self._prepared.get(int(physical_group)),
|
||||
}
|
||||
return {"started": time.perf_counter(), "record": record}
|
||||
|
||||
def after_sampling(
|
||||
self,
|
||||
*,
|
||||
token: dict[str, Any],
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
context_frames: int,
|
||||
latent: dict[str, Any],
|
||||
sampled: dict[str, Any],
|
||||
seed: int,
|
||||
sigmas: torch.Tensor,
|
||||
) -> None:
|
||||
record = dict(token["record"])
|
||||
record["sample_host_elapsed_seconds"] = float(
|
||||
time.perf_counter() - float(token["started"])
|
||||
)
|
||||
source_video, source_audio = list(latent["samples"].unbind())[:2]
|
||||
output_video, output_audio = list(sampled["samples"].unbind())[:2]
|
||||
record["output_video"] = self.tensor_summary(output_video)
|
||||
record["output_audio"] = self.tensor_summary(output_audio)
|
||||
if int(context_frames) == 22:
|
||||
source_video_prefix = source_video[:, :, :_PREFIX_SLOTS]
|
||||
output_video_prefix = output_video[:, :, :_PREFIX_SLOTS]
|
||||
source_audio_prefix = source_audio[..., :_AUDIO_PREFIX_STEPS]
|
||||
output_audio_prefix = output_audio[..., :_AUDIO_PREFIX_STEPS]
|
||||
record["restored_video_prefix_bit_exact"] = bool(
|
||||
torch.equal(source_video_prefix, output_video_prefix)
|
||||
)
|
||||
record["restored_audio_prefix_bit_exact"] = bool(
|
||||
torch.equal(source_audio_prefix, output_audio_prefix)
|
||||
)
|
||||
record["restored_video_prefix"] = self.tensor_summary(output_video_prefix)
|
||||
record["restored_audio_prefix"] = self.tensor_summary(output_audio_prefix)
|
||||
self.sample_calls.append(record)
|
||||
|
||||
def finalize(self) -> dict[str, Any]:
|
||||
return {
|
||||
"format": "h3-continuum-issue13-r28-policy-v1",
|
||||
"mode": self.mode,
|
||||
"transport_context_frames": 22,
|
||||
"transport_video_prefix_slots": _PREFIX_SLOTS,
|
||||
"transport_audio_prefix_steps": _AUDIO_PREFIX_STEPS,
|
||||
"active_video_slots": self.active_video_slots,
|
||||
"active_video_frames": self.active_video_frames,
|
||||
"video_prefix_denoise_mask_value": self.prefix_denoise_mask_value,
|
||||
"audio_contract": "unchanged 37T protected prefix",
|
||||
"sample_calls": list(self.sample_calls),
|
||||
"sampling_count": len(self.sample_calls),
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,164 @@
|
||||
"""Diagnostic-only one-shot appearance re-anchor for Issue #13 R2.7."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
SOFT_REANCHOR_STRENGTH = 0.5
|
||||
_EPSILON = 1.0e-6
|
||||
|
||||
|
||||
def _validate_pixels(value: torch.Tensor, *, label: str) -> torch.Tensor:
|
||||
if not torch.is_tensor(value) or value.ndim != 5 or int(value.shape[-1]) != 3:
|
||||
raise ValueError(f"{label} must be [B,F,H,W,3] RGB pixels")
|
||||
measured = value.detach().float()
|
||||
if not bool(torch.isfinite(measured).all().item()):
|
||||
raise ValueError(f"{label} contains NaN or Inf")
|
||||
return measured
|
||||
|
||||
|
||||
def rgb_appearance_summary(value: torch.Tensor) -> dict[str, float]:
|
||||
"""Return global low-frequency RGB appearance statistics."""
|
||||
|
||||
rgb = _validate_pixels(value, label="pixels")
|
||||
red, green, blue = rgb.unbind(dim=-1)
|
||||
luma = 0.2126 * red + 0.7152 * green + 0.0722 * blue
|
||||
maximum = rgb.amax(dim=-1)
|
||||
minimum = rgb.amin(dim=-1)
|
||||
saturation = (maximum - minimum) / maximum.clamp_min(_EPSILON)
|
||||
return {
|
||||
"luma_mean": float(luma.mean().item()),
|
||||
"luma_std": float(luma.std(unbiased=False).item()),
|
||||
"saturation_mean": float(saturation.mean().item()),
|
||||
"chroma_u_mean": float((blue - luma).mean().item()),
|
||||
"chroma_u_std": float((blue - luma).std(unbiased=False).item()),
|
||||
"chroma_v_mean": float((red - luma).mean().item()),
|
||||
"chroma_v_std": float((red - luma).std(unbiased=False).item()),
|
||||
"highlight_fraction": float((luma >= (250.0 / 255.0)).float().mean().item()),
|
||||
}
|
||||
|
||||
|
||||
def _matched_channel(
|
||||
current: torch.Tensor,
|
||||
reference: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, float]:
|
||||
current_mean = current.mean()
|
||||
current_std = current.std(unbiased=False).clamp_min(_EPSILON)
|
||||
reference_mean = reference.mean()
|
||||
reference_std = reference.std(unbiased=False).clamp_min(_EPSILON)
|
||||
scale = (reference_std / current_std).clamp(0.5, 2.0)
|
||||
return (current - current_mean) * scale + reference_mean, float(scale.item())
|
||||
|
||||
|
||||
def soft_match_rgb_appearance(
|
||||
current_pixels: torch.Tensor,
|
||||
reference_pixels: torch.Tensor,
|
||||
*,
|
||||
strength: float = SOFT_REANCHOR_STRENGTH,
|
||||
) -> tuple[torch.Tensor, dict[str, Any]]:
|
||||
"""Match only global luma/chroma statistics; preserve spatial coordinates."""
|
||||
|
||||
amount = float(strength)
|
||||
if not 0.0 <= amount <= 1.0:
|
||||
raise ValueError("soft re-anchor strength must be in [0,1]")
|
||||
current = _validate_pixels(current_pixels, label="current_pixels")
|
||||
reference = _validate_pixels(reference_pixels, label="reference_pixels")
|
||||
if tuple(current.shape[-3:]) != tuple(reference.shape[-3:]):
|
||||
raise ValueError("current/reference pixel geometry must match")
|
||||
|
||||
current_r, current_g, current_b = current.unbind(dim=-1)
|
||||
reference_r, reference_g, reference_b = reference.unbind(dim=-1)
|
||||
current_y = 0.2126 * current_r + 0.7152 * current_g + 0.0722 * current_b
|
||||
reference_y = (
|
||||
0.2126 * reference_r + 0.7152 * reference_g + 0.0722 * reference_b
|
||||
)
|
||||
current_u = current_b - current_y
|
||||
current_v = current_r - current_y
|
||||
reference_u = reference_b - reference_y
|
||||
reference_v = reference_r - reference_y
|
||||
|
||||
matched_y, luma_scale = _matched_channel(current_y, reference_y)
|
||||
matched_u, chroma_u_scale = _matched_channel(current_u, reference_u)
|
||||
matched_v, chroma_v_scale = _matched_channel(current_v, reference_v)
|
||||
matched_r = matched_y + matched_v
|
||||
matched_b = matched_y + matched_u
|
||||
matched_g = (matched_y - 0.2126 * matched_r - 0.0722 * matched_b) / 0.7152
|
||||
fully_matched = torch.stack((matched_r, matched_g, matched_b), dim=-1).clamp(0.0, 1.0)
|
||||
adjusted = (current + amount * (fully_matched - current)).clamp(0.0, 1.0)
|
||||
delta = adjusted - current
|
||||
report = {
|
||||
"strength": amount,
|
||||
"method": "global Y/U/V mean+std match; 50% blend; no blur/warp/resize",
|
||||
"scales": {
|
||||
"luma": luma_scale,
|
||||
"chroma_u": chroma_u_scale,
|
||||
"chroma_v": chroma_v_scale,
|
||||
},
|
||||
"reference": rgb_appearance_summary(reference),
|
||||
"before": rgb_appearance_summary(current),
|
||||
"fully_matched": rgb_appearance_summary(fully_matched),
|
||||
"after": rgb_appearance_summary(adjusted),
|
||||
"pixel_delta_rms": float(delta.square().mean().sqrt().item()),
|
||||
"pixel_delta_max_abs": float(delta.abs().max().item()),
|
||||
"geometry_unchanged": tuple(adjusted.shape) == tuple(current.shape),
|
||||
}
|
||||
return adjusted.to(dtype=current_pixels.dtype), report
|
||||
|
||||
|
||||
def soft_reanchor_context(
|
||||
current_context: torch.Tensor,
|
||||
reference_context: torch.Tensor,
|
||||
*,
|
||||
video_vae: Any,
|
||||
strength: float = SOFT_REANCHOR_STRENGTH,
|
||||
) -> tuple[torch.Tensor, dict[str, Any]]:
|
||||
"""Decode/adjust/encode one temporary context copy without mutating inputs."""
|
||||
|
||||
if not torch.is_tensor(current_context) or current_context.ndim != 5:
|
||||
raise ValueError("current_context must be [B,C,T,H,W]")
|
||||
if not torch.is_tensor(reference_context) or reference_context.ndim != 5:
|
||||
raise ValueError("reference_context must be [B,C,T,H,W]")
|
||||
if tuple(current_context.shape) != tuple(reference_context.shape):
|
||||
raise ValueError("current/reference latent context shapes must match")
|
||||
if int(current_context.shape[0]) != 1:
|
||||
raise ValueError("R2.7 diagnostic expects a single context batch")
|
||||
if not callable(getattr(video_vae, "decode", None)) or not callable(
|
||||
getattr(video_vae, "encode", None)
|
||||
):
|
||||
raise ValueError("R2.7 Soft Re-anchor requires the connected Video VAE")
|
||||
|
||||
current_guard = current_context.detach().clone()
|
||||
reference_guard = reference_context.detach().clone()
|
||||
reference_pixels = video_vae.decode(reference_context)
|
||||
current_pixels = video_vae.decode(current_context)
|
||||
adjusted_pixels, appearance = soft_match_rgb_appearance(
|
||||
current_pixels,
|
||||
reference_pixels,
|
||||
strength=strength,
|
||||
)
|
||||
encoded = video_vae.encode(adjusted_pixels[0])
|
||||
if not torch.is_tensor(encoded) or tuple(encoded.shape) != tuple(current_context.shape):
|
||||
raise ValueError(
|
||||
"Soft Re-anchor VAE roundtrip changed the continuation latent shape: "
|
||||
f"{getattr(encoded, 'shape', None)} vs {tuple(current_context.shape)}"
|
||||
)
|
||||
encoded = encoded.to(
|
||||
device=current_context.device,
|
||||
dtype=current_context.dtype,
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(encoded.float()).all().item()):
|
||||
raise ValueError("Soft Re-anchor encoded context contains NaN or Inf")
|
||||
appearance.update(
|
||||
{
|
||||
"current_context_unchanged": bool(torch.equal(current_context, current_guard)),
|
||||
"reference_context_unchanged": bool(
|
||||
torch.equal(reference_context, reference_guard)
|
||||
),
|
||||
"latent_shape_unchanged": tuple(encoded.shape)
|
||||
== tuple(current_context.shape),
|
||||
}
|
||||
)
|
||||
return encoded, appearance
|
||||
Reference in New Issue
Block a user