Merge commit 'bf8f76505fdd25ea99770f262eb9371509bf226d' as 'vendor/dumas_h3_continuum'
This commit is contained in:
+5
@@ -0,0 +1,5 @@
|
||||
"""Latent-first H3 Continuum V3 nodes."""
|
||||
|
||||
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
"""Read-only Adaptive Continuity observation and decision trace for A8a.
|
||||
|
||||
The observer records the continuation contract that Production already chose.
|
||||
It never changes context depth, transport, masks, tensors, conditioning, MODEL,
|
||||
Seed, SIGMAS, physical grouping, or execution policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
|
||||
ADAPTIVE_CONTINUITY_OBSERVER_VERSION = 1
|
||||
EXECUTION_APPLIED = False
|
||||
|
||||
INITIAL_TRANSPORT = "initial"
|
||||
REFERENCE_CONTEXT_V1 = "reference_context_v1"
|
||||
MASKED_VIDEO_PREFIX_V1 = "masked_video_prefix_v1"
|
||||
MASKED_AV_PREFIX_22_V1 = "masked_av_prefix_22_v1"
|
||||
MASKED_AV_PREFIX_39_V1 = "masked_av_prefix_39_v1"
|
||||
|
||||
_KNOWN_TRANSPORTS = frozenset(
|
||||
(
|
||||
INITIAL_TRANSPORT,
|
||||
REFERENCE_CONTEXT_V1,
|
||||
MASKED_VIDEO_PREFIX_V1,
|
||||
MASKED_AV_PREFIX_22_V1,
|
||||
MASKED_AV_PREFIX_39_V1,
|
||||
)
|
||||
)
|
||||
_VALID_CONTEXT_FRAMES = frozenset((0, 5, 22, 39))
|
||||
_PLANNER_CONTRACT = {
|
||||
"planner_version": ADAPTIVE_CONTINUITY_OBSERVER_VERSION,
|
||||
"mode": "read_only_observer",
|
||||
"execution_applied": EXECUTION_APPLIED,
|
||||
"confidence_scope": "observation_completeness_not_mitigation_efficacy",
|
||||
"issue_13_mitigation_claim": False,
|
||||
"mutations": [],
|
||||
"observed_fields": [
|
||||
"motion_score",
|
||||
"current_context_frames",
|
||||
"resolved_transport",
|
||||
"terminal_atomic",
|
||||
"boundary_index",
|
||||
"boundary_count",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _canonical_hash(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
ADAPTIVE_CONTINUITY_PLANNER_HASH = _canonical_hash(_PLANNER_CONTRACT)
|
||||
|
||||
|
||||
class AdaptiveContinuityObservationError(ValueError):
|
||||
"""One diagnostic observation is malformed or internally inconsistent."""
|
||||
|
||||
|
||||
def _recommendation(
|
||||
*,
|
||||
current_context_frames: int,
|
||||
resolved_transport: str,
|
||||
terminal_atomic: bool,
|
||||
) -> str:
|
||||
if int(current_context_frames) == 0:
|
||||
return "no_continuation_action_initial_group"
|
||||
if bool(terminal_atomic):
|
||||
return "keep_fixed_terminal_merge_contract"
|
||||
if str(resolved_transport) == REFERENCE_CONTEXT_V1:
|
||||
return "keep_current_reference_context_contract"
|
||||
return "keep_current_masked_prefix_contract"
|
||||
|
||||
|
||||
def make_adaptive_continuity_decision(
|
||||
*,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
observed_motion_score: float,
|
||||
current_context_frames: int,
|
||||
resolved_transport: str,
|
||||
terminal_atomic: bool,
|
||||
boundary_index: int,
|
||||
boundary_count: int,
|
||||
reason: str,
|
||||
fallback_reason: str | None = None,
|
||||
reused: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Build one immutable, JSON-safe observation without retaining tensors."""
|
||||
|
||||
group = int(physical_group)
|
||||
logical = tuple(int(value) for value in logical_chunks)
|
||||
context = int(current_context_frames)
|
||||
boundary = int(boundary_index)
|
||||
boundaries = int(boundary_count)
|
||||
transport = str(resolved_transport)
|
||||
motion = float(observed_motion_score)
|
||||
reason_text = str(reason).strip()
|
||||
|
||||
if group < 1 or not logical or any(value < 1 for value in logical):
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"physical group and logical chunks must be positive"
|
||||
)
|
||||
if tuple(sorted(set(logical))) != logical:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"logical chunks must be ordered and unique"
|
||||
)
|
||||
if context not in _VALID_CONTEXT_FRAMES:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
f"unsupported observed context frame count: {context}"
|
||||
)
|
||||
if transport not in _KNOWN_TRANSPORTS:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
f"unknown resolved continuation transport: {transport!r}"
|
||||
)
|
||||
if context == 0 and transport != INITIAL_TRANSPORT:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"zero-context observations must use the initial transport marker"
|
||||
)
|
||||
if context > 0 and transport == INITIAL_TRANSPORT:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"continuation observations require a resolved transport"
|
||||
)
|
||||
if boundary < 0 or boundaries < 0 or boundary > boundaries:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"boundary index/count are inconsistent"
|
||||
)
|
||||
if not math.isfinite(motion) or motion < 0.0:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"observed motion score must be finite and non-negative"
|
||||
)
|
||||
|
||||
confidence = 1.0 if reason_text else 0.5
|
||||
payload: dict[str, Any] = {
|
||||
"planner_version": ADAPTIVE_CONTINUITY_OBSERVER_VERSION,
|
||||
"planner_hash": ADAPTIVE_CONTINUITY_PLANNER_HASH,
|
||||
"physical_group": group,
|
||||
"logical_chunks": list(logical),
|
||||
"observed_motion_score": motion,
|
||||
"current_context_frames": context,
|
||||
"resolved_transport": transport,
|
||||
"terminal_atomic": bool(terminal_atomic),
|
||||
"boundary_index": boundary,
|
||||
"boundary_count": boundaries,
|
||||
"recommendation": _recommendation(
|
||||
current_context_frames=context,
|
||||
resolved_transport=transport,
|
||||
terminal_atomic=bool(terminal_atomic),
|
||||
),
|
||||
"confidence": confidence,
|
||||
"confidence_scope": "observation completeness only; not mitigation efficacy",
|
||||
"reason": reason_text or "selection reason unavailable",
|
||||
"fallback_reason": (
|
||||
str(fallback_reason).strip() if fallback_reason is not None else None
|
||||
),
|
||||
"reused": bool(reused),
|
||||
"execution_applied": EXECUTION_APPLIED,
|
||||
"issue_13_mitigation_claim": False,
|
||||
}
|
||||
payload["decision_hash"] = _canonical_hash(payload)
|
||||
return payload
|
||||
|
||||
|
||||
class AdaptiveContinuityObserver:
|
||||
"""Collect fail-soft A8a observations in deterministic physical-group order."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._records: list[dict[str, Any]] = []
|
||||
self._advisories: list[str] = []
|
||||
|
||||
@property
|
||||
def records(self) -> tuple[dict[str, Any], ...]:
|
||||
return tuple(dict(record) for record in self._records)
|
||||
|
||||
def observe(self, **kwargs: Any) -> dict[str, Any]:
|
||||
record = make_adaptive_continuity_decision(**kwargs)
|
||||
self._records.append(record)
|
||||
return record
|
||||
|
||||
def note_observation_failure(self, exc: Exception) -> None:
|
||||
self._advisories.append(f"{type(exc).__name__}: {exc}")
|
||||
|
||||
def replay_prefix(
|
||||
self,
|
||||
*,
|
||||
entries: list[dict[str, Any]],
|
||||
total_chunks: int,
|
||||
terminal_merge_enabled: bool,
|
||||
resolved_transport: str,
|
||||
) -> None:
|
||||
"""Reconstruct trace-only records from accepted CPU chunk metadata."""
|
||||
|
||||
chunk_count = int(total_chunks)
|
||||
completed = len(entries)
|
||||
if chunk_count < 1 or completed > chunk_count:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"replayed prefix length is outside the configured sequence"
|
||||
)
|
||||
boundary_count = max(0, chunk_count - 1)
|
||||
normal_count = completed
|
||||
terminal_complete = False
|
||||
pair_start = max(0, chunk_count - 2)
|
||||
if bool(terminal_merge_enabled):
|
||||
if pair_start < completed < chunk_count:
|
||||
raise AdaptiveContinuityObservationError(
|
||||
"terminal merged prefix contains only one logical half"
|
||||
)
|
||||
terminal_complete = completed == chunk_count
|
||||
normal_count = pair_start if terminal_complete else completed
|
||||
|
||||
for index in range(normal_count):
|
||||
entry = entries[index]
|
||||
context = int(entry.get("context_frames", 0))
|
||||
self.observe(
|
||||
physical_group=index + 1,
|
||||
logical_chunks=(index + 1,),
|
||||
observed_motion_score=float(entry.get("motion_score", 0.0)),
|
||||
current_context_frames=context,
|
||||
resolved_transport=(
|
||||
INITIAL_TRANSPORT if context == 0 else str(resolved_transport)
|
||||
),
|
||||
terminal_atomic=False,
|
||||
boundary_index=index,
|
||||
boundary_count=boundary_count,
|
||||
reason="replayed from accepted chunk metadata",
|
||||
fallback_reason=None,
|
||||
reused=True,
|
||||
)
|
||||
|
||||
if terminal_complete:
|
||||
entry = entries[pair_start]
|
||||
context = int(entry.get("context_frames", 0))
|
||||
self.observe(
|
||||
physical_group=pair_start + 1,
|
||||
logical_chunks=(pair_start + 1, chunk_count),
|
||||
observed_motion_score=float(entry.get("motion_score", 0.0)),
|
||||
current_context_frames=context,
|
||||
resolved_transport=(
|
||||
INITIAL_TRANSPORT if context == 0 else str(resolved_transport)
|
||||
),
|
||||
terminal_atomic=True,
|
||||
boundary_index=pair_start,
|
||||
boundary_count=boundary_count,
|
||||
reason="replayed from accepted Terminal Merge metadata",
|
||||
fallback_reason=None,
|
||||
reused=True,
|
||||
)
|
||||
|
||||
def report_lines(self) -> list[str]:
|
||||
lines = [
|
||||
"Adaptive Continuity observer [A8a v1]: "
|
||||
f"planner_hash={ADAPTIVE_CONTINUITY_PLANNER_HASH}, "
|
||||
f"observed_groups={len(self._records)}, "
|
||||
"execution_applied=false; recommendation trace only, "
|
||||
"Issue #13 mitigation claim=none, Production continuity unchanged."
|
||||
]
|
||||
for record in self._records:
|
||||
lines.append(
|
||||
"Adaptive Continuity decision [A8a v1]: "
|
||||
f"physical_group={record['physical_group']}, "
|
||||
f"logical_chunks={record['logical_chunks']}, "
|
||||
f"boundary={record['boundary_index']}/{record['boundary_count']}, "
|
||||
f"motion={record['observed_motion_score']:.6f}, "
|
||||
f"context={record['current_context_frames']}, "
|
||||
f"transport={record['resolved_transport']}, "
|
||||
f"terminal_atomic={str(record['terminal_atomic']).lower()}, "
|
||||
f"reused={str(record['reused']).lower()}, "
|
||||
f"recommendation={record['recommendation']}, "
|
||||
f"confidence={record['confidence']:.2f}, "
|
||||
f"reason={record['reason']}, "
|
||||
f"fallback_reason={record['fallback_reason'] or 'none'}, "
|
||||
f"decision_hash={record['decision_hash']}, "
|
||||
"execution_applied=false."
|
||||
)
|
||||
lines.extend(
|
||||
f"Adaptive Continuity advisory [A8a v1]: {item}; execution_applied=false."
|
||||
for item in self._advisories
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def adaptive_observer_event_fail_soft(
|
||||
observer: Any,
|
||||
method: str,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Invoke one observer event without allowing diagnostics to stop Sampling."""
|
||||
|
||||
if observer is None:
|
||||
return None
|
||||
try:
|
||||
return getattr(observer, str(method))(**kwargs)
|
||||
except Exception as exc:
|
||||
try:
|
||||
observer.note_observation_failure(exc)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ADAPTIVE_CONTINUITY_OBSERVER_VERSION",
|
||||
"ADAPTIVE_CONTINUITY_PLANNER_HASH",
|
||||
"AdaptiveContinuityObservationError",
|
||||
"AdaptiveContinuityObserver",
|
||||
"EXECUTION_APPLIED",
|
||||
"INITIAL_TRANSPORT",
|
||||
"adaptive_observer_event_fail_soft",
|
||||
"make_adaptive_continuity_decision",
|
||||
]
|
||||
+431
@@ -0,0 +1,431 @@
|
||||
"""Decoded chunk assembly for latent-first H3 Continuum V3."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..media import validate_audio
|
||||
from ..v2.decoder import _slice_audio_for_timeline, enforce_total_frames
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_FULL,
|
||||
DIAGNOSTICS_OFF,
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
normalize_diagnostics_mode,
|
||||
)
|
||||
from ..v2.seam_guard import correct_audio_seam
|
||||
from .plan import FPS, validate_assembly_plan
|
||||
|
||||
AUDIO_SEAM_OFF = "Off"
|
||||
AUDIO_SEAM_AUTO = "Auto"
|
||||
AUDIO_SEAM_OPTIONS = (AUDIO_SEAM_OFF, AUDIO_SEAM_AUTO)
|
||||
VIDEO_SEAM_OFF = "Off"
|
||||
VIDEO_SEAM_ANALYZE = "Analyze Only"
|
||||
VIDEO_SEAM_AUTO = "Auto"
|
||||
VIDEO_SEAM_AUTO_2 = "Auto 2"
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS = (
|
||||
VIDEO_SEAM_AUTO,
|
||||
VIDEO_SEAM_AUTO_2,
|
||||
VIDEO_SEAM_ANALYZE,
|
||||
VIDEO_SEAM_OFF,
|
||||
)
|
||||
|
||||
|
||||
def _singleton(value: Any, name: str) -> Any:
|
||||
while isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
value = value[0]
|
||||
return value
|
||||
|
||||
|
||||
def _chunk_list(value: Any) -> list[Any]:
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1 and isinstance(value[0], list):
|
||||
return list(value[0])
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def _format_elapsed(seconds: float) -> str:
|
||||
total = max(0.0, float(seconds))
|
||||
hours, remainder = divmod(total, 3600.0)
|
||||
minutes, seconds = divmod(remainder, 60.0)
|
||||
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
|
||||
|
||||
|
||||
def assemble_decoded_chunks(
|
||||
*,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
diagnostics: str,
|
||||
):
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
|
||||
if audio_seam not in AUDIO_SEAM_OPTIONS:
|
||||
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
|
||||
|
||||
chunk_plans = list(
|
||||
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
|
||||
)
|
||||
if len(images) != len(chunk_plans) or len(audio) != len(chunk_plans):
|
||||
raise ValueError(
|
||||
"decoded chunk count does not match the V3 assembly plan: "
|
||||
f"images={len(images)}, audio={len(audio)}, plan={len(chunk_plans)}"
|
||||
)
|
||||
|
||||
total_retained_frames = sum(int(item["net_frames"]) for item in chunk_plans)
|
||||
image_buffer = None
|
||||
audio_buffer = None
|
||||
audio_rate = None
|
||||
frame_cursor = 0
|
||||
reports = [
|
||||
"H3 Continuum Assemble V3",
|
||||
f"Audio Seam: {audio_seam}. Video seam correction is disabled.",
|
||||
]
|
||||
|
||||
for index, (raw_images, raw_audio, chunk) in enumerate(
|
||||
zip(images, audio, chunk_plans), start=1
|
||||
):
|
||||
if not torch.is_tensor(raw_images) or raw_images.ndim != 4:
|
||||
raise ValueError(f"decoded image chunk {index} must be [frames,H,W,C]")
|
||||
raw_images = raw_images.detach().to("cpu")
|
||||
total_frames = int(chunk["total_frames"])
|
||||
trim_frames = int(chunk["trim_frames"])
|
||||
net_frames = int(chunk["net_frames"])
|
||||
if int(raw_images.shape[0]) < total_frames:
|
||||
raise ValueError(
|
||||
f"decoded image chunk {index} has {raw_images.shape[0]} frames; "
|
||||
f"expected at least {total_frames}"
|
||||
)
|
||||
segment_images = raw_images[:total_frames][trim_frames:].contiguous()
|
||||
if int(segment_images.shape[0]) != net_frames:
|
||||
raise ValueError(
|
||||
f"decoded image chunk {index} retained {segment_images.shape[0]} frames; "
|
||||
f"plan expects {net_frames}"
|
||||
)
|
||||
|
||||
waveform, sample_rate = validate_audio(raw_audio)
|
||||
waveform = waveform.detach().to("cpu")
|
||||
sample_rate = int(sample_rate)
|
||||
raw_audio_cpu = {"waveform": waveform, "sample_rate": sample_rate}
|
||||
if audio_rate is None:
|
||||
audio_rate = sample_rate
|
||||
elif sample_rate != audio_rate:
|
||||
raise ValueError(
|
||||
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
||||
)
|
||||
|
||||
frame_stop = frame_cursor + net_frames
|
||||
segment_waveform = _slice_audio_for_timeline(
|
||||
raw_audio_cpu,
|
||||
trim_frames=trim_frames,
|
||||
frame_start=frame_cursor,
|
||||
frame_stop=frame_stop,
|
||||
)
|
||||
|
||||
if image_buffer is None:
|
||||
image_buffer = torch.empty(
|
||||
(total_retained_frames, *segment_images.shape[1:]),
|
||||
dtype=segment_images.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(segment_images.shape[1:]) != tuple(image_buffer.shape[1:]):
|
||||
raise ValueError(f"decoded image geometry changed at chunk {index}")
|
||||
image_buffer[frame_cursor:frame_stop].copy_(segment_images)
|
||||
|
||||
if audio_buffer is None:
|
||||
audio_buffer = torch.empty(
|
||||
(
|
||||
*waveform.shape[:-1],
|
||||
int(round(total_retained_frames / FPS * sample_rate)),
|
||||
),
|
||||
dtype=waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
|
||||
raise ValueError(
|
||||
"decoded audio batch/channel structure changed between chunks"
|
||||
)
|
||||
|
||||
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
||||
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
||||
seam_report = None
|
||||
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
|
||||
try:
|
||||
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
||||
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
|
||||
audio_buffer[..., :sample_start],
|
||||
waveform,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
if patch is not None and int(patch.shape[-1]) > 0:
|
||||
patch_start = sample_start - int(patch.shape[-1])
|
||||
if patch_start < 0:
|
||||
raise ValueError("audio seam patch starts before the output")
|
||||
audio_buffer[..., patch_start:sample_start].copy_(patch)
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: "
|
||||
f"corr={metrics.correlation_before:.4f}->{metrics.correlation_after:.4f}, "
|
||||
f"jump={metrics.boundary_jump_before:.6f}->{metrics.boundary_jump_after:.6f}, "
|
||||
f"fade={fade_samples} samples"
|
||||
)
|
||||
if diagnostics_mode == DIAGNOSTICS_FULL:
|
||||
seam_report += (
|
||||
f", offset={metrics.offset_samples:+d}, "
|
||||
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
|
||||
)
|
||||
except Exception as exc:
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: fallback to native boundary "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
|
||||
if diagnostics_mode != DIAGNOSTICS_OFF:
|
||||
reports.append(
|
||||
f"assembled decoded chunk {index}: {net_frames} retained frames, "
|
||||
f"cumulative {frame_stop}/{total_retained_frames}"
|
||||
)
|
||||
if seam_report:
|
||||
reports.append(seam_report)
|
||||
frame_cursor = frame_stop
|
||||
|
||||
if image_buffer is None or audio_buffer is None or audio_rate is None:
|
||||
raise RuntimeError("V3 assembler failed to allocate output buffers")
|
||||
if frame_cursor != total_retained_frames:
|
||||
raise RuntimeError(
|
||||
f"V3 assembly cursor mismatch: {frame_cursor} != {total_retained_frames}"
|
||||
)
|
||||
|
||||
result_audio = {
|
||||
"waveform": audio_buffer.contiguous(),
|
||||
"sample_rate": audio_rate,
|
||||
}
|
||||
duration_report = ""
|
||||
if exact_total_duration:
|
||||
image_buffer, result_audio, duration_report = enforce_total_frames(
|
||||
image_buffer.contiguous(),
|
||||
result_audio,
|
||||
target_frames=int(plan["target_frames"]),
|
||||
preserve_final_frame=bool(plan.get("preserve_final_frame")),
|
||||
)
|
||||
|
||||
reports.append(
|
||||
f"Assembled {len(chunk_plans)} externally decoded chunks into "
|
||||
f"{image_buffer.shape[0]} frames with cumulative sample-boundary alignment."
|
||||
)
|
||||
if duration_report:
|
||||
reports.append(duration_report)
|
||||
runtime_started_at = plan.get("_runtime_started_at")
|
||||
if isinstance(runtime_started_at, (int, float)):
|
||||
elapsed = time.perf_counter() - float(runtime_started_at)
|
||||
if math.isfinite(elapsed) and elapsed >= 0.0:
|
||||
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
|
||||
return image_buffer.contiguous(), result_audio, "\n".join(reports)
|
||||
|
||||
|
||||
class H3ContinuumAssembleV3:
|
||||
DESCRIPTION = (
|
||||
"Assemble full AV chunks decoded by ComfyUI Core. Trims decoded context, "
|
||||
"aligns audio on cumulative frame boundaries, and optionally applies Audio Seam Auto."
|
||||
)
|
||||
SEARCH_ALIASES = ["H3 latent assemble", "H3 external VAE decode"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
"audio": ("AUDIO",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"exact_total_duration": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "Adjust the final output to the exact requested total duration.",
|
||||
},
|
||||
),
|
||||
"audio_seam": (
|
||||
AUDIO_SEAM_OPTIONS,
|
||||
{
|
||||
"default": AUDIO_SEAM_AUTO,
|
||||
"display_name": "Audio Seam",
|
||||
"tooltip": (
|
||||
"Auto corrects only decoded audio boundaries; "
|
||||
"video frames are never altered."
|
||||
),
|
||||
},
|
||||
),
|
||||
"diagnostics": (
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
{
|
||||
"default": DIAGNOSTICS_OPTIONS[0],
|
||||
"display_name": "Report Detail",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
|
||||
RETURN_NAMES = ("images", "audio", "report")
|
||||
FUNCTION = "assemble"
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
diagnostics,
|
||||
):
|
||||
return assemble_decoded_chunks(
|
||||
images=_chunk_list(images),
|
||||
audio=_chunk_list(audio),
|
||||
assembly_plan=_singleton(assembly_plan, "assembly_plan"),
|
||||
exact_total_duration=bool(
|
||||
_singleton(exact_total_duration, "exact_total_duration")
|
||||
),
|
||||
audio_seam=str(_singleton(audio_seam, "audio_seam")),
|
||||
diagnostics=str(_singleton(diagnostics, "diagnostics")),
|
||||
)
|
||||
# V3.0.1 hardening integration: preflight before allocation; Detailed Report only.
|
||||
from ..hardening import assemble_with_hardening as _assemble_with_hardening
|
||||
|
||||
_assemble_decoded_chunks_v300 = assemble_decoded_chunks
|
||||
|
||||
|
||||
def assemble_decoded_chunks(*args, **kwargs):
|
||||
return _assemble_with_hardening(_assemble_decoded_chunks_v300, args, kwargs)
|
||||
|
||||
|
||||
class H3ContinuumAssembleSeamExperimental(H3ContinuumAssembleV3):
|
||||
DEPRECATED = False
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
DESCRIPTION = (
|
||||
"Analyze decoded chunk boundaries and apply guarded video seam correction."
|
||||
)
|
||||
SEARCH_ALIASES = ["H3 seam analysis", "H3 boundary flicker analysis"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
schema = super().INPUT_TYPES()
|
||||
required = dict(schema["required"])
|
||||
diagnostics = required.pop("diagnostics")
|
||||
required["video_seam"] = (
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS,
|
||||
{
|
||||
"default": VIDEO_SEAM_AUTO,
|
||||
"display_name": "Video Seam",
|
||||
"tooltip": (
|
||||
"Analyze Only reports decoded video boundaries without changing them. "
|
||||
"Auto applies the validated transient and micro-flash correction. "
|
||||
"Auto 2 experimentally adds qualified exposure-ramp smoothing."
|
||||
),
|
||||
},
|
||||
)
|
||||
required["diagnostics"] = diagnostics
|
||||
schema["required"] = required
|
||||
return schema
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
video_seam,
|
||||
diagnostics,
|
||||
):
|
||||
mode = str(_singleton(video_seam, "video_seam"))
|
||||
if mode not in VIDEO_SEAM_ANALYSIS_OPTIONS:
|
||||
raise ValueError(f"unknown Video Seam mode: {mode!r}")
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return super().assemble(
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
diagnostics,
|
||||
)
|
||||
|
||||
image_chunks = _chunk_list(images)
|
||||
plan = _singleton(assembly_plan, "assembly_plan")
|
||||
analyses = []
|
||||
analysis_error = None
|
||||
actions = {}
|
||||
try:
|
||||
from .video_seam import (
|
||||
analyze_decoded_boundaries,
|
||||
correct_decoded_boundaries,
|
||||
format_video_boundary_analysis,
|
||||
)
|
||||
|
||||
analyses = analyze_decoded_boundaries(
|
||||
images=image_chunks,
|
||||
assembly_plan=plan,
|
||||
)
|
||||
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
|
||||
image_chunks, actions = correct_decoded_boundaries(
|
||||
images=image_chunks,
|
||||
assembly_plan=plan,
|
||||
analyses=analyses,
|
||||
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
|
||||
)
|
||||
except Exception as exc:
|
||||
analysis_error = exc
|
||||
|
||||
result_images, result_audio, report = assemble_decoded_chunks(
|
||||
images=image_chunks,
|
||||
audio=_chunk_list(audio),
|
||||
assembly_plan=plan,
|
||||
exact_total_duration=bool(
|
||||
_singleton(exact_total_duration, "exact_total_duration")
|
||||
),
|
||||
audio_seam=str(_singleton(audio_seam, "audio_seam")),
|
||||
diagnostics=str(_singleton(diagnostics, "diagnostics")),
|
||||
)
|
||||
if mode == VIDEO_SEAM_ANALYZE:
|
||||
status = "Video Seam: Analyze Only; decoded frames and audio are unchanged."
|
||||
elif mode == VIDEO_SEAM_AUTO:
|
||||
status = (
|
||||
"Video Seam: Auto; guarded transient and micro-flash correction enabled."
|
||||
)
|
||||
else:
|
||||
status = (
|
||||
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
|
||||
"and exposure-ramp correction enabled."
|
||||
)
|
||||
report = report.replace("Video seam correction is disabled.", status, 1)
|
||||
if analysis_error is None:
|
||||
lines = [
|
||||
format_video_boundary_analysis(
|
||||
item,
|
||||
action=(
|
||||
"analysis only"
|
||||
if mode == VIDEO_SEAM_ANALYZE
|
||||
else actions.get(item.boundary_index, "kept native boundary")
|
||||
),
|
||||
)
|
||||
for item in analyses
|
||||
]
|
||||
if not lines:
|
||||
lines = [f"Video Seam {mode}: no decoded chunk boundary to analyze."]
|
||||
else:
|
||||
lines = [
|
||||
f"Video Seam {mode}: native output preserved; analysis unavailable "
|
||||
f"({type(analysis_error).__name__}: {analysis_error})"
|
||||
]
|
||||
return result_images, result_audio, report.rstrip() + "\n" + "\n".join(lines)
|
||||
+735
@@ -0,0 +1,735 @@
|
||||
"""V3.5 direct-write assembly with a manual RAM or file-backed IMAGE buffer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_FULL,
|
||||
DIAGNOSTICS_OFF,
|
||||
normalize_diagnostics_mode,
|
||||
)
|
||||
from ..hardening import assemble_with_hardening
|
||||
from ..media import validate_audio
|
||||
from ..v2.seam_guard import correct_audio_seam
|
||||
from .assembly import (
|
||||
AUDIO_SEAM_AUTO,
|
||||
AUDIO_SEAM_OPTIONS,
|
||||
VIDEO_SEAM_ANALYZE,
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS,
|
||||
VIDEO_SEAM_AUTO,
|
||||
VIDEO_SEAM_AUTO_2,
|
||||
VIDEO_SEAM_OFF,
|
||||
)
|
||||
from .file_backed_buffer import (
|
||||
BUFFER_BACKEND_AUTO,
|
||||
BUFFER_BACKEND_DISK,
|
||||
BUFFER_BACKEND_OPTIONS,
|
||||
BUFFER_BACKEND_RAM,
|
||||
allocate_file_backed_image,
|
||||
format_auto_buffer_decision,
|
||||
select_auto_buffer_backend,
|
||||
preflight_manual_buffer,
|
||||
)
|
||||
from .. import storage_settings
|
||||
from .plan import FPS, validate_assembly_plan, REVIEW_AUDIO_PROJECTION_KEY
|
||||
|
||||
|
||||
_COPY_BATCH_FRAMES = 8
|
||||
|
||||
|
||||
def _check_interrupted(
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> None:
|
||||
"""Honor ComfyUI's normal ESC interrupt without requiring Core in tests."""
|
||||
|
||||
if interrupt_check is not None:
|
||||
interrupt_check()
|
||||
return
|
||||
try:
|
||||
from comfy.model_management import throw_exception_if_processing_interrupted
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
throw_exception_if_processing_interrupted()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CopySpan:
|
||||
"""Map a half-open natural-timeline interval into the output buffer."""
|
||||
|
||||
source_start: int
|
||||
source_stop: int
|
||||
destination_start: int
|
||||
|
||||
|
||||
class _RamAllocation(AbstractContextManager):
|
||||
"""Match the provisional file-backed allocation interface for RAM."""
|
||||
|
||||
def __init__(self, shape: tuple[int, ...], dtype: torch.dtype):
|
||||
self.tensor = torch.empty(shape, dtype=dtype, device="cpu")
|
||||
self.record = None
|
||||
self._published = False
|
||||
|
||||
def publish(self) -> torch.Tensor:
|
||||
if self._published:
|
||||
raise RuntimeError("RAM assembly buffer was already published")
|
||||
self._published = True
|
||||
return self.tensor
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _video_copy_spans(
|
||||
*,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
preserve_final_frame: bool,
|
||||
) -> tuple[CopySpan, ...]:
|
||||
if output_frames >= natural_frames:
|
||||
return (CopySpan(0, natural_frames, 0),)
|
||||
if preserve_final_frame and output_frames >= 2:
|
||||
return (
|
||||
CopySpan(0, output_frames - 1, 0),
|
||||
CopySpan(natural_frames - 1, natural_frames, output_frames - 1),
|
||||
)
|
||||
return (CopySpan(0, output_frames, 0),)
|
||||
|
||||
|
||||
def _copy_natural_interval(
|
||||
destination: torch.Tensor,
|
||||
source: torch.Tensor,
|
||||
*,
|
||||
natural_start: int,
|
||||
spans: tuple[CopySpan, ...],
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
natural_stop = natural_start + int(source.shape[0])
|
||||
for span in spans:
|
||||
overlap_start = max(natural_start, int(span.source_start))
|
||||
overlap_stop = min(natural_stop, int(span.source_stop))
|
||||
if overlap_stop <= overlap_start:
|
||||
continue
|
||||
source_start = overlap_start - natural_start
|
||||
destination_start = int(span.destination_start) + (
|
||||
overlap_start - int(span.source_start)
|
||||
)
|
||||
frame_count = overlap_stop - overlap_start
|
||||
for offset in range(0, frame_count, _COPY_BATCH_FRAMES):
|
||||
_check_interrupted(interrupt_check)
|
||||
batch_frames = min(_COPY_BATCH_FRAMES, frame_count - offset)
|
||||
source_batch_start = source_start + offset
|
||||
destination_batch_start = destination_start + offset
|
||||
destination[
|
||||
destination_batch_start : destination_batch_start + batch_frames
|
||||
].copy_(
|
||||
source[source_batch_start : source_batch_start + batch_frames]
|
||||
)
|
||||
|
||||
|
||||
def _repeat_final_video_frame(
|
||||
destination: torch.Tensor,
|
||||
*,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> None:
|
||||
for destination_start in range(
|
||||
natural_frames,
|
||||
output_frames,
|
||||
_COPY_BATCH_FRAMES,
|
||||
):
|
||||
_check_interrupted(interrupt_check)
|
||||
destination_stop = min(
|
||||
output_frames,
|
||||
destination_start + _COPY_BATCH_FRAMES,
|
||||
)
|
||||
destination[destination_start:destination_stop].copy_(
|
||||
destination[natural_frames - 1 : natural_frames].expand(
|
||||
destination_stop - destination_start,
|
||||
*destination.shape[1:],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _copy_audio_segment(
|
||||
destination: torch.Tensor,
|
||||
waveform: torch.Tensor,
|
||||
*,
|
||||
trim_samples: int,
|
||||
) -> None:
|
||||
"""Reproduce timeline slicing/padding without concatenating a waveform."""
|
||||
|
||||
wanted = int(destination.shape[-1])
|
||||
trim_samples = max(0, int(trim_samples))
|
||||
available = max(0, min(wanted, int(waveform.shape[-1]) - trim_samples))
|
||||
if available > 0:
|
||||
destination[..., :available].copy_(
|
||||
waveform[..., trim_samples : trim_samples + available]
|
||||
)
|
||||
if available >= wanted:
|
||||
return
|
||||
if available > 0:
|
||||
destination[..., available:].copy_(
|
||||
destination[..., available - 1 : available].expand(
|
||||
*destination.shape[:-1],
|
||||
wanted - available,
|
||||
)
|
||||
)
|
||||
else:
|
||||
destination.zero_()
|
||||
|
||||
|
||||
def adjust_audio_duration_direct(
|
||||
audio: Mapping[str, Any],
|
||||
*,
|
||||
target_frames: int,
|
||||
preserve_final_audio_anchor: bool,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
"""Apply the legacy audio trim/pad values using direct allocation and copy."""
|
||||
|
||||
waveform, sample_rate = validate_audio(audio)
|
||||
waveform = waveform.detach().to(device="cpu")
|
||||
sample_rate = int(sample_rate)
|
||||
target_samples = int(round(int(target_frames) / FPS * sample_rate))
|
||||
original_samples = int(waveform.shape[-1])
|
||||
delta_samples = target_samples - original_samples
|
||||
if delta_samples == 0:
|
||||
return {"waveform": waveform, "sample_rate": sample_rate}, 0
|
||||
|
||||
output = torch.empty(
|
||||
(*waveform.shape[:-1], target_samples),
|
||||
dtype=waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
if delta_samples < 0:
|
||||
if preserve_final_audio_anchor and target_samples > 1:
|
||||
tail_samples = min(
|
||||
max(1, int(round(sample_rate / FPS))),
|
||||
target_samples,
|
||||
original_samples,
|
||||
)
|
||||
head_samples = target_samples - tail_samples
|
||||
if head_samples > 0:
|
||||
output[..., :head_samples].copy_(waveform[..., :head_samples])
|
||||
output[..., head_samples:].copy_(waveform[..., -tail_samples:])
|
||||
fade = min(
|
||||
int(round(sample_rate * 0.005)),
|
||||
head_samples,
|
||||
tail_samples,
|
||||
)
|
||||
if fade > 0:
|
||||
start = output[..., head_samples - 1 : head_samples].to(
|
||||
dtype=output.dtype
|
||||
)
|
||||
alpha = torch.linspace(
|
||||
0.0,
|
||||
1.0,
|
||||
fade,
|
||||
dtype=output.dtype,
|
||||
device=output.device,
|
||||
).reshape(*([1] * (output.ndim - 1)), fade)
|
||||
tail_head = output[..., head_samples : head_samples + fade]
|
||||
tail_head.copy_(start * (1.0 - alpha) + tail_head * alpha)
|
||||
else:
|
||||
output.copy_(waveform[..., :target_samples])
|
||||
else:
|
||||
if original_samples > 0:
|
||||
output[..., :original_samples].copy_(waveform)
|
||||
output[..., original_samples:].copy_(
|
||||
waveform[..., -1:].expand(
|
||||
*waveform.shape[:-1],
|
||||
delta_samples,
|
||||
)
|
||||
)
|
||||
else:
|
||||
output.zero_()
|
||||
return {"waveform": output, "sample_rate": sample_rate}, delta_samples
|
||||
|
||||
|
||||
def _prepare_video_seam(
|
||||
*,
|
||||
images: list[Any],
|
||||
plan: dict[str, Any],
|
||||
mode: str,
|
||||
) -> tuple[list[Any], dict[int, torch.Tensor], dict[int, str], Exception | None]:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return [], {}, {}, None
|
||||
analyses: list[Any] = []
|
||||
try:
|
||||
from .video_seam import analyze_decoded_boundaries
|
||||
|
||||
analyses = analyze_decoded_boundaries(
|
||||
images=images,
|
||||
assembly_plan=plan,
|
||||
)
|
||||
patches: dict[int, torch.Tensor] = {}
|
||||
actions: dict[int, str] = {}
|
||||
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
|
||||
from .video_seam_v35 import build_boundary_patches
|
||||
|
||||
patches, actions = build_boundary_patches(
|
||||
images=images,
|
||||
assembly_plan=plan,
|
||||
analyses=analyses,
|
||||
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
|
||||
)
|
||||
return analyses, patches, actions, None
|
||||
except Exception as exc:
|
||||
# Match V3.4's all-native fallback: no partial patch is retained.
|
||||
return analyses, {}, {}, exc
|
||||
|
||||
|
||||
def _video_seam_status(mode: str) -> str:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return "Video seam correction is disabled."
|
||||
if mode == VIDEO_SEAM_ANALYZE:
|
||||
return "Video Seam: Analyze Only; decoded frames and audio are unchanged."
|
||||
if mode == VIDEO_SEAM_AUTO:
|
||||
return "Video Seam: Auto; guarded transient and micro-flash correction enabled."
|
||||
return (
|
||||
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
|
||||
"and exposure-ramp correction enabled."
|
||||
)
|
||||
|
||||
|
||||
def _format_video_seam_lines(
|
||||
*,
|
||||
mode: str,
|
||||
analyses: list[Any],
|
||||
actions: dict[int, str],
|
||||
error: Exception | None,
|
||||
) -> list[str]:
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return []
|
||||
if error is not None:
|
||||
return [
|
||||
f"Video Seam {mode}: native output preserved; analysis unavailable "
|
||||
f"({type(error).__name__}: {error})"
|
||||
]
|
||||
from .video_seam import format_video_boundary_analysis
|
||||
|
||||
lines = [
|
||||
format_video_boundary_analysis(
|
||||
item,
|
||||
action=(
|
||||
"analysis only"
|
||||
if mode == VIDEO_SEAM_ANALYZE
|
||||
else actions.get(item.boundary_index, "kept native boundary")
|
||||
),
|
||||
)
|
||||
for item in analyses
|
||||
]
|
||||
if not lines:
|
||||
lines.append(f"Video Seam {mode}: no decoded chunk boundary to analyze.")
|
||||
return lines
|
||||
|
||||
|
||||
def _format_elapsed(seconds: float) -> str:
|
||||
total = max(0.0, float(seconds))
|
||||
hours, remainder = divmod(total, 3600.0)
|
||||
minutes, seconds = divmod(remainder, 60.0)
|
||||
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
|
||||
|
||||
|
||||
def _write_assembly(
|
||||
*,
|
||||
image_buffer: torch.Tensor,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
natural_frames: int,
|
||||
output_frames: int,
|
||||
video_spans: tuple[CopySpan, ...],
|
||||
seam_patches: dict[int, torch.Tensor],
|
||||
seam_analyses: list[Any],
|
||||
seam_actions: dict[int, str],
|
||||
seam_error: Exception | None,
|
||||
buffer_backend: str,
|
||||
auto_decision_report: str | None,
|
||||
final_audio_override: Mapping[str, Any] | None,
|
||||
final_audio_source: str | None,
|
||||
interrupt_check: Callable[[], None] | None,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
|
||||
units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
first_waveform, audio_rate = validate_audio(audio[0])
|
||||
first_waveform = first_waveform.detach().to(device="cpu")
|
||||
audio_rate = int(audio_rate)
|
||||
natural_samples = int(round(natural_frames / FPS * audio_rate))
|
||||
audio_buffer = torch.empty(
|
||||
(*first_waveform.shape[:-1], natural_samples),
|
||||
dtype=first_waveform.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
reports = [
|
||||
"H3 Continuum Assemble V3.5",
|
||||
f"Audio Seam: {audio_seam}. {_video_seam_status(video_seam)}",
|
||||
f"Buffer Backend: {buffer_backend}; IMAGE direct-write "
|
||||
f"frames={output_frames}, bytes={image_buffer.numel() * image_buffer.element_size()}; "
|
||||
"no final full IMAGE copy.",
|
||||
]
|
||||
if auto_decision_report is not None:
|
||||
reports.append(auto_decision_report)
|
||||
|
||||
frame_cursor = 0
|
||||
for index, (raw_images, raw_audio, unit) in enumerate(
|
||||
zip(images, audio, units),
|
||||
start=1,
|
||||
):
|
||||
_check_interrupted(interrupt_check)
|
||||
total_frames = int(unit["total_frames"])
|
||||
trim_frames = int(unit["trim_frames"])
|
||||
net_frames = int(unit["net_frames"])
|
||||
raw_images_cpu = raw_images.detach().to(device="cpu")
|
||||
retained_images = raw_images_cpu[trim_frames:total_frames]
|
||||
if int(retained_images.shape[0]) != net_frames:
|
||||
raise ValueError(
|
||||
f"decoded image chunk {index} retained {retained_images.shape[0]} "
|
||||
f"frames; plan expects {net_frames}"
|
||||
)
|
||||
_copy_natural_interval(
|
||||
image_buffer,
|
||||
retained_images,
|
||||
natural_start=frame_cursor,
|
||||
spans=video_spans,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
_check_interrupted(interrupt_check)
|
||||
waveform, sample_rate = validate_audio(raw_audio)
|
||||
waveform = waveform.detach().to(device="cpu")
|
||||
sample_rate = int(sample_rate)
|
||||
if sample_rate != audio_rate:
|
||||
raise ValueError(
|
||||
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
|
||||
)
|
||||
frame_stop = frame_cursor + net_frames
|
||||
sample_start = int(round(frame_cursor / FPS * sample_rate))
|
||||
sample_stop = int(round(frame_stop / FPS * sample_rate))
|
||||
seam_report = None
|
||||
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
|
||||
try:
|
||||
cut_sample = int(round(trim_frames / FPS * sample_rate))
|
||||
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
|
||||
audio_buffer[..., :sample_start],
|
||||
waveform,
|
||||
sample_rate=sample_rate,
|
||||
cut_sample=cut_sample,
|
||||
)
|
||||
if patch is not None and int(patch.shape[-1]) > 0:
|
||||
patch_start = sample_start - int(patch.shape[-1])
|
||||
if patch_start < 0:
|
||||
raise ValueError("audio seam patch starts before the output")
|
||||
audio_buffer[..., patch_start:sample_start].copy_(patch)
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: "
|
||||
f"corr={metrics.correlation_before:.4f}->"
|
||||
f"{metrics.correlation_after:.4f}, "
|
||||
f"jump={metrics.boundary_jump_before:.6f}->"
|
||||
f"{metrics.boundary_jump_after:.6f}, "
|
||||
f"fade={fade_samples} samples"
|
||||
)
|
||||
if diagnostics_mode == DIAGNOSTICS_FULL:
|
||||
seam_report += (
|
||||
f", offset={metrics.offset_samples:+d}, "
|
||||
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
|
||||
)
|
||||
except Exception as exc:
|
||||
seam_report = (
|
||||
f"audio seam {index-1}->{index}: fallback to native boundary "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
trim_samples = int(round(trim_frames / FPS * sample_rate))
|
||||
_copy_audio_segment(
|
||||
audio_buffer[..., sample_start:sample_stop],
|
||||
waveform,
|
||||
trim_samples=trim_samples,
|
||||
)
|
||||
if diagnostics_mode != DIAGNOSTICS_OFF:
|
||||
reports.append(
|
||||
f"assembled decoded chunk {index}: {net_frames} retained frames, "
|
||||
f"cumulative {frame_stop}/{natural_frames}"
|
||||
)
|
||||
if seam_report:
|
||||
reports.append(seam_report)
|
||||
frame_cursor = frame_stop
|
||||
|
||||
if frame_cursor != natural_frames:
|
||||
raise RuntimeError(
|
||||
f"V3.5 assembly cursor mismatch: {frame_cursor} != {natural_frames}"
|
||||
)
|
||||
|
||||
unit_starts: list[int] = []
|
||||
cursor = 0
|
||||
for unit in units:
|
||||
unit_starts.append(cursor)
|
||||
cursor += int(unit["net_frames"])
|
||||
for boundary, patch in seam_patches.items():
|
||||
_check_interrupted(interrupt_check)
|
||||
if boundary <= 0 or boundary >= len(unit_starts):
|
||||
raise ValueError("V3.5 video seam patch boundary is outside decode groups")
|
||||
_copy_natural_interval(
|
||||
image_buffer,
|
||||
patch,
|
||||
natural_start=unit_starts[boundary],
|
||||
spans=video_spans,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
if output_frames > natural_frames:
|
||||
_repeat_final_video_frame(
|
||||
image_buffer,
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
|
||||
_check_interrupted(interrupt_check)
|
||||
result_audio: dict[str, Any] = {
|
||||
"waveform": audio_buffer,
|
||||
"sample_rate": audio_rate,
|
||||
}
|
||||
duration_report = ""
|
||||
if exact_total_duration:
|
||||
preserve_anchor = bool(
|
||||
plan.get("preserve_final_frame", False)
|
||||
and output_frames < natural_frames
|
||||
and output_frames >= 2
|
||||
)
|
||||
result_audio, delta_samples = adjust_audio_duration_direct(
|
||||
result_audio,
|
||||
target_frames=output_frames,
|
||||
preserve_final_audio_anchor=preserve_anchor,
|
||||
)
|
||||
adjustment = output_frames - natural_frames
|
||||
mode = (
|
||||
"final anchor preserved; pre-tail trim"
|
||||
if preserve_anchor
|
||||
else "tail trim/pad"
|
||||
)
|
||||
duration_report = (
|
||||
f"Exact-duration adjustment: frames {natural_frames}->{output_frames} "
|
||||
f"({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}."
|
||||
)
|
||||
|
||||
driving_audio_report = ""
|
||||
if final_audio_override is not None:
|
||||
result_audio = dict(final_audio_override)
|
||||
override_waveform, _ = validate_audio(result_audio)
|
||||
empty_review = (REVIEW_AUDIO_PROJECTION_KEY in plan and int(override_waveform.shape[-1]) == 0)
|
||||
if exact_total_duration or empty_review:
|
||||
result_audio, _ = adjust_audio_duration_direct(
|
||||
result_audio,
|
||||
target_frames=output_frames,
|
||||
preserve_final_audio_anchor=False,
|
||||
)
|
||||
waveform, selected_rate = validate_audio(result_audio)
|
||||
result_audio = {
|
||||
"waveform": waveform,
|
||||
"sample_rate": int(selected_rate),
|
||||
}
|
||||
driving_audio_report = (
|
||||
"Driving Audio: preserved source selected from "
|
||||
f"{final_audio_source or 'direct input'}; "
|
||||
f"sample_rate={int(selected_rate)}, samples={int(waveform.shape[-1])}; "
|
||||
"generated audio and Audio Seam bypassed."
|
||||
)
|
||||
if empty_review:
|
||||
driving_audio_report += " Empty Review interval: silence added for the selected video duration."
|
||||
|
||||
reports.append(
|
||||
f"Assembled {len(units)} physical decoded group(s) into "
|
||||
f"{output_frames} frames with cumulative sample-boundary alignment."
|
||||
)
|
||||
if duration_report:
|
||||
reports.append(duration_report)
|
||||
if driving_audio_report:
|
||||
reports.append(driving_audio_report)
|
||||
runtime_started_at = plan.get("_runtime_started_at")
|
||||
if isinstance(runtime_started_at, (int, float)):
|
||||
elapsed = time.perf_counter() - float(runtime_started_at)
|
||||
if math.isfinite(elapsed) and elapsed >= 0.0:
|
||||
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
|
||||
reports.extend(
|
||||
_format_video_seam_lines(
|
||||
mode=video_seam,
|
||||
analyses=seam_analyses,
|
||||
actions=seam_actions,
|
||||
error=seam_error,
|
||||
)
|
||||
)
|
||||
return result_audio, "\n".join(reports)
|
||||
|
||||
|
||||
def _assemble_decoded_chunks_v35_core(
|
||||
*,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
buffer_backend: str,
|
||||
backing_root: str | Path | None = None,
|
||||
final_audio_override: Mapping[str, Any] | None = None,
|
||||
final_audio_source: str | None = None,
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
):
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
if audio_seam not in AUDIO_SEAM_OPTIONS:
|
||||
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
|
||||
if video_seam not in VIDEO_SEAM_ANALYSIS_OPTIONS:
|
||||
raise ValueError(f"unknown Video Seam mode: {video_seam!r}")
|
||||
if buffer_backend not in BUFFER_BACKEND_OPTIONS:
|
||||
raise ValueError(f"unknown Buffer Backend: {buffer_backend!r}")
|
||||
units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
natural_frames = sum(int(unit["net_frames"]) for unit in units)
|
||||
output_frames = (
|
||||
int(plan["target_frames"])
|
||||
if bool(exact_total_duration)
|
||||
else natural_frames
|
||||
)
|
||||
if natural_frames <= 0 or output_frames <= 0:
|
||||
raise ValueError("V3.5 assembly frame counts must be positive")
|
||||
first_image = images[0]
|
||||
shape = (output_frames, *tuple(int(value) for value in first_image.shape[1:]))
|
||||
video_spans = _video_copy_spans(
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
preserve_final_frame=bool(plan.get("preserve_final_frame", False)),
|
||||
)
|
||||
analyses, patches, actions, analysis_error = _prepare_video_seam(
|
||||
images=images,
|
||||
plan=plan,
|
||||
mode=video_seam,
|
||||
)
|
||||
|
||||
resolved_backend = buffer_backend
|
||||
resolved_backing_root = backing_root
|
||||
auto_decision_report = None
|
||||
final_image_bytes = math.prod(shape) * first_image.element_size()
|
||||
if buffer_backend == BUFFER_BACKEND_AUTO:
|
||||
final_image_bytes = (
|
||||
math.prod(shape) * first_image.element_size()
|
||||
)
|
||||
decision = select_auto_buffer_backend(
|
||||
final_image_bytes,
|
||||
backing_root=backing_root,
|
||||
)
|
||||
resolved_backend = decision.selected_backend
|
||||
resolved_backing_root = decision.backing_root
|
||||
auto_decision_report = format_auto_buffer_decision(decision)
|
||||
else:
|
||||
preflight_manual_buffer(buffer_backend, final_image_bytes, backing_root)
|
||||
|
||||
allocation: Any
|
||||
if resolved_backend == BUFFER_BACKEND_RAM:
|
||||
allocation = _RamAllocation(shape, first_image.dtype)
|
||||
else:
|
||||
allocation = allocate_file_backed_image(
|
||||
shape,
|
||||
dtype=first_image.dtype,
|
||||
backing_root=resolved_backing_root,
|
||||
)
|
||||
with allocation:
|
||||
settings = storage_settings.get_manager()
|
||||
if settings is not None:
|
||||
from .file_backed_buffer import resolve_backing_root
|
||||
settings.record_backend(resolved_backend, final_image_bytes,
|
||||
resolve_backing_root(resolved_backing_root) if resolved_backend == BUFFER_BACKEND_DISK else None)
|
||||
# The file exists at this point, so an ESC received immediately before
|
||||
# assembly exercises the same unpublished cleanup path as a mid-copy
|
||||
# cancellation.
|
||||
_check_interrupted(interrupt_check)
|
||||
result_audio, report = _write_assembly(
|
||||
image_buffer=allocation.tensor,
|
||||
images=images,
|
||||
audio=audio,
|
||||
plan=plan,
|
||||
exact_total_duration=bool(exact_total_duration),
|
||||
audio_seam=audio_seam,
|
||||
video_seam=video_seam,
|
||||
diagnostics=diagnostics,
|
||||
natural_frames=natural_frames,
|
||||
output_frames=output_frames,
|
||||
video_spans=video_spans,
|
||||
seam_patches=patches,
|
||||
seam_analyses=analyses,
|
||||
seam_actions=actions,
|
||||
seam_error=analysis_error,
|
||||
buffer_backend=(
|
||||
resolved_backend
|
||||
if buffer_backend != BUFFER_BACKEND_AUTO
|
||||
else f"Auto -> {resolved_backend}"
|
||||
),
|
||||
auto_decision_report=auto_decision_report,
|
||||
final_audio_override=final_audio_override,
|
||||
final_audio_source=final_audio_source,
|
||||
interrupt_check=interrupt_check,
|
||||
)
|
||||
result_images = allocation.publish()
|
||||
return result_images, result_audio, report
|
||||
|
||||
|
||||
@storage_settings.storage_operation
|
||||
def assemble_decoded_chunks_v35(
|
||||
*,
|
||||
images: list[Any],
|
||||
audio: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
exact_total_duration: bool,
|
||||
audio_seam: str,
|
||||
video_seam: str,
|
||||
diagnostics: str,
|
||||
buffer_backend: str,
|
||||
backing_root: str | Path | None = None,
|
||||
final_audio_override: Mapping[str, Any] | None = None,
|
||||
final_audio_source: str | None = None,
|
||||
interrupt_check: Callable[[], None] | None = None,
|
||||
):
|
||||
"""Preflight all inputs, then execute the independent V3.5 writer."""
|
||||
|
||||
kwargs = {
|
||||
"images": images,
|
||||
"audio": audio,
|
||||
"assembly_plan": assembly_plan,
|
||||
"exact_total_duration": bool(exact_total_duration),
|
||||
"audio_seam": str(audio_seam),
|
||||
"video_seam": str(video_seam),
|
||||
"diagnostics": str(diagnostics),
|
||||
"buffer_backend": str(buffer_backend),
|
||||
"backing_root": backing_root,
|
||||
"final_audio_override": final_audio_override,
|
||||
"final_audio_source": final_audio_source,
|
||||
"interrupt_check": interrupt_check,
|
||||
}
|
||||
return assemble_with_hardening(
|
||||
_assemble_decoded_chunks_v35_core,
|
||||
(),
|
||||
kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BUFFER_BACKEND_DISK",
|
||||
"BUFFER_BACKEND_AUTO",
|
||||
"BUFFER_BACKEND_OPTIONS",
|
||||
"BUFFER_BACKEND_RAM",
|
||||
"CopySpan",
|
||||
"adjust_audio_duration_direct",
|
||||
"assemble_decoded_chunks_v35",
|
||||
]
|
||||
@@ -0,0 +1,135 @@
|
||||
"""V3.5 Advanced bridge from Continuum refine context to Core conditioning."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .second_pass import prepare_physical_refine_groups
|
||||
from .second_pass_nodes import _single_list_input
|
||||
|
||||
|
||||
class H3ContinuumConditioningBridgeV35:
|
||||
"""Publish paired physical-group MODEL and CONDITIONING values."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"Advanced interoperability bridge for Basic Guider and external samplers. "
|
||||
"It adapts the captured Continuum refine context to externally processed "
|
||||
"physical-group video latents, then returns one paired MODEL and complete "
|
||||
"CONDITIONING per physical group plus the target-geometry Assembly Plan. "
|
||||
"Nested AV LATENT construction, Audio LATENT, Noise, Audio Lock, and external "
|
||||
"sampling behavior remain the workflow's responsibility."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"video_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
|
||||
},
|
||||
"optional": {
|
||||
"video_vae": (
|
||||
"VAE",
|
||||
{
|
||||
"tooltip": (
|
||||
"When available, First/Last keyframes are re-encoded at "
|
||||
"the target geometry. Otherwise the existing conditioning "
|
||||
"adapter uses its documented latent-resize fallback."
|
||||
)
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = (
|
||||
"MODEL",
|
||||
"CONDITIONING",
|
||||
"H3_CONTINUUM_ASSEMBLY_PLAN",
|
||||
"STRING",
|
||||
)
|
||||
RETURN_NAMES = (
|
||||
"group_models",
|
||||
"conditioning",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "prepare"
|
||||
CATEGORY = "MiniMax H3/Continuum/Advanced"
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
video_latents,
|
||||
assembly_plan,
|
||||
refine_context,
|
||||
video_vae=None,
|
||||
):
|
||||
prepared = prepare_physical_refine_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
video_latents=video_latents,
|
||||
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
|
||||
refine_context=_single_list_input("refine_context", refine_context),
|
||||
video_vae=(
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
),
|
||||
)
|
||||
group_models = prepared["group_models"]
|
||||
conditioning = prepared["conditioning"]
|
||||
physical_group_count = int(prepared["physical_group_count"])
|
||||
if not len(group_models) == len(conditioning) == physical_group_count:
|
||||
raise ValueError(
|
||||
"Conditioning Bridge MODEL/CONDITIONING physical group counts differ"
|
||||
)
|
||||
|
||||
status_lines = [
|
||||
"H3 Continuum Conditioning Bridge V3.5",
|
||||
f"Physical groups: {physical_group_count}.",
|
||||
"Output contract: one paired MODEL and one complete CONDITIONING per "
|
||||
"physical group; CONDITIONING entries remain grouped.",
|
||||
"Scope: Advanced MODEL + CONDITIONING interoperability only; AV LATENT, "
|
||||
"Noise, and Audio Lock are controlled by the external workflow.",
|
||||
*prepared["warnings"],
|
||||
]
|
||||
for detail in prepared["details"]:
|
||||
group = detail["group"]
|
||||
status_lines.append(
|
||||
"group "
|
||||
f"{int(detail['group_index']) + 1}: "
|
||||
f"logical_chunks={group.get('logical_chunks')}, "
|
||||
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
|
||||
f"conditioning_source={detail['conditioning_source']}, "
|
||||
f"physical_clip_index={detail['physical_clip_index']}, "
|
||||
f"context_frames={detail['context_frames']}"
|
||||
)
|
||||
status_lines.append(
|
||||
"Target geometry: "
|
||||
f"{prepared['updated_assembly_plan']['width']}x"
|
||||
f"{prepared['updated_assembly_plan']['height']}."
|
||||
)
|
||||
return (
|
||||
group_models,
|
||||
conditioning,
|
||||
prepared["updated_assembly_plan"],
|
||||
"\n".join(status_lines),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumConditioningBridgeV35": H3ContinuumConditioningBridgeV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumConditioningBridgeV35": (
|
||||
"H3 Continuum Conditioning Bridge V3.5"
|
||||
),
|
||||
}
|
||||
+1222
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,67 @@
|
||||
"""Generic Easy workflow audio loader backed by ComfyUI Core LoadAudio."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
try:
|
||||
from comfy_extras.nodes_audio import LoadAudio as CoreLoadAudio
|
||||
except (ImportError, AttributeError): # pragma: no cover - standalone unit tests
|
||||
class CoreLoadAudio:
|
||||
"""Non-runtime shim; real ComfyUI always supplies Core LoadAudio."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return SimpleNamespace(
|
||||
node_id="LoadAudio",
|
||||
display_name="Load Audio",
|
||||
category="audio",
|
||||
essentials_category="Audio",
|
||||
description="",
|
||||
search_aliases=["import audio", "open audio", "audio file"],
|
||||
is_deprecated=False,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, audio):
|
||||
return audio
|
||||
|
||||
load = execute
|
||||
|
||||
|
||||
class H3EasyLoadAudio(CoreLoadAudio):
|
||||
"""Core LoadAudio with an Easy frontend control for native node bypass."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
schema = super().define_schema()
|
||||
schema.node_id = "H3EasyLoadAudio"
|
||||
schema.is_deprecated = True # Kept for saved workflows; new graphs use Core LoadAudio.
|
||||
schema.display_name = "H3 Continuum Load Audio"
|
||||
schema.category = "MiniMax H3/Continuum"
|
||||
schema.description = (
|
||||
"Loads audio through ComfyUI Core. Enable Audio controls the node's "
|
||||
"native Bypass mode without changing the AUDIO contract."
|
||||
)
|
||||
schema.search_aliases = list(
|
||||
dict.fromkeys(
|
||||
[
|
||||
*getattr(schema, "search_aliases", []),
|
||||
"H3 Continuum Load Audio",
|
||||
"H3 Easy Load Audio",
|
||||
"H3 Easy Audio Loader",
|
||||
"Load Audio with Bypass",
|
||||
]
|
||||
)
|
||||
)
|
||||
return schema
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3EasyLoadAudio": H3EasyLoadAudio,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3EasyLoadAudio": "H3 Continuum Load Audio",
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Generic Easy workflow image loader backed by ComfyUI Core LoadImage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
try:
|
||||
from nodes import LoadImage as CoreLoadImage
|
||||
except (ImportError, AttributeError): # pragma: no cover - standalone unit tests
|
||||
class CoreLoadImage:
|
||||
"""Non-runtime shim; real ComfyUI always supplies Core LoadImage."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": (("",), {"image_upload": True}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "MASK")
|
||||
FUNCTION = "load_image"
|
||||
|
||||
|
||||
class H3EasyLoadImage(CoreLoadImage):
|
||||
"""Core LoadImage with an Easy frontend control for native node bypass."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
DESCRIPTION = (
|
||||
"Loads an image through ComfyUI Core. Enable Image controls the node's "
|
||||
"native Bypass mode, so the same node can serve First, Last, or Reference "
|
||||
"image inputs without changing the IMAGE/MASK contract."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Load Image",
|
||||
"H3 Easy Load Image",
|
||||
"H3 Easy Image Loader",
|
||||
"Load Image with Bypass",
|
||||
]
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3EasyLoadImage": H3EasyLoadImage,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3EasyLoadImage": "H3 Continuum Load Image",
|
||||
}
|
||||
+482
@@ -0,0 +1,482 @@
|
||||
"""V3.8 Easy Mode facade and deterministic UI resolvers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from ..conditioning import conditioning_display_label
|
||||
from ..constants import (
|
||||
CONTINUITY_OPTIONS,
|
||||
DIAGNOSTICS_BASIC,
|
||||
PROMPT_FORMAT_AUTO,
|
||||
)
|
||||
from ..reference import REFERENCE_SIZE_MATCH_OUTPUT
|
||||
from ..reference_video import REFERENCE_VIDEO_SIZE_EFFICIENT
|
||||
from .driving_nodes import (
|
||||
CONTINUATION_BACKEND_STANDARD,
|
||||
H3ContinuumSamplerV37,
|
||||
)
|
||||
from .nodes import CATEGORY as CONTINUUM_CATEGORY, REGENERATE_AUTO
|
||||
from .resolution import (
|
||||
H3_ASPECT_AUTO,
|
||||
H3_ASPECT_LANDSCAPE,
|
||||
H3_ASPECT_OPTIONS,
|
||||
H3_ASPECT_PORTRAIT,
|
||||
H3_ASPECT_SQUARE,
|
||||
H3_BALANCED_MP,
|
||||
H3_CANVAS_MULTIPLE,
|
||||
H3_CUSTOM_MP_DEFAULT,
|
||||
H3_CUSTOM_MP_MAX,
|
||||
H3_CUSTOM_MP_MIN,
|
||||
H3_CUSTOM_MP_STRONG_WARNING,
|
||||
H3_DRAFT_MP,
|
||||
H3_MP_PIXELS,
|
||||
H3_NATIVE_LONG_EDGE_CAP,
|
||||
H3_NATIVE_SHORT_EDGE,
|
||||
H3_PRESET_BALANCED,
|
||||
H3_PRESET_CUSTOM,
|
||||
H3_PRESET_DRAFT,
|
||||
H3_PRESET_NATIVE,
|
||||
H3_PRESET_OPTIONS,
|
||||
H3ResolutionPlan,
|
||||
resolve_h3_aspect,
|
||||
resolve_h3_resolution,
|
||||
)
|
||||
|
||||
|
||||
EASY_DURATION_DEFAULT = 10
|
||||
EASY_DURATION_MIN = 4
|
||||
EASY_DURATION_MAX = 480
|
||||
EASY_TARGET_CHUNK_SECONDS = 10.0
|
||||
EASY_MAX_CHUNKS = 16
|
||||
|
||||
EASY_SEED_MODE_RANDOMIZE = "Randomize"
|
||||
EASY_SEED_MODE_FIXED = "Fixed"
|
||||
EASY_SEED_MODE_OPTIONS = (
|
||||
EASY_SEED_MODE_RANDOMIZE,
|
||||
EASY_SEED_MODE_FIXED,
|
||||
)
|
||||
|
||||
EASY_ASPECT_AUTO = H3_ASPECT_AUTO
|
||||
EASY_ASPECT_LANDSCAPE = H3_ASPECT_LANDSCAPE
|
||||
EASY_ASPECT_PORTRAIT = H3_ASPECT_PORTRAIT
|
||||
EASY_ASPECT_SQUARE = H3_ASPECT_SQUARE
|
||||
EASY_ASPECT_OPTIONS = H3_ASPECT_OPTIONS
|
||||
|
||||
EASY_PRESET_DRAFT = H3_PRESET_DRAFT
|
||||
EASY_PRESET_BALANCED = H3_PRESET_BALANCED
|
||||
EASY_PRESET_NATIVE = H3_PRESET_NATIVE
|
||||
EASY_PRESET_CUSTOM = H3_PRESET_CUSTOM
|
||||
EASY_PRESET_OPTIONS = H3_PRESET_OPTIONS
|
||||
|
||||
EASY_DRAFT_MP = H3_DRAFT_MP
|
||||
EASY_BALANCED_MP = H3_BALANCED_MP
|
||||
EASY_CUSTOM_MP_DEFAULT = H3_CUSTOM_MP_DEFAULT
|
||||
EASY_CUSTOM_MP_MIN = H3_CUSTOM_MP_MIN
|
||||
EASY_CUSTOM_MP_MAX = H3_CUSTOM_MP_MAX
|
||||
EASY_CUSTOM_MP_STRONG_WARNING = H3_CUSTOM_MP_STRONG_WARNING
|
||||
EASY_MP_PIXELS = H3_MP_PIXELS
|
||||
EASY_CANVAS_MULTIPLE = H3_CANVAS_MULTIPLE
|
||||
EASY_NATIVE_SHORT_EDGE = H3_NATIVE_SHORT_EDGE
|
||||
EASY_NATIVE_LONG_EDGE_CAP = H3_NATIVE_LONG_EDGE_CAP
|
||||
EasyResolutionPlan = H3ResolutionPlan
|
||||
resolve_easy_aspect = resolve_h3_aspect
|
||||
resolve_easy_resolution = resolve_h3_resolution
|
||||
EASY_REFERENCES_TYPE = "H3_CONTINUUM_EASY_REFERENCES"
|
||||
EASY_REFERENCES_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EasyDurationPlan:
|
||||
duration_seconds: int
|
||||
chunks: int
|
||||
chunk_seconds: float
|
||||
|
||||
|
||||
def make_easy_references(
|
||||
reference_image_1: Any = None,
|
||||
reference_image_2: Any = None,
|
||||
reference_image_3: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Pack fixed Easy reference sockets without preprocessing their images."""
|
||||
|
||||
return {
|
||||
"schema_version": EASY_REFERENCES_SCHEMA_VERSION,
|
||||
"reference_image_1": reference_image_1,
|
||||
"reference_image_2": reference_image_2,
|
||||
"reference_image_3": reference_image_3,
|
||||
}
|
||||
|
||||
|
||||
def resolve_easy_references(references: Any = None) -> tuple[Any, Any, Any]:
|
||||
"""Expand the private Easy bundle back into the mature V3.7 input contract."""
|
||||
|
||||
if references is None:
|
||||
return None, None, None
|
||||
if not isinstance(references, dict):
|
||||
raise TypeError("Easy References must come from H3 Continuum Easy References")
|
||||
if references.get("schema_version") != EASY_REFERENCES_SCHEMA_VERSION:
|
||||
raise ValueError("Unsupported H3 Continuum Easy References schema version")
|
||||
return tuple(
|
||||
references.get(f"reference_image_{index}") for index in range(1, 4)
|
||||
)
|
||||
|
||||
|
||||
def resolve_easy_duration(duration_seconds: int) -> EasyDurationPlan:
|
||||
"""Resolve one total duration around ten-second logical chunks."""
|
||||
|
||||
if isinstance(duration_seconds, bool):
|
||||
raise TypeError("Duration must be an integer number of seconds")
|
||||
duration = int(duration_seconds)
|
||||
if duration != duration_seconds:
|
||||
raise ValueError("Duration must be an integer number of seconds")
|
||||
if not EASY_DURATION_MIN <= duration <= EASY_DURATION_MAX:
|
||||
raise ValueError(
|
||||
f"Duration must be between {EASY_DURATION_MIN} and {EASY_DURATION_MAX} seconds"
|
||||
)
|
||||
chunks = min(
|
||||
EASY_MAX_CHUNKS,
|
||||
max(1, int(math.ceil(float(duration) / EASY_TARGET_CHUNK_SECONDS))),
|
||||
)
|
||||
chunk_seconds = float(duration) / float(chunks)
|
||||
if not 4.0 <= chunk_seconds <= 30.0:
|
||||
raise ValueError("Duration cannot be represented by the Production chunk contract")
|
||||
return EasyDurationPlan(
|
||||
duration_seconds=duration,
|
||||
chunks=chunks,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumEasyReferences:
|
||||
"""Fixed three-image facade for the existing V3.7 reference contract."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = CONTINUUM_CATEGORY
|
||||
DESCRIPTION = (
|
||||
"Collects up to three optional still-image references for H3 Continuum "
|
||||
"Easy V3.8. Images are passed unchanged to the existing V3.7 reference "
|
||||
"preprocessing and conditioning path."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Easy References",
|
||||
"MiniMax H3 Easy Reference Images",
|
||||
]
|
||||
RETURN_TYPES = (EASY_REFERENCES_TYPE,)
|
||||
RETURN_NAMES = ("references",)
|
||||
FUNCTION = "pack"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"optional": {
|
||||
"reference_image_1": (
|
||||
"IMAGE",
|
||||
{"display_name": "Reference Image 1 (Optional)"},
|
||||
),
|
||||
"reference_image_2": (
|
||||
"IMAGE",
|
||||
{"display_name": "Reference Image 2 (Optional)"},
|
||||
),
|
||||
"reference_image_3": (
|
||||
"IMAGE",
|
||||
{"display_name": "Reference Image 3 (Optional)"},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
def pack(
|
||||
self,
|
||||
reference_image_1=None,
|
||||
reference_image_2=None,
|
||||
reference_image_3=None,
|
||||
):
|
||||
return (
|
||||
make_easy_references(
|
||||
reference_image_1=reference_image_1,
|
||||
reference_image_2=reference_image_2,
|
||||
reference_image_3=reference_image_3,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumEasyV38(H3ContinuumSamplerV37):
|
||||
"""Beginner-facing resolver over the unchanged V3.7 Production engine."""
|
||||
|
||||
DEPRECATED = False
|
||||
CATEGORY = CONTINUUM_CATEGORY
|
||||
DESCRIPTION = (
|
||||
"Easy Mode facade over H3 Continuum V3.7 Production. It resolves total "
|
||||
"duration, decimal-MP resolution, seed UI, and image mode without copying "
|
||||
"the sampler or scheduler implementation."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Easy V3.8",
|
||||
"MiniMax H3 Easy Mode",
|
||||
]
|
||||
RETURN_TYPES = (
|
||||
"LATENT",
|
||||
"LATENT",
|
||||
"H3_CONTINUUM_ASSEMBLY_PLAN",
|
||||
"STRING",
|
||||
"AUDIO",
|
||||
)
|
||||
RETURN_NAMES = (
|
||||
"video_latents",
|
||||
"audio_latents",
|
||||
"assembly_plan",
|
||||
"status",
|
||||
"driving_audio",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False, False)
|
||||
FUNCTION = "run"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"video_vae": (
|
||||
"VAE",
|
||||
{
|
||||
"display_name": "Video VAE",
|
||||
"tooltip": "Used only for connected image conditioning.",
|
||||
},
|
||||
),
|
||||
"sampler": ("SAMPLER",),
|
||||
"sigmas": ("SIGMAS",),
|
||||
"prompt_text": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"dynamicPrompts": True,
|
||||
"display_name": "Prompt",
|
||||
},
|
||||
),
|
||||
"duration": (
|
||||
"INT",
|
||||
{
|
||||
"default": EASY_DURATION_DEFAULT,
|
||||
"min": EASY_DURATION_MIN,
|
||||
"max": EASY_DURATION_MAX,
|
||||
"step": 1,
|
||||
"display_name": "Duration (sec)",
|
||||
},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"display_name": "Seed",
|
||||
},
|
||||
),
|
||||
"seed_mode": (
|
||||
EASY_SEED_MODE_OPTIONS,
|
||||
{
|
||||
"default": EASY_SEED_MODE_RANDOMIZE,
|
||||
"display_name": "Seed Mode",
|
||||
},
|
||||
),
|
||||
"aspect": (
|
||||
EASY_ASPECT_OPTIONS,
|
||||
{
|
||||
"default": EASY_ASPECT_AUTO,
|
||||
"display_name": "Aspect",
|
||||
},
|
||||
),
|
||||
"preset": (
|
||||
EASY_PRESET_OPTIONS,
|
||||
{
|
||||
"default": EASY_PRESET_DRAFT,
|
||||
"display_name": "Resolution Preset",
|
||||
},
|
||||
),
|
||||
"custom_mp": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": EASY_CUSTOM_MP_DEFAULT,
|
||||
"min": EASY_CUSTOM_MP_MIN,
|
||||
"max": EASY_CUSTOM_MP_MAX,
|
||||
"step": 0.01,
|
||||
"display_name": "Custom MP",
|
||||
},
|
||||
),
|
||||
"run_storage": (
|
||||
("Off", "Save + Auto Resume"),
|
||||
{
|
||||
"default": "Off",
|
||||
"display_name": "Run Storage",
|
||||
"advanced": True,
|
||||
},
|
||||
),
|
||||
"run_name": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"display_name": "Run Name (Optional Override)",
|
||||
"advanced": True,
|
||||
},
|
||||
),
|
||||
"project_id": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"display_name": "Auto Resume ID Override",
|
||||
"advanced": True,
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"first_frame": (
|
||||
"IMAGE",
|
||||
{"display_name": "First Image (Optional)"},
|
||||
),
|
||||
"last_frame": (
|
||||
"IMAGE",
|
||||
{"display_name": "Last Image (Optional)"},
|
||||
),
|
||||
"references": (
|
||||
EASY_REFERENCES_TYPE,
|
||||
{"display_name": "Easy References (Optional)"},
|
||||
),
|
||||
"driving_audio": (
|
||||
"AUDIO",
|
||||
{
|
||||
"display_name": "Driving Audio",
|
||||
"tooltip": (
|
||||
"Optional original audio timeline. Uses the unchanged "
|
||||
"V3.7 Driving Audio conditioning and final-output contract."
|
||||
),
|
||||
},
|
||||
),
|
||||
"audio_vae": (
|
||||
"VAE",
|
||||
{
|
||||
"display_name": "Driving Audio VAE",
|
||||
"tooltip": (
|
||||
"Required only when Driving Audio is connected. It may be "
|
||||
"shared with the external Core audio decode path."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"hidden": {
|
||||
"prompt": "PROMPT",
|
||||
"unique_id": "UNIQUE_ID",
|
||||
},
|
||||
}
|
||||
|
||||
def run(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
video_vae,
|
||||
sampler,
|
||||
sigmas,
|
||||
prompt_text,
|
||||
duration,
|
||||
seed,
|
||||
seed_mode=EASY_SEED_MODE_RANDOMIZE,
|
||||
aspect=EASY_ASPECT_AUTO,
|
||||
preset=EASY_PRESET_DRAFT,
|
||||
custom_mp=EASY_CUSTOM_MP_DEFAULT,
|
||||
run_storage="Off",
|
||||
run_name="",
|
||||
project_id="",
|
||||
first_frame=None,
|
||||
last_frame=None,
|
||||
references=None,
|
||||
driving_audio=None,
|
||||
audio_vae=None,
|
||||
prompt=None,
|
||||
unique_id=None,
|
||||
):
|
||||
del seed_mode # Queue-time seed mutation remains owned by ComfyUI's frontend.
|
||||
duration_plan = resolve_easy_duration(duration)
|
||||
resolution = resolve_easy_resolution(
|
||||
aspect=aspect,
|
||||
preset=preset,
|
||||
custom_mp=custom_mp,
|
||||
first_frame=first_frame,
|
||||
)
|
||||
reference_image_1, reference_image_2, reference_image_3 = (
|
||||
resolve_easy_references(references)
|
||||
)
|
||||
outputs = super().run(
|
||||
model=model,
|
||||
clip=clip,
|
||||
video_vae=video_vae,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
sequence_prompt=prompt_text,
|
||||
prompt_mode=PROMPT_FORMAT_AUTO,
|
||||
chunks=duration_plan.chunks,
|
||||
chunk_seconds=duration_plan.chunk_seconds,
|
||||
width=resolution.width,
|
||||
height=resolution.height,
|
||||
continuity=CONTINUITY_OPTIONS[0],
|
||||
base_seed=int(seed),
|
||||
audio_continuity=True,
|
||||
diagnostics=DIAGNOSTICS_BASIC,
|
||||
reroll_from_chunk=REGENERATE_AUTO,
|
||||
reroll_nonce=0,
|
||||
strict_compatibility=False,
|
||||
debug=False,
|
||||
show_preview=True,
|
||||
run_storage=run_storage,
|
||||
run_name=run_name,
|
||||
reference_size=REFERENCE_SIZE_MATCH_OUTPUT,
|
||||
reference_image_1=reference_image_1,
|
||||
reference_image_2=reference_image_2,
|
||||
reference_image_3=reference_image_3,
|
||||
driving_audio=driving_audio,
|
||||
audio_vae=audio_vae,
|
||||
project_id=project_id,
|
||||
first_frame=first_frame,
|
||||
last_frame=last_frame,
|
||||
prompt=prompt,
|
||||
unique_id=unique_id,
|
||||
video_reference_size=REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
continuation_backend=CONTINUATION_BACKEND_STANDARD,
|
||||
)
|
||||
video, audio, assembly_plan, status, driving_audio = outputs[:5]
|
||||
mode = conditioning_display_label(
|
||||
has_first=first_frame is not None,
|
||||
has_last=last_frame is not None,
|
||||
has_reference=any(
|
||||
image is not None
|
||||
for image in (
|
||||
reference_image_1,
|
||||
reference_image_2,
|
||||
reference_image_3,
|
||||
)
|
||||
),
|
||||
)
|
||||
warning_lines = "\n".join(resolution.warnings)
|
||||
summary = (
|
||||
f"H3 Continuum Easy V3.8\n"
|
||||
f"{mode} / {duration_plan.duration_seconds} sec\n"
|
||||
f"{resolution.preset}: {resolution.width} x {resolution.height} "
|
||||
f"({resolution.actual_mp:.2f} MP)"
|
||||
)
|
||||
if warning_lines:
|
||||
summary += f"\n{warning_lines}"
|
||||
return video, audio, assembly_plan, f"{summary}\n\n{status}", driving_audio
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumEasyReferences": H3ContinuumEasyReferences,
|
||||
"H3ContinuumEasyV38": H3ContinuumEasyV38,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumEasyReferences": "H3 Continuum Easy References",
|
||||
"H3ContinuumEasyV38": "H3 Continuum Easy V3.8",
|
||||
}
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
"""Simple H3 video loader backed by the current ComfyUI Core video API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fractions import Fraction
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
try:
|
||||
import folder_paths
|
||||
except ImportError: # pragma: no cover - standalone unit tests
|
||||
class _FolderPathsShim:
|
||||
@staticmethod
|
||||
def get_input_directory():
|
||||
return "."
|
||||
|
||||
@staticmethod
|
||||
def filter_files_content_types(files, content_types):
|
||||
del files, content_types
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def get_annotated_filepath(file):
|
||||
raise RuntimeError(f"ComfyUI folder_paths is unavailable for {file}")
|
||||
|
||||
@staticmethod
|
||||
def exists_annotated_filepath(file):
|
||||
del file
|
||||
return False
|
||||
|
||||
folder_paths = _FolderPathsShim()
|
||||
|
||||
try:
|
||||
from comfy_api.latest import InputImpl, io
|
||||
except ImportError: # pragma: no cover - standalone unit tests
|
||||
class _SchemaItem:
|
||||
def __init__(self, item_id=None, **kwargs):
|
||||
self.id = item_id
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
class _InputFactory:
|
||||
Input = _SchemaItem
|
||||
|
||||
class _OutputFactory:
|
||||
@staticmethod
|
||||
def Output(**kwargs):
|
||||
return _SchemaItem(**kwargs)
|
||||
|
||||
class _UploadVideo:
|
||||
value = "video_upload"
|
||||
|
||||
class _UploadType:
|
||||
video = _UploadVideo()
|
||||
|
||||
class _Schema:
|
||||
def __init__(self, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
class _NodeOutput:
|
||||
def __init__(self, *args):
|
||||
self.args = args
|
||||
|
||||
class _ComfyNode:
|
||||
pass
|
||||
|
||||
class _IOShim:
|
||||
ComfyNode = _ComfyNode
|
||||
Schema = _Schema
|
||||
NodeOutput = _NodeOutput
|
||||
Boolean = _InputFactory
|
||||
Combo = _InputFactory
|
||||
Float = _InputFactory
|
||||
Video = _InputFactory
|
||||
Image = _OutputFactory
|
||||
Audio = _OutputFactory
|
||||
UploadType = _UploadType
|
||||
|
||||
class _InputImplShim:
|
||||
@staticmethod
|
||||
def VideoFromFile(path):
|
||||
raise RuntimeError(f"ComfyUI VideoFromFile is unavailable for {path}")
|
||||
|
||||
InputImpl = _InputImplShim
|
||||
io = _IOShim()
|
||||
|
||||
try:
|
||||
from comfy_execution.graph_utils import ExecutionBlocker
|
||||
except ImportError: # pragma: no cover - standalone unit tests
|
||||
class ExecutionBlocker:
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
|
||||
|
||||
def _fraction_from_rate(value: float) -> Fraction:
|
||||
"""Convert a UI frame-rate value without introducing binary-float drift."""
|
||||
|
||||
return Fraction(str(float(value))).limit_denominator(1_000_000)
|
||||
|
||||
|
||||
def _resample_video_frames(
|
||||
images: torch.Tensor,
|
||||
source_rate: Fraction,
|
||||
target_rate: float,
|
||||
) -> torch.Tensor:
|
||||
"""Drop or duplicate IMAGE frames while preserving nominal duration."""
|
||||
|
||||
if images.ndim != 4:
|
||||
raise ValueError(
|
||||
f"Expected IMAGE batch [B,H,W,C], got shape {tuple(images.shape)}"
|
||||
)
|
||||
|
||||
frame_count = int(images.shape[0])
|
||||
if frame_count <= 0:
|
||||
raise ValueError("Video contains no decodable frames")
|
||||
|
||||
source = Fraction(source_rate)
|
||||
if source <= 0:
|
||||
raise ValueError(f"Invalid source frame rate: {source_rate}")
|
||||
|
||||
requested_rate = float(target_rate)
|
||||
if requested_rate <= 0:
|
||||
return images
|
||||
|
||||
target = _fraction_from_rate(requested_rate)
|
||||
if target == source:
|
||||
return images
|
||||
|
||||
target_count = max(1, round(Fraction(frame_count, 1) * target / source))
|
||||
source_numerator = source.numerator
|
||||
source_denominator = source.denominator
|
||||
target_numerator = target.numerator
|
||||
target_denominator = target.denominator
|
||||
indices = [
|
||||
min(
|
||||
frame_count - 1,
|
||||
(
|
||||
index
|
||||
* source_numerator
|
||||
* target_denominator
|
||||
// (source_denominator * target_numerator)
|
||||
),
|
||||
)
|
||||
for index in range(target_count)
|
||||
]
|
||||
index_tensor = torch.tensor(indices, dtype=torch.long, device=images.device)
|
||||
return images.index_select(0, index_tensor)
|
||||
|
||||
|
||||
def load_video_components(
|
||||
*,
|
||||
file: str,
|
||||
enable_video: bool,
|
||||
force_rate: float,
|
||||
):
|
||||
"""Decode through Core and return standard IMAGE/AUDIO outputs."""
|
||||
|
||||
if not enable_video:
|
||||
blocker = ExecutionBlocker(None)
|
||||
return blocker, blocker
|
||||
|
||||
video_path = folder_paths.get_annotated_filepath(file)
|
||||
components = InputImpl.VideoFromFile(video_path).get_components()
|
||||
images = _resample_video_frames(
|
||||
components.images,
|
||||
Fraction(components.frame_rate),
|
||||
float(force_rate),
|
||||
)
|
||||
audio = components.audio
|
||||
if audio is None:
|
||||
audio = ExecutionBlocker(None)
|
||||
return images, audio
|
||||
|
||||
|
||||
class H3ContinuumLoadVideo(io.ComfyNode):
|
||||
"""Minimal Core-backed video loader for H3 Continuum workflows."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
input_dir = folder_paths.get_input_directory()
|
||||
files = [
|
||||
name
|
||||
for name in os.listdir(input_dir)
|
||||
if os.path.isfile(os.path.join(input_dir, name))
|
||||
]
|
||||
files = folder_paths.filter_files_content_types(files, ["video"])
|
||||
return io.Schema(
|
||||
node_id="H3ContinuumLoadVideo",
|
||||
is_deprecated=True,
|
||||
display_name="H3 Continuum Load Video",
|
||||
category=CATEGORY,
|
||||
description=(
|
||||
"Loads IMAGE and AUDIO through ComfyUI Core VideoFromFile. "
|
||||
"Force Rate drops or duplicates frames without changing duration or audio."
|
||||
),
|
||||
search_aliases=[
|
||||
"H3 Continuum Load Video",
|
||||
"H3 Easy Load Video",
|
||||
"MiniMax H3 Video Loader",
|
||||
],
|
||||
inputs=[
|
||||
io.Boolean.Input(
|
||||
"enable_video",
|
||||
default=True,
|
||||
display_name="Enable Video",
|
||||
),
|
||||
io.Combo.Input(
|
||||
"file",
|
||||
options=sorted(files),
|
||||
upload=io.UploadType.video,
|
||||
display_name="Video",
|
||||
),
|
||||
io.Float.Input(
|
||||
"force_rate",
|
||||
default=24.0,
|
||||
min=0.0,
|
||||
max=120.0,
|
||||
step=0.01,
|
||||
display_name="Force Rate",
|
||||
tooltip=(
|
||||
"0 uses the source FPS. A positive value resamples frames "
|
||||
"while preserving nominal duration."
|
||||
),
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
io.Image.Output(display_name="images"),
|
||||
io.Audio.Output(display_name="audio"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, enable_video: bool, file: str, force_rate: float):
|
||||
return io.NodeOutput(
|
||||
*load_video_components(
|
||||
file=file,
|
||||
enable_video=bool(enable_video),
|
||||
force_rate=float(force_rate),
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def fingerprint_inputs(cls, enable_video: bool, file: str, force_rate: float):
|
||||
enabled = bool(enable_video)
|
||||
file_name = str(file)
|
||||
modification_time = None
|
||||
if enabled and folder_paths.exists_annotated_filepath(file_name):
|
||||
video_path = folder_paths.get_annotated_filepath(file_name)
|
||||
modification_time = os.path.getmtime(video_path)
|
||||
return enabled, file_name, float(force_rate), modification_time
|
||||
|
||||
@classmethod
|
||||
def validate_inputs(cls, enable_video: bool, file: str, force_rate: float):
|
||||
del force_rate
|
||||
if not bool(enable_video):
|
||||
return True
|
||||
if not folder_paths.exists_annotated_filepath(file):
|
||||
return f"Invalid video file: {file}"
|
||||
return True
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumLoadVideo": H3ContinuumLoadVideo,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumLoadVideo": "H3 Continuum Load Video",
|
||||
}
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
"""Pure physical-group planner for the staged V3.8 runtime redesign.
|
||||
|
||||
The caller owns continuation-source priority and passes exactly one selected
|
||||
``ContinuationSourceFacts``. This module performs no repository, Session,
|
||||
State, tensor, ComfyUI, or sampling work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..branch_provenance import BranchProvenanceError, physical_groups
|
||||
from .planning_types import (
|
||||
ContinuationSourceFacts,
|
||||
ExecutionPlan,
|
||||
PhysicalGroupDescriptor,
|
||||
ProjectionDecision,
|
||||
)
|
||||
from .review_control import (
|
||||
REVISION_STATUS_COMPLETE,
|
||||
REVISION_STATUS_INTERRUPTED,
|
||||
REVISION_STATUS_REVIEW_READY,
|
||||
ReviewExecution,
|
||||
ReviewUnit,
|
||||
)
|
||||
|
||||
|
||||
PROJECTION_FULL_ACCEPTED_PREFIX = "full_accepted_prefix"
|
||||
PROJECTION_CURRENT_REVIEW_UNIT = "current_review_unit"
|
||||
|
||||
|
||||
class ExecutionPlanningError(ValueError):
|
||||
"""Raised when already-selected planning facts are internally inconsistent."""
|
||||
|
||||
|
||||
def resolve_projection_decision(
|
||||
*,
|
||||
review_execution: ReviewExecution | None,
|
||||
effective_result_status: str,
|
||||
accepted_chunks: int,
|
||||
physical_group_facts: tuple[tuple[int, ...], ...],
|
||||
) -> ProjectionDecision:
|
||||
"""Resolve output-only projection from finalized, source-neutral facts."""
|
||||
|
||||
full = ProjectionDecision(PROJECTION_FULL_ACCEPTED_PREFIX)
|
||||
if str(effective_result_status) != REVISION_STATUS_REVIEW_READY:
|
||||
return full
|
||||
if review_execution is None or bool(
|
||||
getattr(review_execution, "finish_remaining", False)
|
||||
):
|
||||
return full
|
||||
if not (
|
||||
bool(getattr(review_execution, "partial_review", False))
|
||||
or bool(getattr(review_execution, "smart_regenerate", False))
|
||||
):
|
||||
return full
|
||||
start = getattr(review_execution, "next_review_unit_start", None)
|
||||
end = getattr(review_execution, "next_review_unit_end", None)
|
||||
if start is None or end is None:
|
||||
return full
|
||||
start, end = int(start), int(end)
|
||||
if start < 1 or end < start or end > int(accepted_chunks):
|
||||
return full
|
||||
|
||||
selected = set(range(start, end + 1))
|
||||
if not physical_group_facts:
|
||||
return full
|
||||
for group in physical_group_facts:
|
||||
group_indices = set(group)
|
||||
if selected.intersection(group_indices) and not group_indices.issubset(selected):
|
||||
return full
|
||||
return ProjectionDecision(PROJECTION_CURRENT_REVIEW_UNIT, start, end)
|
||||
|
||||
|
||||
def build_execution_plan(
|
||||
*,
|
||||
continuation_source_facts: ContinuationSourceFacts,
|
||||
configured_chunks: int,
|
||||
terminal_merge_enabled: bool,
|
||||
max_new_physical_groups: int | None,
|
||||
review_execution: ReviewExecution | None,
|
||||
retained_review_unit: ReviewUnit | None = None,
|
||||
) -> ExecutionPlan:
|
||||
"""Plan physical groups without selecting or validating runtime sources."""
|
||||
|
||||
if not isinstance(continuation_source_facts, ContinuationSourceFacts):
|
||||
raise ExecutionPlanningError(
|
||||
"continuation_source_facts must be one selected ContinuationSourceFacts"
|
||||
)
|
||||
if continuation_source_facts.kind not in {
|
||||
"run_storage",
|
||||
"explicit_session",
|
||||
"initial_state",
|
||||
"none",
|
||||
}:
|
||||
raise ExecutionPlanningError(
|
||||
f"unknown continuation source kind: {continuation_source_facts.kind!r}"
|
||||
)
|
||||
if isinstance(configured_chunks, bool) or not isinstance(configured_chunks, int):
|
||||
raise ExecutionPlanningError("configured_chunks must be an integer")
|
||||
if configured_chunks < 1:
|
||||
raise ExecutionPlanningError("configured_chunks must be positive")
|
||||
if type(terminal_merge_enabled) is not bool:
|
||||
raise ExecutionPlanningError("terminal_merge_enabled must be a boolean")
|
||||
if max_new_physical_groups is not None and (
|
||||
isinstance(max_new_physical_groups, bool)
|
||||
or not isinstance(max_new_physical_groups, int)
|
||||
or max_new_physical_groups < 0
|
||||
):
|
||||
raise ExecutionPlanningError(
|
||||
"max_new_physical_groups must be None or a non-negative integer"
|
||||
)
|
||||
|
||||
accepted = continuation_source_facts.accepted_chunks
|
||||
if isinstance(accepted, bool) or not isinstance(accepted, int):
|
||||
raise ExecutionPlanningError("accepted_chunks must be an integer")
|
||||
if not 0 <= accepted <= configured_chunks:
|
||||
raise ExecutionPlanningError(
|
||||
"accepted_chunks must be within the configured sequence"
|
||||
)
|
||||
|
||||
try:
|
||||
runtime_groups = physical_groups(
|
||||
chunks=configured_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
)
|
||||
except BranchProvenanceError as exc:
|
||||
raise ExecutionPlanningError(str(exc)) from exc
|
||||
descriptors = tuple(
|
||||
PhysicalGroupDescriptor(
|
||||
physical_group=int(group.physical_group),
|
||||
logical_chunks=tuple(range(int(group.start), int(group.end) + 1)),
|
||||
terminal_atomic=int(group.start) != int(group.end),
|
||||
)
|
||||
for group in runtime_groups
|
||||
)
|
||||
accepted_boundaries = tuple(
|
||||
(descriptor.logical_chunks[0], descriptor.logical_chunks[-1])
|
||||
for descriptor in descriptors
|
||||
if descriptor.logical_chunks[-1] <= accepted
|
||||
)
|
||||
if accepted_boundaries and accepted_boundaries[-1][1] != accepted:
|
||||
raise ExecutionPlanningError("accepted prefix ends inside a physical group")
|
||||
if accepted and not accepted_boundaries:
|
||||
raise ExecutionPlanningError("accepted prefix has no complete physical group")
|
||||
if continuation_source_facts.physical_group_boundaries != accepted_boundaries:
|
||||
raise ExecutionPlanningError(
|
||||
"selected continuation source physical groups do not match its prefix"
|
||||
)
|
||||
|
||||
pending = tuple(
|
||||
descriptor
|
||||
for descriptor in descriptors
|
||||
if descriptor.logical_chunks[0] > accepted
|
||||
)
|
||||
groups_to_generate = (
|
||||
pending
|
||||
if max_new_physical_groups is None
|
||||
else pending[:max_new_physical_groups]
|
||||
)
|
||||
expected_chunks = accepted + sum(
|
||||
len(descriptor.logical_chunks) for descriptor in groups_to_generate
|
||||
)
|
||||
expected_review_unit = retained_review_unit
|
||||
if groups_to_generate and max_new_physical_groups is not None:
|
||||
final_group = groups_to_generate[-1]
|
||||
expected_review_unit = ReviewUnit(
|
||||
start=final_group.logical_chunks[0],
|
||||
end=final_group.logical_chunks[-1],
|
||||
physical_group=final_group.physical_group,
|
||||
)
|
||||
|
||||
if expected_chunks == configured_chunks:
|
||||
expected_status = REVISION_STATUS_COMPLETE
|
||||
elif max_new_physical_groups is not None and expected_review_unit is not None:
|
||||
expected_status = REVISION_STATUS_REVIEW_READY
|
||||
else:
|
||||
expected_status = REVISION_STATUS_INTERRUPTED
|
||||
|
||||
projection = resolve_projection_decision(
|
||||
review_execution=review_execution,
|
||||
effective_result_status=expected_status,
|
||||
accepted_chunks=expected_chunks,
|
||||
physical_group_facts=tuple(
|
||||
descriptor.logical_chunks for descriptor in descriptors
|
||||
),
|
||||
)
|
||||
return ExecutionPlan(
|
||||
groups_to_generate=groups_to_generate,
|
||||
projection=projection,
|
||||
expected_status=expected_status,
|
||||
expected_review_unit=expected_review_unit,
|
||||
)
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
"""V3.5 adapter for the accepted P2-0 file-backed Tensor backend.
|
||||
|
||||
The storage and lifetime implementation remains in the P2-0 module so the
|
||||
Windows-tested locking, sidecar, cancellation, and alias-lifetime contracts
|
||||
have one implementation. This module only exposes the small Production API
|
||||
needed by the V3.5 assembler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
from ..storage_memory import system_memory_bytes
|
||||
from .. import storage_settings
|
||||
|
||||
from ..tools.p20_file_backed_tensor_poc import (
|
||||
FileBackedAllocation,
|
||||
StorageLifetimeManager,
|
||||
inspect_managed_files,
|
||||
get_process_lifetime_manager,
|
||||
)
|
||||
|
||||
|
||||
BUFFER_BACKEND_AUTO = "Auto"
|
||||
BUFFER_BACKEND_RAM = "RAM"
|
||||
BUFFER_BACKEND_DISK = "Disk-backed"
|
||||
BUFFER_BACKEND_OPTIONS = (
|
||||
BUFFER_BACKEND_AUTO,
|
||||
BUFFER_BACKEND_RAM,
|
||||
BUFFER_BACKEND_DISK,
|
||||
)
|
||||
AUTO_RAM_IMAGE_LIMIT_BYTES = 4 * 1024**3
|
||||
AUTO_RAM_RESERVE_MIN_BYTES = 4 * 1024**3
|
||||
AUTO_RAM_RESERVE_FRACTION = 0.10
|
||||
AUTO_DISK_RESERVE_BYTES = 2 * 1024**3
|
||||
_BACKING_DIRECTORY_NAME = "h3-continuum-v35-file-backed"
|
||||
_CLEANED_ROOTS: set[str] = set()
|
||||
_CLEANED_ROOTS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def validate_buffer_backend(value: str) -> str:
|
||||
"""Validate the explicit or automatic V3.5 IMAGE backend request."""
|
||||
|
||||
if value not in BUFFER_BACKEND_OPTIONS:
|
||||
raise ValueError(f"unknown Buffer Backend: {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AutoBufferDecision:
|
||||
selected_backend: str
|
||||
final_image_bytes: int
|
||||
backing_root: Path
|
||||
available_memory_bytes: int | None
|
||||
total_memory_bytes: int | None
|
||||
memory_reserve_bytes: int | None
|
||||
ram_image_limit_bytes: int
|
||||
disk_free_bytes: int | None
|
||||
disk_reserve_bytes: int
|
||||
reason: str
|
||||
|
||||
|
||||
def _system_memory_bytes() -> tuple[int | None, int | None]:
|
||||
return system_memory_bytes()
|
||||
|
||||
|
||||
def _existing_disk_anchor(path: Path) -> Path:
|
||||
candidate = path
|
||||
while not candidate.exists() and candidate != candidate.parent:
|
||||
candidate = candidate.parent
|
||||
if not candidate.exists():
|
||||
raise OSError(f"no existing filesystem anchor for backing root: {path}")
|
||||
return candidate
|
||||
|
||||
|
||||
def _disk_free_bytes(path: Path) -> int:
|
||||
manager = storage_settings.get_manager()
|
||||
if manager is not None:
|
||||
manager.verify_final_root(path)
|
||||
return int(shutil.disk_usage(_existing_disk_anchor(path)).free)
|
||||
|
||||
|
||||
def preflight_manual_buffer(backend, final_image_bytes, backing_root=None):
|
||||
if backend == BUFFER_BACKEND_RAM:
|
||||
available, total = _system_memory_bytes()
|
||||
if available is None or total is None:
|
||||
raise RuntimeError("RAM buffer safety check unavailable; use Auto or Disk-backed, or restore memory counters")
|
||||
reserve = max(AUTO_RAM_RESERVE_MIN_BYTES, int(total * AUTO_RAM_RESERVE_FRACTION))
|
||||
if available < final_image_bytes + reserve:
|
||||
raise RuntimeError(f"RAM buffer needs {final_image_bytes} bytes plus {reserve} bytes reserve; "
|
||||
f"only {available} bytes are available. Select Auto or Disk-backed.")
|
||||
elif backend == BUFFER_BACKEND_DISK:
|
||||
root = resolve_backing_root(backing_root)
|
||||
free = _disk_free_bytes(root)
|
||||
if free < final_image_bytes + AUTO_DISK_RESERVE_BYTES:
|
||||
raise RuntimeError(f"Disk-backed buffer needs {final_image_bytes + AUTO_DISK_RESERVE_BYTES} bytes "
|
||||
f"including reserve; {free} bytes are free at {root}")
|
||||
|
||||
|
||||
def select_auto_buffer_backend(
|
||||
final_image_bytes: int,
|
||||
*,
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> AutoBufferDecision:
|
||||
"""Choose RAM only when both output size and physical headroom are safe.
|
||||
|
||||
Auto never converts a Disk-backed decision into RAM after a disk-space or
|
||||
allocation failure. Users can still explicitly select either manual path.
|
||||
"""
|
||||
|
||||
final_bytes = int(final_image_bytes)
|
||||
if final_bytes <= 0:
|
||||
raise ValueError("Auto Buffer Backend requires a positive final IMAGE size")
|
||||
root = resolve_backing_root(backing_root)
|
||||
available, total = _system_memory_bytes()
|
||||
reserve = None
|
||||
if available is not None and total is not None and total > 0:
|
||||
reserve = max(
|
||||
AUTO_RAM_RESERVE_MIN_BYTES,
|
||||
int(total * AUTO_RAM_RESERVE_FRACTION),
|
||||
)
|
||||
|
||||
if (
|
||||
reserve is not None
|
||||
and final_bytes <= AUTO_RAM_IMAGE_LIMIT_BYTES
|
||||
and available >= final_bytes + reserve
|
||||
):
|
||||
return AutoBufferDecision(
|
||||
selected_backend=BUFFER_BACKEND_RAM,
|
||||
final_image_bytes=final_bytes,
|
||||
backing_root=root,
|
||||
available_memory_bytes=available,
|
||||
total_memory_bytes=total,
|
||||
memory_reserve_bytes=reserve,
|
||||
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
|
||||
disk_free_bytes=None,
|
||||
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
|
||||
reason="final IMAGE fits the Auto RAM size limit and safety reserve",
|
||||
)
|
||||
|
||||
if reserve is None:
|
||||
reason = "physical-memory counters are unavailable"
|
||||
elif final_bytes > AUTO_RAM_IMAGE_LIMIT_BYTES:
|
||||
reason = "final IMAGE exceeds the Auto RAM size limit"
|
||||
else:
|
||||
reason = "available physical memory is below final IMAGE plus reserve"
|
||||
|
||||
try:
|
||||
disk_free = _disk_free_bytes(root)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"Auto Buffer Backend selected Disk-backed but disk capacity could "
|
||||
f"not be measured for {root}: {type(exc).__name__}: {exc}. "
|
||||
"Select RAM manually only if sufficient memory is known to be available."
|
||||
) from exc
|
||||
required_disk = final_bytes + AUTO_DISK_RESERVE_BYTES
|
||||
if disk_free < required_disk:
|
||||
raise RuntimeError(
|
||||
"Auto Buffer Backend selected Disk-backed but the backing volume "
|
||||
f"has {disk_free} free bytes; {required_disk} bytes are required "
|
||||
f"for the final IMAGE plus the {AUTO_DISK_RESERVE_BYTES}-byte safety "
|
||||
"reserve. Free disk space or select RAM manually only if sufficient "
|
||||
"memory is known to be available."
|
||||
)
|
||||
return AutoBufferDecision(
|
||||
selected_backend=BUFFER_BACKEND_DISK,
|
||||
final_image_bytes=final_bytes,
|
||||
backing_root=root,
|
||||
available_memory_bytes=available,
|
||||
total_memory_bytes=total,
|
||||
memory_reserve_bytes=reserve,
|
||||
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
|
||||
disk_free_bytes=disk_free,
|
||||
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def format_auto_buffer_decision(decision: AutoBufferDecision) -> str:
|
||||
def value(number: int | None) -> str:
|
||||
return "n/a" if number is None else str(int(number))
|
||||
|
||||
return (
|
||||
"Auto Buffer Decision: "
|
||||
f"selected={decision.selected_backend}, "
|
||||
f"final_IMAGE={decision.final_image_bytes} bytes, "
|
||||
f"system_available={value(decision.available_memory_bytes)} bytes, "
|
||||
f"system_total={value(decision.total_memory_bytes)} bytes, "
|
||||
f"RAM_reserve={value(decision.memory_reserve_bytes)} bytes, "
|
||||
f"RAM_IMAGE_limit={decision.ram_image_limit_bytes} bytes, "
|
||||
f"disk_free={value(decision.disk_free_bytes)} bytes, "
|
||||
f"disk_reserve={decision.disk_reserve_bytes} bytes; "
|
||||
f"reason={decision.reason}."
|
||||
)
|
||||
|
||||
|
||||
def resolve_backing_root(
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> Path:
|
||||
"""Resolve the dedicated backing root without importing ComfyUI eagerly."""
|
||||
|
||||
if backing_root is not None:
|
||||
return Path(backing_root).expanduser().resolve()
|
||||
|
||||
try:
|
||||
import folder_paths
|
||||
except ImportError:
|
||||
base = Path(tempfile.gettempdir())
|
||||
else:
|
||||
value = folder_paths.get_temp_directory()
|
||||
if value is None or not str(value).strip():
|
||||
raise RuntimeError("ComfyUI returned an empty temporary directory")
|
||||
base = Path(value)
|
||||
default = (base.expanduser().resolve() / _BACKING_DIRECTORY_NAME).resolve()
|
||||
manager = storage_settings.get_manager()
|
||||
return manager.resolve("finalize", default, verify=False) if manager else default
|
||||
|
||||
|
||||
def get_file_backed_image_manager(
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> StorageLifetimeManager:
|
||||
"""Return the accepted process-global manager for one resolved root."""
|
||||
|
||||
root = resolve_backing_root(backing_root)
|
||||
settings = storage_settings.get_manager()
|
||||
if settings is not None:
|
||||
settings.verify_final_root(root)
|
||||
key = os.path.normcase(str(root))
|
||||
with _CLEANED_ROOTS_LOCK:
|
||||
if key not in _CLEANED_ROOTS:
|
||||
# Production startup and the UI share conservative ownership rules.
|
||||
# The standalone recovery probe can still inspect legacy orphans;
|
||||
# an unknown production file is never deleted by its name alone.
|
||||
inspect_managed_files(root, cleanup=True)
|
||||
_CLEANED_ROOTS.add(key)
|
||||
manager = get_process_lifetime_manager(root)
|
||||
if settings is not None:
|
||||
settings.record_root("finalize", root)
|
||||
return manager
|
||||
|
||||
|
||||
def allocate_file_backed_image(
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
backing_root: str | os.PathLike[str] | None = None,
|
||||
) -> FileBackedAllocation:
|
||||
"""Create a provisional Disk-backed IMAGE allocation.
|
||||
|
||||
The caller must use the returned context manager and call ``publish()``
|
||||
only after every output element has been written. Backend failures are
|
||||
propagated; this manual Disk-backed path never falls back to RAM.
|
||||
"""
|
||||
|
||||
manager = get_file_backed_image_manager(backing_root)
|
||||
return manager.allocate(shape, dtype=dtype)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AUTO_DISK_RESERVE_BYTES",
|
||||
"AUTO_RAM_IMAGE_LIMIT_BYTES",
|
||||
"AUTO_RAM_RESERVE_FRACTION",
|
||||
"AUTO_RAM_RESERVE_MIN_BYTES",
|
||||
"AutoBufferDecision",
|
||||
"BUFFER_BACKEND_AUTO",
|
||||
"BUFFER_BACKEND_DISK",
|
||||
"BUFFER_BACKEND_OPTIONS",
|
||||
"BUFFER_BACKEND_RAM",
|
||||
"allocate_file_backed_image",
|
||||
"format_auto_buffer_decision",
|
||||
"get_file_backed_image_manager",
|
||||
"resolve_backing_root",
|
||||
"select_auto_buffer_backend",
|
||||
"validate_buffer_backend",
|
||||
]
|
||||
+527
@@ -0,0 +1,527 @@
|
||||
"""V3.5 Continuum-aware Hi-res fix convenience node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from .pixel_vae_resize import (
|
||||
resize_video_latents_via_vae,
|
||||
resolve_pixel_latent_scale,
|
||||
)
|
||||
from .second_pass import SecondPassContractError, run_second_pass_groups
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
HIRES_UPSCALE_METHODS = (
|
||||
"lanczos",
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
)
|
||||
_CONDITIONING_UPSCALE_METHOD = "bilinear"
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
def _enabled_value(value: Any) -> bool:
|
||||
if isinstance(value, list):
|
||||
value = _single_list_input("enabled", value)
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _lazy_missing(value: Any) -> bool:
|
||||
return value is None or (
|
||||
isinstance(value, (list, tuple))
|
||||
and len(value) == 1
|
||||
and value[0] is None
|
||||
)
|
||||
|
||||
|
||||
def h3_core_native_canvas(
|
||||
width: int,
|
||||
height: int,
|
||||
*,
|
||||
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Return the native canvas selected by ComfyUI Core for this aspect ratio."""
|
||||
|
||||
if width < 1 or height < 1:
|
||||
raise SecondPassContractError("source pixel width and height must be positive")
|
||||
if adapt_canvas_fn is None:
|
||||
from comfy_extras.nodes_minimax_h3 import adapt_canvas
|
||||
|
||||
adapt_canvas_fn = adapt_canvas
|
||||
target = adapt_canvas_fn(int(width), int(height))
|
||||
if not isinstance(target, Sequence) or len(target) != 2:
|
||||
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
|
||||
target_width, target_height = (int(value) for value in target)
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
|
||||
return target_width, target_height
|
||||
|
||||
|
||||
def align_h3_manual_canvas(
|
||||
current_latent_width: int,
|
||||
current_latent_height: int,
|
||||
scale_by: float,
|
||||
*,
|
||||
pixel_scale_width: int,
|
||||
pixel_scale_height: int,
|
||||
canvas_multiple: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Scale and align a manual target to ComfyUI Core's H3 canvas grid."""
|
||||
|
||||
if canvas_multiple is None:
|
||||
from comfy_extras.nodes_minimax_h3 import CANVAS_MULTIPLE
|
||||
|
||||
canvas_multiple = int(CANVAS_MULTIPLE)
|
||||
current_latent_width = int(current_latent_width)
|
||||
current_latent_height = int(current_latent_height)
|
||||
pixel_scale_width = int(pixel_scale_width)
|
||||
pixel_scale_height = int(pixel_scale_height)
|
||||
canvas_multiple = int(canvas_multiple)
|
||||
scale_by = float(scale_by)
|
||||
if min(
|
||||
current_latent_width,
|
||||
current_latent_height,
|
||||
pixel_scale_width,
|
||||
pixel_scale_height,
|
||||
canvas_multiple,
|
||||
) < 1:
|
||||
raise SecondPassContractError("manual H3 canvas geometry is invalid")
|
||||
if (
|
||||
canvas_multiple % pixel_scale_width
|
||||
or canvas_multiple % pixel_scale_height
|
||||
):
|
||||
raise SecondPassContractError(
|
||||
"ComfyUI Core H3 canvas multiple is not aligned to the latent scale"
|
||||
)
|
||||
|
||||
def aligned_target(current_latent: int, pixel_scale: int) -> int:
|
||||
current_pixels = current_latent * pixel_scale
|
||||
scaled_pixels = current_pixels * scale_by
|
||||
nearest_core_canvas = max(
|
||||
canvas_multiple,
|
||||
round(scaled_pixels / canvas_multiple) * canvas_multiple,
|
||||
)
|
||||
preserved_core_canvas = (
|
||||
(current_pixels + canvas_multiple - 1) // canvas_multiple
|
||||
) * canvas_multiple
|
||||
return max(nearest_core_canvas, preserved_core_canvas)
|
||||
|
||||
target_width = aligned_target(current_latent_width, pixel_scale_width)
|
||||
target_height = aligned_target(current_latent_height, pixel_scale_height)
|
||||
return {
|
||||
"target_width": target_width,
|
||||
"target_height": target_height,
|
||||
"target_latent_width": target_width // pixel_scale_width,
|
||||
"target_latent_height": target_height // pixel_scale_height,
|
||||
}
|
||||
|
||||
|
||||
def resolve_h3_native_latent_target(
|
||||
video_latents: Sequence[Mapping[str, Any]],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
*,
|
||||
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
|
||||
) -> dict[str, int | bool]:
|
||||
"""Resolve Core's native canvas without shrinking an existing video latent."""
|
||||
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise SecondPassContractError("assembly_plan has no second_pass_contract")
|
||||
groups = contract.get("physical_groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise SecondPassContractError(
|
||||
"second_pass_contract.physical_groups must be non-empty"
|
||||
)
|
||||
if not isinstance(video_latents, Sequence) or not video_latents:
|
||||
raise SecondPassContractError("video_latents must be a non-empty sequence")
|
||||
if len(video_latents) != len(groups):
|
||||
raise SecondPassContractError("video latent group count changed")
|
||||
|
||||
first_group = groups[0]
|
||||
if not isinstance(first_group, Mapping):
|
||||
raise SecondPassContractError("physical group metadata must be a mapping")
|
||||
source_width = int(first_group.get("source_width", -1))
|
||||
source_height = int(first_group.get("source_height", -1))
|
||||
source_latent_width = int(first_group.get("source_latent_w", -1))
|
||||
source_latent_height = int(first_group.get("source_latent_h", -1))
|
||||
if min(source_width, source_height, source_latent_width, source_latent_height) < 1:
|
||||
raise SecondPassContractError("source pixel/latent geometry is invalid")
|
||||
if source_width % source_latent_width or source_height % source_latent_height:
|
||||
raise SecondPassContractError("source pixel/latent geometry is not integral")
|
||||
pixel_scale_width = source_width // source_latent_width
|
||||
pixel_scale_height = source_height // source_latent_height
|
||||
|
||||
current_geometry: tuple[int, int] | None = None
|
||||
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
|
||||
if not isinstance(group, Mapping):
|
||||
raise SecondPassContractError("physical group metadata must be a mapping")
|
||||
if int(group.get("source_width", -1)) != source_width or int(
|
||||
group.get("source_height", -1)
|
||||
) != source_height:
|
||||
raise SecondPassContractError("source geometry differs between physical groups")
|
||||
if int(group.get("source_latent_w", -1)) != source_latent_width or int(
|
||||
group.get("source_latent_h", -1)
|
||||
) != source_latent_height:
|
||||
raise SecondPassContractError(
|
||||
"source latent geometry differs between physical groups"
|
||||
)
|
||||
if not isinstance(latent, Mapping) or "samples" not in latent:
|
||||
raise SecondPassContractError(
|
||||
f"video_latents[{position}] must be a LATENT mapping with samples"
|
||||
)
|
||||
samples = latent["samples"]
|
||||
if not hasattr(samples, "shape") or len(samples.shape) != 5:
|
||||
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
|
||||
geometry = (int(samples.shape[-2]), int(samples.shape[-1]))
|
||||
if current_geometry is None:
|
||||
current_geometry = geometry
|
||||
elif geometry != current_geometry:
|
||||
raise SecondPassContractError(
|
||||
"all physical groups must use the same current H/W"
|
||||
)
|
||||
|
||||
assert current_geometry is not None
|
||||
native_width, native_height = h3_core_native_canvas(
|
||||
source_width,
|
||||
source_height,
|
||||
adapt_canvas_fn=adapt_canvas_fn,
|
||||
)
|
||||
if native_width % pixel_scale_width or native_height % pixel_scale_height:
|
||||
raise SecondPassContractError(
|
||||
"ComfyUI Core native canvas is not aligned to the H3 latent scale"
|
||||
)
|
||||
native_latent_width = native_width // pixel_scale_width
|
||||
native_latent_height = native_height // pixel_scale_height
|
||||
current_height, current_width = current_geometry
|
||||
preserve_current = (
|
||||
native_latent_width < current_width
|
||||
or native_latent_height < current_height
|
||||
)
|
||||
if preserve_current:
|
||||
target_latent_width = current_width
|
||||
target_latent_height = current_height
|
||||
else:
|
||||
target_latent_width = native_latent_width
|
||||
target_latent_height = native_latent_height
|
||||
|
||||
return {
|
||||
"source_width": source_width,
|
||||
"source_height": source_height,
|
||||
"native_width": native_width,
|
||||
"native_height": native_height,
|
||||
"target_width": target_latent_width * pixel_scale_width,
|
||||
"target_height": target_latent_height * pixel_scale_height,
|
||||
"target_latent_width": target_latent_width,
|
||||
"target_latent_height": target_latent_height,
|
||||
"preserved_current": preserve_current,
|
||||
}
|
||||
|
||||
|
||||
def _prepend_resize_status(
|
||||
second_pass_status: str,
|
||||
*,
|
||||
backend: str,
|
||||
mode: str,
|
||||
upscale_method: str,
|
||||
scale_by: float,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
native_width: int,
|
||||
native_height: int,
|
||||
) -> str:
|
||||
lines = [
|
||||
"H3 Continuum Hi-Res Fix V3.5",
|
||||
"Pixel/VAE resize: "
|
||||
f"backend={backend}, mode={mode}, method={upscale_method}, "
|
||||
f"scale_by={scale_by:g}, "
|
||||
f"source={source_width}x{source_height}, "
|
||||
f"target={target_width}x{target_height}, "
|
||||
f"core_native={native_width}x{native_height}.",
|
||||
]
|
||||
if target_width > native_width or target_height > native_height:
|
||||
lines.append(
|
||||
"INFO: target exceeds the ComfyUI Core H3 base canvas; the Pixel/VAE "
|
||||
"roundtrip is preserved and resource use may increase."
|
||||
)
|
||||
lines.append(second_pass_status)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class H3ContinuumHiResFixV35:
|
||||
"""Pixel/VAE resize and low-sigma refine complete physical video groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"Continuum-aware Hi-res fix. When enabled, each first-pass physical video "
|
||||
"LATENT is decoded with the connected H3 Video VAE, resized in pixel space, "
|
||||
"re-encoded with the same VAE, and refined once with the workflow SIGMAS. "
|
||||
"scale_by 0 uses ComfyUI Core's H3 native canvas; positive values use the "
|
||||
"manual multiplier. Connect the optional Continuum refine context to preserve "
|
||||
"First/Last/Reference conditioning at the target geometry. "
|
||||
"When disabled, first-pass video/audio LATENT objects and the original assembly "
|
||||
"plan are returned unchanged."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
lazy = {"lazy": True}
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL", lazy),
|
||||
"clip": ("CLIP", lazy),
|
||||
"sampler": ("SAMPLER", lazy),
|
||||
"sigmas": ("SIGMAS", lazy),
|
||||
"video_latents": ("LATENT",),
|
||||
"audio_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"enabled": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": (
|
||||
"Enable Resize + Second Pass. Off returns the first-pass "
|
||||
"video/audio LATENT objects and assembly plan unchanged."
|
||||
),
|
||||
},
|
||||
),
|
||||
"upscale_method": (
|
||||
HIRES_UPSCALE_METHODS,
|
||||
{"default": "lanczos"},
|
||||
),
|
||||
"scale_by": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 2.0,
|
||||
"min": 0.0,
|
||||
"max": 4.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the "
|
||||
"ComfyUI Core H3 native canvas automatically. Manual "
|
||||
"values follow the existing Second Pass contract and must "
|
||||
"be 1.0 or greater."
|
||||
),
|
||||
},
|
||||
),
|
||||
"refine_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"tooltip": "Base seed independently derived once per physical group.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"refine_context": (
|
||||
"H3_CONTINUUM_REFINE_CONTEXT",
|
||||
{"lazy": True},
|
||||
),
|
||||
"video_vae": (
|
||||
"VAE",
|
||||
{
|
||||
"lazy": True,
|
||||
"tooltip": (
|
||||
"Required while enabled for the safe Pixel/VAE roundtrip; "
|
||||
"not evaluated while disabled."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
|
||||
RETURN_NAMES = (
|
||||
"video_latents",
|
||||
"audio_latents",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "apply"
|
||||
CATEGORY = "MiniMax H3/Continuum"
|
||||
|
||||
def check_lazy_status(
|
||||
self,
|
||||
enabled,
|
||||
model=None,
|
||||
clip=None,
|
||||
sampler=None,
|
||||
sigmas=None,
|
||||
refine_context=_MISSING,
|
||||
video_vae=_MISSING,
|
||||
**_kwargs,
|
||||
):
|
||||
if not _enabled_value(enabled):
|
||||
return []
|
||||
required = [
|
||||
name
|
||||
for name, value in (
|
||||
("model", model),
|
||||
("clip", clip),
|
||||
("sampler", sampler),
|
||||
("sigmas", sigmas),
|
||||
)
|
||||
if _lazy_missing(value)
|
||||
]
|
||||
required.extend(
|
||||
name
|
||||
for name, value in (
|
||||
("refine_context", refine_context),
|
||||
("video_vae", video_vae),
|
||||
)
|
||||
if value is not _MISSING and _lazy_missing(value)
|
||||
)
|
||||
return required
|
||||
|
||||
def apply(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
sampler,
|
||||
sigmas,
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
enabled,
|
||||
upscale_method,
|
||||
scale_by,
|
||||
refine_seed,
|
||||
refine_context=None,
|
||||
video_vae=None,
|
||||
refine_schedule=None,
|
||||
):
|
||||
plan = _single_list_input("assembly_plan", assembly_plan)
|
||||
if not _enabled_value(enabled):
|
||||
return (
|
||||
video_latents,
|
||||
audio_latents,
|
||||
plan,
|
||||
"H3 Continuum Hi-Res Fix V3.5: disabled; first-pass video/audio "
|
||||
"LATENT objects and assembly_plan returned unchanged.",
|
||||
)
|
||||
|
||||
method = str(_single_list_input("upscale_method", upscale_method))
|
||||
scale = float(_single_list_input("scale_by", scale_by))
|
||||
if 0.0 < scale < 1.0:
|
||||
raise SecondPassContractError(
|
||||
"scale_by must be 0 for H3 native Auto or at least 1.0; the "
|
||||
"existing Second Pass contract preserves or enlarges spatial size"
|
||||
)
|
||||
vae = (
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
)
|
||||
if vae is None:
|
||||
raise SecondPassContractError(
|
||||
"Hi-Res Fix enabled requires video_vae for the safe Pixel/VAE "
|
||||
"roundtrip; legacy direct latent interpolation is disabled"
|
||||
)
|
||||
source_width = int(plan["width"])
|
||||
source_height = int(plan["height"])
|
||||
native_width, native_height = h3_core_native_canvas(
|
||||
source_width,
|
||||
source_height,
|
||||
)
|
||||
first_samples = video_latents[0]["samples"]
|
||||
current_geometry = (
|
||||
int(first_samples.shape[-2]),
|
||||
int(first_samples.shape[-1]),
|
||||
)
|
||||
if scale == 0.0:
|
||||
native_target = resolve_h3_native_latent_target(video_latents, plan)
|
||||
target_latent_width = int(native_target["target_latent_width"])
|
||||
target_latent_height = int(native_target["target_latent_height"])
|
||||
target_width = int(native_target["target_width"])
|
||||
target_height = int(native_target["target_height"])
|
||||
resize_mode = "h3_native_canvas_auto"
|
||||
else:
|
||||
pixel_scale_width, pixel_scale_height = resolve_pixel_latent_scale(plan)
|
||||
manual_target = align_h3_manual_canvas(
|
||||
current_geometry[1],
|
||||
current_geometry[0],
|
||||
scale,
|
||||
pixel_scale_width=pixel_scale_width,
|
||||
pixel_scale_height=pixel_scale_height,
|
||||
)
|
||||
target_latent_width = int(manual_target["target_latent_width"])
|
||||
target_latent_height = int(manual_target["target_latent_height"])
|
||||
target_width = int(manual_target["target_width"])
|
||||
target_height = int(manual_target["target_height"])
|
||||
resize_mode = "manual_scale_by"
|
||||
|
||||
if current_geometry == (target_latent_height, target_latent_width):
|
||||
resized_video_latents = list(video_latents)
|
||||
resize_backend = "direct_second_pass_no_resize"
|
||||
else:
|
||||
resized_video_latents = resize_video_latents_via_vae(
|
||||
video_latents,
|
||||
plan,
|
||||
video_vae=vae,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
upscale_method=method,
|
||||
)
|
||||
resize_backend = "sequential_pixel_vae_roundtrip"
|
||||
|
||||
refined_video, output_audio, updated_plan, status = run_second_pass_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
sampler=_single_list_input("sampler", sampler),
|
||||
sigmas=_single_list_input("sigmas", sigmas),
|
||||
video_latents=resized_video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=plan,
|
||||
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
|
||||
refine_context=(
|
||||
None
|
||||
if refine_context is None
|
||||
else _single_list_input("refine_context", refine_context)
|
||||
),
|
||||
video_vae=vae,
|
||||
conditioning_upscale_method=_CONDITIONING_UPSCALE_METHOD,
|
||||
refine_schedule=refine_schedule,
|
||||
)
|
||||
return (
|
||||
refined_video,
|
||||
output_audio,
|
||||
updated_plan,
|
||||
_prepend_resize_status(
|
||||
status,
|
||||
backend=resize_backend,
|
||||
mode=resize_mode,
|
||||
upscale_method=method,
|
||||
scale_by=scale,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
target_width=int(updated_plan["width"]),
|
||||
target_height=int(updated_plan["height"]),
|
||||
native_width=native_width,
|
||||
native_height=native_height,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumHiResFixV35": H3ContinuumHiResFixV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumHiResFixV35": "H3 Continuum Hi-Res Fix V3.5",
|
||||
}
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""V3.5 Core-compatible spatial resize for physical video latents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
||||
UPSCALE_METHODS = (
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
)
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
def resize_video_latents(
|
||||
video_latents: list[dict[str, Any]],
|
||||
*,
|
||||
upscale_method: str,
|
||||
scale_by: float,
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize only spatial H/W through the ComfyUI Core latent implementation."""
|
||||
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
resized: list[dict[str, Any]] = []
|
||||
for latent in video_latents:
|
||||
samples = latent["samples"]
|
||||
width = round(samples.shape[-1] * scale_by)
|
||||
height = round(samples.shape[-2] * scale_by)
|
||||
output = latent.copy()
|
||||
output["samples"] = upscale_fn(
|
||||
samples,
|
||||
width,
|
||||
height,
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
resized.append(output)
|
||||
return resized
|
||||
|
||||
|
||||
def resize_video_latents_to_size(
|
||||
video_latents: list[dict[str, Any]],
|
||||
*,
|
||||
upscale_method: str,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize every physical video group to one exact latent H/W."""
|
||||
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise ValueError("target latent width and height must be positive")
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
resized: list[dict[str, Any]] = []
|
||||
for latent in video_latents:
|
||||
samples = latent["samples"]
|
||||
output = latent.copy()
|
||||
output["samples"] = upscale_fn(
|
||||
samples,
|
||||
int(target_width),
|
||||
int(target_height),
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
resized.append(output)
|
||||
return resized
|
||||
|
||||
|
||||
class H3ContinuumLatentResizeV35:
|
||||
"""List-aware Core spatial resize for Continuum physical video groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"V3.5 Hi-res fix resize. Spatially resizes every Continuum "
|
||||
"physical video LATENT with the selected ComfyUI Core interpolation method. "
|
||||
"B/C/T, physical group order, and LATENT metadata are preserved."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video_latents": ("LATENT",),
|
||||
"upscale_method": (UPSCALE_METHODS,),
|
||||
"scale_by": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 2.0,
|
||||
"min": 0.01,
|
||||
"max": 8.0,
|
||||
"step": 0.01,
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT",)
|
||||
RETURN_NAMES = ("resized_video_latents",)
|
||||
OUTPUT_IS_LIST = (True,)
|
||||
FUNCTION = "resize"
|
||||
CATEGORY = "MiniMax H3/Continuum/Utilities"
|
||||
|
||||
def resize(self, video_latents, upscale_method, scale_by):
|
||||
return (
|
||||
resize_video_latents(
|
||||
video_latents,
|
||||
upscale_method=str(
|
||||
_single_list_input("upscale_method", upscale_method)
|
||||
),
|
||||
scale_by=float(_single_list_input("scale_by", scale_by)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumLatentResizeV35": H3ContinuumLatentResizeV35,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumLatentResizeV35": "H3 Continuum Latent Resize V3.5",
|
||||
}
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
"""Private V3.6 masked-prefix continuation experiments.
|
||||
|
||||
This module is intentionally not registered as a public node contract. It
|
||||
builds H3 AV targets whose protected prefixes are preserved by Core's denoise
|
||||
mask. R1 protects video only; R2-0 can protect matching video and audio tails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
from ..temporal import audio_latent_t, context_slots
|
||||
|
||||
|
||||
REFERENCE_CONTEXT_V1 = "reference_context_v1"
|
||||
MASKED_VIDEO_PREFIX_V1 = "masked_video_prefix_v1"
|
||||
MASKED_AV_PREFIX_22_V1 = "masked_av_prefix_22_v1"
|
||||
MASKED_AV_PREFIX_39_V1 = "masked_av_prefix_39_v1"
|
||||
CONTINUATION_TRANSPORTS = (
|
||||
REFERENCE_CONTEXT_V1,
|
||||
MASKED_VIDEO_PREFIX_V1,
|
||||
MASKED_AV_PREFIX_22_V1,
|
||||
MASKED_AV_PREFIX_39_V1,
|
||||
)
|
||||
|
||||
|
||||
def _build_masked_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
*,
|
||||
audio_context: torch.Tensor | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy normalized AV tails into a target and attach stream masks."""
|
||||
|
||||
try:
|
||||
import comfy.nested_tensor
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
|
||||
|
||||
target_video, target_audio = extract_av_streams(target_latent)
|
||||
frames = int(context_frames)
|
||||
slots = context_slots(frames)
|
||||
if not torch.is_tensor(video_context) or video_context.ndim != 5:
|
||||
raise ValueError("masked video context must be a five-dimensional Tensor")
|
||||
if tuple(video_context.shape[:2]) != tuple(target_video.shape[:2]):
|
||||
raise ValueError(
|
||||
"masked video context batch/channel shape does not match the target: "
|
||||
f"context={tuple(video_context.shape)}, target={tuple(target_video.shape)}"
|
||||
)
|
||||
if int(video_context.shape[2]) != slots:
|
||||
raise ValueError(
|
||||
f"masked video context has T={video_context.shape[2]}; expected {slots} "
|
||||
f"for {frames} frames"
|
||||
)
|
||||
if tuple(video_context.shape[-2:]) != tuple(target_video.shape[-2:]):
|
||||
raise ValueError(
|
||||
"masked video context geometry does not match the target: "
|
||||
f"context={tuple(video_context.shape[-2:])}, "
|
||||
f"target={tuple(target_video.shape[-2:])}"
|
||||
)
|
||||
if int(target_video.shape[2]) <= slots:
|
||||
raise ValueError("masked video target must contain generated slots after the prefix")
|
||||
|
||||
normalized_video = video_context.detach().to(
|
||||
device=target_video.device,
|
||||
dtype=target_video.dtype,
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(normalized_video.float()).all().item()):
|
||||
raise ValueError("masked video context contains NaN or Inf")
|
||||
|
||||
normalized_audio = None
|
||||
audio_steps = 0
|
||||
if audio_context is not None:
|
||||
audio_steps = audio_latent_t(frames)
|
||||
if not torch.is_tensor(audio_context) or audio_context.ndim != 4:
|
||||
raise ValueError("masked audio context must be a four-dimensional Tensor")
|
||||
if tuple(audio_context.shape[:3]) != tuple(target_audio.shape[:3]):
|
||||
raise ValueError(
|
||||
"masked audio context batch/channel shape does not match the target: "
|
||||
f"context={tuple(audio_context.shape)}, target={tuple(target_audio.shape)}"
|
||||
)
|
||||
if int(audio_context.shape[-1]) != audio_steps:
|
||||
raise ValueError(
|
||||
f"masked audio context has T={audio_context.shape[-1]}; expected "
|
||||
f"{audio_steps} for {frames} frames"
|
||||
)
|
||||
if int(target_audio.shape[-1]) <= audio_steps:
|
||||
raise ValueError("masked audio target must contain generated ticks after the prefix")
|
||||
normalized_audio = audio_context.detach().to(
|
||||
device=target_audio.device,
|
||||
dtype=target_audio.dtype,
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(normalized_audio.float()).all().item()):
|
||||
raise ValueError("masked audio context contains NaN or Inf")
|
||||
|
||||
video = target_video.clone()
|
||||
audio = target_audio.clone()
|
||||
video[:, :, :slots].copy_(normalized_video)
|
||||
if normalized_audio is not None:
|
||||
audio[..., :audio_steps].copy_(normalized_audio)
|
||||
|
||||
video_mask = torch.ones(
|
||||
(
|
||||
int(video.shape[0]),
|
||||
1,
|
||||
int(video.shape[2]),
|
||||
int(video.shape[3]),
|
||||
int(video.shape[4]),
|
||||
),
|
||||
device=video.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
video_mask[:, :, :slots] = 0.0
|
||||
audio_mask = torch.ones(
|
||||
(
|
||||
int(audio.shape[0]),
|
||||
1,
|
||||
int(audio.shape[2]),
|
||||
int(audio.shape[3]),
|
||||
),
|
||||
device=audio.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if normalized_audio is not None:
|
||||
audio_mask[..., :audio_steps] = 0.0
|
||||
|
||||
result = dict(target_latent)
|
||||
result["samples"] = comfy.nested_tensor.NestedTensor((video, audio))
|
||||
result["noise_mask"] = comfy.nested_tensor.NestedTensor(
|
||||
(video_mask, audio_mask)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def build_masked_video_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy a normalized video tail into the target and attach AV noise masks."""
|
||||
return _build_masked_prefix_latent(
|
||||
target_latent,
|
||||
video_context,
|
||||
context_frames,
|
||||
audio_context=None,
|
||||
)
|
||||
|
||||
|
||||
def build_masked_av_prefix_latent(
|
||||
target_latent: dict[str, Any],
|
||||
video_context: torch.Tensor,
|
||||
audio_context: torch.Tensor,
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Copy matching video/audio tails and preserve both with denoise masks."""
|
||||
|
||||
return _build_masked_prefix_latent(
|
||||
target_latent,
|
||||
video_context,
|
||||
context_frames,
|
||||
audio_context=audio_context,
|
||||
)
|
||||
|
||||
|
||||
def _restore_masked_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
*,
|
||||
restore_audio: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only protected prefix regions after sampler integration."""
|
||||
|
||||
try:
|
||||
import comfy.nested_tensor
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
|
||||
|
||||
sampled_video, sampled_audio = extract_av_streams(sampled_latent)
|
||||
target_video, target_audio = extract_av_streams(target_latent)
|
||||
slots = context_slots(int(context_frames))
|
||||
if tuple(sampled_video.shape) != tuple(target_video.shape):
|
||||
raise ValueError(
|
||||
"sampled video shape does not match the masked target: "
|
||||
f"sampled={tuple(sampled_video.shape)}, target={tuple(target_video.shape)}"
|
||||
)
|
||||
restored_video = sampled_video.clone()
|
||||
restored_video[:, :, :slots].copy_(target_video[:, :, :slots])
|
||||
restored_audio = sampled_audio
|
||||
if restore_audio:
|
||||
if tuple(sampled_audio.shape) != tuple(target_audio.shape):
|
||||
raise ValueError(
|
||||
"sampled audio shape does not match the masked target: "
|
||||
f"sampled={tuple(sampled_audio.shape)}, target={tuple(target_audio.shape)}"
|
||||
)
|
||||
audio_steps = audio_latent_t(int(context_frames))
|
||||
restored_audio = sampled_audio.clone()
|
||||
restored_audio[..., :audio_steps].copy_(target_audio[..., :audio_steps])
|
||||
result = dict(sampled_latent)
|
||||
result["samples"] = comfy.nested_tensor.NestedTensor(
|
||||
(restored_video, restored_audio)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def restore_masked_video_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only the preserved prefix after sampler integration rounding."""
|
||||
return _restore_masked_prefix(
|
||||
sampled_latent,
|
||||
target_latent,
|
||||
context_frames,
|
||||
restore_audio=False,
|
||||
)
|
||||
|
||||
|
||||
def restore_masked_av_prefix(
|
||||
sampled_latent: dict[str, Any],
|
||||
target_latent: dict[str, Any],
|
||||
context_frames: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Reapply only protected video and audio prefixes after Sampling."""
|
||||
|
||||
return _restore_masked_prefix(
|
||||
sampled_latent,
|
||||
target_latent,
|
||||
context_frames,
|
||||
restore_audio=True,
|
||||
)
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Explicit opt-in A7b memory actions that preserve Production ownership.
|
||||
|
||||
This module resolves only Reference sizing policy. Model unloading and
|
||||
attention/backend selection remain advisory because ComfyUI Core and external
|
||||
MODEL wrappers own those lifecycles.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from ..reference import REFERENCE_SIZE_MATCH_OUTPUT, REFERENCE_SIZE_MAX_IDENTITY
|
||||
from ..reference_video import (
|
||||
REFERENCE_VIDEO_SIZE_BALANCED,
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
|
||||
)
|
||||
|
||||
|
||||
MEMORY_ACTION_POLICY_TYPE = "H3_CONTINUUM_MEMORY_ACTION_POLICY"
|
||||
MEMORY_ACTION_POLICY_VERSION = 1
|
||||
|
||||
ACTION_DISABLED = "Disabled"
|
||||
ACTION_REDUCE_REFERENCES_ONE_STEP = "Reduce References One Step"
|
||||
ACTION_OPTIONS = (
|
||||
ACTION_DISABLED,
|
||||
ACTION_REDUCE_REFERENCES_ONE_STEP,
|
||||
)
|
||||
|
||||
_IMAGE_DOWNGRADE = {
|
||||
REFERENCE_SIZE_MAX_IDENTITY: REFERENCE_SIZE_MATCH_OUTPUT,
|
||||
REFERENCE_SIZE_MATCH_OUTPUT: REFERENCE_SIZE_MATCH_OUTPUT,
|
||||
}
|
||||
_VIDEO_DOWNGRADE = {
|
||||
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT: REFERENCE_VIDEO_SIZE_BALANCED,
|
||||
REFERENCE_VIDEO_SIZE_BALANCED: REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT: REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
}
|
||||
|
||||
|
||||
def _canonical_hash(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _contract_base(action: str) -> dict[str, Any]:
|
||||
return {
|
||||
"memory_action_policy_version": MEMORY_ACTION_POLICY_VERSION,
|
||||
"action": action,
|
||||
"reference_downgrade": (
|
||||
"one_step"
|
||||
if action == ACTION_REDUCE_REFERENCES_ONE_STEP
|
||||
else "disabled"
|
||||
),
|
||||
"auto_unload": "advisory_only",
|
||||
"backend_switch": "advisory_only",
|
||||
"same_run_actual_row_trigger": False,
|
||||
}
|
||||
|
||||
|
||||
def make_memory_action_policy(action: str = ACTION_DISABLED) -> dict[str, Any]:
|
||||
"""Build the versioned policy object emitted by the public helper node."""
|
||||
|
||||
normalized = str(action)
|
||||
if normalized not in ACTION_OPTIONS:
|
||||
raise ValueError(f"unsupported memory action {action!r}")
|
||||
contract = _contract_base(normalized)
|
||||
return {**contract, "contract_hash": _canonical_hash(contract)}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemoryActionDecision:
|
||||
action: str
|
||||
enabled: bool
|
||||
requested_image_mode: str
|
||||
effective_image_mode: str
|
||||
requested_video_mode: str
|
||||
effective_video_mode: str
|
||||
image_applied: bool
|
||||
video_applied: bool
|
||||
contract_hash: str | None
|
||||
warning: str | None = None
|
||||
|
||||
@property
|
||||
def applied(self) -> bool:
|
||||
return bool(self.image_applied or self.video_applied)
|
||||
|
||||
|
||||
def _disabled_decision(
|
||||
*,
|
||||
image_mode: str,
|
||||
video_mode: str,
|
||||
contract_hash: str | None = None,
|
||||
warning: str | None = None,
|
||||
) -> MemoryActionDecision:
|
||||
return MemoryActionDecision(
|
||||
action=ACTION_DISABLED,
|
||||
enabled=False,
|
||||
requested_image_mode=image_mode,
|
||||
effective_image_mode=image_mode,
|
||||
requested_video_mode=video_mode,
|
||||
effective_video_mode=video_mode,
|
||||
image_applied=False,
|
||||
video_applied=False,
|
||||
contract_hash=contract_hash,
|
||||
warning=warning,
|
||||
)
|
||||
|
||||
|
||||
def resolve_memory_action_policy(
|
||||
policy: Any,
|
||||
*,
|
||||
image_mode: str,
|
||||
video_mode: str,
|
||||
has_reference_images: bool,
|
||||
has_video_guide: bool,
|
||||
) -> MemoryActionDecision:
|
||||
"""Resolve an optional policy without changing tensors or runtime state.
|
||||
|
||||
Malformed or unsupported objects fail soft to Disabled. The caller may
|
||||
surface ``warning`` after generation, but Sampling inputs remain unchanged.
|
||||
"""
|
||||
|
||||
requested_image = str(image_mode)
|
||||
requested_video = str(video_mode)
|
||||
if policy is None:
|
||||
return _disabled_decision(
|
||||
image_mode=requested_image,
|
||||
video_mode=requested_video,
|
||||
)
|
||||
if not isinstance(policy, dict):
|
||||
return _disabled_decision(
|
||||
image_mode=requested_image,
|
||||
video_mode=requested_video,
|
||||
warning="invalid policy object; no memory action was applied",
|
||||
)
|
||||
|
||||
action = str(policy.get("action", ""))
|
||||
try:
|
||||
version = int(policy.get("memory_action_policy_version"))
|
||||
except (TypeError, ValueError):
|
||||
version = -1
|
||||
supplied_hash = policy.get("contract_hash")
|
||||
expected_contract = _contract_base(action) if action in ACTION_OPTIONS else None
|
||||
expected_hash = (
|
||||
_canonical_hash(expected_contract) if expected_contract is not None else None
|
||||
)
|
||||
if (
|
||||
version != MEMORY_ACTION_POLICY_VERSION
|
||||
or action not in ACTION_OPTIONS
|
||||
or not isinstance(supplied_hash, str)
|
||||
or supplied_hash != expected_hash
|
||||
):
|
||||
return _disabled_decision(
|
||||
image_mode=requested_image,
|
||||
video_mode=requested_video,
|
||||
warning="unsupported or modified policy contract; no memory action was applied",
|
||||
)
|
||||
if action == ACTION_DISABLED:
|
||||
return _disabled_decision(
|
||||
image_mode=requested_image,
|
||||
video_mode=requested_video,
|
||||
contract_hash=supplied_hash,
|
||||
)
|
||||
|
||||
effective_image = requested_image
|
||||
effective_video = requested_video
|
||||
if bool(has_reference_images):
|
||||
effective_image = _IMAGE_DOWNGRADE.get(requested_image, requested_image)
|
||||
if bool(has_video_guide):
|
||||
effective_video = _VIDEO_DOWNGRADE.get(requested_video, requested_video)
|
||||
return MemoryActionDecision(
|
||||
action=action,
|
||||
enabled=True,
|
||||
requested_image_mode=requested_image,
|
||||
effective_image_mode=effective_image,
|
||||
requested_video_mode=requested_video,
|
||||
effective_video_mode=effective_video,
|
||||
image_applied=(
|
||||
bool(has_reference_images) and effective_image != requested_image
|
||||
),
|
||||
video_applied=(bool(has_video_guide) and effective_video != requested_video),
|
||||
contract_hash=supplied_hash,
|
||||
)
|
||||
|
||||
|
||||
def format_memory_action_status(decision: MemoryActionDecision) -> str | None:
|
||||
"""Format post-generation status without claiming VRAM prediction."""
|
||||
|
||||
if decision.warning is not None:
|
||||
return "Memory Action Policy [Experimental A7b v1]: " + decision.warning + "."
|
||||
if not decision.enabled:
|
||||
return None
|
||||
changes: list[str] = []
|
||||
if decision.image_applied:
|
||||
changes.append(
|
||||
"Reference Image "
|
||||
f"{decision.requested_image_mode} -> {decision.effective_image_mode}"
|
||||
)
|
||||
if decision.video_applied:
|
||||
changes.append(
|
||||
"Video Guide "
|
||||
f"{decision.requested_video_mode} -> {decision.effective_video_mode}"
|
||||
)
|
||||
action = "; ".join(changes) if changes else "no applicable Reference downgrade"
|
||||
return (
|
||||
"Memory Action Policy [Experimental A7b v1]: explicit opt-in; "
|
||||
f"{action}.\n"
|
||||
"Memory advisory: automatic MODEL unload was not executed; Core retains "
|
||||
"lifecycle ownership.\n"
|
||||
"Memory advisory: automatic attention/backend switching was not executed; "
|
||||
"the external MODEL wrapper was preserved."
|
||||
)
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
"""V3.5-only, non-synchronizing memory attribution helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
import time
|
||||
from typing import Any, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import FPS
|
||||
from ..hardening import tensor_storage_stats
|
||||
from ..state import extract_av_streams
|
||||
from .plan import validate_assembly_plan
|
||||
|
||||
|
||||
_MIB = 1024**2
|
||||
|
||||
_CONDITIONING_SUBPHASES = (
|
||||
"conditioning_identity_video_vae",
|
||||
"conditioning_reference_image_vae",
|
||||
"conditioning_reference_audio_vae",
|
||||
"conditioning_driving_audio_vae",
|
||||
"conditioning_reference_video_vae",
|
||||
"conditioning_timeline_video_vae",
|
||||
"conditioning_prompt_clip",
|
||||
)
|
||||
|
||||
|
||||
def _mib(value: int | None) -> str:
|
||||
if value is None:
|
||||
return "n/a"
|
||||
return f"{int(value) / _MIB:.1f} MiB"
|
||||
|
||||
|
||||
def _bytes_and_mib(value: int) -> str:
|
||||
return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)"
|
||||
|
||||
|
||||
def _process_counters() -> tuple[int | None, int | None]:
|
||||
try:
|
||||
import psutil
|
||||
|
||||
return (
|
||||
int(psutil.Process().memory_info().rss),
|
||||
int(psutil.virtual_memory().available),
|
||||
)
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def _cuda_counters() -> tuple[int | None, int | None, int | None]:
|
||||
"""Read counters only; never synchronize, clear caches, or reset peaks."""
|
||||
|
||||
try:
|
||||
if torch.cuda.is_available():
|
||||
return (
|
||||
int(torch.cuda.memory_allocated()),
|
||||
int(torch.cuda.memory_reserved()),
|
||||
int(torch.cuda.max_memory_allocated()),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _retained_stream_bytes(
|
||||
entries: Any,
|
||||
stream: str,
|
||||
) -> int:
|
||||
values: list[Any] = []
|
||||
if isinstance(entries, (list, tuple)):
|
||||
for entry in entries:
|
||||
if isinstance(entry, Mapping):
|
||||
values.append(entry.get(stream))
|
||||
stats = tensor_storage_stats(values)
|
||||
return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"])
|
||||
|
||||
|
||||
def _observed_streams(latent: Any) -> tuple[Any, ...]:
|
||||
if latent is None:
|
||||
return ()
|
||||
if isinstance(latent, Mapping) and "samples" in latent:
|
||||
try:
|
||||
return tuple(extract_av_streams(latent))
|
||||
except Exception:
|
||||
return ()
|
||||
return (latent,)
|
||||
|
||||
|
||||
def format_attribution_snapshot(
|
||||
label: str,
|
||||
*,
|
||||
retained_entries: Any = None,
|
||||
observed_latent: Any = None,
|
||||
retained_refine_context: Any = None,
|
||||
) -> str:
|
||||
"""Format one fail-soft snapshot without changing allocator state."""
|
||||
try:
|
||||
rss, available = _process_counters()
|
||||
cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters()
|
||||
retained_video = _retained_stream_bytes(retained_entries, "video")
|
||||
retained_audio = _retained_stream_bytes(retained_entries, "audio")
|
||||
observed = tensor_storage_stats(_observed_streams(observed_latent))
|
||||
refine_context = tensor_storage_stats(retained_refine_context)
|
||||
return (
|
||||
f"Memory [{label}]: RSS={_mib(rss)}, "
|
||||
f"system_available={_mib(available)}, "
|
||||
f"CUDA_allocated={_mib(cuda_allocated)}, "
|
||||
f"CUDA_reserved={_mib(cuda_reserved)}, "
|
||||
f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, "
|
||||
f"retained_video={_bytes_and_mib(retained_video)}, "
|
||||
f"retained_audio={_bytes_and_mib(retained_audio)}, "
|
||||
"retained_refine_context_CPU="
|
||||
f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, "
|
||||
"retained_refine_context_GPU="
|
||||
f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, "
|
||||
f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, "
|
||||
f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}"
|
||||
)
|
||||
except Exception as exc:
|
||||
return (
|
||||
f"Memory [{label}]: unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
|
||||
class MemoryAttributionCollector:
|
||||
"""Keep V3.5 memory events in chronological order until report assembly."""
|
||||
|
||||
def __init__(self):
|
||||
self.lines: list[str] = []
|
||||
self._sequence_started_at: float | None = None
|
||||
self._phase_started_at: dict[tuple[str, int | None, tuple[int, ...]], float] = {}
|
||||
self._phase_totals: dict[str, float] = {}
|
||||
self._phase_counts: dict[str, int] = {}
|
||||
self._group_totals: dict[
|
||||
tuple[int, tuple[int, ...]], dict[str, float]
|
||||
] = {}
|
||||
|
||||
def capture_sequence_start(self) -> None:
|
||||
self._sequence_started_at = time.perf_counter()
|
||||
self.lines.append(format_attribution_snapshot("V3.5 sequence start"))
|
||||
self.start_phase(phase="preparation")
|
||||
|
||||
def start_phase(
|
||||
self,
|
||||
*,
|
||||
phase: str,
|
||||
physical_group: int | None = None,
|
||||
logical_chunks: tuple[int, ...] = (),
|
||||
) -> None:
|
||||
key = (
|
||||
str(phase),
|
||||
None if physical_group is None else int(physical_group),
|
||||
tuple(int(value) for value in logical_chunks),
|
||||
)
|
||||
self._phase_started_at[key] = time.perf_counter()
|
||||
|
||||
def finish_phase(
|
||||
self,
|
||||
*,
|
||||
phase: str,
|
||||
physical_group: int | None = None,
|
||||
logical_chunks: tuple[int, ...] = (),
|
||||
) -> None:
|
||||
phase = str(phase)
|
||||
group = None if physical_group is None else int(physical_group)
|
||||
logical = tuple(int(value) for value in logical_chunks)
|
||||
key = (phase, group, logical)
|
||||
started = self._phase_started_at.pop(key, None)
|
||||
if started is None:
|
||||
return
|
||||
elapsed = time.perf_counter() - started
|
||||
if not math.isfinite(elapsed) or elapsed < 0.0:
|
||||
return
|
||||
self._phase_totals[phase] = self._phase_totals.get(phase, 0.0) + elapsed
|
||||
self._phase_counts[phase] = self._phase_counts.get(phase, 0) + 1
|
||||
if group is not None:
|
||||
phases = self._group_totals.setdefault((group, logical), {})
|
||||
phases[phase] = phases.get(phase, 0.0) + elapsed
|
||||
|
||||
@staticmethod
|
||||
def _format_seconds(value: float | None) -> str:
|
||||
return "n/a" if value is None else f"{float(value):.6f}s"
|
||||
|
||||
def _performance_lines(self) -> list[str]:
|
||||
now = time.perf_counter()
|
||||
host_wall = None
|
||||
if self._sequence_started_at is not None:
|
||||
candidate = now - self._sequence_started_at
|
||||
if math.isfinite(candidate) and candidate >= 0.0:
|
||||
host_wall = candidate
|
||||
ordered = (
|
||||
"preparation",
|
||||
"conditioning",
|
||||
*_CONDITIONING_SUBPHASES,
|
||||
"group_prepare",
|
||||
"sampling",
|
||||
"cpu_commit_validation",
|
||||
"refine_context",
|
||||
"finalization",
|
||||
)
|
||||
lines = []
|
||||
for (group, logical), phases in sorted(self._group_totals.items()):
|
||||
logical_label = ",".join(str(value) for value in logical)
|
||||
fields = ", ".join(
|
||||
f"{name}={self._format_seconds(phases.get(name))}"
|
||||
for name in (
|
||||
"group_prepare",
|
||||
"sampling",
|
||||
"cpu_commit_validation",
|
||||
"refine_context",
|
||||
)
|
||||
)
|
||||
lines.append(
|
||||
f"Performance [V3.5 physical group {group} "
|
||||
f"logical=[{logical_label}]]: {fields}"
|
||||
)
|
||||
totals = ", ".join(
|
||||
f"{name}={self._format_seconds(self._phase_totals.get(name))}"
|
||||
for name in ordered
|
||||
)
|
||||
counts = ", ".join(
|
||||
f"{name}={int(self._phase_counts.get(name, 0))}"
|
||||
for name in (
|
||||
"group_prepare",
|
||||
"sampling",
|
||||
"cpu_commit_validation",
|
||||
"refine_context",
|
||||
)
|
||||
)
|
||||
conditioning_counts = ", ".join(
|
||||
f"{name}={int(self._phase_counts.get(name, 0))}"
|
||||
for name in _CONDITIONING_SUBPHASES
|
||||
)
|
||||
lines.append(
|
||||
"Performance [V3.5 summary]: "
|
||||
f"host_wall_including_diagnostics={self._format_seconds(host_wall)}, "
|
||||
f"{totals}; "
|
||||
f"phase_counts({counts}); "
|
||||
f"conditioning_counts({conditioning_counts}); "
|
||||
"CUDA_synchronize=not_used"
|
||||
)
|
||||
return lines
|
||||
|
||||
def capture_group(
|
||||
self,
|
||||
*,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
stage: str,
|
||||
retained_entries: Any,
|
||||
observed_latent: Any = None,
|
||||
retained_refine_context: Any = None,
|
||||
) -> None:
|
||||
logical = ",".join(str(int(value)) for value in logical_chunks)
|
||||
self.lines.append(
|
||||
format_attribution_snapshot(
|
||||
f"V3.5 physical group {int(physical_group)} "
|
||||
f"logical=[{logical}] {stage}",
|
||||
retained_entries=retained_entries,
|
||||
observed_latent=observed_latent,
|
||||
retained_refine_context=retained_refine_context,
|
||||
)
|
||||
)
|
||||
|
||||
def capture_sequence_complete(
|
||||
self,
|
||||
entries: Any,
|
||||
retained_refine_context: Any = None,
|
||||
) -> None:
|
||||
self.lines.extend(self._performance_lines())
|
||||
self.lines.append(
|
||||
format_attribution_snapshot(
|
||||
"V3.5 sequence complete",
|
||||
retained_entries=entries,
|
||||
retained_refine_context=retained_refine_context,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def capture_attribution_fail_soft(
|
||||
collector: Any,
|
||||
method: str,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""A diagnostics failure must never replace a sampling result."""
|
||||
|
||||
try:
|
||||
getattr(collector, method)(**kwargs)
|
||||
except Exception as exc:
|
||||
try:
|
||||
collector.lines.append(
|
||||
f"Memory attribution [{method}]: unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssemblyBufferProjection:
|
||||
natural_frames: int
|
||||
target_frames: int
|
||||
natural_image_bytes: int
|
||||
natural_audio_bytes: int
|
||||
final_image_bytes: int
|
||||
final_audio_bytes: int
|
||||
frame_count_adjustment_required: bool
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[Any]:
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1 and isinstance(value[0], list):
|
||||
return list(value[0])
|
||||
return list(value)
|
||||
if isinstance(value, tuple):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def _singleton(value: Any, name: str) -> Any:
|
||||
while isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
value = value[0]
|
||||
return value
|
||||
|
||||
|
||||
def project_assembly_buffers(
|
||||
*,
|
||||
images: Any,
|
||||
audio: Any,
|
||||
assembly_plan: Any,
|
||||
exact_total_duration: Any,
|
||||
) -> AssemblyBufferProjection:
|
||||
"""Calculate natural and final buffer sizes without allocating a buffer."""
|
||||
|
||||
plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan"))
|
||||
image_chunks = _as_list(images)
|
||||
audio_chunks = _as_list(audio)
|
||||
if not image_chunks or not audio_chunks:
|
||||
raise ValueError("decoded image/audio chunks are required for projection")
|
||||
|
||||
first_image = _singleton(image_chunks[0], "decoded image chunk")
|
||||
if not torch.is_tensor(first_image) or first_image.ndim != 4:
|
||||
raise ValueError("decoded image chunk must be [frames,H,W,C]")
|
||||
frame_shape = tuple(int(value) for value in first_image.shape[1:])
|
||||
for index, value in enumerate(image_chunks, start=1):
|
||||
tensor = _singleton(value, f"decoded image chunk {index}")
|
||||
if (
|
||||
not torch.is_tensor(tensor)
|
||||
or tensor.ndim != 4
|
||||
or tuple(int(item) for item in tensor.shape[1:]) != frame_shape
|
||||
or tensor.dtype != first_image.dtype
|
||||
):
|
||||
raise ValueError("decoded image geometry or dtype changed between chunks")
|
||||
|
||||
first_audio = _singleton(audio_chunks[0], "decoded audio chunk")
|
||||
if not isinstance(first_audio, Mapping):
|
||||
raise ValueError("decoded audio chunk must be an AUDIO mapping")
|
||||
waveform = first_audio.get("waveform")
|
||||
sample_rate = first_audio.get("sample_rate")
|
||||
if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None:
|
||||
raise ValueError("decoded audio chunk has an invalid waveform or sample rate")
|
||||
sample_rate = int(sample_rate)
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("decoded audio sample rate must be positive")
|
||||
|
||||
decode_units = list(plan.get("decode_groups") or plan["chunks"])
|
||||
natural_frames = sum(int(unit["net_frames"]) for unit in decode_units)
|
||||
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
|
||||
target_frames = int(plan["target_frames"]) if exact else natural_frames
|
||||
if natural_frames <= 0 or target_frames <= 0:
|
||||
raise ValueError("assembly frame counts must be positive")
|
||||
|
||||
frame_elements = math.prod(frame_shape)
|
||||
audio_streams = math.prod(int(value) for value in waveform.shape[:-1])
|
||||
natural_samples = int(round(natural_frames / FPS * sample_rate))
|
||||
target_samples = int(round(target_frames / FPS * sample_rate))
|
||||
return AssemblyBufferProjection(
|
||||
natural_frames=natural_frames,
|
||||
target_frames=target_frames,
|
||||
natural_image_bytes=(
|
||||
natural_frames * frame_elements * first_image.element_size()
|
||||
),
|
||||
natural_audio_bytes=(
|
||||
natural_samples * audio_streams * waveform.element_size()
|
||||
),
|
||||
final_image_bytes=(
|
||||
target_frames * frame_elements * first_image.element_size()
|
||||
),
|
||||
final_audio_bytes=(
|
||||
target_samples * audio_streams * waveform.element_size()
|
||||
),
|
||||
frame_count_adjustment_required=bool(
|
||||
exact and target_frames != natural_frames
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def format_assembly_projection(projection: AssemblyBufferProjection) -> str:
|
||||
return (
|
||||
"Projected assembly buffers [V3.5]: "
|
||||
f"natural_frames={projection.natural_frames}, "
|
||||
f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, "
|
||||
"projected_natural_generated_AUDIO="
|
||||
f"{_bytes_and_mib(projection.natural_audio_bytes)}, "
|
||||
f"target_frames={projection.target_frames}, "
|
||||
f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, "
|
||||
f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, "
|
||||
"frame_count_adjustment_required="
|
||||
f"{'yes' if projection.frame_count_adjustment_required else 'no'}"
|
||||
)
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
"""Thin V3.8 public-node to runtime boundary.
|
||||
|
||||
This module deliberately owns no continuation, review, storage, planning, or
|
||||
sampling decision. It only carries already-resolved public-node values into
|
||||
the existing hardening/runtime call and preserves its public return value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
from ..reference import resolve_reference_image_inputs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeRequest:
|
||||
"""Resolved V3.8 node inputs for one existing runtime invocation.
|
||||
|
||||
Tensor and model objects deliberately keep their identity. Only the two
|
||||
keyword containers are copied and frozen so the facade cannot mutate the
|
||||
caller's widget/input mapping after the request boundary.
|
||||
"""
|
||||
|
||||
runtime_kwargs: Mapping[str, Any]
|
||||
resolution: Any
|
||||
size_source: str
|
||||
diagnostics_mode: str
|
||||
diagnostics_inputs: Mapping[str, Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"runtime_kwargs",
|
||||
MappingProxyType(dict(self.runtime_kwargs)),
|
||||
)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"diagnostics_inputs",
|
||||
MappingProxyType(dict(self.diagnostics_inputs)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeOutput:
|
||||
"""Existing runtime result with status-only facade decoration helpers."""
|
||||
|
||||
value: Any
|
||||
|
||||
@property
|
||||
def has_status(self) -> bool:
|
||||
return isinstance(self.value, tuple) and len(self.value) >= 4
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
if not self.has_status:
|
||||
raise TypeError("runtime output has no public status field")
|
||||
return str(self.value[3])
|
||||
|
||||
def with_status(self, status: str) -> "RuntimeOutput":
|
||||
if not self.has_status:
|
||||
return self
|
||||
return RuntimeOutput((*self.value[:3], str(status), *self.value[4:]))
|
||||
|
||||
def as_public_tuple(self) -> Any:
|
||||
"""Return the unchanged nonstandard test double or public tuple."""
|
||||
|
||||
return self.value
|
||||
|
||||
|
||||
def make_runtime_request(
|
||||
*,
|
||||
runtime_kwargs: Mapping[str, Any],
|
||||
fixed_runtime_kwargs: Mapping[str, Any],
|
||||
resolution: Any,
|
||||
size_source: str,
|
||||
diagnostics_mode: str,
|
||||
diagnostics_inputs: Mapping[str, Any],
|
||||
) -> RuntimeRequest:
|
||||
"""Build the one R6 request without silently accepting duplicate inputs."""
|
||||
|
||||
duplicate = set(runtime_kwargs).intersection(fixed_runtime_kwargs)
|
||||
if duplicate:
|
||||
name = sorted(duplicate)[0]
|
||||
raise TypeError(
|
||||
"H3 Continuum Sampler V3.8 received multiple values for "
|
||||
f"runtime keyword {name!r}"
|
||||
)
|
||||
merged = dict(runtime_kwargs)
|
||||
merged.update(fixed_runtime_kwargs)
|
||||
return RuntimeRequest(
|
||||
runtime_kwargs=merged,
|
||||
resolution=resolution,
|
||||
size_source=str(size_source),
|
||||
diagnostics_mode=str(diagnostics_mode),
|
||||
diagnostics_inputs=diagnostics_inputs,
|
||||
)
|
||||
|
||||
|
||||
def execute_v38_runtime_request(
|
||||
request: RuntimeRequest,
|
||||
*,
|
||||
runtime_adapter: Callable[..., Any],
|
||||
legacy_size_source: str,
|
||||
diagnostics_off: str,
|
||||
diagnostics_full: str,
|
||||
reference_video_size_default: str,
|
||||
build_diagnostics: Callable[..., Any],
|
||||
append_status: Callable[..., str],
|
||||
) -> Any:
|
||||
"""Call the existing runtime once and add V3.8 display-only diagnostics."""
|
||||
|
||||
output = RuntimeOutput(runtime_adapter(**request.runtime_kwargs))
|
||||
if not output.has_status:
|
||||
return output.as_public_tuple()
|
||||
|
||||
if request.size_source != legacy_size_source:
|
||||
resolution = request.resolution
|
||||
lines = [
|
||||
(
|
||||
f"Resolution: {resolution.width} x {resolution.height} "
|
||||
f"({resolution.actual_mp:.2f} MP); source={resolution.aspect_source}."
|
||||
)
|
||||
]
|
||||
lines.extend(resolution.warnings)
|
||||
output = output.with_status(output.status.rstrip() + "\n" + "\n".join(lines))
|
||||
|
||||
if request.diagnostics_mode == diagnostics_off:
|
||||
return output.as_public_tuple()
|
||||
|
||||
# The V3.8 reliability summary counts every connected Reference image.
|
||||
# A Custom V3.9 group intentionally prepares only its selected images, so
|
||||
# that legacy summary would describe a different conditioning path.
|
||||
if "_reference_routing_settings" in request.runtime_kwargs:
|
||||
return output.as_public_tuple()
|
||||
|
||||
try:
|
||||
inputs = request.diagnostics_inputs
|
||||
diagnostics = build_diagnostics(
|
||||
video_latents=output.value[0],
|
||||
audio_latents=output.value[1],
|
||||
assembly_plan=output.value[2],
|
||||
output_width=request.resolution.width,
|
||||
output_height=request.resolution.height,
|
||||
chunk_seconds=float(inputs["chunk_seconds"]),
|
||||
reference_images=resolve_reference_image_inputs(
|
||||
inputs.get("reference_image_1"),
|
||||
inputs.get("reference_image_2"),
|
||||
inputs.get("reference_image_3"),
|
||||
inputs.get("reference_image_4"),
|
||||
inputs.get("reference_image_5"),
|
||||
inputs.get("image_references"),
|
||||
),
|
||||
reference_size=str(inputs.get("reference_size", "Match Output")),
|
||||
video_guide=inputs.get("reference_video_1"),
|
||||
video_guide_size=str(
|
||||
inputs.get("video_reference_size", reference_video_size_default)
|
||||
),
|
||||
still_guide_active=inputs.get("guide") is not None,
|
||||
)
|
||||
status = append_status(
|
||||
output.status,
|
||||
diagnostics,
|
||||
mode=request.diagnostics_mode,
|
||||
)
|
||||
except Exception as exc:
|
||||
message = (
|
||||
"V3.8 Reliability\n"
|
||||
"Reliability diagnostics unavailable; generation result was preserved."
|
||||
)
|
||||
if request.diagnostics_mode == diagnostics_full:
|
||||
message += f" ({type(exc).__name__}: {exc})"
|
||||
status = output.status.rstrip() + "\n" + message
|
||||
return output.with_status(status).as_public_tuple()
|
||||
+1350
File diff suppressed because it is too large
Load Diff
+349
@@ -0,0 +1,349 @@
|
||||
"""Read-only A7a planning from actual ComfyUI H3 PackedLayout rows.
|
||||
|
||||
The planner observes layouts that ComfyUI Core already built for Sampling. It
|
||||
does not estimate VRAM, change execution policy, mutate a layout, or influence
|
||||
the physical-group order. Memory counters are point-in-time observations only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
PACKED_ROW_PLAN_VERSION = 1
|
||||
MAX_CONCURRENT_PHYSICAL_GROUPS = 1
|
||||
|
||||
_VIDEO_CONDITION_KINDS = frozenset(("cond", "ref_img"))
|
||||
_AUDIO_CONDITION_KINDS = frozenset(("cond_audio", "ref_audio"))
|
||||
|
||||
|
||||
class PackedRowPlanningError(ValueError):
|
||||
"""An observed object is not an internally consistent H3 PackedLayout."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackedRowBreakdown:
|
||||
seq_len: int
|
||||
text_rows: int
|
||||
condition_video_rows: int
|
||||
condition_audio_rows: int
|
||||
target_video_rows: int
|
||||
target_audio_rows: int
|
||||
unknown_rows: int
|
||||
segment_count: int
|
||||
segment_signature: tuple[tuple[int, int, str], ...]
|
||||
|
||||
@property
|
||||
def condition_rows(self) -> int:
|
||||
return int(self.condition_video_rows + self.condition_audio_rows)
|
||||
|
||||
@property
|
||||
def target_rows(self) -> int:
|
||||
return int(self.target_video_rows + self.target_audio_rows)
|
||||
|
||||
@property
|
||||
def condition_to_target_ratio(self) -> float | None:
|
||||
if self.target_rows <= 0:
|
||||
return None
|
||||
return float(self.condition_rows / self.target_rows)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MemoryObservation:
|
||||
system_available_bytes: int | None = None
|
||||
cuda_allocated_bytes: int | None = None
|
||||
cuda_reserved_bytes: int | None = None
|
||||
core_visible_free_bytes: int | None = None
|
||||
|
||||
|
||||
def analyze_packed_layout(layout: Any) -> PackedRowBreakdown:
|
||||
"""Account actual rows without importing or recreating Core PackedLayout."""
|
||||
|
||||
if layout is None:
|
||||
raise PackedRowPlanningError("Core PackedLayout is unavailable")
|
||||
try:
|
||||
seq_len = int(layout.seq_len)
|
||||
raw_segments = tuple(layout.segments)
|
||||
except Exception as exc:
|
||||
raise PackedRowPlanningError(
|
||||
"Core PackedLayout does not expose seq_len/segments"
|
||||
) from exc
|
||||
if seq_len <= 0 or not raw_segments:
|
||||
raise PackedRowPlanningError("Core PackedLayout is empty")
|
||||
|
||||
signature: list[tuple[int, int, str]] = []
|
||||
previous_stop = 0
|
||||
counts = {
|
||||
"text": 0,
|
||||
"condition_video": 0,
|
||||
"condition_audio": 0,
|
||||
"target_video": 0,
|
||||
"target_audio": 0,
|
||||
"unknown": 0,
|
||||
}
|
||||
target_video_segments = 0
|
||||
target_audio_segments = 0
|
||||
for item in raw_segments:
|
||||
if not isinstance(item, (tuple, list)) or len(item) != 3:
|
||||
raise PackedRowPlanningError("PackedLayout segment is not (start, stop, kind)")
|
||||
start, stop, kind = int(item[0]), int(item[1]), str(item[2])
|
||||
if start != previous_stop or stop <= start or stop > seq_len:
|
||||
raise PackedRowPlanningError(
|
||||
"PackedLayout segments are not a contiguous in-range partition"
|
||||
)
|
||||
rows = stop - start
|
||||
signature.append((start, stop, kind))
|
||||
if kind == "text":
|
||||
counts["text"] += rows
|
||||
elif kind in _VIDEO_CONDITION_KINDS:
|
||||
counts["condition_video"] += rows
|
||||
elif kind in _AUDIO_CONDITION_KINDS:
|
||||
counts["condition_audio"] += rows
|
||||
elif kind == "video":
|
||||
counts["target_video"] += rows
|
||||
target_video_segments += 1
|
||||
elif kind == "audio":
|
||||
counts["target_audio"] += rows
|
||||
target_audio_segments += 1
|
||||
else:
|
||||
counts["unknown"] += rows
|
||||
previous_stop = stop
|
||||
if previous_stop != seq_len:
|
||||
raise PackedRowPlanningError("PackedLayout segment rows do not sum to seq_len")
|
||||
if target_video_segments != 1 or target_audio_segments != 1:
|
||||
raise PackedRowPlanningError(
|
||||
"H3 PackedLayout must contain exactly one target Video and Audio segment"
|
||||
)
|
||||
return PackedRowBreakdown(
|
||||
seq_len=seq_len,
|
||||
text_rows=int(counts["text"]),
|
||||
condition_video_rows=int(counts["condition_video"]),
|
||||
condition_audio_rows=int(counts["condition_audio"]),
|
||||
target_video_rows=int(counts["target_video"]),
|
||||
target_audio_rows=int(counts["target_audio"]),
|
||||
unknown_rows=int(counts["unknown"]),
|
||||
segment_count=len(signature),
|
||||
segment_signature=tuple(signature),
|
||||
)
|
||||
|
||||
|
||||
def read_memory_observation() -> MemoryObservation:
|
||||
"""Read counters without CUDA synchronization, cache clearing, or peak reset."""
|
||||
|
||||
system_available = None
|
||||
cuda_allocated = None
|
||||
cuda_reserved = None
|
||||
core_free = None
|
||||
try:
|
||||
import psutil
|
||||
|
||||
system_available = int(psutil.virtual_memory().available)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
cuda_allocated = int(torch.cuda.memory_allocated())
|
||||
cuda_reserved = int(torch.cuda.memory_reserved())
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import comfy.model_management as model_management
|
||||
|
||||
device = model_management.get_torch_device()
|
||||
core_free = int(model_management.get_free_memory(device))
|
||||
except Exception:
|
||||
pass
|
||||
return MemoryObservation(
|
||||
system_available_bytes=system_available,
|
||||
cuda_allocated_bytes=cuda_allocated,
|
||||
cuda_reserved_bytes=cuda_reserved,
|
||||
core_visible_free_bytes=core_free,
|
||||
)
|
||||
|
||||
|
||||
def _safe_memory_read(reader: Callable[[], MemoryObservation]) -> MemoryObservation:
|
||||
try:
|
||||
value = reader()
|
||||
return value if isinstance(value, MemoryObservation) else MemoryObservation()
|
||||
except Exception:
|
||||
return MemoryObservation()
|
||||
|
||||
|
||||
class PackedRowPlanner:
|
||||
"""Collect one immutable actual-layout plan record per sampled group."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
memory_reader: Callable[[], MemoryObservation] = read_memory_observation,
|
||||
) -> None:
|
||||
self._memory_reader = memory_reader
|
||||
self._active: dict[str, Any] | None = None
|
||||
self._records: list[dict[str, Any]] = []
|
||||
self._advisories: list[str] = []
|
||||
|
||||
@property
|
||||
def records(self) -> tuple[dict[str, Any], ...]:
|
||||
return tuple(self._records)
|
||||
|
||||
def begin_group(
|
||||
self,
|
||||
*,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
terminal_atomic: bool,
|
||||
) -> None:
|
||||
if self._active is not None:
|
||||
self._advisories.append(
|
||||
"a group began before the prior observation was finalized"
|
||||
)
|
||||
self.finish_group()
|
||||
self._active = {
|
||||
"physical_group": int(physical_group),
|
||||
"logical_chunks": tuple(int(value) for value in logical_chunks),
|
||||
"terminal_atomic": bool(terminal_atomic),
|
||||
"pre_memory": _safe_memory_read(self._memory_reader),
|
||||
"breakdown": None,
|
||||
"unavailable": None,
|
||||
}
|
||||
|
||||
def observe_layout(self, layout: Any) -> None:
|
||||
if self._active is None:
|
||||
return
|
||||
try:
|
||||
observed = analyze_packed_layout(layout)
|
||||
except Exception as exc:
|
||||
if self._active["breakdown"] is None:
|
||||
self._active["unavailable"] = f"{type(exc).__name__}: {exc}"
|
||||
return
|
||||
current = self._active["breakdown"]
|
||||
if current is not None and current != observed:
|
||||
self._active["unavailable"] = (
|
||||
"PackedLayout topology changed within one physical group"
|
||||
)
|
||||
self._active["breakdown"] = None
|
||||
return
|
||||
if self._active["unavailable"] is None:
|
||||
self._active["breakdown"] = observed
|
||||
|
||||
def note_observation_failure(self, exc: Exception) -> None:
|
||||
if self._active is not None and self._active["breakdown"] is None:
|
||||
self._active["unavailable"] = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
def finish_group(self) -> None:
|
||||
if self._active is None:
|
||||
return
|
||||
record = dict(self._active)
|
||||
record["post_memory"] = _safe_memory_read(self._memory_reader)
|
||||
if record["breakdown"] is None and record["unavailable"] is None:
|
||||
record["unavailable"] = "Core PackedLayout was not observed"
|
||||
self._records.append(record)
|
||||
self._active = None
|
||||
|
||||
@staticmethod
|
||||
def _mib(value: int | None) -> str:
|
||||
return "n/a" if value is None else f"{int(value) / 1024**2:.1f} MiB"
|
||||
|
||||
def report_lines(self) -> list[str]:
|
||||
if self._active is not None:
|
||||
self.finish_group()
|
||||
valid = [record for record in self._records if record["breakdown"] is not None]
|
||||
peak = None
|
||||
if valid:
|
||||
peak = max(valid, key=lambda record: record["breakdown"].seq_len)
|
||||
order = [int(record["physical_group"]) for record in self._records]
|
||||
lines = [
|
||||
"Packed-row plan [A7a v1]: "
|
||||
f"execution_order={order}, "
|
||||
f"max_concurrent_physical_groups={MAX_CONCURRENT_PHYSICAL_GROUPS}, "
|
||||
f"observed_groups={len(valid)}/{len(self._records)}, "
|
||||
f"peak_candidate={int(peak['physical_group']) if peak else 'unavailable'}, "
|
||||
f"peak_total_rows={peak['breakdown'].seq_len if peak else 'unavailable'}; "
|
||||
"actual Core rows only, VRAM/time prediction=not provided, "
|
||||
"execution policy=unchanged."
|
||||
]
|
||||
for record in self._records:
|
||||
group = int(record["physical_group"])
|
||||
logical = list(record["logical_chunks"])
|
||||
breakdown = record["breakdown"]
|
||||
if breakdown is None:
|
||||
lines.append(
|
||||
"Packed-row group [A7a v1]: "
|
||||
f"physical_group={group}, logical_chunks={logical}, "
|
||||
f"terminal_atomic={str(bool(record['terminal_atomic'])).lower()}, "
|
||||
f"unavailable/advisory={record['unavailable']}."
|
||||
)
|
||||
continue
|
||||
pre = record["pre_memory"]
|
||||
post = record["post_memory"]
|
||||
ratio = breakdown.condition_to_target_ratio
|
||||
ratio_text = "n/a" if ratio is None else f"{ratio:.6f}"
|
||||
lines.append(
|
||||
"Packed-row group [A7a v1]: "
|
||||
f"physical_group={group}, logical_chunks={logical}, "
|
||||
f"terminal_atomic={str(bool(record['terminal_atomic'])).lower()}, "
|
||||
f"total={breakdown.seq_len}, text={breakdown.text_rows}, "
|
||||
f"condition_video={breakdown.condition_video_rows}, "
|
||||
f"condition_audio={breakdown.condition_audio_rows}, "
|
||||
f"target_video={breakdown.target_video_rows}, "
|
||||
f"target_audio={breakdown.target_audio_rows}, "
|
||||
f"unknown={breakdown.unknown_rows}, "
|
||||
f"condition_to_target_ratio={ratio_text}, "
|
||||
f"pre_system_available={self._mib(pre.system_available_bytes)}, "
|
||||
f"pre_CUDA_allocated={self._mib(pre.cuda_allocated_bytes)}, "
|
||||
f"pre_CUDA_reserved={self._mib(pre.cuda_reserved_bytes)}, "
|
||||
f"pre_Core_free={self._mib(pre.core_visible_free_bytes)}, "
|
||||
f"post_system_available={self._mib(post.system_available_bytes)}, "
|
||||
f"post_CUDA_allocated={self._mib(post.cuda_allocated_bytes)}, "
|
||||
f"post_CUDA_reserved={self._mib(post.cuda_reserved_bytes)}, "
|
||||
f"post_Core_free={self._mib(post.core_visible_free_bytes)}."
|
||||
)
|
||||
lines.extend(f"Packed-row advisory [A7a v1]: {item}." for item in self._advisories)
|
||||
return lines
|
||||
|
||||
|
||||
def observe_layout_fail_soft(planner: Any, payload: Any) -> None:
|
||||
"""Diagnostics must never replace a Sampling result."""
|
||||
|
||||
if planner is None:
|
||||
return
|
||||
try:
|
||||
layout = payload.get("layout") if isinstance(payload, dict) else None
|
||||
planner.observe_layout(layout)
|
||||
except Exception as exc:
|
||||
try:
|
||||
planner.note_observation_failure(exc)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def planner_event_fail_soft(planner: Any, method: str, **kwargs: Any) -> Any:
|
||||
"""Invoke one lifecycle event without allowing diagnostics to stop Sampling."""
|
||||
|
||||
if planner is None:
|
||||
return None
|
||||
try:
|
||||
return getattr(planner, str(method))(**kwargs)
|
||||
except Exception as exc:
|
||||
try:
|
||||
planner.note_observation_failure(exc)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_CONCURRENT_PHYSICAL_GROUPS",
|
||||
"MemoryObservation",
|
||||
"PACKED_ROW_PLAN_VERSION",
|
||||
"PackedRowBreakdown",
|
||||
"PackedRowPlanner",
|
||||
"PackedRowPlanningError",
|
||||
"analyze_packed_layout",
|
||||
"observe_layout_fail_soft",
|
||||
"planner_event_fail_soft",
|
||||
"read_memory_observation",
|
||||
]
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
"""Sequential pixel/VAE resize for Continuum physical video groups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .plan import SECOND_PASS_CONTRACT_VERSION
|
||||
|
||||
|
||||
PIXEL_UPSCALE_METHODS = (
|
||||
"nearest-exact",
|
||||
"bilinear",
|
||||
"area",
|
||||
"bicubic",
|
||||
"bislerp",
|
||||
"lanczos",
|
||||
)
|
||||
_PIXEL_RESIZE_FRAME_BATCH = 8
|
||||
|
||||
|
||||
class PixelVaeResizeError(ValueError):
|
||||
"""Raised when a pixel/VAE resize would violate the physical-group contract."""
|
||||
|
||||
|
||||
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise PixelVaeResizeError("assembly_plan must be a mapping")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise PixelVaeResizeError("assembly_plan has no second_pass_contract")
|
||||
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
|
||||
raise PixelVaeResizeError("unsupported second_pass_contract version")
|
||||
groups = contract.get("physical_groups")
|
||||
if (
|
||||
not isinstance(groups, list)
|
||||
or not groups
|
||||
or not all(isinstance(group, Mapping) for group in groups)
|
||||
):
|
||||
raise PixelVaeResizeError(
|
||||
"second_pass_contract.physical_groups must be non-empty"
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def resolve_pixel_latent_scale(
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> tuple[int, int]:
|
||||
"""Return plan-derived pixel-per-latent ratios as ``(width, height)``."""
|
||||
|
||||
groups = _contract_groups(assembly_plan)
|
||||
scale: tuple[int, int] | None = None
|
||||
for position, group in enumerate(groups):
|
||||
if int(group.get("group_id", -1)) != position:
|
||||
raise PixelVaeResizeError("physical group order or group_id changed")
|
||||
source_width = int(group.get("source_width", -1))
|
||||
source_height = int(group.get("source_height", -1))
|
||||
source_latent_width = int(group.get("source_latent_w", -1))
|
||||
source_latent_height = int(group.get("source_latent_h", -1))
|
||||
if min(
|
||||
source_width,
|
||||
source_height,
|
||||
source_latent_width,
|
||||
source_latent_height,
|
||||
) < 1:
|
||||
raise PixelVaeResizeError("source pixel/latent geometry is invalid")
|
||||
if (
|
||||
source_width % source_latent_width
|
||||
or source_height % source_latent_height
|
||||
):
|
||||
raise PixelVaeResizeError(
|
||||
"source pixel/latent geometry is not integral"
|
||||
)
|
||||
group_scale = (
|
||||
source_width // source_latent_width,
|
||||
source_height // source_latent_height,
|
||||
)
|
||||
if scale is None:
|
||||
scale = group_scale
|
||||
elif scale != group_scale:
|
||||
raise PixelVaeResizeError(
|
||||
"source pixel/latent scale differs between physical groups"
|
||||
)
|
||||
|
||||
assert scale is not None
|
||||
return scale
|
||||
|
||||
|
||||
def _latent_samples(
|
||||
latent: Any,
|
||||
*,
|
||||
position: int,
|
||||
group: Mapping[str, Any],
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(latent, Mapping) or "samples" not in latent:
|
||||
raise PixelVaeResizeError(
|
||||
f"video_latents[{position}] must be a LATENT mapping with samples"
|
||||
)
|
||||
samples = latent["samples"]
|
||||
if not isinstance(samples, torch.Tensor) or samples.ndim != 5:
|
||||
raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]")
|
||||
batch, channels, temporal, height, width = (
|
||||
int(value) for value in samples.shape
|
||||
)
|
||||
expected = (
|
||||
int(group.get("source_batch", -1)),
|
||||
int(group.get("latent_channels", -1)),
|
||||
int(group.get("source_latent_t", -1)),
|
||||
int(group.get("source_latent_h", -1)),
|
||||
int(group.get("source_latent_w", -1)),
|
||||
)
|
||||
if (batch, channels, temporal, height, width) != expected:
|
||||
raise PixelVaeResizeError(
|
||||
"video latent shape or physical group order changed"
|
||||
)
|
||||
if channels != 24:
|
||||
raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels")
|
||||
if not bool(torch.isfinite(samples).all()):
|
||||
raise PixelVaeResizeError("input video latent contains NaN or Inf")
|
||||
return samples
|
||||
|
||||
|
||||
def _resize_one_group(
|
||||
samples: torch.Tensor,
|
||||
*,
|
||||
video_vae: Any,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
target_latent_width: int,
|
||||
target_latent_height: int,
|
||||
upscale_method: str,
|
||||
upscale_fn: Callable[..., Any],
|
||||
) -> torch.Tensor:
|
||||
"""Decode, resize, and encode one group so its pixel buffer dies per call."""
|
||||
|
||||
decoded_pixels = video_vae.decode(samples)
|
||||
if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode must return pixels with shape [B,F,H,W,C]"
|
||||
)
|
||||
batch, frames, pixel_height, pixel_width, channels = (
|
||||
int(value) for value in decoded_pixels.shape
|
||||
)
|
||||
if batch != int(samples.shape[0]):
|
||||
raise PixelVaeResizeError("Video VAE decode changed the batch size")
|
||||
if frames < 1 or channels != 3:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode returned invalid frame or channel geometry"
|
||||
)
|
||||
if (pixel_width, pixel_height) != (source_width, source_height):
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE decode geometry differs from the assembly plan"
|
||||
)
|
||||
for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH):
|
||||
if not bool(
|
||||
torch.isfinite(
|
||||
decoded_pixels[
|
||||
:,
|
||||
frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH,
|
||||
]
|
||||
).all()
|
||||
):
|
||||
raise PixelVaeResizeError("Video VAE decode contains NaN or Inf")
|
||||
|
||||
# Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation
|
||||
# is CPU/PIL-based already, and this also avoids retaining a full video-sized
|
||||
# resize workspace in VRAM under --gpu-only configurations.
|
||||
pixels = decoded_pixels.detach().to("cpu").contiguous()
|
||||
del decoded_pixels
|
||||
|
||||
# Core common_upscale is a 4D image boundary. Make B*F explicit so video
|
||||
# frame order is unambiguous. Resize bounded frame batches into one CPU
|
||||
# [B,F,H,W,C] buffer required by the temporal H3 VAE encoder.
|
||||
source_flat = pixels.reshape(
|
||||
batch * frames, pixel_height, pixel_width, channels
|
||||
)
|
||||
resized_pixels = torch.empty(
|
||||
(batch, frames, target_height, target_width, channels),
|
||||
dtype=pixels.dtype,
|
||||
device="cpu",
|
||||
)
|
||||
target_flat = resized_pixels.reshape(
|
||||
batch * frames, target_height, target_width, channels
|
||||
)
|
||||
for frame_start in range(
|
||||
0,
|
||||
batch * frames,
|
||||
_PIXEL_RESIZE_FRAME_BATCH,
|
||||
):
|
||||
frame_stop = min(
|
||||
frame_start + _PIXEL_RESIZE_FRAME_BATCH,
|
||||
batch * frames,
|
||||
)
|
||||
pixel_nchw = (
|
||||
source_flat[frame_start:frame_stop]
|
||||
.movedim(-1, 1)
|
||||
.contiguous()
|
||||
)
|
||||
resized_nchw = upscale_fn(
|
||||
pixel_nchw,
|
||||
int(target_width),
|
||||
int(target_height),
|
||||
upscale_method,
|
||||
"disabled",
|
||||
)
|
||||
expected_resize_shape = (
|
||||
frame_stop - frame_start,
|
||||
channels,
|
||||
target_height,
|
||||
target_width,
|
||||
)
|
||||
if not isinstance(resized_nchw, torch.Tensor) or tuple(
|
||||
resized_nchw.shape
|
||||
) != expected_resize_shape:
|
||||
raise PixelVaeResizeError("Core pixel resize returned invalid geometry")
|
||||
resized_nchw = resized_nchw.detach().to("cpu")
|
||||
if not bool(torch.isfinite(resized_nchw).all()):
|
||||
raise PixelVaeResizeError("Core pixel resize contains NaN or Inf")
|
||||
target_flat[frame_start:frame_stop].copy_(
|
||||
resized_nchw.movedim(1, -1)
|
||||
)
|
||||
del pixel_nchw, resized_nchw
|
||||
|
||||
del pixels, source_flat, target_flat
|
||||
|
||||
# Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch
|
||||
# and promotes its frame axis to one temporal video batch. Passing the 5D
|
||||
# [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial.
|
||||
# Encode each Continuum batch independently, then restore B explicitly.
|
||||
encoded_batches: list[torch.Tensor] = []
|
||||
expected_batch_shape = (
|
||||
1,
|
||||
int(samples.shape[1]),
|
||||
int(samples.shape[2]),
|
||||
int(target_latent_height),
|
||||
int(target_latent_width),
|
||||
)
|
||||
for batch_index in range(batch):
|
||||
encoded_batch = video_vae.encode(resized_pixels[batch_index])
|
||||
if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
|
||||
)
|
||||
if int(encoded_batch.shape[0]) != 1:
|
||||
raise PixelVaeResizeError(
|
||||
"Core Video VAE Encode must return one batch per IMAGE sequence"
|
||||
)
|
||||
encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape)
|
||||
if encoded_batch_shape != expected_batch_shape:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode changed B/C/T or returned the wrong target H/W "
|
||||
f"for batch {batch_index}: expected {expected_batch_shape}, "
|
||||
f"got {encoded_batch_shape}"
|
||||
)
|
||||
encoded_batches.append(encoded_batch)
|
||||
encoded = (
|
||||
encoded_batches[0]
|
||||
if len(encoded_batches) == 1
|
||||
else torch.cat(encoded_batches, dim=0)
|
||||
)
|
||||
if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
|
||||
)
|
||||
expected_shape = (
|
||||
int(samples.shape[0]),
|
||||
int(samples.shape[1]),
|
||||
int(samples.shape[2]),
|
||||
int(target_latent_height),
|
||||
int(target_latent_width),
|
||||
)
|
||||
if tuple(int(value) for value in encoded.shape) != expected_shape:
|
||||
raise PixelVaeResizeError(
|
||||
"Video VAE encode changed B/C/T or returned the wrong target H/W: "
|
||||
f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}"
|
||||
)
|
||||
if not bool(torch.isfinite(encoded).all()):
|
||||
raise PixelVaeResizeError("Video VAE encode contains NaN or Inf")
|
||||
|
||||
# Continuum stores accepted physical latents on CPU; do not retain one GPU
|
||||
# allocation per physical group while later groups are being converted.
|
||||
output = encoded.detach().to("cpu").contiguous()
|
||||
del resized_pixels, encoded_batches, encoded
|
||||
return output
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def resize_video_latents_via_vae(
|
||||
video_latents: Sequence[Mapping[str, Any]],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
*,
|
||||
video_vae: Any,
|
||||
target_width: int,
|
||||
target_height: int,
|
||||
upscale_method: str = "lanczos",
|
||||
upscale_fn: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Resize physical groups sequentially through the connected H3 Video VAE.
|
||||
|
||||
``target_width`` and ``target_height`` are pixel dimensions. The expected
|
||||
encoded latent geometry is derived exclusively from assembly-plan metadata.
|
||||
VAE failures are intentionally not caught: Core owns its OOM/tiled fallback,
|
||||
and this path must never fall back to direct latent interpolation.
|
||||
"""
|
||||
|
||||
groups = _contract_groups(assembly_plan)
|
||||
if (
|
||||
not isinstance(video_latents, Sequence)
|
||||
or isinstance(video_latents, (str, bytes))
|
||||
or len(video_latents) != len(groups)
|
||||
):
|
||||
raise PixelVaeResizeError("video latent group count changed")
|
||||
if (
|
||||
video_vae is None
|
||||
or not callable(getattr(video_vae, "decode", None))
|
||||
or not callable(getattr(video_vae, "encode", None))
|
||||
):
|
||||
raise PixelVaeResizeError("a usable H3 Video VAE is required")
|
||||
if upscale_method not in PIXEL_UPSCALE_METHODS:
|
||||
raise PixelVaeResizeError(
|
||||
f"unsupported pixel upscale method {upscale_method!r}"
|
||||
)
|
||||
target_width = int(target_width)
|
||||
target_height = int(target_height)
|
||||
if target_width < 1 or target_height < 1:
|
||||
raise PixelVaeResizeError("target pixel width and height must be positive")
|
||||
|
||||
scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan)
|
||||
if target_width % scale_width or target_height % scale_height:
|
||||
raise PixelVaeResizeError(
|
||||
"target pixel geometry is not aligned to the plan-derived latent scale"
|
||||
)
|
||||
target_latent_width = target_width // scale_width
|
||||
target_latent_height = target_height // scale_height
|
||||
if upscale_fn is None:
|
||||
from comfy.utils import common_upscale
|
||||
|
||||
upscale_fn = common_upscale
|
||||
|
||||
outputs: list[dict[str, Any]] = []
|
||||
for position, (latent, group) in enumerate(
|
||||
zip(video_latents, groups, strict=True)
|
||||
):
|
||||
samples = _latent_samples(latent, position=position, group=group)
|
||||
source_width = int(group["source_width"])
|
||||
source_height = int(group["source_height"])
|
||||
resized_samples = _resize_one_group(
|
||||
samples,
|
||||
video_vae=video_vae,
|
||||
source_width=source_width,
|
||||
source_height=source_height,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
target_latent_width=target_latent_width,
|
||||
target_latent_height=target_latent_height,
|
||||
upscale_method=upscale_method,
|
||||
upscale_fn=upscale_fn,
|
||||
)
|
||||
output = dict(latent)
|
||||
output["samples"] = resized_samples
|
||||
outputs.append(output)
|
||||
|
||||
return outputs
|
||||
Vendored
+466
@@ -0,0 +1,466 @@
|
||||
"""Assembly-plan contract for latent-first H3 Continuum V3."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import validate_plan
|
||||
|
||||
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
|
||||
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
|
||||
FPS = 24
|
||||
SECOND_PASS_CONTRACT_VERSION = 1
|
||||
REVIEW_AUDIO_PROJECTION_KEY = "review_audio_projection_v1"
|
||||
|
||||
|
||||
class AssemblyPlanError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def make_assembly_plan(
|
||||
entries: list[dict[str, Any]],
|
||||
*,
|
||||
chunk_seconds: float,
|
||||
preserve_final_frame: bool,
|
||||
) -> dict[str, Any]:
|
||||
if not entries:
|
||||
raise AssemblyPlanError("cannot create an assembly plan for an empty sequence")
|
||||
|
||||
chunks: list[dict[str, Any]] = []
|
||||
width = height = None
|
||||
for sequence_index, entry in enumerate(entries, start=1):
|
||||
if not isinstance(entry, dict):
|
||||
raise AssemblyPlanError(f"chunk {sequence_index} entry is not a dictionary")
|
||||
plan = validate_plan(entry.get("plan"))
|
||||
video = entry.get("video")
|
||||
audio = entry.get("audio")
|
||||
if not torch.is_tensor(video) or video.ndim < 3:
|
||||
raise AssemblyPlanError(f"chunk {sequence_index} video latent is invalid")
|
||||
if not torch.is_tensor(audio) or audio.ndim < 3:
|
||||
raise AssemblyPlanError(f"chunk {sequence_index} audio latent is invalid")
|
||||
|
||||
plan_width = int(plan["width"])
|
||||
plan_height = int(plan["height"])
|
||||
if width is None:
|
||||
width, height = plan_width, plan_height
|
||||
elif (plan_width, plan_height) != (width, height):
|
||||
raise AssemblyPlanError("chunk dimensions changed within the sequence")
|
||||
|
||||
chunks.append(
|
||||
{
|
||||
"sequence_index": sequence_index,
|
||||
"chunk_index": int(plan["clip_index"]),
|
||||
"total_frames": int(plan["total_frames"]),
|
||||
"trim_frames": int(plan["trim_frames"]),
|
||||
"net_frames": int(plan["net_frames"]),
|
||||
"context_frames": int(plan["context_frames"]),
|
||||
"expected_video_latent_t": int(video.shape[2]),
|
||||
"expected_audio_latent_t": int(audio.shape[-1]),
|
||||
}
|
||||
)
|
||||
|
||||
result = {
|
||||
"magic": ASSEMBLY_PLAN_MAGIC,
|
||||
"schema_version": ASSEMBLY_PLAN_SCHEMA_VERSION,
|
||||
"fps": FPS,
|
||||
"width": int(width),
|
||||
"height": int(height),
|
||||
"chunk_seconds": float(chunk_seconds),
|
||||
"target_frames": int(round(len(chunks) * float(chunk_seconds) * FPS)),
|
||||
"preserve_final_frame": bool(preserve_final_frame),
|
||||
"chunks": chunks,
|
||||
}
|
||||
return validate_assembly_plan(result)
|
||||
|
||||
|
||||
def validate_assembly_plan(plan: Any) -> dict[str, Any]:
|
||||
if not isinstance(plan, dict) or plan.get("magic") != ASSEMBLY_PLAN_MAGIC:
|
||||
raise AssemblyPlanError("invalid H3 Continuum V3 assembly plan")
|
||||
if (
|
||||
type(plan.get("schema_version")) is not int
|
||||
or plan["schema_version"] != ASSEMBLY_PLAN_SCHEMA_VERSION
|
||||
):
|
||||
raise AssemblyPlanError(
|
||||
f"unsupported assembly plan schema {plan.get('schema_version')!r}"
|
||||
)
|
||||
if int(plan.get("fps", 0)) != FPS:
|
||||
raise AssemblyPlanError("assembly plan FPS must be 24")
|
||||
if int(plan.get("width", 0)) <= 0 or int(plan.get("height", 0)) <= 0:
|
||||
raise AssemblyPlanError("assembly plan dimensions are invalid")
|
||||
if float(plan.get("chunk_seconds", 0.0)) <= 0:
|
||||
raise AssemblyPlanError("assembly plan chunk duration is invalid")
|
||||
if int(plan.get("target_frames", 0)) <= 0:
|
||||
raise AssemblyPlanError("assembly plan target frame count is invalid")
|
||||
|
||||
chunks = plan.get("chunks")
|
||||
if not isinstance(chunks, list) or not chunks:
|
||||
raise AssemblyPlanError("assembly plan contains no chunks")
|
||||
for expected_index, chunk in enumerate(chunks, start=1):
|
||||
if not isinstance(chunk, dict):
|
||||
raise AssemblyPlanError(f"assembly chunk {expected_index} is invalid")
|
||||
if int(chunk.get("sequence_index", 0)) != expected_index:
|
||||
raise AssemblyPlanError("assembly chunk order is not contiguous")
|
||||
total = int(chunk.get("total_frames", 0))
|
||||
trim = int(chunk.get("trim_frames", -1))
|
||||
net = int(chunk.get("net_frames", 0))
|
||||
if total <= 0 or trim < 0 or net <= 0 or total - trim != net:
|
||||
raise AssemblyPlanError(
|
||||
f"assembly chunk {expected_index} frame plan is invalid"
|
||||
)
|
||||
if int(chunk.get("context_frames", -1)) < 0:
|
||||
raise AssemblyPlanError(
|
||||
f"assembly chunk {expected_index} context is invalid"
|
||||
)
|
||||
if int(chunk.get("expected_video_latent_t", 0)) <= 0:
|
||||
raise AssemblyPlanError(
|
||||
f"assembly chunk {expected_index} video latent length is invalid"
|
||||
)
|
||||
if int(chunk.get("expected_audio_latent_t", 0)) <= 0:
|
||||
raise AssemblyPlanError(
|
||||
f"assembly chunk {expected_index} audio latent length is invalid"
|
||||
)
|
||||
return plan
|
||||
# V3.0.1 hardening integration: redundant metadata plus fail-closed validation.
|
||||
from ..hardening import (
|
||||
enrich_assembly_plan as _enrich_assembly_plan,
|
||||
validate_assembly_plan_contract as _validate_assembly_plan_contract,
|
||||
)
|
||||
|
||||
_make_assembly_plan_v300 = make_assembly_plan
|
||||
_validate_assembly_plan_v300 = validate_assembly_plan
|
||||
|
||||
|
||||
def validate_assembly_plan(plan):
|
||||
result = _validate_assembly_plan_v300(plan)
|
||||
_validate_assembly_plan_contract(plan)
|
||||
validate_review_audio_projection(plan)
|
||||
return result
|
||||
|
||||
|
||||
def validate_review_audio_projection(plan):
|
||||
"""Check a scoped PCM coordinate without recomputing its absolute origin."""
|
||||
if REVIEW_AUDIO_PROJECTION_KEY not in plan:
|
||||
return None
|
||||
metadata = plan[REVIEW_AUDIO_PROJECTION_KEY]
|
||||
if not isinstance(metadata, dict):
|
||||
raise AssemblyPlanError("Review audio projection must be a dictionary")
|
||||
if (type(metadata.get("version")) is not int or metadata.get("version") != 1 or metadata.get("kind") != "current_review_unit"
|
||||
or metadata.get("coordinate_space") != "full_source_pcm"):
|
||||
raise AssemblyPlanError("Review audio projection version or coordinate space is invalid")
|
||||
for field in ("fps", "start_chunk", "end_chunk", "source_start_frame", "source_stop_frame"):
|
||||
if type(metadata.get(field)) is not int:
|
||||
raise AssemblyPlanError(f"Review audio projection {field} must be an integer")
|
||||
if metadata["fps"] != FPS or int(plan.get("fps", 0)) != FPS:
|
||||
raise AssemblyPlanError("Review audio projection FPS must be 24")
|
||||
start, end = metadata["start_chunk"], metadata["end_chunk"]
|
||||
first, stop = metadata["source_start_frame"], metadata["source_stop_frame"]
|
||||
chunks = [int(chunk["chunk_index"]) for chunk in plan["chunks"]]
|
||||
units = plan.get("decode_groups") or plan["chunks"]
|
||||
if (start < 1 or end < start or len(chunks) != end - start + 1
|
||||
or any(index != start + position for position, index in enumerate(chunks))
|
||||
or first < 0 or stop <= first
|
||||
or stop - first != sum(int(unit["net_frames"]) for unit in units)):
|
||||
raise AssemblyPlanError("Review audio projection physical range is inconsistent")
|
||||
return metadata
|
||||
|
||||
|
||||
def make_review_audio_projection(plan, *, start_chunk: int, end_chunk: int):
|
||||
"""Read natural retained ranges before narrowing the accepted decode view."""
|
||||
validate_assembly_plan(plan)
|
||||
if REVIEW_AUDIO_PROJECTION_KEY in plan:
|
||||
raise AssemblyPlanError("Review output cannot be projected twice")
|
||||
start, end = int(start_chunk), int(end_chunk)
|
||||
selected = []
|
||||
covered = []
|
||||
for unit in plan.get("decode_groups") or plan["chunks"]:
|
||||
indices = unit.get("logical_chunk_indices") or [unit["chunk_index"]]
|
||||
if not any(start <= int(index) <= end for index in indices):
|
||||
continue
|
||||
if not all(start <= int(index) <= end for index in indices):
|
||||
raise AssemblyPlanError("Review audio projection cannot split a physical group")
|
||||
first, stop = unit.get("frame_start"), unit.get("frame_stop")
|
||||
if type(first) is not int or type(stop) is not int or stop - first != int(unit["net_frames"]):
|
||||
raise AssemblyPlanError("Review audio projection needs retained natural frame ranges")
|
||||
if selected and selected[-1][1] != first:
|
||||
raise AssemblyPlanError("Review audio projection ranges are not contiguous")
|
||||
selected.append((first, stop))
|
||||
covered.extend(int(index) for index in indices)
|
||||
if not selected or covered != list(range(start, end + 1)):
|
||||
raise AssemblyPlanError("Review audio projection is outside the accepted sequence")
|
||||
return dict(version=1, kind="current_review_unit", coordinate_space="full_source_pcm",
|
||||
fps=FPS, start_chunk=start, end_chunk=end,
|
||||
source_start_frame=selected[0][0], source_stop_frame=selected[-1][1])
|
||||
|
||||
|
||||
def make_assembly_plan(*args, **kwargs):
|
||||
plan = _make_assembly_plan_v300(*args, **kwargs)
|
||||
_enrich_assembly_plan(plan)
|
||||
validate_assembly_plan(plan)
|
||||
return plan
|
||||
|
||||
|
||||
def _recombine_terminal_tensor(
|
||||
first,
|
||||
second,
|
||||
*,
|
||||
slices,
|
||||
time_dim: int,
|
||||
name: str,
|
||||
):
|
||||
"""Invert the terminal logical split without changing any tensor value."""
|
||||
|
||||
import torch
|
||||
|
||||
if not torch.is_tensor(first) or not torch.is_tensor(second):
|
||||
raise ValueError(f"terminal {name} entries must be tensors")
|
||||
if first.ndim != second.ndim:
|
||||
raise ValueError(f"terminal {name} tensor ranks differ")
|
||||
dim = int(time_dim)
|
||||
if dim < 0:
|
||||
dim += first.ndim
|
||||
if dim < 0 or dim >= first.ndim:
|
||||
raise ValueError(f"terminal {name} time dimension is invalid")
|
||||
for axis, (left, right) in enumerate(zip(first.shape, second.shape)):
|
||||
if axis != dim and int(left) != int(right):
|
||||
raise ValueError(f"terminal {name} tensor geometry differs")
|
||||
|
||||
(first_start, first_stop), (second_start, second_stop) = slices
|
||||
first_start = int(first_start)
|
||||
first_stop = int(first_stop)
|
||||
second_start = int(second_start)
|
||||
second_stop = int(second_stop)
|
||||
if first_start != 0 or second_start > first_stop or second_stop <= first_stop:
|
||||
raise ValueError(f"terminal {name} split ranges do not form one physical unit")
|
||||
if int(first.shape[dim]) != first_stop - first_start:
|
||||
raise ValueError(f"terminal {name} first slice length is inconsistent")
|
||||
if int(second.shape[dim]) != second_stop - second_start:
|
||||
raise ValueError(f"terminal {name} second slice length is inconsistent")
|
||||
|
||||
overlap_start = max(first_start, second_start)
|
||||
overlap_stop = min(first_stop, second_stop)
|
||||
overlap_length = max(0, overlap_stop - overlap_start)
|
||||
if overlap_length:
|
||||
first_overlap = first.narrow(dim, overlap_start - first_start, overlap_length)
|
||||
second_overlap = second.narrow(dim, overlap_start - second_start, overlap_length)
|
||||
if not torch.equal(first_overlap, second_overlap):
|
||||
raise ValueError(
|
||||
f"terminal {name} overlap differs; refusing lossy physical reconstruction"
|
||||
)
|
||||
|
||||
output_shape = list(first.shape)
|
||||
output_shape[dim] = second_stop
|
||||
output = first.new_empty(output_shape)
|
||||
output.narrow(dim, first_start, first_stop - first_start).copy_(first)
|
||||
output.narrow(dim, second_start, second_stop - second_start).copy_(second)
|
||||
return output.contiguous()
|
||||
|
||||
|
||||
def _terminal_physical_prompt(
|
||||
prompts: list[str],
|
||||
*,
|
||||
chunk_seconds: float,
|
||||
) -> tuple[str, str]:
|
||||
"""Reproduce the established terminal-pair prompt without reparsing it later."""
|
||||
first, second = prompts
|
||||
if first == second:
|
||||
return first, "shared_prompt_v1"
|
||||
split = f"{float(chunk_seconds):g}"
|
||||
stop = f"{float(chunk_seconds) * 2:g}"
|
||||
return (
|
||||
f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}",
|
||||
"paired_timeline_v1",
|
||||
)
|
||||
|
||||
|
||||
def _attach_second_pass_contract(
|
||||
plan: dict[str, Any],
|
||||
*,
|
||||
logical_entries: list[dict[str, Any]],
|
||||
physical_entries: list[dict[str, Any]],
|
||||
chunk_seconds: float,
|
||||
logical_chunk_indices: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Attach additive V3.5 metadata without changing V3.4 generation semantics."""
|
||||
chunk_indices = (
|
||||
list(range(1, len(logical_entries) + 1))
|
||||
if logical_chunk_indices is None
|
||||
else [int(value) for value in logical_chunk_indices]
|
||||
)
|
||||
if len(chunk_indices) != len(logical_entries):
|
||||
raise ValueError("logical chunk identity count does not match entries")
|
||||
decode_groups = plan.get("decode_groups")
|
||||
if not isinstance(decode_groups, list):
|
||||
decode_groups = [
|
||||
{
|
||||
"logical_chunk_indices": [chunk_indices[index]],
|
||||
"terminal_merged": False,
|
||||
"total_frames": plan["chunks"][index]["total_frames"],
|
||||
"trim_frames": plan["chunks"][index]["trim_frames"],
|
||||
}
|
||||
for index in range(len(physical_entries))
|
||||
]
|
||||
|
||||
prompt_by_chunk = {
|
||||
chunk_indices[index]: str(entry.get("prompt", ""))
|
||||
for index, entry in enumerate(logical_entries)
|
||||
}
|
||||
groups: list[dict[str, Any]] = []
|
||||
for group_id, (entry, decode_group) in enumerate(
|
||||
zip(physical_entries, decode_groups, strict=True)
|
||||
):
|
||||
video = entry["video"]
|
||||
audio = entry["audio"]
|
||||
logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]]
|
||||
prompts = [prompt_by_chunk[index] for index in logical_chunks]
|
||||
if len(prompts) == 1:
|
||||
physical_prompt = prompts[0]
|
||||
prompt_policy = "single"
|
||||
elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")):
|
||||
physical_prompt, prompt_policy = _terminal_physical_prompt(
|
||||
prompts,
|
||||
chunk_seconds=chunk_seconds,
|
||||
)
|
||||
else:
|
||||
physical_prompt = "\n\n".join(prompts)
|
||||
prompt_policy = "physical_group_v1"
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"group_id": group_id,
|
||||
"logical_chunks": logical_chunks,
|
||||
"physical_prompt": physical_prompt,
|
||||
"prompt_policy": prompt_policy,
|
||||
"physical_frames": int(decode_group["total_frames"]),
|
||||
"trim_prefix_frames": int(decode_group["trim_frames"]),
|
||||
"terminal_merged": bool(decode_group.get("terminal_merged", False)),
|
||||
"source_width": int(plan["width"]),
|
||||
"source_height": int(plan["height"]),
|
||||
"source_batch": int(video.shape[0]),
|
||||
"latent_channels": int(video.shape[1]),
|
||||
"source_latent_t": int(video.shape[2]),
|
||||
"source_latent_h": int(video.shape[3]),
|
||||
"source_latent_w": int(video.shape[4]),
|
||||
"source_audio_shape": [int(value) for value in audio.shape],
|
||||
}
|
||||
)
|
||||
|
||||
plan["second_pass_contract"] = {
|
||||
"version": SECOND_PASS_CONTRACT_VERSION,
|
||||
"physical_groups": groups,
|
||||
}
|
||||
|
||||
|
||||
def prepare_physical_decode_entries(
|
||||
entries,
|
||||
*,
|
||||
chunk_seconds: float,
|
||||
preserve_final_frame: bool,
|
||||
terminal_merged: bool,
|
||||
terminal_initial_pair: bool | None = None,
|
||||
):
|
||||
"""Return external-VAE decode units while preserving logical plan metadata."""
|
||||
|
||||
logical_entries = list(entries)
|
||||
plan = make_assembly_plan(
|
||||
logical_entries,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
preserve_final_frame=bool(preserve_final_frame),
|
||||
)
|
||||
logical_chunk_indices = [
|
||||
int(chunk["chunk_index"]) for chunk in plan["chunks"]
|
||||
]
|
||||
if not terminal_merged:
|
||||
_attach_second_pass_contract(
|
||||
plan,
|
||||
logical_entries=logical_entries,
|
||||
physical_entries=logical_entries,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
logical_chunk_indices=logical_chunk_indices,
|
||||
)
|
||||
return logical_entries, plan
|
||||
if len(logical_entries) < 2:
|
||||
raise ValueError("terminal physical decode requires two logical entries")
|
||||
|
||||
from ..v2.sequence import _terminal_pair_contract
|
||||
|
||||
initial_pair = (
|
||||
len(logical_entries) == 2
|
||||
if terminal_initial_pair is None
|
||||
else bool(terminal_initial_pair)
|
||||
)
|
||||
contract = _terminal_pair_contract(
|
||||
initial_pair=initial_pair,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
)
|
||||
first_entry = logical_entries[-2]
|
||||
second_entry = logical_entries[-1]
|
||||
physical_video = _recombine_terminal_tensor(
|
||||
first_entry["video"],
|
||||
second_entry["video"],
|
||||
slices=contract["video_slices"],
|
||||
time_dim=2,
|
||||
name="video",
|
||||
)
|
||||
physical_audio = _recombine_terminal_tensor(
|
||||
first_entry["audio"],
|
||||
second_entry["audio"],
|
||||
slices=contract["audio_slices"],
|
||||
time_dim=-1,
|
||||
name="audio",
|
||||
)
|
||||
|
||||
physical_entry = dict(first_entry)
|
||||
physical_entry["video"] = physical_video
|
||||
physical_entry["audio"] = physical_audio
|
||||
decode_entries = [*logical_entries[:-2], physical_entry]
|
||||
|
||||
logical_chunks = list(plan["chunks"])
|
||||
decode_groups = []
|
||||
for chunk in logical_chunks[:-2]:
|
||||
group = dict(chunk)
|
||||
group["logical_chunk_indices"] = [int(chunk["chunk_index"])]
|
||||
group["terminal_merged"] = False
|
||||
decode_groups.append(group)
|
||||
|
||||
first_chunk = logical_chunks[-2]
|
||||
second_chunk = logical_chunks[-1]
|
||||
frame_start = int(first_chunk.get("frame_start", 0))
|
||||
net_frames = int(contract["physical_frames"]) - int(
|
||||
contract["physical_context_frames"]
|
||||
)
|
||||
terminal_group = dict(first_chunk)
|
||||
terminal_group.update(
|
||||
{
|
||||
"total_frames": int(contract["physical_frames"]),
|
||||
"trim_frames": int(contract["physical_context_frames"]),
|
||||
"net_frames": net_frames,
|
||||
"context_frames": int(contract["physical_context_frames"]),
|
||||
"expected_video_latent_t": int(physical_video.shape[2]),
|
||||
"expected_audio_latent_t": int(physical_audio.shape[-1]),
|
||||
"frame_start": frame_start,
|
||||
"frame_stop": frame_start + net_frames,
|
||||
"logical_chunk_indices": [
|
||||
int(first_chunk["chunk_index"]),
|
||||
int(second_chunk["chunk_index"]),
|
||||
],
|
||||
"terminal_merged": True,
|
||||
}
|
||||
)
|
||||
decode_groups.append(terminal_group)
|
||||
|
||||
plan = dict(plan)
|
||||
plan["decode_group_version"] = 1
|
||||
plan["logical_chunk_count"] = len(logical_chunks)
|
||||
plan["physical_decode_group_count"] = len(decode_groups)
|
||||
plan["decode_groups"] = decode_groups
|
||||
_attach_second_pass_contract(
|
||||
plan,
|
||||
logical_entries=logical_entries,
|
||||
physical_entries=decode_entries,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
logical_chunk_indices=logical_chunk_indices,
|
||||
)
|
||||
validate_assembly_plan(plan)
|
||||
return decode_entries, plan
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
"""Immutable facts shared by the pure Continuum execution planner.
|
||||
|
||||
This module must remain free of torch and ComfyUI imports. Runtime payloads
|
||||
stay in their existing adapters and are projected into these facts only after
|
||||
the active continuation source has already been selected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .review_control import ReviewUnit
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PrefixFacts:
|
||||
"""Immutable planning projection of one fully validated storage prefix."""
|
||||
|
||||
accepted_chunks: int
|
||||
configured_chunks: int
|
||||
revision_id: str | None
|
||||
status: str | None
|
||||
review_unit: ReviewUnit | None
|
||||
effective_nonce: int
|
||||
branch_regenerate_from: int
|
||||
physical_group_boundaries: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExplicitSessionFacts:
|
||||
"""Characterized facts for an explicit Session runtime input."""
|
||||
|
||||
present: bool
|
||||
compatible: bool
|
||||
accepted_chunks: int
|
||||
reroll_from_chunk: int | None
|
||||
physical_group_boundaries: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InitialStateFacts:
|
||||
"""Characterized facts for an initial State runtime input."""
|
||||
|
||||
present: bool
|
||||
usable: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ContinuationSourceFacts:
|
||||
"""Planning projection of the single source selected by the runtime."""
|
||||
|
||||
kind: str
|
||||
accepted_chunks: int
|
||||
reroll_from_chunk: int | None
|
||||
physical_group_boundaries: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProjectionDecision:
|
||||
"""Describe which accepted chunks may be exposed to external decode."""
|
||||
|
||||
kind: str
|
||||
start_chunk: int | None = None
|
||||
end_chunk: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhysicalGroupDescriptor:
|
||||
"""One atomic sampling group expressed in one-based logical chunks."""
|
||||
|
||||
physical_group: int
|
||||
logical_chunks: tuple[int, ...]
|
||||
terminal_atomic: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutionPlan:
|
||||
"""Pure plan consumed by the unchanged runtime execution loops."""
|
||||
|
||||
groups_to_generate: tuple[PhysicalGroupDescriptor, ...]
|
||||
projection: ProjectionDecision
|
||||
expected_status: str
|
||||
expected_review_unit: ReviewUnit | None
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
"""Process-local, VAE-scoped cache for V3.8 reference encode tensors.
|
||||
|
||||
Only detached, contiguous CPU tensor copies are retained. The cache never
|
||||
owns source images, video frames, audio waveforms, conditioning objects, or
|
||||
sampling outputs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, OrderedDict
|
||||
from dataclasses import dataclass
|
||||
import threading
|
||||
from typing import Any, Callable
|
||||
import weakref
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
REF_ENCODE_CACHE_VERSION = 1
|
||||
DEFAULT_MAX_ENTRIES = 16
|
||||
DEFAULT_MAX_BYTES = 512 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RefEncodeCacheKey:
|
||||
kind: str
|
||||
preprocess_version: int
|
||||
identity: str
|
||||
cache_version: int = REF_ENCODE_CACHE_VERSION
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RefEncodeCacheEvent:
|
||||
kind: str
|
||||
action: str
|
||||
reason: str = ""
|
||||
payload_bytes: int = 0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _CacheEntry:
|
||||
tensor: torch.Tensor
|
||||
payload_bytes: int
|
||||
last_used: int
|
||||
|
||||
|
||||
CacheEventSink = Callable[[RefEncodeCacheEvent], None]
|
||||
|
||||
|
||||
def make_ref_encode_cache_key(
|
||||
kind: str,
|
||||
preprocess_version: int,
|
||||
identity: str,
|
||||
) -> RefEncodeCacheKey:
|
||||
return RefEncodeCacheKey(
|
||||
kind=str(kind),
|
||||
preprocess_version=int(preprocess_version),
|
||||
identity=str(identity),
|
||||
)
|
||||
|
||||
|
||||
class RefEncodeCache:
|
||||
"""Small process-local LRU split into weak VAE object namespaces."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_entries: int = DEFAULT_MAX_ENTRIES,
|
||||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||||
) -> None:
|
||||
if int(max_entries) < 1:
|
||||
raise ValueError("max_entries must be positive")
|
||||
if int(max_bytes) < 1:
|
||||
raise ValueError("max_bytes must be positive")
|
||||
self.max_entries = int(max_entries)
|
||||
self.max_bytes = int(max_bytes)
|
||||
self._lock = threading.RLock()
|
||||
self._namespaces: weakref.WeakKeyDictionary[
|
||||
Any, OrderedDict[RefEncodeCacheKey, _CacheEntry]
|
||||
] = weakref.WeakKeyDictionary()
|
||||
self._clock = 0
|
||||
self._stats: Counter[str] = Counter()
|
||||
self._kind_stats: dict[str, Counter[str]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _vae_cacheable(vae: Any) -> bool:
|
||||
try:
|
||||
weakref.ref(vae)
|
||||
hash(vae)
|
||||
except TypeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def supports_vae(self, vae: Any) -> bool:
|
||||
return self._vae_cacheable(vae)
|
||||
|
||||
def _next_clock_locked(self) -> int:
|
||||
self._clock += 1
|
||||
return self._clock
|
||||
|
||||
def _record_locked(
|
||||
self,
|
||||
*,
|
||||
key: RefEncodeCacheKey,
|
||||
action: str,
|
||||
reason: str = "",
|
||||
payload_bytes: int = 0,
|
||||
) -> RefEncodeCacheEvent:
|
||||
self._stats[str(action)] += 1
|
||||
self._kind_stats.setdefault(key.kind, Counter())[str(action)] += 1
|
||||
return RefEncodeCacheEvent(
|
||||
kind=key.kind,
|
||||
action=str(action),
|
||||
reason=str(reason),
|
||||
payload_bytes=int(payload_bytes),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _emit(event: RefEncodeCacheEvent, sink: CacheEventSink | None) -> None:
|
||||
if sink is not None:
|
||||
sink(event)
|
||||
|
||||
def _resident_locked(self) -> tuple[int, int]:
|
||||
entries = 0
|
||||
payload_bytes = 0
|
||||
for namespace in list(self._namespaces.values()):
|
||||
entries += len(namespace)
|
||||
payload_bytes += sum(entry.payload_bytes for entry in namespace.values())
|
||||
return entries, payload_bytes
|
||||
|
||||
def _evict_locked(self) -> None:
|
||||
while True:
|
||||
entries, payload_bytes = self._resident_locked()
|
||||
if entries <= self.max_entries and payload_bytes <= self.max_bytes:
|
||||
return
|
||||
oldest_namespace = None
|
||||
oldest_key = None
|
||||
oldest_clock = None
|
||||
for namespace in list(self._namespaces.values()):
|
||||
for key, entry in namespace.items():
|
||||
if oldest_clock is None or entry.last_used < oldest_clock:
|
||||
oldest_namespace = namespace
|
||||
oldest_key = key
|
||||
oldest_clock = entry.last_used
|
||||
if oldest_namespace is None or oldest_key is None:
|
||||
return
|
||||
oldest_namespace.pop(oldest_key, None)
|
||||
self._stats["evictions"] += 1
|
||||
|
||||
def lookup(
|
||||
self,
|
||||
vae: Any,
|
||||
key: RefEncodeCacheKey,
|
||||
*,
|
||||
event_sink: CacheEventSink | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
if not self._vae_cacheable(vae):
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="vae_not_weakrefable_or_hashable",
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return None
|
||||
|
||||
master = None
|
||||
with self._lock:
|
||||
namespace = self._namespaces.get(vae)
|
||||
entry = namespace.get(key) if namespace is not None else None
|
||||
if entry is None:
|
||||
event = self._record_locked(key=key, action="miss")
|
||||
else:
|
||||
entry.last_used = self._next_clock_locked()
|
||||
namespace.move_to_end(key)
|
||||
master = entry.tensor
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="hit",
|
||||
payload_bytes=entry.payload_bytes,
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return None if master is None else master.clone()
|
||||
|
||||
def store(
|
||||
self,
|
||||
vae: Any,
|
||||
key: RefEncodeCacheKey,
|
||||
tensor: Any,
|
||||
*,
|
||||
event_sink: CacheEventSink | None = None,
|
||||
) -> bool:
|
||||
if not self._vae_cacheable(vae):
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="vae_not_weakrefable_or_hashable",
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return False
|
||||
if not torch.is_tensor(tensor):
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="payload_not_tensor",
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return False
|
||||
if tensor.device.type != "cpu":
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="payload_not_cpu",
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return False
|
||||
payload_bytes = int(tensor.numel()) * int(tensor.element_size())
|
||||
if payload_bytes > self.max_bytes:
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="entry_exceeds_byte_limit",
|
||||
payload_bytes=payload_bytes,
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return False
|
||||
try:
|
||||
master = tensor.detach().contiguous().clone()
|
||||
except Exception:
|
||||
with self._lock:
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="bypass",
|
||||
reason="payload_copy_failed",
|
||||
payload_bytes=payload_bytes,
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return False
|
||||
|
||||
with self._lock:
|
||||
namespace = self._namespaces.get(vae)
|
||||
if namespace is None:
|
||||
namespace = OrderedDict()
|
||||
self._namespaces[vae] = namespace
|
||||
namespace[key] = _CacheEntry(
|
||||
tensor=master,
|
||||
payload_bytes=payload_bytes,
|
||||
last_used=self._next_clock_locked(),
|
||||
)
|
||||
namespace.move_to_end(key)
|
||||
self._evict_locked()
|
||||
event = self._record_locked(
|
||||
key=key,
|
||||
action="store",
|
||||
payload_bytes=payload_bytes,
|
||||
)
|
||||
self._emit(event, event_sink)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._namespaces = weakref.WeakKeyDictionary()
|
||||
self._clock = 0
|
||||
self._stats.clear()
|
||||
self._kind_stats.clear()
|
||||
|
||||
def inspect(self) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
entries, payload_bytes = self._resident_locked()
|
||||
return {
|
||||
"cache_version": REF_ENCODE_CACHE_VERSION,
|
||||
"resident_entries": entries,
|
||||
"resident_bytes": payload_bytes,
|
||||
"max_entries": self.max_entries,
|
||||
"max_bytes": self.max_bytes,
|
||||
"stats": dict(self._stats),
|
||||
"kind_stats": {
|
||||
kind: dict(values)
|
||||
for kind, values in sorted(self._kind_stats.items())
|
||||
},
|
||||
"vae_namespaces": len(self._namespaces),
|
||||
}
|
||||
|
||||
|
||||
_REF_ENCODE_CACHE = RefEncodeCache()
|
||||
|
||||
|
||||
def get_ref_encode_cache() -> RefEncodeCache:
|
||||
return _REF_ENCODE_CACHE
|
||||
|
||||
|
||||
def clear_ref_encode_cache() -> None:
|
||||
_REF_ENCODE_CACHE.clear()
|
||||
|
||||
|
||||
def inspect_ref_encode_cache() -> dict[str, Any]:
|
||||
return _REF_ENCODE_CACHE.inspect()
|
||||
|
||||
|
||||
def format_ref_encode_cache_diagnostics(
|
||||
events: list[RefEncodeCacheEvent] | tuple[RefEncodeCacheEvent, ...],
|
||||
) -> str:
|
||||
if not events:
|
||||
return ""
|
||||
labels = {
|
||||
"reference_image": "image",
|
||||
"video_guide": "video",
|
||||
"reference_audio": "audio",
|
||||
}
|
||||
by_kind: dict[str, Counter[str]] = {}
|
||||
totals: Counter[str] = Counter()
|
||||
for event in events:
|
||||
by_kind.setdefault(event.kind, Counter())[event.action] += 1
|
||||
totals[event.action] += 1
|
||||
components = []
|
||||
for kind in ("reference_image", "video_guide", "reference_audio"):
|
||||
values = by_kind.get(kind)
|
||||
if not values:
|
||||
continue
|
||||
components.append(
|
||||
f"{labels[kind]} hit={values['hit']}/miss={values['miss']}"
|
||||
f"/store={values['store']}/bypass={values['bypass']}"
|
||||
)
|
||||
state = inspect_ref_encode_cache()
|
||||
resident_mib = int(state["resident_bytes"]) / float(1024**2)
|
||||
limit_mib = int(state["max_bytes"]) / float(1024**2)
|
||||
components.append(
|
||||
"total "
|
||||
f"hits={totals['hit']}, misses={totals['miss']}, "
|
||||
f"stores={totals['store']}, bypasses={totals['bypass']}"
|
||||
)
|
||||
components.append(
|
||||
f"resident={resident_mib:.1f} MiB/{limit_mib:.1f} MiB "
|
||||
f"({int(state['resident_entries'])}/{int(state['max_entries'])} entries)"
|
||||
)
|
||||
return "Reference encode cache: " + "; ".join(components) + "."
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Pure RR-R3 plan joining fixed Reference identities to Core Picture numbers.
|
||||
|
||||
The caller supplies only immutable routing and Reference metadata. This module
|
||||
does not encode images, rewrite prompts, select runtime sources, or sample.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from .reference_routing import REFERENCE_SLOT_IDS, ReferenceRoutingSchedule
|
||||
|
||||
|
||||
_SLOT_ORDER = {slot_id: index for index, slot_id in enumerate(REFERENCE_SLOT_IDS)}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EffectivePlanIssue:
|
||||
code: str
|
||||
message: str
|
||||
source_slot_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferencePicture:
|
||||
"""A fixed source slot, compact asset, content hash, and Qwen Picture number."""
|
||||
|
||||
source_slot_id: str
|
||||
asset_index: int
|
||||
image_sha256: str
|
||||
picture_number: int
|
||||
|
||||
@property
|
||||
def picture_tag(self) -> str:
|
||||
return f"<Picture {self.picture_number}>"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PictureMap:
|
||||
"""Core's image presentation order; First/Last are not Reference assets."""
|
||||
|
||||
first_picture_number: int | None
|
||||
last_picture_number: int | None
|
||||
references: tuple[ReferencePicture, ...]
|
||||
|
||||
@property
|
||||
def source_slot_ids(self) -> tuple[str, ...]:
|
||||
return tuple(item.source_slot_id for item in self.references)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LogicalEffectiveRoute:
|
||||
logical_chunk: int
|
||||
requested_source_slot_ids: tuple[str, ...]
|
||||
picture_map: PictureMap
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhysicalEffectiveRoute:
|
||||
physical_group: int
|
||||
logical_chunks: tuple[int, ...]
|
||||
terminal_atomic: bool
|
||||
routing_status: str
|
||||
status: str
|
||||
requested_source_slot_ids: tuple[str, ...] | None
|
||||
picture_map: PictureMap | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EffectiveReferencePlan:
|
||||
mode: str
|
||||
legacy_fallback: bool
|
||||
connected_source_slot_ids: tuple[str, ...]
|
||||
logical_routes: tuple[LogicalEffectiveRoute, ...] = ()
|
||||
physical_routes: tuple[PhysicalEffectiveRoute, ...] = ()
|
||||
issues: tuple[EffectivePlanIssue, ...] = ()
|
||||
|
||||
@property
|
||||
def valid(self) -> bool:
|
||||
return not self.issues
|
||||
|
||||
@property
|
||||
def has_route_conflicts(self) -> bool:
|
||||
return any(group.status == "conflict" for group in self.physical_routes)
|
||||
|
||||
|
||||
def _schedule_issues(schedule: ReferenceRoutingSchedule) -> tuple[EffectivePlanIssue, ...]:
|
||||
if not schedule.valid:
|
||||
return tuple(
|
||||
EffectivePlanIssue(f"routing_{issue.code}", issue.message, issue.slot_id)
|
||||
for issue in schedule.issues
|
||||
)
|
||||
total = schedule.total_chunks
|
||||
if isinstance(total, bool) or not isinstance(total, int) or total < 1:
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "missing logical chunk count"),)
|
||||
if tuple(route.logical_chunk for route in schedule.logical_routes) != tuple(range(1, total + 1)):
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "logical chunks are not the full project sequence"),)
|
||||
|
||||
logical_slots = {route.logical_chunk: route.reference_slot_ids for route in schedule.logical_routes}
|
||||
for route in schedule.logical_routes:
|
||||
if any(slot_id not in _SLOT_ORDER for slot_id in route.reference_slot_ids):
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "logical route has an unknown Reference slot"),)
|
||||
if len(set(route.reference_slot_ids)) != len(route.reference_slot_ids):
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "logical route repeats a Reference slot"),)
|
||||
if tuple(sorted(route.reference_slot_ids, key=_SLOT_ORDER.get)) != route.reference_slot_ids:
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "logical Reference slots are not ordered"),)
|
||||
|
||||
covered = tuple(chunk for group in schedule.physical_routes for chunk in group.logical_chunks)
|
||||
if covered != tuple(range(1, total + 1)):
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "physical groups do not cover the logical sequence"),)
|
||||
for group in schedule.physical_routes:
|
||||
if group.terminal_atomic != (len(group.logical_chunks) > 1):
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "physical group atomicity disagrees with its chunks"),)
|
||||
members = tuple(logical_slots[chunk] for chunk in group.logical_chunks)
|
||||
if group.logical_reference_slot_ids != members:
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "physical group member routes disagree with logical routes"),)
|
||||
consistent = all(member == members[0] for member in members[1:])
|
||||
if consistent:
|
||||
expected_status = "empty" if not members[0] else "consistent"
|
||||
if group.status != expected_status or group.reference_slot_ids != members[0]:
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "physical group route is not its shared member route"),)
|
||||
elif group.status != "conflict" or group.reference_slot_ids is not None:
|
||||
return (EffectivePlanIssue("invalid_routing_schedule", "conflicting physical group has an effective route"),)
|
||||
return ()
|
||||
|
||||
|
||||
def _source_issues(
|
||||
source_slot_ids: Any, image_hashes: Any, asset_count: Any
|
||||
) -> tuple[EffectivePlanIssue, ...]:
|
||||
issues: list[EffectivePlanIssue] = []
|
||||
if isinstance(asset_count, bool) or not isinstance(asset_count, int) or asset_count < 0:
|
||||
issues.append(EffectivePlanIssue("invalid_asset_count", "Reference asset count must be a non-negative integer"))
|
||||
if not isinstance(source_slot_ids, tuple):
|
||||
issues.append(EffectivePlanIssue("invalid_source_slots", "source_slot_ids must be a tuple"))
|
||||
return tuple(issues)
|
||||
if not isinstance(image_hashes, tuple):
|
||||
issues.append(EffectivePlanIssue("invalid_image_hashes", "Reference image hashes must be a tuple"))
|
||||
elif len(image_hashes) != len(source_slot_ids):
|
||||
issues.append(EffectivePlanIssue("hash_count_mismatch", "Reference image hashes do not align with fixed source slots"))
|
||||
elif any(not isinstance(value, str) or len(value) != 64 or
|
||||
any(char not in "0123456789abcdef" for char in value) for value in image_hashes):
|
||||
issues.append(EffectivePlanIssue("invalid_image_hash", "Reference image hashes must be lowercase SHA-256 values"))
|
||||
if isinstance(asset_count, int) and not isinstance(asset_count, bool):
|
||||
if asset_count > 0 and not source_slot_ids:
|
||||
issues.append(EffectivePlanIssue("missing_source_slots", "connected Reference assets have no fixed source slot IDs"))
|
||||
elif asset_count != len(source_slot_ids):
|
||||
issues.append(EffectivePlanIssue("asset_count_mismatch", "source slot IDs do not align with compact Reference assets"))
|
||||
for slot_id in source_slot_ids:
|
||||
if not isinstance(slot_id, str) or slot_id not in _SLOT_ORDER:
|
||||
issues.append(EffectivePlanIssue("unknown_source_slot", f"unknown Reference source slot: {slot_id!r}", str(slot_id)))
|
||||
if not any(issue.code == "unknown_source_slot" for issue in issues):
|
||||
if len(set(source_slot_ids)) != len(source_slot_ids):
|
||||
issues.append(EffectivePlanIssue("duplicate_source_slot", "a fixed Reference source slot occurs more than once"))
|
||||
elif tuple(sorted(source_slot_ids, key=_SLOT_ORDER.get)) != source_slot_ids:
|
||||
issues.append(EffectivePlanIssue("source_slot_order", "source slot IDs do not follow the compact R1-R9 order"))
|
||||
return tuple(issues)
|
||||
|
||||
|
||||
def _picture_map(
|
||||
requested_source_slot_ids: tuple[str, ...],
|
||||
connected_asset_indices: dict[str, int],
|
||||
image_hashes: tuple[str, ...],
|
||||
*,
|
||||
has_first_image: bool,
|
||||
has_last_image: bool,
|
||||
) -> PictureMap:
|
||||
first_number = 1 if has_first_image else None
|
||||
last_number = 1 + int(has_first_image) if has_last_image else None
|
||||
picture_offset = int(has_first_image) + int(has_last_image)
|
||||
selected = tuple(slot_id for slot_id in requested_source_slot_ids if slot_id in connected_asset_indices)
|
||||
references = tuple(
|
||||
ReferencePicture(
|
||||
slot_id, connected_asset_indices[slot_id],
|
||||
image_hashes[connected_asset_indices[slot_id]], picture_offset + index,
|
||||
)
|
||||
for index, slot_id in enumerate(selected, start=1)
|
||||
)
|
||||
return PictureMap(first_number, last_number, references)
|
||||
|
||||
|
||||
def compile_group_picture_map(
|
||||
*,
|
||||
source_slot_ids: tuple[str, ...],
|
||||
reference_image_hashes: tuple[str, ...],
|
||||
has_first_image: bool,
|
||||
has_last_image: bool,
|
||||
) -> PictureMap:
|
||||
"""Use RR-R3's Picture numbering for one already-selected physical group."""
|
||||
|
||||
issues = _source_issues(source_slot_ids, reference_image_hashes, len(source_slot_ids))
|
||||
if issues or type(has_first_image) is not bool or type(has_last_image) is not bool:
|
||||
raise ValueError("invalid internal Reference group identity or anchor flags")
|
||||
return _picture_map(
|
||||
source_slot_ids,
|
||||
{slot_id: index for index, slot_id in enumerate(source_slot_ids)},
|
||||
reference_image_hashes,
|
||||
has_first_image=has_first_image,
|
||||
has_last_image=has_last_image,
|
||||
)
|
||||
|
||||
|
||||
def compile_effective_reference_plan(
|
||||
*,
|
||||
routing_schedule: ReferenceRoutingSchedule,
|
||||
source_slot_ids: tuple[str, ...],
|
||||
reference_image_hashes: tuple[str, ...],
|
||||
reference_asset_count: int,
|
||||
has_first_image: bool,
|
||||
has_last_image: bool,
|
||||
) -> EffectiveReferencePlan:
|
||||
"""Map full logical routes to compact assets and Core Qwen Picture numbers.
|
||||
|
||||
Anchor flags describe the presentation being planned. The caller retains
|
||||
responsibility for choosing when First/Last are supplied to conditioning.
|
||||
"""
|
||||
|
||||
if not isinstance(routing_schedule, ReferenceRoutingSchedule):
|
||||
return EffectiveReferencePlan(
|
||||
mode="invalid", legacy_fallback=False, connected_source_slot_ids=(),
|
||||
issues=(EffectivePlanIssue("invalid_routing_schedule", "expected an RR-R2 routing schedule"),),
|
||||
)
|
||||
issues = [
|
||||
*_schedule_issues(routing_schedule),
|
||||
*_source_issues(source_slot_ids, reference_image_hashes, reference_asset_count),
|
||||
]
|
||||
if type(has_first_image) is not bool or type(has_last_image) is not bool:
|
||||
issues.append(EffectivePlanIssue("invalid_anchor_flags", "First/Last presence flags must be booleans"))
|
||||
if issues:
|
||||
return EffectiveReferencePlan(
|
||||
mode=routing_schedule.mode,
|
||||
legacy_fallback=routing_schedule.legacy_fallback,
|
||||
connected_source_slot_ids=source_slot_ids if isinstance(source_slot_ids, tuple) else (),
|
||||
issues=tuple(issues),
|
||||
)
|
||||
|
||||
asset_indices = {slot_id: index for index, slot_id in enumerate(source_slot_ids)}
|
||||
logical_routes = tuple(
|
||||
LogicalEffectiveRoute(
|
||||
logical_chunk=route.logical_chunk,
|
||||
requested_source_slot_ids=route.reference_slot_ids,
|
||||
picture_map=_picture_map(
|
||||
route.reference_slot_ids, asset_indices, reference_image_hashes,
|
||||
has_first_image=has_first_image, has_last_image=has_last_image,
|
||||
),
|
||||
)
|
||||
for route in routing_schedule.logical_routes
|
||||
)
|
||||
physical_routes: list[PhysicalEffectiveRoute] = []
|
||||
for group in routing_schedule.physical_routes:
|
||||
if group.status == "conflict":
|
||||
physical_routes.append(
|
||||
PhysicalEffectiveRoute(
|
||||
physical_group=group.physical_group,
|
||||
logical_chunks=group.logical_chunks,
|
||||
terminal_atomic=group.terminal_atomic,
|
||||
routing_status=group.status,
|
||||
status="conflict",
|
||||
requested_source_slot_ids=None,
|
||||
picture_map=None,
|
||||
)
|
||||
)
|
||||
continue
|
||||
requested = group.reference_slot_ids
|
||||
assert requested is not None
|
||||
picture_map = _picture_map(
|
||||
requested, asset_indices, reference_image_hashes,
|
||||
has_first_image=has_first_image, has_last_image=has_last_image,
|
||||
)
|
||||
physical_routes.append(
|
||||
PhysicalEffectiveRoute(
|
||||
physical_group=group.physical_group,
|
||||
logical_chunks=group.logical_chunks,
|
||||
terminal_atomic=group.terminal_atomic,
|
||||
routing_status=group.status,
|
||||
status="active" if picture_map.references else "empty",
|
||||
requested_source_slot_ids=requested,
|
||||
picture_map=picture_map,
|
||||
)
|
||||
)
|
||||
return EffectiveReferencePlan(
|
||||
mode=routing_schedule.mode,
|
||||
legacy_fallback=routing_schedule.legacy_fallback,
|
||||
connected_source_slot_ids=source_slot_ids,
|
||||
logical_routes=logical_routes,
|
||||
physical_routes=tuple(physical_routes),
|
||||
)
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Fixed nine-slot Reference Images input for the V3.9 sampler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from ..reference import ReferenceImageBundle
|
||||
from .reference_routing import REFERENCE_SLOT_IDS, parse_chunk_selector
|
||||
|
||||
|
||||
REFERENCE_IMAGES_V39_TYPE = "H3_CONTINUUM_REFERENCE_IMAGES_V39"
|
||||
REFERENCE_USE_ALL = "All chunks"
|
||||
REFERENCE_USE_PER_CHUNK = "Per chunk"
|
||||
REFERENCE_USE_OPTIONS = (REFERENCE_USE_ALL, REFERENCE_USE_PER_CHUNK)
|
||||
MAX_CHUNKS = 16 # Current Sampler schema limit; checked in integration tests.
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceImagesV39Bundle:
|
||||
images: tuple[torch.Tensor | None, ...]
|
||||
reference_use: str
|
||||
selectors: tuple[str, ...]
|
||||
|
||||
|
||||
def _canonical_selector(value: str) -> str:
|
||||
parsed = parse_chunk_selector(value, total_chunks=MAX_CHUNKS)
|
||||
if not parsed.valid:
|
||||
details = "; ".join(issue.message for issue in parsed.issues)
|
||||
raise ValueError(f"invalid V3.9 Reference chunk selection: {details}")
|
||||
return ",".join(map(str, parsed.logical_chunks)) or "off"
|
||||
|
||||
|
||||
def effective_reference_inputs(bundle: ReferenceImagesV39Bundle | None, *, chunks: int):
|
||||
"""Return legacy-shaped images and private selectors without changing saved data."""
|
||||
if not isinstance(chunks, int) or isinstance(chunks, bool) or not 1 <= chunks <= MAX_CHUNKS:
|
||||
raise ValueError(f"V3.9 Reference Chunks must be within 1..{MAX_CHUNKS}")
|
||||
if bundle is None:
|
||||
images = (None,) * 9
|
||||
mode = REFERENCE_USE_ALL
|
||||
selections = ("off",) * 9
|
||||
elif (not isinstance(bundle, ReferenceImagesV39Bundle)
|
||||
or len(bundle.images) != 9 or len(bundle.selectors) != 9
|
||||
or bundle.reference_use not in REFERENCE_USE_OPTIONS):
|
||||
raise ValueError("V3.9 Reference Images requires its nine-slot V3.9 bundle")
|
||||
else:
|
||||
images = bundle.images
|
||||
mode = bundle.reference_use
|
||||
selections = bundle.selectors
|
||||
selectors = {}
|
||||
for index, (image, selection) in enumerate(zip(images, selections, strict=True), start=1):
|
||||
if image is None:
|
||||
selectors[f"R{index}"] = "off"
|
||||
elif mode == REFERENCE_USE_ALL:
|
||||
selectors[f"R{index}"] = "all"
|
||||
else:
|
||||
parsed = parse_chunk_selector(selection, total_chunks=MAX_CHUNKS)
|
||||
if not parsed.valid:
|
||||
details = "; ".join(issue.message for issue in parsed.issues)
|
||||
raise ValueError(f"R{index} chunk selection: {details}")
|
||||
active = (n for n in parsed.logical_chunks if n <= chunks)
|
||||
selectors[f"R{index}"] = ",".join(map(str, active)) or "off"
|
||||
if tuple(selectors) != REFERENCE_SLOT_IDS:
|
||||
raise ValueError("V3.9 Reference slot ordering changed")
|
||||
return images[:3], ReferenceImageBundle(images[3:]), selectors
|
||||
|
||||
|
||||
class H3ContinuumReferenceImagesV39:
|
||||
CATEGORY = "MiniMax H3/Continuum/Input"
|
||||
DESCRIPTION = "Connect up to nine fixed Reference Images; choose chunks in this node."
|
||||
RETURN_TYPES = (REFERENCE_IMAGES_V39_TYPE,)
|
||||
RETURN_NAMES = ("reference_images",)
|
||||
FUNCTION = "pack"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"reference_use": (REFERENCE_USE_OPTIONS, {
|
||||
"default": REFERENCE_USE_ALL,
|
||||
"display_name": "Reference Use",
|
||||
"tooltip": "All chunks uses connected images throughout. Per chunk uses the assignments below.",
|
||||
}),
|
||||
**{
|
||||
f"reference_r{index}_chunks": ("STRING", {
|
||||
"default": "off", "advanced": True,
|
||||
"display_name": f"Image {index} chunks",
|
||||
"tooltip": f"Saved assignments for Reference Image {index}; use the in-node table.",
|
||||
}) for index in range(1, 10)
|
||||
},
|
||||
},
|
||||
"optional": {
|
||||
f"reference_image_{index}": ("IMAGE", {
|
||||
"display_name": f"Reference Image {index} (Optional)",
|
||||
"tooltip": f"Fixed Image {index} / @R{index}. Its number never changes with other connections.",
|
||||
}) for index in range(1, 10)
|
||||
},
|
||||
}
|
||||
|
||||
def pack(self, reference_use=REFERENCE_USE_ALL, **kwargs):
|
||||
if reference_use not in REFERENCE_USE_OPTIONS:
|
||||
raise ValueError("Reference Use must be All chunks or Per chunk")
|
||||
images = tuple(kwargs.get(f"reference_image_{index}") for index in range(1, 10))
|
||||
selectors = tuple(
|
||||
(_canonical_selector(kwargs.get(f"reference_r{index}_chunks", "off"))
|
||||
if reference_use == REFERENCE_USE_PER_CHUNK else
|
||||
str(kwargs.get(f"reference_r{index}_chunks", "off")))
|
||||
for index in range(1, 10)
|
||||
)
|
||||
return (ReferenceImagesV39Bundle(images, reference_use, selectors),)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Read-only V3.9 projection of the Reference contract used by this Queue.
|
||||
|
||||
This module never selects images or changes a route. Pending groups are shown
|
||||
as settings only; accepted groups use the frozen/observed RR-R5 descriptor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .reference_runtime import ReferenceRoutingRuntime
|
||||
|
||||
|
||||
def project_reference_plan(runtime: ReferenceRoutingRuntime) -> dict[str, Any]:
|
||||
if not isinstance(runtime, ReferenceRoutingRuntime):
|
||||
raise TypeError("Reference Plan Inspector requires the active routing runtime")
|
||||
accepted = int(runtime.accepted_chunks)
|
||||
generated = set(runtime.generated_groups)
|
||||
groups: list[dict[str, Any]] = []
|
||||
for route in runtime.schedule.physical_routes:
|
||||
group = int(route.physical_group)
|
||||
complete = route.logical_chunks[-1] <= accepted
|
||||
item: dict[str, Any] = {
|
||||
"physical_group": group,
|
||||
"logical_chunks": list(route.logical_chunks),
|
||||
"requested_slots": list(route.reference_slot_ids or ()),
|
||||
"status": (
|
||||
"pending" if not complete else
|
||||
"generated" if group in generated else "reused"
|
||||
),
|
||||
}
|
||||
if complete:
|
||||
contract = runtime.group_contract_for(group)
|
||||
item["contract_sha256"] = contract["sha256"]
|
||||
item["descriptor"] = contract["descriptor"]
|
||||
item["warnings"] = list(runtime.group_warnings.get(group, ()))
|
||||
groups.append(item)
|
||||
return {
|
||||
"version": 1,
|
||||
"mode": runtime.schedule.mode,
|
||||
"accepted_chunks": accepted,
|
||||
"connected_slots": list(runtime.inputs.connected_source_slot_ids),
|
||||
"groups": groups,
|
||||
}
|
||||
|
||||
|
||||
def format_reference_plan(plan: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"Reference Plan Inspector (last Queue, runtime verified)",
|
||||
f"Mode: {plan['mode']}; accepted chunks: {plan['accepted_chunks']}",
|
||||
"Connected: " + (", ".join(plan["connected_slots"]) or "none"),
|
||||
]
|
||||
for group in plan["groups"]:
|
||||
label = ",".join(str(chunk) for chunk in group["logical_chunks"])
|
||||
head = f"Group {group['physical_group']} [C{label}] {group['status']}"
|
||||
descriptor = group.get("descriptor")
|
||||
if descriptor is None:
|
||||
lines.append(head + " — configured " + (
|
||||
", ".join(group["requested_slots"]) or "no References"
|
||||
))
|
||||
continue
|
||||
mapping = ", ".join(
|
||||
f"{ref['source_slot_id']}=<Picture {ref['picture_number']}>"
|
||||
for ref in descriptor["references"]
|
||||
) or "no References"
|
||||
lines.append(head + " — " + mapping + f"; contract={group['contract_sha256'][:12]}")
|
||||
for warning in group["warnings"]:
|
||||
lines.append(" Warning: " + warning)
|
||||
return "\n".join(lines)
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
"""Pure V3.9 Reference selector parsing and group schedule planning.
|
||||
|
||||
V3.9 Custom consumes this plan through the Reference runtime and storage
|
||||
contracts. Planning here remains free of Sampling and persistence side effects.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..branch_provenance import BranchProvenanceError, physical_groups
|
||||
|
||||
|
||||
REFERENCE_SLOT_IDS = tuple(f"R{index}" for index in range(1, 10))
|
||||
|
||||
_INTEGER_TOKEN = re.compile(r"^[0-9]+$")
|
||||
_RANGE_TOKEN = re.compile(r"^([0-9]+)\s*-\s*([0-9]+)$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RoutingIssue:
|
||||
"""A classified parser/planner issue; it does not itself stop execution."""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
slot_id: str | None = None
|
||||
selector: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChunkSelectorResult:
|
||||
"""Parsed selector with all/off kept distinct from explicit chunk lists."""
|
||||
|
||||
kind: str
|
||||
logical_chunks: tuple[int, ...]
|
||||
issues: tuple[RoutingIssue, ...] = ()
|
||||
|
||||
@property
|
||||
def valid(self) -> bool:
|
||||
return self.kind != "invalid" and not self.issues
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LogicalReferenceRoute:
|
||||
"""Reference identities selected for one project-wide logical chunk."""
|
||||
|
||||
logical_chunk: int
|
||||
reference_slot_ids: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PhysicalReferenceRoute:
|
||||
"""Routing facts for one authoritative atomic physical sampling group."""
|
||||
|
||||
physical_group: int
|
||||
logical_chunks: tuple[int, ...]
|
||||
terminal_atomic: bool
|
||||
logical_reference_slot_ids: tuple[tuple[str, ...], ...]
|
||||
status: str
|
||||
reference_slot_ids: tuple[str, ...] | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceRoutingSchedule:
|
||||
"""Full logical schedule plus its physical-group compatibility view."""
|
||||
|
||||
mode: str
|
||||
legacy_fallback: bool
|
||||
total_chunks: int | None
|
||||
terminal_merge_enabled: bool | None
|
||||
logical_routes: tuple[LogicalReferenceRoute, ...] = ()
|
||||
physical_routes: tuple[PhysicalReferenceRoute, ...] = ()
|
||||
issues: tuple[RoutingIssue, ...] = ()
|
||||
|
||||
@property
|
||||
def valid(self) -> bool:
|
||||
return not self.issues
|
||||
|
||||
|
||||
def _invalid_selector(code: str, message: str, selector: str | None) -> ChunkSelectorResult:
|
||||
return ChunkSelectorResult(
|
||||
kind="invalid",
|
||||
logical_chunks=(),
|
||||
issues=(RoutingIssue(code=code, message=message, selector=selector),),
|
||||
)
|
||||
|
||||
|
||||
def parse_chunk_selector(selector: Any, *, total_chunks: int) -> ChunkSelectorResult:
|
||||
"""Parse all/off, a one-based chunk, inclusive ranges, and their combinations.
|
||||
|
||||
Invalid values are returned as structured issues so callers can inspect
|
||||
them without introducing a new runtime execution stop.
|
||||
"""
|
||||
|
||||
if isinstance(total_chunks, bool) or not isinstance(total_chunks, int) or total_chunks < 1:
|
||||
return _invalid_selector(
|
||||
"invalid_chunk_count",
|
||||
"total_chunks must be a positive integer",
|
||||
selector if isinstance(selector, str) else None,
|
||||
)
|
||||
if not isinstance(selector, str):
|
||||
return _invalid_selector(
|
||||
"selector_not_string",
|
||||
"selector must be a string",
|
||||
None,
|
||||
)
|
||||
|
||||
text = selector.strip()
|
||||
normalized = text.casefold()
|
||||
if not normalized:
|
||||
return _invalid_selector("selector_empty", "selector is empty", selector)
|
||||
if normalized == "all":
|
||||
return ChunkSelectorResult("all", tuple(range(1, total_chunks + 1)))
|
||||
if normalized == "off":
|
||||
return ChunkSelectorResult("off", ())
|
||||
|
||||
selected: set[int] = set()
|
||||
issues: list[RoutingIssue] = []
|
||||
for raw_token in text.split(","):
|
||||
token = raw_token.strip()
|
||||
if not token:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"empty_selector_item",
|
||||
"selector contains an empty comma-separated item",
|
||||
selector=selector,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if _INTEGER_TOKEN.fullmatch(token):
|
||||
start = end = int(token)
|
||||
else:
|
||||
match = _RANGE_TOKEN.fullmatch(token)
|
||||
if match is None:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"invalid_selector_item",
|
||||
f"invalid selector item: {token!r}",
|
||||
selector=selector,
|
||||
)
|
||||
)
|
||||
continue
|
||||
start, end = int(match.group(1)), int(match.group(2))
|
||||
if start > end:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"reversed_range",
|
||||
f"range start {start} exceeds end {end}",
|
||||
selector=selector,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
if start < 1 or end > total_chunks:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"chunk_out_of_range",
|
||||
f"selector item {token!r} is outside 1..{total_chunks}",
|
||||
selector=selector,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
values = set(range(start, end + 1))
|
||||
if selected.intersection(values):
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"duplicate_chunk",
|
||||
f"selector item {token!r} overlaps an earlier item",
|
||||
selector=selector,
|
||||
)
|
||||
)
|
||||
continue
|
||||
selected.update(values)
|
||||
|
||||
if issues:
|
||||
return ChunkSelectorResult("invalid", (), tuple(issues))
|
||||
return ChunkSelectorResult("chunks", tuple(sorted(selected)))
|
||||
|
||||
|
||||
def _invalid_schedule(
|
||||
*,
|
||||
mode: str,
|
||||
legacy_fallback: bool,
|
||||
total_chunks: int | None,
|
||||
terminal_merge_enabled: bool | None,
|
||||
issues: tuple[RoutingIssue, ...],
|
||||
) -> ReferenceRoutingSchedule:
|
||||
return ReferenceRoutingSchedule(
|
||||
mode=mode,
|
||||
legacy_fallback=legacy_fallback,
|
||||
total_chunks=total_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
issues=issues,
|
||||
)
|
||||
|
||||
|
||||
def compile_reference_routing_schedule(
|
||||
*,
|
||||
total_chunks: int,
|
||||
terminal_merge_enabled: bool,
|
||||
mode: Any = None,
|
||||
selectors_by_slot: Any = None,
|
||||
) -> ReferenceRoutingSchedule:
|
||||
"""Compile routing over full logical chunk numbers and physical groups.
|
||||
|
||||
An absent mode preserves legacy All behavior. Custom mode requires an
|
||||
explicit selector for each fixed R1-R9 slot; explicit all-off is valid and
|
||||
remains distinct from an absent mode. Terminal atomic groups with
|
||||
different member routes are surfaced as conflicts without auto-correction.
|
||||
"""
|
||||
|
||||
legacy_fallback = mode is None
|
||||
normalized_mode = "all" if legacy_fallback else None
|
||||
issues: list[RoutingIssue] = []
|
||||
|
||||
if isinstance(total_chunks, bool) or not isinstance(total_chunks, int) or total_chunks < 1:
|
||||
issues.append(RoutingIssue("invalid_chunk_count", "total_chunks must be a positive integer"))
|
||||
if type(terminal_merge_enabled) is not bool:
|
||||
issues.append(
|
||||
RoutingIssue("invalid_terminal_merge", "terminal_merge_enabled must be a boolean")
|
||||
)
|
||||
|
||||
if not legacy_fallback:
|
||||
if not isinstance(mode, str):
|
||||
issues.append(RoutingIssue("invalid_mode", "mode must be All or Custom"))
|
||||
else:
|
||||
token = mode.strip().casefold()
|
||||
if token == "all":
|
||||
normalized_mode = "all"
|
||||
elif token == "custom":
|
||||
normalized_mode = "custom"
|
||||
else:
|
||||
issues.append(RoutingIssue("invalid_mode", "mode must be All or Custom"))
|
||||
|
||||
if issues:
|
||||
return _invalid_schedule(
|
||||
mode=normalized_mode or "invalid",
|
||||
legacy_fallback=legacy_fallback,
|
||||
total_chunks=total_chunks if isinstance(total_chunks, int) and not isinstance(total_chunks, bool) else None,
|
||||
terminal_merge_enabled=terminal_merge_enabled if type(terminal_merge_enabled) is bool else None,
|
||||
issues=tuple(issues),
|
||||
)
|
||||
|
||||
slot_selectors: dict[str, ChunkSelectorResult] = {}
|
||||
if normalized_mode == "custom":
|
||||
if not isinstance(selectors_by_slot, Mapping):
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"selectors_required",
|
||||
"Custom mode requires an explicit selector map for R1-R9",
|
||||
)
|
||||
)
|
||||
else:
|
||||
for slot_id in selectors_by_slot:
|
||||
if slot_id not in REFERENCE_SLOT_IDS:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"unknown_reference_slot",
|
||||
f"unknown Reference slot: {slot_id!r}",
|
||||
slot_id=str(slot_id),
|
||||
)
|
||||
)
|
||||
for slot_id in REFERENCE_SLOT_IDS:
|
||||
if slot_id not in selectors_by_slot:
|
||||
issues.append(
|
||||
RoutingIssue(
|
||||
"missing_selector",
|
||||
f"Custom mode has no selector for {slot_id}",
|
||||
slot_id=slot_id,
|
||||
)
|
||||
)
|
||||
continue
|
||||
parsed = parse_chunk_selector(
|
||||
selectors_by_slot[slot_id], total_chunks=total_chunks
|
||||
)
|
||||
if not parsed.valid:
|
||||
issues.extend(
|
||||
RoutingIssue(
|
||||
issue.code,
|
||||
issue.message,
|
||||
slot_id=slot_id,
|
||||
selector=issue.selector,
|
||||
)
|
||||
for issue in parsed.issues
|
||||
)
|
||||
else:
|
||||
slot_selectors[slot_id] = parsed
|
||||
|
||||
if issues:
|
||||
return _invalid_schedule(
|
||||
mode=normalized_mode or "invalid",
|
||||
legacy_fallback=legacy_fallback,
|
||||
total_chunks=total_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
issues=tuple(issues),
|
||||
)
|
||||
|
||||
logical_routes: list[LogicalReferenceRoute] = []
|
||||
for logical_chunk in range(1, total_chunks + 1):
|
||||
if normalized_mode == "all":
|
||||
selected_slots = REFERENCE_SLOT_IDS
|
||||
else:
|
||||
selected_slots = tuple(
|
||||
slot_id
|
||||
for slot_id in REFERENCE_SLOT_IDS
|
||||
if logical_chunk in slot_selectors[slot_id].logical_chunks
|
||||
)
|
||||
logical_routes.append(
|
||||
LogicalReferenceRoute(logical_chunk, tuple(selected_slots))
|
||||
)
|
||||
|
||||
try:
|
||||
groups = physical_groups(
|
||||
chunks=total_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
)
|
||||
except BranchProvenanceError as exc:
|
||||
return _invalid_schedule(
|
||||
mode=normalized_mode or "invalid",
|
||||
legacy_fallback=legacy_fallback,
|
||||
total_chunks=total_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
issues=(RoutingIssue("invalid_physical_groups", str(exc)),),
|
||||
)
|
||||
|
||||
routes_by_chunk = {route.logical_chunk: route.reference_slot_ids for route in logical_routes}
|
||||
physical_routes: list[PhysicalReferenceRoute] = []
|
||||
for group in groups:
|
||||
chunks = tuple(range(group.start, group.end + 1))
|
||||
member_routes = tuple(routes_by_chunk[chunk] for chunk in chunks)
|
||||
first_route = member_routes[0]
|
||||
consistent = all(route == first_route for route in member_routes[1:])
|
||||
if not consistent:
|
||||
group_status = "conflict"
|
||||
group_slots = None
|
||||
elif not first_route:
|
||||
group_status = "empty"
|
||||
group_slots = ()
|
||||
else:
|
||||
group_status = "consistent"
|
||||
group_slots = first_route
|
||||
physical_routes.append(
|
||||
PhysicalReferenceRoute(
|
||||
physical_group=int(group.physical_group),
|
||||
logical_chunks=chunks,
|
||||
terminal_atomic=group.start != group.end,
|
||||
logical_reference_slot_ids=member_routes,
|
||||
status=group_status,
|
||||
reference_slot_ids=group_slots,
|
||||
)
|
||||
)
|
||||
|
||||
return ReferenceRoutingSchedule(
|
||||
mode=normalized_mode or "invalid",
|
||||
legacy_fallback=legacy_fallback,
|
||||
total_chunks=total_chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
logical_routes=tuple(logical_routes),
|
||||
physical_routes=tuple(physical_routes),
|
||||
)
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
"""V3.9 Custom group-local Reference conditioning.
|
||||
|
||||
The V3.9 caller supplies this adapter after routing is resolved. V3.8 and
|
||||
V3.9 All retain the legacy path; saved-run compatibility is handled by the
|
||||
separate Reference storage contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..reference import (
|
||||
ReferenceAssets,
|
||||
ReferenceImageBundle,
|
||||
encode_reference_latents,
|
||||
encode_reference_latents_cached,
|
||||
encode_reference_prompt,
|
||||
prepare_reference_assets,
|
||||
resolve_reference_image_inputs,
|
||||
)
|
||||
from ..v2.h3_builder import encode_prompt_conditioning, encode_prompt_conditioning_cached
|
||||
from .reference_effective_plan import PictureMap, compile_group_picture_map
|
||||
from .reference_routing import REFERENCE_SLOT_IDS, ReferenceRoutingSchedule
|
||||
|
||||
|
||||
_REFERENCE_TAG = re.compile(r"(?<![A-Za-z0-9_])@R([1-9])(?![A-Za-z0-9_])")
|
||||
|
||||
|
||||
class ReferenceRuntimeContractError(ValueError):
|
||||
"""An internal RR-R4 caller supplied an unusable routing contract."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ReferenceRoutingRequest:
|
||||
"""Private V3.9 node-to-V2 request; never part of a saved node schema."""
|
||||
|
||||
selectors_by_slot: dict[str, str]
|
||||
inputs: ReferenceInputSet
|
||||
runtime: ReferenceRoutingRuntime | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceInputSet:
|
||||
"""Raw R1-R9 sockets; no image is validated, resized, or VAE-encoded here."""
|
||||
|
||||
images: tuple[torch.Tensor | None, ...]
|
||||
output_width: int
|
||||
output_height: int
|
||||
size_mode: str
|
||||
|
||||
@classmethod
|
||||
def from_inputs(
|
||||
cls,
|
||||
*,
|
||||
reference_image_1: torch.Tensor | None = None,
|
||||
reference_image_2: torch.Tensor | None = None,
|
||||
reference_image_3: torch.Tensor | None = None,
|
||||
reference_image_4: torch.Tensor | None = None,
|
||||
reference_image_5: torch.Tensor | None = None,
|
||||
image_references: ReferenceImageBundle | None = None,
|
||||
output_width: int,
|
||||
output_height: int,
|
||||
size_mode: str,
|
||||
) -> ReferenceInputSet:
|
||||
images = resolve_reference_image_inputs(
|
||||
reference_image_1, reference_image_2, reference_image_3,
|
||||
reference_image_4, reference_image_5, image_references,
|
||||
)
|
||||
return cls(images, int(output_width), int(output_height), str(size_mode))
|
||||
|
||||
@property
|
||||
def connected_source_slot_ids(self) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
slot_id for slot_id, image in zip(REFERENCE_SLOT_IDS, self.images, strict=True)
|
||||
if image is not None
|
||||
)
|
||||
|
||||
def prepare_selected(self, selected_source_slot_ids: tuple[str, ...]) -> ReferenceAssets | None:
|
||||
return prepare_reference_assets(
|
||||
reference_image_1=self.images[0],
|
||||
reference_image_2=self.images[1],
|
||||
reference_image_3=self.images[2],
|
||||
image_references=ReferenceImageBundle(self.images[3:]),
|
||||
output_width=self.output_width,
|
||||
output_height=self.output_height,
|
||||
size_mode=self.size_mode,
|
||||
selected_source_slot_ids=selected_source_slot_ids,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GroupReferenceConditioning:
|
||||
physical_group: int
|
||||
logical_chunks: tuple[int, ...]
|
||||
requested_source_slot_ids: tuple[str, ...]
|
||||
selected_assets: ReferenceAssets | None
|
||||
picture_map: PictureMap
|
||||
effective_prompt: str
|
||||
conditioning: list[list[Any]]
|
||||
warnings: tuple[str, ...]
|
||||
|
||||
|
||||
def rewrite_reference_tags(prompt: str, picture_map: PictureMap) -> tuple[str, tuple[str, ...]]:
|
||||
"""Rewrite active @R tags only; inactive tags are diagnostic, never blocking."""
|
||||
|
||||
tags = {item.source_slot_id: item.picture_tag for item in picture_map.references}
|
||||
inactive: set[str] = set()
|
||||
|
||||
def replace_tag(match: re.Match[str]) -> str:
|
||||
slot_id = f"R{match.group(1)}"
|
||||
replacement = tags.get(slot_id)
|
||||
if replacement is None:
|
||||
inactive.add(slot_id)
|
||||
return match.group(0)
|
||||
return replacement
|
||||
|
||||
effective = _REFERENCE_TAG.sub(replace_tag, str(prompt))
|
||||
warnings = tuple(
|
||||
f"{slot_id} is not active in this physical group; {slot_id}'s @R tag was left unchanged"
|
||||
for slot_id in REFERENCE_SLOT_IDS if slot_id in inactive
|
||||
)
|
||||
return effective, warnings
|
||||
|
||||
|
||||
def resolve_reference_group_prompt(
|
||||
*, physical_group: int, prompt: str,
|
||||
requested_source_slot_ids: tuple[str, ...],
|
||||
connected_source_slot_ids: tuple[str, ...],
|
||||
picture_map: PictureMap,
|
||||
) -> tuple[str, tuple[str, ...]]:
|
||||
"""Use the same prompt and warning rules for planned and sampled groups."""
|
||||
|
||||
effective_prompt, prompt_warnings = rewrite_reference_tags(prompt, picture_map)
|
||||
connected = set(connected_source_slot_ids)
|
||||
disconnected_warnings = tuple(
|
||||
f"{slot_id} is routed to physical group {physical_group} but no image is connected"
|
||||
for slot_id in requested_source_slot_ids if slot_id not in connected
|
||||
)
|
||||
return effective_prompt, disconnected_warnings + prompt_warnings
|
||||
|
||||
|
||||
class ReferenceRoutingRuntime:
|
||||
"""Prepare and encode only the images selected by a physical group."""
|
||||
|
||||
def __init__(self, *, schedule: ReferenceRoutingSchedule, inputs: ReferenceInputSet):
|
||||
if not isinstance(schedule, ReferenceRoutingSchedule) or not schedule.valid:
|
||||
raise ReferenceRuntimeContractError("RR-R4 requires a valid RR-R2 routing schedule")
|
||||
if not isinstance(inputs, ReferenceInputSet) or len(inputs.images) != len(REFERENCE_SLOT_IDS):
|
||||
raise ReferenceRuntimeContractError("RR-R4 requires nine fixed raw Reference slots")
|
||||
self.schedule = schedule
|
||||
self.inputs = inputs
|
||||
# RR-R4 has no saved-run/prefix contract. Keep its session identity
|
||||
# queue-local without reading images assigned only to later groups.
|
||||
self._run_nonce = uuid.uuid4().hex
|
||||
self.storage_plan: Any = None
|
||||
self.generated_groups: tuple[int, ...] = ()
|
||||
self.accepted_chunks = 0
|
||||
self.observed_group_contracts: dict[int, dict[str, Any]] = {}
|
||||
self.group_warnings: dict[int, tuple[str, ...]] = {}
|
||||
|
||||
def group_contract_for(self, physical_group: int) -> dict[str, Any]:
|
||||
group = int(physical_group)
|
||||
observed = self.observed_group_contracts.get(group)
|
||||
if observed is not None:
|
||||
return observed
|
||||
if self.storage_plan is not None:
|
||||
return self.storage_plan.group_for(group)
|
||||
raise ReferenceRuntimeContractError(
|
||||
f"physical group {group} has no observed Reference contract"
|
||||
)
|
||||
|
||||
def check_physical_contract(self, *, chunks: int, terminal_merge_enabled: bool) -> None:
|
||||
if self.schedule.total_chunks != int(chunks) or self.schedule.terminal_merge_enabled != bool(terminal_merge_enabled):
|
||||
raise ReferenceRuntimeContractError(
|
||||
"RR-R4 routing schedule does not match the current physical group topology"
|
||||
)
|
||||
if any(route.status == "conflict" for route in self.schedule.physical_routes):
|
||||
raise ReferenceRuntimeContractError(
|
||||
"RR-R4 cannot select one Reference set for a conflicting terminal physical group"
|
||||
)
|
||||
|
||||
@property
|
||||
def has_selected_references(self) -> bool:
|
||||
connected = set(self.inputs.connected_source_slot_ids)
|
||||
return any(
|
||||
slot_id in connected
|
||||
for route in self.schedule.physical_routes
|
||||
for slot_id in (route.reference_slot_ids or ())
|
||||
)
|
||||
|
||||
def global_identity(self, keyframe_identity_hash: str) -> str:
|
||||
"""Queue-local identity; RR-R5 will define persistent prefix reuse."""
|
||||
|
||||
payload = {
|
||||
"rr_r4_identity_version": 1,
|
||||
"run_nonce": self._run_nonce,
|
||||
"keyframe_identity_hash": str(keyframe_identity_hash),
|
||||
"mode": self.schedule.mode,
|
||||
"logical_routes": [
|
||||
(route.logical_chunk, route.reference_slot_ids)
|
||||
for route in self.schedule.logical_routes
|
||||
],
|
||||
"physical_groups": [
|
||||
(route.physical_group, route.logical_chunks)
|
||||
for route in self.schedule.physical_routes
|
||||
],
|
||||
"connected_source_slot_ids": self.inputs.connected_source_slot_ids,
|
||||
"size_mode": self.inputs.size_mode,
|
||||
"output_size": (self.inputs.output_width, self.inputs.output_height),
|
||||
}
|
||||
return hashlib.sha256(
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
def _route(self, physical_group: int):
|
||||
for route in self.schedule.physical_routes:
|
||||
if route.physical_group == physical_group:
|
||||
if route.status == "conflict" or route.reference_slot_ids is None:
|
||||
raise ReferenceRuntimeContractError(
|
||||
f"physical group {physical_group} has no single Reference route"
|
||||
)
|
||||
return route
|
||||
raise ReferenceRuntimeContractError(f"unknown physical group {physical_group}")
|
||||
|
||||
def prepare_group(
|
||||
self,
|
||||
*,
|
||||
physical_group: int,
|
||||
prompt: str,
|
||||
clip: Any,
|
||||
video_vae: Any,
|
||||
first_image: torch.Tensor | None,
|
||||
last_image: torch.Tensor | None,
|
||||
include_last_image: bool = True,
|
||||
reference_audio_assets: Any = None,
|
||||
timeline_video_assets: Any = None,
|
||||
reference_encode_cache: bool = False,
|
||||
cache_event: Any = None,
|
||||
prompt_conditioning_cache: bool = False,
|
||||
prompt_cache_event: Any = None,
|
||||
expected_group_contract: dict[str, Any] | None = None,
|
||||
first_frame_hash: str = "none",
|
||||
last_frame_hash: str = "none",
|
||||
terminal_prompt_policy: str | None = None,
|
||||
) -> GroupReferenceConditioning:
|
||||
route = self._route(int(physical_group))
|
||||
requested = route.reference_slot_ids
|
||||
connected = set(self.inputs.connected_source_slot_ids)
|
||||
selected = tuple(slot_id for slot_id in requested if slot_id in connected)
|
||||
assets = self.inputs.prepare_selected(selected)
|
||||
if assets is not None and assets.source_slot_ids != selected:
|
||||
raise ReferenceRuntimeContractError("prepared Reference slots differ from the selected route")
|
||||
# The legacy no-Reference path presents Last Image only to the final
|
||||
# group. With References, Core's hybrid Qwen presentation includes it
|
||||
# in every group even though the Last keyframe is attached only at end.
|
||||
presented_last_image = last_image if assets is not None or include_last_image else None
|
||||
picture_map = compile_group_picture_map(
|
||||
source_slot_ids=assets.source_slot_ids if assets is not None else (),
|
||||
reference_image_hashes=assets.image_hashes if assets is not None else (),
|
||||
has_first_image=first_image is not None,
|
||||
has_last_image=presented_last_image is not None,
|
||||
)
|
||||
effective_prompt, warnings = resolve_reference_group_prompt(
|
||||
physical_group=int(physical_group), prompt=prompt,
|
||||
requested_source_slot_ids=requested,
|
||||
connected_source_slot_ids=self.inputs.connected_source_slot_ids,
|
||||
picture_map=picture_map,
|
||||
)
|
||||
# Direct RR-R4 callers predate the saved-plan fingerprints. The normal
|
||||
# V3.9 Sequence supplies both hashes; for an unsaved direct call only,
|
||||
# derive a truthful anchor from the exact image being presented.
|
||||
if expected_group_contract is None and self.storage_plan is None:
|
||||
from ..v2.h3_builder import _tensor_fingerprint
|
||||
if first_image is not None and first_frame_hash == "none":
|
||||
first_frame_hash = _tensor_fingerprint(first_image)
|
||||
if presented_last_image is not None and last_frame_hash == "none":
|
||||
last_frame_hash = _tensor_fingerprint(presented_last_image)
|
||||
from .reference_storage_contract import (
|
||||
ReferenceStorageContractError, canonical_sha256, descriptor_from_facts,
|
||||
)
|
||||
descriptor = descriptor_from_facts(
|
||||
physical_group=int(physical_group),
|
||||
logical_chunks=route.logical_chunks,
|
||||
selected_source_slot_ids=assets.source_slot_ids if assets is not None else (),
|
||||
prepared_image_hashes=assets.image_hashes if assets is not None else (),
|
||||
picture_map=picture_map,
|
||||
first_frame_hash=first_frame_hash,
|
||||
last_frame_hash=last_frame_hash,
|
||||
effective_prompt=effective_prompt,
|
||||
size_mode=self.inputs.size_mode,
|
||||
terminal_prompt_policy=terminal_prompt_policy,
|
||||
)
|
||||
group_contract = {"descriptor": descriptor, "sha256": canonical_sha256(descriptor)}
|
||||
if expected_group_contract is not None:
|
||||
if (canonical_sha256(descriptor) != expected_group_contract.get("sha256")
|
||||
or descriptor != expected_group_contract.get("descriptor")):
|
||||
raise ReferenceStorageContractError(
|
||||
f"physical group {physical_group} Reference inputs changed after the "
|
||||
"saved-run preflight; no VAE, Prompt, or Sampling was started"
|
||||
)
|
||||
if assets is not None:
|
||||
assets = (
|
||||
encode_reference_latents_cached(video_vae, assets, cache_event=cache_event)
|
||||
if reference_encode_cache else encode_reference_latents(video_vae, assets)
|
||||
)
|
||||
conditioning = encode_reference_prompt(
|
||||
clip, effective_prompt, assets,
|
||||
first_image=first_image,
|
||||
last_image=presented_last_image,
|
||||
reference_audio_assets=reference_audio_assets,
|
||||
timeline_video_assets=timeline_video_assets,
|
||||
)
|
||||
elif (prompt_conditioning_cache and not self.has_selected_references
|
||||
and reference_audio_assets is None and timeline_video_assets is None
|
||||
and terminal_prompt_policy is None):
|
||||
# The Sequence caller enables this only for Fixed, reference-free
|
||||
# runs. Reuse the V3.8 cache and its conservative CLIP guards; keep
|
||||
# route/storage validation above and group metadata below intact.
|
||||
conditioning = encode_prompt_conditioning_cached(
|
||||
clip, effective_prompt,
|
||||
first_image=first_image,
|
||||
last_image=presented_last_image,
|
||||
first_image_fingerprint=first_frame_hash,
|
||||
last_image_fingerprint=(last_frame_hash
|
||||
if presented_last_image is not None else "none"),
|
||||
cache_enabled=True,
|
||||
cache_event=prompt_cache_event,
|
||||
)
|
||||
else:
|
||||
conditioning = encode_prompt_conditioning(
|
||||
clip, effective_prompt,
|
||||
first_image=first_image,
|
||||
last_image=presented_last_image,
|
||||
reference_audio_assets=reference_audio_assets,
|
||||
timeline_video_assets=timeline_video_assets,
|
||||
)
|
||||
existing = self.observed_group_contracts.get(int(physical_group))
|
||||
if existing is not None and existing != group_contract:
|
||||
raise ReferenceRuntimeContractError(
|
||||
f"physical group {physical_group} Reference contract changed within one Queue"
|
||||
)
|
||||
self.observed_group_contracts[int(physical_group)] = group_contract
|
||||
self.group_warnings[int(physical_group)] = warnings
|
||||
return GroupReferenceConditioning(
|
||||
physical_group=int(physical_group),
|
||||
logical_chunks=route.logical_chunks,
|
||||
requested_source_slot_ids=requested,
|
||||
selected_assets=assets,
|
||||
picture_map=picture_map,
|
||||
effective_prompt=effective_prompt,
|
||||
conditioning=conditioning,
|
||||
warnings=warnings,
|
||||
)
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Private RR-R5 physical-group Reference persistence contract.
|
||||
|
||||
The stored descriptor records *effective* conditioning, not selector spelling.
|
||||
No VAE, CLIP, Sampling, or saved-run operation is performed here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from ..branch_provenance import physical_groups
|
||||
from ..reference import HYBRID_PRESENTATION_VERSION, REFERENCE_PREPROCESS_VERSION
|
||||
from .reference_effective_plan import PictureMap, compile_group_picture_map
|
||||
from .reference_routing import REFERENCE_SLOT_IDS
|
||||
|
||||
|
||||
REFERENCE_ROUTING_CONTRACT_VERSION = 1
|
||||
SESSION_ROUTING_IDENTITY_VERSION = 1
|
||||
_HEX = frozenset("0123456789abcdef")
|
||||
|
||||
|
||||
class ReferenceStorageContractError(ValueError):
|
||||
"""A routed producer or accepted-prefix contract cannot be trusted."""
|
||||
|
||||
|
||||
def canonical_json(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
|
||||
|
||||
def canonical_sha256(value: Any) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def routed_session_identity(base_identity_hash: str) -> str:
|
||||
return canonical_sha256({
|
||||
"rr_r5_session_identity_version": SESSION_ROUTING_IDENTITY_VERSION,
|
||||
"base_identity_hash": str(base_identity_hash),
|
||||
})
|
||||
|
||||
|
||||
def _valid_sha(value: Any) -> bool:
|
||||
return isinstance(value, str) and len(value) == 64 and all(c in _HEX for c in value)
|
||||
|
||||
|
||||
def _anchor(value: str, present: bool) -> str | None:
|
||||
if not present:
|
||||
return None
|
||||
if not _valid_sha(value):
|
||||
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _present(value: str) -> bool:
|
||||
if str(value).lower() in {"", "none", "null"}:
|
||||
return False
|
||||
if not _valid_sha(value):
|
||||
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
|
||||
return True
|
||||
|
||||
|
||||
def descriptor_from_facts(
|
||||
*, physical_group: int, logical_chunks: tuple[int, ...],
|
||||
selected_source_slot_ids: tuple[str, ...],
|
||||
prepared_image_hashes: tuple[str, ...], picture_map: PictureMap,
|
||||
first_frame_hash: str, last_frame_hash: str,
|
||||
effective_prompt: str, size_mode: str,
|
||||
terminal_prompt_policy: str | None,
|
||||
) -> dict[str, Any]:
|
||||
if len(selected_source_slot_ids) != len(prepared_image_hashes):
|
||||
raise ReferenceStorageContractError("selected Reference slots and image hashes differ")
|
||||
if picture_map.source_slot_ids != selected_source_slot_ids:
|
||||
raise ReferenceStorageContractError("Reference Picture map differs from selected source slots")
|
||||
if any(slot not in REFERENCE_SLOT_IDS for slot in selected_source_slot_ids):
|
||||
raise ReferenceStorageContractError("selected Reference slot is unknown")
|
||||
if tuple(sorted(selected_source_slot_ids, key=REFERENCE_SLOT_IDS.index)) != selected_source_slot_ids:
|
||||
raise ReferenceStorageContractError("selected Reference slots are not in fixed R1-R9 order")
|
||||
if len(set(selected_source_slot_ids)) != len(selected_source_slot_ids):
|
||||
raise ReferenceStorageContractError("selected Reference slot is duplicated")
|
||||
if any(not _valid_sha(value) for value in prepared_image_hashes):
|
||||
raise ReferenceStorageContractError("prepared Reference image hash is invalid")
|
||||
if any(item.image_sha256 != prepared_image_hashes[index]
|
||||
for index, item in enumerate(picture_map.references)):
|
||||
raise ReferenceStorageContractError("Reference Picture map image hash is inconsistent")
|
||||
if not logical_chunks or tuple(range(logical_chunks[0], logical_chunks[-1] + 1)) != logical_chunks:
|
||||
raise ReferenceStorageContractError("physical group logical chunks are not contiguous")
|
||||
if int(physical_group) != logical_chunks[0]:
|
||||
raise ReferenceStorageContractError("physical group number differs from its first logical chunk")
|
||||
if (len(logical_chunks) > 1) != (terminal_prompt_policy is not None):
|
||||
raise ReferenceStorageContractError("terminal prompt policy does not match physical group")
|
||||
first_present = picture_map.first_picture_number is not None
|
||||
last_present = picture_map.last_picture_number is not None
|
||||
order = []
|
||||
if first_present:
|
||||
order.append("first_frame")
|
||||
if last_present:
|
||||
order.append("last_frame")
|
||||
order.extend(selected_source_slot_ids)
|
||||
descriptor: dict[str, Any] = {
|
||||
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
|
||||
"physical_group": int(physical_group),
|
||||
"logical_chunks": list(logical_chunks),
|
||||
"terminal_atomic": len(logical_chunks) > 1,
|
||||
"references": [
|
||||
{
|
||||
"source_slot_id": slot_id,
|
||||
"prepared_sha256": prepared_image_hashes[index],
|
||||
"picture_number": picture_map.references[index].picture_number,
|
||||
}
|
||||
for index, slot_id in enumerate(selected_source_slot_ids)
|
||||
],
|
||||
"first_anchor": _anchor(first_frame_hash, first_present),
|
||||
"last_anchor": _anchor(last_frame_hash, last_present),
|
||||
"picture_order": order,
|
||||
"picture_presentation_version": HYBRID_PRESENTATION_VERSION,
|
||||
"effective_prompt_sha256": hashlib.sha256(str(effective_prompt).encode("utf-8")).hexdigest(),
|
||||
"terminal_prompt_policy": str(terminal_prompt_policy) if len(logical_chunks) > 1 else None,
|
||||
}
|
||||
if selected_source_slot_ids:
|
||||
descriptor["reference_size_mode"] = str(size_mode)
|
||||
descriptor["reference_preprocess_version"] = REFERENCE_PREPROCESS_VERSION
|
||||
return descriptor
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReferenceStoragePlan:
|
||||
"""Canonical JSON is immutable even if the caller's source dicts are not."""
|
||||
|
||||
descriptor_jsons: tuple[str, ...]
|
||||
routing_config_sha256: str
|
||||
requested_routes: tuple[tuple[str, ...], ...]
|
||||
_group_index: dict[int, int] = field(init=False, repr=False, compare=False, hash=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
index: dict[int, int] = {}
|
||||
for position, payload in enumerate(self.descriptor_jsons):
|
||||
descriptor = json.loads(payload)
|
||||
# Preserve the former first-match lookup for duplicate group IDs.
|
||||
index.setdefault(int(descriptor["physical_group"]), position)
|
||||
object.__setattr__(self, "_group_index", index)
|
||||
|
||||
def _record_at(self, position: int) -> dict[str, Any]:
|
||||
payload = self.descriptor_jsons[position]
|
||||
return {
|
||||
"descriptor": json.loads(payload),
|
||||
"sha256": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
|
||||
}
|
||||
|
||||
@property
|
||||
def group_contracts(self) -> list[dict[str, Any]]:
|
||||
return [self._record_at(position) for position in range(len(self.descriptor_jsons))]
|
||||
|
||||
def group_for(self, physical_group: int) -> dict[str, Any]:
|
||||
position = self._group_index.get(int(physical_group))
|
||||
if position is None:
|
||||
raise ReferenceStorageContractError(
|
||||
f"physical group {physical_group} has no frozen Reference contract"
|
||||
)
|
||||
return self._record_at(position)
|
||||
|
||||
def audit(self) -> dict[str, Any]:
|
||||
return {
|
||||
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
|
||||
"routing_config_sha256": self.routing_config_sha256,
|
||||
"requested_routes": [list(route) for route in self.requested_routes],
|
||||
}
|
||||
|
||||
|
||||
def build_reference_storage_plan(
|
||||
*, runtime: Any, prompts: list[str], first_frame_hash: str,
|
||||
last_frame_hash: str, terminal_prompt: str | None = None,
|
||||
terminal_prompt_policy: str | None = None,
|
||||
through_chunk: int | None = None,
|
||||
) -> ReferenceStoragePlan:
|
||||
"""Prehash only connected R slots selected by the requested group prefix."""
|
||||
from .reference_runtime import ReferenceRoutingRuntime, resolve_reference_group_prompt
|
||||
|
||||
if not isinstance(runtime, ReferenceRoutingRuntime):
|
||||
raise ReferenceStorageContractError("routed storage requires ReferenceRoutingRuntime")
|
||||
schedule = runtime.schedule
|
||||
total = int(schedule.total_chunks or 0)
|
||||
if total < 1 or len(prompts) != total:
|
||||
raise ReferenceStorageContractError("Reference schedule and prompt count differ")
|
||||
runtime.check_physical_contract(
|
||||
chunks=total, terminal_merge_enabled=bool(schedule.terminal_merge_enabled),
|
||||
)
|
||||
limit = total if through_chunk is None else int(through_chunk)
|
||||
if limit < 1 or limit > total:
|
||||
raise ReferenceStorageContractError("Reference plan prefix is outside configured chunks")
|
||||
routes = [route for route in schedule.physical_routes if route.logical_chunks[-1] <= limit]
|
||||
if not routes or routes[-1].logical_chunks[-1] != limit:
|
||||
raise ReferenceStorageContractError("Reference plan prefix ends inside a physical group")
|
||||
connected = set(runtime.inputs.connected_source_slot_ids)
|
||||
selected = {slot for route in routes for slot in (route.reference_slot_ids or ()) if slot in connected}
|
||||
image_hashes: dict[str, str] = {}
|
||||
for slot in REFERENCE_SLOT_IDS:
|
||||
if slot not in selected:
|
||||
continue
|
||||
assets = runtime.inputs.prepare_selected((slot,))
|
||||
if assets is None or assets.source_slot_ids != (slot,) or len(assets.image_hashes) != 1:
|
||||
raise ReferenceStorageContractError(f"selected {slot} cannot be prepared")
|
||||
image_hashes[slot] = assets.image_hashes[0]
|
||||
del assets
|
||||
descriptor_jsons: list[str] = []
|
||||
requested_routes: list[tuple[str, ...]] = []
|
||||
pending_group_warnings: dict[int, tuple[str, ...]] = {}
|
||||
for route in routes:
|
||||
requested = route.reference_slot_ids
|
||||
if requested is None:
|
||||
raise ReferenceStorageContractError("conflicting terminal Reference route")
|
||||
requested_routes.append(requested)
|
||||
source_slots = tuple(slot for slot in requested if slot in connected)
|
||||
hashes = tuple(image_hashes[slot] for slot in source_slots)
|
||||
final = route.logical_chunks[-1] == total
|
||||
has_first = _present(first_frame_hash)
|
||||
has_last = _present(last_frame_hash) and (bool(source_slots) or final)
|
||||
picture_map = compile_group_picture_map(
|
||||
source_slot_ids=source_slots,
|
||||
reference_image_hashes=hashes,
|
||||
has_first_image=has_first,
|
||||
has_last_image=has_last,
|
||||
)
|
||||
prompt = (
|
||||
terminal_prompt if len(route.logical_chunks) > 1
|
||||
else prompts[route.logical_chunks[0] - 1]
|
||||
)
|
||||
if prompt is None:
|
||||
raise ReferenceStorageContractError("terminal physical prompt is unavailable")
|
||||
effective_prompt, warnings = resolve_reference_group_prompt(
|
||||
physical_group=int(route.physical_group), prompt=prompt,
|
||||
requested_source_slot_ids=requested,
|
||||
connected_source_slot_ids=runtime.inputs.connected_source_slot_ids,
|
||||
picture_map=picture_map,
|
||||
)
|
||||
pending_group_warnings[int(route.physical_group)] = warnings
|
||||
descriptor = descriptor_from_facts(
|
||||
physical_group=route.physical_group,
|
||||
logical_chunks=route.logical_chunks,
|
||||
selected_source_slot_ids=source_slots,
|
||||
prepared_image_hashes=hashes,
|
||||
picture_map=picture_map,
|
||||
first_frame_hash=first_frame_hash,
|
||||
last_frame_hash=last_frame_hash,
|
||||
effective_prompt=effective_prompt,
|
||||
size_mode=runtime.inputs.size_mode,
|
||||
terminal_prompt_policy=terminal_prompt_policy if len(route.logical_chunks) > 1 else None,
|
||||
)
|
||||
descriptor_jsons.append(canonical_json(descriptor))
|
||||
routing_config_sha256 = canonical_sha256({
|
||||
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
|
||||
"mode": schedule.mode,
|
||||
"logical_routes": [
|
||||
[route.logical_chunk, list(route.reference_slot_ids)]
|
||||
for route in schedule.logical_routes
|
||||
],
|
||||
})
|
||||
plan = ReferenceStoragePlan(
|
||||
tuple(descriptor_jsons), routing_config_sha256, tuple(requested_routes)
|
||||
)
|
||||
runtime.group_warnings.update({
|
||||
group: warnings for group, warnings in pending_group_warnings.items()
|
||||
if group not in runtime.observed_group_contracts
|
||||
})
|
||||
return plan
|
||||
|
||||
|
||||
def validate_reference_group_contracts(
|
||||
records: Any, *, chunk_count: int, terminal_merge_enabled: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Validate the complete routed stored plan, not just its declared hashes."""
|
||||
expected = physical_groups(chunks=chunk_count, terminal_merge_enabled=terminal_merge_enabled)
|
||||
if not isinstance(records, list) or len(records) != len(expected):
|
||||
raise ReferenceStorageContractError("stored Reference group count does not match physical topology")
|
||||
clean: list[dict[str, Any]] = []
|
||||
for record, group in zip(records, expected):
|
||||
if not isinstance(record, dict) or not isinstance(record.get("descriptor"), dict):
|
||||
raise ReferenceStorageContractError("stored Reference group descriptor is missing")
|
||||
descriptor = record["descriptor"]
|
||||
if descriptor.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
|
||||
raise ReferenceStorageContractError("stored Reference group version is unsupported")
|
||||
if (descriptor.get("physical_group") != group.physical_group
|
||||
or descriptor.get("logical_chunks") != list(range(group.start, group.end + 1))
|
||||
or descriptor.get("terminal_atomic") is not (group.start != group.end)):
|
||||
raise ReferenceStorageContractError("stored Reference group boundary is invalid")
|
||||
if not _valid_sha(record.get("sha256")) or canonical_sha256(descriptor) != record["sha256"]:
|
||||
raise ReferenceStorageContractError("stored Reference group SHA-256 is invalid")
|
||||
refs = descriptor.get("references")
|
||||
if not isinstance(refs, list):
|
||||
raise ReferenceStorageContractError("stored Reference group source list is invalid")
|
||||
slots = [ref.get("source_slot_id") for ref in refs if isinstance(ref, dict)]
|
||||
if (len(slots) != len(refs) or any(slot not in REFERENCE_SLOT_IDS for slot in slots)
|
||||
or slots != sorted(set(slots), key=REFERENCE_SLOT_IDS.index)
|
||||
or any(not _valid_sha(ref.get("prepared_sha256")) for ref in refs)):
|
||||
raise ReferenceStorageContractError("stored Reference group source identity is invalid")
|
||||
expected_order = (["first_frame"] if descriptor.get("first_anchor") else [])
|
||||
expected_order += (["last_frame"] if descriptor.get("last_anchor") else [])
|
||||
expected_order += slots
|
||||
if descriptor.get("picture_order") != expected_order:
|
||||
raise ReferenceStorageContractError("stored Reference Picture order is invalid")
|
||||
if descriptor.get("picture_presentation_version") != HYBRID_PRESENTATION_VERSION:
|
||||
raise ReferenceStorageContractError("stored Reference Picture presentation version is invalid")
|
||||
offset = int(descriptor.get("first_anchor") is not None) + int(
|
||||
descriptor.get("last_anchor") is not None
|
||||
)
|
||||
if any(ref.get("picture_number") != offset + index + 1
|
||||
for index, ref in enumerate(refs)):
|
||||
raise ReferenceStorageContractError("stored Reference Picture numbers are invalid")
|
||||
if (group.start != group.end) != (descriptor.get("terminal_prompt_policy") is not None):
|
||||
raise ReferenceStorageContractError("stored terminal prompt policy is invalid")
|
||||
if any(not _valid_sha(descriptor.get(name)) for name in ("effective_prompt_sha256",)):
|
||||
raise ReferenceStorageContractError("stored Reference effective Prompt hash is invalid")
|
||||
if any(descriptor.get(name) is not None and not _valid_sha(descriptor[name])
|
||||
for name in ("first_anchor", "last_anchor")):
|
||||
raise ReferenceStorageContractError("stored Reference anchor hash is invalid")
|
||||
if refs and (descriptor.get("reference_preprocess_version") != REFERENCE_PREPROCESS_VERSION
|
||||
or not isinstance(descriptor.get("reference_size_mode"), str)):
|
||||
raise ReferenceStorageContractError("stored Reference preprocessing contract is invalid")
|
||||
if not refs and ("reference_preprocess_version" in descriptor or "reference_size_mode" in descriptor):
|
||||
raise ReferenceStorageContractError("empty Reference group has preprocessing metadata")
|
||||
clean.append(record)
|
||||
return clean
|
||||
|
||||
|
||||
def session_routing_settings(plan: ReferenceStoragePlan, accepted_chunks: int) -> dict[str, Any]:
|
||||
accepted: list[dict[str, Any]] = []
|
||||
for record in plan.group_contracts:
|
||||
descriptor = record["descriptor"]
|
||||
end = int(descriptor["logical_chunks"][-1])
|
||||
if end > accepted_chunks:
|
||||
break
|
||||
accepted.append({
|
||||
"physical_group": int(descriptor["physical_group"]),
|
||||
"logical_chunks": list(descriptor["logical_chunks"]),
|
||||
"sha256": record["sha256"],
|
||||
})
|
||||
if not accepted or accepted[-1]["logical_chunks"][-1] != accepted_chunks:
|
||||
raise ReferenceStorageContractError("accepted Session ends inside a physical group")
|
||||
return {
|
||||
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
|
||||
"accepted_groups": accepted,
|
||||
"routing_config_sha256": plan.routing_config_sha256,
|
||||
}
|
||||
|
||||
|
||||
def compatible_session_prefix(
|
||||
evidence: Any, plan: ReferenceStoragePlan, *, saved_chunks: int,
|
||||
) -> int:
|
||||
"""Return a complete-group accepted prefix; malformed evidence is rejected."""
|
||||
if not isinstance(evidence, dict) or evidence.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
|
||||
raise ReferenceStorageContractError("saved Session has no RR-R5 group evidence")
|
||||
groups = evidence.get("accepted_groups")
|
||||
if not isinstance(groups, list) or not groups:
|
||||
raise ReferenceStorageContractError("saved Session group evidence is missing")
|
||||
last_end = 0
|
||||
for position, item in enumerate(groups):
|
||||
if not isinstance(item, dict) or not _valid_sha(item.get("sha256")):
|
||||
raise ReferenceStorageContractError("saved Session group hash is invalid")
|
||||
chunks = item.get("logical_chunks")
|
||||
if (not isinstance(chunks, list) or not chunks or chunks[0] != last_end + 1
|
||||
or chunks != list(range(chunks[0], chunks[-1] + 1))
|
||||
or item.get("physical_group") != chunks[0]):
|
||||
raise ReferenceStorageContractError("saved Session group boundaries are invalid")
|
||||
last_end = chunks[-1]
|
||||
if last_end != saved_chunks:
|
||||
raise ReferenceStorageContractError("saved Session group evidence does not cover its chunks")
|
||||
preserved = 0
|
||||
current = plan.group_contracts
|
||||
for position, item in enumerate(groups):
|
||||
if position >= len(current):
|
||||
break
|
||||
now = current[position]
|
||||
if (item["physical_group"] != now["descriptor"]["physical_group"]
|
||||
or item["logical_chunks"] != now["descriptor"]["logical_chunks"]
|
||||
or item["sha256"] != now["sha256"]):
|
||||
break
|
||||
preserved = item["logical_chunks"][-1]
|
||||
return preserved
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
"""Persistent MiniMax H3 video-reference conditioning for the V3.4 sampler."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..temporal import align_frame_count_up
|
||||
|
||||
|
||||
REFERENCE_VIDEO_CONTRACT_VERSION = 1
|
||||
REFERENCE_VIDEO_PREPROCESS_VERSION = 3
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT = "Efficient - 0.4 MP"
|
||||
REFERENCE_VIDEO_SIZE_BALANCED = "Balanced - 0.6 MP"
|
||||
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT = "Match Output"
|
||||
REFERENCE_VIDEO_SIZE_OPTIONS = (
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
REFERENCE_VIDEO_SIZE_BALANCED,
|
||||
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
|
||||
)
|
||||
_CANVAS_MULTIPLE = 32
|
||||
_EFFICIENT_AREA = 400_000
|
||||
_BALANCED_AREA = 600_000
|
||||
_SOURCE_FPS = 24
|
||||
|
||||
|
||||
class ReferenceVideoError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _canonical_hash(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _tensor_hash(value: torch.Tensor) -> str:
|
||||
tensor = value.detach().to(device="cpu").contiguous()
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(tuple(int(v) for v in tensor.shape)).encode("ascii"))
|
||||
digest.update(str(tensor.dtype).encode("ascii"))
|
||||
digest.update(tensor.numpy().tobytes(order="C"))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _round_canvas(value: float) -> int:
|
||||
return max(
|
||||
_CANVAS_MULTIPLE,
|
||||
int(round(float(value) / _CANVAS_MULTIPLE)) * _CANVAS_MULTIPLE,
|
||||
)
|
||||
|
||||
|
||||
def _resolved_size(
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
*,
|
||||
output_width: int,
|
||||
output_height: int,
|
||||
size_mode: str,
|
||||
) -> tuple[int, int]:
|
||||
if size_mode == REFERENCE_VIDEO_SIZE_MATCH_OUTPUT:
|
||||
target_area = int(output_width) * int(output_height)
|
||||
elif size_mode == REFERENCE_VIDEO_SIZE_BALANCED:
|
||||
target_area = _BALANCED_AREA
|
||||
else:
|
||||
target_area = _EFFICIENT_AREA
|
||||
source_area = int(source_width) * int(source_height)
|
||||
scale = min(1.0, math.sqrt(float(target_area) / float(source_area)))
|
||||
target_width = _round_canvas(int(source_width) * scale)
|
||||
target_height = _round_canvas(int(source_height) * scale)
|
||||
if source_width >= _CANVAS_MULTIPLE:
|
||||
target_width = min(
|
||||
target_width,
|
||||
max(_CANVAS_MULTIPLE, (int(source_width) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
|
||||
)
|
||||
if source_height >= _CANVAS_MULTIPLE:
|
||||
target_height = min(
|
||||
target_height,
|
||||
max(_CANVAS_MULTIPLE, (int(source_height) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
|
||||
)
|
||||
return target_width, target_height
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReferenceVideoSource:
|
||||
frames: torch.Tensor
|
||||
source_shape: tuple[int, ...]
|
||||
source_dtype: str
|
||||
source_sha256: str
|
||||
size_mode: str
|
||||
target_width: int
|
||||
target_height: int
|
||||
frame_count: int
|
||||
combined_hash: str
|
||||
|
||||
@property
|
||||
def contract(self) -> dict[str, Any]:
|
||||
return {
|
||||
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
|
||||
"source_shape": list(self.source_shape),
|
||||
"source_dtype": self.source_dtype,
|
||||
"source_sha256": self.source_sha256,
|
||||
"source_fps": _SOURCE_FPS,
|
||||
"size_mode": self.size_mode,
|
||||
"target_width": self.target_width,
|
||||
"target_height": self.target_height,
|
||||
"frame_count": self.frame_count,
|
||||
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
|
||||
"combined_hash": self.combined_hash,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReferenceVideoAssets:
|
||||
source: ReferenceVideoSource
|
||||
item: dict[str, Any]
|
||||
block: dict[str, Any]
|
||||
|
||||
|
||||
def _resolve_reference_frame_count(source_frames: int, target_frames: int) -> int:
|
||||
source_count = int(source_frames)
|
||||
target_count = int(target_frames)
|
||||
if source_count < 5 or target_count < 5:
|
||||
raise ReferenceVideoError("Video Reference requires at least 5 frames")
|
||||
target_cap = align_frame_count_up(target_count)
|
||||
available_count = min(source_count, target_cap)
|
||||
return min(align_frame_count_up(available_count), target_cap)
|
||||
|
||||
|
||||
def _fit_reference_frames(frames: torch.Tensor, frame_count: int) -> torch.Tensor:
|
||||
requested_count = int(frame_count)
|
||||
current_count = int(frames.shape[0])
|
||||
if current_count >= requested_count:
|
||||
return frames[:requested_count].contiguous()
|
||||
if current_count < 1:
|
||||
raise ReferenceVideoError("Video Reference contains no frames")
|
||||
padding = frames[-1:].expand(requested_count - current_count, -1, -1, -1)
|
||||
return torch.cat((frames, padding), dim=0).contiguous()
|
||||
|
||||
|
||||
def prepare_reference_video_source(
|
||||
reference_video_1: Any,
|
||||
*,
|
||||
target_frames: int,
|
||||
output_width: int,
|
||||
output_height: int,
|
||||
size_mode: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
) -> ReferenceVideoSource | None:
|
||||
if reference_video_1 is None:
|
||||
return None
|
||||
if not torch.is_tensor(reference_video_1) or reference_video_1.ndim != 4:
|
||||
raise ReferenceVideoError(
|
||||
"Video Reference must be a ComfyUI IMAGE batch with shape [T, H, W, C]"
|
||||
)
|
||||
if int(reference_video_1.shape[-1]) < 3:
|
||||
raise ReferenceVideoError("Video Reference must contain RGB channels")
|
||||
frames = reference_video_1[:, :, :, :3].detach().to(
|
||||
device="cpu", dtype=torch.float32
|
||||
).contiguous()
|
||||
if not bool(torch.isfinite(frames).all().item()):
|
||||
raise ReferenceVideoError("Video Reference contains NaN or infinity")
|
||||
source_shape = tuple(int(value) for value in frames.shape)
|
||||
source_sha256 = _tensor_hash(frames)
|
||||
frame_count = _resolve_reference_frame_count(
|
||||
int(frames.shape[0]),
|
||||
int(target_frames),
|
||||
)
|
||||
normalized_size_mode = (
|
||||
str(size_mode)
|
||||
if str(size_mode) in REFERENCE_VIDEO_SIZE_OPTIONS
|
||||
else REFERENCE_VIDEO_SIZE_EFFICIENT
|
||||
)
|
||||
target_width, target_height = _resolved_size(
|
||||
int(frames.shape[2]),
|
||||
int(frames.shape[1]),
|
||||
output_width=int(output_width),
|
||||
output_height=int(output_height),
|
||||
size_mode=normalized_size_mode,
|
||||
)
|
||||
contract_base = {
|
||||
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
|
||||
"source_shape": list(source_shape),
|
||||
"source_dtype": str(frames.dtype),
|
||||
"source_sha256": source_sha256,
|
||||
"source_fps": _SOURCE_FPS,
|
||||
"size_mode": normalized_size_mode,
|
||||
"target_width": target_width,
|
||||
"target_height": target_height,
|
||||
"frame_count": frame_count,
|
||||
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
|
||||
}
|
||||
return ReferenceVideoSource(
|
||||
frames=frames,
|
||||
source_shape=source_shape,
|
||||
source_dtype=str(frames.dtype),
|
||||
source_sha256=source_sha256,
|
||||
size_mode=normalized_size_mode,
|
||||
target_width=target_width,
|
||||
target_height=target_height,
|
||||
frame_count=frame_count,
|
||||
combined_hash=_canonical_hash(contract_base),
|
||||
)
|
||||
|
||||
|
||||
def encode_reference_video(
|
||||
video_vae: Any,
|
||||
source: ReferenceVideoSource,
|
||||
) -> ReferenceVideoAssets:
|
||||
available_count = min(int(source.frames.shape[0]), int(source.frame_count))
|
||||
frames = source.frames[:available_count]
|
||||
if (
|
||||
int(frames.shape[2]) != source.target_width
|
||||
or int(frames.shape[1]) != source.target_height
|
||||
):
|
||||
try:
|
||||
import comfy.utils
|
||||
except Exception as exc:
|
||||
raise ReferenceVideoError(
|
||||
"ComfyUI Core image resize support is unavailable"
|
||||
) from exc
|
||||
frames = comfy.utils.common_upscale(
|
||||
frames.movedim(-1, 1),
|
||||
source.target_width,
|
||||
source.target_height,
|
||||
"lanczos",
|
||||
"disabled",
|
||||
).movedim(1, -1).contiguous()
|
||||
frames = _fit_reference_frames(frames, source.frame_count)
|
||||
try:
|
||||
latent = video_vae.encode(frames).detach().to("cpu").contiguous()
|
||||
except Exception as exc:
|
||||
raise ReferenceVideoError("Video Reference VAE Encode failed") from exc
|
||||
qwen_frames = frames[::_SOURCE_FPS // 2].contiguous()
|
||||
item = {
|
||||
"type": "video",
|
||||
"data": qwen_frames,
|
||||
"timestamps": [index / 2.0 for index in range(int(qwen_frames.shape[0]))],
|
||||
}
|
||||
block = {
|
||||
"kind": "video",
|
||||
"latent_t": int(latent.shape[2]),
|
||||
"latent_h": source.target_height // 16,
|
||||
"latent_w": source.target_width // 16,
|
||||
"ref_audio_t": 0,
|
||||
"latent": latent,
|
||||
"audio_latent": None,
|
||||
}
|
||||
return ReferenceVideoAssets(source=source, item=item, block=block)
|
||||
|
||||
|
||||
def combine_reference_video_identity(
|
||||
visual_identity_hash: str,
|
||||
source: ReferenceVideoSource | None,
|
||||
) -> str:
|
||||
if source is None:
|
||||
return str(visual_identity_hash)
|
||||
return _canonical_hash(
|
||||
{
|
||||
"reference_video_identity_version": 1,
|
||||
"visual_identity_hash": str(visual_identity_hash),
|
||||
"reference_video_hash": source.combined_hash,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def validate_reference_video_prompts(
|
||||
prompts: list[str], source: ReferenceVideoSource | None
|
||||
) -> str:
|
||||
if source is None or any("<Video 1>" in str(prompt) for prompt in prompts):
|
||||
return ""
|
||||
return (
|
||||
"H3C-P103 Warning: Video Reference is connected but the prompt contains no "
|
||||
"<Video 1> tag; video still conditions generation, but an explicit tag is "
|
||||
"recommended."
|
||||
)
|
||||
+699
@@ -0,0 +1,699 @@
|
||||
"""Runtime-only First Pass conditioning for Continuum-aware refinement.
|
||||
|
||||
The context deliberately remains outside the persisted Assembly Plan contract. It
|
||||
owns CPU copies of the First Pass conditioning tensors and records them in physical
|
||||
sampling order. Second Pass callers may then derive a target-canvas conditioning
|
||||
without mutating the captured source context.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
import copy
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..constants import MARK_VIDEO_CONTEXT
|
||||
from ..guide_timeline import MARK_STILL_IMAGE_GUIDE
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_CONTEXT"
|
||||
VERSION = 1
|
||||
OUTPUT_SCOPE_KEY = "output_scope_v1"
|
||||
|
||||
|
||||
class RefineContextError(ValueError):
|
||||
"""Raised when the runtime-only refine context is structurally invalid."""
|
||||
|
||||
|
||||
class RefineConditioningAdaptationError(RefineContextError):
|
||||
"""Raised when a valid captured conditioning cannot be adapted safely."""
|
||||
|
||||
|
||||
def _cpu_tensor(value: torch.Tensor) -> torch.Tensor:
|
||||
return value.detach().to(device="cpu").clone().contiguous()
|
||||
|
||||
|
||||
def _freeze(value: Any) -> Any:
|
||||
if torch.is_tensor(value):
|
||||
return _cpu_tensor(value)
|
||||
if isinstance(value, Mapping):
|
||||
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
||||
if isinstance(value, (list, tuple)):
|
||||
return tuple(_freeze(item) for item in value)
|
||||
if isinstance(value, (set, frozenset)):
|
||||
return frozenset(_freeze(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]:
|
||||
"""Return a CPU-owned conditioning clone with immutable containers.
|
||||
|
||||
Tensors are detached and cloned, so later mutation or device movement of the
|
||||
original First Pass conditioning cannot affect the refine context. Container
|
||||
immutability makes the result safe to share until target adaptation performs
|
||||
copy-on-write replacement of visual fields.
|
||||
"""
|
||||
|
||||
if not isinstance(conditioning, Sequence) or isinstance(
|
||||
conditioning, (str, bytes, bytearray)
|
||||
):
|
||||
raise RefineContextError("conditioning must be a non-empty sequence")
|
||||
output: list[tuple[torch.Tensor, Mapping]] = []
|
||||
for index, entry in enumerate(conditioning):
|
||||
if not isinstance(entry, Sequence) or isinstance(
|
||||
entry, (str, bytes, bytearray)
|
||||
) or len(entry) != 2:
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} is not [tensor, metadata]"
|
||||
)
|
||||
tensor, metadata = entry
|
||||
if not torch.is_tensor(tensor):
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} text value is not a tensor"
|
||||
)
|
||||
if not isinstance(metadata, Mapping):
|
||||
raise RefineContextError(
|
||||
f"conditioning entry {index} metadata is not a mapping"
|
||||
)
|
||||
output.append((_cpu_tensor(tensor), _freeze(metadata)))
|
||||
if not output:
|
||||
raise RefineContextError("conditioning is empty")
|
||||
return tuple(output)
|
||||
|
||||
|
||||
def _optional_image(value: Any, name: str) -> torch.Tensor | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3:
|
||||
raise RefineContextError(
|
||||
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
||||
)
|
||||
return _cpu_tensor(value)
|
||||
|
||||
|
||||
def _validate_optional_image(value: Any, name: str) -> None:
|
||||
if value is not None and (
|
||||
not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3
|
||||
):
|
||||
raise RefineContextError(
|
||||
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
|
||||
)
|
||||
|
||||
|
||||
def _guide_image_map(value: Any) -> Mapping[int, torch.Tensor]:
|
||||
if value is None:
|
||||
return MappingProxyType({})
|
||||
if not isinstance(value, Mapping):
|
||||
raise RefineContextError("guide_images must be a frame-to-IMAGE mapping")
|
||||
output: dict[int, torch.Tensor] = {}
|
||||
for frame_index, image in value.items():
|
||||
if type(frame_index) is not int or frame_index < 0:
|
||||
raise RefineContextError("guide_images frame indices must be non-negative integers")
|
||||
prepared = _optional_image(image, f"guide_images[{frame_index}]")
|
||||
assert prepared is not None
|
||||
output[frame_index] = prepared
|
||||
return MappingProxyType(output)
|
||||
|
||||
|
||||
def _validate_guide_images(value: Any, *, physical_frames: int) -> None:
|
||||
if value is None:
|
||||
return
|
||||
if not isinstance(value, Mapping):
|
||||
raise RefineContextError("refine group guide_images is invalid")
|
||||
for frame_index, image in value.items():
|
||||
if type(frame_index) is not int or not 0 <= frame_index < physical_frames:
|
||||
raise RefineContextError("refine group guide image frame is outside the physical group")
|
||||
_validate_optional_image(image, f"guide_images[{frame_index}]")
|
||||
|
||||
|
||||
def make_refine_group(
|
||||
*,
|
||||
conditioning: Any,
|
||||
group_id: int,
|
||||
logical_chunks: Sequence[int],
|
||||
physical_frames: int,
|
||||
prompt_policy: str,
|
||||
physical_prompt: str,
|
||||
source_video_shape: Sequence[int],
|
||||
physical_clip_index: int,
|
||||
context_frames: int,
|
||||
first_image: torch.Tensor | None = None,
|
||||
last_image: torch.Tensor | None = None,
|
||||
guide_images: Mapping[int, torch.Tensor] | None = None,
|
||||
reference_routing_group_contract: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Capture one physical First Pass sampling group's effective conditioning."""
|
||||
|
||||
group_id = int(group_id)
|
||||
physical_frames = int(physical_frames)
|
||||
physical_clip_index = int(physical_clip_index)
|
||||
context_frames = int(context_frames)
|
||||
chunks = tuple(int(value) for value in logical_chunks)
|
||||
shape = tuple(int(value) for value in source_video_shape)
|
||||
if group_id < 0:
|
||||
raise RefineContextError("refine group id must be non-negative")
|
||||
if not chunks or any(value <= 0 for value in chunks):
|
||||
raise RefineContextError("refine group logical chunks are invalid")
|
||||
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
|
||||
raise RefineContextError("refine group logical chunks must be ordered and unique")
|
||||
if physical_frames <= 0:
|
||||
raise RefineContextError("refine group physical frame count is invalid")
|
||||
if not str(prompt_policy):
|
||||
raise RefineContextError("refine group prompt policy is empty")
|
||||
if len(shape) != 5 or any(value <= 0 for value in shape):
|
||||
raise RefineContextError("refine group source video shape must be [B,C,T,H,W]")
|
||||
if physical_clip_index <= 0:
|
||||
raise RefineContextError("refine group physical clip index is invalid")
|
||||
if context_frames < 0:
|
||||
raise RefineContextError("refine group context frame count is invalid")
|
||||
output = {
|
||||
"group_id": group_id,
|
||||
"logical_chunks": chunks,
|
||||
"physical_frames": physical_frames,
|
||||
"prompt_policy": str(prompt_policy),
|
||||
"physical_prompt": str(physical_prompt),
|
||||
"source_video_shape": shape,
|
||||
"physical_clip_index": physical_clip_index,
|
||||
"context_frames": context_frames,
|
||||
"conditioning": freeze_conditioning(conditioning),
|
||||
"first_image": _optional_image(first_image, "first_image"),
|
||||
"last_image": _optional_image(last_image, "last_image"),
|
||||
}
|
||||
frozen_guides = _guide_image_map(guide_images)
|
||||
if frozen_guides:
|
||||
output["guide_images"] = frozen_guides
|
||||
if reference_routing_group_contract is not None:
|
||||
from .reference_storage_contract import canonical_sha256
|
||||
descriptor = reference_routing_group_contract.get("descriptor")
|
||||
digest = reference_routing_group_contract.get("sha256")
|
||||
if not isinstance(descriptor, dict) or canonical_sha256(descriptor) != digest:
|
||||
raise RefineContextError("refine group Reference contract SHA-256 is invalid")
|
||||
if descriptor.get("logical_chunks") != list(chunks):
|
||||
raise RefineContextError("refine group Reference contract has different logical chunks")
|
||||
output["reference_routing_v1"] = copy.deepcopy(dict(reference_routing_group_contract))
|
||||
return output
|
||||
|
||||
|
||||
def make_refine_context(
|
||||
groups: Sequence[Mapping[str, Any]],
|
||||
*,
|
||||
source_width: int,
|
||||
source_height: int,
|
||||
conditioning_mode: str,
|
||||
complete: bool = True,
|
||||
notes: Sequence[str] = (),
|
||||
) -> dict[str, Any]:
|
||||
"""Build and validate a physical-order runtime refine context."""
|
||||
|
||||
context = {
|
||||
"magic": MAGIC,
|
||||
"version": VERSION,
|
||||
"source_width": int(source_width),
|
||||
"source_height": int(source_height),
|
||||
"conditioning_mode": str(conditioning_mode),
|
||||
"complete": bool(complete),
|
||||
"groups": tuple(dict(group) for group in groups),
|
||||
"notes": tuple(str(note) for note in notes),
|
||||
}
|
||||
return validate_refine_context(context)
|
||||
|
||||
|
||||
def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None:
|
||||
if not isinstance(value, tuple) or not value:
|
||||
raise RefineContextError(f"refine group {group_id} conditioning is invalid")
|
||||
for index, entry in enumerate(value):
|
||||
if not isinstance(entry, tuple) or len(entry) != 2:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning entry {index} is invalid"
|
||||
)
|
||||
tensor, metadata = entry
|
||||
if not torch.is_tensor(tensor) or tensor.device.type != "cpu":
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning tensor {index} is not CPU-owned"
|
||||
)
|
||||
if not isinstance(metadata, Mapping):
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} conditioning metadata {index} is invalid"
|
||||
)
|
||||
|
||||
|
||||
def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]:
|
||||
value = group.get(name)
|
||||
if not isinstance(value, (tuple, list)):
|
||||
raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid")
|
||||
try:
|
||||
return tuple(int(item) for item in value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RefineContextError(
|
||||
f"refine group {group.get('group_id')} {name} is invalid"
|
||||
) from exc
|
||||
|
||||
|
||||
def project_refine_context(context: dict[str, Any], assembly_plan: Mapping[str, Any]):
|
||||
"""Bind a V3.9 runtime-only view to complete physical output groups.
|
||||
|
||||
Sequence completion and saved history are unchanged. A partial capture may
|
||||
serve a smaller decode view only after the ordinary complete-group checks
|
||||
pass, including the frozen Reference route. Missing or mismatched partial
|
||||
evidence retains the legacy diagnostic fallback, never guessed conditions.
|
||||
"""
|
||||
validate_refine_context(context)
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
planned = contract.get("physical_groups") if isinstance(contract, Mapping) else None
|
||||
if not isinstance(planned, list) or not planned:
|
||||
return context
|
||||
source = context["groups"]
|
||||
keys = [tuple(group["logical_chunks"]) for group in source]
|
||||
targets = [tuple(group.get("logical_chunks", ())) for group in planned]
|
||||
if context["complete"] and keys == targets:
|
||||
return validate_refine_context(context, assembly_plan=assembly_plan)
|
||||
if len(set(keys)) != len(keys) or len(set(targets)) != len(targets):
|
||||
return context
|
||||
lookup = dict(zip(keys, source))
|
||||
if any(key not in lookup for key in targets):
|
||||
return context
|
||||
selected = [lookup[key] for key in targets]
|
||||
view = dict(context)
|
||||
view["groups"] = tuple(
|
||||
{**group, "group_id": index} for index, group in enumerate(selected)
|
||||
)
|
||||
view[OUTPUT_SCOPE_KEY] = MappingProxyType({
|
||||
"version": 1,
|
||||
"source_group_ids": tuple(group["group_id"] for group in selected),
|
||||
})
|
||||
try:
|
||||
validate_refine_context(view, assembly_plan=assembly_plan)
|
||||
except RefineContextError as exc:
|
||||
# These groups were previously optional/incomplete. Do not add a new
|
||||
# execution stop for unavailable output-scope evidence.
|
||||
fallback = dict(context)
|
||||
fallback["notes"] = (*context["notes"], f"Output scope unavailable: {exc}")
|
||||
return fallback
|
||||
return view
|
||||
|
||||
|
||||
def validate_refine_context(
|
||||
context: Any, assembly_plan: Mapping[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Validate a runtime context and, when supplied, its Assembly Plan pairing."""
|
||||
|
||||
if not isinstance(context, dict) or context.get("magic") != MAGIC:
|
||||
raise RefineContextError("invalid H3 Continuum refine context")
|
||||
if type(context.get("version")) is not int or context["version"] != VERSION:
|
||||
raise RefineContextError(
|
||||
f"unsupported refine context version {context.get('version')!r}"
|
||||
)
|
||||
if int(context.get("source_width", 0)) <= 0 or int(
|
||||
context.get("source_height", 0)
|
||||
) <= 0:
|
||||
raise RefineContextError("refine context source dimensions are invalid")
|
||||
if not str(context.get("conditioning_mode", "")):
|
||||
raise RefineContextError("refine context conditioning mode is empty")
|
||||
if type(context.get("complete")) is not bool:
|
||||
raise RefineContextError("refine context complete flag is invalid")
|
||||
groups = context.get("groups")
|
||||
if not isinstance(groups, tuple):
|
||||
raise RefineContextError("refine context groups must be an immutable tuple")
|
||||
if context["complete"] and not groups:
|
||||
raise RefineContextError("complete refine context contains no physical groups")
|
||||
notes = context.get("notes")
|
||||
if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes):
|
||||
raise RefineContextError("refine context notes must be an immutable string tuple")
|
||||
scope = context.get(OUTPUT_SCOPE_KEY)
|
||||
if scope is not None:
|
||||
ids = scope.get("source_group_ids") if isinstance(scope, Mapping) else None
|
||||
if (not isinstance(scope, Mapping) or scope.get("version") != 1
|
||||
or not isinstance(ids, tuple) or len(ids) != len(groups)
|
||||
or any(type(value) is not int or value < 0 for value in ids)
|
||||
or tuple(sorted(set(ids))) != ids or not groups):
|
||||
raise RefineContextError("refine context output scope is invalid")
|
||||
|
||||
previous_clip_index = 0
|
||||
for expected_id, group in enumerate(groups):
|
||||
if not isinstance(group, Mapping):
|
||||
raise RefineContextError(f"refine group {expected_id} is not a mapping")
|
||||
if type(group.get("group_id")) is not int or group["group_id"] != expected_id:
|
||||
raise RefineContextError("refine context physical group order is not contiguous")
|
||||
logical_chunks = _group_field_tuple(group, "logical_chunks")
|
||||
if not logical_chunks or any(value <= 0 for value in logical_chunks):
|
||||
raise RefineContextError(f"refine group {expected_id} logical chunks are invalid")
|
||||
if tuple(sorted(logical_chunks)) != logical_chunks or len(
|
||||
set(logical_chunks)
|
||||
) != len(logical_chunks):
|
||||
raise RefineContextError(
|
||||
f"refine group {expected_id} logical chunks are not ordered and unique"
|
||||
)
|
||||
if int(group.get("physical_frames", 0)) <= 0:
|
||||
raise RefineContextError(f"refine group {expected_id} physical frames are invalid")
|
||||
if not str(group.get("prompt_policy", "")):
|
||||
raise RefineContextError(f"refine group {expected_id} prompt policy is empty")
|
||||
if "physical_prompt" not in group:
|
||||
raise RefineContextError(f"refine group {expected_id} physical prompt is absent")
|
||||
shape = _group_field_tuple(group, "source_video_shape")
|
||||
if len(shape) != 5 or any(value <= 0 for value in shape):
|
||||
raise RefineContextError(f"refine group {expected_id} source shape is invalid")
|
||||
clip_index = int(group.get("physical_clip_index", 0))
|
||||
if clip_index <= previous_clip_index:
|
||||
raise RefineContextError(
|
||||
"refine context physical clip indices are not strictly increasing"
|
||||
)
|
||||
previous_clip_index = clip_index
|
||||
if int(group.get("context_frames", -1)) < 0:
|
||||
raise RefineContextError(f"refine group {expected_id} context frames are invalid")
|
||||
_validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id)
|
||||
_validate_optional_image(group.get("first_image"), "first_image")
|
||||
_validate_optional_image(group.get("last_image"), "last_image")
|
||||
_validate_guide_images(
|
||||
group.get("guide_images"),
|
||||
physical_frames=int(group["physical_frames"]),
|
||||
)
|
||||
route_contract = group.get("reference_routing_v1")
|
||||
if route_contract is not None:
|
||||
from .reference_storage_contract import canonical_sha256
|
||||
if not isinstance(route_contract, Mapping):
|
||||
raise RefineContextError("refine group Reference contract is invalid")
|
||||
descriptor = route_contract.get("descriptor")
|
||||
if (not isinstance(descriptor, dict)
|
||||
or canonical_sha256(descriptor) != route_contract.get("sha256")
|
||||
or descriptor.get("logical_chunks") != list(logical_chunks)):
|
||||
raise RefineContextError("refine group Reference contract differs from its content")
|
||||
|
||||
if assembly_plan is not None:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise RefineContextError("assembly plan paired with refine context is invalid")
|
||||
if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int(
|
||||
assembly_plan.get("height", 0)
|
||||
) != int(context["source_height"]):
|
||||
raise RefineContextError("refine context source dimensions differ from assembly plan")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None
|
||||
if not isinstance(plan_groups, Sequence):
|
||||
raise RefineContextError("assembly plan has no Second Pass physical-group contract")
|
||||
# An incomplete context is diagnostic-only (for example, Run Storage may
|
||||
# have reused an earlier group that was never sampled in this execution).
|
||||
# Its local group ids describe capture order, not Assembly Plan position;
|
||||
# validating those partial entries by index would incorrectly turn the
|
||||
# intended prompt-only fallback into a broken typed-contract hard error.
|
||||
if not context["complete"] and scope is None:
|
||||
return context
|
||||
if len(plan_groups) != len(groups):
|
||||
raise RefineContextError(
|
||||
"complete refine context physical-group count differs from assembly plan"
|
||||
)
|
||||
for group in groups:
|
||||
group_id = int(group["group_id"])
|
||||
if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping):
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} has no matching assembly-plan group"
|
||||
)
|
||||
planned = plan_groups[group_id]
|
||||
route_view = assembly_plan.get("reference_routing_v1")
|
||||
if route_view is not None:
|
||||
view_groups = route_view.get("groups") if isinstance(route_view, Mapping) else None
|
||||
expected_route = (
|
||||
view_groups[group_id] if isinstance(view_groups, list)
|
||||
and group_id < len(view_groups) else None
|
||||
)
|
||||
if scope is not None:
|
||||
# Inspector retains the full run, including pending groups.
|
||||
# A Chunk-2 view must not compare against Inspector group 1.
|
||||
matches = [item for item in view_groups
|
||||
if isinstance(item, Mapping)
|
||||
and isinstance(item.get("descriptor"), Mapping)
|
||||
and tuple(item["descriptor"].get("logical_chunks", ()))
|
||||
== tuple(group["logical_chunks"])] if isinstance(view_groups, list) else []
|
||||
expected_route = matches[0] if len(matches) == 1 else None
|
||||
captured_route = group.get("reference_routing_v1")
|
||||
if (not isinstance(expected_route, Mapping)
|
||||
or not isinstance(captured_route, Mapping)
|
||||
or captured_route.get("sha256") != expected_route.get("contract_sha256")
|
||||
or captured_route.get("descriptor") != expected_route.get("descriptor")):
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} Reference route differs from First Pass Plan"
|
||||
)
|
||||
comparisons = (
|
||||
("logical_chunks", tuple(int(v) for v in group["logical_chunks"]),
|
||||
tuple(int(v) for v in planned.get("logical_chunks", ()))),
|
||||
("physical_frames", int(group["physical_frames"]),
|
||||
int(planned.get("physical_frames", 0))),
|
||||
("prompt_policy", str(group["prompt_policy"]),
|
||||
str(planned.get("prompt_policy", ""))),
|
||||
("physical_prompt", str(group["physical_prompt"]),
|
||||
str(planned.get("physical_prompt", ""))),
|
||||
("context_frames", int(group["context_frames"]),
|
||||
int(planned.get("trim_prefix_frames", -1))),
|
||||
)
|
||||
for name, actual, expected in comparisons:
|
||||
if actual != expected:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} {name} differs from assembly plan"
|
||||
)
|
||||
expected_shape = (
|
||||
int(planned.get("source_batch", 0)),
|
||||
int(planned.get("latent_channels", 0)),
|
||||
int(planned.get("source_latent_t", 0)),
|
||||
int(planned.get("source_latent_h", 0)),
|
||||
int(planned.get("source_latent_w", 0)),
|
||||
)
|
||||
if tuple(group["source_video_shape"]) != expected_shape:
|
||||
raise RefineContextError(
|
||||
f"refine group {group_id} source shape differs from assembly plan"
|
||||
)
|
||||
return context
|
||||
|
||||
|
||||
def _common_upscale(
|
||||
samples: torch.Tensor,
|
||||
width: int,
|
||||
height: int,
|
||||
method: str,
|
||||
crop: str,
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
import comfy.utils
|
||||
except Exception as exc: # pragma: no cover - ComfyUI runtime only
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"ComfyUI Core resize support is unavailable: {exc}"
|
||||
) from exc
|
||||
return comfy.utils.common_upscale(samples, width, height, method, crop)
|
||||
|
||||
|
||||
def _resize_latent(
|
||||
latent: Any,
|
||||
*,
|
||||
target_h: int,
|
||||
target_w: int,
|
||||
method: str,
|
||||
) -> torch.Tensor:
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
"H3 visual conditioning latent must be [B,C,T,H,W]"
|
||||
)
|
||||
batch, channels, temporal, _, _ = (int(value) for value in latent.shape)
|
||||
flat = latent.permute(0, 2, 1, 3, 4).reshape(
|
||||
batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1])
|
||||
)
|
||||
try:
|
||||
resized = _common_upscale(flat, target_w, target_h, method, "disabled")
|
||||
except RefineConditioningAdaptationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"visual conditioning latent resize failed: {exc}"
|
||||
) from exc
|
||||
if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
raise RefineConditioningAdaptationError(
|
||||
"Core resize returned an incompatible visual conditioning shape"
|
||||
)
|
||||
return (
|
||||
resized.reshape(batch, temporal, channels, target_h, target_w)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _encode_keyframe_image(
|
||||
image: torch.Tensor,
|
||||
*,
|
||||
target_h: int,
|
||||
target_w: int,
|
||||
crop: str,
|
||||
video_vae: Any,
|
||||
) -> torch.Tensor:
|
||||
pixels_h = target_h * 16
|
||||
pixels_w = target_w * 16
|
||||
samples = image[:1, ..., :3].movedim(-1, 1)
|
||||
try:
|
||||
resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop)
|
||||
latent = video_vae.encode(resized.movedim(1, -1).contiguous())
|
||||
except RefineConditioningAdaptationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"target keyframe VAE encode failed: {exc}"
|
||||
) from exc
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5 or tuple(
|
||||
int(value) for value in latent.shape[-2:]
|
||||
) != (target_h, target_w):
|
||||
raise RefineConditioningAdaptationError(
|
||||
"target keyframe VAE encode returned an incompatible latent shape"
|
||||
)
|
||||
return latent.contiguous()
|
||||
|
||||
|
||||
def adapt_group_conditioning(
|
||||
group: Mapping[str, Any],
|
||||
target_latent_h: int,
|
||||
target_latent_w: int,
|
||||
video_vae: Any = None,
|
||||
upscale_method: str = "bilinear",
|
||||
) -> tuple[list[list[Any]], dict[str, Any]]:
|
||||
"""Copy-on-write target adaptation for keyframes and Continuum video context.
|
||||
|
||||
Ordinary MiniMax references, all audio latents, and text conditioning tensors
|
||||
remain shared with the frozen source. Only keyframe video latents and
|
||||
``MARK_VIDEO_CONTEXT`` reference video latents are target-sized.
|
||||
"""
|
||||
|
||||
if not isinstance(group, Mapping):
|
||||
raise RefineConditioningAdaptationError("refine group is not a mapping")
|
||||
group_id = int(group.get("group_id", -1))
|
||||
_validate_frozen_conditioning(group.get("conditioning"), group_id=group_id)
|
||||
target_h = int(target_latent_h)
|
||||
target_w = int(target_latent_w)
|
||||
if target_h <= 0 or target_w <= 0:
|
||||
raise RefineConditioningAdaptationError("target latent dimensions are invalid")
|
||||
physical_frames = int(group.get("physical_frames", 0))
|
||||
if physical_frames <= 0:
|
||||
raise RefineConditioningAdaptationError("refine group physical frames are invalid")
|
||||
|
||||
warnings: list[str] = []
|
||||
stats: dict[str, Any] = {
|
||||
"group_id": group_id,
|
||||
"target_latent_h": target_h,
|
||||
"target_latent_w": target_w,
|
||||
"keyframes_resized": 0,
|
||||
"keyframes_reencoded": 0,
|
||||
"keyframes_latent_fallback": 0,
|
||||
"guide_keyframes_reencoded": 0,
|
||||
"context_refs_resized": 0,
|
||||
}
|
||||
output: list[list[Any]] = []
|
||||
for text_tensor, frozen_metadata in group["conditioning"]:
|
||||
metadata = dict(frozen_metadata)
|
||||
keyframes = frozen_metadata.get("minimax_keyframes")
|
||||
if keyframes is not None:
|
||||
adapted_keyframes = []
|
||||
for frozen_keyframe in keyframes:
|
||||
if not isinstance(frozen_keyframe, Mapping):
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} contains an invalid keyframe"
|
||||
)
|
||||
keyframe = dict(frozen_keyframe)
|
||||
latent = keyframe.get("latent")
|
||||
audio_latent = keyframe.get("audio_latent")
|
||||
if audio_latent is not None and (
|
||||
not torch.is_tensor(audio_latent) or audio_latent.ndim != 4
|
||||
):
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} keyframe audio latent is invalid"
|
||||
)
|
||||
# Core keyframes may carry audio without a spatial video latent.
|
||||
# Keep its frozen audio and timing; resize only video payloads.
|
||||
if latent is None and audio_latent is not None:
|
||||
adapted_keyframes.append(keyframe)
|
||||
continue
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} keyframe latent is invalid"
|
||||
)
|
||||
if tuple(int(value) for value in latent.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
frame_index = int(keyframe.get("resolved_frame_index", -1))
|
||||
image = None
|
||||
crop = "disabled"
|
||||
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
|
||||
guide_images = group.get("guide_images")
|
||||
if isinstance(guide_images, Mapping):
|
||||
image = guide_images.get(frame_index)
|
||||
crop = "center"
|
||||
elif frame_index == 0:
|
||||
image = group.get("first_image")
|
||||
elif frame_index == physical_frames - 1:
|
||||
image = group.get("last_image")
|
||||
crop = "center"
|
||||
if video_vae is not None and torch.is_tensor(image):
|
||||
keyframe["latent"] = _encode_keyframe_image(
|
||||
image,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
crop=crop,
|
||||
video_vae=video_vae,
|
||||
)
|
||||
stats["keyframes_reencoded"] += 1
|
||||
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
|
||||
stats["guide_keyframes_reencoded"] += 1
|
||||
else:
|
||||
keyframe["latent"] = _resize_latent(
|
||||
latent,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
method=str(upscale_method),
|
||||
)
|
||||
stats["keyframes_latent_fallback"] += 1
|
||||
warning = (
|
||||
f"group {group_id} keyframe {frame_index} used latent resize "
|
||||
"because its source image and video VAE were not both available"
|
||||
)
|
||||
warnings.append(warning)
|
||||
stats["keyframes_resized"] += 1
|
||||
adapted_keyframes.append(keyframe)
|
||||
metadata["minimax_keyframes"] = adapted_keyframes
|
||||
|
||||
refs = frozen_metadata.get("minimax_refs")
|
||||
if refs is not None:
|
||||
adapted_refs = []
|
||||
refs_changed = False
|
||||
for frozen_ref in refs:
|
||||
if not isinstance(frozen_ref, Mapping):
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} contains an invalid MiniMax reference"
|
||||
)
|
||||
if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)):
|
||||
latent = frozen_ref.get("latent")
|
||||
if not torch.is_tensor(latent) or latent.ndim != 5:
|
||||
raise RefineConditioningAdaptationError(
|
||||
f"refine group {group_id} Continuum video context is invalid"
|
||||
)
|
||||
if tuple(int(value) for value in latent.shape[-2:]) != (
|
||||
target_h,
|
||||
target_w,
|
||||
):
|
||||
adapted_ref = dict(frozen_ref)
|
||||
adapted_ref["latent"] = _resize_latent(
|
||||
latent,
|
||||
target_h=target_h,
|
||||
target_w=target_w,
|
||||
method=str(upscale_method),
|
||||
)
|
||||
adapted_ref["latent_h"] = target_h
|
||||
adapted_ref["latent_w"] = target_w
|
||||
adapted_refs.append(adapted_ref)
|
||||
refs_changed = True
|
||||
stats["context_refs_resized"] += 1
|
||||
continue
|
||||
adapted_refs.append(frozen_ref)
|
||||
if refs_changed:
|
||||
metadata["minimax_refs"] = adapted_refs
|
||||
|
||||
output.append([text_tensor, metadata])
|
||||
stats["warnings"] = tuple(warnings)
|
||||
return output, stats
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
|
||||
|
||||
class RefineSamplingRuntimeError(RuntimeError):
|
||||
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
|
||||
|
||||
|
||||
def _make_basic_guider(model: Any, conditioning: list):
|
||||
try:
|
||||
import comfy.samplers
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
|
||||
|
||||
class _BasicGuider(comfy.samplers.CFGGuider):
|
||||
def set_positive(self, positive):
|
||||
self.inner_set_conds({"positive": positive})
|
||||
|
||||
guider = _BasicGuider(model)
|
||||
guider.set_positive(conditioning)
|
||||
return guider
|
||||
|
||||
|
||||
def _load_runtime_modules():
|
||||
try:
|
||||
import comfy.model_management
|
||||
import comfy.nested_tensor
|
||||
import comfy.sample
|
||||
import comfy.utils
|
||||
import latent_preview
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"ComfyUI refine sampling runtime unavailable: {exc}"
|
||||
) from exc
|
||||
return (
|
||||
comfy.model_management,
|
||||
comfy.nested_tensor,
|
||||
comfy.sample,
|
||||
comfy.utils,
|
||||
latent_preview,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_refine_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build random video noise and a Core-expandable audio-lock mask."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
|
||||
|
||||
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
|
||||
# device so NestedTensor packing remains valid before Core moves it to the model.
|
||||
audio_noise = torch.zeros_like(audio, device="cpu")
|
||||
|
||||
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
|
||||
# Core expands them to the full stream geometry on the sampling device.
|
||||
video_mask = torch.ones(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.zeros(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def sample_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
|
||||
|
||||
if not isinstance(latent, dict) or "samples" not in latent:
|
||||
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
||||
|
||||
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
||||
_load_runtime_modules()
|
||||
)
|
||||
working = latent.copy()
|
||||
latent_image = comfy_sample.fix_empty_latent_channels(
|
||||
model,
|
||||
working["samples"],
|
||||
working.get("downscale_ratio_spacial"),
|
||||
working.get("downscale_ratio_temporal"),
|
||||
)
|
||||
working["samples"] = latent_image
|
||||
video, audio = extract_av_streams(working)
|
||||
noise, noise_mask = _prepare_refine_noise_and_mask(
|
||||
video,
|
||||
audio,
|
||||
seed=int(seed),
|
||||
batch_inds=working.get("batch_index"),
|
||||
prepare_noise_fn=comfy_sample.prepare_noise,
|
||||
nested_builder=nested_tensor.NestedTensor,
|
||||
)
|
||||
guider = _make_basic_guider(model, conditioning)
|
||||
|
||||
x0_output: dict[str, Any] = {}
|
||||
callback = None
|
||||
if enable_preview:
|
||||
callback = preview.prepare_callback(
|
||||
model,
|
||||
int(sigmas.shape[-1]) - 1,
|
||||
x0_output,
|
||||
)
|
||||
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
|
||||
samples = guider.sample(
|
||||
noise,
|
||||
latent_image,
|
||||
sampler,
|
||||
sigmas,
|
||||
denoise_mask=noise_mask,
|
||||
callback=callback,
|
||||
disable_pbar=disable_pbar,
|
||||
seed=int(seed),
|
||||
)
|
||||
samples = samples.to(model_management.intermediate_device())
|
||||
output = working.copy()
|
||||
output.pop("downscale_ratio_spacial", None)
|
||||
output.pop("downscale_ratio_temporal", None)
|
||||
output["samples"] = samples
|
||||
extract_av_streams(output)
|
||||
return output
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
"""Versioned, UI-independent sigma contracts for Continuum refinement.
|
||||
|
||||
The public Second Pass SIGMAS socket remains authoritative. These helpers attach
|
||||
meaning to that tensor without recreating it: External and Full retain the exact
|
||||
input object, while Tail and Partial return views into the same source values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_SCHEDULE"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
MODE_EXTERNAL = "external"
|
||||
MODE_FULL = "full"
|
||||
MODE_TAIL = "tail"
|
||||
MODE_PARTIAL = "partial"
|
||||
MODES = (MODE_EXTERNAL, MODE_FULL, MODE_TAIL, MODE_PARTIAL)
|
||||
|
||||
NOISE_MODE_VIDEO_RANDOM_AUDIO_ZERO = "video_random_mask_1_audio_zero_mask_0"
|
||||
AUDIO_POLICY_LOCKED_PASSTHROUGH = "locked_first_pass_bit_exact_passthrough"
|
||||
|
||||
|
||||
class RefineScheduleError(ValueError):
|
||||
"""Raised when a refinement schedule cannot safely describe its SIGMAS."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefineSchedule:
|
||||
"""One validated effective SIGMAS tensor and its serializable identity."""
|
||||
|
||||
sigmas: torch.Tensor
|
||||
contract: Mapping[str, Any]
|
||||
source_sigma_hash: str
|
||||
|
||||
|
||||
def _validate_sigmas(sigmas: Any, *, name: str) -> torch.Tensor:
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineScheduleError(f"{name} must be a one-dimensional tensor with at least two values")
|
||||
detached = sigmas.detach()
|
||||
if not bool(torch.isfinite(detached.float()).all().item()):
|
||||
raise RefineScheduleError(f"{name} contains NaN or Inf")
|
||||
values = detached.float().to(device="cpu")
|
||||
if bool((values[1:] > values[:-1]).any().item()):
|
||||
raise RefineScheduleError(f"{name} must be monotonically non-increasing")
|
||||
return sigmas
|
||||
|
||||
|
||||
def sigma_hash(sigmas: torch.Tensor) -> str:
|
||||
"""Hash dtype, shape, and exact tensor bytes without changing the input."""
|
||||
|
||||
_validate_sigmas(sigmas, name="sigmas")
|
||||
cpu = sigmas.detach().to(device="cpu").contiguous()
|
||||
payload = cpu.view(torch.uint8).numpy().tobytes()
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(cpu.dtype).encode("ascii"))
|
||||
digest.update(b"|")
|
||||
digest.update(str(tuple(int(value) for value in cpu.shape)).encode("ascii"))
|
||||
digest.update(b"|")
|
||||
digest.update(payload)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _schedule_identity(contract: Mapping[str, Any]) -> str:
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in contract.items()
|
||||
if key != "schedule_hash"
|
||||
}
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _build_schedule(
|
||||
source_sigmas: torch.Tensor,
|
||||
*,
|
||||
mode: str,
|
||||
start_index: int,
|
||||
end_index: int,
|
||||
) -> RefineSchedule:
|
||||
source = _validate_sigmas(source_sigmas, name="source_sigmas")
|
||||
canonical_mode = str(mode).lower()
|
||||
if canonical_mode not in MODES:
|
||||
raise RefineScheduleError(f"unsupported refine schedule mode {mode!r}")
|
||||
source_evaluations = int(source.numel()) - 1
|
||||
if type(start_index) is not int or type(end_index) is not int:
|
||||
raise RefineScheduleError("schedule indices must be integers")
|
||||
if start_index < 0 or end_index <= start_index or end_index > source_evaluations:
|
||||
raise RefineScheduleError(
|
||||
f"schedule range {start_index}:{end_index} is outside 0:{source_evaluations}"
|
||||
)
|
||||
|
||||
if start_index == 0 and end_index == source_evaluations:
|
||||
effective = source
|
||||
else:
|
||||
effective = source[start_index : end_index + 1]
|
||||
_validate_sigmas(effective, name="effective sigmas")
|
||||
effective_hash = sigma_hash(effective)
|
||||
source_hash = sigma_hash(source)
|
||||
values = tuple(float(value) for value in effective.detach().to(device="cpu").tolist())
|
||||
contract: dict[str, Any] = {
|
||||
"magic": MAGIC,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": canonical_mode,
|
||||
"sigma_values": values,
|
||||
"sigma_hash": effective_hash,
|
||||
"source_sigma_hash": source_hash,
|
||||
"start_sigma": values[0],
|
||||
"end_sigma": values[-1],
|
||||
"evaluation_count": len(values) - 1,
|
||||
"source_evaluation_count": source_evaluations,
|
||||
"start_index": start_index,
|
||||
"end_index": end_index,
|
||||
"noise_mode": NOISE_MODE_VIDEO_RANDOM_AUDIO_ZERO,
|
||||
"audio_policy": AUDIO_POLICY_LOCKED_PASSTHROUGH,
|
||||
}
|
||||
contract["schedule_hash"] = _schedule_identity(contract)
|
||||
return RefineSchedule(
|
||||
sigmas=effective,
|
||||
contract=MappingProxyType(contract),
|
||||
source_sigma_hash=source_hash,
|
||||
)
|
||||
|
||||
|
||||
def make_external_schedule(sigmas: torch.Tensor) -> RefineSchedule:
|
||||
evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
|
||||
return _build_schedule(
|
||||
sigmas,
|
||||
mode=MODE_EXTERNAL,
|
||||
start_index=0,
|
||||
end_index=evaluations,
|
||||
)
|
||||
|
||||
|
||||
def make_full_schedule(sigmas: torch.Tensor) -> RefineSchedule:
|
||||
evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
|
||||
return _build_schedule(
|
||||
sigmas,
|
||||
mode=MODE_FULL,
|
||||
start_index=0,
|
||||
end_index=evaluations,
|
||||
)
|
||||
|
||||
|
||||
def make_tail_schedule(
|
||||
sigmas: torch.Tensor,
|
||||
*,
|
||||
evaluation_count: int,
|
||||
) -> RefineSchedule:
|
||||
source_evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
|
||||
if type(evaluation_count) is not int or not 1 <= evaluation_count <= source_evaluations:
|
||||
raise RefineScheduleError(
|
||||
f"tail evaluation_count must be between 1 and {source_evaluations}"
|
||||
)
|
||||
return _build_schedule(
|
||||
sigmas,
|
||||
mode=MODE_TAIL,
|
||||
start_index=source_evaluations - evaluation_count,
|
||||
end_index=source_evaluations,
|
||||
)
|
||||
|
||||
|
||||
def make_partial_schedule(
|
||||
sigmas: torch.Tensor,
|
||||
*,
|
||||
start_index: int,
|
||||
end_index: int,
|
||||
) -> RefineSchedule:
|
||||
return _build_schedule(
|
||||
sigmas,
|
||||
mode=MODE_PARTIAL,
|
||||
start_index=start_index,
|
||||
end_index=end_index,
|
||||
)
|
||||
|
||||
|
||||
def resolve_refine_schedule(
|
||||
sigmas: torch.Tensor,
|
||||
schedule: RefineSchedule | None = None,
|
||||
) -> RefineSchedule:
|
||||
"""Return External compatibility or validate an explicit internal schedule."""
|
||||
|
||||
source = _validate_sigmas(sigmas, name="sigmas")
|
||||
if schedule is None:
|
||||
return make_external_schedule(source)
|
||||
if not isinstance(schedule, RefineSchedule):
|
||||
raise RefineScheduleError("refine_schedule must be a RefineSchedule")
|
||||
if sigma_hash(source) != schedule.source_sigma_hash:
|
||||
raise RefineScheduleError("refine_schedule was derived from different source SIGMAS")
|
||||
rebuilt = _build_schedule(
|
||||
source,
|
||||
mode=str(schedule.contract.get("mode", "")),
|
||||
start_index=int(schedule.contract.get("start_index", -1)),
|
||||
end_index=int(schedule.contract.get("end_index", -1)),
|
||||
)
|
||||
if rebuilt.contract["schedule_hash"] != schedule.contract.get("schedule_hash"):
|
||||
raise RefineScheduleError("refine_schedule contract identity is inconsistent")
|
||||
return schedule
|
||||
|
||||
|
||||
def serializable_schedule_contract(schedule: RefineSchedule) -> dict[str, Any]:
|
||||
"""Return a mutable JSON-safe copy for Assembly/diagnostic identity."""
|
||||
|
||||
if not isinstance(schedule, RefineSchedule):
|
||||
raise RefineScheduleError("schedule must be a RefineSchedule")
|
||||
return dict(schedule.contract)
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
"""Versioned physical-group selection for Experimental selective refinement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_SCOPE"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
MODE_ALL = "all"
|
||||
MODE_PHYSICAL_GROUP = "physical_group"
|
||||
MODE_LOGICAL_CHUNK = "logical_chunk"
|
||||
MODES = (MODE_ALL, MODE_PHYSICAL_GROUP, MODE_LOGICAL_CHUNK)
|
||||
|
||||
|
||||
class RefineScopeError(ValueError):
|
||||
"""Raised before preparation when a requested scope cannot be resolved."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefineScope:
|
||||
"""One immutable scope resolution containing no Tensor or runtime object."""
|
||||
|
||||
mode: str
|
||||
selected_group_indices: tuple[int, ...]
|
||||
contract: Mapping[str, Any]
|
||||
|
||||
@property
|
||||
def is_all(self) -> bool:
|
||||
return self.mode == MODE_ALL
|
||||
|
||||
|
||||
def _identity(payload: Mapping[str, Any]) -> str:
|
||||
encoded = json.dumps(
|
||||
dict(payload),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _physical_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise RefineScopeError("assembly_plan must be a mapping")
|
||||
second_pass = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(second_pass, Mapping) or second_pass.get("version") != 1:
|
||||
raise RefineScopeError("assembly_plan has no Second Pass contract version 1")
|
||||
groups = second_pass.get("physical_groups")
|
||||
if (
|
||||
not isinstance(groups, Sequence)
|
||||
or isinstance(groups, (str, bytes))
|
||||
or not groups
|
||||
or not all(isinstance(group, Mapping) for group in groups)
|
||||
):
|
||||
raise RefineScopeError("assembly_plan has no physical groups")
|
||||
return list(groups)
|
||||
|
||||
|
||||
def _logical_chunks(group: Mapping[str, Any], group_index: int) -> tuple[int, ...]:
|
||||
values = group.get("logical_chunks")
|
||||
if (
|
||||
not isinstance(values, Sequence)
|
||||
or isinstance(values, (str, bytes))
|
||||
or not values
|
||||
or any(type(value) is not int or value <= 0 for value in values)
|
||||
):
|
||||
raise RefineScopeError(
|
||||
f"physical group {group_index + 1} has invalid logical chunk identity"
|
||||
)
|
||||
chunks = tuple(int(value) for value in values)
|
||||
if len(set(chunks)) != len(chunks):
|
||||
raise RefineScopeError(
|
||||
f"physical group {group_index + 1} repeats a logical chunk"
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def resolve_refine_scope(
|
||||
mode: str | None,
|
||||
scope_index: int,
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> RefineScope:
|
||||
"""Resolve one public 1-based selection to complete physical groups.
|
||||
|
||||
Logical chunks never split a physical group. Selecting either logical half of
|
||||
a Terminal Merge group therefore selects the complete terminal pair.
|
||||
"""
|
||||
|
||||
if mode is None:
|
||||
canonical_mode = MODE_ALL
|
||||
elif not isinstance(mode, str):
|
||||
raise RefineScopeError(f"unsupported refine scope mode {mode!r}")
|
||||
else:
|
||||
canonical_mode = mode.strip().lower()
|
||||
if canonical_mode not in MODES:
|
||||
raise RefineScopeError(f"unsupported refine scope mode {mode!r}")
|
||||
if type(scope_index) is not int or scope_index <= 0:
|
||||
raise RefineScopeError("refine scope index must be a positive integer")
|
||||
|
||||
groups = _physical_groups(assembly_plan)
|
||||
group_chunks = [
|
||||
_logical_chunks(group, group_index)
|
||||
for group_index, group in enumerate(groups)
|
||||
]
|
||||
terminal_expanded = False
|
||||
requested_index: int | None = None
|
||||
|
||||
if canonical_mode == MODE_ALL:
|
||||
selected = tuple(range(len(groups)))
|
||||
elif canonical_mode == MODE_PHYSICAL_GROUP:
|
||||
requested_index = int(scope_index)
|
||||
selected_index = requested_index - 1
|
||||
if selected_index >= len(groups):
|
||||
raise RefineScopeError(
|
||||
f"physical group {requested_index} is outside 1..{len(groups)}"
|
||||
)
|
||||
selected = (selected_index,)
|
||||
else:
|
||||
requested_index = int(scope_index)
|
||||
matches = [
|
||||
group_index
|
||||
for group_index, chunks in enumerate(group_chunks)
|
||||
if requested_index in chunks
|
||||
]
|
||||
if len(matches) != 1:
|
||||
if matches:
|
||||
raise RefineScopeError(
|
||||
f"logical chunk {requested_index} belongs to multiple physical groups"
|
||||
)
|
||||
raise RefineScopeError(
|
||||
f"logical chunk {requested_index} is not present in the assembly plan"
|
||||
)
|
||||
selected = (matches[0],)
|
||||
selected_group = groups[selected[0]]
|
||||
selected_chunks = group_chunks[selected[0]]
|
||||
terminal_expanded = bool(
|
||||
selected_group.get("terminal_merged") and len(selected_chunks) > 1
|
||||
)
|
||||
|
||||
selected_chunks = tuple(
|
||||
chunk
|
||||
for group_index in selected
|
||||
for chunk in group_chunks[group_index]
|
||||
)
|
||||
contract: dict[str, Any] = {
|
||||
"magic": MAGIC,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": canonical_mode,
|
||||
"requested_index": requested_index,
|
||||
"selected_physical_group_indices": tuple(selected),
|
||||
"selected_physical_groups": tuple(index + 1 for index in selected),
|
||||
"selected_logical_chunks": tuple(selected_chunks),
|
||||
"terminal_expanded": terminal_expanded,
|
||||
}
|
||||
contract["scope_hash"] = _identity(contract)
|
||||
return RefineScope(
|
||||
mode=canonical_mode,
|
||||
selected_group_indices=selected,
|
||||
contract=MappingProxyType(contract),
|
||||
)
|
||||
|
||||
|
||||
def serializable_scope_contract(scope: RefineScope) -> dict[str, Any]:
|
||||
if not isinstance(scope, RefineScope):
|
||||
raise RefineScopeError("refine_scope must be a RefineScope")
|
||||
return {
|
||||
**dict(scope.contract),
|
||||
"selected_physical_group_indices": list(
|
||||
scope.contract["selected_physical_group_indices"]
|
||||
),
|
||||
"selected_physical_groups": list(scope.contract["selected_physical_groups"]),
|
||||
"selected_logical_chunks": list(scope.contract["selected_logical_chunks"]),
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
"""Versioned stream-target policies for future Continuum refinement modes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_TARGET"
|
||||
SCHEMA_VERSION = 1
|
||||
EXECUTION_MAGIC = "H3_CONTINUUM_TARGETED_REFINE_EXECUTION"
|
||||
EXECUTION_SCHEMA_VERSION = 1
|
||||
|
||||
MODE_VIDEO_ONLY = "video_only"
|
||||
MODE_AUDIO_ONLY = "audio_only"
|
||||
MODE_VIDEO_AUDIO = "video_audio"
|
||||
MODES = (MODE_VIDEO_ONLY, MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO)
|
||||
|
||||
NOISE_SEEDED_RANDOM = "seeded_random"
|
||||
NOISE_ZERO = "zero"
|
||||
OUTPUT_SAMPLED = "sampled"
|
||||
OUTPUT_INPUT_PASSTHROUGH = "input_passthrough_bit_exact"
|
||||
|
||||
VIDEO_ONLY_SEED_NAMESPACE = "h3-continuum-refine-v1"
|
||||
AUDIO_ONLY_SEED_NAMESPACE = "h3-continuum-refine-audio-v1"
|
||||
VIDEO_AUDIO_SEED_NAMESPACE = "h3-continuum-refine-av-v1"
|
||||
|
||||
|
||||
class RefineTargetError(ValueError):
|
||||
"""Raised when a target policy or target execution contract is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefineTarget:
|
||||
"""One immutable target policy; it contains no Tensor or runtime object."""
|
||||
|
||||
mode: str
|
||||
contract: Mapping[str, Any]
|
||||
|
||||
|
||||
def _identity(payload: Mapping[str, Any]) -> str:
|
||||
encoded = json.dumps(
|
||||
dict(payload),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _stream_policy(*, noise: str, mask: int, output: str) -> dict[str, Any]:
|
||||
return {
|
||||
"noise": str(noise),
|
||||
"mask": int(mask),
|
||||
"output": str(output),
|
||||
}
|
||||
|
||||
|
||||
_POLICIES: dict[str, dict[str, Any]] = {
|
||||
MODE_VIDEO_ONLY: {
|
||||
"video": _stream_policy(
|
||||
noise=NOISE_SEEDED_RANDOM,
|
||||
mask=1,
|
||||
output=OUTPUT_SAMPLED,
|
||||
),
|
||||
"audio": _stream_policy(
|
||||
noise=NOISE_ZERO,
|
||||
mask=0,
|
||||
output=OUTPUT_INPUT_PASSTHROUGH,
|
||||
),
|
||||
"seed_namespace": VIDEO_ONLY_SEED_NAMESPACE,
|
||||
},
|
||||
MODE_AUDIO_ONLY: {
|
||||
"video": _stream_policy(
|
||||
noise=NOISE_ZERO,
|
||||
mask=0,
|
||||
output=OUTPUT_INPUT_PASSTHROUGH,
|
||||
),
|
||||
"audio": _stream_policy(
|
||||
noise=NOISE_SEEDED_RANDOM,
|
||||
mask=1,
|
||||
output=OUTPUT_SAMPLED,
|
||||
),
|
||||
"seed_namespace": AUDIO_ONLY_SEED_NAMESPACE,
|
||||
},
|
||||
MODE_VIDEO_AUDIO: {
|
||||
"video": _stream_policy(
|
||||
noise=NOISE_SEEDED_RANDOM,
|
||||
mask=1,
|
||||
output=OUTPUT_SAMPLED,
|
||||
),
|
||||
"audio": _stream_policy(
|
||||
noise=NOISE_SEEDED_RANDOM,
|
||||
mask=1,
|
||||
output=OUTPUT_SAMPLED,
|
||||
),
|
||||
"seed_namespace": VIDEO_AUDIO_SEED_NAMESPACE,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def make_refine_target(mode: str) -> RefineTarget:
|
||||
canonical_mode = str(mode).strip().lower()
|
||||
if canonical_mode not in MODES:
|
||||
raise RefineTargetError(f"unsupported refine target mode {mode!r}")
|
||||
policy = _POLICIES[canonical_mode]
|
||||
contract: dict[str, Any] = {
|
||||
"magic": MAGIC,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": canonical_mode,
|
||||
"video": dict(policy["video"]),
|
||||
"audio": dict(policy["audio"]),
|
||||
"seed_namespace": str(policy["seed_namespace"]),
|
||||
}
|
||||
contract["target_hash"] = _identity(contract)
|
||||
return RefineTarget(
|
||||
mode=canonical_mode,
|
||||
contract=MappingProxyType(contract),
|
||||
)
|
||||
|
||||
|
||||
def resolve_refine_target(value: RefineTarget | str | None) -> RefineTarget:
|
||||
if value is None:
|
||||
return make_refine_target(MODE_VIDEO_ONLY)
|
||||
if isinstance(value, str):
|
||||
return make_refine_target(value)
|
||||
if not isinstance(value, RefineTarget):
|
||||
raise RefineTargetError("refine_target must be a RefineTarget or mode string")
|
||||
rebuilt = make_refine_target(value.mode)
|
||||
if dict(rebuilt.contract) != dict(value.contract):
|
||||
raise RefineTargetError("refine_target contract identity is inconsistent")
|
||||
return value
|
||||
|
||||
|
||||
def serializable_target_contract(target: RefineTarget) -> dict[str, Any]:
|
||||
resolved = resolve_refine_target(target)
|
||||
return {
|
||||
**dict(resolved.contract),
|
||||
"video": dict(resolved.contract["video"]),
|
||||
"audio": dict(resolved.contract["audio"]),
|
||||
}
|
||||
|
||||
|
||||
def derive_target_refine_seed(
|
||||
refine_seed: int,
|
||||
physical_group_index: int,
|
||||
target: RefineTarget | str,
|
||||
) -> int:
|
||||
if type(refine_seed) is not int or refine_seed < 0:
|
||||
raise RefineTargetError("refine_seed must be a non-negative integer")
|
||||
if type(physical_group_index) is not int or physical_group_index < 0:
|
||||
raise RefineTargetError("physical_group_index must be a non-negative integer")
|
||||
resolved = resolve_refine_target(target)
|
||||
namespace = str(resolved.contract["seed_namespace"])
|
||||
payload = f"{namespace}:{refine_seed}:{physical_group_index}".encode("ascii")
|
||||
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
|
||||
|
||||
|
||||
_SIGMA_RANGE_FIELDS = (
|
||||
"schema_version",
|
||||
"mode",
|
||||
"source_sigma_hash",
|
||||
"sigma_hash",
|
||||
"start_sigma",
|
||||
"end_sigma",
|
||||
"evaluation_count",
|
||||
"source_evaluation_count",
|
||||
"start_index",
|
||||
"end_index",
|
||||
)
|
||||
|
||||
|
||||
def build_target_execution_contract(
|
||||
target: RefineTarget,
|
||||
refine_schedule_contract: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Combine target policy with sigma-range identity without mutating V1."""
|
||||
|
||||
resolved = resolve_refine_target(target)
|
||||
if not isinstance(refine_schedule_contract, Mapping):
|
||||
raise RefineTargetError("refine_schedule_contract must be a mapping")
|
||||
missing = [
|
||||
name for name in _SIGMA_RANGE_FIELDS if name not in refine_schedule_contract
|
||||
]
|
||||
if missing or not refine_schedule_contract.get("schedule_hash"):
|
||||
details = ", ".join([*missing, "schedule_hash"] if not refine_schedule_contract.get("schedule_hash") else missing)
|
||||
raise RefineTargetError(f"refine schedule range identity is incomplete: {details}")
|
||||
|
||||
sigma_range = {
|
||||
name: refine_schedule_contract[name] for name in _SIGMA_RANGE_FIELDS
|
||||
}
|
||||
sigma_range["source_schedule_hash"] = str(
|
||||
refine_schedule_contract["schedule_hash"]
|
||||
)
|
||||
contract: dict[str, Any] = {
|
||||
"magic": EXECUTION_MAGIC,
|
||||
"schema_version": EXECUTION_SCHEMA_VERSION,
|
||||
"policy_authority": "refine_target_contract_v1",
|
||||
"target": serializable_target_contract(resolved),
|
||||
"sigma_range": sigma_range,
|
||||
}
|
||||
contract["execution_hash"] = _identity(contract)
|
||||
return contract
|
||||
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
"""Pure visible-timeline resolution for Experimental temporal refinement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
|
||||
from ..temporal import audio_latent_t, latent_slot_offsets, pixel_frames_for_latent_t
|
||||
|
||||
|
||||
MAGIC = "H3_CONTINUUM_REFINE_WINDOW"
|
||||
SCHEMA_VERSION = 1
|
||||
MODE_TIME_WINDOW = "time_window"
|
||||
|
||||
|
||||
class RefineWindowError(ValueError):
|
||||
"""Raised before preparation when a visible time window is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GroupTemporalWindow:
|
||||
"""One physical group's immutable temporal mask ranges."""
|
||||
|
||||
group_index: int
|
||||
logical_chunks: tuple[int, ...]
|
||||
terminal_merged: bool
|
||||
visible_frame_ranges: tuple[tuple[int, int], ...]
|
||||
physical_frame_ranges: tuple[tuple[int, int], ...]
|
||||
video_slot_ranges: tuple[tuple[int, int], ...]
|
||||
audio_tick_ranges: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RefineWindow:
|
||||
"""One versioned visible-output window with no Tensor or runtime object."""
|
||||
|
||||
selected_group_indices: tuple[int, ...]
|
||||
groups: tuple[GroupTemporalWindow, ...]
|
||||
contract: Mapping[str, Any]
|
||||
|
||||
def group(self, group_index: int) -> GroupTemporalWindow | None:
|
||||
for group in self.groups:
|
||||
if group.group_index == int(group_index):
|
||||
return group
|
||||
return None
|
||||
|
||||
|
||||
def _identity(payload: Mapping[str, Any]) -> str:
|
||||
encoded = json.dumps(
|
||||
dict(payload),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _merge_ranges(values: Sequence[tuple[int, int]]) -> tuple[tuple[int, int], ...]:
|
||||
ordered = sorted(
|
||||
(int(start), int(stop))
|
||||
for start, stop in values
|
||||
if int(stop) > int(start)
|
||||
)
|
||||
merged: list[list[int]] = []
|
||||
for start, stop in ordered:
|
||||
if merged and start <= merged[-1][1]:
|
||||
merged[-1][1] = max(merged[-1][1], stop)
|
||||
else:
|
||||
merged.append([start, stop])
|
||||
return tuple((start, stop) for start, stop in merged)
|
||||
|
||||
|
||||
def _indices_to_ranges(indices: Sequence[int]) -> tuple[tuple[int, int], ...]:
|
||||
ordered = sorted(set(int(value) for value in indices))
|
||||
if not ordered:
|
||||
return ()
|
||||
ranges: list[tuple[int, int]] = []
|
||||
start = previous = ordered[0]
|
||||
for value in ordered[1:]:
|
||||
if value == previous + 1:
|
||||
previous = value
|
||||
continue
|
||||
ranges.append((start, previous + 1))
|
||||
start = previous = value
|
||||
ranges.append((start, previous + 1))
|
||||
return tuple(ranges)
|
||||
|
||||
|
||||
def _overlaps(
|
||||
start: int,
|
||||
stop: int,
|
||||
ranges: Sequence[tuple[int, int]],
|
||||
) -> bool:
|
||||
return any(start < range_stop and stop > range_start for range_start, range_stop in ranges)
|
||||
|
||||
|
||||
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise RefineWindowError("assembly_plan must be a mapping")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping) or contract.get("version") != 1:
|
||||
raise RefineWindowError("assembly_plan has no Second Pass contract version 1")
|
||||
groups = contract.get("physical_groups")
|
||||
if (
|
||||
not isinstance(groups, Sequence)
|
||||
or isinstance(groups, (str, bytes))
|
||||
or not groups
|
||||
or not all(isinstance(group, Mapping) for group in groups)
|
||||
):
|
||||
raise RefineWindowError("assembly_plan has no physical groups")
|
||||
return list(groups)
|
||||
|
||||
|
||||
def _logical_chunks(group: Mapping[str, Any], group_index: int) -> tuple[int, ...]:
|
||||
values = group.get("logical_chunks")
|
||||
if (
|
||||
not isinstance(values, Sequence)
|
||||
or isinstance(values, (str, bytes))
|
||||
or not values
|
||||
or any(type(value) is not int or value <= 0 for value in values)
|
||||
):
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} has invalid logical chunk identity"
|
||||
)
|
||||
chunks = tuple(int(value) for value in values)
|
||||
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} logical chunks are not ordered and unique"
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def _visible_units(
|
||||
assembly_plan: Mapping[str, Any],
|
||||
groups: Sequence[Mapping[str, Any]],
|
||||
) -> tuple[list[dict[str, int]], int]:
|
||||
decode_groups = assembly_plan.get("decode_groups")
|
||||
if isinstance(decode_groups, Sequence) and not isinstance(
|
||||
decode_groups, (str, bytes)
|
||||
):
|
||||
if len(decode_groups) != len(groups) or not all(
|
||||
isinstance(group, Mapping) for group in decode_groups
|
||||
):
|
||||
raise RefineWindowError(
|
||||
"assembly_plan decode groups differ from Second Pass physical groups"
|
||||
)
|
||||
source_units: Sequence[Mapping[str, Any]] = decode_groups
|
||||
else:
|
||||
source_units = groups
|
||||
|
||||
cursor = 0
|
||||
units: list[dict[str, int]] = []
|
||||
for group_index, (group, source) in enumerate(zip(groups, source_units, strict=True)):
|
||||
physical_frames = int(group.get("physical_frames", 0))
|
||||
trim_frames = int(group.get("trim_prefix_frames", -1))
|
||||
source_latent_t = int(group.get("source_latent_t", 0))
|
||||
audio_shape = group.get("source_audio_shape")
|
||||
if physical_frames <= 0 or trim_frames < 0 or trim_frames >= physical_frames:
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} frame geometry is invalid"
|
||||
)
|
||||
if source_latent_t <= 0 or pixel_frames_for_latent_t(source_latent_t) != physical_frames:
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} Video latent T differs from its frame grid"
|
||||
)
|
||||
if (
|
||||
not isinstance(audio_shape, Sequence)
|
||||
or isinstance(audio_shape, (str, bytes))
|
||||
or len(audio_shape) != 4
|
||||
or int(audio_shape[-1]) != audio_latent_t(physical_frames)
|
||||
):
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} Audio latent T differs from its frame grid"
|
||||
)
|
||||
net_frames = physical_frames - trim_frames
|
||||
source_total = int(source.get("total_frames", physical_frames))
|
||||
source_trim = int(source.get("trim_frames", trim_frames))
|
||||
source_net = int(source.get("net_frames", source_total - source_trim))
|
||||
source_start = int(source.get("frame_start", cursor))
|
||||
source_stop = int(source.get("frame_stop", source_start + source_net))
|
||||
if (
|
||||
source_total != physical_frames
|
||||
or source_trim != trim_frames
|
||||
or source_net != net_frames
|
||||
or source_start != cursor
|
||||
or source_stop != cursor + net_frames
|
||||
):
|
||||
raise RefineWindowError(
|
||||
f"physical group {group_index + 1} visible range is inconsistent"
|
||||
)
|
||||
units.append(
|
||||
{
|
||||
"group_index": group_index,
|
||||
"visible_start": source_start,
|
||||
"visible_stop": source_stop,
|
||||
"physical_frames": physical_frames,
|
||||
"trim_frames": trim_frames,
|
||||
"video_t": source_latent_t,
|
||||
"audio_t": int(audio_shape[-1]),
|
||||
}
|
||||
)
|
||||
cursor = source_stop
|
||||
return units, cursor
|
||||
|
||||
|
||||
def _output_to_natural_ranges(
|
||||
output_start: int,
|
||||
output_stop: int,
|
||||
*,
|
||||
natural_frames: int,
|
||||
target_frames: int,
|
||||
preserve_final_frame: bool,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
mappings: list[tuple[int, int, int, bool]] = []
|
||||
if target_frames >= natural_frames:
|
||||
mappings.append((0, natural_frames, 0, False))
|
||||
if target_frames > natural_frames:
|
||||
mappings.append((natural_frames, target_frames, natural_frames - 1, True))
|
||||
elif preserve_final_frame and target_frames >= 2:
|
||||
mappings.extend(
|
||||
(
|
||||
(0, target_frames - 1, 0, False),
|
||||
(target_frames - 1, target_frames, natural_frames - 1, True),
|
||||
)
|
||||
)
|
||||
else:
|
||||
mappings.append((0, target_frames, 0, False))
|
||||
|
||||
natural_ranges: list[tuple[int, int]] = []
|
||||
for destination_start, destination_stop, source_start, repeated in mappings:
|
||||
overlap_start = max(output_start, destination_start)
|
||||
overlap_stop = min(output_stop, destination_stop)
|
||||
if overlap_stop <= overlap_start:
|
||||
continue
|
||||
if repeated:
|
||||
natural_ranges.append((source_start, source_start + 1))
|
||||
else:
|
||||
mapped_start = source_start + overlap_start - destination_start
|
||||
mapped_stop = source_start + overlap_stop - destination_start
|
||||
natural_ranges.append((mapped_start, mapped_stop))
|
||||
return _merge_ranges(natural_ranges)
|
||||
|
||||
|
||||
def _group_window(
|
||||
group: Mapping[str, Any],
|
||||
unit: Mapping[str, int],
|
||||
natural_ranges: Sequence[tuple[int, int]],
|
||||
) -> GroupTemporalWindow | None:
|
||||
visible_start = int(unit["visible_start"])
|
||||
visible_stop = int(unit["visible_stop"])
|
||||
visible_ranges = _merge_ranges(
|
||||
(
|
||||
max(start, visible_start),
|
||||
min(stop, visible_stop),
|
||||
)
|
||||
for start, stop in natural_ranges
|
||||
if start < visible_stop and stop > visible_start
|
||||
)
|
||||
if not visible_ranges:
|
||||
return None
|
||||
|
||||
trim_frames = int(unit["trim_frames"])
|
||||
physical_ranges = _merge_ranges(
|
||||
(
|
||||
trim_frames + start - visible_start,
|
||||
trim_frames + stop - visible_start,
|
||||
)
|
||||
for start, stop in visible_ranges
|
||||
)
|
||||
video_t = int(unit["video_t"])
|
||||
offsets = latent_slot_offsets(video_t)
|
||||
selected_slots = []
|
||||
for slot, slot_start in enumerate(offsets):
|
||||
slot_stop = offsets[slot + 1] if slot + 1 < video_t else int(unit["physical_frames"])
|
||||
if _overlaps(slot_start, slot_stop, physical_ranges):
|
||||
selected_slots.append(slot)
|
||||
video_ranges = _indices_to_ranges(selected_slots)
|
||||
if video_ranges and offsets[video_ranges[0][0]] < trim_frames:
|
||||
raise RefineWindowError(
|
||||
f"physical group {int(unit['group_index']) + 1} Video grid would expose "
|
||||
"the protected continuation prefix"
|
||||
)
|
||||
|
||||
audio_ranges = _merge_ranges(
|
||||
(
|
||||
max(0, min(int(unit["audio_t"]), audio_latent_t(start))),
|
||||
max(0, min(int(unit["audio_t"]), audio_latent_t(stop))),
|
||||
)
|
||||
for start, stop in physical_ranges
|
||||
)
|
||||
protected_audio_stop = audio_latent_t(trim_frames)
|
||||
if audio_ranges and audio_ranges[0][0] < protected_audio_stop:
|
||||
raise RefineWindowError(
|
||||
f"physical group {int(unit['group_index']) + 1} Audio grid would expose "
|
||||
"the protected continuation prefix"
|
||||
)
|
||||
if not video_ranges or not audio_ranges:
|
||||
raise RefineWindowError(
|
||||
f"physical group {int(unit['group_index']) + 1} window resolves to an empty AV grid"
|
||||
)
|
||||
return GroupTemporalWindow(
|
||||
group_index=int(unit["group_index"]),
|
||||
logical_chunks=_logical_chunks(group, int(unit["group_index"])),
|
||||
terminal_merged=bool(group.get("terminal_merged", False)),
|
||||
visible_frame_ranges=visible_ranges,
|
||||
physical_frame_ranges=physical_ranges,
|
||||
video_slot_ranges=video_ranges,
|
||||
audio_tick_ranges=audio_ranges,
|
||||
)
|
||||
|
||||
|
||||
def resolve_refine_window(
|
||||
start_sec: float,
|
||||
end_sec: float,
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> RefineWindow:
|
||||
"""Resolve final visible output time to protected physical AV ranges.
|
||||
|
||||
The requested interval is half-open. Fractions select every output frame
|
||||
interval they overlap. Continuation trim prefixes are never selected.
|
||||
"""
|
||||
|
||||
try:
|
||||
requested_start = float(start_sec)
|
||||
requested_end = float(end_sec)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RefineWindowError("refine window seconds must be numeric") from exc
|
||||
if not math.isfinite(requested_start) or not math.isfinite(requested_end):
|
||||
raise RefineWindowError("refine window seconds must be finite")
|
||||
if requested_start >= requested_end:
|
||||
raise RefineWindowError("refine window start must be less than end")
|
||||
|
||||
groups = _contract_groups(assembly_plan)
|
||||
units, natural_frames = _visible_units(assembly_plan, groups)
|
||||
fps = int(assembly_plan.get("fps", 0))
|
||||
target_frames = int(assembly_plan.get("target_frames", 0))
|
||||
if fps <= 0 or target_frames <= 0 or natural_frames <= 0:
|
||||
raise RefineWindowError("assembly_plan visible timeline is invalid")
|
||||
|
||||
clamped_start = min(max(requested_start, 0.0), target_frames / fps)
|
||||
clamped_end = min(max(requested_end, 0.0), target_frames / fps)
|
||||
output_start = max(0, min(target_frames, math.floor(clamped_start * fps + 1e-9)))
|
||||
output_stop = max(0, min(target_frames, math.ceil(clamped_end * fps - 1e-9)))
|
||||
if output_stop <= output_start:
|
||||
raise RefineWindowError("refine window does not overlap the visible output")
|
||||
|
||||
natural_ranges = _output_to_natural_ranges(
|
||||
output_start,
|
||||
output_stop,
|
||||
natural_frames=natural_frames,
|
||||
target_frames=target_frames,
|
||||
preserve_final_frame=bool(assembly_plan.get("preserve_final_frame", False)),
|
||||
)
|
||||
if not natural_ranges:
|
||||
raise RefineWindowError("refine window has no natural timeline source")
|
||||
|
||||
resolved_groups = tuple(
|
||||
group_window
|
||||
for group, unit in zip(groups, units, strict=True)
|
||||
if (
|
||||
group_window := _group_window(
|
||||
group,
|
||||
unit,
|
||||
natural_ranges,
|
||||
)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
if not resolved_groups:
|
||||
raise RefineWindowError("refine window selects no physical group")
|
||||
selected = tuple(group.group_index for group in resolved_groups)
|
||||
group_contracts = [
|
||||
{
|
||||
"group_index": group.group_index,
|
||||
"physical_group": group.group_index + 1,
|
||||
"logical_chunks": list(group.logical_chunks),
|
||||
"terminal_merged": group.terminal_merged,
|
||||
"visible_frame_ranges": [list(value) for value in group.visible_frame_ranges],
|
||||
"physical_frame_ranges": [list(value) for value in group.physical_frame_ranges],
|
||||
"video_slot_ranges": [list(value) for value in group.video_slot_ranges],
|
||||
"audio_tick_ranges": [list(value) for value in group.audio_tick_ranges],
|
||||
}
|
||||
for group in resolved_groups
|
||||
]
|
||||
contract: dict[str, Any] = {
|
||||
"magic": MAGIC,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": MODE_TIME_WINDOW,
|
||||
"timeline": "final_visible_output",
|
||||
"interval": "half_open",
|
||||
"requested_start_sec": requested_start,
|
||||
"requested_end_sec": requested_end,
|
||||
"effective_start_sec": output_start / fps,
|
||||
"effective_end_sec": output_stop / fps,
|
||||
"output_frame_range": [output_start, output_stop],
|
||||
"target_frames": target_frames,
|
||||
"natural_frames": natural_frames,
|
||||
"natural_frame_ranges": [list(value) for value in natural_ranges],
|
||||
"selected_physical_groups": [index + 1 for index in selected],
|
||||
"groups": group_contracts,
|
||||
"continuation_prefix_policy": "always_protected",
|
||||
"outside_window_policy": "mask_zero_then_bit_exact_restore",
|
||||
"physical_group_atomicity": "preserved",
|
||||
"temporal_quantization": "intersecting_video_slots_and_audio_ticks",
|
||||
}
|
||||
contract["window_hash"] = _identity(contract)
|
||||
return RefineWindow(
|
||||
selected_group_indices=selected,
|
||||
groups=resolved_groups,
|
||||
contract=MappingProxyType(contract),
|
||||
)
|
||||
|
||||
|
||||
def serializable_window_contract(window: RefineWindow) -> dict[str, Any]:
|
||||
if not isinstance(window, RefineWindow):
|
||||
raise RefineWindowError("refine_window must be a RefineWindow")
|
||||
return json.loads(json.dumps(dict(window.contract)))
|
||||
+920
@@ -0,0 +1,920 @@
|
||||
"""Read-only V3.8 reliability diagnostics.
|
||||
|
||||
The helpers in this module inspect shapes and existing metadata only. They do
|
||||
not resize, encode, hash, copy, or otherwise modify user tensors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..compatibility import check_comfy_h3_runtime
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_BASIC,
|
||||
DIAGNOSTICS_FULL,
|
||||
DIAGNOSTICS_OFF,
|
||||
FPS,
|
||||
normalize_diagnostics_mode,
|
||||
)
|
||||
from ..reference import resolve_reference_image_size
|
||||
from ..reference_video import (
|
||||
REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
resolve_reference_video_frame_count,
|
||||
resolve_reference_video_size,
|
||||
)
|
||||
from ..temporal import (
|
||||
align_frame_count_up,
|
||||
audio_grid_offset,
|
||||
audio_latent_t,
|
||||
context_slots,
|
||||
is_valid_frame_count,
|
||||
video_latent_t,
|
||||
)
|
||||
|
||||
|
||||
VISUAL_LOAD_LOW_MAX = 0.75
|
||||
VISUAL_LOAD_MEDIUM_MAX = 1.75
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DiagnosticSection:
|
||||
basic: tuple[str, ...] = ()
|
||||
detailed: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class V38Diagnostics:
|
||||
basic: tuple[str, ...]
|
||||
detailed: tuple[str, ...]
|
||||
|
||||
|
||||
def _combine_sections(*sections: DiagnosticSection) -> V38Diagnostics:
|
||||
return V38Diagnostics(
|
||||
basic=tuple(line for section in sections for line in section.basic),
|
||||
detailed=tuple(line for section in sections for line in section.detailed),
|
||||
)
|
||||
|
||||
|
||||
def _physical_groups(assembly_plan: Any) -> list[Any]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
return []
|
||||
decode_groups = assembly_plan.get("decode_groups")
|
||||
if isinstance(decode_groups, list) and decode_groups:
|
||||
return list(decode_groups)
|
||||
chunks = assembly_plan.get("chunks")
|
||||
return list(chunks) if isinstance(chunks, list) else []
|
||||
|
||||
|
||||
def _latent_outputs(value: Any) -> list[Any]:
|
||||
return list(value) if isinstance(value, (list, tuple)) else []
|
||||
|
||||
|
||||
def _latent_samples(value: Any) -> Any:
|
||||
return value.get("samples") if isinstance(value, Mapping) else None
|
||||
|
||||
|
||||
def _image_geometry(value: Any) -> tuple[int, int]:
|
||||
shape = getattr(value, "shape", None)
|
||||
if shape is None or len(shape) != 4:
|
||||
raise ValueError("IMAGE must have shape [B,H,W,C]")
|
||||
height = int(shape[1])
|
||||
width = int(shape[2])
|
||||
if height < 1 or width < 1:
|
||||
raise ValueError("IMAGE geometry must be positive")
|
||||
return width, height
|
||||
|
||||
|
||||
def _video_guide_geometry(value: Any) -> tuple[int, int, int]:
|
||||
shape = getattr(value, "shape", None)
|
||||
if shape is None or len(shape) != 4:
|
||||
raise ValueError("Video Guide must have shape [T,H,W,C]")
|
||||
frames = int(shape[0])
|
||||
height = int(shape[1])
|
||||
width = int(shape[2])
|
||||
if frames < 1 or height < 1 or width < 1:
|
||||
raise ValueError("Video Guide geometry must be positive")
|
||||
return frames, width, height
|
||||
|
||||
|
||||
def _native_chunk_frames(chunk_seconds: float) -> int:
|
||||
return align_frame_count_up(int(round(float(chunk_seconds) * FPS)))
|
||||
|
||||
|
||||
def build_frame_alignment_diagnostics(
|
||||
*,
|
||||
video_latents: Any,
|
||||
assembly_plan: Any,
|
||||
video_guide: Any = None,
|
||||
chunk_seconds: float,
|
||||
) -> DiagnosticSection:
|
||||
"""Inspect Video Guide alignment and physical video latent lengths."""
|
||||
|
||||
basic: list[str] = []
|
||||
detailed: list[str] = []
|
||||
if video_guide is not None:
|
||||
try:
|
||||
source_frames, _source_width, _source_height = _video_guide_geometry(
|
||||
video_guide
|
||||
)
|
||||
aligned_frames = resolve_reference_video_frame_count(
|
||||
source_frames,
|
||||
_native_chunk_frames(chunk_seconds),
|
||||
)
|
||||
if source_frames < aligned_frames:
|
||||
action = "final-frame pad"
|
||||
elif source_frames > aligned_frames:
|
||||
action = "trimmed to one-chunk H3 limit"
|
||||
else:
|
||||
action = "unchanged"
|
||||
line = (
|
||||
f"Frame alignment: Video Guide {source_frames} -> {aligned_frames} "
|
||||
f"({action}, H3 17k+5)."
|
||||
)
|
||||
if action != "unchanged":
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
except Exception as exc:
|
||||
line = (
|
||||
"Frame alignment advisory: Video Guide alignment could not be "
|
||||
f"inspected ({type(exc).__name__}: {exc})."
|
||||
)
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
|
||||
groups = _physical_groups(assembly_plan)
|
||||
outputs = _latent_outputs(video_latents)
|
||||
if not groups:
|
||||
line = "Frame alignment advisory: physical group metadata is unavailable."
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
return DiagnosticSection(tuple(basic), tuple(detailed))
|
||||
if len(outputs) != len(groups):
|
||||
line = (
|
||||
"Frame alignment advisory: physical video output count "
|
||||
f"{len(outputs)} does not match group count {len(groups)}."
|
||||
)
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
|
||||
for index, group in enumerate(groups, start=1):
|
||||
problems: list[str] = []
|
||||
try:
|
||||
if not isinstance(group, Mapping):
|
||||
raise ValueError("group metadata is not a mapping")
|
||||
frames = int(group["total_frames"])
|
||||
trim = int(group["trim_frames"])
|
||||
net = int(group["net_frames"])
|
||||
metadata_t = int(group["expected_video_latent_t"])
|
||||
if not is_valid_frame_count(frames):
|
||||
problems.append("frames are not on H3 17k+5")
|
||||
expected_t = None
|
||||
else:
|
||||
expected_t = video_latent_t(frames)
|
||||
if metadata_t != expected_t:
|
||||
problems.append("metadata T differs from native grid")
|
||||
if frames - trim != net:
|
||||
problems.append("total-trim differs from net")
|
||||
samples = (
|
||||
_latent_samples(outputs[index - 1])
|
||||
if index <= len(outputs)
|
||||
else None
|
||||
)
|
||||
if not torch.is_tensor(samples) or samples.ndim < 3:
|
||||
actual_t = None
|
||||
problems.append("video LATENT samples are missing or invalid")
|
||||
else:
|
||||
actual_t = int(samples.shape[2])
|
||||
if actual_t != metadata_t:
|
||||
problems.append("actual T differs from metadata")
|
||||
if expected_t is not None and actual_t != expected_t:
|
||||
problems.append("actual T differs from native grid")
|
||||
result = "PASS" if not problems else "ADVISORY"
|
||||
expected_text = "n/a" if expected_t is None else str(expected_t)
|
||||
actual_text = "n/a" if actual_t is None else str(actual_t)
|
||||
detailed.append(
|
||||
f"Frame grid: group={index} frames={frames} trim={trim} net={net} "
|
||||
f"video_t={actual_text} metadata={metadata_t} "
|
||||
f"expected={expected_text} {result}."
|
||||
)
|
||||
except Exception as exc:
|
||||
problems.append(f"{type(exc).__name__}: {exc}")
|
||||
detailed.append(
|
||||
f"Frame grid: group={index} unavailable ADVISORY "
|
||||
f"({problems[-1]})."
|
||||
)
|
||||
if problems:
|
||||
basic.append(
|
||||
f"Frame alignment advisory: group {index}: "
|
||||
+ "; ".join(problems)
|
||||
+ "."
|
||||
)
|
||||
return DiagnosticSection(tuple(basic), tuple(detailed))
|
||||
|
||||
|
||||
def validate_audio_latent_lengths(
|
||||
*,
|
||||
audio_latents: Any,
|
||||
assembly_plan: Any,
|
||||
) -> DiagnosticSection:
|
||||
"""Validate physical audio latent shapes without reading tensor values."""
|
||||
|
||||
basic: list[str] = []
|
||||
detailed: list[str] = []
|
||||
groups = _physical_groups(assembly_plan)
|
||||
outputs = _latent_outputs(audio_latents)
|
||||
if not groups:
|
||||
line = "Audio latent advisory: physical group metadata is unavailable."
|
||||
return DiagnosticSection((line,), (line,))
|
||||
if len(outputs) != len(groups):
|
||||
line = (
|
||||
f"Audio latent advisory: physical output count {len(outputs)} "
|
||||
f"does not match group count {len(groups)}."
|
||||
)
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
|
||||
for index, group in enumerate(groups, start=1):
|
||||
problems: list[str] = []
|
||||
frames = metadata_t = nominal_t = actual_t = None
|
||||
phase = None
|
||||
try:
|
||||
if not isinstance(group, Mapping):
|
||||
raise ValueError("group metadata is not a mapping")
|
||||
frames = int(group["total_frames"])
|
||||
metadata_t = int(group["expected_audio_latent_t"])
|
||||
nominal_t = audio_latent_t(frames)
|
||||
samples = (
|
||||
_latent_samples(outputs[index - 1])
|
||||
if index <= len(outputs)
|
||||
else None
|
||||
)
|
||||
if not torch.is_tensor(samples):
|
||||
problems.append("LATENT samples are missing or not a Tensor")
|
||||
elif samples.ndim != 4:
|
||||
problems.append(f"rank is {samples.ndim}, expected 4")
|
||||
else:
|
||||
if int(samples.shape[0]) < 1:
|
||||
problems.append("batch dimension is empty")
|
||||
if int(samples.shape[1]) != 32:
|
||||
problems.append(f"channels={int(samples.shape[1])}, expected 32")
|
||||
if int(samples.shape[2]) != 2:
|
||||
problems.append(f"stereo={int(samples.shape[2])}, expected 2")
|
||||
actual_t = int(samples.shape[-1])
|
||||
if actual_t <= 0:
|
||||
problems.append("T must be positive")
|
||||
else:
|
||||
phase = audio_grid_offset(frames, actual_t)
|
||||
if actual_t != metadata_t:
|
||||
problems.append(
|
||||
f"metadata={metadata_t} actual={actual_t} "
|
||||
f"delta={actual_t - metadata_t}"
|
||||
)
|
||||
if not -0.500001 <= phase <= 0.500001:
|
||||
problems.append(f"phase={phase:.6f} is outside [-0.5,0.5]")
|
||||
result = "PASS" if not problems else "ADVISORY"
|
||||
detailed.append(
|
||||
f"Audio latent check: group={index} frames={frames} "
|
||||
f"metadata={metadata_t} actual="
|
||||
f"{'n/a' if actual_t is None else actual_t} nominal={nominal_t} "
|
||||
f"phase={'n/a' if phase is None else f'{phase:.6f}'} {result}."
|
||||
)
|
||||
except Exception as exc:
|
||||
problems.append(f"{type(exc).__name__}: {exc}")
|
||||
detailed.append(
|
||||
f"Audio latent check: group={index} unavailable ADVISORY "
|
||||
f"({problems[-1]})."
|
||||
)
|
||||
if problems:
|
||||
basic.append(
|
||||
f"Audio latent advisory: group {index}: "
|
||||
+ "; ".join(problems)
|
||||
+ "."
|
||||
)
|
||||
return DiagnosticSection(tuple(basic), tuple(detailed))
|
||||
|
||||
|
||||
def collect_core_compatibility_advisory(
|
||||
checker: Callable[[], list[str]] | None = None,
|
||||
) -> DiagnosticSection:
|
||||
"""Collect the existing feature-based Core advisory after successful generation."""
|
||||
|
||||
active_checker = checker or check_comfy_h3_runtime
|
||||
try:
|
||||
issues = tuple(dict.fromkeys(str(issue) for issue in active_checker() if issue))
|
||||
except Exception as exc:
|
||||
basic = (
|
||||
"Core compatibility advisory unavailable; generation result was preserved.",
|
||||
)
|
||||
detailed = (
|
||||
basic[0][:-1] + f" ({type(exc).__name__}: {exc}).",
|
||||
)
|
||||
return DiagnosticSection(basic, detailed)
|
||||
if not issues:
|
||||
return DiagnosticSection(
|
||||
(),
|
||||
("Core compatibility: native MiniMax H3 contracts detected.",),
|
||||
)
|
||||
return DiagnosticSection(
|
||||
(
|
||||
"Core compatibility advisory: "
|
||||
f"{len(issues)} native H3 contract difference(s) detected; "
|
||||
"generation completed; enable Detailed Report for details.",
|
||||
),
|
||||
tuple(f"Core compatibility advisory: {issue}." for issue in issues),
|
||||
)
|
||||
|
||||
|
||||
def _visual_load_class(total: float) -> str:
|
||||
if total < VISUAL_LOAD_LOW_MAX:
|
||||
return "Low"
|
||||
if total < VISUAL_LOAD_MEDIUM_MAX:
|
||||
return "Medium"
|
||||
return "High"
|
||||
|
||||
|
||||
def estimate_visual_conditioning_load(
|
||||
*,
|
||||
output_width: int,
|
||||
output_height: int,
|
||||
reference_images: Sequence[Any],
|
||||
reference_size: str,
|
||||
video_guide: Any,
|
||||
video_guide_size: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
chunk_seconds: float,
|
||||
still_guide_active: bool,
|
||||
) -> DiagnosticSection:
|
||||
"""Estimate peak per-group relative visual conditioning from geometry only."""
|
||||
|
||||
output_area = int(output_width) * int(output_height)
|
||||
if output_area <= 0:
|
||||
line = "Visual conditioning load advisory: output geometry is invalid."
|
||||
return DiagnosticSection((line,), (line,))
|
||||
|
||||
components: list[tuple[str, float]] = []
|
||||
problems: list[str] = []
|
||||
for socket_index, image in enumerate(reference_images, start=1):
|
||||
if image is None:
|
||||
continue
|
||||
try:
|
||||
source_width, source_height = _image_geometry(image)
|
||||
target_width, target_height = resolve_reference_image_size(
|
||||
source_width,
|
||||
source_height,
|
||||
output_width=int(output_width),
|
||||
output_height=int(output_height),
|
||||
size_mode=str(reference_size),
|
||||
)
|
||||
load = (target_width * target_height) / float(output_area)
|
||||
components.append(
|
||||
(f"Ref{socket_index}={target_width}x{target_height}", load)
|
||||
)
|
||||
except Exception as exc:
|
||||
problems.append(
|
||||
f"Reference Image {socket_index} unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
if video_guide is not None:
|
||||
try:
|
||||
source_frames, source_width, source_height = _video_guide_geometry(
|
||||
video_guide
|
||||
)
|
||||
native_frames = _native_chunk_frames(chunk_seconds)
|
||||
aligned_frames = resolve_reference_video_frame_count(
|
||||
source_frames,
|
||||
native_frames,
|
||||
)
|
||||
target_width, target_height = resolve_reference_video_size(
|
||||
source_width,
|
||||
source_height,
|
||||
output_width=int(output_width),
|
||||
output_height=int(output_height),
|
||||
size_mode=str(video_guide_size),
|
||||
)
|
||||
spatial_ratio = (target_width * target_height) / float(output_area)
|
||||
temporal_ratio = video_latent_t(aligned_frames) / float(
|
||||
video_latent_t(native_frames)
|
||||
)
|
||||
components.append(
|
||||
(
|
||||
f"Video Guide={target_width}x{target_height}/{aligned_frames}f",
|
||||
spatial_ratio * temporal_ratio,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
problems.append(
|
||||
f"Video Guide unavailable ({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
basic: list[str] = []
|
||||
detailed: list[str] = []
|
||||
if components:
|
||||
total = sum(load for _name, load in components)
|
||||
classification = _visual_load_class(total)
|
||||
basic_line = (
|
||||
f"Visual conditioning load: {classification} "
|
||||
"(peak per physical group)."
|
||||
)
|
||||
if still_guide_active:
|
||||
basic_line = basic_line[:-1] + "; Still Guide active."
|
||||
basic.append(basic_line)
|
||||
details = "; ".join(
|
||||
f"{name}={load:.3f}" for name, load in components
|
||||
)
|
||||
detailed_line = (
|
||||
f"Visual conditioning load: total={total:.3f} "
|
||||
f"class={classification}; {details}"
|
||||
)
|
||||
if still_guide_active:
|
||||
detailed_line += "; Still Guide=active"
|
||||
detailed.append(detailed_line + ".")
|
||||
elif still_guide_active:
|
||||
basic.append("Still Guide: active.")
|
||||
detailed.append("Still Guide: active (excluded from numeric visual load).")
|
||||
|
||||
for problem in problems:
|
||||
line = f"Visual conditioning load advisory: {problem}."
|
||||
basic.append(line)
|
||||
detailed.append(line)
|
||||
return DiagnosticSection(tuple(basic), tuple(detailed))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LogicalChunkGeometry:
|
||||
index: int
|
||||
total_frames: int | None
|
||||
trim_frames: int | None
|
||||
net_frames: int | None
|
||||
derived_start: int
|
||||
derived_stop: int
|
||||
visible_start: int | None
|
||||
visible_stop: int | None
|
||||
problems: tuple[str, ...]
|
||||
|
||||
|
||||
def _chunk_boundary_advisory(message: str) -> str:
|
||||
return f"Chunk boundary advisory: {str(message).rstrip('.')}."
|
||||
|
||||
|
||||
def _logical_chunk_geometries(
|
||||
assembly_plan: Any,
|
||||
) -> tuple[list[_LogicalChunkGeometry], list[str]]:
|
||||
problems: list[str] = []
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
return [], ["Assembly Plan metadata is unavailable"]
|
||||
raw_chunks = assembly_plan.get("chunks")
|
||||
if not isinstance(raw_chunks, list) or not raw_chunks:
|
||||
return [], ["logical chunk metadata is unavailable"]
|
||||
|
||||
geometries: list[_LogicalChunkGeometry] = []
|
||||
cursor = 0
|
||||
for position, chunk in enumerate(raw_chunks, start=1):
|
||||
local: list[str] = []
|
||||
total = trim = net = None
|
||||
visible_start = visible_stop = None
|
||||
index = position
|
||||
if not isinstance(chunk, Mapping):
|
||||
local.append("metadata is not a mapping")
|
||||
else:
|
||||
try:
|
||||
index = int(chunk.get("chunk_index", chunk.get("sequence_index", position)))
|
||||
if index != position:
|
||||
local.append(
|
||||
f"logical order has Chunk {index} at position {position}"
|
||||
)
|
||||
except Exception:
|
||||
index = position
|
||||
local.append("logical chunk index is invalid")
|
||||
try:
|
||||
total = int(chunk["total_frames"])
|
||||
trim = int(chunk["trim_frames"])
|
||||
net = int(chunk["net_frames"])
|
||||
if total < 0:
|
||||
local.append("total_frames is negative")
|
||||
if trim < 0:
|
||||
local.append("trim_frames is negative")
|
||||
if trim > total:
|
||||
local.append("trim_frames exceeds total_frames")
|
||||
if net < 0:
|
||||
local.append("net_frames is negative")
|
||||
if total - trim != net:
|
||||
local.append("total_frames - trim_frames differs from net_frames")
|
||||
except Exception as exc:
|
||||
local.append(
|
||||
"required geometry is unavailable "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
derived_start = cursor
|
||||
derived_stop = cursor + (net if net is not None and net >= 0 else 0)
|
||||
if isinstance(chunk, Mapping) and "frame_start" in chunk:
|
||||
try:
|
||||
visible_start = int(chunk["frame_start"])
|
||||
if visible_start != derived_start:
|
||||
local.append(
|
||||
f"frame_start={visible_start} differs from derived {derived_start}"
|
||||
)
|
||||
except Exception:
|
||||
local.append("frame_start is invalid")
|
||||
else:
|
||||
visible_start = derived_start if net is not None else None
|
||||
if isinstance(chunk, Mapping) and "frame_stop" in chunk:
|
||||
try:
|
||||
visible_stop = int(chunk["frame_stop"])
|
||||
if visible_stop != derived_stop:
|
||||
local.append(
|
||||
f"frame_stop={visible_stop} differs from derived {derived_stop}"
|
||||
)
|
||||
except Exception:
|
||||
local.append("frame_stop is invalid")
|
||||
else:
|
||||
visible_stop = derived_stop if net is not None else None
|
||||
|
||||
geometries.append(
|
||||
_LogicalChunkGeometry(
|
||||
index=index,
|
||||
total_frames=total,
|
||||
trim_frames=trim,
|
||||
net_frames=net,
|
||||
derived_start=derived_start,
|
||||
derived_stop=derived_stop,
|
||||
visible_start=visible_start,
|
||||
visible_stop=visible_stop,
|
||||
problems=tuple(local),
|
||||
)
|
||||
)
|
||||
problems.extend(f"Chunk {position}: {problem}" for problem in local)
|
||||
cursor = derived_stop
|
||||
return geometries, problems
|
||||
|
||||
|
||||
def _physical_boundary_mapping(
|
||||
*,
|
||||
assembly_plan: Any,
|
||||
logical_count: int,
|
||||
video_latents: Any,
|
||||
audio_latents: Any,
|
||||
) -> tuple[list[Mapping[str, Any]], list[tuple[int, ...]], dict[int, int], list[str]]:
|
||||
problems: list[str] = []
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
return [], [], {}, ["physical group metadata is unavailable"]
|
||||
|
||||
if "decode_groups" in assembly_plan:
|
||||
raw_groups = assembly_plan.get("decode_groups")
|
||||
if not isinstance(raw_groups, list) or not raw_groups:
|
||||
return [], [], {}, ["physical groups are not a non-empty list"]
|
||||
groups = list(raw_groups)
|
||||
fallback = False
|
||||
else:
|
||||
raw_chunks = assembly_plan.get("chunks")
|
||||
if not isinstance(raw_chunks, list) or not raw_chunks:
|
||||
return [], [], {}, ["physical group metadata is unavailable"]
|
||||
groups = list(raw_chunks)
|
||||
fallback = True
|
||||
|
||||
normalized_groups: list[Mapping[str, Any]] = []
|
||||
logical_groups: list[tuple[int, ...]] = []
|
||||
logical_to_group: dict[int, int] = {}
|
||||
flattened: list[int] = []
|
||||
for group_index, group in enumerate(groups, start=1):
|
||||
if not isinstance(group, Mapping):
|
||||
problems.append(f"physical group {group_index} metadata is not a mapping")
|
||||
normalized_groups.append({})
|
||||
logical_groups.append(())
|
||||
continue
|
||||
normalized_groups.append(group)
|
||||
raw_logical = (
|
||||
[group.get("chunk_index", group.get("sequence_index", group_index))]
|
||||
if fallback
|
||||
else group.get("logical_chunk_indices")
|
||||
)
|
||||
if not isinstance(raw_logical, (list, tuple)) or not raw_logical:
|
||||
problems.append(f"physical group {group_index} has no logical chunks")
|
||||
logical_groups.append(())
|
||||
continue
|
||||
try:
|
||||
logical = tuple(int(value) for value in raw_logical)
|
||||
except Exception:
|
||||
problems.append(f"physical group {group_index} logical chunks are invalid")
|
||||
logical_groups.append(())
|
||||
continue
|
||||
logical_groups.append(logical)
|
||||
flattened.extend(logical)
|
||||
if tuple(sorted(logical)) != logical:
|
||||
problems.append(f"physical group {group_index} logical order is reversed")
|
||||
terminal = bool(group.get("terminal_merged", False))
|
||||
if terminal:
|
||||
expected_terminal = (logical_count - 1, logical_count)
|
||||
if len(logical) != 2 or logical != expected_terminal:
|
||||
problems.append(
|
||||
f"physical group {group_index} Terminal Merge mapping is invalid"
|
||||
)
|
||||
elif len(logical) != 1:
|
||||
problems.append(
|
||||
f"physical group {group_index} has multiple logical chunks without Terminal Merge"
|
||||
)
|
||||
for chunk_index in logical:
|
||||
if chunk_index < 1 or chunk_index > logical_count:
|
||||
problems.append(
|
||||
f"physical group {group_index} references invalid Chunk {chunk_index}"
|
||||
)
|
||||
elif chunk_index in logical_to_group:
|
||||
problems.append(f"logical Chunk {chunk_index} is mapped more than once")
|
||||
else:
|
||||
logical_to_group[chunk_index] = group_index
|
||||
|
||||
expected = list(range(1, logical_count + 1))
|
||||
if flattened != expected:
|
||||
missing = [value for value in expected if value not in flattened]
|
||||
if missing:
|
||||
problems.append(
|
||||
"physical group mapping does not cover all logical chunks "
|
||||
f"(missing={missing})"
|
||||
)
|
||||
if not missing and flattened != sorted(flattened):
|
||||
problems.append("physical group logical chunk order is not contiguous")
|
||||
|
||||
video_count = len(_latent_outputs(video_latents))
|
||||
audio_count = len(_latent_outputs(audio_latents))
|
||||
if video_count != len(groups) or audio_count != len(groups):
|
||||
problems.append(
|
||||
"physical output/group counts differ "
|
||||
f"(groups={len(groups)}, video={video_count}, audio={audio_count})"
|
||||
)
|
||||
return normalized_groups, logical_groups, logical_to_group, problems
|
||||
|
||||
|
||||
def _second_pass_boundary_contract_problems(
|
||||
*,
|
||||
assembly_plan: Any,
|
||||
groups: Sequence[Mapping[str, Any]],
|
||||
logical_groups: Sequence[tuple[int, ...]],
|
||||
) -> tuple[list[str], str | None]:
|
||||
if not isinstance(assembly_plan, Mapping) or "second_pass_contract" not in assembly_plan:
|
||||
return [], None
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
return ["Second Pass contract is not a mapping"], "Second Pass contract: UNAVAILABLE."
|
||||
contract_groups = contract.get("physical_groups")
|
||||
if not isinstance(contract_groups, list):
|
||||
return ["Second Pass physical_groups is not a list"], "Second Pass contract: UNAVAILABLE."
|
||||
|
||||
problems: list[str] = []
|
||||
if len(contract_groups) != len(groups):
|
||||
problems.append(
|
||||
"Second Pass physical group count differs from decode groups "
|
||||
f"({len(contract_groups)} != {len(groups)})"
|
||||
)
|
||||
for index, (decode_group, logical) in enumerate(
|
||||
zip(groups, logical_groups, strict=False)
|
||||
):
|
||||
if index >= len(contract_groups):
|
||||
break
|
||||
contract_group = contract_groups[index]
|
||||
if not isinstance(contract_group, Mapping):
|
||||
problems.append(f"Second Pass group {index} is not a mapping")
|
||||
continue
|
||||
raw_contract_logical = contract_group.get("logical_chunks")
|
||||
contract_logical = (
|
||||
tuple(raw_contract_logical)
|
||||
if isinstance(raw_contract_logical, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
checks = (
|
||||
("group_id", contract_group.get("group_id"), index),
|
||||
("logical_chunks", contract_logical, logical),
|
||||
(
|
||||
"physical_frames",
|
||||
contract_group.get("physical_frames"),
|
||||
decode_group.get("total_frames"),
|
||||
),
|
||||
(
|
||||
"trim_prefix_frames",
|
||||
contract_group.get("trim_prefix_frames"),
|
||||
decode_group.get("trim_frames"),
|
||||
),
|
||||
(
|
||||
"terminal_merged",
|
||||
bool(contract_group.get("terminal_merged", False)),
|
||||
bool(decode_group.get("terminal_merged", False)),
|
||||
),
|
||||
)
|
||||
for name, actual, expected in checks:
|
||||
if actual != expected:
|
||||
problems.append(
|
||||
f"Second Pass group {index} {name} differs "
|
||||
f"({actual!r} != {expected!r})"
|
||||
)
|
||||
result = "PASS" if not problems else "ADVISORY"
|
||||
return problems, f"Second Pass contract: groups={len(contract_groups)} {result}."
|
||||
|
||||
|
||||
def build_chunk_boundary_diagnostics(
|
||||
*,
|
||||
video_latents: Any,
|
||||
audio_latents: Any,
|
||||
assembly_plan: Any,
|
||||
) -> DiagnosticSection:
|
||||
"""Inspect logical/physical chunk boundaries using metadata and shapes only."""
|
||||
|
||||
detailed = [
|
||||
"Chunk boundary contract: structural diagnostic only; "
|
||||
"no image/audio similarity measurement."
|
||||
]
|
||||
basic: list[str] = []
|
||||
geometries, logical_problems = _logical_chunk_geometries(assembly_plan)
|
||||
groups, logical_groups, logical_to_group, mapping_problems = (
|
||||
_physical_boundary_mapping(
|
||||
assembly_plan=assembly_plan,
|
||||
logical_count=len(geometries),
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
)
|
||||
)
|
||||
second_pass_problems, second_pass_line = _second_pass_boundary_contract_problems(
|
||||
assembly_plan=assembly_plan,
|
||||
groups=groups,
|
||||
logical_groups=logical_groups,
|
||||
)
|
||||
global_problems = logical_problems + mapping_problems + second_pass_problems
|
||||
for problem in dict.fromkeys(global_problems):
|
||||
basic.append(_chunk_boundary_advisory(problem))
|
||||
if second_pass_line is not None:
|
||||
detailed.append(second_pass_line)
|
||||
|
||||
for boundary_index in range(1, len(geometries)):
|
||||
left = geometries[boundary_index - 1]
|
||||
right = geometries[boundary_index]
|
||||
local = list(left.problems) + list(right.problems)
|
||||
if left.visible_stop is None or right.visible_start is None:
|
||||
gap = None
|
||||
local.append("visible range is unavailable")
|
||||
else:
|
||||
gap = int(right.visible_start) - int(left.visible_stop)
|
||||
if gap != 0:
|
||||
local.append("non-contiguous visible range")
|
||||
|
||||
context_frames = right.trim_frames
|
||||
context_video_t = context_audio_t = None
|
||||
if context_frames is None:
|
||||
local.append("incoming context is unavailable")
|
||||
else:
|
||||
try:
|
||||
context_video_t = context_slots(context_frames)
|
||||
context_audio_t = audio_latent_t(context_frames)
|
||||
except Exception as exc:
|
||||
local.append(
|
||||
f"incoming context {context_frames}f is invalid "
|
||||
f"({type(exc).__name__}: {exc})"
|
||||
)
|
||||
|
||||
left_group = logical_to_group.get(left.index)
|
||||
right_group = logical_to_group.get(right.index)
|
||||
if left_group is None or right_group is None:
|
||||
local.append("physical group mapping is unavailable")
|
||||
classification = "UNAVAILABLE"
|
||||
detailed.append(
|
||||
f"Boundary {left.index}→{right.index}: physical mapping unavailable; "
|
||||
f"UNAVAILABLE ({'; '.join(dict.fromkeys(local))})."
|
||||
)
|
||||
elif left_group == right_group:
|
||||
group = groups[right_group - 1] if right_group <= len(groups) else {}
|
||||
terminal = bool(group.get("terminal_merged", False))
|
||||
expected_pair = (left.index, right.index)
|
||||
actual_pair = (
|
||||
logical_groups[right_group - 1]
|
||||
if right_group <= len(logical_groups)
|
||||
else ()
|
||||
)
|
||||
if not terminal:
|
||||
local.append("shared physical group lacks terminal_merged=yes")
|
||||
if actual_pair != expected_pair:
|
||||
local.append("shared physical group does not match this boundary")
|
||||
classification = "PASS" if not local else "ADVISORY"
|
||||
detailed.append(
|
||||
f"Boundary {left.index}→{right.index}: shared physical group={right_group}; "
|
||||
f"terminal_merged={'yes' if terminal else 'no'}; "
|
||||
"no external decode boundary; "
|
||||
f"{classification}"
|
||||
+ (f" ({'; '.join(dict.fromkeys(local))})" if local else "")
|
||||
+ "."
|
||||
)
|
||||
else:
|
||||
classification = "PASS" if not local else "ADVISORY"
|
||||
visible_left = "n/a" if left.visible_stop is None else str(left.visible_stop)
|
||||
visible_right = "n/a" if right.visible_start is None else str(right.visible_start)
|
||||
gap_text = "n/a" if gap is None else str(gap)
|
||||
context_text = "n/a" if context_frames is None else f"{context_frames}f"
|
||||
video_text = "n/a" if context_video_t is None else f"{context_video_t}T"
|
||||
audio_text = "n/a" if context_audio_t is None else f"{context_audio_t}T"
|
||||
right_text = (
|
||||
"right=unavailable"
|
||||
if right.total_frames is None
|
||||
or right.trim_frames is None
|
||||
or right.net_frames is None
|
||||
else (
|
||||
f"right={right.total_frames}f trim={right.trim_frames} "
|
||||
f"net={right.net_frames}"
|
||||
)
|
||||
)
|
||||
detailed.append(
|
||||
f"Boundary {left.index}→{right.index}: physical; "
|
||||
f"visible={visible_left}→{visible_right} gap={gap_text}; "
|
||||
f"context={context_text} video={video_text} audio={audio_text}; "
|
||||
f"{right_text}; {classification}"
|
||||
+ (f" ({'; '.join(dict.fromkeys(local))})" if local else "")
|
||||
+ "."
|
||||
)
|
||||
|
||||
if local:
|
||||
if gap not in (None, 0):
|
||||
message = (
|
||||
f"boundary {left.index}→{right.index} has a non-contiguous "
|
||||
"visible range"
|
||||
)
|
||||
elif classification == "UNAVAILABLE":
|
||||
message = f"boundary {left.index}→{right.index} is unavailable"
|
||||
else:
|
||||
message = (
|
||||
f"boundary {left.index}→{right.index}: "
|
||||
+ "; ".join(dict.fromkeys(local))
|
||||
)
|
||||
advisory = _chunk_boundary_advisory(message)
|
||||
if advisory not in basic:
|
||||
basic.append(advisory)
|
||||
|
||||
return DiagnosticSection(tuple(basic), tuple(detailed))
|
||||
|
||||
|
||||
def build_v38_diagnostics(
|
||||
*,
|
||||
video_latents: Any,
|
||||
audio_latents: Any,
|
||||
assembly_plan: Any,
|
||||
output_width: int,
|
||||
output_height: int,
|
||||
chunk_seconds: float,
|
||||
reference_images: Sequence[Any] = (),
|
||||
reference_size: str = "Match Output",
|
||||
video_guide: Any = None,
|
||||
video_guide_size: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
|
||||
still_guide_active: bool = False,
|
||||
core_checker: Callable[[], list[str]] | None = None,
|
||||
) -> V38Diagnostics:
|
||||
return _combine_sections(
|
||||
build_frame_alignment_diagnostics(
|
||||
video_latents=video_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
video_guide=video_guide,
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
),
|
||||
validate_audio_latent_lengths(
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
),
|
||||
build_chunk_boundary_diagnostics(
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
),
|
||||
collect_core_compatibility_advisory(core_checker),
|
||||
estimate_visual_conditioning_load(
|
||||
output_width=int(output_width),
|
||||
output_height=int(output_height),
|
||||
reference_images=tuple(reference_images),
|
||||
reference_size=str(reference_size),
|
||||
video_guide=video_guide,
|
||||
video_guide_size=str(video_guide_size),
|
||||
chunk_seconds=float(chunk_seconds),
|
||||
still_guide_active=bool(still_guide_active),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def format_v38_diagnostics(
|
||||
diagnostics: V38Diagnostics,
|
||||
*,
|
||||
mode: str = DIAGNOSTICS_BASIC,
|
||||
) -> str:
|
||||
normalized = normalize_diagnostics_mode(str(mode))
|
||||
if normalized == DIAGNOSTICS_OFF:
|
||||
return ""
|
||||
lines = diagnostics.detailed if normalized == DIAGNOSTICS_FULL else diagnostics.basic
|
||||
if not lines:
|
||||
return ""
|
||||
return "V3.8 Reliability\n" + "\n".join(lines)
|
||||
|
||||
|
||||
def append_v38_status(
|
||||
status: Any,
|
||||
diagnostics: V38Diagnostics,
|
||||
*,
|
||||
mode: str = DIAGNOSTICS_BASIC,
|
||||
) -> str:
|
||||
suffix = format_v38_diagnostics(diagnostics, mode=mode)
|
||||
original = str(status)
|
||||
return original if not suffix else original.rstrip() + "\n" + suffix
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
"""Shared H3 resolution presets for Easy and V3.8 sampler facades."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
|
||||
H3_ASPECT_AUTO = "Auto from First Image"
|
||||
H3_ASPECT_LANDSCAPE = "Landscape 16:9"
|
||||
H3_ASPECT_PORTRAIT = "Portrait 9:16"
|
||||
H3_ASPECT_SQUARE = "Square 1:1"
|
||||
H3_ASPECT_OPTIONS = (
|
||||
H3_ASPECT_AUTO,
|
||||
H3_ASPECT_LANDSCAPE,
|
||||
H3_ASPECT_PORTRAIT,
|
||||
H3_ASPECT_SQUARE,
|
||||
)
|
||||
|
||||
H3_SIZE_SOURCE_FIRST_IMAGE = "First Image"
|
||||
H3_SIZE_SOURCE_MANUAL = "Manual"
|
||||
H3_SIZE_SOURCE_LEGACY = "Legacy Aspect"
|
||||
H3_SIZE_SOURCE_OPTIONS = (
|
||||
H3_SIZE_SOURCE_FIRST_IMAGE,
|
||||
H3_SIZE_SOURCE_MANUAL,
|
||||
H3_SIZE_SOURCE_LEGACY,
|
||||
)
|
||||
|
||||
H3_PRESET_DRAFT = "Draft — 0.30 MP"
|
||||
H3_PRESET_BALANCED = "Balanced — 0.60 MP"
|
||||
H3_PRESET_NATIVE = "Native 768"
|
||||
H3_PRESET_CUSTOM = "Custom"
|
||||
H3_PRESET_OPTIONS = (
|
||||
H3_PRESET_DRAFT,
|
||||
H3_PRESET_BALANCED,
|
||||
H3_PRESET_NATIVE,
|
||||
H3_PRESET_CUSTOM,
|
||||
)
|
||||
|
||||
H3_DRAFT_MP = 0.30
|
||||
H3_BALANCED_MP = 0.60
|
||||
H3_CUSTOM_MP_DEFAULT = 0.30
|
||||
H3_CUSTOM_MP_MIN = 0.10
|
||||
H3_CUSTOM_MP_MAX = 16.0
|
||||
H3_CUSTOM_MP_STRONG_WARNING = 2.50
|
||||
H3_MP_PIXELS = 1_000_000.0
|
||||
H3_CANVAS_MULTIPLE = 32
|
||||
H3_NATIVE_SHORT_EDGE = 768
|
||||
H3_NATIVE_LONG_EDGE_CAP = 1344
|
||||
H3_MANUAL_WIDTH_DEFAULT = 736
|
||||
H3_MANUAL_HEIGHT_DEFAULT = 416
|
||||
H3_CANVAS_MIN = 32
|
||||
H3_CANVAS_MAX = 16384
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class H3ResolutionPlan:
|
||||
width: int
|
||||
height: int
|
||||
aspect_ratio: float
|
||||
aspect_source: str
|
||||
preset: str
|
||||
target_mp: float | None
|
||||
warnings: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def actual_mp(self) -> float:
|
||||
return float(self.width * self.height) / H3_MP_PIXELS
|
||||
|
||||
|
||||
def _round_canvas(value: float) -> int:
|
||||
units = math.floor(float(value) / H3_CANVAS_MULTIPLE + 0.5)
|
||||
return max(H3_CANVAS_MULTIPLE, int(units) * H3_CANVAS_MULTIPLE)
|
||||
|
||||
|
||||
def _first_image_aspect(first_frame: Any) -> float | None:
|
||||
shape = getattr(first_frame, "shape", None)
|
||||
if shape is None or len(shape) != 4:
|
||||
return None
|
||||
height = int(shape[1])
|
||||
width = int(shape[2])
|
||||
if width < 1 or height < 1:
|
||||
return None
|
||||
return float(width) / float(height)
|
||||
|
||||
|
||||
def _manual_dimension(value: Any, name: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise TypeError(f"Manual {name} must be an integer")
|
||||
resolved = int(value)
|
||||
if float(value) != float(resolved):
|
||||
raise ValueError(f"Manual {name} must be an integer")
|
||||
if not H3_CANVAS_MIN <= resolved <= H3_CANVAS_MAX:
|
||||
raise ValueError(
|
||||
f"Manual {name} must be between {H3_CANVAS_MIN} and {H3_CANVAS_MAX}"
|
||||
)
|
||||
if resolved % H3_CANVAS_MULTIPLE:
|
||||
raise ValueError(
|
||||
f"Manual {name} must be aligned to {H3_CANVAS_MULTIPLE} pixels"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _manual_resolution(width: Any, height: Any, *, warning: str | None = None) -> H3ResolutionPlan:
|
||||
resolved_width = _manual_dimension(width, "Width")
|
||||
resolved_height = _manual_dimension(height, "Height")
|
||||
return H3ResolutionPlan(
|
||||
width=resolved_width,
|
||||
height=resolved_height,
|
||||
aspect_ratio=float(resolved_width) / float(resolved_height),
|
||||
aspect_source="Manual Width / Height",
|
||||
preset="Manual",
|
||||
target_mp=None,
|
||||
warnings=(warning,) if warning else (),
|
||||
)
|
||||
|
||||
|
||||
def resolve_h3_aspect(aspect: str, first_frame: Any = None) -> tuple[float, str]:
|
||||
"""Resolve the existing Easy aspect policy without creating a second fallback."""
|
||||
|
||||
if aspect == H3_ASPECT_AUTO:
|
||||
image_aspect = _first_image_aspect(first_frame)
|
||||
if image_aspect is not None:
|
||||
return image_aspect, "First Image"
|
||||
return 16.0 / 9.0, "Landscape 16:9 fallback"
|
||||
if aspect == H3_ASPECT_LANDSCAPE:
|
||||
return 16.0 / 9.0, H3_ASPECT_LANDSCAPE
|
||||
if aspect == H3_ASPECT_PORTRAIT:
|
||||
return 9.0 / 16.0, H3_ASPECT_PORTRAIT
|
||||
if aspect == H3_ASPECT_SQUARE:
|
||||
return 1.0, H3_ASPECT_SQUARE
|
||||
return 16.0 / 9.0, "Landscape 16:9 fallback"
|
||||
|
||||
|
||||
def _area_resolution(aspect_ratio: float, megapixels: float) -> tuple[int, int]:
|
||||
target_pixels = float(megapixels) * H3_MP_PIXELS
|
||||
raw_width = math.sqrt(target_pixels * float(aspect_ratio))
|
||||
raw_height = math.sqrt(target_pixels / float(aspect_ratio))
|
||||
return _round_canvas(raw_width), _round_canvas(raw_height)
|
||||
|
||||
|
||||
def _native_resolution(aspect_ratio: float) -> tuple[int, int]:
|
||||
landscape = float(aspect_ratio) >= 1.0
|
||||
major_ratio = float(aspect_ratio) if landscape else 1.0 / float(aspect_ratio)
|
||||
short_edge = float(H3_NATIVE_SHORT_EDGE)
|
||||
long_edge = short_edge * major_ratio
|
||||
if long_edge > H3_NATIVE_LONG_EDGE_CAP:
|
||||
long_edge = float(H3_NATIVE_LONG_EDGE_CAP)
|
||||
short_edge = long_edge / major_ratio
|
||||
long_px = min(H3_NATIVE_LONG_EDGE_CAP, _round_canvas(long_edge))
|
||||
short_px = min(H3_NATIVE_SHORT_EDGE, _round_canvas(short_edge))
|
||||
if landscape:
|
||||
return long_px, short_px
|
||||
return short_px, long_px
|
||||
|
||||
|
||||
def resolve_h3_resolution(
|
||||
*,
|
||||
aspect: str,
|
||||
preset: str,
|
||||
custom_mp: float = H3_CUSTOM_MP_DEFAULT,
|
||||
first_frame: Any = None,
|
||||
) -> H3ResolutionPlan:
|
||||
"""Resolve decimal-MP or Native 768 dimensions on the H3 32-pixel grid."""
|
||||
|
||||
aspect_ratio, aspect_source = resolve_h3_aspect(aspect, first_frame)
|
||||
warnings: list[str] = []
|
||||
target_mp: float | None
|
||||
if preset == H3_PRESET_DRAFT:
|
||||
target_mp = H3_DRAFT_MP
|
||||
width, height = _area_resolution(aspect_ratio, target_mp)
|
||||
elif preset == H3_PRESET_BALANCED:
|
||||
target_mp = H3_BALANCED_MP
|
||||
width, height = _area_resolution(aspect_ratio, target_mp)
|
||||
elif preset == H3_PRESET_NATIVE:
|
||||
target_mp = None
|
||||
width, height = _native_resolution(aspect_ratio)
|
||||
else:
|
||||
preset = H3_PRESET_CUSTOM
|
||||
if isinstance(custom_mp, bool):
|
||||
raise TypeError("Custom MP must be numeric")
|
||||
target_mp = float(custom_mp)
|
||||
if not H3_CUSTOM_MP_MIN <= target_mp <= H3_CUSTOM_MP_MAX:
|
||||
raise ValueError(
|
||||
f"Custom MP must be between {H3_CUSTOM_MP_MIN:.2f} and "
|
||||
f"{H3_CUSTOM_MP_MAX:.2f}"
|
||||
)
|
||||
width, height = _area_resolution(aspect_ratio, target_mp)
|
||||
native_width, native_height = _native_resolution(aspect_ratio)
|
||||
native_pixels = native_width * native_height
|
||||
if width * height > native_pixels:
|
||||
warnings.append(
|
||||
"Above Native 768: processing time and memory usage may increase."
|
||||
)
|
||||
if target_mp > H3_CUSTOM_MP_STRONG_WARNING:
|
||||
warnings.append(
|
||||
"High Custom MP: substantial VRAM use and generation time are possible."
|
||||
)
|
||||
return H3ResolutionPlan(
|
||||
width=width,
|
||||
height=height,
|
||||
aspect_ratio=aspect_ratio,
|
||||
aspect_source=aspect_source,
|
||||
preset=preset,
|
||||
target_mp=target_mp,
|
||||
warnings=tuple(warnings),
|
||||
)
|
||||
|
||||
|
||||
def resolve_h3_size_source(
|
||||
*,
|
||||
size_source: str,
|
||||
width: Any = H3_MANUAL_WIDTH_DEFAULT,
|
||||
height: Any = H3_MANUAL_HEIGHT_DEFAULT,
|
||||
aspect: str = H3_ASPECT_AUTO,
|
||||
preset: str = H3_PRESET_DRAFT,
|
||||
custom_mp: float = H3_CUSTOM_MP_DEFAULT,
|
||||
first_frame: Any = None,
|
||||
) -> H3ResolutionPlan:
|
||||
"""Resolve the V3.8 two-choice Size Source while retaining legacy API input."""
|
||||
|
||||
if size_source == H3_SIZE_SOURCE_MANUAL:
|
||||
return _manual_resolution(width, height)
|
||||
if size_source == H3_SIZE_SOURCE_FIRST_IMAGE:
|
||||
if _first_image_aspect(first_frame) is None:
|
||||
return _manual_resolution(
|
||||
width,
|
||||
height,
|
||||
warning=(
|
||||
"First Image is unavailable for Size Source = First Image; "
|
||||
"the displayed Manual Width / Height were used."
|
||||
),
|
||||
)
|
||||
return resolve_h3_resolution(
|
||||
aspect=H3_ASPECT_AUTO,
|
||||
preset=preset,
|
||||
custom_mp=custom_mp,
|
||||
first_frame=first_frame,
|
||||
)
|
||||
# Existing API prompts and saved workflows omit size_source. Preserve their
|
||||
# exact V3.8 aspect resolver until the frontend migrates the visible widgets.
|
||||
return resolve_h3_resolution(
|
||||
aspect=aspect,
|
||||
preset=preset,
|
||||
custom_mp=custom_mp,
|
||||
first_frame=first_frame,
|
||||
)
|
||||
+812
@@ -0,0 +1,812 @@
|
||||
"""Pure execution-policy resolvers for V3.8 chunk review.
|
||||
|
||||
This module deliberately owns no ComfyUI, torch, filesystem, Run Storage, or
|
||||
sampling behavior. It turns already-validated storage facts into an execution
|
||||
decision that later phases can apply to the existing Production engine.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from ..branch_provenance import TAKE_ACTION_CONTINUE, TAKE_ACTION_USE
|
||||
|
||||
|
||||
GENERATION_MODE_FULL_RUN = "Full Run"
|
||||
GENERATION_MODE_REVIEW = "Review Each Chunk"
|
||||
GENERATION_MODE_OPTIONS = (
|
||||
GENERATION_MODE_FULL_RUN,
|
||||
GENERATION_MODE_REVIEW,
|
||||
)
|
||||
|
||||
REVIEW_ACTION_CONTINUE = "Continue / Next"
|
||||
REVIEW_ACTION_REGENERATE_CURRENT = "Regenerate Current"
|
||||
REVIEW_ACTION_FINISH_REMAINING = "Finish Remaining"
|
||||
REVIEW_ACTION_OPTIONS = (
|
||||
REVIEW_ACTION_CONTINUE,
|
||||
REVIEW_ACTION_REGENERATE_CURRENT,
|
||||
REVIEW_ACTION_FINISH_REMAINING,
|
||||
)
|
||||
|
||||
RUN_STORAGE_OFF = "Off"
|
||||
RUN_STORAGE_SAVE_AUTO_RESUME = "Save + Auto Resume"
|
||||
|
||||
REVISION_STATUS_IN_PROGRESS = "in_progress"
|
||||
REVISION_STATUS_INTERRUPTED = "interrupted"
|
||||
REVISION_STATUS_REVIEW_READY = "review_ready"
|
||||
REVISION_STATUS_COMPLETE = "complete"
|
||||
REVISION_STATUSES = (
|
||||
REVISION_STATUS_IN_PROGRESS,
|
||||
REVISION_STATUS_INTERRUPTED,
|
||||
REVISION_STATUS_REVIEW_READY,
|
||||
REVISION_STATUS_COMPLETE,
|
||||
)
|
||||
|
||||
EXECUTION_MODE_FULL_RUN = "full_run"
|
||||
EXECUTION_MODE_REVIEW_CONTINUE = "review_continue"
|
||||
EXECUTION_MODE_REVIEW_REGENERATE = "review_regenerate_current"
|
||||
EXECUTION_MODE_REVIEW_FINISH = "review_finish_remaining"
|
||||
EXECUTION_MODE_REVIEW_USE_TAKE = "review_use_take"
|
||||
EXECUTION_MODE_REVIEW_CONTINUE_FROM_TAKE = "review_continue_from_take"
|
||||
|
||||
NONCE_POLICY_INACTIVE = "inactive"
|
||||
NONCE_POLICY_INHERIT = "inherit_branch"
|
||||
NONCE_POLICY_ADVANCE = "advance_variation"
|
||||
NONCE_POLICY_MANUAL = "manual"
|
||||
|
||||
REVIEW_PAUSE_REASON = "review_each_chunk"
|
||||
|
||||
|
||||
class ReviewControlError(ValueError):
|
||||
"""Raised when a requested review policy is structurally ambiguous."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewUnit:
|
||||
"""One atomic physical sampling group expressed in logical chunk numbers."""
|
||||
|
||||
start: int
|
||||
end: int
|
||||
physical_group: int
|
||||
|
||||
def as_metadata(self) -> dict[str, int]:
|
||||
return {
|
||||
"start": self.start,
|
||||
"end": self.end,
|
||||
"physical_group": self.physical_group,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewExecution:
|
||||
"""Side-effect-free decision consumed by later execution phases."""
|
||||
|
||||
execution_mode: str
|
||||
requires_run_storage: bool
|
||||
effective_regenerate_from: int
|
||||
max_new_physical_groups: int | None
|
||||
next_review_unit_start: int | None
|
||||
next_review_unit_end: int | None
|
||||
next_review_physical_group: int | None
|
||||
finish_remaining: bool
|
||||
smart_regenerate: bool
|
||||
one_shot_reset_required: bool
|
||||
partial_review: bool
|
||||
projected_prefix_count: int
|
||||
nonce_policy: str
|
||||
branch_regenerate_from: int
|
||||
base_effective_nonce: int
|
||||
requested_effective_nonce: int | None
|
||||
status_hint: str
|
||||
|
||||
|
||||
def _integer(value: Any, *, name: str, minimum: int, maximum: int) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise ReviewControlError(f"{name} must be an integer")
|
||||
if not minimum <= value <= maximum:
|
||||
raise ReviewControlError(
|
||||
f"{name} must be between {minimum} and {maximum}"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _manual_regenerate_from(value: Any, *, configured_chunks: int) -> int:
|
||||
if value is None or value == "Auto" or value == 0:
|
||||
return 0
|
||||
if isinstance(value, str) and value.startswith("Chunk "):
|
||||
try:
|
||||
value = int(value[6:])
|
||||
except ValueError as exc:
|
||||
raise ReviewControlError(
|
||||
"manual_regenerate_from must be Auto or a valid chunk"
|
||||
) from exc
|
||||
return _integer(
|
||||
value,
|
||||
name="manual_regenerate_from",
|
||||
minimum=1,
|
||||
maximum=configured_chunks,
|
||||
)
|
||||
|
||||
|
||||
def _terminal_pair_start(
|
||||
*,
|
||||
configured_chunks: int,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: Any,
|
||||
) -> int | None:
|
||||
if type(terminal_merge_enabled) is not bool:
|
||||
raise ReviewControlError("terminal_merge_enabled must be a boolean")
|
||||
if not terminal_merge_enabled:
|
||||
return None
|
||||
if configured_chunks < 2:
|
||||
raise ReviewControlError(
|
||||
"Terminal Merge requires at least two configured chunks"
|
||||
)
|
||||
start = _integer(
|
||||
terminal_pair_start,
|
||||
name="terminal_pair_start",
|
||||
minimum=1,
|
||||
maximum=configured_chunks - 1,
|
||||
)
|
||||
if start != configured_chunks - 1:
|
||||
raise ReviewControlError(
|
||||
"Terminal Merge must cover the final two configured chunks"
|
||||
)
|
||||
return start
|
||||
|
||||
|
||||
def _review_geometry(
|
||||
*,
|
||||
configured_chunks: Any,
|
||||
validated_prefix_count: Any,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: Any,
|
||||
) -> tuple[int, int, int | None]:
|
||||
chunks = _integer(
|
||||
configured_chunks,
|
||||
name="configured_chunks",
|
||||
minimum=1,
|
||||
maximum=0x7FFFFFFF,
|
||||
)
|
||||
prefix = _integer(
|
||||
validated_prefix_count,
|
||||
name="validated_prefix_count",
|
||||
minimum=0,
|
||||
maximum=chunks,
|
||||
)
|
||||
pair_start = _terminal_pair_start(
|
||||
configured_chunks=chunks,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=terminal_pair_start,
|
||||
)
|
||||
if pair_start is not None and pair_start <= prefix < chunks:
|
||||
raise ReviewControlError(
|
||||
"validated prefix ends inside the atomic Terminal Merge pair"
|
||||
)
|
||||
return chunks, prefix, pair_start
|
||||
|
||||
|
||||
def resolve_next_review_unit(
|
||||
*,
|
||||
configured_chunks: int,
|
||||
validated_prefix_count: int,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: int | None,
|
||||
) -> ReviewUnit | None:
|
||||
"""Return the next atomic physical group after a validated logical prefix."""
|
||||
|
||||
chunks, prefix, pair_start = _review_geometry(
|
||||
configured_chunks=configured_chunks,
|
||||
validated_prefix_count=validated_prefix_count,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=terminal_pair_start,
|
||||
)
|
||||
if prefix == chunks:
|
||||
return None
|
||||
start = prefix + 1
|
||||
if pair_start is not None and start == pair_start:
|
||||
return ReviewUnit(
|
||||
start=pair_start,
|
||||
end=chunks,
|
||||
physical_group=pair_start,
|
||||
)
|
||||
return ReviewUnit(start=start, end=start, physical_group=start)
|
||||
|
||||
|
||||
def _review_unit_for_chunk(
|
||||
chunk: int,
|
||||
*,
|
||||
configured_chunks: int,
|
||||
terminal_pair_start: int | None,
|
||||
) -> ReviewUnit:
|
||||
if terminal_pair_start is not None and chunk >= terminal_pair_start:
|
||||
return ReviewUnit(
|
||||
start=terminal_pair_start,
|
||||
end=configured_chunks,
|
||||
physical_group=terminal_pair_start,
|
||||
)
|
||||
return ReviewUnit(start=chunk, end=chunk, physical_group=chunk)
|
||||
|
||||
|
||||
def _validated_review_unit(
|
||||
value: Any,
|
||||
*,
|
||||
configured_chunks: int,
|
||||
validated_prefix_count: int,
|
||||
terminal_pair_start: int | None,
|
||||
) -> ReviewUnit | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, ReviewUnit):
|
||||
unit = value
|
||||
elif isinstance(value, Mapping):
|
||||
try:
|
||||
unit = ReviewUnit(
|
||||
start=value["start"],
|
||||
end=value["end"],
|
||||
physical_group=value["physical_group"],
|
||||
)
|
||||
except KeyError as exc:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit is missing required metadata"
|
||||
) from exc
|
||||
else:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit must be review metadata or None"
|
||||
)
|
||||
start = _integer(
|
||||
unit.start,
|
||||
name="latest_review_unit.start",
|
||||
minimum=1,
|
||||
maximum=configured_chunks,
|
||||
)
|
||||
end = _integer(
|
||||
unit.end,
|
||||
name="latest_review_unit.end",
|
||||
minimum=start,
|
||||
maximum=configured_chunks,
|
||||
)
|
||||
physical_group = _integer(
|
||||
unit.physical_group,
|
||||
name="latest_review_unit.physical_group",
|
||||
minimum=1,
|
||||
maximum=configured_chunks,
|
||||
)
|
||||
if physical_group != start:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit physical group must equal its first logical chunk"
|
||||
)
|
||||
if terminal_pair_start is not None and start == terminal_pair_start:
|
||||
if end != configured_chunks:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit splits the atomic Terminal Merge pair"
|
||||
)
|
||||
elif end != start:
|
||||
raise ReviewControlError(
|
||||
"a non-terminal review unit must contain exactly one logical chunk"
|
||||
)
|
||||
if terminal_pair_start is not None and start > terminal_pair_start:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit starts inside the atomic Terminal Merge pair"
|
||||
)
|
||||
if end != validated_prefix_count:
|
||||
raise ReviewControlError(
|
||||
"latest_review_unit is not the head of the validated prefix"
|
||||
)
|
||||
return ReviewUnit(start=start, end=end, physical_group=physical_group)
|
||||
|
||||
|
||||
def make_review_pause_metadata(review_unit: ReviewUnit) -> dict[str, Any]:
|
||||
"""Build non-identity manifest metadata for a successful review pause."""
|
||||
|
||||
if not isinstance(review_unit, ReviewUnit):
|
||||
raise ReviewControlError("review_unit must be a ReviewUnit")
|
||||
start = _integer(
|
||||
review_unit.start,
|
||||
name="review_unit.start",
|
||||
minimum=1,
|
||||
maximum=0x7FFFFFFF,
|
||||
)
|
||||
end = _integer(
|
||||
review_unit.end,
|
||||
name="review_unit.end",
|
||||
minimum=start,
|
||||
maximum=0x7FFFFFFF,
|
||||
)
|
||||
physical_group = _integer(
|
||||
review_unit.physical_group,
|
||||
name="review_unit.physical_group",
|
||||
minimum=1,
|
||||
maximum=0x7FFFFFFF,
|
||||
)
|
||||
if physical_group != start:
|
||||
raise ReviewControlError(
|
||||
"review_unit physical group must equal its first logical chunk"
|
||||
)
|
||||
return {
|
||||
"review_unit": {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"physical_group": physical_group,
|
||||
},
|
||||
"review_pause_reason": REVIEW_PAUSE_REASON,
|
||||
}
|
||||
|
||||
|
||||
def _revision_status(value: Any, *, prefix: int, chunks: int) -> str | None:
|
||||
if value is None or value == "":
|
||||
if prefix:
|
||||
raise ReviewControlError(
|
||||
"a non-empty validated prefix requires a revision status"
|
||||
)
|
||||
return None
|
||||
status = str(value)
|
||||
if status not in REVISION_STATUSES:
|
||||
raise ReviewControlError(f"unknown latest revision status: {status!r}")
|
||||
if status == REVISION_STATUS_COMPLETE and prefix != chunks:
|
||||
raise ReviewControlError(
|
||||
"a complete revision must contain every configured chunk"
|
||||
)
|
||||
if status == REVISION_STATUS_REVIEW_READY and not 0 < prefix < chunks:
|
||||
raise ReviewControlError(
|
||||
"review_ready requires a non-empty incomplete prefix"
|
||||
)
|
||||
return status
|
||||
|
||||
|
||||
def _branch_contract(
|
||||
*,
|
||||
latest_effective_nonce: Any,
|
||||
latest_branch_regenerate_from: Any,
|
||||
validated_prefix_count: int,
|
||||
configured_chunks: int,
|
||||
latest_revision_status: str | None,
|
||||
) -> tuple[int, int]:
|
||||
nonce = _integer(
|
||||
latest_effective_nonce,
|
||||
name="latest_effective_nonce",
|
||||
minimum=0,
|
||||
maximum=0xFFFFFFFF,
|
||||
)
|
||||
branch_start = _manual_regenerate_from(
|
||||
latest_branch_regenerate_from,
|
||||
configured_chunks=configured_chunks,
|
||||
)
|
||||
if nonce == 0 and branch_start != 0:
|
||||
raise ReviewControlError(
|
||||
"a regenerate branch requires a positive effective nonce"
|
||||
)
|
||||
if nonce > 0 and branch_start == 0:
|
||||
raise ReviewControlError(
|
||||
"a positive effective nonce requires its regenerate branch boundary"
|
||||
)
|
||||
if branch_start > validated_prefix_count + 1:
|
||||
raise ReviewControlError(
|
||||
"regenerate branch boundary skips the validated prefix head"
|
||||
)
|
||||
if (
|
||||
latest_revision_status
|
||||
in (REVISION_STATUS_REVIEW_READY, REVISION_STATUS_COMPLETE)
|
||||
and branch_start > validated_prefix_count
|
||||
):
|
||||
raise ReviewControlError(
|
||||
"a ready regenerate branch has not committed its boundary chunk"
|
||||
)
|
||||
return nonce, branch_start
|
||||
|
||||
|
||||
def validate_review_prefix_metadata(
|
||||
*,
|
||||
configured_chunks: int,
|
||||
validated_prefix_count: int,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: int | None,
|
||||
latest_review_unit: ReviewUnit | Mapping[str, Any] | None,
|
||||
latest_revision_status: str | None,
|
||||
latest_effective_nonce: int,
|
||||
latest_branch_regenerate_from: Any = 0,
|
||||
) -> ReviewUnit | None:
|
||||
"""Validate persisted Review metadata without creating an execution decision.
|
||||
|
||||
Repository reads and finalization use this boundary after raw prefix
|
||||
validation. It deliberately returns no ``ReviewExecution`` so those paths
|
||||
cannot become a second policy owner.
|
||||
"""
|
||||
|
||||
chunks, prefix, pair_start = _review_geometry(
|
||||
configured_chunks=configured_chunks,
|
||||
validated_prefix_count=validated_prefix_count,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=terminal_pair_start,
|
||||
)
|
||||
status = _revision_status(
|
||||
latest_revision_status,
|
||||
prefix=prefix,
|
||||
chunks=chunks,
|
||||
)
|
||||
unit = _validated_review_unit(
|
||||
latest_review_unit,
|
||||
configured_chunks=chunks,
|
||||
validated_prefix_count=prefix,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
_branch_contract(
|
||||
latest_effective_nonce=latest_effective_nonce,
|
||||
latest_branch_regenerate_from=latest_branch_regenerate_from,
|
||||
validated_prefix_count=prefix,
|
||||
configured_chunks=chunks,
|
||||
latest_revision_status=status,
|
||||
)
|
||||
return unit
|
||||
|
||||
|
||||
def _nonce_decision(
|
||||
*,
|
||||
manual_regenerate_from: int,
|
||||
smart_regenerate: bool,
|
||||
latest_effective_nonce: int,
|
||||
latest_branch_regenerate_from: int,
|
||||
smart_target: int = 0,
|
||||
) -> tuple[str, int, int | None]:
|
||||
if smart_regenerate:
|
||||
if latest_effective_nonce >= 0xFFFFFFFF:
|
||||
raise ReviewControlError(
|
||||
"effective reroll nonce variation space is exhausted"
|
||||
)
|
||||
return (
|
||||
NONCE_POLICY_ADVANCE,
|
||||
smart_target,
|
||||
latest_effective_nonce + 1,
|
||||
)
|
||||
if manual_regenerate_from:
|
||||
return NONCE_POLICY_MANUAL, manual_regenerate_from, None
|
||||
if latest_effective_nonce:
|
||||
return (
|
||||
NONCE_POLICY_INHERIT,
|
||||
latest_branch_regenerate_from,
|
||||
latest_effective_nonce,
|
||||
)
|
||||
return NONCE_POLICY_INACTIVE, 0, 0
|
||||
|
||||
|
||||
def _unit_status(unit: ReviewUnit, *, chunks: int, verb: str) -> str:
|
||||
if unit.start == unit.end:
|
||||
return f"{verb}: Chunk {unit.start} / {chunks}."
|
||||
return (
|
||||
f"{verb}: Chunks {unit.start}-{unit.end} / {chunks}; "
|
||||
"Terminal Merge atomic group."
|
||||
)
|
||||
|
||||
|
||||
def resolve_review_execution(
|
||||
*,
|
||||
generation_mode: str,
|
||||
review_action: str,
|
||||
configured_chunks: int,
|
||||
validated_prefix_count: int,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: int | None,
|
||||
manual_regenerate_from: Any,
|
||||
run_storage_mode: str,
|
||||
latest_review_unit: ReviewUnit | Mapping[str, Any] | None,
|
||||
latest_revision_status: str | None,
|
||||
latest_effective_nonce: int,
|
||||
latest_branch_regenerate_from: Any = 0,
|
||||
) -> ReviewExecution:
|
||||
"""Resolve V3.8 review intent without touching Production or storage state."""
|
||||
|
||||
if generation_mode not in GENERATION_MODE_OPTIONS:
|
||||
raise ReviewControlError(f"unknown Generation Mode: {generation_mode!r}")
|
||||
if review_action not in REVIEW_ACTION_OPTIONS:
|
||||
raise ReviewControlError(f"unknown Review Action: {review_action!r}")
|
||||
chunks, prefix, pair_start = _review_geometry(
|
||||
configured_chunks=configured_chunks,
|
||||
validated_prefix_count=validated_prefix_count,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=terminal_pair_start,
|
||||
)
|
||||
status = _revision_status(
|
||||
latest_revision_status,
|
||||
prefix=prefix,
|
||||
chunks=chunks,
|
||||
)
|
||||
manual_from = _manual_regenerate_from(
|
||||
manual_regenerate_from,
|
||||
configured_chunks=chunks,
|
||||
)
|
||||
nonce, existing_branch_start = _branch_contract(
|
||||
latest_effective_nonce=latest_effective_nonce,
|
||||
latest_branch_regenerate_from=latest_branch_regenerate_from,
|
||||
validated_prefix_count=prefix,
|
||||
configured_chunks=chunks,
|
||||
latest_revision_status=status,
|
||||
)
|
||||
|
||||
if generation_mode == GENERATION_MODE_FULL_RUN:
|
||||
nonce_policy, branch_start, requested_nonce = _nonce_decision(
|
||||
manual_regenerate_from=manual_from,
|
||||
smart_regenerate=False,
|
||||
latest_effective_nonce=nonce,
|
||||
latest_branch_regenerate_from=existing_branch_start,
|
||||
)
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_FULL_RUN,
|
||||
requires_run_storage=False,
|
||||
effective_regenerate_from=(manual_from or branch_start),
|
||||
max_new_physical_groups=None,
|
||||
next_review_unit_start=None,
|
||||
next_review_unit_end=None,
|
||||
next_review_physical_group=None,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=False,
|
||||
partial_review=False,
|
||||
projected_prefix_count=chunks,
|
||||
nonce_policy=nonce_policy,
|
||||
branch_regenerate_from=branch_start,
|
||||
base_effective_nonce=nonce,
|
||||
requested_effective_nonce=requested_nonce,
|
||||
status_hint="Full Run: Review execution limit disabled.",
|
||||
)
|
||||
|
||||
if run_storage_mode != RUN_STORAGE_SAVE_AUTO_RESUME:
|
||||
raise ReviewControlError(
|
||||
"Review Each Chunk requires Run Storage = Save + Auto Resume."
|
||||
)
|
||||
|
||||
latest_unit = _validated_review_unit(
|
||||
latest_review_unit,
|
||||
configured_chunks=chunks,
|
||||
validated_prefix_count=prefix,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
|
||||
if review_action == REVIEW_ACTION_REGENERATE_CURRENT:
|
||||
if manual_from:
|
||||
raise ReviewControlError(
|
||||
"Regenerate Current cannot be combined with manual Regenerate From. "
|
||||
"Set Regenerate From to Auto."
|
||||
)
|
||||
if prefix == 0:
|
||||
raise ReviewControlError(
|
||||
"Regenerate Current requires an existing reviewed unit"
|
||||
)
|
||||
if latest_unit is None:
|
||||
raise ReviewControlError(
|
||||
"Regenerate Current requires latest review_unit metadata"
|
||||
)
|
||||
if status not in (
|
||||
REVISION_STATUS_REVIEW_READY,
|
||||
REVISION_STATUS_COMPLETE,
|
||||
):
|
||||
raise ReviewControlError(
|
||||
"Regenerate Current requires a review_ready revision or a complete "
|
||||
"revision with review_unit metadata"
|
||||
)
|
||||
nonce_policy, branch_start, requested_nonce = _nonce_decision(
|
||||
manual_regenerate_from=0,
|
||||
smart_regenerate=True,
|
||||
latest_effective_nonce=nonce,
|
||||
latest_branch_regenerate_from=existing_branch_start,
|
||||
smart_target=latest_unit.start,
|
||||
)
|
||||
projected = latest_unit.end
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_REGENERATE,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=latest_unit.start,
|
||||
max_new_physical_groups=1,
|
||||
next_review_unit_start=latest_unit.start,
|
||||
next_review_unit_end=latest_unit.end,
|
||||
next_review_physical_group=latest_unit.physical_group,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=True,
|
||||
one_shot_reset_required=True,
|
||||
partial_review=projected < chunks,
|
||||
projected_prefix_count=projected,
|
||||
nonce_policy=nonce_policy,
|
||||
branch_regenerate_from=branch_start,
|
||||
base_effective_nonce=nonce,
|
||||
requested_effective_nonce=requested_nonce,
|
||||
status_hint=_unit_status(
|
||||
latest_unit,
|
||||
chunks=chunks,
|
||||
verb="Smart Regenerate",
|
||||
),
|
||||
)
|
||||
|
||||
nonce_policy, branch_start, requested_nonce = _nonce_decision(
|
||||
manual_regenerate_from=manual_from,
|
||||
smart_regenerate=False,
|
||||
latest_effective_nonce=nonce,
|
||||
latest_branch_regenerate_from=existing_branch_start,
|
||||
)
|
||||
if review_action == REVIEW_ACTION_FINISH_REMAINING:
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_FINISH,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=(manual_from or branch_start),
|
||||
max_new_physical_groups=None,
|
||||
next_review_unit_start=None,
|
||||
next_review_unit_end=None,
|
||||
next_review_physical_group=None,
|
||||
finish_remaining=True,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=True,
|
||||
partial_review=False,
|
||||
projected_prefix_count=chunks,
|
||||
nonce_policy=nonce_policy,
|
||||
branch_regenerate_from=branch_start,
|
||||
base_effective_nonce=nonce,
|
||||
requested_effective_nonce=requested_nonce,
|
||||
status_hint=(
|
||||
f"Review Mode: finish remaining after accepted prefix "
|
||||
f"{prefix} / {chunks}."
|
||||
),
|
||||
)
|
||||
|
||||
if manual_from:
|
||||
next_unit = _review_unit_for_chunk(
|
||||
manual_from,
|
||||
configured_chunks=chunks,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
else:
|
||||
next_unit = resolve_next_review_unit(
|
||||
configured_chunks=chunks,
|
||||
validated_prefix_count=prefix,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
if next_unit is None:
|
||||
projected = chunks
|
||||
status_hint = "Review Mode: Sequence complete."
|
||||
unit_start = unit_end = physical_group = None
|
||||
else:
|
||||
projected = next_unit.end
|
||||
status_hint = _unit_status(
|
||||
next_unit,
|
||||
chunks=chunks,
|
||||
verb="Review next unit",
|
||||
)
|
||||
unit_start = next_unit.start
|
||||
unit_end = next_unit.end
|
||||
physical_group = next_unit.physical_group
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_CONTINUE,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=(manual_from or branch_start),
|
||||
max_new_physical_groups=1,
|
||||
next_review_unit_start=unit_start,
|
||||
next_review_unit_end=unit_end,
|
||||
next_review_physical_group=physical_group,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=False,
|
||||
partial_review=projected < chunks,
|
||||
projected_prefix_count=projected,
|
||||
nonce_policy=nonce_policy,
|
||||
branch_regenerate_from=branch_start,
|
||||
base_effective_nonce=nonce,
|
||||
requested_effective_nonce=requested_nonce,
|
||||
status_hint=status_hint,
|
||||
)
|
||||
|
||||
|
||||
def resolve_take_execution(
|
||||
*,
|
||||
take_action: str,
|
||||
configured_chunks: int,
|
||||
selected_prefix_count: int,
|
||||
terminal_merge_enabled: bool,
|
||||
terminal_pair_start: int | None,
|
||||
selected_review_unit: ReviewUnit | Mapping[str, Any],
|
||||
selected_effective_nonce: int,
|
||||
selected_branch_regenerate_from: int,
|
||||
next_effective_nonce: int,
|
||||
) -> ReviewExecution:
|
||||
"""Resolve an already-validated immutable Take selection.
|
||||
|
||||
The selected prefix is a Run Storage fact. Continuing from it starts a
|
||||
fresh downstream nonce boundary without changing the Sampling Contract.
|
||||
"""
|
||||
|
||||
if take_action not in (TAKE_ACTION_USE, TAKE_ACTION_CONTINUE):
|
||||
raise ReviewControlError(f"unknown Take action: {take_action!r}")
|
||||
chunks, prefix, pair_start = _review_geometry(
|
||||
configured_chunks=configured_chunks,
|
||||
validated_prefix_count=selected_prefix_count,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=terminal_pair_start,
|
||||
)
|
||||
unit = _validated_review_unit(
|
||||
selected_review_unit,
|
||||
configured_chunks=chunks,
|
||||
validated_prefix_count=prefix,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
if unit is None:
|
||||
raise ReviewControlError("selected Take is missing its physical review unit")
|
||||
selected_nonce, selected_branch = _branch_contract(
|
||||
latest_effective_nonce=selected_effective_nonce,
|
||||
latest_branch_regenerate_from=selected_branch_regenerate_from,
|
||||
validated_prefix_count=prefix,
|
||||
configured_chunks=chunks,
|
||||
latest_revision_status=(
|
||||
REVISION_STATUS_COMPLETE if prefix == chunks else REVISION_STATUS_REVIEW_READY
|
||||
),
|
||||
)
|
||||
if take_action == TAKE_ACTION_USE:
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_USE_TAKE,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=selected_branch,
|
||||
max_new_physical_groups=0,
|
||||
next_review_unit_start=None,
|
||||
next_review_unit_end=None,
|
||||
next_review_physical_group=None,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=True,
|
||||
partial_review=prefix < chunks,
|
||||
projected_prefix_count=prefix,
|
||||
nonce_policy=NONCE_POLICY_INHERIT if selected_nonce else NONCE_POLICY_INACTIVE,
|
||||
branch_regenerate_from=selected_branch,
|
||||
base_effective_nonce=selected_nonce,
|
||||
requested_effective_nonce=selected_nonce,
|
||||
status_hint=_unit_status(unit, chunks=chunks, verb="Use This Take"),
|
||||
)
|
||||
|
||||
next_unit = resolve_next_review_unit(
|
||||
configured_chunks=chunks,
|
||||
validated_prefix_count=prefix,
|
||||
terminal_merge_enabled=terminal_merge_enabled,
|
||||
terminal_pair_start=pair_start,
|
||||
)
|
||||
if next_unit is None:
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_USE_TAKE,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=selected_branch,
|
||||
max_new_physical_groups=0,
|
||||
next_review_unit_start=None,
|
||||
next_review_unit_end=None,
|
||||
next_review_physical_group=None,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=True,
|
||||
partial_review=False,
|
||||
projected_prefix_count=chunks,
|
||||
nonce_policy=NONCE_POLICY_INHERIT if selected_nonce else NONCE_POLICY_INACTIVE,
|
||||
branch_regenerate_from=selected_branch,
|
||||
base_effective_nonce=selected_nonce,
|
||||
requested_effective_nonce=selected_nonce,
|
||||
status_hint="Continue From Here: selected Take is already complete.",
|
||||
)
|
||||
nonce = _integer(
|
||||
next_effective_nonce,
|
||||
name="next_effective_nonce",
|
||||
minimum=1,
|
||||
maximum=0xFFFFFFFF,
|
||||
)
|
||||
return ReviewExecution(
|
||||
execution_mode=EXECUTION_MODE_REVIEW_CONTINUE_FROM_TAKE,
|
||||
requires_run_storage=True,
|
||||
effective_regenerate_from=next_unit.start,
|
||||
max_new_physical_groups=1,
|
||||
next_review_unit_start=next_unit.start,
|
||||
next_review_unit_end=next_unit.end,
|
||||
next_review_physical_group=next_unit.physical_group,
|
||||
finish_remaining=False,
|
||||
smart_regenerate=False,
|
||||
one_shot_reset_required=True,
|
||||
partial_review=next_unit.end < chunks,
|
||||
projected_prefix_count=next_unit.end,
|
||||
nonce_policy=NONCE_POLICY_ADVANCE,
|
||||
branch_regenerate_from=next_unit.start,
|
||||
base_effective_nonce=selected_nonce,
|
||||
requested_effective_nonce=nonce,
|
||||
status_hint=_unit_status(next_unit, chunks=chunks, verb="Continue From Here"),
|
||||
)
|
||||
+223
@@ -0,0 +1,223 @@
|
||||
"""Internal owner of Continuum runtime continuation-source priority.
|
||||
|
||||
The pure execution planner receives only the source selected here. Runtime
|
||||
payload validation remains outside the planner so Run Storage, explicit
|
||||
Session, and initial State can never become competing planner inputs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from ..branch_provenance import physical_groups
|
||||
from ..run_storage import RunStorageError
|
||||
from ..state import validate_state
|
||||
from ..v2.session import SessionValidationError, entry_to_state
|
||||
from .planning_types import ContinuationSourceFacts
|
||||
|
||||
|
||||
LOG = logging.getLogger("h3_continuum_join")
|
||||
|
||||
|
||||
class RuntimeCoordinatorError(ValueError):
|
||||
"""Raised when the characterized runtime source contract is invalid."""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeContinuationSelection:
|
||||
"""Selected runtime payload plus its single planning projection."""
|
||||
|
||||
entries: list[dict[str, Any]]
|
||||
previous_state: dict[str, Any] | None
|
||||
continuation_source_facts: ContinuationSourceFacts
|
||||
notes: tuple[str, ...]
|
||||
|
||||
|
||||
def project_selected_continuation_source(
|
||||
*,
|
||||
kind: str,
|
||||
selected_source: dict[str, Any] | None,
|
||||
) -> ContinuationSourceFacts:
|
||||
"""Project one already-selected runtime source into planning-only facts."""
|
||||
|
||||
normalized_kind = str(kind)
|
||||
if normalized_kind == "none":
|
||||
if selected_source is not None:
|
||||
raise RuntimeCoordinatorError("the none continuation source has no payload")
|
||||
return ContinuationSourceFacts(
|
||||
kind="none",
|
||||
accepted_chunks=0,
|
||||
reroll_from_chunk=0,
|
||||
physical_group_boundaries=(),
|
||||
)
|
||||
if normalized_kind not in {
|
||||
"run_storage",
|
||||
"explicit_session",
|
||||
"initial_state",
|
||||
}:
|
||||
raise RuntimeCoordinatorError(
|
||||
f"unknown selected continuation source: {normalized_kind!r}"
|
||||
)
|
||||
if not isinstance(selected_source, dict):
|
||||
raise RuntimeCoordinatorError(
|
||||
"the selected continuation source requires normalized runtime facts"
|
||||
)
|
||||
try:
|
||||
accepted_chunks = int(selected_source["accepted_chunks"])
|
||||
reroll_from_chunk = int(selected_source["reroll_from_chunk"])
|
||||
boundaries = tuple(
|
||||
(int(start), int(end))
|
||||
for start, end in selected_source["physical_group_boundaries"]
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise RuntimeCoordinatorError(
|
||||
"selected continuation source facts are invalid"
|
||||
) from exc
|
||||
return ContinuationSourceFacts(
|
||||
kind=normalized_kind,
|
||||
accepted_chunks=accepted_chunks,
|
||||
reroll_from_chunk=reroll_from_chunk,
|
||||
physical_group_boundaries=boundaries,
|
||||
)
|
||||
|
||||
|
||||
class InternalRuntimeCoordinator:
|
||||
"""Own the established Storage / Session / State priority contract."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
storage_controller: Any,
|
||||
input_session: dict[str, Any] | None,
|
||||
initial_state: dict[str, Any] | None,
|
||||
) -> None:
|
||||
self.storage_controller = storage_controller
|
||||
self.input_session = input_session
|
||||
self.initial_state = initial_state
|
||||
self.session_present = input_session is not None
|
||||
|
||||
@property
|
||||
def storage_enabled(self) -> bool:
|
||||
return self.storage_controller is not None
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
||||
"""Suppress State as soon as an explicit Session is present."""
|
||||
|
||||
if self.session_present and self.initial_state is not None:
|
||||
LOG.warning(
|
||||
"Both session and initial_state were supplied; using the session "
|
||||
"and ignoring initial_state"
|
||||
)
|
||||
self.initial_state = None
|
||||
return self.input_session, self.initial_state
|
||||
|
||||
def assert_storage_session_compatible(self) -> None:
|
||||
"""Preserve the existing Run Storage plus explicit Session hard error."""
|
||||
|
||||
if self.storage_enabled and self.session_present:
|
||||
raise RunStorageError(
|
||||
"Run Storage cannot be combined with an explicit Session"
|
||||
)
|
||||
|
||||
def validate_initial_state_reroll(self, reroll_from_chunk: int) -> None:
|
||||
"""Apply the existing reroll restriction to the effective State only."""
|
||||
|
||||
if self.initial_state is not None and int(reroll_from_chunk) not in (0, 1):
|
||||
raise RuntimeCoordinatorError(
|
||||
"with initial_state, reroll_from_chunk can only be 0 or 1"
|
||||
)
|
||||
|
||||
def select_continuation_source(
|
||||
self,
|
||||
*,
|
||||
preserved: list[dict[str, Any]],
|
||||
initial_state: dict[str, Any] | None,
|
||||
chunks: int,
|
||||
width: int,
|
||||
height: int,
|
||||
reroll_from_chunk: int,
|
||||
effective_reroll_from_chunk: int,
|
||||
terminal_merge_enabled: bool,
|
||||
) -> RuntimeContinuationSelection:
|
||||
"""Select exactly one runtime continuation source, then project it."""
|
||||
|
||||
entries = preserved[:]
|
||||
previous_state = None
|
||||
notes: list[str] = []
|
||||
|
||||
if entries:
|
||||
try:
|
||||
previous_state = entry_to_state(entries[-1])
|
||||
except (SessionValidationError, ValueError) as exc:
|
||||
notes.append(
|
||||
"saved continuation state was rejected; generated a fresh run "
|
||||
f"({exc})"
|
||||
)
|
||||
entries = []
|
||||
previous_state = None
|
||||
|
||||
# An initial State is considered only if no usable Storage/Session prefix
|
||||
# survived. Explicit Session presence already suppressed it in
|
||||
# prepare_inputs(), before Session validity was known.
|
||||
if not entries and initial_state is not None:
|
||||
try:
|
||||
candidate = validate_state(initial_state)
|
||||
if int(candidate["width"]) != int(width) or int(
|
||||
candidate["height"]
|
||||
) != int(height):
|
||||
notes.append(
|
||||
"initial_state resolution differs; generated a fresh run"
|
||||
)
|
||||
else:
|
||||
previous_state = candidate
|
||||
except ValueError as exc:
|
||||
notes.append(
|
||||
f"initial_state was rejected; generated a fresh run ({exc})"
|
||||
)
|
||||
|
||||
if entries:
|
||||
selected_kind = (
|
||||
"run_storage" if self.storage_enabled else "explicit_session"
|
||||
)
|
||||
source_reroll = (
|
||||
int(reroll_from_chunk)
|
||||
if self.storage_enabled
|
||||
else int(effective_reroll_from_chunk)
|
||||
)
|
||||
accepted_boundaries = tuple(
|
||||
(int(group.start), int(group.end))
|
||||
for group in physical_groups(
|
||||
chunks=int(chunks),
|
||||
terminal_merge_enabled=bool(terminal_merge_enabled),
|
||||
)
|
||||
if int(group.end) <= len(entries)
|
||||
)
|
||||
selected_source = {
|
||||
"accepted_chunks": len(entries),
|
||||
"reroll_from_chunk": source_reroll,
|
||||
"physical_group_boundaries": accepted_boundaries,
|
||||
}
|
||||
elif previous_state is not None and initial_state is not None:
|
||||
selected_kind = "initial_state"
|
||||
selected_source = {
|
||||
"accepted_chunks": 0,
|
||||
"reroll_from_chunk": int(reroll_from_chunk),
|
||||
"physical_group_boundaries": (),
|
||||
}
|
||||
else:
|
||||
selected_kind = "none"
|
||||
selected_source = None
|
||||
|
||||
return RuntimeContinuationSelection(
|
||||
entries=entries,
|
||||
previous_state=previous_state,
|
||||
continuation_source_facts=project_selected_continuation_source(
|
||||
kind=selected_kind,
|
||||
selected_source=selected_source,
|
||||
),
|
||||
notes=tuple(notes),
|
||||
)
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
"""One-physical-group Sampling Engine for the Continuum runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
def sample_physical_group(
|
||||
*,
|
||||
sample_callable: Callable[..., Any],
|
||||
model: Any,
|
||||
conditioning: Any,
|
||||
latent: Any,
|
||||
sampler: Any,
|
||||
sigmas: Any,
|
||||
seed: int,
|
||||
enable_preview: bool,
|
||||
physical_group: int,
|
||||
logical_chunks: tuple[int, ...],
|
||||
terminal_atomic: bool,
|
||||
layout_validation_profiler: Any = None,
|
||||
packed_row_planner: Any = None,
|
||||
planner_event: Any = None,
|
||||
diagnostic_policy: Any = None,
|
||||
context_frames: int = 0,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Run exactly one physical sample with unchanged observer ordering."""
|
||||
|
||||
normalized_chunks = tuple(int(value) for value in logical_chunks)
|
||||
layout_profile_token = (
|
||||
layout_validation_profiler.begin_sampling_group(
|
||||
physical_group=int(physical_group),
|
||||
logical_chunks=normalized_chunks,
|
||||
sampling_steps=max(0, int(sigmas.shape[-1]) - 1),
|
||||
)
|
||||
if layout_validation_profiler is not None
|
||||
else None
|
||||
)
|
||||
diagnostic_sample_token = None
|
||||
if diagnostic_policy is not None:
|
||||
diagnostic_sample_token = diagnostic_policy.before_sampling(
|
||||
physical_group=int(physical_group),
|
||||
logical_chunks=normalized_chunks,
|
||||
context_frames=int(context_frames),
|
||||
latent=latent,
|
||||
conditioning=conditioning,
|
||||
seed=int(seed),
|
||||
sigmas=sigmas,
|
||||
)
|
||||
if planner_event is not None:
|
||||
planner_event(
|
||||
packed_row_planner,
|
||||
"begin_group",
|
||||
physical_group=int(physical_group),
|
||||
logical_chunks=normalized_chunks,
|
||||
terminal_atomic=bool(terminal_atomic),
|
||||
)
|
||||
sampled = sample_callable(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
if planner_event is not None:
|
||||
planner_event(packed_row_planner, "finish_group")
|
||||
if layout_validation_profiler is not None and layout_profile_token is not None:
|
||||
layout_validation_profiler.finish_sampling_group(layout_profile_token)
|
||||
return sampled, diagnostic_sample_token
|
||||
+644
@@ -0,0 +1,644 @@
|
||||
"""V3.5 Second Pass contracts and the physical-group sampling runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..model_patch import clone_model_for_chunk
|
||||
from ..state import extract_av_streams
|
||||
from ..v2.h3_builder import encode_prompt_conditioning
|
||||
from ..v2.sampling import latent_from_cpu
|
||||
from .plan import SECOND_PASS_CONTRACT_VERSION
|
||||
from .refine_context import (
|
||||
MAGIC as REFINE_CONTEXT_MAGIC,
|
||||
OUTPUT_SCOPE_KEY,
|
||||
RefineConditioningAdaptationError,
|
||||
RefineContextError,
|
||||
adapt_group_conditioning,
|
||||
validate_refine_context,
|
||||
)
|
||||
from .refine_schedule import (
|
||||
RefineSchedule,
|
||||
RefineScheduleError,
|
||||
resolve_refine_schedule,
|
||||
serializable_schedule_contract,
|
||||
)
|
||||
from .refine_sampling import sample_refine_chunk
|
||||
|
||||
|
||||
class SecondPassContractError(ValueError):
|
||||
"""Raised only when a Second Pass structural contract is impossible."""
|
||||
|
||||
|
||||
def derive_refine_seed(refine_seed: int, physical_group_index: int) -> int:
|
||||
"""Derive one deterministic seed per physical group in an independent namespace."""
|
||||
if type(refine_seed) is not int or refine_seed < 0:
|
||||
raise SecondPassContractError("refine_seed must be a non-negative integer")
|
||||
if type(physical_group_index) is not int or physical_group_index < 0:
|
||||
raise SecondPassContractError("physical_group_index must be a non-negative integer")
|
||||
payload = f"h3-continuum-refine-v1:{refine_seed}:{physical_group_index}".encode("ascii")
|
||||
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
|
||||
|
||||
|
||||
def _latent_samples(value: Any, *, name: str) -> torch.Tensor:
|
||||
if not isinstance(value, Mapping) or "samples" not in value:
|
||||
raise SecondPassContractError(f"{name} must be a LATENT mapping with samples")
|
||||
samples = value["samples"]
|
||||
if not isinstance(samples, torch.Tensor):
|
||||
raise SecondPassContractError(f"{name}.samples must be a tensor")
|
||||
return samples
|
||||
|
||||
|
||||
def _require_finite_samples(samples: torch.Tensor, *, name: str) -> None:
|
||||
"""Reject unusable processor output before it reaches Second Pass Sampling."""
|
||||
|
||||
if not bool(torch.isfinite(samples).all().item()):
|
||||
raise SecondPassContractError(f"{name}.samples contains NaN or Inf")
|
||||
|
||||
|
||||
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
|
||||
if not isinstance(assembly_plan, Mapping):
|
||||
raise SecondPassContractError("assembly_plan must be a mapping")
|
||||
contract = assembly_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, Mapping):
|
||||
raise SecondPassContractError("assembly_plan has no second_pass_contract")
|
||||
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
|
||||
raise SecondPassContractError("unsupported second_pass_contract version")
|
||||
groups = contract.get("physical_groups")
|
||||
if not isinstance(groups, list) or not groups or not all(isinstance(g, Mapping) for g in groups):
|
||||
raise SecondPassContractError("second_pass_contract.physical_groups must be non-empty")
|
||||
return groups
|
||||
|
||||
|
||||
def _validate_video_groups(
|
||||
video_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> tuple[list[Mapping[str, Any]], tuple[int, int]]:
|
||||
groups = _contract_groups(assembly_plan)
|
||||
if not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)):
|
||||
raise SecondPassContractError("video_latents must be a sequence")
|
||||
if len(video_latents) != len(groups):
|
||||
raise SecondPassContractError("video latent group count changed")
|
||||
|
||||
target_geometry: tuple[int, int] | None = None
|
||||
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
|
||||
if group.get("group_id") != position:
|
||||
raise SecondPassContractError("physical group order or group_id changed")
|
||||
samples = _latent_samples(latent, name=f"video_latents[{position}]")
|
||||
if samples.ndim != 5:
|
||||
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
|
||||
batch, channels, temporal, height, width = (int(value) for value in samples.shape)
|
||||
if channels != 24:
|
||||
raise SecondPassContractError("MiniMax H3 video latent must have 24 channels")
|
||||
if batch != int(group.get("source_batch", -1)):
|
||||
raise SecondPassContractError("video latent batch changed")
|
||||
if temporal != int(group.get("source_latent_t", -1)):
|
||||
raise SecondPassContractError("video latent temporal length changed")
|
||||
source_h = int(group.get("source_latent_h", -1))
|
||||
source_w = int(group.get("source_latent_w", -1))
|
||||
if source_h < 1 or source_w < 1 or height < source_h or width < source_w:
|
||||
raise SecondPassContractError("video latent spatial size must be preserved or enlarged")
|
||||
_require_finite_samples(samples, name=f"video_latents[{position}]")
|
||||
if target_geometry is None:
|
||||
target_geometry = (height, width)
|
||||
elif target_geometry != (height, width):
|
||||
raise SecondPassContractError("all physical groups must use the same target H/W")
|
||||
|
||||
assert target_geometry is not None
|
||||
return groups, target_geometry
|
||||
|
||||
|
||||
def validate_second_pass_inputs(
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Validate only structural invariants required by a future Second Pass node."""
|
||||
groups, target_geometry = _validate_video_groups(video_latents, assembly_plan)
|
||||
if not isinstance(audio_latents, Sequence) or isinstance(audio_latents, (str, bytes)):
|
||||
raise SecondPassContractError("audio_latents must be a sequence")
|
||||
if len(audio_latents) != len(groups):
|
||||
raise SecondPassContractError("audio latent group count changed")
|
||||
|
||||
for position, (latent, group) in enumerate(zip(audio_latents, groups, strict=True)):
|
||||
samples = _latent_samples(latent, name=f"audio_latents[{position}]")
|
||||
expected_shape = group.get("source_audio_shape")
|
||||
if not isinstance(expected_shape, list) or tuple(expected_shape) != tuple(samples.shape):
|
||||
raise SecondPassContractError("audio latent shape or physical group order changed")
|
||||
_require_finite_samples(samples, name=f"audio_latents[{position}]")
|
||||
|
||||
return {
|
||||
"physical_group_count": len(groups),
|
||||
"target_latent_h": target_geometry[0],
|
||||
"target_latent_w": target_geometry[1],
|
||||
"audio_passthrough": True,
|
||||
}
|
||||
|
||||
|
||||
def passthrough_audio_latents(audio_latents: Sequence[Any]) -> list[Any]:
|
||||
"""Return original LATENT objects; temporary refined audio is not adopted in V1."""
|
||||
return list(audio_latents)
|
||||
|
||||
|
||||
def _normalize_selected_group_indices(
|
||||
selected_group_indices: Sequence[int] | None,
|
||||
physical_group_count: int,
|
||||
) -> tuple[tuple[int, ...], bool]:
|
||||
"""Validate an optional internal physical-group selection."""
|
||||
|
||||
if selected_group_indices is None:
|
||||
return tuple(range(physical_group_count)), False
|
||||
if (
|
||||
not isinstance(selected_group_indices, Sequence)
|
||||
or isinstance(selected_group_indices, (str, bytes))
|
||||
or not selected_group_indices
|
||||
):
|
||||
raise SecondPassContractError(
|
||||
"selected physical groups must be a non-empty integer sequence"
|
||||
)
|
||||
selected = tuple(selected_group_indices)
|
||||
if any(type(index) is not int for index in selected):
|
||||
raise SecondPassContractError(
|
||||
"selected physical group indices must be integers"
|
||||
)
|
||||
if len(set(selected)) != len(selected):
|
||||
raise SecondPassContractError("selected physical groups contain duplicates")
|
||||
if tuple(sorted(selected)) != selected:
|
||||
raise SecondPassContractError(
|
||||
"selected physical groups must follow physical execution order"
|
||||
)
|
||||
if selected[0] < 0 or selected[-1] >= physical_group_count:
|
||||
raise SecondPassContractError("selected physical group index is out of range")
|
||||
return selected, True
|
||||
|
||||
|
||||
def update_second_pass_geometry(
|
||||
assembly_plan: Mapping[str, Any],
|
||||
video_latents: Sequence[Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Return a copied plan whose output geometry follows the upscaled physical latents."""
|
||||
groups, (target_h, target_w) = _validate_video_groups(video_latents, assembly_plan)
|
||||
scale_h: int | None = None
|
||||
scale_w: int | None = None
|
||||
for group in groups:
|
||||
source_h = int(group["source_latent_h"])
|
||||
source_w = int(group["source_latent_w"])
|
||||
source_height = int(group["source_height"])
|
||||
source_width = int(group["source_width"])
|
||||
if source_height % source_h or source_width % source_w:
|
||||
raise SecondPassContractError("source pixel/latent geometry is not integral")
|
||||
group_scale_h = source_height // source_h
|
||||
group_scale_w = source_width // source_w
|
||||
if scale_h is None:
|
||||
scale_h, scale_w = group_scale_h, group_scale_w
|
||||
elif (scale_h, scale_w) != (group_scale_h, group_scale_w):
|
||||
raise SecondPassContractError("source geometry scale differs between groups")
|
||||
|
||||
assert scale_h is not None and scale_w is not None
|
||||
target_height = target_h * scale_h
|
||||
target_width = target_w * scale_w
|
||||
updated = copy.deepcopy(dict(assembly_plan))
|
||||
updated["width"] = target_width
|
||||
updated["height"] = target_height
|
||||
contract = updated["second_pass_contract"]
|
||||
contract["target_width"] = target_width
|
||||
contract["target_height"] = target_height
|
||||
for group in contract["physical_groups"]:
|
||||
group["target_latent_h"] = target_h
|
||||
group["target_latent_w"] = target_w
|
||||
group["target_width"] = target_width
|
||||
group["target_height"] = target_height
|
||||
return updated
|
||||
|
||||
|
||||
def prepare_physical_refine_groups(
|
||||
*,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
video_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
refine_context: Any = None,
|
||||
video_vae: Any = None,
|
||||
conditioning_upscale_method: str = "bilinear",
|
||||
encode_prompt_fn=None,
|
||||
clone_model_fn=None,
|
||||
validate_refine_context_fn=None,
|
||||
adapt_group_conditioning_fn=None,
|
||||
group_consumer_fn=None,
|
||||
retain_group_outputs: bool = True,
|
||||
selected_group_indices: Sequence[int] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Prepare one MODEL and one complete CONDITIONING per physical group.
|
||||
|
||||
This is the shared, sampling-independent contract used by the built-in
|
||||
Second Pass and the public Advanced Conditioning Bridge. The returned
|
||||
``conditioning`` value is an outer physical-group list; every item inside
|
||||
it remains one complete ComfyUI CONDITIONING object.
|
||||
"""
|
||||
|
||||
groups, (target_h, target_w) = _validate_video_groups(
|
||||
video_latents,
|
||||
assembly_plan,
|
||||
)
|
||||
selected_indices, selection_active = _normalize_selected_group_indices(
|
||||
selected_group_indices,
|
||||
len(groups),
|
||||
)
|
||||
selected_index_set = set(selected_indices)
|
||||
encode_prompt_fn = encode_prompt_fn or encode_prompt_conditioning
|
||||
clone_model_fn = clone_model_fn or clone_model_for_chunk
|
||||
validate_refine_context_fn = validate_refine_context_fn or validate_refine_context
|
||||
adapt_group_conditioning_fn = (
|
||||
adapt_group_conditioning_fn or adapt_group_conditioning
|
||||
)
|
||||
|
||||
captured_groups: Sequence[Mapping[str, Any]] | None = None
|
||||
warnings: list[str] = []
|
||||
if refine_context is None:
|
||||
warnings.append(
|
||||
"WARNING: refine_context is not connected; using legacy prompt-only "
|
||||
"conditioning."
|
||||
)
|
||||
else:
|
||||
typed_context = (
|
||||
isinstance(refine_context, Mapping)
|
||||
and refine_context.get("magic") == REFINE_CONTEXT_MAGIC
|
||||
)
|
||||
try:
|
||||
validated_context = validate_refine_context_fn(
|
||||
refine_context,
|
||||
assembly_plan=assembly_plan,
|
||||
)
|
||||
except RefineContextError as exc:
|
||||
if typed_context:
|
||||
raise SecondPassContractError(
|
||||
f"refine_context contract is invalid: {exc}"
|
||||
) from exc
|
||||
warnings.append(
|
||||
"WARNING: optional refine_context was not a typed Continuum context; "
|
||||
f"using legacy prompt-only conditioning ({exc})."
|
||||
)
|
||||
else:
|
||||
if (bool(validated_context.get("complete"))
|
||||
or validated_context.get(OUTPUT_SCOPE_KEY) is not None):
|
||||
captured_groups = validated_context["groups"]
|
||||
else:
|
||||
warnings.append(
|
||||
"WARNING: refine_context capture is incomplete; using legacy "
|
||||
"prompt-only conditioning."
|
||||
)
|
||||
warnings.extend(str(note) for note in validated_context.get("notes", ())
|
||||
if str(note).startswith("Output scope unavailable:"))
|
||||
|
||||
group_models: list[Any] = []
|
||||
group_conditioning: list[list[Any]] = []
|
||||
details: list[dict[str, Any]] = []
|
||||
for group_index, group in enumerate(groups):
|
||||
if group_index not in selected_index_set:
|
||||
continue
|
||||
captured_group = (
|
||||
captured_groups[group_index] if captured_groups is not None else None
|
||||
)
|
||||
model_group = captured_group if captured_group is not None else group
|
||||
adaptation_stats: Mapping[str, Any] | None = None
|
||||
context_conditioning_available = False
|
||||
if captured_group is not None:
|
||||
try:
|
||||
conditioning, adaptation_stats = adapt_group_conditioning_fn(
|
||||
captured_group,
|
||||
target_latent_h=target_h,
|
||||
target_latent_w=target_w,
|
||||
video_vae=video_vae,
|
||||
upscale_method=str(conditioning_upscale_method),
|
||||
)
|
||||
conditioning_source = "refine_context"
|
||||
context_conditioning_available = True
|
||||
except RefineConditioningAdaptationError as exc:
|
||||
warnings.append(
|
||||
f"WARNING: group {group_index + 1} refine_context target "
|
||||
f"adaptation was unavailable; using legacy prompt-only "
|
||||
f"conditioning ({exc})."
|
||||
)
|
||||
except RefineContextError as exc:
|
||||
raise SecondPassContractError(
|
||||
f"refine_context group {group_index} is invalid: {exc}"
|
||||
) from exc
|
||||
|
||||
if not context_conditioning_available:
|
||||
prompt = str(group.get("physical_prompt", ""))
|
||||
conditioning = encode_prompt_fn(
|
||||
clip,
|
||||
prompt,
|
||||
first_image=None,
|
||||
last_image=None,
|
||||
reference_audio_assets=None,
|
||||
timeline_video_assets=None,
|
||||
)
|
||||
conditioning_source = "prompt_only_fallback"
|
||||
if not isinstance(conditioning, list):
|
||||
raise SecondPassContractError(
|
||||
f"physical group {group_index} conditioning is not a complete "
|
||||
"ComfyUI CONDITIONING object"
|
||||
)
|
||||
|
||||
logical_chunks = model_group.get("logical_chunks")
|
||||
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
|
||||
logical_chunks = group.get("logical_chunks")
|
||||
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
|
||||
raise SecondPassContractError(
|
||||
f"physical group {group_index} has no logical chunk identity"
|
||||
)
|
||||
physical_clip_index = int(
|
||||
model_group.get("physical_clip_index", logical_chunks[0])
|
||||
)
|
||||
context_frames = int(
|
||||
model_group.get(
|
||||
"context_frames",
|
||||
group.get("trim_prefix_frames", 0),
|
||||
)
|
||||
)
|
||||
if physical_clip_index <= 0 or context_frames < 0:
|
||||
raise SecondPassContractError(
|
||||
f"physical group {group_index} MODEL context is invalid"
|
||||
)
|
||||
chunk_model = clone_model_fn(
|
||||
model,
|
||||
strict=False,
|
||||
debug=False,
|
||||
chunk_index=physical_clip_index,
|
||||
context_frames=context_frames if context_frames > 0 else None,
|
||||
)
|
||||
detail = {
|
||||
"group_index": group_index,
|
||||
"group": group,
|
||||
"conditioning_source": conditioning_source,
|
||||
"adaptation_stats": adaptation_stats,
|
||||
"physical_clip_index": physical_clip_index,
|
||||
"context_frames": context_frames,
|
||||
"reference_routing_v1": (
|
||||
captured_group.get("reference_routing_v1")
|
||||
if captured_group is not None else None
|
||||
),
|
||||
}
|
||||
if group_consumer_fn is not None:
|
||||
group_consumer_fn(
|
||||
group_index,
|
||||
video_latents[group_index],
|
||||
group,
|
||||
chunk_model,
|
||||
conditioning,
|
||||
detail,
|
||||
)
|
||||
if retain_group_outputs:
|
||||
group_models.append(chunk_model)
|
||||
# Append, never extend: CONDITIONING's internal entries belong together.
|
||||
group_conditioning.append(conditioning)
|
||||
details.append(detail)
|
||||
|
||||
physical_group_count = len(groups)
|
||||
selected_group_count = len(selected_indices)
|
||||
if retain_group_outputs and not (
|
||||
len(group_models)
|
||||
== len(group_conditioning)
|
||||
== len(details)
|
||||
== selected_group_count
|
||||
):
|
||||
raise SecondPassContractError(
|
||||
"prepared MODEL/CONDITIONING physical group counts differ"
|
||||
)
|
||||
if not retain_group_outputs and group_consumer_fn is None:
|
||||
raise SecondPassContractError(
|
||||
"non-retained physical group preparation requires a consumer"
|
||||
)
|
||||
return {
|
||||
"group_models": group_models,
|
||||
"conditioning": group_conditioning,
|
||||
"updated_assembly_plan": update_second_pass_geometry(
|
||||
assembly_plan,
|
||||
video_latents,
|
||||
),
|
||||
"details": details,
|
||||
"warnings": warnings,
|
||||
"physical_group_count": physical_group_count,
|
||||
"selected_group_count": selected_group_count,
|
||||
"selected_group_indices": list(selected_indices),
|
||||
"selection_active": selection_active,
|
||||
"target_latent_h": target_h,
|
||||
"target_latent_w": target_w,
|
||||
}
|
||||
|
||||
|
||||
def run_second_pass_groups(
|
||||
*,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
refine_seed: int,
|
||||
refine_context: Any = None,
|
||||
video_vae: Any = None,
|
||||
conditioning_upscale_method: str = "bilinear",
|
||||
enable_preview: bool = True,
|
||||
encode_prompt_fn=None,
|
||||
latent_builder=None,
|
||||
sample_fn=None,
|
||||
stream_extractor=None,
|
||||
clone_model_fn=None,
|
||||
validate_refine_context_fn=None,
|
||||
adapt_group_conditioning_fn=None,
|
||||
refine_schedule: RefineSchedule | None = None,
|
||||
selected_group_indices: Sequence[int] | None = None,
|
||||
temporal_windows_by_group: Mapping[int, Any] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
||||
"""Refine complete physical groups while preserving first-pass audio exactly."""
|
||||
|
||||
validation = validate_second_pass_inputs(video_latents, audio_latents, assembly_plan)
|
||||
try:
|
||||
resolved_schedule = resolve_refine_schedule(sigmas, refine_schedule)
|
||||
except RefineScheduleError as exc:
|
||||
raise SecondPassContractError(f"refine schedule is invalid: {exc}") from exc
|
||||
effective_sigmas = resolved_schedule.sigmas
|
||||
schedule_contract = serializable_schedule_contract(resolved_schedule)
|
||||
groups = _contract_groups(assembly_plan)
|
||||
selected_indices, selection_active = _normalize_selected_group_indices(
|
||||
selected_group_indices,
|
||||
len(groups),
|
||||
)
|
||||
latent_builder = latent_builder or latent_from_cpu
|
||||
sample_fn = sample_fn or sample_refine_chunk
|
||||
stream_extractor = stream_extractor or extract_av_streams
|
||||
refined_videos: list[dict[str, Any]] = (
|
||||
list(video_latents) if selection_active else []
|
||||
)
|
||||
group_seeds: list[int] = []
|
||||
conditioning_sources: list[str] = []
|
||||
reference_inheritance: list[dict[str, Any]] = []
|
||||
group_report_lines: list[str] = []
|
||||
routed_first_pass = isinstance(assembly_plan.get("reference_routing_v1"), Mapping)
|
||||
|
||||
def sample_prepared_group(
|
||||
group_index,
|
||||
video_latent,
|
||||
group,
|
||||
chunk_model,
|
||||
conditioning,
|
||||
detail,
|
||||
):
|
||||
audio_latent = audio_latents[group_index]
|
||||
video_samples = _latent_samples(video_latent, name=f"video_latents[{group_index}]")
|
||||
audio_samples = _latent_samples(audio_latent, name=f"audio_latents[{group_index}]")
|
||||
conditioning_source = str(detail["conditioning_source"])
|
||||
adaptation_stats = detail["adaptation_stats"]
|
||||
physical_clip_index = int(detail["physical_clip_index"])
|
||||
context_frames = int(detail["context_frames"])
|
||||
physical_seed = derive_refine_seed(int(refine_seed), group_index)
|
||||
group_seeds.append(physical_seed)
|
||||
conditioning_sources.append(conditioning_source)
|
||||
route_record = detail.get("reference_routing_v1")
|
||||
inheritance = (
|
||||
"verified" if conditioning_source == "refine_context" and route_record is not None
|
||||
else "not_inherited" if routed_first_pass
|
||||
else "not_applicable"
|
||||
)
|
||||
reference_inheritance.append({
|
||||
"physical_group": group_index + 1,
|
||||
"status": inheritance,
|
||||
"contract_sha256": route_record.get("sha256") if route_record is not None else None,
|
||||
})
|
||||
nested_latent = latent_builder(video_samples, audio_samples)
|
||||
sample_arguments = dict(
|
||||
model=chunk_model,
|
||||
conditioning=conditioning,
|
||||
latent=nested_latent,
|
||||
sampler=sampler,
|
||||
sigmas=effective_sigmas,
|
||||
seed=physical_seed,
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
if temporal_windows_by_group is not None:
|
||||
temporal_window = temporal_windows_by_group.get(group_index)
|
||||
if temporal_window is None:
|
||||
raise SecondPassContractError(
|
||||
f"selected physical group {group_index + 1} has no temporal window"
|
||||
)
|
||||
sample_arguments["temporal_window"] = temporal_window
|
||||
sampled = sample_fn(**sample_arguments)
|
||||
refined_video, _temporary_audio = stream_extractor(sampled)
|
||||
if tuple(refined_video.shape) != tuple(video_samples.shape):
|
||||
raise SecondPassContractError(
|
||||
f"refined video group {group_index} changed B/C/T/H/W geometry"
|
||||
)
|
||||
if not bool(torch.isfinite(refined_video.float()).all().item()):
|
||||
raise SecondPassContractError(
|
||||
f"refined video group {group_index} contains NaN or Inf"
|
||||
)
|
||||
output_latent = dict(video_latent)
|
||||
output_latent["samples"] = refined_video
|
||||
if selection_active:
|
||||
refined_videos[group_index] = output_latent
|
||||
else:
|
||||
refined_videos.append(output_latent)
|
||||
group_report_lines.append(
|
||||
"group "
|
||||
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
|
||||
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
|
||||
f"conditioning_source={conditioning_source}, "
|
||||
f"reference_inheritance={inheritance}, "
|
||||
f"physical_clip_index={physical_clip_index}, "
|
||||
f"context_frames={context_frames}, "
|
||||
f"refine_seed={physical_seed}, shape={tuple(refined_video.shape)}, "
|
||||
"sampling_passes=1, temporary_audio_discarded=true"
|
||||
)
|
||||
if adaptation_stats is not None:
|
||||
group_report_lines.append(
|
||||
"group "
|
||||
f"{group_index + 1} target conditioning: "
|
||||
f"keyframes_resized={int(adaptation_stats.get('keyframes_resized', 0))}, "
|
||||
f"keyframes_reencoded={int(adaptation_stats.get('keyframes_reencoded', 0))}, "
|
||||
f"guide_keyframes_reencoded={int(adaptation_stats.get('guide_keyframes_reencoded', 0))}, "
|
||||
f"context_refs_resized={int(adaptation_stats.get('context_refs_resized', 0))}"
|
||||
)
|
||||
group_report_lines.extend(
|
||||
f"WARNING: {warning}"
|
||||
for warning in adaptation_stats.get("warnings", ())
|
||||
)
|
||||
|
||||
prepared = prepare_physical_refine_groups(
|
||||
model=model,
|
||||
clip=clip,
|
||||
video_latents=video_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
refine_context=refine_context,
|
||||
video_vae=video_vae,
|
||||
conditioning_upscale_method=conditioning_upscale_method,
|
||||
encode_prompt_fn=encode_prompt_fn,
|
||||
clone_model_fn=clone_model_fn,
|
||||
validate_refine_context_fn=validate_refine_context_fn,
|
||||
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
|
||||
group_consumer_fn=sample_prepared_group,
|
||||
retain_group_outputs=False,
|
||||
selected_group_indices=(selected_indices if selection_active else None),
|
||||
)
|
||||
if len(group_seeds) != len(selected_indices):
|
||||
raise SecondPassContractError(
|
||||
"Video refine Sampling count differs from selected physical group count"
|
||||
)
|
||||
report_lines = [
|
||||
"H3 Continuum Second Pass V3.5",
|
||||
"Mode: context-aware physical-group refine when a complete First Pass "
|
||||
"refine_context is available.",
|
||||
f"Physical groups: {validation['physical_group_count']}.",
|
||||
"Sampling contract: video random noise/mask=1; audio zero noise/mask=0; "
|
||||
"the resolved RefineSchedule SIGMAS are applied directly by Core.",
|
||||
"RefineSchedule: "
|
||||
f"schema={schedule_contract['schema_version']}, "
|
||||
f"mode={schedule_contract['mode']}, "
|
||||
f"evaluations={schedule_contract['evaluation_count']}, "
|
||||
f"start_sigma={schedule_contract['start_sigma']}, "
|
||||
f"end_sigma={schedule_contract['end_sigma']}, "
|
||||
f"sigma_hash={schedule_contract['sigma_hash']}.",
|
||||
*prepared["warnings"],
|
||||
*group_report_lines,
|
||||
]
|
||||
if selection_active:
|
||||
report_lines.insert(
|
||||
3,
|
||||
"Selected physical groups: "
|
||||
+ ", ".join(str(index + 1) for index in selected_indices)
|
||||
+ "; non-selected groups were exact object passthrough with no "
|
||||
"conditioning, MODEL clone, Sampling, or seed derivation.",
|
||||
)
|
||||
|
||||
passthrough_audio = passthrough_audio_latents(audio_latents)
|
||||
updated_plan = prepared["updated_assembly_plan"]
|
||||
contract = updated_plan["second_pass_contract"]
|
||||
if conditioning_sources and all(
|
||||
source == "refine_context" for source in conditioning_sources
|
||||
):
|
||||
contract["execution"] = "context_aware_physical_groups_audio_locked_v3"
|
||||
elif any(source == "refine_context" for source in conditioning_sources):
|
||||
contract["execution"] = "mixed_context_physical_groups_audio_locked_v3"
|
||||
else:
|
||||
contract["execution"] = "t2va_physical_groups_audio_locked_v2"
|
||||
contract["conditioning_sources"] = conditioning_sources
|
||||
if any(item["status"] != "not_applicable" for item in reference_inheritance):
|
||||
contract["reference_inheritance"] = reference_inheritance
|
||||
contract["refine_seed_base"] = int(refine_seed)
|
||||
contract["refine_group_seeds"] = group_seeds
|
||||
contract["audio_output"] = "bit_exact_first_pass_passthrough"
|
||||
contract["audio_sampling"] = "zero_noise_mask_locked"
|
||||
contract["refine_schedule"] = schedule_contract
|
||||
contract["refine_schedule_identity"] = schedule_contract["schedule_hash"]
|
||||
report_lines.append(
|
||||
"Audio output: first-pass physical audio LATENT objects returned bit-exact; "
|
||||
"temporary refined audio discarded."
|
||||
)
|
||||
return refined_videos, passthrough_audio, updated_plan, "\n".join(report_lines)
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
"""V3.5 Second Pass public node."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .refine_target import MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO, MODE_VIDEO_ONLY
|
||||
from .refine_scope import MODE_ALL, MODE_LOGICAL_CHUNK, MODE_PHYSICAL_GROUP
|
||||
from .refine_window import MODE_TIME_WINDOW
|
||||
from .second_pass import run_second_pass_groups
|
||||
from .targeted_second_pass import run_targeted_second_pass_groups
|
||||
|
||||
|
||||
REFINE_TARGET_VIDEO_ONLY = "Video Only"
|
||||
REFINE_TARGET_AUDIO_ONLY = "Audio Only"
|
||||
REFINE_TARGET_VIDEO_AUDIO = "Video + Audio"
|
||||
REFINE_TARGET_OPTIONS = (
|
||||
REFINE_TARGET_VIDEO_ONLY,
|
||||
REFINE_TARGET_AUDIO_ONLY,
|
||||
REFINE_TARGET_VIDEO_AUDIO,
|
||||
)
|
||||
_REFINE_TARGET_MODES = {
|
||||
REFINE_TARGET_VIDEO_ONLY: MODE_VIDEO_ONLY,
|
||||
REFINE_TARGET_AUDIO_ONLY: MODE_AUDIO_ONLY,
|
||||
REFINE_TARGET_VIDEO_AUDIO: MODE_VIDEO_AUDIO,
|
||||
}
|
||||
|
||||
REFINE_SCOPE_ALL = "All"
|
||||
REFINE_SCOPE_PHYSICAL_GROUP = "Physical Group"
|
||||
REFINE_SCOPE_LOGICAL_CHUNK = "Logical Chunk"
|
||||
REFINE_SCOPE_TIME_WINDOW = "Time Window"
|
||||
REFINE_SCOPE_OPTIONS = (
|
||||
REFINE_SCOPE_ALL,
|
||||
REFINE_SCOPE_PHYSICAL_GROUP,
|
||||
REFINE_SCOPE_LOGICAL_CHUNK,
|
||||
REFINE_SCOPE_TIME_WINDOW,
|
||||
)
|
||||
_REFINE_SCOPE_MODES = {
|
||||
REFINE_SCOPE_ALL: MODE_ALL,
|
||||
REFINE_SCOPE_PHYSICAL_GROUP: MODE_PHYSICAL_GROUP,
|
||||
REFINE_SCOPE_LOGICAL_CHUNK: MODE_LOGICAL_CHUNK,
|
||||
REFINE_SCOPE_TIME_WINDOW: MODE_TIME_WINDOW,
|
||||
}
|
||||
|
||||
|
||||
def _single_list_input(name: str, value: Any) -> Any:
|
||||
if not isinstance(value, list) or len(value) != 1:
|
||||
raise ValueError(f"{name} must contain exactly one value")
|
||||
return value[0]
|
||||
|
||||
|
||||
class H3ContinuumSecondPassV35:
|
||||
"""Low-sigma refinement over complete physical decode groups."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"V3.5 context-aware Second Pass. Connect externally upscaled H3 video "
|
||||
"latents, the matching first-pass audio latents, and the original assembly plan. "
|
||||
"Connect the optional Continuum refine context to preserve the physical group's "
|
||||
"First/Last/Reference conditioning. "
|
||||
"SIGMAS controls the low-denoise refine schedule; audio output is preserved "
|
||||
"bit-exact from the first pass."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"sampler": ("SAMPLER",),
|
||||
"sigmas": ("SIGMAS",),
|
||||
"video_latents": ("LATENT",),
|
||||
"audio_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"refine_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"tooltip": "Base seed independently derived once per physical group.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
|
||||
"video_vae": ("VAE",),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
|
||||
RETURN_NAMES = (
|
||||
"refined_video_latents",
|
||||
"audio_latents",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "refine"
|
||||
CATEGORY = "MiniMax H3/Continuum/Advanced"
|
||||
|
||||
def refine(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
sampler,
|
||||
sigmas,
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
refine_seed,
|
||||
refine_context=None,
|
||||
video_vae=None,
|
||||
refine_schedule=None,
|
||||
):
|
||||
return run_second_pass_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
sampler=_single_list_input("sampler", sampler),
|
||||
sigmas=_single_list_input("sigmas", sigmas),
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
|
||||
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
|
||||
refine_context=(
|
||||
None
|
||||
if refine_context is None
|
||||
else _single_list_input("refine_context", refine_context)
|
||||
),
|
||||
video_vae=(
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
),
|
||||
refine_schedule=refine_schedule,
|
||||
)
|
||||
|
||||
|
||||
def resolve_selective_refine_target(value: str) -> str:
|
||||
"""Resolve one public UI label to the versioned internal target mode."""
|
||||
|
||||
try:
|
||||
return _REFINE_TARGET_MODES[value]
|
||||
except (KeyError, TypeError) as exc:
|
||||
raise ValueError(f"unsupported selective refine target {value!r}") from exc
|
||||
|
||||
|
||||
def resolve_selective_refine_scope(value: str) -> str:
|
||||
"""Resolve one public UI label to the versioned internal scope mode."""
|
||||
|
||||
try:
|
||||
return _REFINE_SCOPE_MODES[value]
|
||||
except (KeyError, TypeError) as exc:
|
||||
raise ValueError(f"unsupported selective refine scope {value!r}") from exc
|
||||
|
||||
|
||||
def _selective_status_prefix(
|
||||
label: str,
|
||||
physical_group_count: int,
|
||||
scope_label: str,
|
||||
scope_index: int,
|
||||
window_start_sec: float,
|
||||
window_end_sec: float,
|
||||
) -> str:
|
||||
if label == REFINE_TARGET_VIDEO_ONLY:
|
||||
detail = "Video=refined; Audio=bit-exact passthrough"
|
||||
elif label == REFINE_TARGET_AUDIO_ONLY:
|
||||
detail = "Experimental / Production HOLD; Video=locked; Audio=refined"
|
||||
elif label == REFINE_TARGET_VIDEO_AUDIO:
|
||||
detail = (
|
||||
"GPU Experimental PASS / Production HOLD; "
|
||||
"Video=refined; Audio=refined"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"unsupported selective refine target {label!r}")
|
||||
if scope_label == REFINE_SCOPE_ALL:
|
||||
scope_detail = "."
|
||||
elif scope_label == REFINE_SCOPE_TIME_WINDOW:
|
||||
scope_detail = f" [{window_start_sec:g}, {window_end_sec:g}) sec."
|
||||
else:
|
||||
scope_detail = f" {scope_index}."
|
||||
return (
|
||||
f"Selective Refine: {label}; {detail}; "
|
||||
f"physical groups={physical_group_count}; scope={scope_label}"
|
||||
+ scope_detail
|
||||
)
|
||||
|
||||
|
||||
class H3ContinuumSelectiveSecondPassExperimental:
|
||||
"""Experimental public adapter for versioned selective-refine targets."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"Experimental Selective Second Pass for MiniMax H3. Video Only preserves "
|
||||
"compatibility with the existing Continuum Second Pass. Audio Only keeps "
|
||||
"the input video locked and refines audio. Video + Audio refines both AV "
|
||||
"streams. Audio-targeted modes remain Experimental while upstream H3 "
|
||||
"denoise-mask correctness is under review. Chunk scope skips complete "
|
||||
"physical groups without conditioning, model clone, Sampling, or seed use; "
|
||||
"a logical chunk inside Terminal Merge expands to the complete pair. Time "
|
||||
"Window uses the final visible output timeline, protects continuation "
|
||||
"prefixes, and restores every out-of-window latent position bit-exactly."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
"clip": ("CLIP",),
|
||||
"sampler": ("SAMPLER",),
|
||||
"sigmas": ("SIGMAS",),
|
||||
"video_latents": ("LATENT",),
|
||||
"audio_latents": ("LATENT",),
|
||||
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
|
||||
"refine_seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": True,
|
||||
"tooltip": "Base seed independently derived once per physical group.",
|
||||
},
|
||||
),
|
||||
"refine_target": (
|
||||
REFINE_TARGET_OPTIONS,
|
||||
{"default": REFINE_TARGET_VIDEO_ONLY},
|
||||
),
|
||||
"refine_scope": (
|
||||
REFINE_SCOPE_OPTIONS,
|
||||
{"default": REFINE_SCOPE_ALL},
|
||||
),
|
||||
"refine_scope_index": (
|
||||
"INT",
|
||||
{
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 0x7FFFFFFF,
|
||||
"tooltip": (
|
||||
"1-based physical-group or logical-chunk number. "
|
||||
"Ignored when Refine Scope is All or Time Window."
|
||||
),
|
||||
},
|
||||
),
|
||||
"window_start_sec": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 86400.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"Time Window start on the final visible output timeline. "
|
||||
"Ignored by other Refine Scope modes."
|
||||
),
|
||||
},
|
||||
),
|
||||
"window_end_sec": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 0.01,
|
||||
"max": 86400.0,
|
||||
"step": 0.01,
|
||||
"tooltip": (
|
||||
"Exclusive Time Window end on the final visible output "
|
||||
"timeline. Ignored by other Refine Scope modes."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
|
||||
"video_vae": ("VAE",),
|
||||
},
|
||||
}
|
||||
|
||||
INPUT_IS_LIST = True
|
||||
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
|
||||
RETURN_NAMES = (
|
||||
"refined_video_latents",
|
||||
"audio_latents",
|
||||
"updated_assembly_plan",
|
||||
"status",
|
||||
)
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "refine"
|
||||
CATEGORY = "MiniMax H3/Continuum/Advanced"
|
||||
|
||||
def refine(
|
||||
self,
|
||||
model,
|
||||
clip,
|
||||
sampler,
|
||||
sigmas,
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
refine_seed,
|
||||
refine_target,
|
||||
refine_scope=None,
|
||||
refine_scope_index=None,
|
||||
window_start_sec=None,
|
||||
window_end_sec=None,
|
||||
refine_context=None,
|
||||
video_vae=None,
|
||||
refine_schedule=None,
|
||||
):
|
||||
label = _single_list_input("refine_target", refine_target)
|
||||
target_mode = resolve_selective_refine_target(label)
|
||||
scope_label = (
|
||||
REFINE_SCOPE_ALL
|
||||
if refine_scope is None
|
||||
else _single_list_input("refine_scope", refine_scope)
|
||||
)
|
||||
scope_mode = resolve_selective_refine_scope(scope_label)
|
||||
resolved_scope_index = (
|
||||
1
|
||||
if refine_scope_index is None
|
||||
else int(_single_list_input("refine_scope_index", refine_scope_index))
|
||||
)
|
||||
resolved_window_start = (
|
||||
0.0
|
||||
if window_start_sec is None
|
||||
else float(_single_list_input("window_start_sec", window_start_sec))
|
||||
)
|
||||
resolved_window_end = (
|
||||
5.0
|
||||
if window_end_sec is None
|
||||
else float(_single_list_input("window_end_sec", window_end_sec))
|
||||
)
|
||||
videos, audios, updated_plan, status = run_targeted_second_pass_groups(
|
||||
model=_single_list_input("model", model),
|
||||
clip=_single_list_input("clip", clip),
|
||||
sampler=_single_list_input("sampler", sampler),
|
||||
sigmas=_single_list_input("sigmas", sigmas),
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
|
||||
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
|
||||
refine_target=target_mode,
|
||||
refine_scope=scope_mode,
|
||||
refine_scope_index=resolved_scope_index,
|
||||
window_start_sec=resolved_window_start,
|
||||
window_end_sec=resolved_window_end,
|
||||
refine_context=(
|
||||
None
|
||||
if refine_context is None
|
||||
else _single_list_input("refine_context", refine_context)
|
||||
),
|
||||
video_vae=(
|
||||
None
|
||||
if video_vae is None
|
||||
else _single_list_input("video_vae", video_vae)
|
||||
),
|
||||
refine_schedule=refine_schedule,
|
||||
)
|
||||
prefix = _selective_status_prefix(
|
||||
label,
|
||||
len(video_latents),
|
||||
scope_label,
|
||||
resolved_scope_index,
|
||||
resolved_window_start,
|
||||
resolved_window_end,
|
||||
)
|
||||
return videos, audios, updated_plan, f"{prefix}\n{status}"
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSecondPassV35": H3ContinuumSecondPassV35,
|
||||
"H3ContinuumSelectiveSecondPassExperimental": (
|
||||
H3ContinuumSelectiveSecondPassExperimental
|
||||
),
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSecondPassV35": "H3 Continuum Second Pass",
|
||||
"H3ContinuumSelectiveSecondPassExperimental": (
|
||||
"H3 Continuum Selective Second Pass (Experimental)"
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
"""Target-aware refinement sampling for frozen Video and Audio policies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
from .refine_sampling import (
|
||||
RefineSamplingRuntimeError,
|
||||
_load_runtime_modules,
|
||||
_make_basic_guider,
|
||||
sample_refine_chunk,
|
||||
)
|
||||
from .refine_target import (
|
||||
MODE_AUDIO_ONLY,
|
||||
MODE_VIDEO_AUDIO,
|
||||
MODE_VIDEO_ONLY,
|
||||
RefineTarget,
|
||||
RefineTargetError,
|
||||
resolve_refine_target,
|
||||
)
|
||||
from .refine_window import GroupTemporalWindow
|
||||
|
||||
|
||||
class TargetedRefineExecutionUnavailable(RefineTargetError):
|
||||
"""Raised before Sampling when a future target mode is not implemented."""
|
||||
|
||||
|
||||
def _validate_ranges(
|
||||
ranges: tuple[tuple[int, int], ...],
|
||||
length: int,
|
||||
*,
|
||||
name: str,
|
||||
) -> tuple[tuple[int, int], ...]:
|
||||
if not ranges:
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal window is empty")
|
||||
previous_stop = 0
|
||||
for start, stop in ranges:
|
||||
if start < previous_stop or stop <= start or stop > int(length):
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal window is outside its latent length"
|
||||
)
|
||||
previous_stop = stop
|
||||
return ranges
|
||||
|
||||
|
||||
def _window_masks(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
temporal_window: GroupTemporalWindow,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not isinstance(temporal_window, GroupTemporalWindow):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"temporal window must be a resolved GroupTemporalWindow"
|
||||
)
|
||||
video_ranges = _validate_ranges(
|
||||
temporal_window.video_slot_ranges,
|
||||
int(video.shape[2]),
|
||||
name="Video",
|
||||
)
|
||||
audio_ranges = _validate_ranges(
|
||||
temporal_window.audio_tick_ranges,
|
||||
int(audio.shape[-1]),
|
||||
name="Audio",
|
||||
)
|
||||
video_mask = torch.zeros(
|
||||
(
|
||||
int(video.shape[0]),
|
||||
1,
|
||||
int(video.shape[2]),
|
||||
int(video.shape[3]),
|
||||
int(video.shape[4]),
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.zeros(
|
||||
(
|
||||
int(audio.shape[0]),
|
||||
1,
|
||||
int(audio.shape[2]),
|
||||
int(audio.shape[3]),
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
for start, stop in video_ranges:
|
||||
video_mask[:, :, start:stop] = 1.0
|
||||
for start, stop in audio_ranges:
|
||||
audio_mask[..., start:stop] = 1.0
|
||||
return video_mask, audio_mask
|
||||
|
||||
|
||||
def _prepare_video_only_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build windowed Video noise/mask while locking Audio completely."""
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video Only noise geometry"
|
||||
)
|
||||
video_mask, _audio_window_mask = _window_masks(video, audio, temporal_window)
|
||||
audio_noise = torch.zeros_like(audio, device="cpu")
|
||||
audio_mask = torch.zeros(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
return (
|
||||
nested_builder((video_noise, audio_noise)),
|
||||
nested_builder((video_mask, audio_mask)),
|
||||
)
|
||||
|
||||
|
||||
def require_video_only_execution(
|
||||
refine_target: RefineTarget | str | None,
|
||||
) -> RefineTarget:
|
||||
target = resolve_refine_target(refine_target)
|
||||
if target.mode != MODE_VIDEO_ONLY:
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is contract-only in Phase A1"
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
def _prepare_audio_only_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build zero/locked Video and seeded-random/sampled Audio streams."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
||||
|
||||
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
||||
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Audio Only noise geometry"
|
||||
)
|
||||
video_noise = torch.zeros_like(video, device="cpu")
|
||||
if temporal_window is None:
|
||||
video_mask = torch.zeros(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.ones(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
_video_window_mask, audio_mask = _window_masks(
|
||||
video,
|
||||
audio,
|
||||
temporal_window,
|
||||
)
|
||||
video_mask = torch.zeros(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def _prepare_video_audio_noise_and_mask(
|
||||
video: torch.Tensor,
|
||||
audio: torch.Tensor,
|
||||
*,
|
||||
seed: int,
|
||||
batch_inds: Any = None,
|
||||
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
|
||||
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Build seeded-random noise and sampled masks for both AV streams."""
|
||||
|
||||
if video.ndim != 5 or audio.ndim != 4:
|
||||
raise RefineSamplingRuntimeError(
|
||||
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
|
||||
)
|
||||
if int(video.shape[0]) != int(audio.shape[0]):
|
||||
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
|
||||
|
||||
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
|
||||
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
|
||||
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video + Audio video noise geometry"
|
||||
)
|
||||
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
"ComfyUI returned invalid Video + Audio audio noise geometry"
|
||||
)
|
||||
if temporal_window is None:
|
||||
video_mask = torch.ones(
|
||||
(int(video.shape[0]), 1, 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
audio_mask = torch.ones(
|
||||
(int(audio.shape[0]), 1, 1, 1),
|
||||
dtype=torch.float32,
|
||||
device="cpu",
|
||||
)
|
||||
else:
|
||||
video_mask, audio_mask = _window_masks(video, audio, temporal_window)
|
||||
noise = nested_builder((video_noise, audio_noise))
|
||||
noise_mask = nested_builder((video_mask, audio_mask))
|
||||
return noise, noise_mask
|
||||
|
||||
|
||||
def _sample_with_target_noise_and_mask(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
enable_preview: bool,
|
||||
noise_mask_builder: Callable[..., tuple[Any, Any]],
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
restore_video_outside_window: bool = False,
|
||||
restore_audio_outside_window: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Run Core once after constructing one explicit AV target policy."""
|
||||
|
||||
if not isinstance(latent, dict) or "samples" not in latent:
|
||||
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
|
||||
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
|
||||
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
|
||||
|
||||
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
|
||||
_load_runtime_modules()
|
||||
)
|
||||
working = latent.copy()
|
||||
latent_image = comfy_sample.fix_empty_latent_channels(
|
||||
model,
|
||||
working["samples"],
|
||||
working.get("downscale_ratio_spacial"),
|
||||
working.get("downscale_ratio_temporal"),
|
||||
)
|
||||
working["samples"] = latent_image
|
||||
video, audio = extract_av_streams(working)
|
||||
noise, noise_mask = noise_mask_builder(
|
||||
video,
|
||||
audio,
|
||||
seed=int(seed),
|
||||
batch_inds=working.get("batch_index"),
|
||||
prepare_noise_fn=comfy_sample.prepare_noise,
|
||||
nested_builder=nested_tensor.NestedTensor,
|
||||
temporal_window=temporal_window,
|
||||
)
|
||||
guider = _make_basic_guider(model, conditioning)
|
||||
|
||||
x0_output: dict[str, Any] = {}
|
||||
callback = None
|
||||
if enable_preview:
|
||||
callback = preview.prepare_callback(
|
||||
model,
|
||||
int(sigmas.shape[-1]) - 1,
|
||||
x0_output,
|
||||
)
|
||||
samples = guider.sample(
|
||||
noise,
|
||||
latent_image,
|
||||
sampler,
|
||||
sigmas,
|
||||
denoise_mask=noise_mask,
|
||||
callback=callback,
|
||||
disable_pbar=not comfy_utils.PROGRESS_BAR_ENABLED,
|
||||
seed=int(seed),
|
||||
)
|
||||
samples = samples.to(model_management.intermediate_device())
|
||||
if temporal_window is not None:
|
||||
sampled_output = {"samples": samples}
|
||||
sampled_video, sampled_audio = extract_av_streams(sampled_output)
|
||||
if restore_video_outside_window:
|
||||
sampled_video = _restore_outside_temporal_ranges(
|
||||
sampled_video,
|
||||
video,
|
||||
temporal_window.video_slot_ranges,
|
||||
time_dim=2,
|
||||
name="Video",
|
||||
)
|
||||
if restore_audio_outside_window:
|
||||
sampled_audio = _restore_outside_temporal_ranges(
|
||||
sampled_audio,
|
||||
audio,
|
||||
temporal_window.audio_tick_ranges,
|
||||
time_dim=-1,
|
||||
name="Audio",
|
||||
)
|
||||
samples = nested_tensor.NestedTensor((sampled_video, sampled_audio))
|
||||
|
||||
output = working.copy()
|
||||
output.pop("downscale_ratio_spacial", None)
|
||||
output.pop("downscale_ratio_temporal", None)
|
||||
output["samples"] = samples
|
||||
extract_av_streams(output)
|
||||
return output
|
||||
|
||||
|
||||
def _restore_outside_temporal_ranges(
|
||||
sampled: torch.Tensor,
|
||||
source: torch.Tensor,
|
||||
ranges: tuple[tuple[int, int], ...],
|
||||
*,
|
||||
time_dim: int,
|
||||
name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Restore every inactive temporal position from the input bit-exactly."""
|
||||
|
||||
if not torch.is_tensor(sampled) or not torch.is_tensor(source):
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal restore requires Tensors")
|
||||
if tuple(sampled.shape) != tuple(source.shape):
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal restore input/output shapes differ"
|
||||
)
|
||||
if sampled.dtype != source.dtype:
|
||||
raise RefineSamplingRuntimeError(
|
||||
f"{name} temporal restore input/output dtypes differ"
|
||||
)
|
||||
dim = int(time_dim)
|
||||
if dim < 0:
|
||||
dim += sampled.ndim
|
||||
if dim < 0 or dim >= sampled.ndim:
|
||||
raise RefineSamplingRuntimeError(f"{name} temporal restore dimension is invalid")
|
||||
validated = _validate_ranges(ranges, int(sampled.shape[dim]), name=name)
|
||||
restored = sampled.clone()
|
||||
original = source.to(device=restored.device)
|
||||
cursor = 0
|
||||
for start, stop in validated:
|
||||
if start > cursor:
|
||||
restored.narrow(dim, cursor, start - cursor).copy_(
|
||||
original.narrow(dim, cursor, start - cursor)
|
||||
)
|
||||
cursor = stop
|
||||
if cursor < int(restored.shape[dim]):
|
||||
restored.narrow(dim, cursor, int(restored.shape[dim]) - cursor).copy_(
|
||||
original.narrow(dim, cursor, int(restored.shape[dim]) - cursor)
|
||||
)
|
||||
return restored
|
||||
|
||||
|
||||
def sample_video_only_window_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Refine only selected Video slots and restore all other AV positions."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_video_only_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=True,
|
||||
restore_audio_outside_window=True,
|
||||
)
|
||||
|
||||
|
||||
def sample_audio_only_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Sample Audio while Core carries Video with zero noise and mask zero."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_audio_only_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=temporal_window is not None,
|
||||
restore_audio_outside_window=temporal_window is not None,
|
||||
)
|
||||
|
||||
|
||||
def sample_video_audio_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Sample and adopt both Video and Audio using mask one for each stream."""
|
||||
|
||||
return _sample_with_target_noise_and_mask(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
noise_mask_builder=_prepare_video_audio_noise_and_mask,
|
||||
temporal_window=temporal_window,
|
||||
restore_video_outside_window=temporal_window is not None,
|
||||
restore_audio_outside_window=temporal_window is not None,
|
||||
)
|
||||
|
||||
|
||||
def sample_targeted_refine_chunk(
|
||||
*,
|
||||
model: Any,
|
||||
conditioning: list,
|
||||
latent: dict[str, Any],
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
seed: int,
|
||||
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
|
||||
temporal_window: GroupTemporalWindow | None = None,
|
||||
enable_preview: bool = True,
|
||||
legacy_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
video_only_window_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
audio_only_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
video_audio_sample_fn: Callable[..., dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Dispatch an implemented target without changing its stream policy."""
|
||||
|
||||
target = resolve_refine_target(refine_target)
|
||||
if target.mode == MODE_VIDEO_ONLY and temporal_window is not None:
|
||||
sample_fn = video_only_window_sample_fn or sample_video_only_window_refine_chunk
|
||||
elif target.mode == MODE_VIDEO_ONLY:
|
||||
sample_fn = legacy_sample_fn or sample_refine_chunk
|
||||
elif target.mode == MODE_AUDIO_ONLY:
|
||||
sample_fn = audio_only_sample_fn or sample_audio_only_refine_chunk
|
||||
elif target.mode == MODE_VIDEO_AUDIO:
|
||||
sample_fn = video_audio_sample_fn or sample_video_audio_refine_chunk
|
||||
else: # pragma: no cover - resolve_refine_target validates the mode.
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is unavailable"
|
||||
)
|
||||
arguments = dict(
|
||||
model=model,
|
||||
conditioning=conditioning,
|
||||
latent=latent,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
seed=int(seed),
|
||||
enable_preview=bool(enable_preview),
|
||||
)
|
||||
if temporal_window is not None:
|
||||
arguments["temporal_window"] = temporal_window
|
||||
return sample_fn(**arguments)
|
||||
@@ -0,0 +1,491 @@
|
||||
"""Target-aware Second Pass orchestration over complete physical groups."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from ..state import extract_av_streams
|
||||
from ..v2.sampling import latent_from_cpu
|
||||
from .refine_schedule import (
|
||||
RefineScheduleError,
|
||||
resolve_refine_schedule,
|
||||
serializable_schedule_contract,
|
||||
)
|
||||
from .refine_scope import (
|
||||
MODE_ALL as SCOPE_ALL,
|
||||
resolve_refine_scope,
|
||||
)
|
||||
from .refine_target import (
|
||||
MODE_AUDIO_ONLY,
|
||||
MODE_VIDEO_AUDIO,
|
||||
MODE_VIDEO_ONLY,
|
||||
RefineTarget,
|
||||
build_target_execution_contract,
|
||||
derive_target_refine_seed,
|
||||
resolve_refine_target,
|
||||
serializable_target_contract,
|
||||
)
|
||||
from .refine_window import (
|
||||
MODE_TIME_WINDOW,
|
||||
GroupTemporalWindow,
|
||||
RefineWindow,
|
||||
resolve_refine_window,
|
||||
)
|
||||
from .second_pass import (
|
||||
SecondPassContractError,
|
||||
_normalize_selected_group_indices,
|
||||
_latent_samples,
|
||||
prepare_physical_refine_groups,
|
||||
run_second_pass_groups,
|
||||
validate_second_pass_inputs,
|
||||
)
|
||||
from .targeted_refine_sampling import (
|
||||
TargetedRefineExecutionUnavailable,
|
||||
sample_targeted_refine_chunk,
|
||||
)
|
||||
|
||||
|
||||
def _attach_target_contract(
|
||||
updated_plan: dict[str, Any],
|
||||
target: RefineTarget,
|
||||
) -> None:
|
||||
contract = updated_plan.get("second_pass_contract")
|
||||
if not isinstance(contract, dict):
|
||||
raise SecondPassContractError(
|
||||
"targeted Second Pass output has no mutable contract"
|
||||
)
|
||||
if contract.get("version") != 1:
|
||||
raise SecondPassContractError(
|
||||
"targeted Second Pass must preserve contract version 1"
|
||||
)
|
||||
schedule_contract = contract.get("refine_schedule")
|
||||
contract["refine_target_contract"] = serializable_target_contract(target)
|
||||
contract["refine_execution_contract"] = build_target_execution_contract(
|
||||
target,
|
||||
schedule_contract,
|
||||
)
|
||||
|
||||
|
||||
def _run_video_only_groups(
|
||||
*,
|
||||
target: RefineTarget,
|
||||
sample_fn,
|
||||
arguments: dict[str, Any],
|
||||
selected_group_indices: Sequence[int] | None = None,
|
||||
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
||||
def targeted_sample(**kwargs):
|
||||
return sample_targeted_refine_chunk(
|
||||
refine_target=target,
|
||||
legacy_sample_fn=sample_fn,
|
||||
video_only_window_sample_fn=sample_fn,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
videos, audios, updated_plan, status = run_second_pass_groups(
|
||||
**arguments,
|
||||
sample_fn=targeted_sample,
|
||||
selected_group_indices=selected_group_indices,
|
||||
temporal_windows_by_group=temporal_windows_by_group,
|
||||
)
|
||||
_attach_target_contract(updated_plan, target)
|
||||
return videos, audios, updated_plan, status
|
||||
|
||||
|
||||
def _run_sampled_target_groups(
|
||||
*,
|
||||
target: RefineTarget,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
refine_seed: int,
|
||||
refine_context: Any,
|
||||
video_vae: Any,
|
||||
conditioning_upscale_method: str,
|
||||
enable_preview: bool,
|
||||
encode_prompt_fn,
|
||||
latent_builder,
|
||||
sample_fn,
|
||||
stream_extractor,
|
||||
clone_model_fn,
|
||||
validate_refine_context_fn,
|
||||
adapt_group_conditioning_fn,
|
||||
refine_schedule,
|
||||
selected_group_indices: Sequence[int] | None = None,
|
||||
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
|
||||
) -> tuple[list[Any], list[dict[str, Any]], dict[str, Any], str]:
|
||||
"""Sample one non-legacy target and adopt only its configured outputs."""
|
||||
|
||||
if target.mode not in (MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO):
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is not a sampled-stream target"
|
||||
)
|
||||
adopt_sampled_video = target.mode == MODE_VIDEO_AUDIO
|
||||
|
||||
validation = validate_second_pass_inputs(
|
||||
video_latents,
|
||||
audio_latents,
|
||||
assembly_plan,
|
||||
)
|
||||
selected_indices, selection_active = _normalize_selected_group_indices(
|
||||
selected_group_indices,
|
||||
int(validation["physical_group_count"]),
|
||||
)
|
||||
try:
|
||||
resolved_schedule = resolve_refine_schedule(sigmas, refine_schedule)
|
||||
except RefineScheduleError as exc:
|
||||
raise SecondPassContractError(
|
||||
f"refine schedule is invalid: {exc}"
|
||||
) from exc
|
||||
effective_sigmas = resolved_schedule.sigmas
|
||||
schedule_contract = serializable_schedule_contract(resolved_schedule)
|
||||
latent_builder = latent_builder or latent_from_cpu
|
||||
stream_extractor = stream_extractor or extract_av_streams
|
||||
refined_videos: list[dict[str, Any]] = (
|
||||
list(video_latents) if selection_active else []
|
||||
)
|
||||
refined_audios: list[dict[str, Any]] = (
|
||||
list(audio_latents) if selection_active else []
|
||||
)
|
||||
group_seeds: list[int] = []
|
||||
conditioning_sources: list[str] = []
|
||||
group_report_lines: list[str] = []
|
||||
|
||||
def sample_prepared_group(
|
||||
group_index,
|
||||
video_latent,
|
||||
group,
|
||||
chunk_model,
|
||||
conditioning,
|
||||
detail,
|
||||
):
|
||||
audio_latent = audio_latents[group_index]
|
||||
video_samples = _latent_samples(
|
||||
video_latent,
|
||||
name=f"video_latents[{group_index}]",
|
||||
)
|
||||
audio_samples = _latent_samples(
|
||||
audio_latent,
|
||||
name=f"audio_latents[{group_index}]",
|
||||
)
|
||||
physical_seed = derive_target_refine_seed(
|
||||
int(refine_seed),
|
||||
group_index,
|
||||
target,
|
||||
)
|
||||
group_seeds.append(physical_seed)
|
||||
conditioning_source = str(detail["conditioning_source"])
|
||||
conditioning_sources.append(conditioning_source)
|
||||
nested_latent = latent_builder(video_samples, audio_samples)
|
||||
sample_arguments = dict(
|
||||
model=chunk_model,
|
||||
conditioning=conditioning,
|
||||
latent=nested_latent,
|
||||
sampler=sampler,
|
||||
sigmas=effective_sigmas,
|
||||
seed=physical_seed,
|
||||
refine_target=target,
|
||||
enable_preview=bool(enable_preview),
|
||||
audio_only_sample_fn=sample_fn,
|
||||
video_audio_sample_fn=sample_fn,
|
||||
)
|
||||
if temporal_windows_by_group is not None:
|
||||
temporal_window = temporal_windows_by_group.get(group_index)
|
||||
if temporal_window is None:
|
||||
raise SecondPassContractError(
|
||||
f"selected physical group {group_index + 1} has no temporal window"
|
||||
)
|
||||
sample_arguments["temporal_window"] = temporal_window
|
||||
sampled = sample_targeted_refine_chunk(**sample_arguments)
|
||||
sampled_video, sampled_audio = stream_extractor(sampled)
|
||||
if tuple(sampled_video.shape) != tuple(video_samples.shape):
|
||||
raise SecondPassContractError(
|
||||
f"sampled video group {group_index} changed B/C/T/H/W geometry"
|
||||
)
|
||||
if tuple(sampled_audio.shape) != tuple(audio_samples.shape):
|
||||
raise SecondPassContractError(
|
||||
f"refined audio group {group_index} changed B/C/stereo/T geometry"
|
||||
)
|
||||
if not bool(torch.isfinite(sampled_audio.float()).all().item()):
|
||||
raise SecondPassContractError(
|
||||
f"refined audio group {group_index} contains NaN or Inf"
|
||||
)
|
||||
if adopt_sampled_video:
|
||||
if not bool(torch.isfinite(sampled_video.float()).all().item()):
|
||||
raise SecondPassContractError(
|
||||
f"refined video group {group_index} contains NaN or Inf"
|
||||
)
|
||||
output_video = dict(video_latent)
|
||||
output_video["samples"] = sampled_video
|
||||
if selection_active:
|
||||
refined_videos[group_index] = output_video
|
||||
else:
|
||||
refined_videos.append(output_video)
|
||||
output_audio = dict(audio_latent)
|
||||
output_audio["samples"] = sampled_audio
|
||||
if selection_active:
|
||||
refined_audios[group_index] = output_audio
|
||||
else:
|
||||
refined_audios.append(output_audio)
|
||||
adoption = (
|
||||
f"video_shape={tuple(sampled_video.shape)}, "
|
||||
"sampled_video_adopted=true, sampled_audio_adopted=true"
|
||||
if adopt_sampled_video
|
||||
else "temporary_video_discarded=true"
|
||||
)
|
||||
group_report_lines.append(
|
||||
"group "
|
||||
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
|
||||
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
|
||||
f"conditioning_source={conditioning_source}, "
|
||||
f"refine_seed={physical_seed}, "
|
||||
f"audio_shape={tuple(sampled_audio.shape)}, sampling_passes=1, "
|
||||
f"{adoption}"
|
||||
)
|
||||
|
||||
prepared = prepare_physical_refine_groups(
|
||||
model=model,
|
||||
clip=clip,
|
||||
video_latents=video_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
refine_context=refine_context,
|
||||
video_vae=video_vae,
|
||||
conditioning_upscale_method=conditioning_upscale_method,
|
||||
encode_prompt_fn=encode_prompt_fn,
|
||||
clone_model_fn=clone_model_fn,
|
||||
validate_refine_context_fn=validate_refine_context_fn,
|
||||
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
|
||||
group_consumer_fn=sample_prepared_group,
|
||||
retain_group_outputs=False,
|
||||
selected_group_indices=(selected_indices if selection_active else None),
|
||||
)
|
||||
expected_groups = int(validation["physical_group_count"])
|
||||
expected_sampled_groups = len(selected_indices)
|
||||
if len(group_seeds) != expected_sampled_groups:
|
||||
raise SecondPassContractError(
|
||||
"targeted Sampling count differs from selected physical group count"
|
||||
)
|
||||
if len(refined_audios) != expected_groups:
|
||||
raise SecondPassContractError(
|
||||
"targeted Audio output count differs from physical group count"
|
||||
)
|
||||
if adopt_sampled_video and len(refined_videos) != expected_groups:
|
||||
raise SecondPassContractError(
|
||||
"targeted Video output count differs from physical group count"
|
||||
)
|
||||
|
||||
output_videos = refined_videos if adopt_sampled_video else list(video_latents)
|
||||
updated_plan = prepared["updated_assembly_plan"]
|
||||
contract = updated_plan["second_pass_contract"]
|
||||
execution_target = "both_sampled" if adopt_sampled_video else "video_locked_audio"
|
||||
if conditioning_sources and all(
|
||||
source == "refine_context" for source in conditioning_sources
|
||||
):
|
||||
contract["execution"] = (
|
||||
f"context_aware_physical_groups_{execution_target}_v1"
|
||||
)
|
||||
elif any(source == "refine_context" for source in conditioning_sources):
|
||||
contract["execution"] = f"mixed_context_physical_groups_{execution_target}_v1"
|
||||
else:
|
||||
contract["execution"] = f"t2va_physical_groups_{execution_target}_v1"
|
||||
contract["conditioning_sources"] = conditioning_sources
|
||||
contract["refine_seed_base"] = int(refine_seed)
|
||||
contract["refine_group_seeds"] = group_seeds
|
||||
contract["video_output"] = (
|
||||
"sampled" if adopt_sampled_video else "bit_exact_first_pass_passthrough"
|
||||
)
|
||||
contract["video_sampling"] = (
|
||||
"seeded_random_mask_1" if adopt_sampled_video else "zero_noise_mask_locked"
|
||||
)
|
||||
contract["audio_output"] = "sampled"
|
||||
contract["audio_sampling"] = "seeded_random_mask_1"
|
||||
contract["refine_schedule"] = schedule_contract
|
||||
contract["refine_schedule_identity"] = schedule_contract["schedule_hash"]
|
||||
_attach_target_contract(updated_plan, target)
|
||||
|
||||
if adopt_sampled_video:
|
||||
target_line = (
|
||||
"Target: Video + Audio "
|
||||
"(GPU Experimental PASS / Production HOLD)."
|
||||
)
|
||||
sampling_line = (
|
||||
"Sampling contract: video seeded random noise/mask=1 and adopted "
|
||||
"sampled output; audio seeded random noise/mask=1 and adopted "
|
||||
"sampled output."
|
||||
)
|
||||
output_line = (
|
||||
"Outputs: sampled Video and sampled Audio adopted for every complete "
|
||||
"physical group."
|
||||
)
|
||||
else:
|
||||
target_line = "Target: Audio Only (Experimental / Production HOLD)."
|
||||
sampling_line = (
|
||||
"Sampling contract: video zero noise/mask=0 and discarded sampled "
|
||||
"output; audio seeded random noise/mask=1 and adopted sampled output."
|
||||
)
|
||||
output_line = (
|
||||
"Video output: original first-pass physical Video LATENT objects "
|
||||
"returned bit-exact; temporary sampled Video discarded."
|
||||
)
|
||||
report_lines = [
|
||||
"H3 Continuum Targeted Second Pass",
|
||||
target_line,
|
||||
f"Physical groups: {validation['physical_group_count']}.",
|
||||
sampling_line,
|
||||
"RefineSchedule: "
|
||||
f"schema={schedule_contract['schema_version']}, "
|
||||
f"mode={schedule_contract['mode']}, "
|
||||
f"evaluations={schedule_contract['evaluation_count']}, "
|
||||
f"sigma_hash={schedule_contract['sigma_hash']}.",
|
||||
*prepared["warnings"],
|
||||
*group_report_lines,
|
||||
output_line,
|
||||
]
|
||||
if selection_active:
|
||||
report_lines.insert(
|
||||
3,
|
||||
"Selected physical groups: "
|
||||
+ ", ".join(str(index + 1) for index in selected_indices)
|
||||
+ "; non-selected groups were exact object passthrough with no "
|
||||
"conditioning, MODEL clone, Sampling, or seed derivation.",
|
||||
)
|
||||
return output_videos, refined_audios, updated_plan, "\n".join(report_lines)
|
||||
|
||||
|
||||
def run_targeted_second_pass_groups(
|
||||
*,
|
||||
model: Any,
|
||||
clip: Any,
|
||||
sampler: Any,
|
||||
sigmas: torch.Tensor,
|
||||
video_latents: Sequence[Any],
|
||||
audio_latents: Sequence[Any],
|
||||
assembly_plan: Mapping[str, Any],
|
||||
refine_seed: int,
|
||||
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
|
||||
refine_context: Any = None,
|
||||
video_vae: Any = None,
|
||||
conditioning_upscale_method: str = "bilinear",
|
||||
enable_preview: bool = True,
|
||||
encode_prompt_fn=None,
|
||||
latent_builder=None,
|
||||
sample_fn=None,
|
||||
stream_extractor=None,
|
||||
clone_model_fn=None,
|
||||
validate_refine_context_fn=None,
|
||||
adapt_group_conditioning_fn=None,
|
||||
refine_schedule=None,
|
||||
refine_scope: str | None = SCOPE_ALL,
|
||||
refine_scope_index: int = 1,
|
||||
window_start_sec: float = 0.0,
|
||||
window_end_sec: float = 5.0,
|
||||
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
||||
"""Run an implemented target without changing the V3.5 public node."""
|
||||
|
||||
target = resolve_refine_target(refine_target)
|
||||
window: RefineWindow | None = None
|
||||
if isinstance(refine_scope, str) and refine_scope.strip().lower() == MODE_TIME_WINDOW:
|
||||
scope = None
|
||||
window = resolve_refine_window(
|
||||
window_start_sec,
|
||||
window_end_sec,
|
||||
assembly_plan,
|
||||
)
|
||||
selected_group_indices = window.selected_group_indices
|
||||
temporal_windows_by_group = {
|
||||
group.group_index: group for group in window.groups
|
||||
}
|
||||
else:
|
||||
scope = resolve_refine_scope(
|
||||
refine_scope,
|
||||
refine_scope_index,
|
||||
assembly_plan,
|
||||
)
|
||||
selected_group_indices = (
|
||||
None if scope.is_all else scope.selected_group_indices
|
||||
)
|
||||
temporal_windows_by_group = None
|
||||
arguments = dict(
|
||||
model=model,
|
||||
clip=clip,
|
||||
sampler=sampler,
|
||||
sigmas=sigmas,
|
||||
video_latents=video_latents,
|
||||
audio_latents=audio_latents,
|
||||
assembly_plan=assembly_plan,
|
||||
refine_seed=int(refine_seed),
|
||||
refine_context=refine_context,
|
||||
video_vae=video_vae,
|
||||
conditioning_upscale_method=str(conditioning_upscale_method),
|
||||
enable_preview=bool(enable_preview),
|
||||
encode_prompt_fn=encode_prompt_fn,
|
||||
latent_builder=latent_builder,
|
||||
stream_extractor=stream_extractor,
|
||||
clone_model_fn=clone_model_fn,
|
||||
validate_refine_context_fn=validate_refine_context_fn,
|
||||
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
|
||||
refine_schedule=refine_schedule,
|
||||
)
|
||||
if target.mode == MODE_VIDEO_ONLY:
|
||||
result = _run_video_only_groups(
|
||||
target=target,
|
||||
sample_fn=sample_fn,
|
||||
arguments=arguments,
|
||||
selected_group_indices=selected_group_indices,
|
||||
temporal_windows_by_group=temporal_windows_by_group,
|
||||
)
|
||||
elif target.mode == MODE_AUDIO_ONLY:
|
||||
result = _run_sampled_target_groups(
|
||||
target=target,
|
||||
sample_fn=sample_fn,
|
||||
selected_group_indices=selected_group_indices,
|
||||
temporal_windows_by_group=temporal_windows_by_group,
|
||||
**arguments,
|
||||
)
|
||||
elif target.mode == MODE_VIDEO_AUDIO:
|
||||
result = _run_sampled_target_groups(
|
||||
target=target,
|
||||
sample_fn=sample_fn,
|
||||
selected_group_indices=selected_group_indices,
|
||||
temporal_windows_by_group=temporal_windows_by_group,
|
||||
**arguments,
|
||||
)
|
||||
else:
|
||||
raise TargetedRefineExecutionUnavailable(
|
||||
f"refine target {target.mode!r} is unavailable"
|
||||
)
|
||||
|
||||
videos, audios, updated_plan, status = result
|
||||
if window is not None:
|
||||
contract = window.contract
|
||||
window_line = (
|
||||
"Refine Window: timeline=final_visible_output, interval=["
|
||||
f"{contract['effective_start_sec']:.6f}, "
|
||||
f"{contract['effective_end_sec']:.6f}) sec, output_frames="
|
||||
f"{contract['output_frame_range']}, selected_physical_groups="
|
||||
f"{contract['selected_physical_groups']}, continuation_prefix=protected, "
|
||||
"outside_window=mask_zero_then_bit_exact_restore, "
|
||||
"physical_group_atomicity=preserved."
|
||||
)
|
||||
return videos, audios, updated_plan, f"{status}\n{window_line}"
|
||||
assert scope is not None
|
||||
if scope.is_all:
|
||||
return videos, audios, updated_plan, status
|
||||
scope_contract = scope.contract
|
||||
scope_line = (
|
||||
"Refine Scope: "
|
||||
f"mode={scope.mode}, requested_index={scope_contract['requested_index']}, "
|
||||
"selected_physical_groups="
|
||||
f"{list(scope_contract['selected_physical_groups'])}, "
|
||||
"selected_logical_chunks="
|
||||
f"{list(scope_contract['selected_logical_chunks'])}, "
|
||||
f"terminal_expanded={str(scope_contract['terminal_expanded']).lower()}."
|
||||
)
|
||||
return videos, audios, updated_plan, f"{status}\n{scope_line}"
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Core VIDEO to H3 frame batches, with no file/upload/Enable frontend."""
|
||||
from __future__ import annotations
|
||||
|
||||
from fractions import Fraction
|
||||
|
||||
from .easy_video_nodes import ExecutionBlocker, _resample_video_frames, io
|
||||
|
||||
|
||||
class H3ContinuumVideoAdapter(io.ComfyNode):
|
||||
"""Reuse the established frame-rate conversion; audio is passed unchanged."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="H3ContinuumVideoAdapter",
|
||||
display_name="H3 Continuum Video Adapter",
|
||||
category="MiniMax H3/Continuum/Input",
|
||||
description=(
|
||||
"Converts Core Load Video's VIDEO to IMAGE frames and AUDIO. "
|
||||
"Use Force Rate 24 for H3 Video Guide. No file selector or Enable. "
|
||||
"Disable the whole video-input group, or bypass this adapter, "
|
||||
"to disconnect its optional Sampler inputs."
|
||||
),
|
||||
inputs=[
|
||||
io.Video.Input("video"),
|
||||
io.Float.Input(
|
||||
"force_rate", default=24.0, min=0.0, max=120.0, step=0.01,
|
||||
display_name="Force Rate",
|
||||
tooltip=(
|
||||
"24 resamples to H3's expected FPS without changing nominal "
|
||||
"duration or audio. 0 keeps source frames (not normally for H3)."
|
||||
),
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
io.Image.Output(display_name="images"),
|
||||
io.Audio.Output(display_name="audio"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, video, force_rate: float = 24.0):
|
||||
components = video.get_components()
|
||||
images = _resample_video_frames(
|
||||
components.images, Fraction(components.frame_rate), float(force_rate)
|
||||
)
|
||||
audio = components.audio
|
||||
if audio is None:
|
||||
audio = ExecutionBlocker(None)
|
||||
return io.NodeOutput(images, audio)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"H3ContinuumVideoAdapter": H3ContinuumVideoAdapter}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumVideoAdapter": "H3 Continuum Video Adapter",
|
||||
}
|
||||
+532
@@ -0,0 +1,532 @@
|
||||
"""Read-only decoded video-boundary analysis for H3 Continuum V3."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .plan import validate_assembly_plan
|
||||
|
||||
|
||||
COMPARE_LONG_SIDE = 192
|
||||
MAX_REWIND_FRAMES = 2
|
||||
FLASH_WINDOW_FRAMES = 6
|
||||
FLASH_TILE_GRID = 4
|
||||
MIN_FLASH_LUMA_SHIFT = 0.01
|
||||
MIN_FLASH_REVERSAL = 0.35
|
||||
MIN_FLASH_GLOBAL_FRACTION = 0.75
|
||||
MIN_ANOMALY_RATIO = 1.35
|
||||
MIN_RELATIVE_IMPROVEMENT = 0.06
|
||||
MIN_AUTO_FLASH_REVERSAL = 0.70
|
||||
MIN_AUTO_FLASH_IMPROVEMENT = 0.70
|
||||
SMALLER_SHIFT_TOLERANCE = 0.03
|
||||
MIN_MICRO_FLASH_LUMA_SHIFT = 0.003
|
||||
MIN_MICRO_FLASH_REVERSAL = 0.25
|
||||
MIN_MICRO_FLASH_GLOBAL_FRACTION = 0.70
|
||||
MAX_EXPOSURE_RAMP_REVERSAL = 0.25
|
||||
MIN_MOTION_CROSS_DELTA = 0.006
|
||||
MIN_MOTION_JUMP_RATIO = 1.60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VideoBoundaryAnalysis:
|
||||
boundary_index: int
|
||||
nominal_error: float
|
||||
baseline_error: float
|
||||
anomaly_ratio: float
|
||||
recommended_rewind: int
|
||||
best_error: float
|
||||
improvement: float
|
||||
luma_delta: float
|
||||
chroma_delta: float
|
||||
edge_delta: float
|
||||
scene_cut_score: float
|
||||
scene_cut: bool
|
||||
flash_luma_shift: float
|
||||
flash_reversal: float
|
||||
flash_global_fraction: float
|
||||
transient_flash_candidate: bool
|
||||
cross_frame_delta: float
|
||||
motion_baseline_delta: float
|
||||
motion_jump_ratio: float
|
||||
micro_flash_candidate: bool
|
||||
exposure_ramp_candidate: bool
|
||||
motion_hitch_candidate: bool
|
||||
classification: str
|
||||
reason: str
|
||||
|
||||
|
||||
def _validate_frames(frames: Any, label: str) -> None:
|
||||
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
|
||||
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
|
||||
if int(frames.shape[0]) < 1 or int(frames.shape[1]) < 1 or int(frames.shape[2]) < 1:
|
||||
raise ValueError(f"{label} must not be empty")
|
||||
|
||||
|
||||
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
|
||||
_validate_frames(frames, "video seam frames")
|
||||
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
|
||||
rgb = rgb.permute(0, 3, 1, 2)
|
||||
height, width = int(rgb.shape[-2]), int(rgb.shape[-1])
|
||||
scale = min(1.0, COMPARE_LONG_SIDE / float(max(height, width)))
|
||||
target_height = max(1, int(round(height * scale)))
|
||||
target_width = max(1, int(round(width * scale)))
|
||||
if (target_height, target_width) != (height, width):
|
||||
rgb = F.interpolate(
|
||||
rgb,
|
||||
size=(target_height, target_width),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
|
||||
if not bool(torch.isfinite(rgb).all().item()):
|
||||
raise ValueError("video seam comparison contains NaN or Inf")
|
||||
return rgb
|
||||
|
||||
|
||||
def _luma(frame: torch.Tensor) -> torch.Tensor:
|
||||
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
|
||||
return torch.sum(frame * weights, dim=-1)
|
||||
|
||||
|
||||
def _frame_delta(left: torch.Tensor, right: torch.Tensor) -> float:
|
||||
return float(torch.mean(torch.abs(left - right)).item())
|
||||
|
||||
|
||||
def _prediction_error(history: torch.Tensor, candidate: torch.Tensor) -> float:
|
||||
if int(history.shape[0]) < 3:
|
||||
raise ValueError("temporal prediction requires three history frames")
|
||||
recent_velocity = history[-1] - history[-2]
|
||||
prior_velocity = history[-2] - history[-3]
|
||||
predicted = (
|
||||
history[-1] + recent_velocity * 0.65 + prior_velocity * 0.35
|
||||
).clamp(0.0, 1.0)
|
||||
motion = torch.mean(torch.abs(recent_velocity), dim=-1)
|
||||
motion_scale = motion / motion.mean().clamp_min(1.0e-6)
|
||||
weights = (1.0 + motion_scale).clamp(1.0, 4.0).unsqueeze(-1)
|
||||
value = float(torch.mean((predicted - candidate).square() * weights).item())
|
||||
if not math.isfinite(value):
|
||||
raise ValueError("video seam prediction score is not finite")
|
||||
return max(0.0, value)
|
||||
|
||||
|
||||
def _baseline_prediction_error(previous: torch.Tensor) -> float:
|
||||
recent = previous[-min(8, int(previous.shape[0])) :]
|
||||
errors = [
|
||||
_prediction_error(recent[:index], recent[index])
|
||||
for index in range(3, int(recent.shape[0]))
|
||||
]
|
||||
if not errors:
|
||||
return 0.0
|
||||
return float(torch.tensor(errors, dtype=torch.float64).median().item())
|
||||
|
||||
|
||||
def _edge_delta(left: torch.Tensor, right: torch.Tensor) -> float:
|
||||
left_y = _luma(left)
|
||||
right_y = _luma(right)
|
||||
left_x = left_y[:, 1:] - left_y[:, :-1]
|
||||
right_x = right_y[:, 1:] - right_y[:, :-1]
|
||||
left_v = left_y[1:, :] - left_y[:-1, :]
|
||||
right_v = right_y[1:, :] - right_y[:-1, :]
|
||||
horizontal = torch.mean(torch.abs(left_x - right_x)) if left_x.numel() else left_y.new_tensor(0.0)
|
||||
vertical = torch.mean(torch.abs(left_v - right_v)) if left_v.numel() else left_y.new_tensor(0.0)
|
||||
return float(((horizontal + vertical) * 0.5).item())
|
||||
|
||||
|
||||
def _scene_cut_metrics(previous: torch.Tensor, current: torch.Tensor) -> tuple[float, bool]:
|
||||
cross_delta = _frame_delta(previous[-1], current[0])
|
||||
previous_internal = _frame_delta(previous[-2], previous[-1])
|
||||
if int(current.shape[0]) >= 3:
|
||||
current_internal = 0.5 * (
|
||||
_frame_delta(current[0], current[1])
|
||||
+ _frame_delta(current[1], current[2])
|
||||
)
|
||||
elif int(current.shape[0]) == 2:
|
||||
current_internal = _frame_delta(current[0], current[1])
|
||||
else:
|
||||
current_internal = cross_delta
|
||||
denominator = cross_delta + previous_internal + current_internal + 1.0e-8
|
||||
score = cross_delta / denominator
|
||||
persistent_new_scene = (
|
||||
int(current.shape[0]) >= 3
|
||||
and cross_delta >= 0.12
|
||||
and current_internal <= cross_delta * 0.35
|
||||
and cross_delta >= max(0.12, previous_internal * 3.0)
|
||||
)
|
||||
return max(0.0, min(1.0, score)), persistent_new_scene
|
||||
|
||||
|
||||
def _transient_flash_metrics(
|
||||
previous: torch.Tensor,
|
||||
current: torch.Tensor,
|
||||
) -> tuple[float, float, float]:
|
||||
"""Measure a broad boundary exposure pulse that quickly returns."""
|
||||
|
||||
previous_y = _luma(previous[-1])
|
||||
current_y = _luma(current[:FLASH_WINDOW_FRAMES])
|
||||
shifts = torch.mean(current_y - previous_y.unsqueeze(0), dim=(1, 2))
|
||||
initial_shift = float(torch.abs(shifts[0]).item())
|
||||
if int(shifts.shape[0]) > 1 and initial_shift > 1.0e-8:
|
||||
closest_later = float(torch.min(torch.abs(shifts[1:])).item())
|
||||
reversal = max(0.0, min(1.0, 1.0 - closest_later / initial_shift))
|
||||
else:
|
||||
reversal = 0.0
|
||||
|
||||
height, width = int(previous_y.shape[0]), int(previous_y.shape[1])
|
||||
tile_height = max(1, height // FLASH_TILE_GRID)
|
||||
tile_width = max(1, width // FLASH_TILE_GRID)
|
||||
delta = current_y[0] - previous_y
|
||||
sign = 1.0 if float(shifts[0].item()) >= 0.0 else -1.0
|
||||
tile_matches = []
|
||||
for top in range(0, height, tile_height):
|
||||
for left in range(0, width, tile_width):
|
||||
tile = delta[
|
||||
top : min(height, top + tile_height),
|
||||
left : min(width, left + tile_width),
|
||||
]
|
||||
tile_shift = float(torch.mean(tile).item())
|
||||
tile_matches.append(
|
||||
tile_shift * sign >= min(MIN_FLASH_LUMA_SHIFT * 0.5, initial_shift * 0.5)
|
||||
)
|
||||
global_fraction = sum(tile_matches) / max(1, len(tile_matches))
|
||||
return initial_shift, reversal, float(global_fraction)
|
||||
|
||||
|
||||
def _temporal_motion_metrics(
|
||||
previous: torch.Tensor,
|
||||
current: torch.Tensor,
|
||||
) -> tuple[float, float, float]:
|
||||
"""Compare the boundary transition with nearby native frame motion."""
|
||||
|
||||
cross_delta = _frame_delta(previous[-1], current[0])
|
||||
internal_deltas: list[float] = []
|
||||
previous_start = max(1, int(previous.shape[0]) - 5)
|
||||
for index in range(previous_start, int(previous.shape[0])):
|
||||
internal_deltas.append(_frame_delta(previous[index - 1], previous[index]))
|
||||
current_limit = min(5, int(current.shape[0]))
|
||||
for index in range(1, current_limit):
|
||||
internal_deltas.append(_frame_delta(current[index - 1], current[index]))
|
||||
if internal_deltas:
|
||||
baseline = float(
|
||||
torch.tensor(internal_deltas, dtype=torch.float64).median().item()
|
||||
)
|
||||
else:
|
||||
baseline = 0.0
|
||||
if cross_delta <= 1.0e-8:
|
||||
ratio = 0.0
|
||||
else:
|
||||
ratio = cross_delta / max(baseline, 1.0e-5)
|
||||
return cross_delta, baseline, ratio
|
||||
|
||||
|
||||
def analyze_video_boundary(
|
||||
previous_images: torch.Tensor,
|
||||
current_raw_images: torch.Tensor,
|
||||
*,
|
||||
trim_frames: int,
|
||||
boundary_index: int = 1,
|
||||
) -> VideoBoundaryAnalysis:
|
||||
"""Analyze one boundary without modifying either input tensor."""
|
||||
|
||||
_validate_frames(previous_images, "previous decoded image chunk")
|
||||
_validate_frames(current_raw_images, "current decoded image chunk")
|
||||
trim_frames = int(trim_frames)
|
||||
if int(previous_images.shape[0]) < 4:
|
||||
raise ValueError("video seam analysis requires four previous frames")
|
||||
if trim_frames < 0 or trim_frames >= int(current_raw_images.shape[0]):
|
||||
raise ValueError("video seam trim position is outside the current chunk")
|
||||
if trim_frames + 3 > int(current_raw_images.shape[0]):
|
||||
raise ValueError("video seam analysis requires three retained current frames")
|
||||
|
||||
previous = _downsample_rgb(previous_images[-8:])
|
||||
candidate_start = max(0, trim_frames - MAX_REWIND_FRAMES)
|
||||
current = _downsample_rgb(
|
||||
current_raw_images[
|
||||
candidate_start : trim_frames + FLASH_WINDOW_FRAMES
|
||||
]
|
||||
)
|
||||
nominal_index = trim_frames - candidate_start
|
||||
nominal = current[nominal_index]
|
||||
|
||||
candidate_errors: dict[int, float] = {}
|
||||
maximum_rewind = min(MAX_REWIND_FRAMES, trim_frames)
|
||||
for rewind in range(maximum_rewind + 1):
|
||||
candidate = current[nominal_index - rewind]
|
||||
candidate_errors[rewind] = _prediction_error(previous, candidate)
|
||||
|
||||
nominal_error = candidate_errors[0]
|
||||
minimum_error = min(candidate_errors.values())
|
||||
tolerance = minimum_error * SMALLER_SHIFT_TOLERANCE + 1.0e-10
|
||||
best_rewind = min(
|
||||
rewind
|
||||
for rewind, error in candidate_errors.items()
|
||||
if error <= minimum_error + tolerance
|
||||
)
|
||||
best_error = candidate_errors[best_rewind]
|
||||
baseline_error = _baseline_prediction_error(previous)
|
||||
anomaly_ratio = nominal_error / max(baseline_error, 1.0e-5)
|
||||
improvement = (nominal_error - best_error) / max(nominal_error, 1.0e-12)
|
||||
|
||||
previous_edge = previous[-1]
|
||||
previous_y = _luma(previous_edge)
|
||||
nominal_y = _luma(nominal)
|
||||
luma_delta = float(torch.mean(torch.abs(previous_y - nominal_y)).item())
|
||||
previous_chroma = previous_edge - previous_y.unsqueeze(-1)
|
||||
nominal_chroma = nominal - nominal_y.unsqueeze(-1)
|
||||
chroma_delta = float(
|
||||
torch.mean(torch.abs(previous_chroma - nominal_chroma)).item()
|
||||
)
|
||||
edge_delta = _edge_delta(previous_edge, nominal)
|
||||
scene_score, scene_cut = _scene_cut_metrics(
|
||||
previous,
|
||||
current[nominal_index : nominal_index + 3],
|
||||
)
|
||||
flash_luma_shift, flash_reversal, flash_global_fraction = (
|
||||
_transient_flash_metrics(previous, current[nominal_index:])
|
||||
)
|
||||
cross_frame_delta, motion_baseline_delta, motion_jump_ratio = (
|
||||
_temporal_motion_metrics(previous, current[nominal_index:])
|
||||
)
|
||||
transient_flash_candidate = (
|
||||
not scene_cut
|
||||
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
|
||||
and flash_reversal >= MIN_FLASH_REVERSAL
|
||||
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
micro_flash_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and flash_luma_shift >= MIN_MICRO_FLASH_LUMA_SHIFT
|
||||
and flash_reversal >= MIN_MICRO_FLASH_REVERSAL
|
||||
and flash_global_fraction >= MIN_MICRO_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
exposure_ramp_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and not micro_flash_candidate
|
||||
and anomaly_ratio >= MIN_ANOMALY_RATIO
|
||||
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
|
||||
and flash_reversal < MAX_EXPOSURE_RAMP_REVERSAL
|
||||
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
motion_hitch_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and not micro_flash_candidate
|
||||
and not exposure_ramp_candidate
|
||||
and cross_frame_delta >= MIN_MOTION_CROSS_DELTA
|
||||
and motion_jump_ratio >= MIN_MOTION_JUMP_RATIO
|
||||
)
|
||||
if scene_cut:
|
||||
classification = "scene_cut"
|
||||
elif transient_flash_candidate:
|
||||
classification = "transient_flash"
|
||||
elif micro_flash_candidate:
|
||||
classification = "micro_flash"
|
||||
elif exposure_ramp_candidate:
|
||||
classification = "exposure_ramp"
|
||||
elif motion_hitch_candidate:
|
||||
classification = "motion_hitch"
|
||||
else:
|
||||
classification = "clean_boundary"
|
||||
|
||||
recommended_rewind = 0
|
||||
if scene_cut:
|
||||
reason = "kept nominal boundary; persistent content change resembles a scene cut"
|
||||
elif best_rewind == 0:
|
||||
reason = "kept nominal boundary; it has the lowest prediction error"
|
||||
elif (
|
||||
transient_flash_candidate
|
||||
and flash_reversal >= MIN_AUTO_FLASH_REVERSAL
|
||||
and improvement >= MIN_AUTO_FLASH_IMPROVEMENT
|
||||
):
|
||||
recommended_rewind = best_rewind
|
||||
reason = f"mild transient flash recommends correcting {best_rewind} frame(s)"
|
||||
elif anomaly_ratio < MIN_ANOMALY_RATIO:
|
||||
reason = "kept nominal boundary; anomaly ratio is below the analysis threshold"
|
||||
elif improvement < MIN_RELATIVE_IMPROVEMENT:
|
||||
reason = "kept nominal boundary; candidate improvement is below 6%"
|
||||
else:
|
||||
recommended_rewind = best_rewind
|
||||
reason = f"analysis recommends shift {-best_rewind:+d} frame(s)"
|
||||
|
||||
return VideoBoundaryAnalysis(
|
||||
boundary_index=int(boundary_index),
|
||||
nominal_error=nominal_error,
|
||||
baseline_error=baseline_error,
|
||||
anomaly_ratio=anomaly_ratio,
|
||||
recommended_rewind=recommended_rewind,
|
||||
best_error=best_error,
|
||||
improvement=improvement,
|
||||
luma_delta=luma_delta,
|
||||
chroma_delta=chroma_delta,
|
||||
edge_delta=edge_delta,
|
||||
scene_cut_score=scene_score,
|
||||
scene_cut=scene_cut,
|
||||
flash_luma_shift=flash_luma_shift,
|
||||
flash_reversal=flash_reversal,
|
||||
flash_global_fraction=flash_global_fraction,
|
||||
transient_flash_candidate=transient_flash_candidate,
|
||||
cross_frame_delta=cross_frame_delta,
|
||||
motion_baseline_delta=motion_baseline_delta,
|
||||
motion_jump_ratio=motion_jump_ratio,
|
||||
micro_flash_candidate=micro_flash_candidate,
|
||||
exposure_ramp_candidate=exposure_ramp_candidate,
|
||||
motion_hitch_candidate=motion_hitch_candidate,
|
||||
classification=classification,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def analyze_decoded_boundaries(
|
||||
*,
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
) -> list[VideoBoundaryAnalysis]:
|
||||
"""Analyze every decoded chunk boundary without changing decoded content."""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
chunks = list(
|
||||
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
|
||||
)
|
||||
if len(images) != len(chunks):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
analyses: list[VideoBoundaryAnalysis] = []
|
||||
for index in range(1, len(chunks)):
|
||||
previous_raw = images[index - 1]
|
||||
current_raw = images[index]
|
||||
previous_plan = chunks[index - 1]
|
||||
current_plan = chunks[index]
|
||||
_validate_frames(previous_raw, f"decoded image chunk {index}")
|
||||
_validate_frames(current_raw, f"decoded image chunk {index + 1}")
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
current_segment = current_raw[:current_total]
|
||||
analyses.append(
|
||||
analyze_video_boundary(
|
||||
previous_segment,
|
||||
current_segment,
|
||||
trim_frames=current_trim,
|
||||
boundary_index=index,
|
||||
)
|
||||
)
|
||||
return analyses
|
||||
|
||||
|
||||
def correct_decoded_boundaries(
|
||||
*,
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
analyses: list[VideoBoundaryAnalysis],
|
||||
enable_exposure_ramp: bool = False,
|
||||
) -> tuple[list[Any], dict[int, str]]:
|
||||
"""Normalize qualified transient exposure shifts without changing motion."""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
chunks = list(
|
||||
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
|
||||
)
|
||||
if len(images) != len(chunks):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
corrected = list(images)
|
||||
actions: dict[int, str] = {}
|
||||
for analysis in analyses:
|
||||
boundary = int(analysis.boundary_index)
|
||||
correction_kind = "transient flash"
|
||||
replace_frames = int(analysis.recommended_rewind)
|
||||
if analysis.micro_flash_candidate:
|
||||
correction_kind = "micro-flash"
|
||||
replace_frames = 1
|
||||
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
|
||||
correction_kind = "exposure ramp"
|
||||
replace_frames = 4
|
||||
if replace_frames <= 0:
|
||||
actions[boundary] = "kept native boundary"
|
||||
continue
|
||||
if analysis.scene_cut or not (
|
||||
analysis.transient_flash_candidate
|
||||
or analysis.micro_flash_candidate
|
||||
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
|
||||
):
|
||||
actions[boundary] = "kept native boundary; correction guard rejected candidate"
|
||||
continue
|
||||
previous_index = boundary - 1
|
||||
current_index = boundary
|
||||
if previous_index < 0 or current_index >= len(images):
|
||||
raise ValueError("video seam boundary index is outside decoded chunks")
|
||||
previous_raw = images[previous_index]
|
||||
current_raw = images[current_index]
|
||||
_validate_frames(previous_raw, f"decoded image chunk {previous_index + 1}")
|
||||
_validate_frames(current_raw, f"decoded image chunk {current_index + 1}")
|
||||
previous_plan = chunks[previous_index]
|
||||
current_plan = chunks[current_index]
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
if current_trim + replace_frames >= current_total:
|
||||
raise ValueError("video seam correction has no retained recovery frame")
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
if int(previous_segment.shape[0]) < 1:
|
||||
raise ValueError("video seam correction has no previous retained frame")
|
||||
current_cpu = current_raw.detach().to(device="cpu").clone()
|
||||
previous_anchor = previous_segment[-1].detach().to(
|
||||
device="cpu", dtype=torch.float32
|
||||
)
|
||||
recovery_anchor = current_cpu[current_trim + replace_frames].to(
|
||||
dtype=torch.float32
|
||||
)
|
||||
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
|
||||
raise ValueError("decoded image geometry changed at video seam")
|
||||
for offset in range(replace_frames):
|
||||
alpha = float(offset + 1) / float(replace_frames + 1)
|
||||
original = current_cpu[current_trim + offset].to(dtype=torch.float32)
|
||||
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
|
||||
original_mean = original.mean(dim=(0, 1), keepdim=True)
|
||||
target_mean = target.mean(dim=(0, 1), keepdim=True)
|
||||
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
|
||||
current_cpu[current_trim + offset].copy_(
|
||||
normalized.to(dtype=current_cpu.dtype)
|
||||
)
|
||||
corrected[current_index] = current_cpu
|
||||
actions[boundary] = (
|
||||
f"normalized exposure/color on {replace_frames} {correction_kind} "
|
||||
"boundary frame(s)"
|
||||
)
|
||||
return corrected, actions
|
||||
|
||||
|
||||
def format_video_boundary_analysis(
|
||||
analysis: VideoBoundaryAnalysis,
|
||||
*,
|
||||
action: str = "analysis only",
|
||||
) -> str:
|
||||
shift = -int(analysis.recommended_rewind)
|
||||
return (
|
||||
f"video seam {analysis.boundary_index}->{analysis.boundary_index + 1}: "
|
||||
f"anomaly={analysis.anomaly_ratio:.3f}, "
|
||||
f"prediction={analysis.nominal_error:.6f}->{analysis.best_error:.6f}, "
|
||||
f"improvement={analysis.improvement:.1%}, shift={shift:+d}, "
|
||||
f"luma={analysis.luma_delta:.6f}, chroma={analysis.chroma_delta:.6f}, "
|
||||
f"edge={analysis.edge_delta:.6f}, "
|
||||
f"scene_cut={'yes' if analysis.scene_cut else 'no'} "
|
||||
f"flash_shift={analysis.flash_luma_shift:.6f}, "
|
||||
f"flash_reversal={analysis.flash_reversal:.1%}, "
|
||||
f"flash_global={analysis.flash_global_fraction:.1%}, "
|
||||
f"flash_candidate={'yes' if analysis.transient_flash_candidate else 'no'} "
|
||||
f"class={analysis.classification}, "
|
||||
f"cross={analysis.cross_frame_delta:.6f}, "
|
||||
f"motion_base={analysis.motion_baseline_delta:.6f}, "
|
||||
f"motion_jump={analysis.motion_jump_ratio:.2f}x, "
|
||||
f"micro_flash={'yes' if analysis.micro_flash_candidate else 'no'}, "
|
||||
f"exposure_ramp={'yes' if analysis.exposure_ramp_candidate else 'no'}, "
|
||||
f"motion_hitch={'yes' if analysis.motion_hitch_candidate else 'no'} "
|
||||
f"({analysis.reason}); action={action}"
|
||||
)
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""Small decoded-boundary patches for the V3.5 direct-write assembler.
|
||||
|
||||
The established V3.4 seam analyzer remains the source of all boundary
|
||||
decisions. This module reproduces only the correction step while avoiding the
|
||||
full physical-group clone performed by ``correct_decoded_boundaries``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .plan import validate_assembly_plan
|
||||
from .video_seam import VideoBoundaryAnalysis, _validate_frames
|
||||
|
||||
|
||||
def build_boundary_patches(
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
analyses: list[VideoBoundaryAnalysis],
|
||||
*,
|
||||
enable_exposure_ramp: bool = False,
|
||||
) -> tuple[dict[int, torch.Tensor], dict[int, str]]:
|
||||
"""Return qualified 1--4 frame corrections without changing ``images``.
|
||||
|
||||
Patch keys are the existing one-based ``boundary_index`` values. They are
|
||||
also the zero-based index of the current physical decode group, so the
|
||||
direct writer can add its natural-frame cursor without this module knowing
|
||||
the final buffer layout.
|
||||
|
||||
The result is transactional: all patches and actions are built in local
|
||||
containers and are returned only after every requested boundary succeeds.
|
||||
Any exception therefore leaves the caller with no partial correction set.
|
||||
"""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
groups = list(
|
||||
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
|
||||
)
|
||||
if len(images) != len(groups):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
|
||||
patches: dict[int, torch.Tensor] = {}
|
||||
actions: dict[int, str] = {}
|
||||
for analysis in analyses:
|
||||
boundary = int(analysis.boundary_index)
|
||||
correction_kind = "transient flash"
|
||||
replace_frames = int(analysis.recommended_rewind)
|
||||
if analysis.micro_flash_candidate:
|
||||
correction_kind = "micro-flash"
|
||||
replace_frames = 1
|
||||
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
|
||||
correction_kind = "exposure ramp"
|
||||
replace_frames = 4
|
||||
|
||||
if replace_frames <= 0:
|
||||
actions[boundary] = "kept native boundary"
|
||||
continue
|
||||
if analysis.scene_cut or not (
|
||||
analysis.transient_flash_candidate
|
||||
or analysis.micro_flash_candidate
|
||||
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
|
||||
):
|
||||
actions[boundary] = (
|
||||
"kept native boundary; correction guard rejected candidate"
|
||||
)
|
||||
continue
|
||||
|
||||
previous_index = boundary - 1
|
||||
current_index = boundary
|
||||
if previous_index < 0 or current_index >= len(images):
|
||||
raise ValueError("video seam boundary index is outside decoded chunks")
|
||||
|
||||
previous_raw = images[previous_index]
|
||||
current_raw = images[current_index]
|
||||
_validate_frames(
|
||||
previous_raw,
|
||||
f"decoded image chunk {previous_index + 1}",
|
||||
)
|
||||
_validate_frames(
|
||||
current_raw,
|
||||
f"decoded image chunk {current_index + 1}",
|
||||
)
|
||||
previous_plan = groups[previous_index]
|
||||
current_plan = groups[current_index]
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
if current_trim + replace_frames >= current_total:
|
||||
raise ValueError("video seam correction has no retained recovery frame")
|
||||
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
if int(previous_segment.shape[0]) < 1:
|
||||
raise ValueError("video seam correction has no previous retained frame")
|
||||
|
||||
# Only the frames that may be changed are cloned. The recovery frame
|
||||
# and previous anchor remain read-only views of their physical groups.
|
||||
patch = current_raw[
|
||||
current_trim : current_trim + replace_frames
|
||||
].detach().to(device="cpu").clone()
|
||||
if int(patch.shape[0]) != replace_frames:
|
||||
raise ValueError("video seam correction frames are outside the current chunk")
|
||||
previous_anchor = previous_segment[-1].detach().to(
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
recovery_anchor = current_raw[current_trim + replace_frames].detach().to(
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
|
||||
raise ValueError("decoded image geometry changed at video seam")
|
||||
|
||||
for offset in range(replace_frames):
|
||||
alpha = float(offset + 1) / float(replace_frames + 1)
|
||||
original = patch[offset].to(dtype=torch.float32)
|
||||
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
|
||||
original_mean = original.mean(dim=(0, 1), keepdim=True)
|
||||
target_mean = target.mean(dim=(0, 1), keepdim=True)
|
||||
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
|
||||
patch[offset].copy_(normalized.to(dtype=patch.dtype))
|
||||
|
||||
patches[boundary] = patch
|
||||
actions[boundary] = (
|
||||
f"normalized exposure/color on {replace_frames} {correction_kind} "
|
||||
"boundary frame(s)"
|
||||
)
|
||||
|
||||
return patches, actions
|
||||
Reference in New Issue
Block a user