Merge commit 'bf8f76505fdd25ea99770f262eb9371509bf226d' as 'vendor/dumas_h3_continuum'

This commit is contained in:
2026-10-20 13:43:38 +00:00
481 changed files with 131517 additions and 0 deletions
+5
View File
@@ -0,0 +1,5 @@
"""Latent-first H3 Continuum V3 nodes."""
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+320
View File
@@ -0,0 +1,320 @@
"""Read-only Adaptive Continuity observation and decision trace for A8a.
The observer records the continuation contract that Production already chose.
It never changes context depth, transport, masks, tensors, conditioning, MODEL,
Seed, SIGMAS, physical grouping, or execution policy.
"""
from __future__ import annotations
import hashlib
import json
import math
from typing import Any
ADAPTIVE_CONTINUITY_OBSERVER_VERSION = 1
EXECUTION_APPLIED = False
INITIAL_TRANSPORT = "initial"
REFERENCE_CONTEXT_V1 = "reference_context_v1"
MASKED_VIDEO_PREFIX_V1 = "masked_video_prefix_v1"
MASKED_AV_PREFIX_22_V1 = "masked_av_prefix_22_v1"
MASKED_AV_PREFIX_39_V1 = "masked_av_prefix_39_v1"
_KNOWN_TRANSPORTS = frozenset(
(
INITIAL_TRANSPORT,
REFERENCE_CONTEXT_V1,
MASKED_VIDEO_PREFIX_V1,
MASKED_AV_PREFIX_22_V1,
MASKED_AV_PREFIX_39_V1,
)
)
_VALID_CONTEXT_FRAMES = frozenset((0, 5, 22, 39))
_PLANNER_CONTRACT = {
"planner_version": ADAPTIVE_CONTINUITY_OBSERVER_VERSION,
"mode": "read_only_observer",
"execution_applied": EXECUTION_APPLIED,
"confidence_scope": "observation_completeness_not_mitigation_efficacy",
"issue_13_mitigation_claim": False,
"mutations": [],
"observed_fields": [
"motion_score",
"current_context_frames",
"resolved_transport",
"terminal_atomic",
"boundary_index",
"boundary_count",
],
}
def _canonical_hash(value: Any) -> str:
payload = json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
ADAPTIVE_CONTINUITY_PLANNER_HASH = _canonical_hash(_PLANNER_CONTRACT)
class AdaptiveContinuityObservationError(ValueError):
"""One diagnostic observation is malformed or internally inconsistent."""
def _recommendation(
*,
current_context_frames: int,
resolved_transport: str,
terminal_atomic: bool,
) -> str:
if int(current_context_frames) == 0:
return "no_continuation_action_initial_group"
if bool(terminal_atomic):
return "keep_fixed_terminal_merge_contract"
if str(resolved_transport) == REFERENCE_CONTEXT_V1:
return "keep_current_reference_context_contract"
return "keep_current_masked_prefix_contract"
def make_adaptive_continuity_decision(
*,
physical_group: int,
logical_chunks: tuple[int, ...],
observed_motion_score: float,
current_context_frames: int,
resolved_transport: str,
terminal_atomic: bool,
boundary_index: int,
boundary_count: int,
reason: str,
fallback_reason: str | None = None,
reused: bool = False,
) -> dict[str, Any]:
"""Build one immutable, JSON-safe observation without retaining tensors."""
group = int(physical_group)
logical = tuple(int(value) for value in logical_chunks)
context = int(current_context_frames)
boundary = int(boundary_index)
boundaries = int(boundary_count)
transport = str(resolved_transport)
motion = float(observed_motion_score)
reason_text = str(reason).strip()
if group < 1 or not logical or any(value < 1 for value in logical):
raise AdaptiveContinuityObservationError(
"physical group and logical chunks must be positive"
)
if tuple(sorted(set(logical))) != logical:
raise AdaptiveContinuityObservationError(
"logical chunks must be ordered and unique"
)
if context not in _VALID_CONTEXT_FRAMES:
raise AdaptiveContinuityObservationError(
f"unsupported observed context frame count: {context}"
)
if transport not in _KNOWN_TRANSPORTS:
raise AdaptiveContinuityObservationError(
f"unknown resolved continuation transport: {transport!r}"
)
if context == 0 and transport != INITIAL_TRANSPORT:
raise AdaptiveContinuityObservationError(
"zero-context observations must use the initial transport marker"
)
if context > 0 and transport == INITIAL_TRANSPORT:
raise AdaptiveContinuityObservationError(
"continuation observations require a resolved transport"
)
if boundary < 0 or boundaries < 0 or boundary > boundaries:
raise AdaptiveContinuityObservationError(
"boundary index/count are inconsistent"
)
if not math.isfinite(motion) or motion < 0.0:
raise AdaptiveContinuityObservationError(
"observed motion score must be finite and non-negative"
)
confidence = 1.0 if reason_text else 0.5
payload: dict[str, Any] = {
"planner_version": ADAPTIVE_CONTINUITY_OBSERVER_VERSION,
"planner_hash": ADAPTIVE_CONTINUITY_PLANNER_HASH,
"physical_group": group,
"logical_chunks": list(logical),
"observed_motion_score": motion,
"current_context_frames": context,
"resolved_transport": transport,
"terminal_atomic": bool(terminal_atomic),
"boundary_index": boundary,
"boundary_count": boundaries,
"recommendation": _recommendation(
current_context_frames=context,
resolved_transport=transport,
terminal_atomic=bool(terminal_atomic),
),
"confidence": confidence,
"confidence_scope": "observation completeness only; not mitigation efficacy",
"reason": reason_text or "selection reason unavailable",
"fallback_reason": (
str(fallback_reason).strip() if fallback_reason is not None else None
),
"reused": bool(reused),
"execution_applied": EXECUTION_APPLIED,
"issue_13_mitigation_claim": False,
}
payload["decision_hash"] = _canonical_hash(payload)
return payload
class AdaptiveContinuityObserver:
"""Collect fail-soft A8a observations in deterministic physical-group order."""
def __init__(self) -> None:
self._records: list[dict[str, Any]] = []
self._advisories: list[str] = []
@property
def records(self) -> tuple[dict[str, Any], ...]:
return tuple(dict(record) for record in self._records)
def observe(self, **kwargs: Any) -> dict[str, Any]:
record = make_adaptive_continuity_decision(**kwargs)
self._records.append(record)
return record
def note_observation_failure(self, exc: Exception) -> None:
self._advisories.append(f"{type(exc).__name__}: {exc}")
def replay_prefix(
self,
*,
entries: list[dict[str, Any]],
total_chunks: int,
terminal_merge_enabled: bool,
resolved_transport: str,
) -> None:
"""Reconstruct trace-only records from accepted CPU chunk metadata."""
chunk_count = int(total_chunks)
completed = len(entries)
if chunk_count < 1 or completed > chunk_count:
raise AdaptiveContinuityObservationError(
"replayed prefix length is outside the configured sequence"
)
boundary_count = max(0, chunk_count - 1)
normal_count = completed
terminal_complete = False
pair_start = max(0, chunk_count - 2)
if bool(terminal_merge_enabled):
if pair_start < completed < chunk_count:
raise AdaptiveContinuityObservationError(
"terminal merged prefix contains only one logical half"
)
terminal_complete = completed == chunk_count
normal_count = pair_start if terminal_complete else completed
for index in range(normal_count):
entry = entries[index]
context = int(entry.get("context_frames", 0))
self.observe(
physical_group=index + 1,
logical_chunks=(index + 1,),
observed_motion_score=float(entry.get("motion_score", 0.0)),
current_context_frames=context,
resolved_transport=(
INITIAL_TRANSPORT if context == 0 else str(resolved_transport)
),
terminal_atomic=False,
boundary_index=index,
boundary_count=boundary_count,
reason="replayed from accepted chunk metadata",
fallback_reason=None,
reused=True,
)
if terminal_complete:
entry = entries[pair_start]
context = int(entry.get("context_frames", 0))
self.observe(
physical_group=pair_start + 1,
logical_chunks=(pair_start + 1, chunk_count),
observed_motion_score=float(entry.get("motion_score", 0.0)),
current_context_frames=context,
resolved_transport=(
INITIAL_TRANSPORT if context == 0 else str(resolved_transport)
),
terminal_atomic=True,
boundary_index=pair_start,
boundary_count=boundary_count,
reason="replayed from accepted Terminal Merge metadata",
fallback_reason=None,
reused=True,
)
def report_lines(self) -> list[str]:
lines = [
"Adaptive Continuity observer [A8a v1]: "
f"planner_hash={ADAPTIVE_CONTINUITY_PLANNER_HASH}, "
f"observed_groups={len(self._records)}, "
"execution_applied=false; recommendation trace only, "
"Issue #13 mitigation claim=none, Production continuity unchanged."
]
for record in self._records:
lines.append(
"Adaptive Continuity decision [A8a v1]: "
f"physical_group={record['physical_group']}, "
f"logical_chunks={record['logical_chunks']}, "
f"boundary={record['boundary_index']}/{record['boundary_count']}, "
f"motion={record['observed_motion_score']:.6f}, "
f"context={record['current_context_frames']}, "
f"transport={record['resolved_transport']}, "
f"terminal_atomic={str(record['terminal_atomic']).lower()}, "
f"reused={str(record['reused']).lower()}, "
f"recommendation={record['recommendation']}, "
f"confidence={record['confidence']:.2f}, "
f"reason={record['reason']}, "
f"fallback_reason={record['fallback_reason'] or 'none'}, "
f"decision_hash={record['decision_hash']}, "
"execution_applied=false."
)
lines.extend(
f"Adaptive Continuity advisory [A8a v1]: {item}; execution_applied=false."
for item in self._advisories
)
return lines
def adaptive_observer_event_fail_soft(
observer: Any,
method: str,
**kwargs: Any,
) -> Any:
"""Invoke one observer event without allowing diagnostics to stop Sampling."""
if observer is None:
return None
try:
return getattr(observer, str(method))(**kwargs)
except Exception as exc:
try:
observer.note_observation_failure(exc)
except Exception:
pass
return None
__all__ = [
"ADAPTIVE_CONTINUITY_OBSERVER_VERSION",
"ADAPTIVE_CONTINUITY_PLANNER_HASH",
"AdaptiveContinuityObservationError",
"AdaptiveContinuityObserver",
"EXECUTION_APPLIED",
"INITIAL_TRANSPORT",
"adaptive_observer_event_fail_soft",
"make_adaptive_continuity_decision",
]
+431
View File
@@ -0,0 +1,431 @@
"""Decoded chunk assembly for latent-first H3 Continuum V3."""
from __future__ import annotations
import math
import time
from typing import Any
import torch
from ..media import validate_audio
from ..v2.decoder import _slice_audio_for_timeline, enforce_total_frames
from ..constants import (
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
DIAGNOSTICS_OPTIONS,
normalize_diagnostics_mode,
)
from ..v2.seam_guard import correct_audio_seam
from .plan import FPS, validate_assembly_plan
AUDIO_SEAM_OFF = "Off"
AUDIO_SEAM_AUTO = "Auto"
AUDIO_SEAM_OPTIONS = (AUDIO_SEAM_OFF, AUDIO_SEAM_AUTO)
VIDEO_SEAM_OFF = "Off"
VIDEO_SEAM_ANALYZE = "Analyze Only"
VIDEO_SEAM_AUTO = "Auto"
VIDEO_SEAM_AUTO_2 = "Auto 2"
VIDEO_SEAM_ANALYSIS_OPTIONS = (
VIDEO_SEAM_AUTO,
VIDEO_SEAM_AUTO_2,
VIDEO_SEAM_ANALYZE,
VIDEO_SEAM_OFF,
)
def _singleton(value: Any, name: str) -> Any:
while isinstance(value, list):
if len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
value = value[0]
return value
def _chunk_list(value: Any) -> list[Any]:
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], list):
return list(value[0])
return list(value)
return [value]
def _format_elapsed(seconds: float) -> str:
total = max(0.0, float(seconds))
hours, remainder = divmod(total, 3600.0)
minutes, seconds = divmod(remainder, 60.0)
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
def assemble_decoded_chunks(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
diagnostics: str,
):
plan = validate_assembly_plan(assembly_plan)
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
if audio_seam not in AUDIO_SEAM_OPTIONS:
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
chunk_plans = list(
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
)
if len(images) != len(chunk_plans) or len(audio) != len(chunk_plans):
raise ValueError(
"decoded chunk count does not match the V3 assembly plan: "
f"images={len(images)}, audio={len(audio)}, plan={len(chunk_plans)}"
)
total_retained_frames = sum(int(item["net_frames"]) for item in chunk_plans)
image_buffer = None
audio_buffer = None
audio_rate = None
frame_cursor = 0
reports = [
"H3 Continuum Assemble V3",
f"Audio Seam: {audio_seam}. Video seam correction is disabled.",
]
for index, (raw_images, raw_audio, chunk) in enumerate(
zip(images, audio, chunk_plans), start=1
):
if not torch.is_tensor(raw_images) or raw_images.ndim != 4:
raise ValueError(f"decoded image chunk {index} must be [frames,H,W,C]")
raw_images = raw_images.detach().to("cpu")
total_frames = int(chunk["total_frames"])
trim_frames = int(chunk["trim_frames"])
net_frames = int(chunk["net_frames"])
if int(raw_images.shape[0]) < total_frames:
raise ValueError(
f"decoded image chunk {index} has {raw_images.shape[0]} frames; "
f"expected at least {total_frames}"
)
segment_images = raw_images[:total_frames][trim_frames:].contiguous()
if int(segment_images.shape[0]) != net_frames:
raise ValueError(
f"decoded image chunk {index} retained {segment_images.shape[0]} frames; "
f"plan expects {net_frames}"
)
waveform, sample_rate = validate_audio(raw_audio)
waveform = waveform.detach().to("cpu")
sample_rate = int(sample_rate)
raw_audio_cpu = {"waveform": waveform, "sample_rate": sample_rate}
if audio_rate is None:
audio_rate = sample_rate
elif sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
frame_stop = frame_cursor + net_frames
segment_waveform = _slice_audio_for_timeline(
raw_audio_cpu,
trim_frames=trim_frames,
frame_start=frame_cursor,
frame_stop=frame_stop,
)
if image_buffer is None:
image_buffer = torch.empty(
(total_retained_frames, *segment_images.shape[1:]),
dtype=segment_images.dtype,
device="cpu",
)
elif tuple(segment_images.shape[1:]) != tuple(image_buffer.shape[1:]):
raise ValueError(f"decoded image geometry changed at chunk {index}")
image_buffer[frame_cursor:frame_stop].copy_(segment_images)
if audio_buffer is None:
audio_buffer = torch.empty(
(
*waveform.shape[:-1],
int(round(total_retained_frames / FPS * sample_rate)),
),
dtype=waveform.dtype,
device="cpu",
)
elif tuple(segment_waveform.shape[:-1]) != tuple(audio_buffer.shape[:-1]):
raise ValueError(
"decoded audio batch/channel structure changed between chunks"
)
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
seam_report = None
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
try:
cut_sample = int(round(trim_frames / FPS * sample_rate))
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
if patch is not None and int(patch.shape[-1]) > 0:
patch_start = sample_start - int(patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(patch)
seam_report = (
f"audio seam {index-1}->{index}: "
f"corr={metrics.correlation_before:.4f}->{metrics.correlation_after:.4f}, "
f"jump={metrics.boundary_jump_before:.6f}->{metrics.boundary_jump_after:.6f}, "
f"fade={fade_samples} samples"
)
if diagnostics_mode == DIAGNOSTICS_FULL:
seam_report += (
f", offset={metrics.offset_samples:+d}, "
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
)
except Exception as exc:
seam_report = (
f"audio seam {index-1}->{index}: fallback to native boundary "
f"({type(exc).__name__}: {exc})"
)
audio_buffer[..., sample_start:sample_stop].copy_(segment_waveform)
if diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
f"assembled decoded chunk {index}: {net_frames} retained frames, "
f"cumulative {frame_stop}/{total_retained_frames}"
)
if seam_report:
reports.append(seam_report)
frame_cursor = frame_stop
if image_buffer is None or audio_buffer is None or audio_rate is None:
raise RuntimeError("V3 assembler failed to allocate output buffers")
if frame_cursor != total_retained_frames:
raise RuntimeError(
f"V3 assembly cursor mismatch: {frame_cursor} != {total_retained_frames}"
)
result_audio = {
"waveform": audio_buffer.contiguous(),
"sample_rate": audio_rate,
}
duration_report = ""
if exact_total_duration:
image_buffer, result_audio, duration_report = enforce_total_frames(
image_buffer.contiguous(),
result_audio,
target_frames=int(plan["target_frames"]),
preserve_final_frame=bool(plan.get("preserve_final_frame")),
)
reports.append(
f"Assembled {len(chunk_plans)} externally decoded chunks into "
f"{image_buffer.shape[0]} frames with cumulative sample-boundary alignment."
)
if duration_report:
reports.append(duration_report)
runtime_started_at = plan.get("_runtime_started_at")
if isinstance(runtime_started_at, (int, float)):
elapsed = time.perf_counter() - float(runtime_started_at)
if math.isfinite(elapsed) and elapsed >= 0.0:
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
return image_buffer.contiguous(), result_audio, "\n".join(reports)
class H3ContinuumAssembleV3:
DESCRIPTION = (
"Assemble full AV chunks decoded by ComfyUI Core. Trims decoded context, "
"aligns audio on cumulative frame boundaries, and optionally applies Audio Seam Auto."
)
SEARCH_ALIASES = ["H3 latent assemble", "H3 external VAE decode"]
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"images": ("IMAGE",),
"audio": ("AUDIO",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"exact_total_duration": (
"BOOLEAN",
{
"default": True,
"tooltip": "Adjust the final output to the exact requested total duration.",
},
),
"audio_seam": (
AUDIO_SEAM_OPTIONS,
{
"default": AUDIO_SEAM_AUTO,
"display_name": "Audio Seam",
"tooltip": (
"Auto corrects only decoded audio boundaries; "
"video frames are never altered."
),
},
),
"diagnostics": (
DIAGNOSTICS_OPTIONS,
{
"default": DIAGNOSTICS_OPTIONS[0],
"display_name": "Report Detail",
},
),
}
}
INPUT_IS_LIST = True
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING")
RETURN_NAMES = ("images", "audio", "report")
FUNCTION = "assemble"
CATEGORY = "MiniMax H3/Continuum"
def assemble(
self,
images,
audio,
assembly_plan,
exact_total_duration,
audio_seam,
diagnostics,
):
return assemble_decoded_chunks(
images=_chunk_list(images),
audio=_chunk_list(audio),
assembly_plan=_singleton(assembly_plan, "assembly_plan"),
exact_total_duration=bool(
_singleton(exact_total_duration, "exact_total_duration")
),
audio_seam=str(_singleton(audio_seam, "audio_seam")),
diagnostics=str(_singleton(diagnostics, "diagnostics")),
)
# V3.0.1 hardening integration: preflight before allocation; Detailed Report only.
from ..hardening import assemble_with_hardening as _assemble_with_hardening
_assemble_decoded_chunks_v300 = assemble_decoded_chunks
def assemble_decoded_chunks(*args, **kwargs):
return _assemble_with_hardening(_assemble_decoded_chunks_v300, args, kwargs)
class H3ContinuumAssembleSeamExperimental(H3ContinuumAssembleV3):
DEPRECATED = False
CATEGORY = "MiniMax H3/Continuum"
DESCRIPTION = (
"Analyze decoded chunk boundaries and apply guarded video seam correction."
)
SEARCH_ALIASES = ["H3 seam analysis", "H3 boundary flicker analysis"]
@classmethod
def INPUT_TYPES(cls):
schema = super().INPUT_TYPES()
required = dict(schema["required"])
diagnostics = required.pop("diagnostics")
required["video_seam"] = (
VIDEO_SEAM_ANALYSIS_OPTIONS,
{
"default": VIDEO_SEAM_AUTO,
"display_name": "Video Seam",
"tooltip": (
"Analyze Only reports decoded video boundaries without changing them. "
"Auto applies the validated transient and micro-flash correction. "
"Auto 2 experimentally adds qualified exposure-ramp smoothing."
),
},
)
required["diagnostics"] = diagnostics
schema["required"] = required
return schema
def assemble(
self,
images,
audio,
assembly_plan,
exact_total_duration,
audio_seam,
video_seam,
diagnostics,
):
mode = str(_singleton(video_seam, "video_seam"))
if mode not in VIDEO_SEAM_ANALYSIS_OPTIONS:
raise ValueError(f"unknown Video Seam mode: {mode!r}")
if mode == VIDEO_SEAM_OFF:
return super().assemble(
images,
audio,
assembly_plan,
exact_total_duration,
audio_seam,
diagnostics,
)
image_chunks = _chunk_list(images)
plan = _singleton(assembly_plan, "assembly_plan")
analyses = []
analysis_error = None
actions = {}
try:
from .video_seam import (
analyze_decoded_boundaries,
correct_decoded_boundaries,
format_video_boundary_analysis,
)
analyses = analyze_decoded_boundaries(
images=image_chunks,
assembly_plan=plan,
)
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
image_chunks, actions = correct_decoded_boundaries(
images=image_chunks,
assembly_plan=plan,
analyses=analyses,
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
)
except Exception as exc:
analysis_error = exc
result_images, result_audio, report = assemble_decoded_chunks(
images=image_chunks,
audio=_chunk_list(audio),
assembly_plan=plan,
exact_total_duration=bool(
_singleton(exact_total_duration, "exact_total_duration")
),
audio_seam=str(_singleton(audio_seam, "audio_seam")),
diagnostics=str(_singleton(diagnostics, "diagnostics")),
)
if mode == VIDEO_SEAM_ANALYZE:
status = "Video Seam: Analyze Only; decoded frames and audio are unchanged."
elif mode == VIDEO_SEAM_AUTO:
status = (
"Video Seam: Auto; guarded transient and micro-flash correction enabled."
)
else:
status = (
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
"and exposure-ramp correction enabled."
)
report = report.replace("Video seam correction is disabled.", status, 1)
if analysis_error is None:
lines = [
format_video_boundary_analysis(
item,
action=(
"analysis only"
if mode == VIDEO_SEAM_ANALYZE
else actions.get(item.boundary_index, "kept native boundary")
),
)
for item in analyses
]
if not lines:
lines = [f"Video Seam {mode}: no decoded chunk boundary to analyze."]
else:
lines = [
f"Video Seam {mode}: native output preserved; analysis unavailable "
f"({type(analysis_error).__name__}: {analysis_error})"
]
return result_images, result_audio, report.rstrip() + "\n" + "\n".join(lines)
+735
View File
@@ -0,0 +1,735 @@
"""V3.5 direct-write assembly with a manual RAM or file-backed IMAGE buffer."""
from __future__ import annotations
from contextlib import AbstractContextManager
from dataclasses import dataclass
import math
from pathlib import Path
import time
from typing import Any, Callable, Mapping
import torch
from ..constants import (
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
normalize_diagnostics_mode,
)
from ..hardening import assemble_with_hardening
from ..media import validate_audio
from ..v2.seam_guard import correct_audio_seam
from .assembly import (
AUDIO_SEAM_AUTO,
AUDIO_SEAM_OPTIONS,
VIDEO_SEAM_ANALYZE,
VIDEO_SEAM_ANALYSIS_OPTIONS,
VIDEO_SEAM_AUTO,
VIDEO_SEAM_AUTO_2,
VIDEO_SEAM_OFF,
)
from .file_backed_buffer import (
BUFFER_BACKEND_AUTO,
BUFFER_BACKEND_DISK,
BUFFER_BACKEND_OPTIONS,
BUFFER_BACKEND_RAM,
allocate_file_backed_image,
format_auto_buffer_decision,
select_auto_buffer_backend,
preflight_manual_buffer,
)
from .. import storage_settings
from .plan import FPS, validate_assembly_plan, REVIEW_AUDIO_PROJECTION_KEY
_COPY_BATCH_FRAMES = 8
def _check_interrupted(
interrupt_check: Callable[[], None] | None,
) -> None:
"""Honor ComfyUI's normal ESC interrupt without requiring Core in tests."""
if interrupt_check is not None:
interrupt_check()
return
try:
from comfy.model_management import throw_exception_if_processing_interrupted
except ModuleNotFoundError:
return
throw_exception_if_processing_interrupted()
@dataclass(frozen=True)
class CopySpan:
"""Map a half-open natural-timeline interval into the output buffer."""
source_start: int
source_stop: int
destination_start: int
class _RamAllocation(AbstractContextManager):
"""Match the provisional file-backed allocation interface for RAM."""
def __init__(self, shape: tuple[int, ...], dtype: torch.dtype):
self.tensor = torch.empty(shape, dtype=dtype, device="cpu")
self.record = None
self._published = False
def publish(self) -> torch.Tensor:
if self._published:
raise RuntimeError("RAM assembly buffer was already published")
self._published = True
return self.tensor
def __exit__(self, exc_type, exc, traceback) -> bool:
return False
def _video_copy_spans(
*,
natural_frames: int,
output_frames: int,
preserve_final_frame: bool,
) -> tuple[CopySpan, ...]:
if output_frames >= natural_frames:
return (CopySpan(0, natural_frames, 0),)
if preserve_final_frame and output_frames >= 2:
return (
CopySpan(0, output_frames - 1, 0),
CopySpan(natural_frames - 1, natural_frames, output_frames - 1),
)
return (CopySpan(0, output_frames, 0),)
def _copy_natural_interval(
destination: torch.Tensor,
source: torch.Tensor,
*,
natural_start: int,
spans: tuple[CopySpan, ...],
interrupt_check: Callable[[], None] | None = None,
) -> None:
natural_stop = natural_start + int(source.shape[0])
for span in spans:
overlap_start = max(natural_start, int(span.source_start))
overlap_stop = min(natural_stop, int(span.source_stop))
if overlap_stop <= overlap_start:
continue
source_start = overlap_start - natural_start
destination_start = int(span.destination_start) + (
overlap_start - int(span.source_start)
)
frame_count = overlap_stop - overlap_start
for offset in range(0, frame_count, _COPY_BATCH_FRAMES):
_check_interrupted(interrupt_check)
batch_frames = min(_COPY_BATCH_FRAMES, frame_count - offset)
source_batch_start = source_start + offset
destination_batch_start = destination_start + offset
destination[
destination_batch_start : destination_batch_start + batch_frames
].copy_(
source[source_batch_start : source_batch_start + batch_frames]
)
def _repeat_final_video_frame(
destination: torch.Tensor,
*,
natural_frames: int,
output_frames: int,
interrupt_check: Callable[[], None] | None,
) -> None:
for destination_start in range(
natural_frames,
output_frames,
_COPY_BATCH_FRAMES,
):
_check_interrupted(interrupt_check)
destination_stop = min(
output_frames,
destination_start + _COPY_BATCH_FRAMES,
)
destination[destination_start:destination_stop].copy_(
destination[natural_frames - 1 : natural_frames].expand(
destination_stop - destination_start,
*destination.shape[1:],
)
)
def _copy_audio_segment(
destination: torch.Tensor,
waveform: torch.Tensor,
*,
trim_samples: int,
) -> None:
"""Reproduce timeline slicing/padding without concatenating a waveform."""
wanted = int(destination.shape[-1])
trim_samples = max(0, int(trim_samples))
available = max(0, min(wanted, int(waveform.shape[-1]) - trim_samples))
if available > 0:
destination[..., :available].copy_(
waveform[..., trim_samples : trim_samples + available]
)
if available >= wanted:
return
if available > 0:
destination[..., available:].copy_(
destination[..., available - 1 : available].expand(
*destination.shape[:-1],
wanted - available,
)
)
else:
destination.zero_()
def adjust_audio_duration_direct(
audio: Mapping[str, Any],
*,
target_frames: int,
preserve_final_audio_anchor: bool,
) -> tuple[dict[str, Any], int]:
"""Apply the legacy audio trim/pad values using direct allocation and copy."""
waveform, sample_rate = validate_audio(audio)
waveform = waveform.detach().to(device="cpu")
sample_rate = int(sample_rate)
target_samples = int(round(int(target_frames) / FPS * sample_rate))
original_samples = int(waveform.shape[-1])
delta_samples = target_samples - original_samples
if delta_samples == 0:
return {"waveform": waveform, "sample_rate": sample_rate}, 0
output = torch.empty(
(*waveform.shape[:-1], target_samples),
dtype=waveform.dtype,
device="cpu",
)
if delta_samples < 0:
if preserve_final_audio_anchor and target_samples > 1:
tail_samples = min(
max(1, int(round(sample_rate / FPS))),
target_samples,
original_samples,
)
head_samples = target_samples - tail_samples
if head_samples > 0:
output[..., :head_samples].copy_(waveform[..., :head_samples])
output[..., head_samples:].copy_(waveform[..., -tail_samples:])
fade = min(
int(round(sample_rate * 0.005)),
head_samples,
tail_samples,
)
if fade > 0:
start = output[..., head_samples - 1 : head_samples].to(
dtype=output.dtype
)
alpha = torch.linspace(
0.0,
1.0,
fade,
dtype=output.dtype,
device=output.device,
).reshape(*([1] * (output.ndim - 1)), fade)
tail_head = output[..., head_samples : head_samples + fade]
tail_head.copy_(start * (1.0 - alpha) + tail_head * alpha)
else:
output.copy_(waveform[..., :target_samples])
else:
if original_samples > 0:
output[..., :original_samples].copy_(waveform)
output[..., original_samples:].copy_(
waveform[..., -1:].expand(
*waveform.shape[:-1],
delta_samples,
)
)
else:
output.zero_()
return {"waveform": output, "sample_rate": sample_rate}, delta_samples
def _prepare_video_seam(
*,
images: list[Any],
plan: dict[str, Any],
mode: str,
) -> tuple[list[Any], dict[int, torch.Tensor], dict[int, str], Exception | None]:
if mode == VIDEO_SEAM_OFF:
return [], {}, {}, None
analyses: list[Any] = []
try:
from .video_seam import analyze_decoded_boundaries
analyses = analyze_decoded_boundaries(
images=images,
assembly_plan=plan,
)
patches: dict[int, torch.Tensor] = {}
actions: dict[int, str] = {}
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
from .video_seam_v35 import build_boundary_patches
patches, actions = build_boundary_patches(
images=images,
assembly_plan=plan,
analyses=analyses,
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
)
return analyses, patches, actions, None
except Exception as exc:
# Match V3.4's all-native fallback: no partial patch is retained.
return analyses, {}, {}, exc
def _video_seam_status(mode: str) -> str:
if mode == VIDEO_SEAM_OFF:
return "Video seam correction is disabled."
if mode == VIDEO_SEAM_ANALYZE:
return "Video Seam: Analyze Only; decoded frames and audio are unchanged."
if mode == VIDEO_SEAM_AUTO:
return "Video Seam: Auto; guarded transient and micro-flash correction enabled."
return (
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
"and exposure-ramp correction enabled."
)
def _format_video_seam_lines(
*,
mode: str,
analyses: list[Any],
actions: dict[int, str],
error: Exception | None,
) -> list[str]:
if mode == VIDEO_SEAM_OFF:
return []
if error is not None:
return [
f"Video Seam {mode}: native output preserved; analysis unavailable "
f"({type(error).__name__}: {error})"
]
from .video_seam import format_video_boundary_analysis
lines = [
format_video_boundary_analysis(
item,
action=(
"analysis only"
if mode == VIDEO_SEAM_ANALYZE
else actions.get(item.boundary_index, "kept native boundary")
),
)
for item in analyses
]
if not lines:
lines.append(f"Video Seam {mode}: no decoded chunk boundary to analyze.")
return lines
def _format_elapsed(seconds: float) -> str:
total = max(0.0, float(seconds))
hours, remainder = divmod(total, 3600.0)
minutes, seconds = divmod(remainder, 60.0)
return f"{int(hours):02d}:{int(minutes):02d}:{seconds:05.2f}"
def _write_assembly(
*,
image_buffer: torch.Tensor,
images: list[Any],
audio: list[Any],
plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
natural_frames: int,
output_frames: int,
video_spans: tuple[CopySpan, ...],
seam_patches: dict[int, torch.Tensor],
seam_analyses: list[Any],
seam_actions: dict[int, str],
seam_error: Exception | None,
buffer_backend: str,
auto_decision_report: str | None,
final_audio_override: Mapping[str, Any] | None,
final_audio_source: str | None,
interrupt_check: Callable[[], None] | None,
) -> tuple[dict[str, Any], str]:
diagnostics_mode = normalize_diagnostics_mode(diagnostics)
units = list(plan.get("decode_groups") or plan["chunks"])
first_waveform, audio_rate = validate_audio(audio[0])
first_waveform = first_waveform.detach().to(device="cpu")
audio_rate = int(audio_rate)
natural_samples = int(round(natural_frames / FPS * audio_rate))
audio_buffer = torch.empty(
(*first_waveform.shape[:-1], natural_samples),
dtype=first_waveform.dtype,
device="cpu",
)
reports = [
"H3 Continuum Assemble V3.5",
f"Audio Seam: {audio_seam}. {_video_seam_status(video_seam)}",
f"Buffer Backend: {buffer_backend}; IMAGE direct-write "
f"frames={output_frames}, bytes={image_buffer.numel() * image_buffer.element_size()}; "
"no final full IMAGE copy.",
]
if auto_decision_report is not None:
reports.append(auto_decision_report)
frame_cursor = 0
for index, (raw_images, raw_audio, unit) in enumerate(
zip(images, audio, units),
start=1,
):
_check_interrupted(interrupt_check)
total_frames = int(unit["total_frames"])
trim_frames = int(unit["trim_frames"])
net_frames = int(unit["net_frames"])
raw_images_cpu = raw_images.detach().to(device="cpu")
retained_images = raw_images_cpu[trim_frames:total_frames]
if int(retained_images.shape[0]) != net_frames:
raise ValueError(
f"decoded image chunk {index} retained {retained_images.shape[0]} "
f"frames; plan expects {net_frames}"
)
_copy_natural_interval(
image_buffer,
retained_images,
natural_start=frame_cursor,
spans=video_spans,
interrupt_check=interrupt_check,
)
_check_interrupted(interrupt_check)
waveform, sample_rate = validate_audio(raw_audio)
waveform = waveform.detach().to(device="cpu")
sample_rate = int(sample_rate)
if sample_rate != audio_rate:
raise ValueError(
f"audio sample rate changed between chunks: {audio_rate} -> {sample_rate}"
)
frame_stop = frame_cursor + net_frames
sample_start = int(round(frame_cursor / FPS * sample_rate))
sample_stop = int(round(frame_stop / FPS * sample_rate))
seam_report = None
if index > 1 and audio_seam == AUDIO_SEAM_AUTO:
try:
cut_sample = int(round(trim_frames / FPS * sample_rate))
patch, metrics, fade_samples, level_gain, dc_bias = correct_audio_seam(
audio_buffer[..., :sample_start],
waveform,
sample_rate=sample_rate,
cut_sample=cut_sample,
)
if patch is not None and int(patch.shape[-1]) > 0:
patch_start = sample_start - int(patch.shape[-1])
if patch_start < 0:
raise ValueError("audio seam patch starts before the output")
audio_buffer[..., patch_start:sample_start].copy_(patch)
seam_report = (
f"audio seam {index-1}->{index}: "
f"corr={metrics.correlation_before:.4f}->"
f"{metrics.correlation_after:.4f}, "
f"jump={metrics.boundary_jump_before:.6f}->"
f"{metrics.boundary_jump_after:.6f}, "
f"fade={fade_samples} samples"
)
if diagnostics_mode == DIAGNOSTICS_FULL:
seam_report += (
f", offset={metrics.offset_samples:+d}, "
f"gain={level_gain:.4f}, dc={dc_bias:+.6f}"
)
except Exception as exc:
seam_report = (
f"audio seam {index-1}->{index}: fallback to native boundary "
f"({type(exc).__name__}: {exc})"
)
trim_samples = int(round(trim_frames / FPS * sample_rate))
_copy_audio_segment(
audio_buffer[..., sample_start:sample_stop],
waveform,
trim_samples=trim_samples,
)
if diagnostics_mode != DIAGNOSTICS_OFF:
reports.append(
f"assembled decoded chunk {index}: {net_frames} retained frames, "
f"cumulative {frame_stop}/{natural_frames}"
)
if seam_report:
reports.append(seam_report)
frame_cursor = frame_stop
if frame_cursor != natural_frames:
raise RuntimeError(
f"V3.5 assembly cursor mismatch: {frame_cursor} != {natural_frames}"
)
unit_starts: list[int] = []
cursor = 0
for unit in units:
unit_starts.append(cursor)
cursor += int(unit["net_frames"])
for boundary, patch in seam_patches.items():
_check_interrupted(interrupt_check)
if boundary <= 0 or boundary >= len(unit_starts):
raise ValueError("V3.5 video seam patch boundary is outside decode groups")
_copy_natural_interval(
image_buffer,
patch,
natural_start=unit_starts[boundary],
spans=video_spans,
interrupt_check=interrupt_check,
)
if output_frames > natural_frames:
_repeat_final_video_frame(
image_buffer,
natural_frames=natural_frames,
output_frames=output_frames,
interrupt_check=interrupt_check,
)
_check_interrupted(interrupt_check)
result_audio: dict[str, Any] = {
"waveform": audio_buffer,
"sample_rate": audio_rate,
}
duration_report = ""
if exact_total_duration:
preserve_anchor = bool(
plan.get("preserve_final_frame", False)
and output_frames < natural_frames
and output_frames >= 2
)
result_audio, delta_samples = adjust_audio_duration_direct(
result_audio,
target_frames=output_frames,
preserve_final_audio_anchor=preserve_anchor,
)
adjustment = output_frames - natural_frames
mode = (
"final anchor preserved; pre-tail trim"
if preserve_anchor
else "tail trim/pad"
)
duration_report = (
f"Exact-duration adjustment: frames {natural_frames}->{output_frames} "
f"({adjustment:+d}), audio samples correction {delta_samples:+d}; {mode}."
)
driving_audio_report = ""
if final_audio_override is not None:
result_audio = dict(final_audio_override)
override_waveform, _ = validate_audio(result_audio)
empty_review = (REVIEW_AUDIO_PROJECTION_KEY in plan and int(override_waveform.shape[-1]) == 0)
if exact_total_duration or empty_review:
result_audio, _ = adjust_audio_duration_direct(
result_audio,
target_frames=output_frames,
preserve_final_audio_anchor=False,
)
waveform, selected_rate = validate_audio(result_audio)
result_audio = {
"waveform": waveform,
"sample_rate": int(selected_rate),
}
driving_audio_report = (
"Driving Audio: preserved source selected from "
f"{final_audio_source or 'direct input'}; "
f"sample_rate={int(selected_rate)}, samples={int(waveform.shape[-1])}; "
"generated audio and Audio Seam bypassed."
)
if empty_review:
driving_audio_report += " Empty Review interval: silence added for the selected video duration."
reports.append(
f"Assembled {len(units)} physical decoded group(s) into "
f"{output_frames} frames with cumulative sample-boundary alignment."
)
if duration_report:
reports.append(duration_report)
if driving_audio_report:
reports.append(driving_audio_report)
runtime_started_at = plan.get("_runtime_started_at")
if isinstance(runtime_started_at, (int, float)):
elapsed = time.perf_counter() - float(runtime_started_at)
if math.isfinite(elapsed) and elapsed >= 0.0:
reports.append(f"Total workflow elapsed: {_format_elapsed(elapsed)}")
reports.extend(
_format_video_seam_lines(
mode=video_seam,
analyses=seam_analyses,
actions=seam_actions,
error=seam_error,
)
)
return result_audio, "\n".join(reports)
def _assemble_decoded_chunks_v35_core(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
buffer_backend: str,
backing_root: str | Path | None = None,
final_audio_override: Mapping[str, Any] | None = None,
final_audio_source: str | None = None,
interrupt_check: Callable[[], None] | None = None,
):
plan = validate_assembly_plan(assembly_plan)
if audio_seam not in AUDIO_SEAM_OPTIONS:
raise ValueError(f"unknown Audio Seam mode: {audio_seam!r}")
if video_seam not in VIDEO_SEAM_ANALYSIS_OPTIONS:
raise ValueError(f"unknown Video Seam mode: {video_seam!r}")
if buffer_backend not in BUFFER_BACKEND_OPTIONS:
raise ValueError(f"unknown Buffer Backend: {buffer_backend!r}")
units = list(plan.get("decode_groups") or plan["chunks"])
natural_frames = sum(int(unit["net_frames"]) for unit in units)
output_frames = (
int(plan["target_frames"])
if bool(exact_total_duration)
else natural_frames
)
if natural_frames <= 0 or output_frames <= 0:
raise ValueError("V3.5 assembly frame counts must be positive")
first_image = images[0]
shape = (output_frames, *tuple(int(value) for value in first_image.shape[1:]))
video_spans = _video_copy_spans(
natural_frames=natural_frames,
output_frames=output_frames,
preserve_final_frame=bool(plan.get("preserve_final_frame", False)),
)
analyses, patches, actions, analysis_error = _prepare_video_seam(
images=images,
plan=plan,
mode=video_seam,
)
resolved_backend = buffer_backend
resolved_backing_root = backing_root
auto_decision_report = None
final_image_bytes = math.prod(shape) * first_image.element_size()
if buffer_backend == BUFFER_BACKEND_AUTO:
final_image_bytes = (
math.prod(shape) * first_image.element_size()
)
decision = select_auto_buffer_backend(
final_image_bytes,
backing_root=backing_root,
)
resolved_backend = decision.selected_backend
resolved_backing_root = decision.backing_root
auto_decision_report = format_auto_buffer_decision(decision)
else:
preflight_manual_buffer(buffer_backend, final_image_bytes, backing_root)
allocation: Any
if resolved_backend == BUFFER_BACKEND_RAM:
allocation = _RamAllocation(shape, first_image.dtype)
else:
allocation = allocate_file_backed_image(
shape,
dtype=first_image.dtype,
backing_root=resolved_backing_root,
)
with allocation:
settings = storage_settings.get_manager()
if settings is not None:
from .file_backed_buffer import resolve_backing_root
settings.record_backend(resolved_backend, final_image_bytes,
resolve_backing_root(resolved_backing_root) if resolved_backend == BUFFER_BACKEND_DISK else None)
# The file exists at this point, so an ESC received immediately before
# assembly exercises the same unpublished cleanup path as a mid-copy
# cancellation.
_check_interrupted(interrupt_check)
result_audio, report = _write_assembly(
image_buffer=allocation.tensor,
images=images,
audio=audio,
plan=plan,
exact_total_duration=bool(exact_total_duration),
audio_seam=audio_seam,
video_seam=video_seam,
diagnostics=diagnostics,
natural_frames=natural_frames,
output_frames=output_frames,
video_spans=video_spans,
seam_patches=patches,
seam_analyses=analyses,
seam_actions=actions,
seam_error=analysis_error,
buffer_backend=(
resolved_backend
if buffer_backend != BUFFER_BACKEND_AUTO
else f"Auto -> {resolved_backend}"
),
auto_decision_report=auto_decision_report,
final_audio_override=final_audio_override,
final_audio_source=final_audio_source,
interrupt_check=interrupt_check,
)
result_images = allocation.publish()
return result_images, result_audio, report
@storage_settings.storage_operation
def assemble_decoded_chunks_v35(
*,
images: list[Any],
audio: list[Any],
assembly_plan: dict[str, Any],
exact_total_duration: bool,
audio_seam: str,
video_seam: str,
diagnostics: str,
buffer_backend: str,
backing_root: str | Path | None = None,
final_audio_override: Mapping[str, Any] | None = None,
final_audio_source: str | None = None,
interrupt_check: Callable[[], None] | None = None,
):
"""Preflight all inputs, then execute the independent V3.5 writer."""
kwargs = {
"images": images,
"audio": audio,
"assembly_plan": assembly_plan,
"exact_total_duration": bool(exact_total_duration),
"audio_seam": str(audio_seam),
"video_seam": str(video_seam),
"diagnostics": str(diagnostics),
"buffer_backend": str(buffer_backend),
"backing_root": backing_root,
"final_audio_override": final_audio_override,
"final_audio_source": final_audio_source,
"interrupt_check": interrupt_check,
}
return assemble_with_hardening(
_assemble_decoded_chunks_v35_core,
(),
kwargs,
)
__all__ = [
"BUFFER_BACKEND_DISK",
"BUFFER_BACKEND_AUTO",
"BUFFER_BACKEND_OPTIONS",
"BUFFER_BACKEND_RAM",
"CopySpan",
"adjust_audio_duration_direct",
"assemble_decoded_chunks_v35",
]
@@ -0,0 +1,135 @@
"""V3.5 Advanced bridge from Continuum refine context to Core conditioning."""
from __future__ import annotations
from typing import Any
from .second_pass import prepare_physical_refine_groups
from .second_pass_nodes import _single_list_input
class H3ContinuumConditioningBridgeV35:
"""Publish paired physical-group MODEL and CONDITIONING values."""
DEPRECATED = False
DESCRIPTION = (
"Advanced interoperability bridge for Basic Guider and external samplers. "
"It adapts the captured Continuum refine context to externally processed "
"physical-group video latents, then returns one paired MODEL and complete "
"CONDITIONING per physical group plus the target-geometry Assembly Plan. "
"Nested AV LATENT construction, Audio LATENT, Noise, Audio Lock, and external "
"sampling behavior remain the workflow's responsibility."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"video_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
},
"optional": {
"video_vae": (
"VAE",
{
"tooltip": (
"When available, First/Last keyframes are re-encoded at "
"the target geometry. Otherwise the existing conditioning "
"adapter uses its documented latent-resize fallback."
)
},
),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = (
"MODEL",
"CONDITIONING",
"H3_CONTINUUM_ASSEMBLY_PLAN",
"STRING",
)
RETURN_NAMES = (
"group_models",
"conditioning",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "prepare"
CATEGORY = "MiniMax H3/Continuum/Advanced"
def prepare(
self,
model,
clip,
video_latents,
assembly_plan,
refine_context,
video_vae=None,
):
prepared = prepare_physical_refine_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
video_latents=video_latents,
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
refine_context=_single_list_input("refine_context", refine_context),
video_vae=(
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
),
)
group_models = prepared["group_models"]
conditioning = prepared["conditioning"]
physical_group_count = int(prepared["physical_group_count"])
if not len(group_models) == len(conditioning) == physical_group_count:
raise ValueError(
"Conditioning Bridge MODEL/CONDITIONING physical group counts differ"
)
status_lines = [
"H3 Continuum Conditioning Bridge V3.5",
f"Physical groups: {physical_group_count}.",
"Output contract: one paired MODEL and one complete CONDITIONING per "
"physical group; CONDITIONING entries remain grouped.",
"Scope: Advanced MODEL + CONDITIONING interoperability only; AV LATENT, "
"Noise, and Audio Lock are controlled by the external workflow.",
*prepared["warnings"],
]
for detail in prepared["details"]:
group = detail["group"]
status_lines.append(
"group "
f"{int(detail['group_index']) + 1}: "
f"logical_chunks={group.get('logical_chunks')}, "
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
f"conditioning_source={detail['conditioning_source']}, "
f"physical_clip_index={detail['physical_clip_index']}, "
f"context_frames={detail['context_frames']}"
)
status_lines.append(
"Target geometry: "
f"{prepared['updated_assembly_plan']['width']}x"
f"{prepared['updated_assembly_plan']['height']}."
)
return (
group_models,
conditioning,
prepared["updated_assembly_plan"],
"\n".join(status_lines),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumConditioningBridgeV35": H3ContinuumConditioningBridgeV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumConditioningBridgeV35": (
"H3 Continuum Conditioning Bridge V3.5"
),
}
File diff suppressed because it is too large Load Diff
+67
View File
@@ -0,0 +1,67 @@
"""Generic Easy workflow audio loader backed by ComfyUI Core LoadAudio."""
from __future__ import annotations
from types import SimpleNamespace
try:
from comfy_extras.nodes_audio import LoadAudio as CoreLoadAudio
except (ImportError, AttributeError): # pragma: no cover - standalone unit tests
class CoreLoadAudio:
"""Non-runtime shim; real ComfyUI always supplies Core LoadAudio."""
@classmethod
def define_schema(cls):
return SimpleNamespace(
node_id="LoadAudio",
display_name="Load Audio",
category="audio",
essentials_category="Audio",
description="",
search_aliases=["import audio", "open audio", "audio file"],
is_deprecated=False,
)
@classmethod
def execute(cls, audio):
return audio
load = execute
class H3EasyLoadAudio(CoreLoadAudio):
"""Core LoadAudio with an Easy frontend control for native node bypass."""
@classmethod
def define_schema(cls):
schema = super().define_schema()
schema.node_id = "H3EasyLoadAudio"
schema.is_deprecated = True # Kept for saved workflows; new graphs use Core LoadAudio.
schema.display_name = "H3 Continuum Load Audio"
schema.category = "MiniMax H3/Continuum"
schema.description = (
"Loads audio through ComfyUI Core. Enable Audio controls the node's "
"native Bypass mode without changing the AUDIO contract."
)
schema.search_aliases = list(
dict.fromkeys(
[
*getattr(schema, "search_aliases", []),
"H3 Continuum Load Audio",
"H3 Easy Load Audio",
"H3 Easy Audio Loader",
"Load Audio with Bypass",
]
)
)
return schema
NODE_CLASS_MAPPINGS = {
"H3EasyLoadAudio": H3EasyLoadAudio,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3EasyLoadAudio": "H3 Continuum Load Audio",
}
+48
View File
@@ -0,0 +1,48 @@
"""Generic Easy workflow image loader backed by ComfyUI Core LoadImage."""
from __future__ import annotations
try:
from nodes import LoadImage as CoreLoadImage
except (ImportError, AttributeError): # pragma: no cover - standalone unit tests
class CoreLoadImage:
"""Non-runtime shim; real ComfyUI always supplies Core LoadImage."""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": (("",), {"image_upload": True}),
}
}
RETURN_TYPES = ("IMAGE", "MASK")
FUNCTION = "load_image"
class H3EasyLoadImage(CoreLoadImage):
"""Core LoadImage with an Easy frontend control for native node bypass."""
DEPRECATED = False
CATEGORY = "MiniMax H3/Continuum"
DESCRIPTION = (
"Loads an image through ComfyUI Core. Enable Image controls the node's "
"native Bypass mode, so the same node can serve First, Last, or Reference "
"image inputs without changing the IMAGE/MASK contract."
)
SEARCH_ALIASES = [
"H3 Continuum Load Image",
"H3 Easy Load Image",
"H3 Easy Image Loader",
"Load Image with Bypass",
]
NODE_CLASS_MAPPINGS = {
"H3EasyLoadImage": H3EasyLoadImage,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3EasyLoadImage": "H3 Continuum Load Image",
}
+482
View File
@@ -0,0 +1,482 @@
"""V3.8 Easy Mode facade and deterministic UI resolvers."""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any
from ..conditioning import conditioning_display_label
from ..constants import (
CONTINUITY_OPTIONS,
DIAGNOSTICS_BASIC,
PROMPT_FORMAT_AUTO,
)
from ..reference import REFERENCE_SIZE_MATCH_OUTPUT
from ..reference_video import REFERENCE_VIDEO_SIZE_EFFICIENT
from .driving_nodes import (
CONTINUATION_BACKEND_STANDARD,
H3ContinuumSamplerV37,
)
from .nodes import CATEGORY as CONTINUUM_CATEGORY, REGENERATE_AUTO
from .resolution import (
H3_ASPECT_AUTO,
H3_ASPECT_LANDSCAPE,
H3_ASPECT_OPTIONS,
H3_ASPECT_PORTRAIT,
H3_ASPECT_SQUARE,
H3_BALANCED_MP,
H3_CANVAS_MULTIPLE,
H3_CUSTOM_MP_DEFAULT,
H3_CUSTOM_MP_MAX,
H3_CUSTOM_MP_MIN,
H3_CUSTOM_MP_STRONG_WARNING,
H3_DRAFT_MP,
H3_MP_PIXELS,
H3_NATIVE_LONG_EDGE_CAP,
H3_NATIVE_SHORT_EDGE,
H3_PRESET_BALANCED,
H3_PRESET_CUSTOM,
H3_PRESET_DRAFT,
H3_PRESET_NATIVE,
H3_PRESET_OPTIONS,
H3ResolutionPlan,
resolve_h3_aspect,
resolve_h3_resolution,
)
EASY_DURATION_DEFAULT = 10
EASY_DURATION_MIN = 4
EASY_DURATION_MAX = 480
EASY_TARGET_CHUNK_SECONDS = 10.0
EASY_MAX_CHUNKS = 16
EASY_SEED_MODE_RANDOMIZE = "Randomize"
EASY_SEED_MODE_FIXED = "Fixed"
EASY_SEED_MODE_OPTIONS = (
EASY_SEED_MODE_RANDOMIZE,
EASY_SEED_MODE_FIXED,
)
EASY_ASPECT_AUTO = H3_ASPECT_AUTO
EASY_ASPECT_LANDSCAPE = H3_ASPECT_LANDSCAPE
EASY_ASPECT_PORTRAIT = H3_ASPECT_PORTRAIT
EASY_ASPECT_SQUARE = H3_ASPECT_SQUARE
EASY_ASPECT_OPTIONS = H3_ASPECT_OPTIONS
EASY_PRESET_DRAFT = H3_PRESET_DRAFT
EASY_PRESET_BALANCED = H3_PRESET_BALANCED
EASY_PRESET_NATIVE = H3_PRESET_NATIVE
EASY_PRESET_CUSTOM = H3_PRESET_CUSTOM
EASY_PRESET_OPTIONS = H3_PRESET_OPTIONS
EASY_DRAFT_MP = H3_DRAFT_MP
EASY_BALANCED_MP = H3_BALANCED_MP
EASY_CUSTOM_MP_DEFAULT = H3_CUSTOM_MP_DEFAULT
EASY_CUSTOM_MP_MIN = H3_CUSTOM_MP_MIN
EASY_CUSTOM_MP_MAX = H3_CUSTOM_MP_MAX
EASY_CUSTOM_MP_STRONG_WARNING = H3_CUSTOM_MP_STRONG_WARNING
EASY_MP_PIXELS = H3_MP_PIXELS
EASY_CANVAS_MULTIPLE = H3_CANVAS_MULTIPLE
EASY_NATIVE_SHORT_EDGE = H3_NATIVE_SHORT_EDGE
EASY_NATIVE_LONG_EDGE_CAP = H3_NATIVE_LONG_EDGE_CAP
EasyResolutionPlan = H3ResolutionPlan
resolve_easy_aspect = resolve_h3_aspect
resolve_easy_resolution = resolve_h3_resolution
EASY_REFERENCES_TYPE = "H3_CONTINUUM_EASY_REFERENCES"
EASY_REFERENCES_SCHEMA_VERSION = 1
@dataclass(frozen=True)
class EasyDurationPlan:
duration_seconds: int
chunks: int
chunk_seconds: float
def make_easy_references(
reference_image_1: Any = None,
reference_image_2: Any = None,
reference_image_3: Any = None,
) -> dict[str, Any]:
"""Pack fixed Easy reference sockets without preprocessing their images."""
return {
"schema_version": EASY_REFERENCES_SCHEMA_VERSION,
"reference_image_1": reference_image_1,
"reference_image_2": reference_image_2,
"reference_image_3": reference_image_3,
}
def resolve_easy_references(references: Any = None) -> tuple[Any, Any, Any]:
"""Expand the private Easy bundle back into the mature V3.7 input contract."""
if references is None:
return None, None, None
if not isinstance(references, dict):
raise TypeError("Easy References must come from H3 Continuum Easy References")
if references.get("schema_version") != EASY_REFERENCES_SCHEMA_VERSION:
raise ValueError("Unsupported H3 Continuum Easy References schema version")
return tuple(
references.get(f"reference_image_{index}") for index in range(1, 4)
)
def resolve_easy_duration(duration_seconds: int) -> EasyDurationPlan:
"""Resolve one total duration around ten-second logical chunks."""
if isinstance(duration_seconds, bool):
raise TypeError("Duration must be an integer number of seconds")
duration = int(duration_seconds)
if duration != duration_seconds:
raise ValueError("Duration must be an integer number of seconds")
if not EASY_DURATION_MIN <= duration <= EASY_DURATION_MAX:
raise ValueError(
f"Duration must be between {EASY_DURATION_MIN} and {EASY_DURATION_MAX} seconds"
)
chunks = min(
EASY_MAX_CHUNKS,
max(1, int(math.ceil(float(duration) / EASY_TARGET_CHUNK_SECONDS))),
)
chunk_seconds = float(duration) / float(chunks)
if not 4.0 <= chunk_seconds <= 30.0:
raise ValueError("Duration cannot be represented by the Production chunk contract")
return EasyDurationPlan(
duration_seconds=duration,
chunks=chunks,
chunk_seconds=chunk_seconds,
)
class H3ContinuumEasyReferences:
"""Fixed three-image facade for the existing V3.7 reference contract."""
DEPRECATED = False
CATEGORY = CONTINUUM_CATEGORY
DESCRIPTION = (
"Collects up to three optional still-image references for H3 Continuum "
"Easy V3.8. Images are passed unchanged to the existing V3.7 reference "
"preprocessing and conditioning path."
)
SEARCH_ALIASES = [
"H3 Continuum Easy References",
"MiniMax H3 Easy Reference Images",
]
RETURN_TYPES = (EASY_REFERENCES_TYPE,)
RETURN_NAMES = ("references",)
FUNCTION = "pack"
@classmethod
def INPUT_TYPES(cls):
return {
"optional": {
"reference_image_1": (
"IMAGE",
{"display_name": "Reference Image 1 (Optional)"},
),
"reference_image_2": (
"IMAGE",
{"display_name": "Reference Image 2 (Optional)"},
),
"reference_image_3": (
"IMAGE",
{"display_name": "Reference Image 3 (Optional)"},
),
}
}
def pack(
self,
reference_image_1=None,
reference_image_2=None,
reference_image_3=None,
):
return (
make_easy_references(
reference_image_1=reference_image_1,
reference_image_2=reference_image_2,
reference_image_3=reference_image_3,
),
)
class H3ContinuumEasyV38(H3ContinuumSamplerV37):
"""Beginner-facing resolver over the unchanged V3.7 Production engine."""
DEPRECATED = False
CATEGORY = CONTINUUM_CATEGORY
DESCRIPTION = (
"Easy Mode facade over H3 Continuum V3.7 Production. It resolves total "
"duration, decimal-MP resolution, seed UI, and image mode without copying "
"the sampler or scheduler implementation."
)
SEARCH_ALIASES = [
"H3 Continuum Easy V3.8",
"MiniMax H3 Easy Mode",
]
RETURN_TYPES = (
"LATENT",
"LATENT",
"H3_CONTINUUM_ASSEMBLY_PLAN",
"STRING",
"AUDIO",
)
RETURN_NAMES = (
"video_latents",
"audio_latents",
"assembly_plan",
"status",
"driving_audio",
)
OUTPUT_IS_LIST = (True, True, False, False, False)
FUNCTION = "run"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"video_vae": (
"VAE",
{
"display_name": "Video VAE",
"tooltip": "Used only for connected image conditioning.",
},
),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"prompt_text": (
"STRING",
{
"default": "",
"multiline": True,
"dynamicPrompts": True,
"display_name": "Prompt",
},
),
"duration": (
"INT",
{
"default": EASY_DURATION_DEFAULT,
"min": EASY_DURATION_MIN,
"max": EASY_DURATION_MAX,
"step": 1,
"display_name": "Duration (sec)",
},
),
"seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"display_name": "Seed",
},
),
"seed_mode": (
EASY_SEED_MODE_OPTIONS,
{
"default": EASY_SEED_MODE_RANDOMIZE,
"display_name": "Seed Mode",
},
),
"aspect": (
EASY_ASPECT_OPTIONS,
{
"default": EASY_ASPECT_AUTO,
"display_name": "Aspect",
},
),
"preset": (
EASY_PRESET_OPTIONS,
{
"default": EASY_PRESET_DRAFT,
"display_name": "Resolution Preset",
},
),
"custom_mp": (
"FLOAT",
{
"default": EASY_CUSTOM_MP_DEFAULT,
"min": EASY_CUSTOM_MP_MIN,
"max": EASY_CUSTOM_MP_MAX,
"step": 0.01,
"display_name": "Custom MP",
},
),
"run_storage": (
("Off", "Save + Auto Resume"),
{
"default": "Off",
"display_name": "Run Storage",
"advanced": True,
},
),
"run_name": (
"STRING",
{
"default": "",
"display_name": "Run Name (Optional Override)",
"advanced": True,
},
),
"project_id": (
"STRING",
{
"default": "",
"display_name": "Auto Resume ID Override",
"advanced": True,
},
),
},
"optional": {
"first_frame": (
"IMAGE",
{"display_name": "First Image (Optional)"},
),
"last_frame": (
"IMAGE",
{"display_name": "Last Image (Optional)"},
),
"references": (
EASY_REFERENCES_TYPE,
{"display_name": "Easy References (Optional)"},
),
"driving_audio": (
"AUDIO",
{
"display_name": "Driving Audio",
"tooltip": (
"Optional original audio timeline. Uses the unchanged "
"V3.7 Driving Audio conditioning and final-output contract."
),
},
),
"audio_vae": (
"VAE",
{
"display_name": "Driving Audio VAE",
"tooltip": (
"Required only when Driving Audio is connected. It may be "
"shared with the external Core audio decode path."
),
},
),
},
"hidden": {
"prompt": "PROMPT",
"unique_id": "UNIQUE_ID",
},
}
def run(
self,
model,
clip,
video_vae,
sampler,
sigmas,
prompt_text,
duration,
seed,
seed_mode=EASY_SEED_MODE_RANDOMIZE,
aspect=EASY_ASPECT_AUTO,
preset=EASY_PRESET_DRAFT,
custom_mp=EASY_CUSTOM_MP_DEFAULT,
run_storage="Off",
run_name="",
project_id="",
first_frame=None,
last_frame=None,
references=None,
driving_audio=None,
audio_vae=None,
prompt=None,
unique_id=None,
):
del seed_mode # Queue-time seed mutation remains owned by ComfyUI's frontend.
duration_plan = resolve_easy_duration(duration)
resolution = resolve_easy_resolution(
aspect=aspect,
preset=preset,
custom_mp=custom_mp,
first_frame=first_frame,
)
reference_image_1, reference_image_2, reference_image_3 = (
resolve_easy_references(references)
)
outputs = super().run(
model=model,
clip=clip,
video_vae=video_vae,
sampler=sampler,
sigmas=sigmas,
sequence_prompt=prompt_text,
prompt_mode=PROMPT_FORMAT_AUTO,
chunks=duration_plan.chunks,
chunk_seconds=duration_plan.chunk_seconds,
width=resolution.width,
height=resolution.height,
continuity=CONTINUITY_OPTIONS[0],
base_seed=int(seed),
audio_continuity=True,
diagnostics=DIAGNOSTICS_BASIC,
reroll_from_chunk=REGENERATE_AUTO,
reroll_nonce=0,
strict_compatibility=False,
debug=False,
show_preview=True,
run_storage=run_storage,
run_name=run_name,
reference_size=REFERENCE_SIZE_MATCH_OUTPUT,
reference_image_1=reference_image_1,
reference_image_2=reference_image_2,
reference_image_3=reference_image_3,
driving_audio=driving_audio,
audio_vae=audio_vae,
project_id=project_id,
first_frame=first_frame,
last_frame=last_frame,
prompt=prompt,
unique_id=unique_id,
video_reference_size=REFERENCE_VIDEO_SIZE_EFFICIENT,
continuation_backend=CONTINUATION_BACKEND_STANDARD,
)
video, audio, assembly_plan, status, driving_audio = outputs[:5]
mode = conditioning_display_label(
has_first=first_frame is not None,
has_last=last_frame is not None,
has_reference=any(
image is not None
for image in (
reference_image_1,
reference_image_2,
reference_image_3,
)
),
)
warning_lines = "\n".join(resolution.warnings)
summary = (
f"H3 Continuum Easy V3.8\n"
f"{mode} / {duration_plan.duration_seconds} sec\n"
f"{resolution.preset}: {resolution.width} x {resolution.height} "
f"({resolution.actual_mp:.2f} MP)"
)
if warning_lines:
summary += f"\n{warning_lines}"
return video, audio, assembly_plan, f"{summary}\n\n{status}", driving_audio
NODE_CLASS_MAPPINGS = {
"H3ContinuumEasyReferences": H3ContinuumEasyReferences,
"H3ContinuumEasyV38": H3ContinuumEasyV38,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumEasyReferences": "H3 Continuum Easy References",
"H3ContinuumEasyV38": "H3 Continuum Easy V3.8",
}
+274
View File
@@ -0,0 +1,274 @@
"""Simple H3 video loader backed by the current ComfyUI Core video API."""
from __future__ import annotations
from fractions import Fraction
import os
import torch
try:
import folder_paths
except ImportError: # pragma: no cover - standalone unit tests
class _FolderPathsShim:
@staticmethod
def get_input_directory():
return "."
@staticmethod
def filter_files_content_types(files, content_types):
del files, content_types
return []
@staticmethod
def get_annotated_filepath(file):
raise RuntimeError(f"ComfyUI folder_paths is unavailable for {file}")
@staticmethod
def exists_annotated_filepath(file):
del file
return False
folder_paths = _FolderPathsShim()
try:
from comfy_api.latest import InputImpl, io
except ImportError: # pragma: no cover - standalone unit tests
class _SchemaItem:
def __init__(self, item_id=None, **kwargs):
self.id = item_id
for key, value in kwargs.items():
setattr(self, key, value)
class _InputFactory:
Input = _SchemaItem
class _OutputFactory:
@staticmethod
def Output(**kwargs):
return _SchemaItem(**kwargs)
class _UploadVideo:
value = "video_upload"
class _UploadType:
video = _UploadVideo()
class _Schema:
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
class _NodeOutput:
def __init__(self, *args):
self.args = args
class _ComfyNode:
pass
class _IOShim:
ComfyNode = _ComfyNode
Schema = _Schema
NodeOutput = _NodeOutput
Boolean = _InputFactory
Combo = _InputFactory
Float = _InputFactory
Video = _InputFactory
Image = _OutputFactory
Audio = _OutputFactory
UploadType = _UploadType
class _InputImplShim:
@staticmethod
def VideoFromFile(path):
raise RuntimeError(f"ComfyUI VideoFromFile is unavailable for {path}")
InputImpl = _InputImplShim
io = _IOShim()
try:
from comfy_execution.graph_utils import ExecutionBlocker
except ImportError: # pragma: no cover - standalone unit tests
class ExecutionBlocker:
def __init__(self, message):
self.message = message
CATEGORY = "MiniMax H3/Continuum"
def _fraction_from_rate(value: float) -> Fraction:
"""Convert a UI frame-rate value without introducing binary-float drift."""
return Fraction(str(float(value))).limit_denominator(1_000_000)
def _resample_video_frames(
images: torch.Tensor,
source_rate: Fraction,
target_rate: float,
) -> torch.Tensor:
"""Drop or duplicate IMAGE frames while preserving nominal duration."""
if images.ndim != 4:
raise ValueError(
f"Expected IMAGE batch [B,H,W,C], got shape {tuple(images.shape)}"
)
frame_count = int(images.shape[0])
if frame_count <= 0:
raise ValueError("Video contains no decodable frames")
source = Fraction(source_rate)
if source <= 0:
raise ValueError(f"Invalid source frame rate: {source_rate}")
requested_rate = float(target_rate)
if requested_rate <= 0:
return images
target = _fraction_from_rate(requested_rate)
if target == source:
return images
target_count = max(1, round(Fraction(frame_count, 1) * target / source))
source_numerator = source.numerator
source_denominator = source.denominator
target_numerator = target.numerator
target_denominator = target.denominator
indices = [
min(
frame_count - 1,
(
index
* source_numerator
* target_denominator
// (source_denominator * target_numerator)
),
)
for index in range(target_count)
]
index_tensor = torch.tensor(indices, dtype=torch.long, device=images.device)
return images.index_select(0, index_tensor)
def load_video_components(
*,
file: str,
enable_video: bool,
force_rate: float,
):
"""Decode through Core and return standard IMAGE/AUDIO outputs."""
if not enable_video:
blocker = ExecutionBlocker(None)
return blocker, blocker
video_path = folder_paths.get_annotated_filepath(file)
components = InputImpl.VideoFromFile(video_path).get_components()
images = _resample_video_frames(
components.images,
Fraction(components.frame_rate),
float(force_rate),
)
audio = components.audio
if audio is None:
audio = ExecutionBlocker(None)
return images, audio
class H3ContinuumLoadVideo(io.ComfyNode):
"""Minimal Core-backed video loader for H3 Continuum workflows."""
@classmethod
def define_schema(cls):
input_dir = folder_paths.get_input_directory()
files = [
name
for name in os.listdir(input_dir)
if os.path.isfile(os.path.join(input_dir, name))
]
files = folder_paths.filter_files_content_types(files, ["video"])
return io.Schema(
node_id="H3ContinuumLoadVideo",
is_deprecated=True,
display_name="H3 Continuum Load Video",
category=CATEGORY,
description=(
"Loads IMAGE and AUDIO through ComfyUI Core VideoFromFile. "
"Force Rate drops or duplicates frames without changing duration or audio."
),
search_aliases=[
"H3 Continuum Load Video",
"H3 Easy Load Video",
"MiniMax H3 Video Loader",
],
inputs=[
io.Boolean.Input(
"enable_video",
default=True,
display_name="Enable Video",
),
io.Combo.Input(
"file",
options=sorted(files),
upload=io.UploadType.video,
display_name="Video",
),
io.Float.Input(
"force_rate",
default=24.0,
min=0.0,
max=120.0,
step=0.01,
display_name="Force Rate",
tooltip=(
"0 uses the source FPS. A positive value resamples frames "
"while preserving nominal duration."
),
),
],
outputs=[
io.Image.Output(display_name="images"),
io.Audio.Output(display_name="audio"),
],
)
@classmethod
def execute(cls, enable_video: bool, file: str, force_rate: float):
return io.NodeOutput(
*load_video_components(
file=file,
enable_video=bool(enable_video),
force_rate=float(force_rate),
)
)
@classmethod
def fingerprint_inputs(cls, enable_video: bool, file: str, force_rate: float):
enabled = bool(enable_video)
file_name = str(file)
modification_time = None
if enabled and folder_paths.exists_annotated_filepath(file_name):
video_path = folder_paths.get_annotated_filepath(file_name)
modification_time = os.path.getmtime(video_path)
return enabled, file_name, float(force_rate), modification_time
@classmethod
def validate_inputs(cls, enable_video: bool, file: str, force_rate: float):
del force_rate
if not bool(enable_video):
return True
if not folder_paths.exists_annotated_filepath(file):
return f"Invalid video file: {file}"
return True
NODE_CLASS_MAPPINGS = {
"H3ContinuumLoadVideo": H3ContinuumLoadVideo,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumLoadVideo": "H3 Continuum Load Video",
}
+191
View File
@@ -0,0 +1,191 @@
"""Pure physical-group planner for the staged V3.8 runtime redesign.
The caller owns continuation-source priority and passes exactly one selected
``ContinuationSourceFacts``. This module performs no repository, Session,
State, tensor, ComfyUI, or sampling work.
"""
from __future__ import annotations
from ..branch_provenance import BranchProvenanceError, physical_groups
from .planning_types import (
ContinuationSourceFacts,
ExecutionPlan,
PhysicalGroupDescriptor,
ProjectionDecision,
)
from .review_control import (
REVISION_STATUS_COMPLETE,
REVISION_STATUS_INTERRUPTED,
REVISION_STATUS_REVIEW_READY,
ReviewExecution,
ReviewUnit,
)
PROJECTION_FULL_ACCEPTED_PREFIX = "full_accepted_prefix"
PROJECTION_CURRENT_REVIEW_UNIT = "current_review_unit"
class ExecutionPlanningError(ValueError):
"""Raised when already-selected planning facts are internally inconsistent."""
def resolve_projection_decision(
*,
review_execution: ReviewExecution | None,
effective_result_status: str,
accepted_chunks: int,
physical_group_facts: tuple[tuple[int, ...], ...],
) -> ProjectionDecision:
"""Resolve output-only projection from finalized, source-neutral facts."""
full = ProjectionDecision(PROJECTION_FULL_ACCEPTED_PREFIX)
if str(effective_result_status) != REVISION_STATUS_REVIEW_READY:
return full
if review_execution is None or bool(
getattr(review_execution, "finish_remaining", False)
):
return full
if not (
bool(getattr(review_execution, "partial_review", False))
or bool(getattr(review_execution, "smart_regenerate", False))
):
return full
start = getattr(review_execution, "next_review_unit_start", None)
end = getattr(review_execution, "next_review_unit_end", None)
if start is None or end is None:
return full
start, end = int(start), int(end)
if start < 1 or end < start or end > int(accepted_chunks):
return full
selected = set(range(start, end + 1))
if not physical_group_facts:
return full
for group in physical_group_facts:
group_indices = set(group)
if selected.intersection(group_indices) and not group_indices.issubset(selected):
return full
return ProjectionDecision(PROJECTION_CURRENT_REVIEW_UNIT, start, end)
def build_execution_plan(
*,
continuation_source_facts: ContinuationSourceFacts,
configured_chunks: int,
terminal_merge_enabled: bool,
max_new_physical_groups: int | None,
review_execution: ReviewExecution | None,
retained_review_unit: ReviewUnit | None = None,
) -> ExecutionPlan:
"""Plan physical groups without selecting or validating runtime sources."""
if not isinstance(continuation_source_facts, ContinuationSourceFacts):
raise ExecutionPlanningError(
"continuation_source_facts must be one selected ContinuationSourceFacts"
)
if continuation_source_facts.kind not in {
"run_storage",
"explicit_session",
"initial_state",
"none",
}:
raise ExecutionPlanningError(
f"unknown continuation source kind: {continuation_source_facts.kind!r}"
)
if isinstance(configured_chunks, bool) or not isinstance(configured_chunks, int):
raise ExecutionPlanningError("configured_chunks must be an integer")
if configured_chunks < 1:
raise ExecutionPlanningError("configured_chunks must be positive")
if type(terminal_merge_enabled) is not bool:
raise ExecutionPlanningError("terminal_merge_enabled must be a boolean")
if max_new_physical_groups is not None and (
isinstance(max_new_physical_groups, bool)
or not isinstance(max_new_physical_groups, int)
or max_new_physical_groups < 0
):
raise ExecutionPlanningError(
"max_new_physical_groups must be None or a non-negative integer"
)
accepted = continuation_source_facts.accepted_chunks
if isinstance(accepted, bool) or not isinstance(accepted, int):
raise ExecutionPlanningError("accepted_chunks must be an integer")
if not 0 <= accepted <= configured_chunks:
raise ExecutionPlanningError(
"accepted_chunks must be within the configured sequence"
)
try:
runtime_groups = physical_groups(
chunks=configured_chunks,
terminal_merge_enabled=terminal_merge_enabled,
)
except BranchProvenanceError as exc:
raise ExecutionPlanningError(str(exc)) from exc
descriptors = tuple(
PhysicalGroupDescriptor(
physical_group=int(group.physical_group),
logical_chunks=tuple(range(int(group.start), int(group.end) + 1)),
terminal_atomic=int(group.start) != int(group.end),
)
for group in runtime_groups
)
accepted_boundaries = tuple(
(descriptor.logical_chunks[0], descriptor.logical_chunks[-1])
for descriptor in descriptors
if descriptor.logical_chunks[-1] <= accepted
)
if accepted_boundaries and accepted_boundaries[-1][1] != accepted:
raise ExecutionPlanningError("accepted prefix ends inside a physical group")
if accepted and not accepted_boundaries:
raise ExecutionPlanningError("accepted prefix has no complete physical group")
if continuation_source_facts.physical_group_boundaries != accepted_boundaries:
raise ExecutionPlanningError(
"selected continuation source physical groups do not match its prefix"
)
pending = tuple(
descriptor
for descriptor in descriptors
if descriptor.logical_chunks[0] > accepted
)
groups_to_generate = (
pending
if max_new_physical_groups is None
else pending[:max_new_physical_groups]
)
expected_chunks = accepted + sum(
len(descriptor.logical_chunks) for descriptor in groups_to_generate
)
expected_review_unit = retained_review_unit
if groups_to_generate and max_new_physical_groups is not None:
final_group = groups_to_generate[-1]
expected_review_unit = ReviewUnit(
start=final_group.logical_chunks[0],
end=final_group.logical_chunks[-1],
physical_group=final_group.physical_group,
)
if expected_chunks == configured_chunks:
expected_status = REVISION_STATUS_COMPLETE
elif max_new_physical_groups is not None and expected_review_unit is not None:
expected_status = REVISION_STATUS_REVIEW_READY
else:
expected_status = REVISION_STATUS_INTERRUPTED
projection = resolve_projection_decision(
review_execution=review_execution,
effective_result_status=expected_status,
accepted_chunks=expected_chunks,
physical_group_facts=tuple(
descriptor.logical_chunks for descriptor in descriptors
),
)
return ExecutionPlan(
groups_to_generate=groups_to_generate,
projection=projection,
expected_status=expected_status,
expected_review_unit=expected_review_unit,
)
+282
View File
@@ -0,0 +1,282 @@
"""V3.5 adapter for the accepted P2-0 file-backed Tensor backend.
The storage and lifetime implementation remains in the P2-0 module so the
Windows-tested locking, sidecar, cancellation, and alias-lifetime contracts
have one implementation. This module only exposes the small Production API
needed by the V3.5 assembler.
"""
from __future__ import annotations
from dataclasses import dataclass
import os
from pathlib import Path
import shutil
import tempfile
import threading
import torch
from ..storage_memory import system_memory_bytes
from .. import storage_settings
from ..tools.p20_file_backed_tensor_poc import (
FileBackedAllocation,
StorageLifetimeManager,
inspect_managed_files,
get_process_lifetime_manager,
)
BUFFER_BACKEND_AUTO = "Auto"
BUFFER_BACKEND_RAM = "RAM"
BUFFER_BACKEND_DISK = "Disk-backed"
BUFFER_BACKEND_OPTIONS = (
BUFFER_BACKEND_AUTO,
BUFFER_BACKEND_RAM,
BUFFER_BACKEND_DISK,
)
AUTO_RAM_IMAGE_LIMIT_BYTES = 4 * 1024**3
AUTO_RAM_RESERVE_MIN_BYTES = 4 * 1024**3
AUTO_RAM_RESERVE_FRACTION = 0.10
AUTO_DISK_RESERVE_BYTES = 2 * 1024**3
_BACKING_DIRECTORY_NAME = "h3-continuum-v35-file-backed"
_CLEANED_ROOTS: set[str] = set()
_CLEANED_ROOTS_LOCK = threading.Lock()
def validate_buffer_backend(value: str) -> str:
"""Validate the explicit or automatic V3.5 IMAGE backend request."""
if value not in BUFFER_BACKEND_OPTIONS:
raise ValueError(f"unknown Buffer Backend: {value!r}")
return value
@dataclass(frozen=True)
class AutoBufferDecision:
selected_backend: str
final_image_bytes: int
backing_root: Path
available_memory_bytes: int | None
total_memory_bytes: int | None
memory_reserve_bytes: int | None
ram_image_limit_bytes: int
disk_free_bytes: int | None
disk_reserve_bytes: int
reason: str
def _system_memory_bytes() -> tuple[int | None, int | None]:
return system_memory_bytes()
def _existing_disk_anchor(path: Path) -> Path:
candidate = path
while not candidate.exists() and candidate != candidate.parent:
candidate = candidate.parent
if not candidate.exists():
raise OSError(f"no existing filesystem anchor for backing root: {path}")
return candidate
def _disk_free_bytes(path: Path) -> int:
manager = storage_settings.get_manager()
if manager is not None:
manager.verify_final_root(path)
return int(shutil.disk_usage(_existing_disk_anchor(path)).free)
def preflight_manual_buffer(backend, final_image_bytes, backing_root=None):
if backend == BUFFER_BACKEND_RAM:
available, total = _system_memory_bytes()
if available is None or total is None:
raise RuntimeError("RAM buffer safety check unavailable; use Auto or Disk-backed, or restore memory counters")
reserve = max(AUTO_RAM_RESERVE_MIN_BYTES, int(total * AUTO_RAM_RESERVE_FRACTION))
if available < final_image_bytes + reserve:
raise RuntimeError(f"RAM buffer needs {final_image_bytes} bytes plus {reserve} bytes reserve; "
f"only {available} bytes are available. Select Auto or Disk-backed.")
elif backend == BUFFER_BACKEND_DISK:
root = resolve_backing_root(backing_root)
free = _disk_free_bytes(root)
if free < final_image_bytes + AUTO_DISK_RESERVE_BYTES:
raise RuntimeError(f"Disk-backed buffer needs {final_image_bytes + AUTO_DISK_RESERVE_BYTES} bytes "
f"including reserve; {free} bytes are free at {root}")
def select_auto_buffer_backend(
final_image_bytes: int,
*,
backing_root: str | os.PathLike[str] | None = None,
) -> AutoBufferDecision:
"""Choose RAM only when both output size and physical headroom are safe.
Auto never converts a Disk-backed decision into RAM after a disk-space or
allocation failure. Users can still explicitly select either manual path.
"""
final_bytes = int(final_image_bytes)
if final_bytes <= 0:
raise ValueError("Auto Buffer Backend requires a positive final IMAGE size")
root = resolve_backing_root(backing_root)
available, total = _system_memory_bytes()
reserve = None
if available is not None and total is not None and total > 0:
reserve = max(
AUTO_RAM_RESERVE_MIN_BYTES,
int(total * AUTO_RAM_RESERVE_FRACTION),
)
if (
reserve is not None
and final_bytes <= AUTO_RAM_IMAGE_LIMIT_BYTES
and available >= final_bytes + reserve
):
return AutoBufferDecision(
selected_backend=BUFFER_BACKEND_RAM,
final_image_bytes=final_bytes,
backing_root=root,
available_memory_bytes=available,
total_memory_bytes=total,
memory_reserve_bytes=reserve,
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
disk_free_bytes=None,
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
reason="final IMAGE fits the Auto RAM size limit and safety reserve",
)
if reserve is None:
reason = "physical-memory counters are unavailable"
elif final_bytes > AUTO_RAM_IMAGE_LIMIT_BYTES:
reason = "final IMAGE exceeds the Auto RAM size limit"
else:
reason = "available physical memory is below final IMAGE plus reserve"
try:
disk_free = _disk_free_bytes(root)
except Exception as exc:
raise RuntimeError(
"Auto Buffer Backend selected Disk-backed but disk capacity could "
f"not be measured for {root}: {type(exc).__name__}: {exc}. "
"Select RAM manually only if sufficient memory is known to be available."
) from exc
required_disk = final_bytes + AUTO_DISK_RESERVE_BYTES
if disk_free < required_disk:
raise RuntimeError(
"Auto Buffer Backend selected Disk-backed but the backing volume "
f"has {disk_free} free bytes; {required_disk} bytes are required "
f"for the final IMAGE plus the {AUTO_DISK_RESERVE_BYTES}-byte safety "
"reserve. Free disk space or select RAM manually only if sufficient "
"memory is known to be available."
)
return AutoBufferDecision(
selected_backend=BUFFER_BACKEND_DISK,
final_image_bytes=final_bytes,
backing_root=root,
available_memory_bytes=available,
total_memory_bytes=total,
memory_reserve_bytes=reserve,
ram_image_limit_bytes=AUTO_RAM_IMAGE_LIMIT_BYTES,
disk_free_bytes=disk_free,
disk_reserve_bytes=AUTO_DISK_RESERVE_BYTES,
reason=reason,
)
def format_auto_buffer_decision(decision: AutoBufferDecision) -> str:
def value(number: int | None) -> str:
return "n/a" if number is None else str(int(number))
return (
"Auto Buffer Decision: "
f"selected={decision.selected_backend}, "
f"final_IMAGE={decision.final_image_bytes} bytes, "
f"system_available={value(decision.available_memory_bytes)} bytes, "
f"system_total={value(decision.total_memory_bytes)} bytes, "
f"RAM_reserve={value(decision.memory_reserve_bytes)} bytes, "
f"RAM_IMAGE_limit={decision.ram_image_limit_bytes} bytes, "
f"disk_free={value(decision.disk_free_bytes)} bytes, "
f"disk_reserve={decision.disk_reserve_bytes} bytes; "
f"reason={decision.reason}."
)
def resolve_backing_root(
backing_root: str | os.PathLike[str] | None = None,
) -> Path:
"""Resolve the dedicated backing root without importing ComfyUI eagerly."""
if backing_root is not None:
return Path(backing_root).expanduser().resolve()
try:
import folder_paths
except ImportError:
base = Path(tempfile.gettempdir())
else:
value = folder_paths.get_temp_directory()
if value is None or not str(value).strip():
raise RuntimeError("ComfyUI returned an empty temporary directory")
base = Path(value)
default = (base.expanduser().resolve() / _BACKING_DIRECTORY_NAME).resolve()
manager = storage_settings.get_manager()
return manager.resolve("finalize", default, verify=False) if manager else default
def get_file_backed_image_manager(
backing_root: str | os.PathLike[str] | None = None,
) -> StorageLifetimeManager:
"""Return the accepted process-global manager for one resolved root."""
root = resolve_backing_root(backing_root)
settings = storage_settings.get_manager()
if settings is not None:
settings.verify_final_root(root)
key = os.path.normcase(str(root))
with _CLEANED_ROOTS_LOCK:
if key not in _CLEANED_ROOTS:
# Production startup and the UI share conservative ownership rules.
# The standalone recovery probe can still inspect legacy orphans;
# an unknown production file is never deleted by its name alone.
inspect_managed_files(root, cleanup=True)
_CLEANED_ROOTS.add(key)
manager = get_process_lifetime_manager(root)
if settings is not None:
settings.record_root("finalize", root)
return manager
def allocate_file_backed_image(
shape: tuple[int, ...],
*,
dtype: torch.dtype,
backing_root: str | os.PathLike[str] | None = None,
) -> FileBackedAllocation:
"""Create a provisional Disk-backed IMAGE allocation.
The caller must use the returned context manager and call ``publish()``
only after every output element has been written. Backend failures are
propagated; this manual Disk-backed path never falls back to RAM.
"""
manager = get_file_backed_image_manager(backing_root)
return manager.allocate(shape, dtype=dtype)
__all__ = [
"AUTO_DISK_RESERVE_BYTES",
"AUTO_RAM_IMAGE_LIMIT_BYTES",
"AUTO_RAM_RESERVE_FRACTION",
"AUTO_RAM_RESERVE_MIN_BYTES",
"AutoBufferDecision",
"BUFFER_BACKEND_AUTO",
"BUFFER_BACKEND_DISK",
"BUFFER_BACKEND_OPTIONS",
"BUFFER_BACKEND_RAM",
"allocate_file_backed_image",
"format_auto_buffer_decision",
"get_file_backed_image_manager",
"resolve_backing_root",
"select_auto_buffer_backend",
"validate_buffer_backend",
]
+527
View File
@@ -0,0 +1,527 @@
"""V3.5 Continuum-aware Hi-res fix convenience node."""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from typing import Any
from .pixel_vae_resize import (
resize_video_latents_via_vae,
resolve_pixel_latent_scale,
)
from .second_pass import SecondPassContractError, run_second_pass_groups
_MISSING = object()
HIRES_UPSCALE_METHODS = (
"lanczos",
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
)
_CONDITIONING_UPSCALE_METHOD = "bilinear"
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
def _enabled_value(value: Any) -> bool:
if isinstance(value, list):
value = _single_list_input("enabled", value)
return bool(value)
def _lazy_missing(value: Any) -> bool:
return value is None or (
isinstance(value, (list, tuple))
and len(value) == 1
and value[0] is None
)
def h3_core_native_canvas(
width: int,
height: int,
*,
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
) -> tuple[int, int]:
"""Return the native canvas selected by ComfyUI Core for this aspect ratio."""
if width < 1 or height < 1:
raise SecondPassContractError("source pixel width and height must be positive")
if adapt_canvas_fn is None:
from comfy_extras.nodes_minimax_h3 import adapt_canvas
adapt_canvas_fn = adapt_canvas
target = adapt_canvas_fn(int(width), int(height))
if not isinstance(target, Sequence) or len(target) != 2:
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
target_width, target_height = (int(value) for value in target)
if target_width < 1 or target_height < 1:
raise SecondPassContractError("ComfyUI Core adapt_canvas returned invalid geometry")
return target_width, target_height
def align_h3_manual_canvas(
current_latent_width: int,
current_latent_height: int,
scale_by: float,
*,
pixel_scale_width: int,
pixel_scale_height: int,
canvas_multiple: int | None = None,
) -> dict[str, int]:
"""Scale and align a manual target to ComfyUI Core's H3 canvas grid."""
if canvas_multiple is None:
from comfy_extras.nodes_minimax_h3 import CANVAS_MULTIPLE
canvas_multiple = int(CANVAS_MULTIPLE)
current_latent_width = int(current_latent_width)
current_latent_height = int(current_latent_height)
pixel_scale_width = int(pixel_scale_width)
pixel_scale_height = int(pixel_scale_height)
canvas_multiple = int(canvas_multiple)
scale_by = float(scale_by)
if min(
current_latent_width,
current_latent_height,
pixel_scale_width,
pixel_scale_height,
canvas_multiple,
) < 1:
raise SecondPassContractError("manual H3 canvas geometry is invalid")
if (
canvas_multiple % pixel_scale_width
or canvas_multiple % pixel_scale_height
):
raise SecondPassContractError(
"ComfyUI Core H3 canvas multiple is not aligned to the latent scale"
)
def aligned_target(current_latent: int, pixel_scale: int) -> int:
current_pixels = current_latent * pixel_scale
scaled_pixels = current_pixels * scale_by
nearest_core_canvas = max(
canvas_multiple,
round(scaled_pixels / canvas_multiple) * canvas_multiple,
)
preserved_core_canvas = (
(current_pixels + canvas_multiple - 1) // canvas_multiple
) * canvas_multiple
return max(nearest_core_canvas, preserved_core_canvas)
target_width = aligned_target(current_latent_width, pixel_scale_width)
target_height = aligned_target(current_latent_height, pixel_scale_height)
return {
"target_width": target_width,
"target_height": target_height,
"target_latent_width": target_width // pixel_scale_width,
"target_latent_height": target_height // pixel_scale_height,
}
def resolve_h3_native_latent_target(
video_latents: Sequence[Mapping[str, Any]],
assembly_plan: Mapping[str, Any],
*,
adapt_canvas_fn: Callable[[int, int], tuple[int, int]] | None = None,
) -> dict[str, int | bool]:
"""Resolve Core's native canvas without shrinking an existing video latent."""
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise SecondPassContractError("assembly_plan has no second_pass_contract")
groups = contract.get("physical_groups")
if not isinstance(groups, list) or not groups:
raise SecondPassContractError(
"second_pass_contract.physical_groups must be non-empty"
)
if not isinstance(video_latents, Sequence) or not video_latents:
raise SecondPassContractError("video_latents must be a non-empty sequence")
if len(video_latents) != len(groups):
raise SecondPassContractError("video latent group count changed")
first_group = groups[0]
if not isinstance(first_group, Mapping):
raise SecondPassContractError("physical group metadata must be a mapping")
source_width = int(first_group.get("source_width", -1))
source_height = int(first_group.get("source_height", -1))
source_latent_width = int(first_group.get("source_latent_w", -1))
source_latent_height = int(first_group.get("source_latent_h", -1))
if min(source_width, source_height, source_latent_width, source_latent_height) < 1:
raise SecondPassContractError("source pixel/latent geometry is invalid")
if source_width % source_latent_width or source_height % source_latent_height:
raise SecondPassContractError("source pixel/latent geometry is not integral")
pixel_scale_width = source_width // source_latent_width
pixel_scale_height = source_height // source_latent_height
current_geometry: tuple[int, int] | None = None
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
if not isinstance(group, Mapping):
raise SecondPassContractError("physical group metadata must be a mapping")
if int(group.get("source_width", -1)) != source_width or int(
group.get("source_height", -1)
) != source_height:
raise SecondPassContractError("source geometry differs between physical groups")
if int(group.get("source_latent_w", -1)) != source_latent_width or int(
group.get("source_latent_h", -1)
) != source_latent_height:
raise SecondPassContractError(
"source latent geometry differs between physical groups"
)
if not isinstance(latent, Mapping) or "samples" not in latent:
raise SecondPassContractError(
f"video_latents[{position}] must be a LATENT mapping with samples"
)
samples = latent["samples"]
if not hasattr(samples, "shape") or len(samples.shape) != 5:
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
geometry = (int(samples.shape[-2]), int(samples.shape[-1]))
if current_geometry is None:
current_geometry = geometry
elif geometry != current_geometry:
raise SecondPassContractError(
"all physical groups must use the same current H/W"
)
assert current_geometry is not None
native_width, native_height = h3_core_native_canvas(
source_width,
source_height,
adapt_canvas_fn=adapt_canvas_fn,
)
if native_width % pixel_scale_width or native_height % pixel_scale_height:
raise SecondPassContractError(
"ComfyUI Core native canvas is not aligned to the H3 latent scale"
)
native_latent_width = native_width // pixel_scale_width
native_latent_height = native_height // pixel_scale_height
current_height, current_width = current_geometry
preserve_current = (
native_latent_width < current_width
or native_latent_height < current_height
)
if preserve_current:
target_latent_width = current_width
target_latent_height = current_height
else:
target_latent_width = native_latent_width
target_latent_height = native_latent_height
return {
"source_width": source_width,
"source_height": source_height,
"native_width": native_width,
"native_height": native_height,
"target_width": target_latent_width * pixel_scale_width,
"target_height": target_latent_height * pixel_scale_height,
"target_latent_width": target_latent_width,
"target_latent_height": target_latent_height,
"preserved_current": preserve_current,
}
def _prepend_resize_status(
second_pass_status: str,
*,
backend: str,
mode: str,
upscale_method: str,
scale_by: float,
source_width: int,
source_height: int,
target_width: int,
target_height: int,
native_width: int,
native_height: int,
) -> str:
lines = [
"H3 Continuum Hi-Res Fix V3.5",
"Pixel/VAE resize: "
f"backend={backend}, mode={mode}, method={upscale_method}, "
f"scale_by={scale_by:g}, "
f"source={source_width}x{source_height}, "
f"target={target_width}x{target_height}, "
f"core_native={native_width}x{native_height}.",
]
if target_width > native_width or target_height > native_height:
lines.append(
"INFO: target exceeds the ComfyUI Core H3 base canvas; the Pixel/VAE "
"roundtrip is preserved and resource use may increase."
)
lines.append(second_pass_status)
return "\n".join(lines)
class H3ContinuumHiResFixV35:
"""Pixel/VAE resize and low-sigma refine complete physical video groups."""
DEPRECATED = False
DESCRIPTION = (
"Continuum-aware Hi-res fix. When enabled, each first-pass physical video "
"LATENT is decoded with the connected H3 Video VAE, resized in pixel space, "
"re-encoded with the same VAE, and refined once with the workflow SIGMAS. "
"scale_by 0 uses ComfyUI Core's H3 native canvas; positive values use the "
"manual multiplier. Connect the optional Continuum refine context to preserve "
"First/Last/Reference conditioning at the target geometry. "
"When disabled, first-pass video/audio LATENT objects and the original assembly "
"plan are returned unchanged."
)
@classmethod
def INPUT_TYPES(cls):
lazy = {"lazy": True}
return {
"required": {
"model": ("MODEL", lazy),
"clip": ("CLIP", lazy),
"sampler": ("SAMPLER", lazy),
"sigmas": ("SIGMAS", lazy),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"enabled": (
"BOOLEAN",
{
"default": True,
"tooltip": (
"Enable Resize + Second Pass. Off returns the first-pass "
"video/audio LATENT objects and assembly plan unchanged."
),
},
),
"upscale_method": (
HIRES_UPSCALE_METHODS,
{"default": "lanczos"},
),
"scale_by": (
"FLOAT",
{
"default": 2.0,
"min": 0.0,
"max": 4.0,
"step": 0.01,
"tooltip": (
"2.0 = validated 2x Pixel/VAE Hi-res Fix. 0 uses the "
"ComfyUI Core H3 native canvas automatically. Manual "
"values follow the existing Second Pass contract and must "
"be 1.0 or greater."
),
},
),
"refine_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"tooltip": "Base seed independently derived once per physical group.",
},
),
},
"optional": {
"refine_context": (
"H3_CONTINUUM_REFINE_CONTEXT",
{"lazy": True},
),
"video_vae": (
"VAE",
{
"lazy": True,
"tooltip": (
"Required while enabled for the safe Pixel/VAE roundtrip; "
"not evaluated while disabled."
),
},
),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = (
"video_latents",
"audio_latents",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "apply"
CATEGORY = "MiniMax H3/Continuum"
def check_lazy_status(
self,
enabled,
model=None,
clip=None,
sampler=None,
sigmas=None,
refine_context=_MISSING,
video_vae=_MISSING,
**_kwargs,
):
if not _enabled_value(enabled):
return []
required = [
name
for name, value in (
("model", model),
("clip", clip),
("sampler", sampler),
("sigmas", sigmas),
)
if _lazy_missing(value)
]
required.extend(
name
for name, value in (
("refine_context", refine_context),
("video_vae", video_vae),
)
if value is not _MISSING and _lazy_missing(value)
)
return required
def apply(
self,
model,
clip,
sampler,
sigmas,
video_latents,
audio_latents,
assembly_plan,
enabled,
upscale_method,
scale_by,
refine_seed,
refine_context=None,
video_vae=None,
refine_schedule=None,
):
plan = _single_list_input("assembly_plan", assembly_plan)
if not _enabled_value(enabled):
return (
video_latents,
audio_latents,
plan,
"H3 Continuum Hi-Res Fix V3.5: disabled; first-pass video/audio "
"LATENT objects and assembly_plan returned unchanged.",
)
method = str(_single_list_input("upscale_method", upscale_method))
scale = float(_single_list_input("scale_by", scale_by))
if 0.0 < scale < 1.0:
raise SecondPassContractError(
"scale_by must be 0 for H3 native Auto or at least 1.0; the "
"existing Second Pass contract preserves or enlarges spatial size"
)
vae = (
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
)
if vae is None:
raise SecondPassContractError(
"Hi-Res Fix enabled requires video_vae for the safe Pixel/VAE "
"roundtrip; legacy direct latent interpolation is disabled"
)
source_width = int(plan["width"])
source_height = int(plan["height"])
native_width, native_height = h3_core_native_canvas(
source_width,
source_height,
)
first_samples = video_latents[0]["samples"]
current_geometry = (
int(first_samples.shape[-2]),
int(first_samples.shape[-1]),
)
if scale == 0.0:
native_target = resolve_h3_native_latent_target(video_latents, plan)
target_latent_width = int(native_target["target_latent_width"])
target_latent_height = int(native_target["target_latent_height"])
target_width = int(native_target["target_width"])
target_height = int(native_target["target_height"])
resize_mode = "h3_native_canvas_auto"
else:
pixel_scale_width, pixel_scale_height = resolve_pixel_latent_scale(plan)
manual_target = align_h3_manual_canvas(
current_geometry[1],
current_geometry[0],
scale,
pixel_scale_width=pixel_scale_width,
pixel_scale_height=pixel_scale_height,
)
target_latent_width = int(manual_target["target_latent_width"])
target_latent_height = int(manual_target["target_latent_height"])
target_width = int(manual_target["target_width"])
target_height = int(manual_target["target_height"])
resize_mode = "manual_scale_by"
if current_geometry == (target_latent_height, target_latent_width):
resized_video_latents = list(video_latents)
resize_backend = "direct_second_pass_no_resize"
else:
resized_video_latents = resize_video_latents_via_vae(
video_latents,
plan,
video_vae=vae,
target_width=target_width,
target_height=target_height,
upscale_method=method,
)
resize_backend = "sequential_pixel_vae_roundtrip"
refined_video, output_audio, updated_plan, status = run_second_pass_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
sampler=_single_list_input("sampler", sampler),
sigmas=_single_list_input("sigmas", sigmas),
video_latents=resized_video_latents,
audio_latents=audio_latents,
assembly_plan=plan,
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
refine_context=(
None
if refine_context is None
else _single_list_input("refine_context", refine_context)
),
video_vae=vae,
conditioning_upscale_method=_CONDITIONING_UPSCALE_METHOD,
refine_schedule=refine_schedule,
)
return (
refined_video,
output_audio,
updated_plan,
_prepend_resize_status(
status,
backend=resize_backend,
mode=resize_mode,
upscale_method=method,
scale_by=scale,
source_width=source_width,
source_height=source_height,
target_width=int(updated_plan["width"]),
target_height=int(updated_plan["height"]),
native_width=native_width,
native_height=native_height,
),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumHiResFixV35": H3ContinuumHiResFixV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumHiResFixV35": "H3 Continuum Hi-Res Fix V3.5",
}
+140
View File
@@ -0,0 +1,140 @@
"""V3.5 Core-compatible spatial resize for physical video latents."""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
UPSCALE_METHODS = (
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
)
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
def resize_video_latents(
video_latents: list[dict[str, Any]],
*,
upscale_method: str,
scale_by: float,
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize only spatial H/W through the ComfyUI Core latent implementation."""
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
resized: list[dict[str, Any]] = []
for latent in video_latents:
samples = latent["samples"]
width = round(samples.shape[-1] * scale_by)
height = round(samples.shape[-2] * scale_by)
output = latent.copy()
output["samples"] = upscale_fn(
samples,
width,
height,
upscale_method,
"disabled",
)
resized.append(output)
return resized
def resize_video_latents_to_size(
video_latents: list[dict[str, Any]],
*,
upscale_method: str,
target_width: int,
target_height: int,
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize every physical video group to one exact latent H/W."""
if target_width < 1 or target_height < 1:
raise ValueError("target latent width and height must be positive")
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
resized: list[dict[str, Any]] = []
for latent in video_latents:
samples = latent["samples"]
output = latent.copy()
output["samples"] = upscale_fn(
samples,
int(target_width),
int(target_height),
upscale_method,
"disabled",
)
resized.append(output)
return resized
class H3ContinuumLatentResizeV35:
"""List-aware Core spatial resize for Continuum physical video groups."""
DEPRECATED = False
DESCRIPTION = (
"V3.5 Hi-res fix resize. Spatially resizes every Continuum "
"physical video LATENT with the selected ComfyUI Core interpolation method. "
"B/C/T, physical group order, and LATENT metadata are preserved."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"video_latents": ("LATENT",),
"upscale_method": (UPSCALE_METHODS,),
"scale_by": (
"FLOAT",
{
"default": 2.0,
"min": 0.01,
"max": 8.0,
"step": 0.01,
},
),
}
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT",)
RETURN_NAMES = ("resized_video_latents",)
OUTPUT_IS_LIST = (True,)
FUNCTION = "resize"
CATEGORY = "MiniMax H3/Continuum/Utilities"
def resize(self, video_latents, upscale_method, scale_by):
return (
resize_video_latents(
video_latents,
upscale_method=str(
_single_list_input("upscale_method", upscale_method)
),
scale_by=float(_single_list_input("scale_by", scale_by)),
),
)
NODE_CLASS_MAPPINGS = {
"H3ContinuumLatentResizeV35": H3ContinuumLatentResizeV35,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumLatentResizeV35": "H3 Continuum Latent Resize V3.5",
}
+236
View File
@@ -0,0 +1,236 @@
"""Private V3.6 masked-prefix continuation experiments.
This module is intentionally not registered as a public node contract. It
builds H3 AV targets whose protected prefixes are preserved by Core's denoise
mask. R1 protects video only; R2-0 can protect matching video and audio tails.
"""
from __future__ import annotations
from typing import Any
import torch
from ..state import extract_av_streams
from ..temporal import audio_latent_t, context_slots
REFERENCE_CONTEXT_V1 = "reference_context_v1"
MASKED_VIDEO_PREFIX_V1 = "masked_video_prefix_v1"
MASKED_AV_PREFIX_22_V1 = "masked_av_prefix_22_v1"
MASKED_AV_PREFIX_39_V1 = "masked_av_prefix_39_v1"
CONTINUATION_TRANSPORTS = (
REFERENCE_CONTEXT_V1,
MASKED_VIDEO_PREFIX_V1,
MASKED_AV_PREFIX_22_V1,
MASKED_AV_PREFIX_39_V1,
)
def _build_masked_prefix_latent(
target_latent: dict[str, Any],
video_context: torch.Tensor,
context_frames: int,
*,
audio_context: torch.Tensor | None,
) -> dict[str, Any]:
"""Copy normalized AV tails into a target and attach stream masks."""
try:
import comfy.nested_tensor
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
target_video, target_audio = extract_av_streams(target_latent)
frames = int(context_frames)
slots = context_slots(frames)
if not torch.is_tensor(video_context) or video_context.ndim != 5:
raise ValueError("masked video context must be a five-dimensional Tensor")
if tuple(video_context.shape[:2]) != tuple(target_video.shape[:2]):
raise ValueError(
"masked video context batch/channel shape does not match the target: "
f"context={tuple(video_context.shape)}, target={tuple(target_video.shape)}"
)
if int(video_context.shape[2]) != slots:
raise ValueError(
f"masked video context has T={video_context.shape[2]}; expected {slots} "
f"for {frames} frames"
)
if tuple(video_context.shape[-2:]) != tuple(target_video.shape[-2:]):
raise ValueError(
"masked video context geometry does not match the target: "
f"context={tuple(video_context.shape[-2:])}, "
f"target={tuple(target_video.shape[-2:])}"
)
if int(target_video.shape[2]) <= slots:
raise ValueError("masked video target must contain generated slots after the prefix")
normalized_video = video_context.detach().to(
device=target_video.device,
dtype=target_video.dtype,
).contiguous()
if not bool(torch.isfinite(normalized_video.float()).all().item()):
raise ValueError("masked video context contains NaN or Inf")
normalized_audio = None
audio_steps = 0
if audio_context is not None:
audio_steps = audio_latent_t(frames)
if not torch.is_tensor(audio_context) or audio_context.ndim != 4:
raise ValueError("masked audio context must be a four-dimensional Tensor")
if tuple(audio_context.shape[:3]) != tuple(target_audio.shape[:3]):
raise ValueError(
"masked audio context batch/channel shape does not match the target: "
f"context={tuple(audio_context.shape)}, target={tuple(target_audio.shape)}"
)
if int(audio_context.shape[-1]) != audio_steps:
raise ValueError(
f"masked audio context has T={audio_context.shape[-1]}; expected "
f"{audio_steps} for {frames} frames"
)
if int(target_audio.shape[-1]) <= audio_steps:
raise ValueError("masked audio target must contain generated ticks after the prefix")
normalized_audio = audio_context.detach().to(
device=target_audio.device,
dtype=target_audio.dtype,
).contiguous()
if not bool(torch.isfinite(normalized_audio.float()).all().item()):
raise ValueError("masked audio context contains NaN or Inf")
video = target_video.clone()
audio = target_audio.clone()
video[:, :, :slots].copy_(normalized_video)
if normalized_audio is not None:
audio[..., :audio_steps].copy_(normalized_audio)
video_mask = torch.ones(
(
int(video.shape[0]),
1,
int(video.shape[2]),
int(video.shape[3]),
int(video.shape[4]),
),
device=video.device,
dtype=torch.float32,
)
video_mask[:, :, :slots] = 0.0
audio_mask = torch.ones(
(
int(audio.shape[0]),
1,
int(audio.shape[2]),
int(audio.shape[3]),
),
device=audio.device,
dtype=torch.float32,
)
if normalized_audio is not None:
audio_mask[..., :audio_steps] = 0.0
result = dict(target_latent)
result["samples"] = comfy.nested_tensor.NestedTensor((video, audio))
result["noise_mask"] = comfy.nested_tensor.NestedTensor(
(video_mask, audio_mask)
)
return result
def build_masked_video_prefix_latent(
target_latent: dict[str, Any],
video_context: torch.Tensor,
context_frames: int,
) -> dict[str, Any]:
"""Copy a normalized video tail into the target and attach AV noise masks."""
return _build_masked_prefix_latent(
target_latent,
video_context,
context_frames,
audio_context=None,
)
def build_masked_av_prefix_latent(
target_latent: dict[str, Any],
video_context: torch.Tensor,
audio_context: torch.Tensor,
context_frames: int,
) -> dict[str, Any]:
"""Copy matching video/audio tails and preserve both with denoise masks."""
return _build_masked_prefix_latent(
target_latent,
video_context,
context_frames,
audio_context=audio_context,
)
def _restore_masked_prefix(
sampled_latent: dict[str, Any],
target_latent: dict[str, Any],
context_frames: int,
*,
restore_audio: bool,
) -> dict[str, Any]:
"""Reapply only protected prefix regions after sampler integration."""
try:
import comfy.nested_tensor
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RuntimeError(f"ComfyUI NestedTensor helper unavailable: {exc}") from exc
sampled_video, sampled_audio = extract_av_streams(sampled_latent)
target_video, target_audio = extract_av_streams(target_latent)
slots = context_slots(int(context_frames))
if tuple(sampled_video.shape) != tuple(target_video.shape):
raise ValueError(
"sampled video shape does not match the masked target: "
f"sampled={tuple(sampled_video.shape)}, target={tuple(target_video.shape)}"
)
restored_video = sampled_video.clone()
restored_video[:, :, :slots].copy_(target_video[:, :, :slots])
restored_audio = sampled_audio
if restore_audio:
if tuple(sampled_audio.shape) != tuple(target_audio.shape):
raise ValueError(
"sampled audio shape does not match the masked target: "
f"sampled={tuple(sampled_audio.shape)}, target={tuple(target_audio.shape)}"
)
audio_steps = audio_latent_t(int(context_frames))
restored_audio = sampled_audio.clone()
restored_audio[..., :audio_steps].copy_(target_audio[..., :audio_steps])
result = dict(sampled_latent)
result["samples"] = comfy.nested_tensor.NestedTensor(
(restored_video, restored_audio)
)
return result
def restore_masked_video_prefix(
sampled_latent: dict[str, Any],
target_latent: dict[str, Any],
context_frames: int,
) -> dict[str, Any]:
"""Reapply only the preserved prefix after sampler integration rounding."""
return _restore_masked_prefix(
sampled_latent,
target_latent,
context_frames,
restore_audio=False,
)
def restore_masked_av_prefix(
sampled_latent: dict[str, Any],
target_latent: dict[str, Any],
context_frames: int,
) -> dict[str, Any]:
"""Reapply only protected video and audio prefixes after Sampling."""
return _restore_masked_prefix(
sampled_latent,
target_latent,
context_frames,
restore_audio=True,
)
+221
View File
@@ -0,0 +1,221 @@
"""Explicit opt-in A7b memory actions that preserve Production ownership.
This module resolves only Reference sizing policy. Model unloading and
attention/backend selection remain advisory because ComfyUI Core and external
MODEL wrappers own those lifecycles.
"""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
from typing import Any
from ..reference import REFERENCE_SIZE_MATCH_OUTPUT, REFERENCE_SIZE_MAX_IDENTITY
from ..reference_video import (
REFERENCE_VIDEO_SIZE_BALANCED,
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
)
MEMORY_ACTION_POLICY_TYPE = "H3_CONTINUUM_MEMORY_ACTION_POLICY"
MEMORY_ACTION_POLICY_VERSION = 1
ACTION_DISABLED = "Disabled"
ACTION_REDUCE_REFERENCES_ONE_STEP = "Reduce References One Step"
ACTION_OPTIONS = (
ACTION_DISABLED,
ACTION_REDUCE_REFERENCES_ONE_STEP,
)
_IMAGE_DOWNGRADE = {
REFERENCE_SIZE_MAX_IDENTITY: REFERENCE_SIZE_MATCH_OUTPUT,
REFERENCE_SIZE_MATCH_OUTPUT: REFERENCE_SIZE_MATCH_OUTPUT,
}
_VIDEO_DOWNGRADE = {
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT: REFERENCE_VIDEO_SIZE_BALANCED,
REFERENCE_VIDEO_SIZE_BALANCED: REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_EFFICIENT: REFERENCE_VIDEO_SIZE_EFFICIENT,
}
def _canonical_hash(value: Any) -> str:
payload = json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def _contract_base(action: str) -> dict[str, Any]:
return {
"memory_action_policy_version": MEMORY_ACTION_POLICY_VERSION,
"action": action,
"reference_downgrade": (
"one_step"
if action == ACTION_REDUCE_REFERENCES_ONE_STEP
else "disabled"
),
"auto_unload": "advisory_only",
"backend_switch": "advisory_only",
"same_run_actual_row_trigger": False,
}
def make_memory_action_policy(action: str = ACTION_DISABLED) -> dict[str, Any]:
"""Build the versioned policy object emitted by the public helper node."""
normalized = str(action)
if normalized not in ACTION_OPTIONS:
raise ValueError(f"unsupported memory action {action!r}")
contract = _contract_base(normalized)
return {**contract, "contract_hash": _canonical_hash(contract)}
@dataclass(frozen=True)
class MemoryActionDecision:
action: str
enabled: bool
requested_image_mode: str
effective_image_mode: str
requested_video_mode: str
effective_video_mode: str
image_applied: bool
video_applied: bool
contract_hash: str | None
warning: str | None = None
@property
def applied(self) -> bool:
return bool(self.image_applied or self.video_applied)
def _disabled_decision(
*,
image_mode: str,
video_mode: str,
contract_hash: str | None = None,
warning: str | None = None,
) -> MemoryActionDecision:
return MemoryActionDecision(
action=ACTION_DISABLED,
enabled=False,
requested_image_mode=image_mode,
effective_image_mode=image_mode,
requested_video_mode=video_mode,
effective_video_mode=video_mode,
image_applied=False,
video_applied=False,
contract_hash=contract_hash,
warning=warning,
)
def resolve_memory_action_policy(
policy: Any,
*,
image_mode: str,
video_mode: str,
has_reference_images: bool,
has_video_guide: bool,
) -> MemoryActionDecision:
"""Resolve an optional policy without changing tensors or runtime state.
Malformed or unsupported objects fail soft to Disabled. The caller may
surface ``warning`` after generation, but Sampling inputs remain unchanged.
"""
requested_image = str(image_mode)
requested_video = str(video_mode)
if policy is None:
return _disabled_decision(
image_mode=requested_image,
video_mode=requested_video,
)
if not isinstance(policy, dict):
return _disabled_decision(
image_mode=requested_image,
video_mode=requested_video,
warning="invalid policy object; no memory action was applied",
)
action = str(policy.get("action", ""))
try:
version = int(policy.get("memory_action_policy_version"))
except (TypeError, ValueError):
version = -1
supplied_hash = policy.get("contract_hash")
expected_contract = _contract_base(action) if action in ACTION_OPTIONS else None
expected_hash = (
_canonical_hash(expected_contract) if expected_contract is not None else None
)
if (
version != MEMORY_ACTION_POLICY_VERSION
or action not in ACTION_OPTIONS
or not isinstance(supplied_hash, str)
or supplied_hash != expected_hash
):
return _disabled_decision(
image_mode=requested_image,
video_mode=requested_video,
warning="unsupported or modified policy contract; no memory action was applied",
)
if action == ACTION_DISABLED:
return _disabled_decision(
image_mode=requested_image,
video_mode=requested_video,
contract_hash=supplied_hash,
)
effective_image = requested_image
effective_video = requested_video
if bool(has_reference_images):
effective_image = _IMAGE_DOWNGRADE.get(requested_image, requested_image)
if bool(has_video_guide):
effective_video = _VIDEO_DOWNGRADE.get(requested_video, requested_video)
return MemoryActionDecision(
action=action,
enabled=True,
requested_image_mode=requested_image,
effective_image_mode=effective_image,
requested_video_mode=requested_video,
effective_video_mode=effective_video,
image_applied=(
bool(has_reference_images) and effective_image != requested_image
),
video_applied=(bool(has_video_guide) and effective_video != requested_video),
contract_hash=supplied_hash,
)
def format_memory_action_status(decision: MemoryActionDecision) -> str | None:
"""Format post-generation status without claiming VRAM prediction."""
if decision.warning is not None:
return "Memory Action Policy [Experimental A7b v1]: " + decision.warning + "."
if not decision.enabled:
return None
changes: list[str] = []
if decision.image_applied:
changes.append(
"Reference Image "
f"{decision.requested_image_mode} -> {decision.effective_image_mode}"
)
if decision.video_applied:
changes.append(
"Video Guide "
f"{decision.requested_video_mode} -> {decision.effective_video_mode}"
)
action = "; ".join(changes) if changes else "no applicable Reference downgrade"
return (
"Memory Action Policy [Experimental A7b v1]: explicit opt-in; "
f"{action}.\n"
"Memory advisory: automatic MODEL unload was not executed; Core retains "
"lifecycle ownership.\n"
"Memory advisory: automatic attention/backend switching was not executed; "
"the external MODEL wrapper was preserved."
)
+416
View File
@@ -0,0 +1,416 @@
"""V3.5-only, non-synchronizing memory attribution helpers."""
from __future__ import annotations
from dataclasses import dataclass
import math
import time
from typing import Any, Mapping
import torch
from ..constants import FPS
from ..hardening import tensor_storage_stats
from ..state import extract_av_streams
from .plan import validate_assembly_plan
_MIB = 1024**2
_CONDITIONING_SUBPHASES = (
"conditioning_identity_video_vae",
"conditioning_reference_image_vae",
"conditioning_reference_audio_vae",
"conditioning_driving_audio_vae",
"conditioning_reference_video_vae",
"conditioning_timeline_video_vae",
"conditioning_prompt_clip",
)
def _mib(value: int | None) -> str:
if value is None:
return "n/a"
return f"{int(value) / _MIB:.1f} MiB"
def _bytes_and_mib(value: int) -> str:
return f"{int(value)} bytes ({int(value) / _MIB:.1f} MiB)"
def _process_counters() -> tuple[int | None, int | None]:
try:
import psutil
return (
int(psutil.Process().memory_info().rss),
int(psutil.virtual_memory().available),
)
except Exception:
return None, None
def _cuda_counters() -> tuple[int | None, int | None, int | None]:
"""Read counters only; never synchronize, clear caches, or reset peaks."""
try:
if torch.cuda.is_available():
return (
int(torch.cuda.memory_allocated()),
int(torch.cuda.memory_reserved()),
int(torch.cuda.max_memory_allocated()),
)
except Exception:
pass
return None, None, None
def _retained_stream_bytes(
entries: Any,
stream: str,
) -> int:
values: list[Any] = []
if isinstance(entries, (list, tuple)):
for entry in entries:
if isinstance(entry, Mapping):
values.append(entry.get(stream))
stats = tensor_storage_stats(values)
return int(stats["cpu_bytes"]) + int(stats["gpu_bytes"])
def _observed_streams(latent: Any) -> tuple[Any, ...]:
if latent is None:
return ()
if isinstance(latent, Mapping) and "samples" in latent:
try:
return tuple(extract_av_streams(latent))
except Exception:
return ()
return (latent,)
def format_attribution_snapshot(
label: str,
*,
retained_entries: Any = None,
observed_latent: Any = None,
retained_refine_context: Any = None,
) -> str:
"""Format one fail-soft snapshot without changing allocator state."""
try:
rss, available = _process_counters()
cuda_allocated, cuda_reserved, cuda_peak = _cuda_counters()
retained_video = _retained_stream_bytes(retained_entries, "video")
retained_audio = _retained_stream_bytes(retained_entries, "audio")
observed = tensor_storage_stats(_observed_streams(observed_latent))
refine_context = tensor_storage_stats(retained_refine_context)
return (
f"Memory [{label}]: RSS={_mib(rss)}, "
f"system_available={_mib(available)}, "
f"CUDA_allocated={_mib(cuda_allocated)}, "
f"CUDA_reserved={_mib(cuda_reserved)}, "
f"CUDA_peak_allocated_since_external_reset={_mib(cuda_peak)}, "
f"retained_video={_bytes_and_mib(retained_video)}, "
f"retained_audio={_bytes_and_mib(retained_audio)}, "
"retained_refine_context_CPU="
f"{_bytes_and_mib(int(refine_context['cpu_bytes']))}, "
"retained_refine_context_GPU="
f"{_bytes_and_mib(int(refine_context['gpu_bytes']))}, "
f"observed_CPU_storage={_bytes_and_mib(int(observed['cpu_bytes']))}, "
f"observed_GPU_storage={_bytes_and_mib(int(observed['gpu_bytes']))}"
)
except Exception as exc:
return (
f"Memory [{label}]: unavailable "
f"({type(exc).__name__}: {exc})"
)
class MemoryAttributionCollector:
"""Keep V3.5 memory events in chronological order until report assembly."""
def __init__(self):
self.lines: list[str] = []
self._sequence_started_at: float | None = None
self._phase_started_at: dict[tuple[str, int | None, tuple[int, ...]], float] = {}
self._phase_totals: dict[str, float] = {}
self._phase_counts: dict[str, int] = {}
self._group_totals: dict[
tuple[int, tuple[int, ...]], dict[str, float]
] = {}
def capture_sequence_start(self) -> None:
self._sequence_started_at = time.perf_counter()
self.lines.append(format_attribution_snapshot("V3.5 sequence start"))
self.start_phase(phase="preparation")
def start_phase(
self,
*,
phase: str,
physical_group: int | None = None,
logical_chunks: tuple[int, ...] = (),
) -> None:
key = (
str(phase),
None if physical_group is None else int(physical_group),
tuple(int(value) for value in logical_chunks),
)
self._phase_started_at[key] = time.perf_counter()
def finish_phase(
self,
*,
phase: str,
physical_group: int | None = None,
logical_chunks: tuple[int, ...] = (),
) -> None:
phase = str(phase)
group = None if physical_group is None else int(physical_group)
logical = tuple(int(value) for value in logical_chunks)
key = (phase, group, logical)
started = self._phase_started_at.pop(key, None)
if started is None:
return
elapsed = time.perf_counter() - started
if not math.isfinite(elapsed) or elapsed < 0.0:
return
self._phase_totals[phase] = self._phase_totals.get(phase, 0.0) + elapsed
self._phase_counts[phase] = self._phase_counts.get(phase, 0) + 1
if group is not None:
phases = self._group_totals.setdefault((group, logical), {})
phases[phase] = phases.get(phase, 0.0) + elapsed
@staticmethod
def _format_seconds(value: float | None) -> str:
return "n/a" if value is None else f"{float(value):.6f}s"
def _performance_lines(self) -> list[str]:
now = time.perf_counter()
host_wall = None
if self._sequence_started_at is not None:
candidate = now - self._sequence_started_at
if math.isfinite(candidate) and candidate >= 0.0:
host_wall = candidate
ordered = (
"preparation",
"conditioning",
*_CONDITIONING_SUBPHASES,
"group_prepare",
"sampling",
"cpu_commit_validation",
"refine_context",
"finalization",
)
lines = []
for (group, logical), phases in sorted(self._group_totals.items()):
logical_label = ",".join(str(value) for value in logical)
fields = ", ".join(
f"{name}={self._format_seconds(phases.get(name))}"
for name in (
"group_prepare",
"sampling",
"cpu_commit_validation",
"refine_context",
)
)
lines.append(
f"Performance [V3.5 physical group {group} "
f"logical=[{logical_label}]]: {fields}"
)
totals = ", ".join(
f"{name}={self._format_seconds(self._phase_totals.get(name))}"
for name in ordered
)
counts = ", ".join(
f"{name}={int(self._phase_counts.get(name, 0))}"
for name in (
"group_prepare",
"sampling",
"cpu_commit_validation",
"refine_context",
)
)
conditioning_counts = ", ".join(
f"{name}={int(self._phase_counts.get(name, 0))}"
for name in _CONDITIONING_SUBPHASES
)
lines.append(
"Performance [V3.5 summary]: "
f"host_wall_including_diagnostics={self._format_seconds(host_wall)}, "
f"{totals}; "
f"phase_counts({counts}); "
f"conditioning_counts({conditioning_counts}); "
"CUDA_synchronize=not_used"
)
return lines
def capture_group(
self,
*,
physical_group: int,
logical_chunks: tuple[int, ...],
stage: str,
retained_entries: Any,
observed_latent: Any = None,
retained_refine_context: Any = None,
) -> None:
logical = ",".join(str(int(value)) for value in logical_chunks)
self.lines.append(
format_attribution_snapshot(
f"V3.5 physical group {int(physical_group)} "
f"logical=[{logical}] {stage}",
retained_entries=retained_entries,
observed_latent=observed_latent,
retained_refine_context=retained_refine_context,
)
)
def capture_sequence_complete(
self,
entries: Any,
retained_refine_context: Any = None,
) -> None:
self.lines.extend(self._performance_lines())
self.lines.append(
format_attribution_snapshot(
"V3.5 sequence complete",
retained_entries=entries,
retained_refine_context=retained_refine_context,
)
)
def capture_attribution_fail_soft(
collector: Any,
method: str,
**kwargs: Any,
) -> None:
"""A diagnostics failure must never replace a sampling result."""
try:
getattr(collector, method)(**kwargs)
except Exception as exc:
try:
collector.lines.append(
f"Memory attribution [{method}]: unavailable "
f"({type(exc).__name__}: {exc})"
)
except Exception:
pass
@dataclass(frozen=True)
class AssemblyBufferProjection:
natural_frames: int
target_frames: int
natural_image_bytes: int
natural_audio_bytes: int
final_image_bytes: int
final_audio_bytes: int
frame_count_adjustment_required: bool
def _as_list(value: Any) -> list[Any]:
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], list):
return list(value[0])
return list(value)
if isinstance(value, tuple):
return list(value)
return [value]
def _singleton(value: Any, name: str) -> Any:
while isinstance(value, list):
if len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
value = value[0]
return value
def project_assembly_buffers(
*,
images: Any,
audio: Any,
assembly_plan: Any,
exact_total_duration: Any,
) -> AssemblyBufferProjection:
"""Calculate natural and final buffer sizes without allocating a buffer."""
plan = validate_assembly_plan(_singleton(assembly_plan, "assembly_plan"))
image_chunks = _as_list(images)
audio_chunks = _as_list(audio)
if not image_chunks or not audio_chunks:
raise ValueError("decoded image/audio chunks are required for projection")
first_image = _singleton(image_chunks[0], "decoded image chunk")
if not torch.is_tensor(first_image) or first_image.ndim != 4:
raise ValueError("decoded image chunk must be [frames,H,W,C]")
frame_shape = tuple(int(value) for value in first_image.shape[1:])
for index, value in enumerate(image_chunks, start=1):
tensor = _singleton(value, f"decoded image chunk {index}")
if (
not torch.is_tensor(tensor)
or tensor.ndim != 4
or tuple(int(item) for item in tensor.shape[1:]) != frame_shape
or tensor.dtype != first_image.dtype
):
raise ValueError("decoded image geometry or dtype changed between chunks")
first_audio = _singleton(audio_chunks[0], "decoded audio chunk")
if not isinstance(first_audio, Mapping):
raise ValueError("decoded audio chunk must be an AUDIO mapping")
waveform = first_audio.get("waveform")
sample_rate = first_audio.get("sample_rate")
if not torch.is_tensor(waveform) or waveform.ndim < 1 or sample_rate is None:
raise ValueError("decoded audio chunk has an invalid waveform or sample rate")
sample_rate = int(sample_rate)
if sample_rate <= 0:
raise ValueError("decoded audio sample rate must be positive")
decode_units = list(plan.get("decode_groups") or plan["chunks"])
natural_frames = sum(int(unit["net_frames"]) for unit in decode_units)
exact = bool(_singleton(exact_total_duration, "exact_total_duration"))
target_frames = int(plan["target_frames"]) if exact else natural_frames
if natural_frames <= 0 or target_frames <= 0:
raise ValueError("assembly frame counts must be positive")
frame_elements = math.prod(frame_shape)
audio_streams = math.prod(int(value) for value in waveform.shape[:-1])
natural_samples = int(round(natural_frames / FPS * sample_rate))
target_samples = int(round(target_frames / FPS * sample_rate))
return AssemblyBufferProjection(
natural_frames=natural_frames,
target_frames=target_frames,
natural_image_bytes=(
natural_frames * frame_elements * first_image.element_size()
),
natural_audio_bytes=(
natural_samples * audio_streams * waveform.element_size()
),
final_image_bytes=(
target_frames * frame_elements * first_image.element_size()
),
final_audio_bytes=(
target_samples * audio_streams * waveform.element_size()
),
frame_count_adjustment_required=bool(
exact and target_frames != natural_frames
),
)
def format_assembly_projection(projection: AssemblyBufferProjection) -> str:
return (
"Projected assembly buffers [V3.5]: "
f"natural_frames={projection.natural_frames}, "
f"projected_natural_IMAGE={_bytes_and_mib(projection.natural_image_bytes)}, "
"projected_natural_generated_AUDIO="
f"{_bytes_and_mib(projection.natural_audio_bytes)}, "
f"target_frames={projection.target_frames}, "
f"final_IMAGE={_bytes_and_mib(projection.final_image_bytes)}, "
f"final_generated_AUDIO={_bytes_and_mib(projection.final_audio_bytes)}, "
"frame_count_adjustment_required="
f"{'yes' if projection.frame_count_adjustment_required else 'no'}"
)
+176
View File
@@ -0,0 +1,176 @@
"""Thin V3.8 public-node to runtime boundary.
This module deliberately owns no continuation, review, storage, planning, or
sampling decision. It only carries already-resolved public-node values into
the existing hardening/runtime call and preserves its public return value.
"""
from __future__ import annotations
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any
from ..reference import resolve_reference_image_inputs
@dataclass(frozen=True)
class RuntimeRequest:
"""Resolved V3.8 node inputs for one existing runtime invocation.
Tensor and model objects deliberately keep their identity. Only the two
keyword containers are copied and frozen so the facade cannot mutate the
caller's widget/input mapping after the request boundary.
"""
runtime_kwargs: Mapping[str, Any]
resolution: Any
size_source: str
diagnostics_mode: str
diagnostics_inputs: Mapping[str, Any]
def __post_init__(self) -> None:
object.__setattr__(
self,
"runtime_kwargs",
MappingProxyType(dict(self.runtime_kwargs)),
)
object.__setattr__(
self,
"diagnostics_inputs",
MappingProxyType(dict(self.diagnostics_inputs)),
)
@dataclass(frozen=True)
class RuntimeOutput:
"""Existing runtime result with status-only facade decoration helpers."""
value: Any
@property
def has_status(self) -> bool:
return isinstance(self.value, tuple) and len(self.value) >= 4
@property
def status(self) -> str:
if not self.has_status:
raise TypeError("runtime output has no public status field")
return str(self.value[3])
def with_status(self, status: str) -> "RuntimeOutput":
if not self.has_status:
return self
return RuntimeOutput((*self.value[:3], str(status), *self.value[4:]))
def as_public_tuple(self) -> Any:
"""Return the unchanged nonstandard test double or public tuple."""
return self.value
def make_runtime_request(
*,
runtime_kwargs: Mapping[str, Any],
fixed_runtime_kwargs: Mapping[str, Any],
resolution: Any,
size_source: str,
diagnostics_mode: str,
diagnostics_inputs: Mapping[str, Any],
) -> RuntimeRequest:
"""Build the one R6 request without silently accepting duplicate inputs."""
duplicate = set(runtime_kwargs).intersection(fixed_runtime_kwargs)
if duplicate:
name = sorted(duplicate)[0]
raise TypeError(
"H3 Continuum Sampler V3.8 received multiple values for "
f"runtime keyword {name!r}"
)
merged = dict(runtime_kwargs)
merged.update(fixed_runtime_kwargs)
return RuntimeRequest(
runtime_kwargs=merged,
resolution=resolution,
size_source=str(size_source),
diagnostics_mode=str(diagnostics_mode),
diagnostics_inputs=diagnostics_inputs,
)
def execute_v38_runtime_request(
request: RuntimeRequest,
*,
runtime_adapter: Callable[..., Any],
legacy_size_source: str,
diagnostics_off: str,
diagnostics_full: str,
reference_video_size_default: str,
build_diagnostics: Callable[..., Any],
append_status: Callable[..., str],
) -> Any:
"""Call the existing runtime once and add V3.8 display-only diagnostics."""
output = RuntimeOutput(runtime_adapter(**request.runtime_kwargs))
if not output.has_status:
return output.as_public_tuple()
if request.size_source != legacy_size_source:
resolution = request.resolution
lines = [
(
f"Resolution: {resolution.width} x {resolution.height} "
f"({resolution.actual_mp:.2f} MP); source={resolution.aspect_source}."
)
]
lines.extend(resolution.warnings)
output = output.with_status(output.status.rstrip() + "\n" + "\n".join(lines))
if request.diagnostics_mode == diagnostics_off:
return output.as_public_tuple()
# The V3.8 reliability summary counts every connected Reference image.
# A Custom V3.9 group intentionally prepares only its selected images, so
# that legacy summary would describe a different conditioning path.
if "_reference_routing_settings" in request.runtime_kwargs:
return output.as_public_tuple()
try:
inputs = request.diagnostics_inputs
diagnostics = build_diagnostics(
video_latents=output.value[0],
audio_latents=output.value[1],
assembly_plan=output.value[2],
output_width=request.resolution.width,
output_height=request.resolution.height,
chunk_seconds=float(inputs["chunk_seconds"]),
reference_images=resolve_reference_image_inputs(
inputs.get("reference_image_1"),
inputs.get("reference_image_2"),
inputs.get("reference_image_3"),
inputs.get("reference_image_4"),
inputs.get("reference_image_5"),
inputs.get("image_references"),
),
reference_size=str(inputs.get("reference_size", "Match Output")),
video_guide=inputs.get("reference_video_1"),
video_guide_size=str(
inputs.get("video_reference_size", reference_video_size_default)
),
still_guide_active=inputs.get("guide") is not None,
)
status = append_status(
output.status,
diagnostics,
mode=request.diagnostics_mode,
)
except Exception as exc:
message = (
"V3.8 Reliability\n"
"Reliability diagnostics unavailable; generation result was preserved."
)
if request.diagnostics_mode == diagnostics_full:
message += f" ({type(exc).__name__}: {exc})"
status = output.status.rstrip() + "\n" + message
return output.with_status(status).as_public_tuple()
File diff suppressed because it is too large Load Diff
+349
View File
@@ -0,0 +1,349 @@
"""Read-only A7a planning from actual ComfyUI H3 PackedLayout rows.
The planner observes layouts that ComfyUI Core already built for Sampling. It
does not estimate VRAM, change execution policy, mutate a layout, or influence
the physical-group order. Memory counters are point-in-time observations only.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Callable
PACKED_ROW_PLAN_VERSION = 1
MAX_CONCURRENT_PHYSICAL_GROUPS = 1
_VIDEO_CONDITION_KINDS = frozenset(("cond", "ref_img"))
_AUDIO_CONDITION_KINDS = frozenset(("cond_audio", "ref_audio"))
class PackedRowPlanningError(ValueError):
"""An observed object is not an internally consistent H3 PackedLayout."""
@dataclass(frozen=True)
class PackedRowBreakdown:
seq_len: int
text_rows: int
condition_video_rows: int
condition_audio_rows: int
target_video_rows: int
target_audio_rows: int
unknown_rows: int
segment_count: int
segment_signature: tuple[tuple[int, int, str], ...]
@property
def condition_rows(self) -> int:
return int(self.condition_video_rows + self.condition_audio_rows)
@property
def target_rows(self) -> int:
return int(self.target_video_rows + self.target_audio_rows)
@property
def condition_to_target_ratio(self) -> float | None:
if self.target_rows <= 0:
return None
return float(self.condition_rows / self.target_rows)
@dataclass(frozen=True)
class MemoryObservation:
system_available_bytes: int | None = None
cuda_allocated_bytes: int | None = None
cuda_reserved_bytes: int | None = None
core_visible_free_bytes: int | None = None
def analyze_packed_layout(layout: Any) -> PackedRowBreakdown:
"""Account actual rows without importing or recreating Core PackedLayout."""
if layout is None:
raise PackedRowPlanningError("Core PackedLayout is unavailable")
try:
seq_len = int(layout.seq_len)
raw_segments = tuple(layout.segments)
except Exception as exc:
raise PackedRowPlanningError(
"Core PackedLayout does not expose seq_len/segments"
) from exc
if seq_len <= 0 or not raw_segments:
raise PackedRowPlanningError("Core PackedLayout is empty")
signature: list[tuple[int, int, str]] = []
previous_stop = 0
counts = {
"text": 0,
"condition_video": 0,
"condition_audio": 0,
"target_video": 0,
"target_audio": 0,
"unknown": 0,
}
target_video_segments = 0
target_audio_segments = 0
for item in raw_segments:
if not isinstance(item, (tuple, list)) or len(item) != 3:
raise PackedRowPlanningError("PackedLayout segment is not (start, stop, kind)")
start, stop, kind = int(item[0]), int(item[1]), str(item[2])
if start != previous_stop or stop <= start or stop > seq_len:
raise PackedRowPlanningError(
"PackedLayout segments are not a contiguous in-range partition"
)
rows = stop - start
signature.append((start, stop, kind))
if kind == "text":
counts["text"] += rows
elif kind in _VIDEO_CONDITION_KINDS:
counts["condition_video"] += rows
elif kind in _AUDIO_CONDITION_KINDS:
counts["condition_audio"] += rows
elif kind == "video":
counts["target_video"] += rows
target_video_segments += 1
elif kind == "audio":
counts["target_audio"] += rows
target_audio_segments += 1
else:
counts["unknown"] += rows
previous_stop = stop
if previous_stop != seq_len:
raise PackedRowPlanningError("PackedLayout segment rows do not sum to seq_len")
if target_video_segments != 1 or target_audio_segments != 1:
raise PackedRowPlanningError(
"H3 PackedLayout must contain exactly one target Video and Audio segment"
)
return PackedRowBreakdown(
seq_len=seq_len,
text_rows=int(counts["text"]),
condition_video_rows=int(counts["condition_video"]),
condition_audio_rows=int(counts["condition_audio"]),
target_video_rows=int(counts["target_video"]),
target_audio_rows=int(counts["target_audio"]),
unknown_rows=int(counts["unknown"]),
segment_count=len(signature),
segment_signature=tuple(signature),
)
def read_memory_observation() -> MemoryObservation:
"""Read counters without CUDA synchronization, cache clearing, or peak reset."""
system_available = None
cuda_allocated = None
cuda_reserved = None
core_free = None
try:
import psutil
system_available = int(psutil.virtual_memory().available)
except Exception:
pass
try:
import torch
if torch.cuda.is_available():
cuda_allocated = int(torch.cuda.memory_allocated())
cuda_reserved = int(torch.cuda.memory_reserved())
except Exception:
pass
try:
import comfy.model_management as model_management
device = model_management.get_torch_device()
core_free = int(model_management.get_free_memory(device))
except Exception:
pass
return MemoryObservation(
system_available_bytes=system_available,
cuda_allocated_bytes=cuda_allocated,
cuda_reserved_bytes=cuda_reserved,
core_visible_free_bytes=core_free,
)
def _safe_memory_read(reader: Callable[[], MemoryObservation]) -> MemoryObservation:
try:
value = reader()
return value if isinstance(value, MemoryObservation) else MemoryObservation()
except Exception:
return MemoryObservation()
class PackedRowPlanner:
"""Collect one immutable actual-layout plan record per sampled group."""
def __init__(
self,
*,
memory_reader: Callable[[], MemoryObservation] = read_memory_observation,
) -> None:
self._memory_reader = memory_reader
self._active: dict[str, Any] | None = None
self._records: list[dict[str, Any]] = []
self._advisories: list[str] = []
@property
def records(self) -> tuple[dict[str, Any], ...]:
return tuple(self._records)
def begin_group(
self,
*,
physical_group: int,
logical_chunks: tuple[int, ...],
terminal_atomic: bool,
) -> None:
if self._active is not None:
self._advisories.append(
"a group began before the prior observation was finalized"
)
self.finish_group()
self._active = {
"physical_group": int(physical_group),
"logical_chunks": tuple(int(value) for value in logical_chunks),
"terminal_atomic": bool(terminal_atomic),
"pre_memory": _safe_memory_read(self._memory_reader),
"breakdown": None,
"unavailable": None,
}
def observe_layout(self, layout: Any) -> None:
if self._active is None:
return
try:
observed = analyze_packed_layout(layout)
except Exception as exc:
if self._active["breakdown"] is None:
self._active["unavailable"] = f"{type(exc).__name__}: {exc}"
return
current = self._active["breakdown"]
if current is not None and current != observed:
self._active["unavailable"] = (
"PackedLayout topology changed within one physical group"
)
self._active["breakdown"] = None
return
if self._active["unavailable"] is None:
self._active["breakdown"] = observed
def note_observation_failure(self, exc: Exception) -> None:
if self._active is not None and self._active["breakdown"] is None:
self._active["unavailable"] = f"{type(exc).__name__}: {exc}"
def finish_group(self) -> None:
if self._active is None:
return
record = dict(self._active)
record["post_memory"] = _safe_memory_read(self._memory_reader)
if record["breakdown"] is None and record["unavailable"] is None:
record["unavailable"] = "Core PackedLayout was not observed"
self._records.append(record)
self._active = None
@staticmethod
def _mib(value: int | None) -> str:
return "n/a" if value is None else f"{int(value) / 1024**2:.1f} MiB"
def report_lines(self) -> list[str]:
if self._active is not None:
self.finish_group()
valid = [record for record in self._records if record["breakdown"] is not None]
peak = None
if valid:
peak = max(valid, key=lambda record: record["breakdown"].seq_len)
order = [int(record["physical_group"]) for record in self._records]
lines = [
"Packed-row plan [A7a v1]: "
f"execution_order={order}, "
f"max_concurrent_physical_groups={MAX_CONCURRENT_PHYSICAL_GROUPS}, "
f"observed_groups={len(valid)}/{len(self._records)}, "
f"peak_candidate={int(peak['physical_group']) if peak else 'unavailable'}, "
f"peak_total_rows={peak['breakdown'].seq_len if peak else 'unavailable'}; "
"actual Core rows only, VRAM/time prediction=not provided, "
"execution policy=unchanged."
]
for record in self._records:
group = int(record["physical_group"])
logical = list(record["logical_chunks"])
breakdown = record["breakdown"]
if breakdown is None:
lines.append(
"Packed-row group [A7a v1]: "
f"physical_group={group}, logical_chunks={logical}, "
f"terminal_atomic={str(bool(record['terminal_atomic'])).lower()}, "
f"unavailable/advisory={record['unavailable']}."
)
continue
pre = record["pre_memory"]
post = record["post_memory"]
ratio = breakdown.condition_to_target_ratio
ratio_text = "n/a" if ratio is None else f"{ratio:.6f}"
lines.append(
"Packed-row group [A7a v1]: "
f"physical_group={group}, logical_chunks={logical}, "
f"terminal_atomic={str(bool(record['terminal_atomic'])).lower()}, "
f"total={breakdown.seq_len}, text={breakdown.text_rows}, "
f"condition_video={breakdown.condition_video_rows}, "
f"condition_audio={breakdown.condition_audio_rows}, "
f"target_video={breakdown.target_video_rows}, "
f"target_audio={breakdown.target_audio_rows}, "
f"unknown={breakdown.unknown_rows}, "
f"condition_to_target_ratio={ratio_text}, "
f"pre_system_available={self._mib(pre.system_available_bytes)}, "
f"pre_CUDA_allocated={self._mib(pre.cuda_allocated_bytes)}, "
f"pre_CUDA_reserved={self._mib(pre.cuda_reserved_bytes)}, "
f"pre_Core_free={self._mib(pre.core_visible_free_bytes)}, "
f"post_system_available={self._mib(post.system_available_bytes)}, "
f"post_CUDA_allocated={self._mib(post.cuda_allocated_bytes)}, "
f"post_CUDA_reserved={self._mib(post.cuda_reserved_bytes)}, "
f"post_Core_free={self._mib(post.core_visible_free_bytes)}."
)
lines.extend(f"Packed-row advisory [A7a v1]: {item}." for item in self._advisories)
return lines
def observe_layout_fail_soft(planner: Any, payload: Any) -> None:
"""Diagnostics must never replace a Sampling result."""
if planner is None:
return
try:
layout = payload.get("layout") if isinstance(payload, dict) else None
planner.observe_layout(layout)
except Exception as exc:
try:
planner.note_observation_failure(exc)
except Exception:
pass
def planner_event_fail_soft(planner: Any, method: str, **kwargs: Any) -> Any:
"""Invoke one lifecycle event without allowing diagnostics to stop Sampling."""
if planner is None:
return None
try:
return getattr(planner, str(method))(**kwargs)
except Exception as exc:
try:
planner.note_observation_failure(exc)
except Exception:
pass
return None
__all__ = [
"MAX_CONCURRENT_PHYSICAL_GROUPS",
"MemoryObservation",
"PACKED_ROW_PLAN_VERSION",
"PackedRowBreakdown",
"PackedRowPlanner",
"PackedRowPlanningError",
"analyze_packed_layout",
"observe_layout_fail_soft",
"planner_event_fail_soft",
"read_memory_observation",
]
+367
View File
@@ -0,0 +1,367 @@
"""Sequential pixel/VAE resize for Continuum physical video groups."""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from typing import Any
import torch
from .plan import SECOND_PASS_CONTRACT_VERSION
PIXEL_UPSCALE_METHODS = (
"nearest-exact",
"bilinear",
"area",
"bicubic",
"bislerp",
"lanczos",
)
_PIXEL_RESIZE_FRAME_BATCH = 8
class PixelVaeResizeError(ValueError):
"""Raised when a pixel/VAE resize would violate the physical-group contract."""
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise PixelVaeResizeError("assembly_plan must be a mapping")
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise PixelVaeResizeError("assembly_plan has no second_pass_contract")
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
raise PixelVaeResizeError("unsupported second_pass_contract version")
groups = contract.get("physical_groups")
if (
not isinstance(groups, list)
or not groups
or not all(isinstance(group, Mapping) for group in groups)
):
raise PixelVaeResizeError(
"second_pass_contract.physical_groups must be non-empty"
)
return groups
def resolve_pixel_latent_scale(
assembly_plan: Mapping[str, Any],
) -> tuple[int, int]:
"""Return plan-derived pixel-per-latent ratios as ``(width, height)``."""
groups = _contract_groups(assembly_plan)
scale: tuple[int, int] | None = None
for position, group in enumerate(groups):
if int(group.get("group_id", -1)) != position:
raise PixelVaeResizeError("physical group order or group_id changed")
source_width = int(group.get("source_width", -1))
source_height = int(group.get("source_height", -1))
source_latent_width = int(group.get("source_latent_w", -1))
source_latent_height = int(group.get("source_latent_h", -1))
if min(
source_width,
source_height,
source_latent_width,
source_latent_height,
) < 1:
raise PixelVaeResizeError("source pixel/latent geometry is invalid")
if (
source_width % source_latent_width
or source_height % source_latent_height
):
raise PixelVaeResizeError(
"source pixel/latent geometry is not integral"
)
group_scale = (
source_width // source_latent_width,
source_height // source_latent_height,
)
if scale is None:
scale = group_scale
elif scale != group_scale:
raise PixelVaeResizeError(
"source pixel/latent scale differs between physical groups"
)
assert scale is not None
return scale
def _latent_samples(
latent: Any,
*,
position: int,
group: Mapping[str, Any],
) -> torch.Tensor:
if not isinstance(latent, Mapping) or "samples" not in latent:
raise PixelVaeResizeError(
f"video_latents[{position}] must be a LATENT mapping with samples"
)
samples = latent["samples"]
if not isinstance(samples, torch.Tensor) or samples.ndim != 5:
raise PixelVaeResizeError("video latent must have shape [B,C,T,H,W]")
batch, channels, temporal, height, width = (
int(value) for value in samples.shape
)
expected = (
int(group.get("source_batch", -1)),
int(group.get("latent_channels", -1)),
int(group.get("source_latent_t", -1)),
int(group.get("source_latent_h", -1)),
int(group.get("source_latent_w", -1)),
)
if (batch, channels, temporal, height, width) != expected:
raise PixelVaeResizeError(
"video latent shape or physical group order changed"
)
if channels != 24:
raise PixelVaeResizeError("MiniMax H3 video latent must have 24 channels")
if not bool(torch.isfinite(samples).all()):
raise PixelVaeResizeError("input video latent contains NaN or Inf")
return samples
def _resize_one_group(
samples: torch.Tensor,
*,
video_vae: Any,
source_width: int,
source_height: int,
target_width: int,
target_height: int,
target_latent_width: int,
target_latent_height: int,
upscale_method: str,
upscale_fn: Callable[..., Any],
) -> torch.Tensor:
"""Decode, resize, and encode one group so its pixel buffer dies per call."""
decoded_pixels = video_vae.decode(samples)
if not isinstance(decoded_pixels, torch.Tensor) or decoded_pixels.ndim != 5:
raise PixelVaeResizeError(
"Video VAE decode must return pixels with shape [B,F,H,W,C]"
)
batch, frames, pixel_height, pixel_width, channels = (
int(value) for value in decoded_pixels.shape
)
if batch != int(samples.shape[0]):
raise PixelVaeResizeError("Video VAE decode changed the batch size")
if frames < 1 or channels != 3:
raise PixelVaeResizeError(
"Video VAE decode returned invalid frame or channel geometry"
)
if (pixel_width, pixel_height) != (source_width, source_height):
raise PixelVaeResizeError(
"Video VAE decode geometry differs from the assembly plan"
)
for frame_start in range(0, frames, _PIXEL_RESIZE_FRAME_BATCH):
if not bool(
torch.isfinite(
decoded_pixels[
:,
frame_start : frame_start + _PIXEL_RESIZE_FRAME_BATCH,
]
).all()
):
raise PixelVaeResizeError("Video VAE decode contains NaN or Inf")
# Pixel resizing is intentionally CPU-owned. Core's Lanczos implementation
# is CPU/PIL-based already, and this also avoids retaining a full video-sized
# resize workspace in VRAM under --gpu-only configurations.
pixels = decoded_pixels.detach().to("cpu").contiguous()
del decoded_pixels
# Core common_upscale is a 4D image boundary. Make B*F explicit so video
# frame order is unambiguous. Resize bounded frame batches into one CPU
# [B,F,H,W,C] buffer required by the temporal H3 VAE encoder.
source_flat = pixels.reshape(
batch * frames, pixel_height, pixel_width, channels
)
resized_pixels = torch.empty(
(batch, frames, target_height, target_width, channels),
dtype=pixels.dtype,
device="cpu",
)
target_flat = resized_pixels.reshape(
batch * frames, target_height, target_width, channels
)
for frame_start in range(
0,
batch * frames,
_PIXEL_RESIZE_FRAME_BATCH,
):
frame_stop = min(
frame_start + _PIXEL_RESIZE_FRAME_BATCH,
batch * frames,
)
pixel_nchw = (
source_flat[frame_start:frame_stop]
.movedim(-1, 1)
.contiguous()
)
resized_nchw = upscale_fn(
pixel_nchw,
int(target_width),
int(target_height),
upscale_method,
"disabled",
)
expected_resize_shape = (
frame_stop - frame_start,
channels,
target_height,
target_width,
)
if not isinstance(resized_nchw, torch.Tensor) or tuple(
resized_nchw.shape
) != expected_resize_shape:
raise PixelVaeResizeError("Core pixel resize returned invalid geometry")
resized_nchw = resized_nchw.detach().to("cpu")
if not bool(torch.isfinite(resized_nchw).all()):
raise PixelVaeResizeError("Core pixel resize contains NaN or Inf")
target_flat[frame_start:frame_stop].copy_(
resized_nchw.movedim(1, -1)
)
del pixel_nchw, resized_nchw
del pixels, source_flat, target_flat
# Match Core VAEEncode exactly: H3's VAE wrapper expects a 4D IMAGE batch
# and promotes its frame axis to one temporal video batch. Passing the 5D
# [B,F,H,W,C] tensor directly makes Core crop F as if it were spatial.
# Encode each Continuum batch independently, then restore B explicitly.
encoded_batches: list[torch.Tensor] = []
expected_batch_shape = (
1,
int(samples.shape[1]),
int(samples.shape[2]),
int(target_latent_height),
int(target_latent_width),
)
for batch_index in range(batch):
encoded_batch = video_vae.encode(resized_pixels[batch_index])
if not isinstance(encoded_batch, torch.Tensor) or encoded_batch.ndim != 5:
raise PixelVaeResizeError(
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
)
if int(encoded_batch.shape[0]) != 1:
raise PixelVaeResizeError(
"Core Video VAE Encode must return one batch per IMAGE sequence"
)
encoded_batch_shape = tuple(int(value) for value in encoded_batch.shape)
if encoded_batch_shape != expected_batch_shape:
raise PixelVaeResizeError(
"Video VAE encode changed B/C/T or returned the wrong target H/W "
f"for batch {batch_index}: expected {expected_batch_shape}, "
f"got {encoded_batch_shape}"
)
encoded_batches.append(encoded_batch)
encoded = (
encoded_batches[0]
if len(encoded_batches) == 1
else torch.cat(encoded_batches, dim=0)
)
if not isinstance(encoded, torch.Tensor) or encoded.ndim != 5:
raise PixelVaeResizeError(
"Video VAE encode must return a latent with shape [B,C,T,H,W]"
)
expected_shape = (
int(samples.shape[0]),
int(samples.shape[1]),
int(samples.shape[2]),
int(target_latent_height),
int(target_latent_width),
)
if tuple(int(value) for value in encoded.shape) != expected_shape:
raise PixelVaeResizeError(
"Video VAE encode changed B/C/T or returned the wrong target H/W: "
f"expected {expected_shape}, got {tuple(int(value) for value in encoded.shape)}"
)
if not bool(torch.isfinite(encoded).all()):
raise PixelVaeResizeError("Video VAE encode contains NaN or Inf")
# Continuum stores accepted physical latents on CPU; do not retain one GPU
# allocation per physical group while later groups are being converted.
output = encoded.detach().to("cpu").contiguous()
del resized_pixels, encoded_batches, encoded
return output
@torch.inference_mode()
def resize_video_latents_via_vae(
video_latents: Sequence[Mapping[str, Any]],
assembly_plan: Mapping[str, Any],
*,
video_vae: Any,
target_width: int,
target_height: int,
upscale_method: str = "lanczos",
upscale_fn: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Resize physical groups sequentially through the connected H3 Video VAE.
``target_width`` and ``target_height`` are pixel dimensions. The expected
encoded latent geometry is derived exclusively from assembly-plan metadata.
VAE failures are intentionally not caught: Core owns its OOM/tiled fallback,
and this path must never fall back to direct latent interpolation.
"""
groups = _contract_groups(assembly_plan)
if (
not isinstance(video_latents, Sequence)
or isinstance(video_latents, (str, bytes))
or len(video_latents) != len(groups)
):
raise PixelVaeResizeError("video latent group count changed")
if (
video_vae is None
or not callable(getattr(video_vae, "decode", None))
or not callable(getattr(video_vae, "encode", None))
):
raise PixelVaeResizeError("a usable H3 Video VAE is required")
if upscale_method not in PIXEL_UPSCALE_METHODS:
raise PixelVaeResizeError(
f"unsupported pixel upscale method {upscale_method!r}"
)
target_width = int(target_width)
target_height = int(target_height)
if target_width < 1 or target_height < 1:
raise PixelVaeResizeError("target pixel width and height must be positive")
scale_width, scale_height = resolve_pixel_latent_scale(assembly_plan)
if target_width % scale_width or target_height % scale_height:
raise PixelVaeResizeError(
"target pixel geometry is not aligned to the plan-derived latent scale"
)
target_latent_width = target_width // scale_width
target_latent_height = target_height // scale_height
if upscale_fn is None:
from comfy.utils import common_upscale
upscale_fn = common_upscale
outputs: list[dict[str, Any]] = []
for position, (latent, group) in enumerate(
zip(video_latents, groups, strict=True)
):
samples = _latent_samples(latent, position=position, group=group)
source_width = int(group["source_width"])
source_height = int(group["source_height"])
resized_samples = _resize_one_group(
samples,
video_vae=video_vae,
source_width=source_width,
source_height=source_height,
target_width=target_width,
target_height=target_height,
target_latent_width=target_latent_width,
target_latent_height=target_latent_height,
upscale_method=upscale_method,
upscale_fn=upscale_fn,
)
output = dict(latent)
output["samples"] = resized_samples
outputs.append(output)
return outputs
+466
View File
@@ -0,0 +1,466 @@
"""Assembly-plan contract for latent-first H3 Continuum V3."""
from __future__ import annotations
from typing import Any
import torch
from ..state import validate_plan
ASSEMBLY_PLAN_MAGIC = "H3_CONTINUUM_ASSEMBLY_PLAN"
ASSEMBLY_PLAN_SCHEMA_VERSION = 1
FPS = 24
SECOND_PASS_CONTRACT_VERSION = 1
REVIEW_AUDIO_PROJECTION_KEY = "review_audio_projection_v1"
class AssemblyPlanError(ValueError):
pass
def make_assembly_plan(
entries: list[dict[str, Any]],
*,
chunk_seconds: float,
preserve_final_frame: bool,
) -> dict[str, Any]:
if not entries:
raise AssemblyPlanError("cannot create an assembly plan for an empty sequence")
chunks: list[dict[str, Any]] = []
width = height = None
for sequence_index, entry in enumerate(entries, start=1):
if not isinstance(entry, dict):
raise AssemblyPlanError(f"chunk {sequence_index} entry is not a dictionary")
plan = validate_plan(entry.get("plan"))
video = entry.get("video")
audio = entry.get("audio")
if not torch.is_tensor(video) or video.ndim < 3:
raise AssemblyPlanError(f"chunk {sequence_index} video latent is invalid")
if not torch.is_tensor(audio) or audio.ndim < 3:
raise AssemblyPlanError(f"chunk {sequence_index} audio latent is invalid")
plan_width = int(plan["width"])
plan_height = int(plan["height"])
if width is None:
width, height = plan_width, plan_height
elif (plan_width, plan_height) != (width, height):
raise AssemblyPlanError("chunk dimensions changed within the sequence")
chunks.append(
{
"sequence_index": sequence_index,
"chunk_index": int(plan["clip_index"]),
"total_frames": int(plan["total_frames"]),
"trim_frames": int(plan["trim_frames"]),
"net_frames": int(plan["net_frames"]),
"context_frames": int(plan["context_frames"]),
"expected_video_latent_t": int(video.shape[2]),
"expected_audio_latent_t": int(audio.shape[-1]),
}
)
result = {
"magic": ASSEMBLY_PLAN_MAGIC,
"schema_version": ASSEMBLY_PLAN_SCHEMA_VERSION,
"fps": FPS,
"width": int(width),
"height": int(height),
"chunk_seconds": float(chunk_seconds),
"target_frames": int(round(len(chunks) * float(chunk_seconds) * FPS)),
"preserve_final_frame": bool(preserve_final_frame),
"chunks": chunks,
}
return validate_assembly_plan(result)
def validate_assembly_plan(plan: Any) -> dict[str, Any]:
if not isinstance(plan, dict) or plan.get("magic") != ASSEMBLY_PLAN_MAGIC:
raise AssemblyPlanError("invalid H3 Continuum V3 assembly plan")
if (
type(plan.get("schema_version")) is not int
or plan["schema_version"] != ASSEMBLY_PLAN_SCHEMA_VERSION
):
raise AssemblyPlanError(
f"unsupported assembly plan schema {plan.get('schema_version')!r}"
)
if int(plan.get("fps", 0)) != FPS:
raise AssemblyPlanError("assembly plan FPS must be 24")
if int(plan.get("width", 0)) <= 0 or int(plan.get("height", 0)) <= 0:
raise AssemblyPlanError("assembly plan dimensions are invalid")
if float(plan.get("chunk_seconds", 0.0)) <= 0:
raise AssemblyPlanError("assembly plan chunk duration is invalid")
if int(plan.get("target_frames", 0)) <= 0:
raise AssemblyPlanError("assembly plan target frame count is invalid")
chunks = plan.get("chunks")
if not isinstance(chunks, list) or not chunks:
raise AssemblyPlanError("assembly plan contains no chunks")
for expected_index, chunk in enumerate(chunks, start=1):
if not isinstance(chunk, dict):
raise AssemblyPlanError(f"assembly chunk {expected_index} is invalid")
if int(chunk.get("sequence_index", 0)) != expected_index:
raise AssemblyPlanError("assembly chunk order is not contiguous")
total = int(chunk.get("total_frames", 0))
trim = int(chunk.get("trim_frames", -1))
net = int(chunk.get("net_frames", 0))
if total <= 0 or trim < 0 or net <= 0 or total - trim != net:
raise AssemblyPlanError(
f"assembly chunk {expected_index} frame plan is invalid"
)
if int(chunk.get("context_frames", -1)) < 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} context is invalid"
)
if int(chunk.get("expected_video_latent_t", 0)) <= 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} video latent length is invalid"
)
if int(chunk.get("expected_audio_latent_t", 0)) <= 0:
raise AssemblyPlanError(
f"assembly chunk {expected_index} audio latent length is invalid"
)
return plan
# V3.0.1 hardening integration: redundant metadata plus fail-closed validation.
from ..hardening import (
enrich_assembly_plan as _enrich_assembly_plan,
validate_assembly_plan_contract as _validate_assembly_plan_contract,
)
_make_assembly_plan_v300 = make_assembly_plan
_validate_assembly_plan_v300 = validate_assembly_plan
def validate_assembly_plan(plan):
result = _validate_assembly_plan_v300(plan)
_validate_assembly_plan_contract(plan)
validate_review_audio_projection(plan)
return result
def validate_review_audio_projection(plan):
"""Check a scoped PCM coordinate without recomputing its absolute origin."""
if REVIEW_AUDIO_PROJECTION_KEY not in plan:
return None
metadata = plan[REVIEW_AUDIO_PROJECTION_KEY]
if not isinstance(metadata, dict):
raise AssemblyPlanError("Review audio projection must be a dictionary")
if (type(metadata.get("version")) is not int or metadata.get("version") != 1 or metadata.get("kind") != "current_review_unit"
or metadata.get("coordinate_space") != "full_source_pcm"):
raise AssemblyPlanError("Review audio projection version or coordinate space is invalid")
for field in ("fps", "start_chunk", "end_chunk", "source_start_frame", "source_stop_frame"):
if type(metadata.get(field)) is not int:
raise AssemblyPlanError(f"Review audio projection {field} must be an integer")
if metadata["fps"] != FPS or int(plan.get("fps", 0)) != FPS:
raise AssemblyPlanError("Review audio projection FPS must be 24")
start, end = metadata["start_chunk"], metadata["end_chunk"]
first, stop = metadata["source_start_frame"], metadata["source_stop_frame"]
chunks = [int(chunk["chunk_index"]) for chunk in plan["chunks"]]
units = plan.get("decode_groups") or plan["chunks"]
if (start < 1 or end < start or len(chunks) != end - start + 1
or any(index != start + position for position, index in enumerate(chunks))
or first < 0 or stop <= first
or stop - first != sum(int(unit["net_frames"]) for unit in units)):
raise AssemblyPlanError("Review audio projection physical range is inconsistent")
return metadata
def make_review_audio_projection(plan, *, start_chunk: int, end_chunk: int):
"""Read natural retained ranges before narrowing the accepted decode view."""
validate_assembly_plan(plan)
if REVIEW_AUDIO_PROJECTION_KEY in plan:
raise AssemblyPlanError("Review output cannot be projected twice")
start, end = int(start_chunk), int(end_chunk)
selected = []
covered = []
for unit in plan.get("decode_groups") or plan["chunks"]:
indices = unit.get("logical_chunk_indices") or [unit["chunk_index"]]
if not any(start <= int(index) <= end for index in indices):
continue
if not all(start <= int(index) <= end for index in indices):
raise AssemblyPlanError("Review audio projection cannot split a physical group")
first, stop = unit.get("frame_start"), unit.get("frame_stop")
if type(first) is not int or type(stop) is not int or stop - first != int(unit["net_frames"]):
raise AssemblyPlanError("Review audio projection needs retained natural frame ranges")
if selected and selected[-1][1] != first:
raise AssemblyPlanError("Review audio projection ranges are not contiguous")
selected.append((first, stop))
covered.extend(int(index) for index in indices)
if not selected or covered != list(range(start, end + 1)):
raise AssemblyPlanError("Review audio projection is outside the accepted sequence")
return dict(version=1, kind="current_review_unit", coordinate_space="full_source_pcm",
fps=FPS, start_chunk=start, end_chunk=end,
source_start_frame=selected[0][0], source_stop_frame=selected[-1][1])
def make_assembly_plan(*args, **kwargs):
plan = _make_assembly_plan_v300(*args, **kwargs)
_enrich_assembly_plan(plan)
validate_assembly_plan(plan)
return plan
def _recombine_terminal_tensor(
first,
second,
*,
slices,
time_dim: int,
name: str,
):
"""Invert the terminal logical split without changing any tensor value."""
import torch
if not torch.is_tensor(first) or not torch.is_tensor(second):
raise ValueError(f"terminal {name} entries must be tensors")
if first.ndim != second.ndim:
raise ValueError(f"terminal {name} tensor ranks differ")
dim = int(time_dim)
if dim < 0:
dim += first.ndim
if dim < 0 or dim >= first.ndim:
raise ValueError(f"terminal {name} time dimension is invalid")
for axis, (left, right) in enumerate(zip(first.shape, second.shape)):
if axis != dim and int(left) != int(right):
raise ValueError(f"terminal {name} tensor geometry differs")
(first_start, first_stop), (second_start, second_stop) = slices
first_start = int(first_start)
first_stop = int(first_stop)
second_start = int(second_start)
second_stop = int(second_stop)
if first_start != 0 or second_start > first_stop or second_stop <= first_stop:
raise ValueError(f"terminal {name} split ranges do not form one physical unit")
if int(first.shape[dim]) != first_stop - first_start:
raise ValueError(f"terminal {name} first slice length is inconsistent")
if int(second.shape[dim]) != second_stop - second_start:
raise ValueError(f"terminal {name} second slice length is inconsistent")
overlap_start = max(first_start, second_start)
overlap_stop = min(first_stop, second_stop)
overlap_length = max(0, overlap_stop - overlap_start)
if overlap_length:
first_overlap = first.narrow(dim, overlap_start - first_start, overlap_length)
second_overlap = second.narrow(dim, overlap_start - second_start, overlap_length)
if not torch.equal(first_overlap, second_overlap):
raise ValueError(
f"terminal {name} overlap differs; refusing lossy physical reconstruction"
)
output_shape = list(first.shape)
output_shape[dim] = second_stop
output = first.new_empty(output_shape)
output.narrow(dim, first_start, first_stop - first_start).copy_(first)
output.narrow(dim, second_start, second_stop - second_start).copy_(second)
return output.contiguous()
def _terminal_physical_prompt(
prompts: list[str],
*,
chunk_seconds: float,
) -> tuple[str, str]:
"""Reproduce the established terminal-pair prompt without reparsing it later."""
first, second = prompts
if first == second:
return first, "shared_prompt_v1"
split = f"{float(chunk_seconds):g}"
stop = f"{float(chunk_seconds) * 2:g}"
return (
f"[0-{split}s]\n{first}\n\n[{split}-{stop}s]\n{second}",
"paired_timeline_v1",
)
def _attach_second_pass_contract(
plan: dict[str, Any],
*,
logical_entries: list[dict[str, Any]],
physical_entries: list[dict[str, Any]],
chunk_seconds: float,
logical_chunk_indices: list[int] | None = None,
) -> None:
"""Attach additive V3.5 metadata without changing V3.4 generation semantics."""
chunk_indices = (
list(range(1, len(logical_entries) + 1))
if logical_chunk_indices is None
else [int(value) for value in logical_chunk_indices]
)
if len(chunk_indices) != len(logical_entries):
raise ValueError("logical chunk identity count does not match entries")
decode_groups = plan.get("decode_groups")
if not isinstance(decode_groups, list):
decode_groups = [
{
"logical_chunk_indices": [chunk_indices[index]],
"terminal_merged": False,
"total_frames": plan["chunks"][index]["total_frames"],
"trim_frames": plan["chunks"][index]["trim_frames"],
}
for index in range(len(physical_entries))
]
prompt_by_chunk = {
chunk_indices[index]: str(entry.get("prompt", ""))
for index, entry in enumerate(logical_entries)
}
groups: list[dict[str, Any]] = []
for group_id, (entry, decode_group) in enumerate(
zip(physical_entries, decode_groups, strict=True)
):
video = entry["video"]
audio = entry["audio"]
logical_chunks = [int(value) for value in decode_group["logical_chunk_indices"]]
prompts = [prompt_by_chunk[index] for index in logical_chunks]
if len(prompts) == 1:
physical_prompt = prompts[0]
prompt_policy = "single"
elif len(prompts) == 2 and bool(decode_group.get("terminal_merged")):
physical_prompt, prompt_policy = _terminal_physical_prompt(
prompts,
chunk_seconds=chunk_seconds,
)
else:
physical_prompt = "\n\n".join(prompts)
prompt_policy = "physical_group_v1"
groups.append(
{
"group_id": group_id,
"logical_chunks": logical_chunks,
"physical_prompt": physical_prompt,
"prompt_policy": prompt_policy,
"physical_frames": int(decode_group["total_frames"]),
"trim_prefix_frames": int(decode_group["trim_frames"]),
"terminal_merged": bool(decode_group.get("terminal_merged", False)),
"source_width": int(plan["width"]),
"source_height": int(plan["height"]),
"source_batch": int(video.shape[0]),
"latent_channels": int(video.shape[1]),
"source_latent_t": int(video.shape[2]),
"source_latent_h": int(video.shape[3]),
"source_latent_w": int(video.shape[4]),
"source_audio_shape": [int(value) for value in audio.shape],
}
)
plan["second_pass_contract"] = {
"version": SECOND_PASS_CONTRACT_VERSION,
"physical_groups": groups,
}
def prepare_physical_decode_entries(
entries,
*,
chunk_seconds: float,
preserve_final_frame: bool,
terminal_merged: bool,
terminal_initial_pair: bool | None = None,
):
"""Return external-VAE decode units while preserving logical plan metadata."""
logical_entries = list(entries)
plan = make_assembly_plan(
logical_entries,
chunk_seconds=float(chunk_seconds),
preserve_final_frame=bool(preserve_final_frame),
)
logical_chunk_indices = [
int(chunk["chunk_index"]) for chunk in plan["chunks"]
]
if not terminal_merged:
_attach_second_pass_contract(
plan,
logical_entries=logical_entries,
physical_entries=logical_entries,
chunk_seconds=float(chunk_seconds),
logical_chunk_indices=logical_chunk_indices,
)
return logical_entries, plan
if len(logical_entries) < 2:
raise ValueError("terminal physical decode requires two logical entries")
from ..v2.sequence import _terminal_pair_contract
initial_pair = (
len(logical_entries) == 2
if terminal_initial_pair is None
else bool(terminal_initial_pair)
)
contract = _terminal_pair_contract(
initial_pair=initial_pair,
chunk_seconds=float(chunk_seconds),
)
first_entry = logical_entries[-2]
second_entry = logical_entries[-1]
physical_video = _recombine_terminal_tensor(
first_entry["video"],
second_entry["video"],
slices=contract["video_slices"],
time_dim=2,
name="video",
)
physical_audio = _recombine_terminal_tensor(
first_entry["audio"],
second_entry["audio"],
slices=contract["audio_slices"],
time_dim=-1,
name="audio",
)
physical_entry = dict(first_entry)
physical_entry["video"] = physical_video
physical_entry["audio"] = physical_audio
decode_entries = [*logical_entries[:-2], physical_entry]
logical_chunks = list(plan["chunks"])
decode_groups = []
for chunk in logical_chunks[:-2]:
group = dict(chunk)
group["logical_chunk_indices"] = [int(chunk["chunk_index"])]
group["terminal_merged"] = False
decode_groups.append(group)
first_chunk = logical_chunks[-2]
second_chunk = logical_chunks[-1]
frame_start = int(first_chunk.get("frame_start", 0))
net_frames = int(contract["physical_frames"]) - int(
contract["physical_context_frames"]
)
terminal_group = dict(first_chunk)
terminal_group.update(
{
"total_frames": int(contract["physical_frames"]),
"trim_frames": int(contract["physical_context_frames"]),
"net_frames": net_frames,
"context_frames": int(contract["physical_context_frames"]),
"expected_video_latent_t": int(physical_video.shape[2]),
"expected_audio_latent_t": int(physical_audio.shape[-1]),
"frame_start": frame_start,
"frame_stop": frame_start + net_frames,
"logical_chunk_indices": [
int(first_chunk["chunk_index"]),
int(second_chunk["chunk_index"]),
],
"terminal_merged": True,
}
)
decode_groups.append(terminal_group)
plan = dict(plan)
plan["decode_group_version"] = 1
plan["logical_chunk_count"] = len(logical_chunks)
plan["physical_decode_group_count"] = len(decode_groups)
plan["decode_groups"] = decode_groups
_attach_second_pass_contract(
plan,
logical_entries=logical_entries,
physical_entries=decode_entries,
chunk_seconds=float(chunk_seconds),
logical_chunk_indices=logical_chunk_indices,
)
validate_assembly_plan(plan)
return decode_entries, plan
+83
View File
@@ -0,0 +1,83 @@
"""Immutable facts shared by the pure Continuum execution planner.
This module must remain free of torch and ComfyUI imports. Runtime payloads
stay in their existing adapters and are projected into these facts only after
the active continuation source has already been selected.
"""
from __future__ import annotations
from dataclasses import dataclass
from .review_control import ReviewUnit
@dataclass(frozen=True, slots=True)
class PrefixFacts:
"""Immutable planning projection of one fully validated storage prefix."""
accepted_chunks: int
configured_chunks: int
revision_id: str | None
status: str | None
review_unit: ReviewUnit | None
effective_nonce: int
branch_regenerate_from: int
physical_group_boundaries: tuple[tuple[int, int], ...]
@dataclass(frozen=True, slots=True)
class ExplicitSessionFacts:
"""Characterized facts for an explicit Session runtime input."""
present: bool
compatible: bool
accepted_chunks: int
reroll_from_chunk: int | None
physical_group_boundaries: tuple[tuple[int, int], ...]
@dataclass(frozen=True, slots=True)
class InitialStateFacts:
"""Characterized facts for an initial State runtime input."""
present: bool
usable: bool
@dataclass(frozen=True, slots=True)
class ContinuationSourceFacts:
"""Planning projection of the single source selected by the runtime."""
kind: str
accepted_chunks: int
reroll_from_chunk: int | None
physical_group_boundaries: tuple[tuple[int, int], ...]
@dataclass(frozen=True, slots=True)
class ProjectionDecision:
"""Describe which accepted chunks may be exposed to external decode."""
kind: str
start_chunk: int | None = None
end_chunk: int | None = None
@dataclass(frozen=True, slots=True)
class PhysicalGroupDescriptor:
"""One atomic sampling group expressed in one-based logical chunks."""
physical_group: int
logical_chunks: tuple[int, ...]
terminal_atomic: bool
@dataclass(frozen=True, slots=True)
class ExecutionPlan:
"""Pure plan consumed by the unchanged runtime execution loops."""
groups_to_generate: tuple[PhysicalGroupDescriptor, ...]
projection: ProjectionDecision
expected_status: str
expected_review_unit: ReviewUnit | None
+340
View File
@@ -0,0 +1,340 @@
"""Process-local, VAE-scoped cache for V3.8 reference encode tensors.
Only detached, contiguous CPU tensor copies are retained. The cache never
owns source images, video frames, audio waveforms, conditioning objects, or
sampling outputs.
"""
from __future__ import annotations
from collections import Counter, OrderedDict
from dataclasses import dataclass
import threading
from typing import Any, Callable
import weakref
import torch
REF_ENCODE_CACHE_VERSION = 1
DEFAULT_MAX_ENTRIES = 16
DEFAULT_MAX_BYTES = 512 * 1024 * 1024
@dataclass(frozen=True, slots=True)
class RefEncodeCacheKey:
kind: str
preprocess_version: int
identity: str
cache_version: int = REF_ENCODE_CACHE_VERSION
@dataclass(frozen=True, slots=True)
class RefEncodeCacheEvent:
kind: str
action: str
reason: str = ""
payload_bytes: int = 0
@dataclass(slots=True)
class _CacheEntry:
tensor: torch.Tensor
payload_bytes: int
last_used: int
CacheEventSink = Callable[[RefEncodeCacheEvent], None]
def make_ref_encode_cache_key(
kind: str,
preprocess_version: int,
identity: str,
) -> RefEncodeCacheKey:
return RefEncodeCacheKey(
kind=str(kind),
preprocess_version=int(preprocess_version),
identity=str(identity),
)
class RefEncodeCache:
"""Small process-local LRU split into weak VAE object namespaces."""
def __init__(
self,
*,
max_entries: int = DEFAULT_MAX_ENTRIES,
max_bytes: int = DEFAULT_MAX_BYTES,
) -> None:
if int(max_entries) < 1:
raise ValueError("max_entries must be positive")
if int(max_bytes) < 1:
raise ValueError("max_bytes must be positive")
self.max_entries = int(max_entries)
self.max_bytes = int(max_bytes)
self._lock = threading.RLock()
self._namespaces: weakref.WeakKeyDictionary[
Any, OrderedDict[RefEncodeCacheKey, _CacheEntry]
] = weakref.WeakKeyDictionary()
self._clock = 0
self._stats: Counter[str] = Counter()
self._kind_stats: dict[str, Counter[str]] = {}
@staticmethod
def _vae_cacheable(vae: Any) -> bool:
try:
weakref.ref(vae)
hash(vae)
except TypeError:
return False
return True
def supports_vae(self, vae: Any) -> bool:
return self._vae_cacheable(vae)
def _next_clock_locked(self) -> int:
self._clock += 1
return self._clock
def _record_locked(
self,
*,
key: RefEncodeCacheKey,
action: str,
reason: str = "",
payload_bytes: int = 0,
) -> RefEncodeCacheEvent:
self._stats[str(action)] += 1
self._kind_stats.setdefault(key.kind, Counter())[str(action)] += 1
return RefEncodeCacheEvent(
kind=key.kind,
action=str(action),
reason=str(reason),
payload_bytes=int(payload_bytes),
)
@staticmethod
def _emit(event: RefEncodeCacheEvent, sink: CacheEventSink | None) -> None:
if sink is not None:
sink(event)
def _resident_locked(self) -> tuple[int, int]:
entries = 0
payload_bytes = 0
for namespace in list(self._namespaces.values()):
entries += len(namespace)
payload_bytes += sum(entry.payload_bytes for entry in namespace.values())
return entries, payload_bytes
def _evict_locked(self) -> None:
while True:
entries, payload_bytes = self._resident_locked()
if entries <= self.max_entries and payload_bytes <= self.max_bytes:
return
oldest_namespace = None
oldest_key = None
oldest_clock = None
for namespace in list(self._namespaces.values()):
for key, entry in namespace.items():
if oldest_clock is None or entry.last_used < oldest_clock:
oldest_namespace = namespace
oldest_key = key
oldest_clock = entry.last_used
if oldest_namespace is None or oldest_key is None:
return
oldest_namespace.pop(oldest_key, None)
self._stats["evictions"] += 1
def lookup(
self,
vae: Any,
key: RefEncodeCacheKey,
*,
event_sink: CacheEventSink | None = None,
) -> torch.Tensor | None:
if not self._vae_cacheable(vae):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="vae_not_weakrefable_or_hashable",
)
self._emit(event, event_sink)
return None
master = None
with self._lock:
namespace = self._namespaces.get(vae)
entry = namespace.get(key) if namespace is not None else None
if entry is None:
event = self._record_locked(key=key, action="miss")
else:
entry.last_used = self._next_clock_locked()
namespace.move_to_end(key)
master = entry.tensor
event = self._record_locked(
key=key,
action="hit",
payload_bytes=entry.payload_bytes,
)
self._emit(event, event_sink)
return None if master is None else master.clone()
def store(
self,
vae: Any,
key: RefEncodeCacheKey,
tensor: Any,
*,
event_sink: CacheEventSink | None = None,
) -> bool:
if not self._vae_cacheable(vae):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="vae_not_weakrefable_or_hashable",
)
self._emit(event, event_sink)
return False
if not torch.is_tensor(tensor):
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_not_tensor",
)
self._emit(event, event_sink)
return False
if tensor.device.type != "cpu":
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_not_cpu",
)
self._emit(event, event_sink)
return False
payload_bytes = int(tensor.numel()) * int(tensor.element_size())
if payload_bytes > self.max_bytes:
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="entry_exceeds_byte_limit",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return False
try:
master = tensor.detach().contiguous().clone()
except Exception:
with self._lock:
event = self._record_locked(
key=key,
action="bypass",
reason="payload_copy_failed",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return False
with self._lock:
namespace = self._namespaces.get(vae)
if namespace is None:
namespace = OrderedDict()
self._namespaces[vae] = namespace
namespace[key] = _CacheEntry(
tensor=master,
payload_bytes=payload_bytes,
last_used=self._next_clock_locked(),
)
namespace.move_to_end(key)
self._evict_locked()
event = self._record_locked(
key=key,
action="store",
payload_bytes=payload_bytes,
)
self._emit(event, event_sink)
return True
def clear(self) -> None:
with self._lock:
self._namespaces = weakref.WeakKeyDictionary()
self._clock = 0
self._stats.clear()
self._kind_stats.clear()
def inspect(self) -> dict[str, Any]:
with self._lock:
entries, payload_bytes = self._resident_locked()
return {
"cache_version": REF_ENCODE_CACHE_VERSION,
"resident_entries": entries,
"resident_bytes": payload_bytes,
"max_entries": self.max_entries,
"max_bytes": self.max_bytes,
"stats": dict(self._stats),
"kind_stats": {
kind: dict(values)
for kind, values in sorted(self._kind_stats.items())
},
"vae_namespaces": len(self._namespaces),
}
_REF_ENCODE_CACHE = RefEncodeCache()
def get_ref_encode_cache() -> RefEncodeCache:
return _REF_ENCODE_CACHE
def clear_ref_encode_cache() -> None:
_REF_ENCODE_CACHE.clear()
def inspect_ref_encode_cache() -> dict[str, Any]:
return _REF_ENCODE_CACHE.inspect()
def format_ref_encode_cache_diagnostics(
events: list[RefEncodeCacheEvent] | tuple[RefEncodeCacheEvent, ...],
) -> str:
if not events:
return ""
labels = {
"reference_image": "image",
"video_guide": "video",
"reference_audio": "audio",
}
by_kind: dict[str, Counter[str]] = {}
totals: Counter[str] = Counter()
for event in events:
by_kind.setdefault(event.kind, Counter())[event.action] += 1
totals[event.action] += 1
components = []
for kind in ("reference_image", "video_guide", "reference_audio"):
values = by_kind.get(kind)
if not values:
continue
components.append(
f"{labels[kind]} hit={values['hit']}/miss={values['miss']}"
f"/store={values['store']}/bypass={values['bypass']}"
)
state = inspect_ref_encode_cache()
resident_mib = int(state["resident_bytes"]) / float(1024**2)
limit_mib = int(state["max_bytes"]) / float(1024**2)
components.append(
"total "
f"hits={totals['hit']}, misses={totals['miss']}, "
f"stores={totals['store']}, bypasses={totals['bypass']}"
)
components.append(
f"resident={resident_mib:.1f} MiB/{limit_mib:.1f} MiB "
f"({int(state['resident_entries'])}/{int(state['max_entries'])} entries)"
)
return "Reference encode cache: " + "; ".join(components) + "."
+287
View File
@@ -0,0 +1,287 @@
"""Pure RR-R3 plan joining fixed Reference identities to Core Picture numbers.
The caller supplies only immutable routing and Reference metadata. This module
does not encode images, rewrite prompts, select runtime sources, or sample.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from .reference_routing import REFERENCE_SLOT_IDS, ReferenceRoutingSchedule
_SLOT_ORDER = {slot_id: index for index, slot_id in enumerate(REFERENCE_SLOT_IDS)}
@dataclass(frozen=True, slots=True)
class EffectivePlanIssue:
code: str
message: str
source_slot_id: str | None = None
@dataclass(frozen=True, slots=True)
class ReferencePicture:
"""A fixed source slot, compact asset, content hash, and Qwen Picture number."""
source_slot_id: str
asset_index: int
image_sha256: str
picture_number: int
@property
def picture_tag(self) -> str:
return f"<Picture {self.picture_number}>"
@dataclass(frozen=True, slots=True)
class PictureMap:
"""Core's image presentation order; First/Last are not Reference assets."""
first_picture_number: int | None
last_picture_number: int | None
references: tuple[ReferencePicture, ...]
@property
def source_slot_ids(self) -> tuple[str, ...]:
return tuple(item.source_slot_id for item in self.references)
@dataclass(frozen=True, slots=True)
class LogicalEffectiveRoute:
logical_chunk: int
requested_source_slot_ids: tuple[str, ...]
picture_map: PictureMap
@dataclass(frozen=True, slots=True)
class PhysicalEffectiveRoute:
physical_group: int
logical_chunks: tuple[int, ...]
terminal_atomic: bool
routing_status: str
status: str
requested_source_slot_ids: tuple[str, ...] | None
picture_map: PictureMap | None
@dataclass(frozen=True, slots=True)
class EffectiveReferencePlan:
mode: str
legacy_fallback: bool
connected_source_slot_ids: tuple[str, ...]
logical_routes: tuple[LogicalEffectiveRoute, ...] = ()
physical_routes: tuple[PhysicalEffectiveRoute, ...] = ()
issues: tuple[EffectivePlanIssue, ...] = ()
@property
def valid(self) -> bool:
return not self.issues
@property
def has_route_conflicts(self) -> bool:
return any(group.status == "conflict" for group in self.physical_routes)
def _schedule_issues(schedule: ReferenceRoutingSchedule) -> tuple[EffectivePlanIssue, ...]:
if not schedule.valid:
return tuple(
EffectivePlanIssue(f"routing_{issue.code}", issue.message, issue.slot_id)
for issue in schedule.issues
)
total = schedule.total_chunks
if isinstance(total, bool) or not isinstance(total, int) or total < 1:
return (EffectivePlanIssue("invalid_routing_schedule", "missing logical chunk count"),)
if tuple(route.logical_chunk for route in schedule.logical_routes) != tuple(range(1, total + 1)):
return (EffectivePlanIssue("invalid_routing_schedule", "logical chunks are not the full project sequence"),)
logical_slots = {route.logical_chunk: route.reference_slot_ids for route in schedule.logical_routes}
for route in schedule.logical_routes:
if any(slot_id not in _SLOT_ORDER for slot_id in route.reference_slot_ids):
return (EffectivePlanIssue("invalid_routing_schedule", "logical route has an unknown Reference slot"),)
if len(set(route.reference_slot_ids)) != len(route.reference_slot_ids):
return (EffectivePlanIssue("invalid_routing_schedule", "logical route repeats a Reference slot"),)
if tuple(sorted(route.reference_slot_ids, key=_SLOT_ORDER.get)) != route.reference_slot_ids:
return (EffectivePlanIssue("invalid_routing_schedule", "logical Reference slots are not ordered"),)
covered = tuple(chunk for group in schedule.physical_routes for chunk in group.logical_chunks)
if covered != tuple(range(1, total + 1)):
return (EffectivePlanIssue("invalid_routing_schedule", "physical groups do not cover the logical sequence"),)
for group in schedule.physical_routes:
if group.terminal_atomic != (len(group.logical_chunks) > 1):
return (EffectivePlanIssue("invalid_routing_schedule", "physical group atomicity disagrees with its chunks"),)
members = tuple(logical_slots[chunk] for chunk in group.logical_chunks)
if group.logical_reference_slot_ids != members:
return (EffectivePlanIssue("invalid_routing_schedule", "physical group member routes disagree with logical routes"),)
consistent = all(member == members[0] for member in members[1:])
if consistent:
expected_status = "empty" if not members[0] else "consistent"
if group.status != expected_status or group.reference_slot_ids != members[0]:
return (EffectivePlanIssue("invalid_routing_schedule", "physical group route is not its shared member route"),)
elif group.status != "conflict" or group.reference_slot_ids is not None:
return (EffectivePlanIssue("invalid_routing_schedule", "conflicting physical group has an effective route"),)
return ()
def _source_issues(
source_slot_ids: Any, image_hashes: Any, asset_count: Any
) -> tuple[EffectivePlanIssue, ...]:
issues: list[EffectivePlanIssue] = []
if isinstance(asset_count, bool) or not isinstance(asset_count, int) or asset_count < 0:
issues.append(EffectivePlanIssue("invalid_asset_count", "Reference asset count must be a non-negative integer"))
if not isinstance(source_slot_ids, tuple):
issues.append(EffectivePlanIssue("invalid_source_slots", "source_slot_ids must be a tuple"))
return tuple(issues)
if not isinstance(image_hashes, tuple):
issues.append(EffectivePlanIssue("invalid_image_hashes", "Reference image hashes must be a tuple"))
elif len(image_hashes) != len(source_slot_ids):
issues.append(EffectivePlanIssue("hash_count_mismatch", "Reference image hashes do not align with fixed source slots"))
elif any(not isinstance(value, str) or len(value) != 64 or
any(char not in "0123456789abcdef" for char in value) for value in image_hashes):
issues.append(EffectivePlanIssue("invalid_image_hash", "Reference image hashes must be lowercase SHA-256 values"))
if isinstance(asset_count, int) and not isinstance(asset_count, bool):
if asset_count > 0 and not source_slot_ids:
issues.append(EffectivePlanIssue("missing_source_slots", "connected Reference assets have no fixed source slot IDs"))
elif asset_count != len(source_slot_ids):
issues.append(EffectivePlanIssue("asset_count_mismatch", "source slot IDs do not align with compact Reference assets"))
for slot_id in source_slot_ids:
if not isinstance(slot_id, str) or slot_id not in _SLOT_ORDER:
issues.append(EffectivePlanIssue("unknown_source_slot", f"unknown Reference source slot: {slot_id!r}", str(slot_id)))
if not any(issue.code == "unknown_source_slot" for issue in issues):
if len(set(source_slot_ids)) != len(source_slot_ids):
issues.append(EffectivePlanIssue("duplicate_source_slot", "a fixed Reference source slot occurs more than once"))
elif tuple(sorted(source_slot_ids, key=_SLOT_ORDER.get)) != source_slot_ids:
issues.append(EffectivePlanIssue("source_slot_order", "source slot IDs do not follow the compact R1-R9 order"))
return tuple(issues)
def _picture_map(
requested_source_slot_ids: tuple[str, ...],
connected_asset_indices: dict[str, int],
image_hashes: tuple[str, ...],
*,
has_first_image: bool,
has_last_image: bool,
) -> PictureMap:
first_number = 1 if has_first_image else None
last_number = 1 + int(has_first_image) if has_last_image else None
picture_offset = int(has_first_image) + int(has_last_image)
selected = tuple(slot_id for slot_id in requested_source_slot_ids if slot_id in connected_asset_indices)
references = tuple(
ReferencePicture(
slot_id, connected_asset_indices[slot_id],
image_hashes[connected_asset_indices[slot_id]], picture_offset + index,
)
for index, slot_id in enumerate(selected, start=1)
)
return PictureMap(first_number, last_number, references)
def compile_group_picture_map(
*,
source_slot_ids: tuple[str, ...],
reference_image_hashes: tuple[str, ...],
has_first_image: bool,
has_last_image: bool,
) -> PictureMap:
"""Use RR-R3's Picture numbering for one already-selected physical group."""
issues = _source_issues(source_slot_ids, reference_image_hashes, len(source_slot_ids))
if issues or type(has_first_image) is not bool or type(has_last_image) is not bool:
raise ValueError("invalid internal Reference group identity or anchor flags")
return _picture_map(
source_slot_ids,
{slot_id: index for index, slot_id in enumerate(source_slot_ids)},
reference_image_hashes,
has_first_image=has_first_image,
has_last_image=has_last_image,
)
def compile_effective_reference_plan(
*,
routing_schedule: ReferenceRoutingSchedule,
source_slot_ids: tuple[str, ...],
reference_image_hashes: tuple[str, ...],
reference_asset_count: int,
has_first_image: bool,
has_last_image: bool,
) -> EffectiveReferencePlan:
"""Map full logical routes to compact assets and Core Qwen Picture numbers.
Anchor flags describe the presentation being planned. The caller retains
responsibility for choosing when First/Last are supplied to conditioning.
"""
if not isinstance(routing_schedule, ReferenceRoutingSchedule):
return EffectiveReferencePlan(
mode="invalid", legacy_fallback=False, connected_source_slot_ids=(),
issues=(EffectivePlanIssue("invalid_routing_schedule", "expected an RR-R2 routing schedule"),),
)
issues = [
*_schedule_issues(routing_schedule),
*_source_issues(source_slot_ids, reference_image_hashes, reference_asset_count),
]
if type(has_first_image) is not bool or type(has_last_image) is not bool:
issues.append(EffectivePlanIssue("invalid_anchor_flags", "First/Last presence flags must be booleans"))
if issues:
return EffectiveReferencePlan(
mode=routing_schedule.mode,
legacy_fallback=routing_schedule.legacy_fallback,
connected_source_slot_ids=source_slot_ids if isinstance(source_slot_ids, tuple) else (),
issues=tuple(issues),
)
asset_indices = {slot_id: index for index, slot_id in enumerate(source_slot_ids)}
logical_routes = tuple(
LogicalEffectiveRoute(
logical_chunk=route.logical_chunk,
requested_source_slot_ids=route.reference_slot_ids,
picture_map=_picture_map(
route.reference_slot_ids, asset_indices, reference_image_hashes,
has_first_image=has_first_image, has_last_image=has_last_image,
),
)
for route in routing_schedule.logical_routes
)
physical_routes: list[PhysicalEffectiveRoute] = []
for group in routing_schedule.physical_routes:
if group.status == "conflict":
physical_routes.append(
PhysicalEffectiveRoute(
physical_group=group.physical_group,
logical_chunks=group.logical_chunks,
terminal_atomic=group.terminal_atomic,
routing_status=group.status,
status="conflict",
requested_source_slot_ids=None,
picture_map=None,
)
)
continue
requested = group.reference_slot_ids
assert requested is not None
picture_map = _picture_map(
requested, asset_indices, reference_image_hashes,
has_first_image=has_first_image, has_last_image=has_last_image,
)
physical_routes.append(
PhysicalEffectiveRoute(
physical_group=group.physical_group,
logical_chunks=group.logical_chunks,
terminal_atomic=group.terminal_atomic,
routing_status=group.status,
status="active" if picture_map.references else "empty",
requested_source_slot_ids=requested,
picture_map=picture_map,
)
)
return EffectiveReferencePlan(
mode=routing_schedule.mode,
legacy_fallback=routing_schedule.legacy_fallback,
connected_source_slot_ids=source_slot_ids,
logical_routes=logical_routes,
physical_routes=tuple(physical_routes),
)
+111
View File
@@ -0,0 +1,111 @@
"""Fixed nine-slot Reference Images input for the V3.9 sampler."""
from __future__ import annotations
from dataclasses import dataclass
import torch
from ..reference import ReferenceImageBundle
from .reference_routing import REFERENCE_SLOT_IDS, parse_chunk_selector
REFERENCE_IMAGES_V39_TYPE = "H3_CONTINUUM_REFERENCE_IMAGES_V39"
REFERENCE_USE_ALL = "All chunks"
REFERENCE_USE_PER_CHUNK = "Per chunk"
REFERENCE_USE_OPTIONS = (REFERENCE_USE_ALL, REFERENCE_USE_PER_CHUNK)
MAX_CHUNKS = 16 # Current Sampler schema limit; checked in integration tests.
@dataclass(frozen=True, slots=True)
class ReferenceImagesV39Bundle:
images: tuple[torch.Tensor | None, ...]
reference_use: str
selectors: tuple[str, ...]
def _canonical_selector(value: str) -> str:
parsed = parse_chunk_selector(value, total_chunks=MAX_CHUNKS)
if not parsed.valid:
details = "; ".join(issue.message for issue in parsed.issues)
raise ValueError(f"invalid V3.9 Reference chunk selection: {details}")
return ",".join(map(str, parsed.logical_chunks)) or "off"
def effective_reference_inputs(bundle: ReferenceImagesV39Bundle | None, *, chunks: int):
"""Return legacy-shaped images and private selectors without changing saved data."""
if not isinstance(chunks, int) or isinstance(chunks, bool) or not 1 <= chunks <= MAX_CHUNKS:
raise ValueError(f"V3.9 Reference Chunks must be within 1..{MAX_CHUNKS}")
if bundle is None:
images = (None,) * 9
mode = REFERENCE_USE_ALL
selections = ("off",) * 9
elif (not isinstance(bundle, ReferenceImagesV39Bundle)
or len(bundle.images) != 9 or len(bundle.selectors) != 9
or bundle.reference_use not in REFERENCE_USE_OPTIONS):
raise ValueError("V3.9 Reference Images requires its nine-slot V3.9 bundle")
else:
images = bundle.images
mode = bundle.reference_use
selections = bundle.selectors
selectors = {}
for index, (image, selection) in enumerate(zip(images, selections, strict=True), start=1):
if image is None:
selectors[f"R{index}"] = "off"
elif mode == REFERENCE_USE_ALL:
selectors[f"R{index}"] = "all"
else:
parsed = parse_chunk_selector(selection, total_chunks=MAX_CHUNKS)
if not parsed.valid:
details = "; ".join(issue.message for issue in parsed.issues)
raise ValueError(f"R{index} chunk selection: {details}")
active = (n for n in parsed.logical_chunks if n <= chunks)
selectors[f"R{index}"] = ",".join(map(str, active)) or "off"
if tuple(selectors) != REFERENCE_SLOT_IDS:
raise ValueError("V3.9 Reference slot ordering changed")
return images[:3], ReferenceImageBundle(images[3:]), selectors
class H3ContinuumReferenceImagesV39:
CATEGORY = "MiniMax H3/Continuum/Input"
DESCRIPTION = "Connect up to nine fixed Reference Images; choose chunks in this node."
RETURN_TYPES = (REFERENCE_IMAGES_V39_TYPE,)
RETURN_NAMES = ("reference_images",)
FUNCTION = "pack"
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"reference_use": (REFERENCE_USE_OPTIONS, {
"default": REFERENCE_USE_ALL,
"display_name": "Reference Use",
"tooltip": "All chunks uses connected images throughout. Per chunk uses the assignments below.",
}),
**{
f"reference_r{index}_chunks": ("STRING", {
"default": "off", "advanced": True,
"display_name": f"Image {index} chunks",
"tooltip": f"Saved assignments for Reference Image {index}; use the in-node table.",
}) for index in range(1, 10)
},
},
"optional": {
f"reference_image_{index}": ("IMAGE", {
"display_name": f"Reference Image {index} (Optional)",
"tooltip": f"Fixed Image {index} / @R{index}. Its number never changes with other connections.",
}) for index in range(1, 10)
},
}
def pack(self, reference_use=REFERENCE_USE_ALL, **kwargs):
if reference_use not in REFERENCE_USE_OPTIONS:
raise ValueError("Reference Use must be All chunks or Per chunk")
images = tuple(kwargs.get(f"reference_image_{index}") for index in range(1, 10))
selectors = tuple(
(_canonical_selector(kwargs.get(f"reference_r{index}_chunks", "off"))
if reference_use == REFERENCE_USE_PER_CHUNK else
str(kwargs.get(f"reference_r{index}_chunks", "off")))
for index in range(1, 10)
)
return (ReferenceImagesV39Bundle(images, reference_use, selectors),)
@@ -0,0 +1,69 @@
"""Read-only V3.9 projection of the Reference contract used by this Queue.
This module never selects images or changes a route. Pending groups are shown
as settings only; accepted groups use the frozen/observed RR-R5 descriptor.
"""
from __future__ import annotations
from typing import Any
from .reference_runtime import ReferenceRoutingRuntime
def project_reference_plan(runtime: ReferenceRoutingRuntime) -> dict[str, Any]:
if not isinstance(runtime, ReferenceRoutingRuntime):
raise TypeError("Reference Plan Inspector requires the active routing runtime")
accepted = int(runtime.accepted_chunks)
generated = set(runtime.generated_groups)
groups: list[dict[str, Any]] = []
for route in runtime.schedule.physical_routes:
group = int(route.physical_group)
complete = route.logical_chunks[-1] <= accepted
item: dict[str, Any] = {
"physical_group": group,
"logical_chunks": list(route.logical_chunks),
"requested_slots": list(route.reference_slot_ids or ()),
"status": (
"pending" if not complete else
"generated" if group in generated else "reused"
),
}
if complete:
contract = runtime.group_contract_for(group)
item["contract_sha256"] = contract["sha256"]
item["descriptor"] = contract["descriptor"]
item["warnings"] = list(runtime.group_warnings.get(group, ()))
groups.append(item)
return {
"version": 1,
"mode": runtime.schedule.mode,
"accepted_chunks": accepted,
"connected_slots": list(runtime.inputs.connected_source_slot_ids),
"groups": groups,
}
def format_reference_plan(plan: dict[str, Any]) -> str:
lines = [
"Reference Plan Inspector (last Queue, runtime verified)",
f"Mode: {plan['mode']}; accepted chunks: {plan['accepted_chunks']}",
"Connected: " + (", ".join(plan["connected_slots"]) or "none"),
]
for group in plan["groups"]:
label = ",".join(str(chunk) for chunk in group["logical_chunks"])
head = f"Group {group['physical_group']} [C{label}] {group['status']}"
descriptor = group.get("descriptor")
if descriptor is None:
lines.append(head + " — configured " + (
", ".join(group["requested_slots"]) or "no References"
))
continue
mapping = ", ".join(
f"{ref['source_slot_id']}=<Picture {ref['picture_number']}>"
for ref in descriptor["references"]
) or "no References"
lines.append(head + " — " + mapping + f"; contract={group['contract_sha256'][:12]}")
for warning in group["warnings"]:
lines.append(" Warning: " + warning)
return "\n".join(lines)
+365
View File
@@ -0,0 +1,365 @@
"""Pure V3.9 Reference selector parsing and group schedule planning.
V3.9 Custom consumes this plan through the Reference runtime and storage
contracts. Planning here remains free of Sampling and persistence side effects.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
import re
from typing import Any
from ..branch_provenance import BranchProvenanceError, physical_groups
REFERENCE_SLOT_IDS = tuple(f"R{index}" for index in range(1, 10))
_INTEGER_TOKEN = re.compile(r"^[0-9]+$")
_RANGE_TOKEN = re.compile(r"^([0-9]+)\s*-\s*([0-9]+)$")
@dataclass(frozen=True, slots=True)
class RoutingIssue:
"""A classified parser/planner issue; it does not itself stop execution."""
code: str
message: str
slot_id: str | None = None
selector: str | None = None
@dataclass(frozen=True, slots=True)
class ChunkSelectorResult:
"""Parsed selector with all/off kept distinct from explicit chunk lists."""
kind: str
logical_chunks: tuple[int, ...]
issues: tuple[RoutingIssue, ...] = ()
@property
def valid(self) -> bool:
return self.kind != "invalid" and not self.issues
@dataclass(frozen=True, slots=True)
class LogicalReferenceRoute:
"""Reference identities selected for one project-wide logical chunk."""
logical_chunk: int
reference_slot_ids: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class PhysicalReferenceRoute:
"""Routing facts for one authoritative atomic physical sampling group."""
physical_group: int
logical_chunks: tuple[int, ...]
terminal_atomic: bool
logical_reference_slot_ids: tuple[tuple[str, ...], ...]
status: str
reference_slot_ids: tuple[str, ...] | None
@dataclass(frozen=True, slots=True)
class ReferenceRoutingSchedule:
"""Full logical schedule plus its physical-group compatibility view."""
mode: str
legacy_fallback: bool
total_chunks: int | None
terminal_merge_enabled: bool | None
logical_routes: tuple[LogicalReferenceRoute, ...] = ()
physical_routes: tuple[PhysicalReferenceRoute, ...] = ()
issues: tuple[RoutingIssue, ...] = ()
@property
def valid(self) -> bool:
return not self.issues
def _invalid_selector(code: str, message: str, selector: str | None) -> ChunkSelectorResult:
return ChunkSelectorResult(
kind="invalid",
logical_chunks=(),
issues=(RoutingIssue(code=code, message=message, selector=selector),),
)
def parse_chunk_selector(selector: Any, *, total_chunks: int) -> ChunkSelectorResult:
"""Parse all/off, a one-based chunk, inclusive ranges, and their combinations.
Invalid values are returned as structured issues so callers can inspect
them without introducing a new runtime execution stop.
"""
if isinstance(total_chunks, bool) or not isinstance(total_chunks, int) or total_chunks < 1:
return _invalid_selector(
"invalid_chunk_count",
"total_chunks must be a positive integer",
selector if isinstance(selector, str) else None,
)
if not isinstance(selector, str):
return _invalid_selector(
"selector_not_string",
"selector must be a string",
None,
)
text = selector.strip()
normalized = text.casefold()
if not normalized:
return _invalid_selector("selector_empty", "selector is empty", selector)
if normalized == "all":
return ChunkSelectorResult("all", tuple(range(1, total_chunks + 1)))
if normalized == "off":
return ChunkSelectorResult("off", ())
selected: set[int] = set()
issues: list[RoutingIssue] = []
for raw_token in text.split(","):
token = raw_token.strip()
if not token:
issues.append(
RoutingIssue(
"empty_selector_item",
"selector contains an empty comma-separated item",
selector=selector,
)
)
continue
if _INTEGER_TOKEN.fullmatch(token):
start = end = int(token)
else:
match = _RANGE_TOKEN.fullmatch(token)
if match is None:
issues.append(
RoutingIssue(
"invalid_selector_item",
f"invalid selector item: {token!r}",
selector=selector,
)
)
continue
start, end = int(match.group(1)), int(match.group(2))
if start > end:
issues.append(
RoutingIssue(
"reversed_range",
f"range start {start} exceeds end {end}",
selector=selector,
)
)
continue
if start < 1 or end > total_chunks:
issues.append(
RoutingIssue(
"chunk_out_of_range",
f"selector item {token!r} is outside 1..{total_chunks}",
selector=selector,
)
)
continue
values = set(range(start, end + 1))
if selected.intersection(values):
issues.append(
RoutingIssue(
"duplicate_chunk",
f"selector item {token!r} overlaps an earlier item",
selector=selector,
)
)
continue
selected.update(values)
if issues:
return ChunkSelectorResult("invalid", (), tuple(issues))
return ChunkSelectorResult("chunks", tuple(sorted(selected)))
def _invalid_schedule(
*,
mode: str,
legacy_fallback: bool,
total_chunks: int | None,
terminal_merge_enabled: bool | None,
issues: tuple[RoutingIssue, ...],
) -> ReferenceRoutingSchedule:
return ReferenceRoutingSchedule(
mode=mode,
legacy_fallback=legacy_fallback,
total_chunks=total_chunks,
terminal_merge_enabled=terminal_merge_enabled,
issues=issues,
)
def compile_reference_routing_schedule(
*,
total_chunks: int,
terminal_merge_enabled: bool,
mode: Any = None,
selectors_by_slot: Any = None,
) -> ReferenceRoutingSchedule:
"""Compile routing over full logical chunk numbers and physical groups.
An absent mode preserves legacy All behavior. Custom mode requires an
explicit selector for each fixed R1-R9 slot; explicit all-off is valid and
remains distinct from an absent mode. Terminal atomic groups with
different member routes are surfaced as conflicts without auto-correction.
"""
legacy_fallback = mode is None
normalized_mode = "all" if legacy_fallback else None
issues: list[RoutingIssue] = []
if isinstance(total_chunks, bool) or not isinstance(total_chunks, int) or total_chunks < 1:
issues.append(RoutingIssue("invalid_chunk_count", "total_chunks must be a positive integer"))
if type(terminal_merge_enabled) is not bool:
issues.append(
RoutingIssue("invalid_terminal_merge", "terminal_merge_enabled must be a boolean")
)
if not legacy_fallback:
if not isinstance(mode, str):
issues.append(RoutingIssue("invalid_mode", "mode must be All or Custom"))
else:
token = mode.strip().casefold()
if token == "all":
normalized_mode = "all"
elif token == "custom":
normalized_mode = "custom"
else:
issues.append(RoutingIssue("invalid_mode", "mode must be All or Custom"))
if issues:
return _invalid_schedule(
mode=normalized_mode or "invalid",
legacy_fallback=legacy_fallback,
total_chunks=total_chunks if isinstance(total_chunks, int) and not isinstance(total_chunks, bool) else None,
terminal_merge_enabled=terminal_merge_enabled if type(terminal_merge_enabled) is bool else None,
issues=tuple(issues),
)
slot_selectors: dict[str, ChunkSelectorResult] = {}
if normalized_mode == "custom":
if not isinstance(selectors_by_slot, Mapping):
issues.append(
RoutingIssue(
"selectors_required",
"Custom mode requires an explicit selector map for R1-R9",
)
)
else:
for slot_id in selectors_by_slot:
if slot_id not in REFERENCE_SLOT_IDS:
issues.append(
RoutingIssue(
"unknown_reference_slot",
f"unknown Reference slot: {slot_id!r}",
slot_id=str(slot_id),
)
)
for slot_id in REFERENCE_SLOT_IDS:
if slot_id not in selectors_by_slot:
issues.append(
RoutingIssue(
"missing_selector",
f"Custom mode has no selector for {slot_id}",
slot_id=slot_id,
)
)
continue
parsed = parse_chunk_selector(
selectors_by_slot[slot_id], total_chunks=total_chunks
)
if not parsed.valid:
issues.extend(
RoutingIssue(
issue.code,
issue.message,
slot_id=slot_id,
selector=issue.selector,
)
for issue in parsed.issues
)
else:
slot_selectors[slot_id] = parsed
if issues:
return _invalid_schedule(
mode=normalized_mode or "invalid",
legacy_fallback=legacy_fallback,
total_chunks=total_chunks,
terminal_merge_enabled=terminal_merge_enabled,
issues=tuple(issues),
)
logical_routes: list[LogicalReferenceRoute] = []
for logical_chunk in range(1, total_chunks + 1):
if normalized_mode == "all":
selected_slots = REFERENCE_SLOT_IDS
else:
selected_slots = tuple(
slot_id
for slot_id in REFERENCE_SLOT_IDS
if logical_chunk in slot_selectors[slot_id].logical_chunks
)
logical_routes.append(
LogicalReferenceRoute(logical_chunk, tuple(selected_slots))
)
try:
groups = physical_groups(
chunks=total_chunks,
terminal_merge_enabled=terminal_merge_enabled,
)
except BranchProvenanceError as exc:
return _invalid_schedule(
mode=normalized_mode or "invalid",
legacy_fallback=legacy_fallback,
total_chunks=total_chunks,
terminal_merge_enabled=terminal_merge_enabled,
issues=(RoutingIssue("invalid_physical_groups", str(exc)),),
)
routes_by_chunk = {route.logical_chunk: route.reference_slot_ids for route in logical_routes}
physical_routes: list[PhysicalReferenceRoute] = []
for group in groups:
chunks = tuple(range(group.start, group.end + 1))
member_routes = tuple(routes_by_chunk[chunk] for chunk in chunks)
first_route = member_routes[0]
consistent = all(route == first_route for route in member_routes[1:])
if not consistent:
group_status = "conflict"
group_slots = None
elif not first_route:
group_status = "empty"
group_slots = ()
else:
group_status = "consistent"
group_slots = first_route
physical_routes.append(
PhysicalReferenceRoute(
physical_group=int(group.physical_group),
logical_chunks=chunks,
terminal_atomic=group.start != group.end,
logical_reference_slot_ids=member_routes,
status=group_status,
reference_slot_ids=group_slots,
)
)
return ReferenceRoutingSchedule(
mode=normalized_mode or "invalid",
legacy_fallback=legacy_fallback,
total_chunks=total_chunks,
terminal_merge_enabled=terminal_merge_enabled,
logical_routes=tuple(logical_routes),
physical_routes=tuple(physical_routes),
)
+360
View File
@@ -0,0 +1,360 @@
"""V3.9 Custom group-local Reference conditioning.
The V3.9 caller supplies this adapter after routing is resolved. V3.8 and
V3.9 All retain the legacy path; saved-run compatibility is handled by the
separate Reference storage contract.
"""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
import re
import uuid
from typing import Any
import torch
from ..reference import (
ReferenceAssets,
ReferenceImageBundle,
encode_reference_latents,
encode_reference_latents_cached,
encode_reference_prompt,
prepare_reference_assets,
resolve_reference_image_inputs,
)
from ..v2.h3_builder import encode_prompt_conditioning, encode_prompt_conditioning_cached
from .reference_effective_plan import PictureMap, compile_group_picture_map
from .reference_routing import REFERENCE_SLOT_IDS, ReferenceRoutingSchedule
_REFERENCE_TAG = re.compile(r"(?<![A-Za-z0-9_])@R([1-9])(?![A-Za-z0-9_])")
class ReferenceRuntimeContractError(ValueError):
"""An internal RR-R4 caller supplied an unusable routing contract."""
@dataclass(slots=True)
class ReferenceRoutingRequest:
"""Private V3.9 node-to-V2 request; never part of a saved node schema."""
selectors_by_slot: dict[str, str]
inputs: ReferenceInputSet
runtime: ReferenceRoutingRuntime | None = None
@dataclass(frozen=True, slots=True)
class ReferenceInputSet:
"""Raw R1-R9 sockets; no image is validated, resized, or VAE-encoded here."""
images: tuple[torch.Tensor | None, ...]
output_width: int
output_height: int
size_mode: str
@classmethod
def from_inputs(
cls,
*,
reference_image_1: torch.Tensor | None = None,
reference_image_2: torch.Tensor | None = None,
reference_image_3: torch.Tensor | None = None,
reference_image_4: torch.Tensor | None = None,
reference_image_5: torch.Tensor | None = None,
image_references: ReferenceImageBundle | None = None,
output_width: int,
output_height: int,
size_mode: str,
) -> ReferenceInputSet:
images = resolve_reference_image_inputs(
reference_image_1, reference_image_2, reference_image_3,
reference_image_4, reference_image_5, image_references,
)
return cls(images, int(output_width), int(output_height), str(size_mode))
@property
def connected_source_slot_ids(self) -> tuple[str, ...]:
return tuple(
slot_id for slot_id, image in zip(REFERENCE_SLOT_IDS, self.images, strict=True)
if image is not None
)
def prepare_selected(self, selected_source_slot_ids: tuple[str, ...]) -> ReferenceAssets | None:
return prepare_reference_assets(
reference_image_1=self.images[0],
reference_image_2=self.images[1],
reference_image_3=self.images[2],
image_references=ReferenceImageBundle(self.images[3:]),
output_width=self.output_width,
output_height=self.output_height,
size_mode=self.size_mode,
selected_source_slot_ids=selected_source_slot_ids,
)
@dataclass(frozen=True, slots=True)
class GroupReferenceConditioning:
physical_group: int
logical_chunks: tuple[int, ...]
requested_source_slot_ids: tuple[str, ...]
selected_assets: ReferenceAssets | None
picture_map: PictureMap
effective_prompt: str
conditioning: list[list[Any]]
warnings: tuple[str, ...]
def rewrite_reference_tags(prompt: str, picture_map: PictureMap) -> tuple[str, tuple[str, ...]]:
"""Rewrite active @R tags only; inactive tags are diagnostic, never blocking."""
tags = {item.source_slot_id: item.picture_tag for item in picture_map.references}
inactive: set[str] = set()
def replace_tag(match: re.Match[str]) -> str:
slot_id = f"R{match.group(1)}"
replacement = tags.get(slot_id)
if replacement is None:
inactive.add(slot_id)
return match.group(0)
return replacement
effective = _REFERENCE_TAG.sub(replace_tag, str(prompt))
warnings = tuple(
f"{slot_id} is not active in this physical group; {slot_id}'s @R tag was left unchanged"
for slot_id in REFERENCE_SLOT_IDS if slot_id in inactive
)
return effective, warnings
def resolve_reference_group_prompt(
*, physical_group: int, prompt: str,
requested_source_slot_ids: tuple[str, ...],
connected_source_slot_ids: tuple[str, ...],
picture_map: PictureMap,
) -> tuple[str, tuple[str, ...]]:
"""Use the same prompt and warning rules for planned and sampled groups."""
effective_prompt, prompt_warnings = rewrite_reference_tags(prompt, picture_map)
connected = set(connected_source_slot_ids)
disconnected_warnings = tuple(
f"{slot_id} is routed to physical group {physical_group} but no image is connected"
for slot_id in requested_source_slot_ids if slot_id not in connected
)
return effective_prompt, disconnected_warnings + prompt_warnings
class ReferenceRoutingRuntime:
"""Prepare and encode only the images selected by a physical group."""
def __init__(self, *, schedule: ReferenceRoutingSchedule, inputs: ReferenceInputSet):
if not isinstance(schedule, ReferenceRoutingSchedule) or not schedule.valid:
raise ReferenceRuntimeContractError("RR-R4 requires a valid RR-R2 routing schedule")
if not isinstance(inputs, ReferenceInputSet) or len(inputs.images) != len(REFERENCE_SLOT_IDS):
raise ReferenceRuntimeContractError("RR-R4 requires nine fixed raw Reference slots")
self.schedule = schedule
self.inputs = inputs
# RR-R4 has no saved-run/prefix contract. Keep its session identity
# queue-local without reading images assigned only to later groups.
self._run_nonce = uuid.uuid4().hex
self.storage_plan: Any = None
self.generated_groups: tuple[int, ...] = ()
self.accepted_chunks = 0
self.observed_group_contracts: dict[int, dict[str, Any]] = {}
self.group_warnings: dict[int, tuple[str, ...]] = {}
def group_contract_for(self, physical_group: int) -> dict[str, Any]:
group = int(physical_group)
observed = self.observed_group_contracts.get(group)
if observed is not None:
return observed
if self.storage_plan is not None:
return self.storage_plan.group_for(group)
raise ReferenceRuntimeContractError(
f"physical group {group} has no observed Reference contract"
)
def check_physical_contract(self, *, chunks: int, terminal_merge_enabled: bool) -> None:
if self.schedule.total_chunks != int(chunks) or self.schedule.terminal_merge_enabled != bool(terminal_merge_enabled):
raise ReferenceRuntimeContractError(
"RR-R4 routing schedule does not match the current physical group topology"
)
if any(route.status == "conflict" for route in self.schedule.physical_routes):
raise ReferenceRuntimeContractError(
"RR-R4 cannot select one Reference set for a conflicting terminal physical group"
)
@property
def has_selected_references(self) -> bool:
connected = set(self.inputs.connected_source_slot_ids)
return any(
slot_id in connected
for route in self.schedule.physical_routes
for slot_id in (route.reference_slot_ids or ())
)
def global_identity(self, keyframe_identity_hash: str) -> str:
"""Queue-local identity; RR-R5 will define persistent prefix reuse."""
payload = {
"rr_r4_identity_version": 1,
"run_nonce": self._run_nonce,
"keyframe_identity_hash": str(keyframe_identity_hash),
"mode": self.schedule.mode,
"logical_routes": [
(route.logical_chunk, route.reference_slot_ids)
for route in self.schedule.logical_routes
],
"physical_groups": [
(route.physical_group, route.logical_chunks)
for route in self.schedule.physical_routes
],
"connected_source_slot_ids": self.inputs.connected_source_slot_ids,
"size_mode": self.inputs.size_mode,
"output_size": (self.inputs.output_width, self.inputs.output_height),
}
return hashlib.sha256(
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
def _route(self, physical_group: int):
for route in self.schedule.physical_routes:
if route.physical_group == physical_group:
if route.status == "conflict" or route.reference_slot_ids is None:
raise ReferenceRuntimeContractError(
f"physical group {physical_group} has no single Reference route"
)
return route
raise ReferenceRuntimeContractError(f"unknown physical group {physical_group}")
def prepare_group(
self,
*,
physical_group: int,
prompt: str,
clip: Any,
video_vae: Any,
first_image: torch.Tensor | None,
last_image: torch.Tensor | None,
include_last_image: bool = True,
reference_audio_assets: Any = None,
timeline_video_assets: Any = None,
reference_encode_cache: bool = False,
cache_event: Any = None,
prompt_conditioning_cache: bool = False,
prompt_cache_event: Any = None,
expected_group_contract: dict[str, Any] | None = None,
first_frame_hash: str = "none",
last_frame_hash: str = "none",
terminal_prompt_policy: str | None = None,
) -> GroupReferenceConditioning:
route = self._route(int(physical_group))
requested = route.reference_slot_ids
connected = set(self.inputs.connected_source_slot_ids)
selected = tuple(slot_id for slot_id in requested if slot_id in connected)
assets = self.inputs.prepare_selected(selected)
if assets is not None and assets.source_slot_ids != selected:
raise ReferenceRuntimeContractError("prepared Reference slots differ from the selected route")
# The legacy no-Reference path presents Last Image only to the final
# group. With References, Core's hybrid Qwen presentation includes it
# in every group even though the Last keyframe is attached only at end.
presented_last_image = last_image if assets is not None or include_last_image else None
picture_map = compile_group_picture_map(
source_slot_ids=assets.source_slot_ids if assets is not None else (),
reference_image_hashes=assets.image_hashes if assets is not None else (),
has_first_image=first_image is not None,
has_last_image=presented_last_image is not None,
)
effective_prompt, warnings = resolve_reference_group_prompt(
physical_group=int(physical_group), prompt=prompt,
requested_source_slot_ids=requested,
connected_source_slot_ids=self.inputs.connected_source_slot_ids,
picture_map=picture_map,
)
# Direct RR-R4 callers predate the saved-plan fingerprints. The normal
# V3.9 Sequence supplies both hashes; for an unsaved direct call only,
# derive a truthful anchor from the exact image being presented.
if expected_group_contract is None and self.storage_plan is None:
from ..v2.h3_builder import _tensor_fingerprint
if first_image is not None and first_frame_hash == "none":
first_frame_hash = _tensor_fingerprint(first_image)
if presented_last_image is not None and last_frame_hash == "none":
last_frame_hash = _tensor_fingerprint(presented_last_image)
from .reference_storage_contract import (
ReferenceStorageContractError, canonical_sha256, descriptor_from_facts,
)
descriptor = descriptor_from_facts(
physical_group=int(physical_group),
logical_chunks=route.logical_chunks,
selected_source_slot_ids=assets.source_slot_ids if assets is not None else (),
prepared_image_hashes=assets.image_hashes if assets is not None else (),
picture_map=picture_map,
first_frame_hash=first_frame_hash,
last_frame_hash=last_frame_hash,
effective_prompt=effective_prompt,
size_mode=self.inputs.size_mode,
terminal_prompt_policy=terminal_prompt_policy,
)
group_contract = {"descriptor": descriptor, "sha256": canonical_sha256(descriptor)}
if expected_group_contract is not None:
if (canonical_sha256(descriptor) != expected_group_contract.get("sha256")
or descriptor != expected_group_contract.get("descriptor")):
raise ReferenceStorageContractError(
f"physical group {physical_group} Reference inputs changed after the "
"saved-run preflight; no VAE, Prompt, or Sampling was started"
)
if assets is not None:
assets = (
encode_reference_latents_cached(video_vae, assets, cache_event=cache_event)
if reference_encode_cache else encode_reference_latents(video_vae, assets)
)
conditioning = encode_reference_prompt(
clip, effective_prompt, assets,
first_image=first_image,
last_image=presented_last_image,
reference_audio_assets=reference_audio_assets,
timeline_video_assets=timeline_video_assets,
)
elif (prompt_conditioning_cache and not self.has_selected_references
and reference_audio_assets is None and timeline_video_assets is None
and terminal_prompt_policy is None):
# The Sequence caller enables this only for Fixed, reference-free
# runs. Reuse the V3.8 cache and its conservative CLIP guards; keep
# route/storage validation above and group metadata below intact.
conditioning = encode_prompt_conditioning_cached(
clip, effective_prompt,
first_image=first_image,
last_image=presented_last_image,
first_image_fingerprint=first_frame_hash,
last_image_fingerprint=(last_frame_hash
if presented_last_image is not None else "none"),
cache_enabled=True,
cache_event=prompt_cache_event,
)
else:
conditioning = encode_prompt_conditioning(
clip, effective_prompt,
first_image=first_image,
last_image=presented_last_image,
reference_audio_assets=reference_audio_assets,
timeline_video_assets=timeline_video_assets,
)
existing = self.observed_group_contracts.get(int(physical_group))
if existing is not None and existing != group_contract:
raise ReferenceRuntimeContractError(
f"physical group {physical_group} Reference contract changed within one Queue"
)
self.observed_group_contracts[int(physical_group)] = group_contract
self.group_warnings[int(physical_group)] = warnings
return GroupReferenceConditioning(
physical_group=int(physical_group),
logical_chunks=route.logical_chunks,
requested_source_slot_ids=requested,
selected_assets=assets,
picture_map=picture_map,
effective_prompt=effective_prompt,
conditioning=conditioning,
warnings=warnings,
)
@@ -0,0 +1,379 @@
"""Private RR-R5 physical-group Reference persistence contract.
The stored descriptor records *effective* conditioning, not selector spelling.
No VAE, CLIP, Sampling, or saved-run operation is performed here.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import hashlib
import json
from typing import Any
from ..branch_provenance import physical_groups
from ..reference import HYBRID_PRESENTATION_VERSION, REFERENCE_PREPROCESS_VERSION
from .reference_effective_plan import PictureMap, compile_group_picture_map
from .reference_routing import REFERENCE_SLOT_IDS
REFERENCE_ROUTING_CONTRACT_VERSION = 1
SESSION_ROUTING_IDENTITY_VERSION = 1
_HEX = frozenset("0123456789abcdef")
class ReferenceStorageContractError(ValueError):
"""A routed producer or accepted-prefix contract cannot be trusted."""
def canonical_json(value: Any) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
def canonical_sha256(value: Any) -> str:
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
def routed_session_identity(base_identity_hash: str) -> str:
return canonical_sha256({
"rr_r5_session_identity_version": SESSION_ROUTING_IDENTITY_VERSION,
"base_identity_hash": str(base_identity_hash),
})
def _valid_sha(value: Any) -> bool:
return isinstance(value, str) and len(value) == 64 and all(c in _HEX for c in value)
def _anchor(value: str, present: bool) -> str | None:
if not present:
return None
if not _valid_sha(value):
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
return value
def _present(value: str) -> bool:
if str(value).lower() in {"", "none", "null"}:
return False
if not _valid_sha(value):
raise ReferenceStorageContractError("routed First/Last image fingerprint is invalid")
return True
def descriptor_from_facts(
*, physical_group: int, logical_chunks: tuple[int, ...],
selected_source_slot_ids: tuple[str, ...],
prepared_image_hashes: tuple[str, ...], picture_map: PictureMap,
first_frame_hash: str, last_frame_hash: str,
effective_prompt: str, size_mode: str,
terminal_prompt_policy: str | None,
) -> dict[str, Any]:
if len(selected_source_slot_ids) != len(prepared_image_hashes):
raise ReferenceStorageContractError("selected Reference slots and image hashes differ")
if picture_map.source_slot_ids != selected_source_slot_ids:
raise ReferenceStorageContractError("Reference Picture map differs from selected source slots")
if any(slot not in REFERENCE_SLOT_IDS for slot in selected_source_slot_ids):
raise ReferenceStorageContractError("selected Reference slot is unknown")
if tuple(sorted(selected_source_slot_ids, key=REFERENCE_SLOT_IDS.index)) != selected_source_slot_ids:
raise ReferenceStorageContractError("selected Reference slots are not in fixed R1-R9 order")
if len(set(selected_source_slot_ids)) != len(selected_source_slot_ids):
raise ReferenceStorageContractError("selected Reference slot is duplicated")
if any(not _valid_sha(value) for value in prepared_image_hashes):
raise ReferenceStorageContractError("prepared Reference image hash is invalid")
if any(item.image_sha256 != prepared_image_hashes[index]
for index, item in enumerate(picture_map.references)):
raise ReferenceStorageContractError("Reference Picture map image hash is inconsistent")
if not logical_chunks or tuple(range(logical_chunks[0], logical_chunks[-1] + 1)) != logical_chunks:
raise ReferenceStorageContractError("physical group logical chunks are not contiguous")
if int(physical_group) != logical_chunks[0]:
raise ReferenceStorageContractError("physical group number differs from its first logical chunk")
if (len(logical_chunks) > 1) != (terminal_prompt_policy is not None):
raise ReferenceStorageContractError("terminal prompt policy does not match physical group")
first_present = picture_map.first_picture_number is not None
last_present = picture_map.last_picture_number is not None
order = []
if first_present:
order.append("first_frame")
if last_present:
order.append("last_frame")
order.extend(selected_source_slot_ids)
descriptor: dict[str, Any] = {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"physical_group": int(physical_group),
"logical_chunks": list(logical_chunks),
"terminal_atomic": len(logical_chunks) > 1,
"references": [
{
"source_slot_id": slot_id,
"prepared_sha256": prepared_image_hashes[index],
"picture_number": picture_map.references[index].picture_number,
}
for index, slot_id in enumerate(selected_source_slot_ids)
],
"first_anchor": _anchor(first_frame_hash, first_present),
"last_anchor": _anchor(last_frame_hash, last_present),
"picture_order": order,
"picture_presentation_version": HYBRID_PRESENTATION_VERSION,
"effective_prompt_sha256": hashlib.sha256(str(effective_prompt).encode("utf-8")).hexdigest(),
"terminal_prompt_policy": str(terminal_prompt_policy) if len(logical_chunks) > 1 else None,
}
if selected_source_slot_ids:
descriptor["reference_size_mode"] = str(size_mode)
descriptor["reference_preprocess_version"] = REFERENCE_PREPROCESS_VERSION
return descriptor
@dataclass(frozen=True, slots=True)
class ReferenceStoragePlan:
"""Canonical JSON is immutable even if the caller's source dicts are not."""
descriptor_jsons: tuple[str, ...]
routing_config_sha256: str
requested_routes: tuple[tuple[str, ...], ...]
_group_index: dict[int, int] = field(init=False, repr=False, compare=False, hash=False)
def __post_init__(self) -> None:
index: dict[int, int] = {}
for position, payload in enumerate(self.descriptor_jsons):
descriptor = json.loads(payload)
# Preserve the former first-match lookup for duplicate group IDs.
index.setdefault(int(descriptor["physical_group"]), position)
object.__setattr__(self, "_group_index", index)
def _record_at(self, position: int) -> dict[str, Any]:
payload = self.descriptor_jsons[position]
return {
"descriptor": json.loads(payload),
"sha256": hashlib.sha256(payload.encode("utf-8")).hexdigest(),
}
@property
def group_contracts(self) -> list[dict[str, Any]]:
return [self._record_at(position) for position in range(len(self.descriptor_jsons))]
def group_for(self, physical_group: int) -> dict[str, Any]:
position = self._group_index.get(int(physical_group))
if position is None:
raise ReferenceStorageContractError(
f"physical group {physical_group} has no frozen Reference contract"
)
return self._record_at(position)
def audit(self) -> dict[str, Any]:
return {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"routing_config_sha256": self.routing_config_sha256,
"requested_routes": [list(route) for route in self.requested_routes],
}
def build_reference_storage_plan(
*, runtime: Any, prompts: list[str], first_frame_hash: str,
last_frame_hash: str, terminal_prompt: str | None = None,
terminal_prompt_policy: str | None = None,
through_chunk: int | None = None,
) -> ReferenceStoragePlan:
"""Prehash only connected R slots selected by the requested group prefix."""
from .reference_runtime import ReferenceRoutingRuntime, resolve_reference_group_prompt
if not isinstance(runtime, ReferenceRoutingRuntime):
raise ReferenceStorageContractError("routed storage requires ReferenceRoutingRuntime")
schedule = runtime.schedule
total = int(schedule.total_chunks or 0)
if total < 1 or len(prompts) != total:
raise ReferenceStorageContractError("Reference schedule and prompt count differ")
runtime.check_physical_contract(
chunks=total, terminal_merge_enabled=bool(schedule.terminal_merge_enabled),
)
limit = total if through_chunk is None else int(through_chunk)
if limit < 1 or limit > total:
raise ReferenceStorageContractError("Reference plan prefix is outside configured chunks")
routes = [route for route in schedule.physical_routes if route.logical_chunks[-1] <= limit]
if not routes or routes[-1].logical_chunks[-1] != limit:
raise ReferenceStorageContractError("Reference plan prefix ends inside a physical group")
connected = set(runtime.inputs.connected_source_slot_ids)
selected = {slot for route in routes for slot in (route.reference_slot_ids or ()) if slot in connected}
image_hashes: dict[str, str] = {}
for slot in REFERENCE_SLOT_IDS:
if slot not in selected:
continue
assets = runtime.inputs.prepare_selected((slot,))
if assets is None or assets.source_slot_ids != (slot,) or len(assets.image_hashes) != 1:
raise ReferenceStorageContractError(f"selected {slot} cannot be prepared")
image_hashes[slot] = assets.image_hashes[0]
del assets
descriptor_jsons: list[str] = []
requested_routes: list[tuple[str, ...]] = []
pending_group_warnings: dict[int, tuple[str, ...]] = {}
for route in routes:
requested = route.reference_slot_ids
if requested is None:
raise ReferenceStorageContractError("conflicting terminal Reference route")
requested_routes.append(requested)
source_slots = tuple(slot for slot in requested if slot in connected)
hashes = tuple(image_hashes[slot] for slot in source_slots)
final = route.logical_chunks[-1] == total
has_first = _present(first_frame_hash)
has_last = _present(last_frame_hash) and (bool(source_slots) or final)
picture_map = compile_group_picture_map(
source_slot_ids=source_slots,
reference_image_hashes=hashes,
has_first_image=has_first,
has_last_image=has_last,
)
prompt = (
terminal_prompt if len(route.logical_chunks) > 1
else prompts[route.logical_chunks[0] - 1]
)
if prompt is None:
raise ReferenceStorageContractError("terminal physical prompt is unavailable")
effective_prompt, warnings = resolve_reference_group_prompt(
physical_group=int(route.physical_group), prompt=prompt,
requested_source_slot_ids=requested,
connected_source_slot_ids=runtime.inputs.connected_source_slot_ids,
picture_map=picture_map,
)
pending_group_warnings[int(route.physical_group)] = warnings
descriptor = descriptor_from_facts(
physical_group=route.physical_group,
logical_chunks=route.logical_chunks,
selected_source_slot_ids=source_slots,
prepared_image_hashes=hashes,
picture_map=picture_map,
first_frame_hash=first_frame_hash,
last_frame_hash=last_frame_hash,
effective_prompt=effective_prompt,
size_mode=runtime.inputs.size_mode,
terminal_prompt_policy=terminal_prompt_policy if len(route.logical_chunks) > 1 else None,
)
descriptor_jsons.append(canonical_json(descriptor))
routing_config_sha256 = canonical_sha256({
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"mode": schedule.mode,
"logical_routes": [
[route.logical_chunk, list(route.reference_slot_ids)]
for route in schedule.logical_routes
],
})
plan = ReferenceStoragePlan(
tuple(descriptor_jsons), routing_config_sha256, tuple(requested_routes)
)
runtime.group_warnings.update({
group: warnings for group, warnings in pending_group_warnings.items()
if group not in runtime.observed_group_contracts
})
return plan
def validate_reference_group_contracts(
records: Any, *, chunk_count: int, terminal_merge_enabled: bool,
) -> list[dict[str, Any]]:
"""Validate the complete routed stored plan, not just its declared hashes."""
expected = physical_groups(chunks=chunk_count, terminal_merge_enabled=terminal_merge_enabled)
if not isinstance(records, list) or len(records) != len(expected):
raise ReferenceStorageContractError("stored Reference group count does not match physical topology")
clean: list[dict[str, Any]] = []
for record, group in zip(records, expected):
if not isinstance(record, dict) or not isinstance(record.get("descriptor"), dict):
raise ReferenceStorageContractError("stored Reference group descriptor is missing")
descriptor = record["descriptor"]
if descriptor.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
raise ReferenceStorageContractError("stored Reference group version is unsupported")
if (descriptor.get("physical_group") != group.physical_group
or descriptor.get("logical_chunks") != list(range(group.start, group.end + 1))
or descriptor.get("terminal_atomic") is not (group.start != group.end)):
raise ReferenceStorageContractError("stored Reference group boundary is invalid")
if not _valid_sha(record.get("sha256")) or canonical_sha256(descriptor) != record["sha256"]:
raise ReferenceStorageContractError("stored Reference group SHA-256 is invalid")
refs = descriptor.get("references")
if not isinstance(refs, list):
raise ReferenceStorageContractError("stored Reference group source list is invalid")
slots = [ref.get("source_slot_id") for ref in refs if isinstance(ref, dict)]
if (len(slots) != len(refs) or any(slot not in REFERENCE_SLOT_IDS for slot in slots)
or slots != sorted(set(slots), key=REFERENCE_SLOT_IDS.index)
or any(not _valid_sha(ref.get("prepared_sha256")) for ref in refs)):
raise ReferenceStorageContractError("stored Reference group source identity is invalid")
expected_order = (["first_frame"] if descriptor.get("first_anchor") else [])
expected_order += (["last_frame"] if descriptor.get("last_anchor") else [])
expected_order += slots
if descriptor.get("picture_order") != expected_order:
raise ReferenceStorageContractError("stored Reference Picture order is invalid")
if descriptor.get("picture_presentation_version") != HYBRID_PRESENTATION_VERSION:
raise ReferenceStorageContractError("stored Reference Picture presentation version is invalid")
offset = int(descriptor.get("first_anchor") is not None) + int(
descriptor.get("last_anchor") is not None
)
if any(ref.get("picture_number") != offset + index + 1
for index, ref in enumerate(refs)):
raise ReferenceStorageContractError("stored Reference Picture numbers are invalid")
if (group.start != group.end) != (descriptor.get("terminal_prompt_policy") is not None):
raise ReferenceStorageContractError("stored terminal prompt policy is invalid")
if any(not _valid_sha(descriptor.get(name)) for name in ("effective_prompt_sha256",)):
raise ReferenceStorageContractError("stored Reference effective Prompt hash is invalid")
if any(descriptor.get(name) is not None and not _valid_sha(descriptor[name])
for name in ("first_anchor", "last_anchor")):
raise ReferenceStorageContractError("stored Reference anchor hash is invalid")
if refs and (descriptor.get("reference_preprocess_version") != REFERENCE_PREPROCESS_VERSION
or not isinstance(descriptor.get("reference_size_mode"), str)):
raise ReferenceStorageContractError("stored Reference preprocessing contract is invalid")
if not refs and ("reference_preprocess_version" in descriptor or "reference_size_mode" in descriptor):
raise ReferenceStorageContractError("empty Reference group has preprocessing metadata")
clean.append(record)
return clean
def session_routing_settings(plan: ReferenceStoragePlan, accepted_chunks: int) -> dict[str, Any]:
accepted: list[dict[str, Any]] = []
for record in plan.group_contracts:
descriptor = record["descriptor"]
end = int(descriptor["logical_chunks"][-1])
if end > accepted_chunks:
break
accepted.append({
"physical_group": int(descriptor["physical_group"]),
"logical_chunks": list(descriptor["logical_chunks"]),
"sha256": record["sha256"],
})
if not accepted or accepted[-1]["logical_chunks"][-1] != accepted_chunks:
raise ReferenceStorageContractError("accepted Session ends inside a physical group")
return {
"version": REFERENCE_ROUTING_CONTRACT_VERSION,
"accepted_groups": accepted,
"routing_config_sha256": plan.routing_config_sha256,
}
def compatible_session_prefix(
evidence: Any, plan: ReferenceStoragePlan, *, saved_chunks: int,
) -> int:
"""Return a complete-group accepted prefix; malformed evidence is rejected."""
if not isinstance(evidence, dict) or evidence.get("version") != REFERENCE_ROUTING_CONTRACT_VERSION:
raise ReferenceStorageContractError("saved Session has no RR-R5 group evidence")
groups = evidence.get("accepted_groups")
if not isinstance(groups, list) or not groups:
raise ReferenceStorageContractError("saved Session group evidence is missing")
last_end = 0
for position, item in enumerate(groups):
if not isinstance(item, dict) or not _valid_sha(item.get("sha256")):
raise ReferenceStorageContractError("saved Session group hash is invalid")
chunks = item.get("logical_chunks")
if (not isinstance(chunks, list) or not chunks or chunks[0] != last_end + 1
or chunks != list(range(chunks[0], chunks[-1] + 1))
or item.get("physical_group") != chunks[0]):
raise ReferenceStorageContractError("saved Session group boundaries are invalid")
last_end = chunks[-1]
if last_end != saved_chunks:
raise ReferenceStorageContractError("saved Session group evidence does not cover its chunks")
preserved = 0
current = plan.group_contracts
for position, item in enumerate(groups):
if position >= len(current):
break
now = current[position]
if (item["physical_group"] != now["descriptor"]["physical_group"]
or item["logical_chunks"] != now["descriptor"]["logical_chunks"]
or item["sha256"] != now["sha256"]):
break
preserved = item["logical_chunks"][-1]
return preserved
+281
View File
@@ -0,0 +1,281 @@
"""Persistent MiniMax H3 video-reference conditioning for the V3.4 sampler."""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
import math
from typing import Any
import torch
from ..temporal import align_frame_count_up
REFERENCE_VIDEO_CONTRACT_VERSION = 1
REFERENCE_VIDEO_PREPROCESS_VERSION = 3
REFERENCE_VIDEO_SIZE_EFFICIENT = "Efficient - 0.4 MP"
REFERENCE_VIDEO_SIZE_BALANCED = "Balanced - 0.6 MP"
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT = "Match Output"
REFERENCE_VIDEO_SIZE_OPTIONS = (
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_BALANCED,
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
)
_CANVAS_MULTIPLE = 32
_EFFICIENT_AREA = 400_000
_BALANCED_AREA = 600_000
_SOURCE_FPS = 24
class ReferenceVideoError(ValueError):
pass
def _canonical_hash(value: Any) -> str:
payload = json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def _tensor_hash(value: torch.Tensor) -> str:
tensor = value.detach().to(device="cpu").contiguous()
digest = hashlib.sha256()
digest.update(str(tuple(int(v) for v in tensor.shape)).encode("ascii"))
digest.update(str(tensor.dtype).encode("ascii"))
digest.update(tensor.numpy().tobytes(order="C"))
return digest.hexdigest()
def _round_canvas(value: float) -> int:
return max(
_CANVAS_MULTIPLE,
int(round(float(value) / _CANVAS_MULTIPLE)) * _CANVAS_MULTIPLE,
)
def _resolved_size(
source_width: int,
source_height: int,
*,
output_width: int,
output_height: int,
size_mode: str,
) -> tuple[int, int]:
if size_mode == REFERENCE_VIDEO_SIZE_MATCH_OUTPUT:
target_area = int(output_width) * int(output_height)
elif size_mode == REFERENCE_VIDEO_SIZE_BALANCED:
target_area = _BALANCED_AREA
else:
target_area = _EFFICIENT_AREA
source_area = int(source_width) * int(source_height)
scale = min(1.0, math.sqrt(float(target_area) / float(source_area)))
target_width = _round_canvas(int(source_width) * scale)
target_height = _round_canvas(int(source_height) * scale)
if source_width >= _CANVAS_MULTIPLE:
target_width = min(
target_width,
max(_CANVAS_MULTIPLE, (int(source_width) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
)
if source_height >= _CANVAS_MULTIPLE:
target_height = min(
target_height,
max(_CANVAS_MULTIPLE, (int(source_height) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
)
return target_width, target_height
@dataclass(frozen=True)
class ReferenceVideoSource:
frames: torch.Tensor
source_shape: tuple[int, ...]
source_dtype: str
source_sha256: str
size_mode: str
target_width: int
target_height: int
frame_count: int
combined_hash: str
@property
def contract(self) -> dict[str, Any]:
return {
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
"source_shape": list(self.source_shape),
"source_dtype": self.source_dtype,
"source_sha256": self.source_sha256,
"source_fps": _SOURCE_FPS,
"size_mode": self.size_mode,
"target_width": self.target_width,
"target_height": self.target_height,
"frame_count": self.frame_count,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
"combined_hash": self.combined_hash,
}
@dataclass(frozen=True)
class ReferenceVideoAssets:
source: ReferenceVideoSource
item: dict[str, Any]
block: dict[str, Any]
def _resolve_reference_frame_count(source_frames: int, target_frames: int) -> int:
source_count = int(source_frames)
target_count = int(target_frames)
if source_count < 5 or target_count < 5:
raise ReferenceVideoError("Video Reference requires at least 5 frames")
target_cap = align_frame_count_up(target_count)
available_count = min(source_count, target_cap)
return min(align_frame_count_up(available_count), target_cap)
def _fit_reference_frames(frames: torch.Tensor, frame_count: int) -> torch.Tensor:
requested_count = int(frame_count)
current_count = int(frames.shape[0])
if current_count >= requested_count:
return frames[:requested_count].contiguous()
if current_count < 1:
raise ReferenceVideoError("Video Reference contains no frames")
padding = frames[-1:].expand(requested_count - current_count, -1, -1, -1)
return torch.cat((frames, padding), dim=0).contiguous()
def prepare_reference_video_source(
reference_video_1: Any,
*,
target_frames: int,
output_width: int,
output_height: int,
size_mode: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
) -> ReferenceVideoSource | None:
if reference_video_1 is None:
return None
if not torch.is_tensor(reference_video_1) or reference_video_1.ndim != 4:
raise ReferenceVideoError(
"Video Reference must be a ComfyUI IMAGE batch with shape [T, H, W, C]"
)
if int(reference_video_1.shape[-1]) < 3:
raise ReferenceVideoError("Video Reference must contain RGB channels")
frames = reference_video_1[:, :, :, :3].detach().to(
device="cpu", dtype=torch.float32
).contiguous()
if not bool(torch.isfinite(frames).all().item()):
raise ReferenceVideoError("Video Reference contains NaN or infinity")
source_shape = tuple(int(value) for value in frames.shape)
source_sha256 = _tensor_hash(frames)
frame_count = _resolve_reference_frame_count(
int(frames.shape[0]),
int(target_frames),
)
normalized_size_mode = (
str(size_mode)
if str(size_mode) in REFERENCE_VIDEO_SIZE_OPTIONS
else REFERENCE_VIDEO_SIZE_EFFICIENT
)
target_width, target_height = _resolved_size(
int(frames.shape[2]),
int(frames.shape[1]),
output_width=int(output_width),
output_height=int(output_height),
size_mode=normalized_size_mode,
)
contract_base = {
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
"source_shape": list(source_shape),
"source_dtype": str(frames.dtype),
"source_sha256": source_sha256,
"source_fps": _SOURCE_FPS,
"size_mode": normalized_size_mode,
"target_width": target_width,
"target_height": target_height,
"frame_count": frame_count,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
}
return ReferenceVideoSource(
frames=frames,
source_shape=source_shape,
source_dtype=str(frames.dtype),
source_sha256=source_sha256,
size_mode=normalized_size_mode,
target_width=target_width,
target_height=target_height,
frame_count=frame_count,
combined_hash=_canonical_hash(contract_base),
)
def encode_reference_video(
video_vae: Any,
source: ReferenceVideoSource,
) -> ReferenceVideoAssets:
available_count = min(int(source.frames.shape[0]), int(source.frame_count))
frames = source.frames[:available_count]
if (
int(frames.shape[2]) != source.target_width
or int(frames.shape[1]) != source.target_height
):
try:
import comfy.utils
except Exception as exc:
raise ReferenceVideoError(
"ComfyUI Core image resize support is unavailable"
) from exc
frames = comfy.utils.common_upscale(
frames.movedim(-1, 1),
source.target_width,
source.target_height,
"lanczos",
"disabled",
).movedim(1, -1).contiguous()
frames = _fit_reference_frames(frames, source.frame_count)
try:
latent = video_vae.encode(frames).detach().to("cpu").contiguous()
except Exception as exc:
raise ReferenceVideoError("Video Reference VAE Encode failed") from exc
qwen_frames = frames[::_SOURCE_FPS // 2].contiguous()
item = {
"type": "video",
"data": qwen_frames,
"timestamps": [index / 2.0 for index in range(int(qwen_frames.shape[0]))],
}
block = {
"kind": "video",
"latent_t": int(latent.shape[2]),
"latent_h": source.target_height // 16,
"latent_w": source.target_width // 16,
"ref_audio_t": 0,
"latent": latent,
"audio_latent": None,
}
return ReferenceVideoAssets(source=source, item=item, block=block)
def combine_reference_video_identity(
visual_identity_hash: str,
source: ReferenceVideoSource | None,
) -> str:
if source is None:
return str(visual_identity_hash)
return _canonical_hash(
{
"reference_video_identity_version": 1,
"visual_identity_hash": str(visual_identity_hash),
"reference_video_hash": source.combined_hash,
}
)
def validate_reference_video_prompts(
prompts: list[str], source: ReferenceVideoSource | None
) -> str:
if source is None or any("<Video 1>" in str(prompt) for prompt in prompts):
return ""
return (
"H3C-P103 Warning: Video Reference is connected but the prompt contains no "
"<Video 1> tag; video still conditions generation, but an explicit tag is "
"recommended."
)
+699
View File
@@ -0,0 +1,699 @@
"""Runtime-only First Pass conditioning for Continuum-aware refinement.
The context deliberately remains outside the persisted Assembly Plan contract. It
owns CPU copies of the First Pass conditioning tensors and records them in physical
sampling order. Second Pass callers may then derive a target-canvas conditioning
without mutating the captured source context.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
import copy
from types import MappingProxyType
from typing import Any
import torch
from ..constants import MARK_VIDEO_CONTEXT
from ..guide_timeline import MARK_STILL_IMAGE_GUIDE
MAGIC = "H3_CONTINUUM_REFINE_CONTEXT"
VERSION = 1
OUTPUT_SCOPE_KEY = "output_scope_v1"
class RefineContextError(ValueError):
"""Raised when the runtime-only refine context is structurally invalid."""
class RefineConditioningAdaptationError(RefineContextError):
"""Raised when a valid captured conditioning cannot be adapted safely."""
def _cpu_tensor(value: torch.Tensor) -> torch.Tensor:
return value.detach().to(device="cpu").clone().contiguous()
def _freeze(value: Any) -> Any:
if torch.is_tensor(value):
return _cpu_tensor(value)
if isinstance(value, Mapping):
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
if isinstance(value, (list, tuple)):
return tuple(_freeze(item) for item in value)
if isinstance(value, (set, frozenset)):
return frozenset(_freeze(item) for item in value)
return value
def freeze_conditioning(conditioning: Any) -> tuple[tuple[torch.Tensor, Mapping], ...]:
"""Return a CPU-owned conditioning clone with immutable containers.
Tensors are detached and cloned, so later mutation or device movement of the
original First Pass conditioning cannot affect the refine context. Container
immutability makes the result safe to share until target adaptation performs
copy-on-write replacement of visual fields.
"""
if not isinstance(conditioning, Sequence) or isinstance(
conditioning, (str, bytes, bytearray)
):
raise RefineContextError("conditioning must be a non-empty sequence")
output: list[tuple[torch.Tensor, Mapping]] = []
for index, entry in enumerate(conditioning):
if not isinstance(entry, Sequence) or isinstance(
entry, (str, bytes, bytearray)
) or len(entry) != 2:
raise RefineContextError(
f"conditioning entry {index} is not [tensor, metadata]"
)
tensor, metadata = entry
if not torch.is_tensor(tensor):
raise RefineContextError(
f"conditioning entry {index} text value is not a tensor"
)
if not isinstance(metadata, Mapping):
raise RefineContextError(
f"conditioning entry {index} metadata is not a mapping"
)
output.append((_cpu_tensor(tensor), _freeze(metadata)))
if not output:
raise RefineContextError("conditioning is empty")
return tuple(output)
def _optional_image(value: Any, name: str) -> torch.Tensor | None:
if value is None:
return None
if not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3:
raise RefineContextError(
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
)
return _cpu_tensor(value)
def _validate_optional_image(value: Any, name: str) -> None:
if value is not None and (
not torch.is_tensor(value) or value.ndim != 4 or int(value.shape[-1]) < 3
):
raise RefineContextError(
f"{name} must be an IMAGE tensor [B,H,W,C] with at least three channels"
)
def _guide_image_map(value: Any) -> Mapping[int, torch.Tensor]:
if value is None:
return MappingProxyType({})
if not isinstance(value, Mapping):
raise RefineContextError("guide_images must be a frame-to-IMAGE mapping")
output: dict[int, torch.Tensor] = {}
for frame_index, image in value.items():
if type(frame_index) is not int or frame_index < 0:
raise RefineContextError("guide_images frame indices must be non-negative integers")
prepared = _optional_image(image, f"guide_images[{frame_index}]")
assert prepared is not None
output[frame_index] = prepared
return MappingProxyType(output)
def _validate_guide_images(value: Any, *, physical_frames: int) -> None:
if value is None:
return
if not isinstance(value, Mapping):
raise RefineContextError("refine group guide_images is invalid")
for frame_index, image in value.items():
if type(frame_index) is not int or not 0 <= frame_index < physical_frames:
raise RefineContextError("refine group guide image frame is outside the physical group")
_validate_optional_image(image, f"guide_images[{frame_index}]")
def make_refine_group(
*,
conditioning: Any,
group_id: int,
logical_chunks: Sequence[int],
physical_frames: int,
prompt_policy: str,
physical_prompt: str,
source_video_shape: Sequence[int],
physical_clip_index: int,
context_frames: int,
first_image: torch.Tensor | None = None,
last_image: torch.Tensor | None = None,
guide_images: Mapping[int, torch.Tensor] | None = None,
reference_routing_group_contract: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Capture one physical First Pass sampling group's effective conditioning."""
group_id = int(group_id)
physical_frames = int(physical_frames)
physical_clip_index = int(physical_clip_index)
context_frames = int(context_frames)
chunks = tuple(int(value) for value in logical_chunks)
shape = tuple(int(value) for value in source_video_shape)
if group_id < 0:
raise RefineContextError("refine group id must be non-negative")
if not chunks or any(value <= 0 for value in chunks):
raise RefineContextError("refine group logical chunks are invalid")
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
raise RefineContextError("refine group logical chunks must be ordered and unique")
if physical_frames <= 0:
raise RefineContextError("refine group physical frame count is invalid")
if not str(prompt_policy):
raise RefineContextError("refine group prompt policy is empty")
if len(shape) != 5 or any(value <= 0 for value in shape):
raise RefineContextError("refine group source video shape must be [B,C,T,H,W]")
if physical_clip_index <= 0:
raise RefineContextError("refine group physical clip index is invalid")
if context_frames < 0:
raise RefineContextError("refine group context frame count is invalid")
output = {
"group_id": group_id,
"logical_chunks": chunks,
"physical_frames": physical_frames,
"prompt_policy": str(prompt_policy),
"physical_prompt": str(physical_prompt),
"source_video_shape": shape,
"physical_clip_index": physical_clip_index,
"context_frames": context_frames,
"conditioning": freeze_conditioning(conditioning),
"first_image": _optional_image(first_image, "first_image"),
"last_image": _optional_image(last_image, "last_image"),
}
frozen_guides = _guide_image_map(guide_images)
if frozen_guides:
output["guide_images"] = frozen_guides
if reference_routing_group_contract is not None:
from .reference_storage_contract import canonical_sha256
descriptor = reference_routing_group_contract.get("descriptor")
digest = reference_routing_group_contract.get("sha256")
if not isinstance(descriptor, dict) or canonical_sha256(descriptor) != digest:
raise RefineContextError("refine group Reference contract SHA-256 is invalid")
if descriptor.get("logical_chunks") != list(chunks):
raise RefineContextError("refine group Reference contract has different logical chunks")
output["reference_routing_v1"] = copy.deepcopy(dict(reference_routing_group_contract))
return output
def make_refine_context(
groups: Sequence[Mapping[str, Any]],
*,
source_width: int,
source_height: int,
conditioning_mode: str,
complete: bool = True,
notes: Sequence[str] = (),
) -> dict[str, Any]:
"""Build and validate a physical-order runtime refine context."""
context = {
"magic": MAGIC,
"version": VERSION,
"source_width": int(source_width),
"source_height": int(source_height),
"conditioning_mode": str(conditioning_mode),
"complete": bool(complete),
"groups": tuple(dict(group) for group in groups),
"notes": tuple(str(note) for note in notes),
}
return validate_refine_context(context)
def _validate_frozen_conditioning(value: Any, *, group_id: int) -> None:
if not isinstance(value, tuple) or not value:
raise RefineContextError(f"refine group {group_id} conditioning is invalid")
for index, entry in enumerate(value):
if not isinstance(entry, tuple) or len(entry) != 2:
raise RefineContextError(
f"refine group {group_id} conditioning entry {index} is invalid"
)
tensor, metadata = entry
if not torch.is_tensor(tensor) or tensor.device.type != "cpu":
raise RefineContextError(
f"refine group {group_id} conditioning tensor {index} is not CPU-owned"
)
if not isinstance(metadata, Mapping):
raise RefineContextError(
f"refine group {group_id} conditioning metadata {index} is invalid"
)
def _group_field_tuple(group: Mapping[str, Any], name: str) -> tuple[int, ...]:
value = group.get(name)
if not isinstance(value, (tuple, list)):
raise RefineContextError(f"refine group {group.get('group_id')} {name} is invalid")
try:
return tuple(int(item) for item in value)
except (TypeError, ValueError) as exc:
raise RefineContextError(
f"refine group {group.get('group_id')} {name} is invalid"
) from exc
def project_refine_context(context: dict[str, Any], assembly_plan: Mapping[str, Any]):
"""Bind a V3.9 runtime-only view to complete physical output groups.
Sequence completion and saved history are unchanged. A partial capture may
serve a smaller decode view only after the ordinary complete-group checks
pass, including the frozen Reference route. Missing or mismatched partial
evidence retains the legacy diagnostic fallback, never guessed conditions.
"""
validate_refine_context(context)
contract = assembly_plan.get("second_pass_contract")
planned = contract.get("physical_groups") if isinstance(contract, Mapping) else None
if not isinstance(planned, list) or not planned:
return context
source = context["groups"]
keys = [tuple(group["logical_chunks"]) for group in source]
targets = [tuple(group.get("logical_chunks", ())) for group in planned]
if context["complete"] and keys == targets:
return validate_refine_context(context, assembly_plan=assembly_plan)
if len(set(keys)) != len(keys) or len(set(targets)) != len(targets):
return context
lookup = dict(zip(keys, source))
if any(key not in lookup for key in targets):
return context
selected = [lookup[key] for key in targets]
view = dict(context)
view["groups"] = tuple(
{**group, "group_id": index} for index, group in enumerate(selected)
)
view[OUTPUT_SCOPE_KEY] = MappingProxyType({
"version": 1,
"source_group_ids": tuple(group["group_id"] for group in selected),
})
try:
validate_refine_context(view, assembly_plan=assembly_plan)
except RefineContextError as exc:
# These groups were previously optional/incomplete. Do not add a new
# execution stop for unavailable output-scope evidence.
fallback = dict(context)
fallback["notes"] = (*context["notes"], f"Output scope unavailable: {exc}")
return fallback
return view
def validate_refine_context(
context: Any, assembly_plan: Mapping[str, Any] | None = None
) -> dict[str, Any]:
"""Validate a runtime context and, when supplied, its Assembly Plan pairing."""
if not isinstance(context, dict) or context.get("magic") != MAGIC:
raise RefineContextError("invalid H3 Continuum refine context")
if type(context.get("version")) is not int or context["version"] != VERSION:
raise RefineContextError(
f"unsupported refine context version {context.get('version')!r}"
)
if int(context.get("source_width", 0)) <= 0 or int(
context.get("source_height", 0)
) <= 0:
raise RefineContextError("refine context source dimensions are invalid")
if not str(context.get("conditioning_mode", "")):
raise RefineContextError("refine context conditioning mode is empty")
if type(context.get("complete")) is not bool:
raise RefineContextError("refine context complete flag is invalid")
groups = context.get("groups")
if not isinstance(groups, tuple):
raise RefineContextError("refine context groups must be an immutable tuple")
if context["complete"] and not groups:
raise RefineContextError("complete refine context contains no physical groups")
notes = context.get("notes")
if not isinstance(notes, tuple) or any(not isinstance(note, str) for note in notes):
raise RefineContextError("refine context notes must be an immutable string tuple")
scope = context.get(OUTPUT_SCOPE_KEY)
if scope is not None:
ids = scope.get("source_group_ids") if isinstance(scope, Mapping) else None
if (not isinstance(scope, Mapping) or scope.get("version") != 1
or not isinstance(ids, tuple) or len(ids) != len(groups)
or any(type(value) is not int or value < 0 for value in ids)
or tuple(sorted(set(ids))) != ids or not groups):
raise RefineContextError("refine context output scope is invalid")
previous_clip_index = 0
for expected_id, group in enumerate(groups):
if not isinstance(group, Mapping):
raise RefineContextError(f"refine group {expected_id} is not a mapping")
if type(group.get("group_id")) is not int or group["group_id"] != expected_id:
raise RefineContextError("refine context physical group order is not contiguous")
logical_chunks = _group_field_tuple(group, "logical_chunks")
if not logical_chunks or any(value <= 0 for value in logical_chunks):
raise RefineContextError(f"refine group {expected_id} logical chunks are invalid")
if tuple(sorted(logical_chunks)) != logical_chunks or len(
set(logical_chunks)
) != len(logical_chunks):
raise RefineContextError(
f"refine group {expected_id} logical chunks are not ordered and unique"
)
if int(group.get("physical_frames", 0)) <= 0:
raise RefineContextError(f"refine group {expected_id} physical frames are invalid")
if not str(group.get("prompt_policy", "")):
raise RefineContextError(f"refine group {expected_id} prompt policy is empty")
if "physical_prompt" not in group:
raise RefineContextError(f"refine group {expected_id} physical prompt is absent")
shape = _group_field_tuple(group, "source_video_shape")
if len(shape) != 5 or any(value <= 0 for value in shape):
raise RefineContextError(f"refine group {expected_id} source shape is invalid")
clip_index = int(group.get("physical_clip_index", 0))
if clip_index <= previous_clip_index:
raise RefineContextError(
"refine context physical clip indices are not strictly increasing"
)
previous_clip_index = clip_index
if int(group.get("context_frames", -1)) < 0:
raise RefineContextError(f"refine group {expected_id} context frames are invalid")
_validate_frozen_conditioning(group.get("conditioning"), group_id=expected_id)
_validate_optional_image(group.get("first_image"), "first_image")
_validate_optional_image(group.get("last_image"), "last_image")
_validate_guide_images(
group.get("guide_images"),
physical_frames=int(group["physical_frames"]),
)
route_contract = group.get("reference_routing_v1")
if route_contract is not None:
from .reference_storage_contract import canonical_sha256
if not isinstance(route_contract, Mapping):
raise RefineContextError("refine group Reference contract is invalid")
descriptor = route_contract.get("descriptor")
if (not isinstance(descriptor, dict)
or canonical_sha256(descriptor) != route_contract.get("sha256")
or descriptor.get("logical_chunks") != list(logical_chunks)):
raise RefineContextError("refine group Reference contract differs from its content")
if assembly_plan is not None:
if not isinstance(assembly_plan, Mapping):
raise RefineContextError("assembly plan paired with refine context is invalid")
if int(assembly_plan.get("width", 0)) != int(context["source_width"]) or int(
assembly_plan.get("height", 0)
) != int(context["source_height"]):
raise RefineContextError("refine context source dimensions differ from assembly plan")
contract = assembly_plan.get("second_pass_contract")
plan_groups = contract.get("physical_groups") if isinstance(contract, Mapping) else None
if not isinstance(plan_groups, Sequence):
raise RefineContextError("assembly plan has no Second Pass physical-group contract")
# An incomplete context is diagnostic-only (for example, Run Storage may
# have reused an earlier group that was never sampled in this execution).
# Its local group ids describe capture order, not Assembly Plan position;
# validating those partial entries by index would incorrectly turn the
# intended prompt-only fallback into a broken typed-contract hard error.
if not context["complete"] and scope is None:
return context
if len(plan_groups) != len(groups):
raise RefineContextError(
"complete refine context physical-group count differs from assembly plan"
)
for group in groups:
group_id = int(group["group_id"])
if group_id >= len(plan_groups) or not isinstance(plan_groups[group_id], Mapping):
raise RefineContextError(
f"refine group {group_id} has no matching assembly-plan group"
)
planned = plan_groups[group_id]
route_view = assembly_plan.get("reference_routing_v1")
if route_view is not None:
view_groups = route_view.get("groups") if isinstance(route_view, Mapping) else None
expected_route = (
view_groups[group_id] if isinstance(view_groups, list)
and group_id < len(view_groups) else None
)
if scope is not None:
# Inspector retains the full run, including pending groups.
# A Chunk-2 view must not compare against Inspector group 1.
matches = [item for item in view_groups
if isinstance(item, Mapping)
and isinstance(item.get("descriptor"), Mapping)
and tuple(item["descriptor"].get("logical_chunks", ()))
== tuple(group["logical_chunks"])] if isinstance(view_groups, list) else []
expected_route = matches[0] if len(matches) == 1 else None
captured_route = group.get("reference_routing_v1")
if (not isinstance(expected_route, Mapping)
or not isinstance(captured_route, Mapping)
or captured_route.get("sha256") != expected_route.get("contract_sha256")
or captured_route.get("descriptor") != expected_route.get("descriptor")):
raise RefineContextError(
f"refine group {group_id} Reference route differs from First Pass Plan"
)
comparisons = (
("logical_chunks", tuple(int(v) for v in group["logical_chunks"]),
tuple(int(v) for v in planned.get("logical_chunks", ()))),
("physical_frames", int(group["physical_frames"]),
int(planned.get("physical_frames", 0))),
("prompt_policy", str(group["prompt_policy"]),
str(planned.get("prompt_policy", ""))),
("physical_prompt", str(group["physical_prompt"]),
str(planned.get("physical_prompt", ""))),
("context_frames", int(group["context_frames"]),
int(planned.get("trim_prefix_frames", -1))),
)
for name, actual, expected in comparisons:
if actual != expected:
raise RefineContextError(
f"refine group {group_id} {name} differs from assembly plan"
)
expected_shape = (
int(planned.get("source_batch", 0)),
int(planned.get("latent_channels", 0)),
int(planned.get("source_latent_t", 0)),
int(planned.get("source_latent_h", 0)),
int(planned.get("source_latent_w", 0)),
)
if tuple(group["source_video_shape"]) != expected_shape:
raise RefineContextError(
f"refine group {group_id} source shape differs from assembly plan"
)
return context
def _common_upscale(
samples: torch.Tensor,
width: int,
height: int,
method: str,
crop: str,
) -> torch.Tensor:
try:
import comfy.utils
except Exception as exc: # pragma: no cover - ComfyUI runtime only
raise RefineConditioningAdaptationError(
f"ComfyUI Core resize support is unavailable: {exc}"
) from exc
return comfy.utils.common_upscale(samples, width, height, method, crop)
def _resize_latent(
latent: Any,
*,
target_h: int,
target_w: int,
method: str,
) -> torch.Tensor:
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
"H3 visual conditioning latent must be [B,C,T,H,W]"
)
batch, channels, temporal, _, _ = (int(value) for value in latent.shape)
flat = latent.permute(0, 2, 1, 3, 4).reshape(
batch * temporal, channels, int(latent.shape[-2]), int(latent.shape[-1])
)
try:
resized = _common_upscale(flat, target_w, target_h, method, "disabled")
except RefineConditioningAdaptationError:
raise
except Exception as exc:
raise RefineConditioningAdaptationError(
f"visual conditioning latent resize failed: {exc}"
) from exc
if not torch.is_tensor(resized) or tuple(resized.shape[-2:]) != (
target_h,
target_w,
):
raise RefineConditioningAdaptationError(
"Core resize returned an incompatible visual conditioning shape"
)
return (
resized.reshape(batch, temporal, channels, target_h, target_w)
.permute(0, 2, 1, 3, 4)
.contiguous()
)
def _encode_keyframe_image(
image: torch.Tensor,
*,
target_h: int,
target_w: int,
crop: str,
video_vae: Any,
) -> torch.Tensor:
pixels_h = target_h * 16
pixels_w = target_w * 16
samples = image[:1, ..., :3].movedim(-1, 1)
try:
resized = _common_upscale(samples, pixels_w, pixels_h, "lanczos", crop)
latent = video_vae.encode(resized.movedim(1, -1).contiguous())
except RefineConditioningAdaptationError:
raise
except Exception as exc:
raise RefineConditioningAdaptationError(
f"target keyframe VAE encode failed: {exc}"
) from exc
if not torch.is_tensor(latent) or latent.ndim != 5 or tuple(
int(value) for value in latent.shape[-2:]
) != (target_h, target_w):
raise RefineConditioningAdaptationError(
"target keyframe VAE encode returned an incompatible latent shape"
)
return latent.contiguous()
def adapt_group_conditioning(
group: Mapping[str, Any],
target_latent_h: int,
target_latent_w: int,
video_vae: Any = None,
upscale_method: str = "bilinear",
) -> tuple[list[list[Any]], dict[str, Any]]:
"""Copy-on-write target adaptation for keyframes and Continuum video context.
Ordinary MiniMax references, all audio latents, and text conditioning tensors
remain shared with the frozen source. Only keyframe video latents and
``MARK_VIDEO_CONTEXT`` reference video latents are target-sized.
"""
if not isinstance(group, Mapping):
raise RefineConditioningAdaptationError("refine group is not a mapping")
group_id = int(group.get("group_id", -1))
_validate_frozen_conditioning(group.get("conditioning"), group_id=group_id)
target_h = int(target_latent_h)
target_w = int(target_latent_w)
if target_h <= 0 or target_w <= 0:
raise RefineConditioningAdaptationError("target latent dimensions are invalid")
physical_frames = int(group.get("physical_frames", 0))
if physical_frames <= 0:
raise RefineConditioningAdaptationError("refine group physical frames are invalid")
warnings: list[str] = []
stats: dict[str, Any] = {
"group_id": group_id,
"target_latent_h": target_h,
"target_latent_w": target_w,
"keyframes_resized": 0,
"keyframes_reencoded": 0,
"keyframes_latent_fallback": 0,
"guide_keyframes_reencoded": 0,
"context_refs_resized": 0,
}
output: list[list[Any]] = []
for text_tensor, frozen_metadata in group["conditioning"]:
metadata = dict(frozen_metadata)
keyframes = frozen_metadata.get("minimax_keyframes")
if keyframes is not None:
adapted_keyframes = []
for frozen_keyframe in keyframes:
if not isinstance(frozen_keyframe, Mapping):
raise RefineConditioningAdaptationError(
f"refine group {group_id} contains an invalid keyframe"
)
keyframe = dict(frozen_keyframe)
latent = keyframe.get("latent")
audio_latent = keyframe.get("audio_latent")
if audio_latent is not None and (
not torch.is_tensor(audio_latent) or audio_latent.ndim != 4
):
raise RefineConditioningAdaptationError(
f"refine group {group_id} keyframe audio latent is invalid"
)
# Core keyframes may carry audio without a spatial video latent.
# Keep its frozen audio and timing; resize only video payloads.
if latent is None and audio_latent is not None:
adapted_keyframes.append(keyframe)
continue
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
f"refine group {group_id} keyframe latent is invalid"
)
if tuple(int(value) for value in latent.shape[-2:]) != (
target_h,
target_w,
):
frame_index = int(keyframe.get("resolved_frame_index", -1))
image = None
crop = "disabled"
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
guide_images = group.get("guide_images")
if isinstance(guide_images, Mapping):
image = guide_images.get(frame_index)
crop = "center"
elif frame_index == 0:
image = group.get("first_image")
elif frame_index == physical_frames - 1:
image = group.get("last_image")
crop = "center"
if video_vae is not None and torch.is_tensor(image):
keyframe["latent"] = _encode_keyframe_image(
image,
target_h=target_h,
target_w=target_w,
crop=crop,
video_vae=video_vae,
)
stats["keyframes_reencoded"] += 1
if bool(keyframe.get(MARK_STILL_IMAGE_GUIDE)):
stats["guide_keyframes_reencoded"] += 1
else:
keyframe["latent"] = _resize_latent(
latent,
target_h=target_h,
target_w=target_w,
method=str(upscale_method),
)
stats["keyframes_latent_fallback"] += 1
warning = (
f"group {group_id} keyframe {frame_index} used latent resize "
"because its source image and video VAE were not both available"
)
warnings.append(warning)
stats["keyframes_resized"] += 1
adapted_keyframes.append(keyframe)
metadata["minimax_keyframes"] = adapted_keyframes
refs = frozen_metadata.get("minimax_refs")
if refs is not None:
adapted_refs = []
refs_changed = False
for frozen_ref in refs:
if not isinstance(frozen_ref, Mapping):
raise RefineConditioningAdaptationError(
f"refine group {group_id} contains an invalid MiniMax reference"
)
if bool(frozen_ref.get(MARK_VIDEO_CONTEXT)):
latent = frozen_ref.get("latent")
if not torch.is_tensor(latent) or latent.ndim != 5:
raise RefineConditioningAdaptationError(
f"refine group {group_id} Continuum video context is invalid"
)
if tuple(int(value) for value in latent.shape[-2:]) != (
target_h,
target_w,
):
adapted_ref = dict(frozen_ref)
adapted_ref["latent"] = _resize_latent(
latent,
target_h=target_h,
target_w=target_w,
method=str(upscale_method),
)
adapted_ref["latent_h"] = target_h
adapted_ref["latent_w"] = target_w
adapted_refs.append(adapted_ref)
refs_changed = True
stats["context_refs_resized"] += 1
continue
adapted_refs.append(frozen_ref)
if refs_changed:
metadata["minimax_refs"] = adapted_refs
output.append([text_tensor, metadata])
stats["warnings"] = tuple(warnings)
return output, stats
+158
View File
@@ -0,0 +1,158 @@
"""V3.5 Second Pass sampling with the first-pass audio stream locked."""
from __future__ import annotations
from typing import Any, Callable
import torch
from ..state import extract_av_streams
class RefineSamplingRuntimeError(RuntimeError):
"""Raised when the ComfyUI sampling runtime cannot satisfy the refine contract."""
def _make_basic_guider(model: Any, conditioning: list):
try:
import comfy.samplers
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(f"ComfyUI sampler API unavailable: {exc}") from exc
class _BasicGuider(comfy.samplers.CFGGuider):
def set_positive(self, positive):
self.inner_set_conds({"positive": positive})
guider = _BasicGuider(model)
guider.set_positive(conditioning)
return guider
def _load_runtime_modules():
try:
import comfy.model_management
import comfy.nested_tensor
import comfy.sample
import comfy.utils
import latent_preview
except Exception as exc: # pragma: no cover
raise RefineSamplingRuntimeError(
f"ComfyUI refine sampling runtime unavailable: {exc}"
) from exc
return (
comfy.model_management,
comfy.nested_tensor,
comfy.sample,
comfy.utils,
latent_preview,
)
def _prepare_refine_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
) -> tuple[Any, Any]:
"""Build random video noise and a Core-expandable audio-lock mask."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"refine latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("refine video/audio batch sizes differ")
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError("ComfyUI returned invalid video noise geometry")
# Core prepare_noise creates CPU noise. Keep both streams on that same handoff
# device so NestedTensor packing remains valid before Core moves it to the model.
audio_noise = torch.zeros_like(audio, device="cpu")
# These masks deliberately use the smallest shapes accepted by Core reshape_mask.
# Core expands them to the full stream geometry on the sampling device.
video_mask = torch.ones(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.zeros(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def sample_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Refine video while Core carries the first-pass audio as a locked AV stream."""
if not isinstance(latent, dict) or "samples" not in latent:
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
_load_runtime_modules()
)
working = latent.copy()
latent_image = comfy_sample.fix_empty_latent_channels(
model,
working["samples"],
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
video, audio = extract_av_streams(working)
noise, noise_mask = _prepare_refine_noise_and_mask(
video,
audio,
seed=int(seed),
batch_inds=working.get("batch_index"),
prepare_noise_fn=comfy_sample.prepare_noise,
nested_builder=nested_tensor.NestedTensor,
)
guider = _make_basic_guider(model, conditioning)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = preview.prepare_callback(
model,
int(sigmas.shape[-1]) - 1,
x0_output,
)
disable_pbar = not comfy_utils.PROGRESS_BAR_ENABLED
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=disable_pbar,
seed=int(seed),
)
samples = samples.to(model_management.intermediate_device())
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
extract_av_streams(output)
return output
+214
View File
@@ -0,0 +1,214 @@
"""Versioned, UI-independent sigma contracts for Continuum refinement.
The public Second Pass SIGMAS socket remains authoritative. These helpers attach
meaning to that tensor without recreating it: External and Full retain the exact
input object, while Tail and Partial return views into the same source values.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Mapping
import torch
MAGIC = "H3_CONTINUUM_REFINE_SCHEDULE"
SCHEMA_VERSION = 1
MODE_EXTERNAL = "external"
MODE_FULL = "full"
MODE_TAIL = "tail"
MODE_PARTIAL = "partial"
MODES = (MODE_EXTERNAL, MODE_FULL, MODE_TAIL, MODE_PARTIAL)
NOISE_MODE_VIDEO_RANDOM_AUDIO_ZERO = "video_random_mask_1_audio_zero_mask_0"
AUDIO_POLICY_LOCKED_PASSTHROUGH = "locked_first_pass_bit_exact_passthrough"
class RefineScheduleError(ValueError):
"""Raised when a refinement schedule cannot safely describe its SIGMAS."""
@dataclass(frozen=True)
class RefineSchedule:
"""One validated effective SIGMAS tensor and its serializable identity."""
sigmas: torch.Tensor
contract: Mapping[str, Any]
source_sigma_hash: str
def _validate_sigmas(sigmas: Any, *, name: str) -> torch.Tensor:
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineScheduleError(f"{name} must be a one-dimensional tensor with at least two values")
detached = sigmas.detach()
if not bool(torch.isfinite(detached.float()).all().item()):
raise RefineScheduleError(f"{name} contains NaN or Inf")
values = detached.float().to(device="cpu")
if bool((values[1:] > values[:-1]).any().item()):
raise RefineScheduleError(f"{name} must be monotonically non-increasing")
return sigmas
def sigma_hash(sigmas: torch.Tensor) -> str:
"""Hash dtype, shape, and exact tensor bytes without changing the input."""
_validate_sigmas(sigmas, name="sigmas")
cpu = sigmas.detach().to(device="cpu").contiguous()
payload = cpu.view(torch.uint8).numpy().tobytes()
digest = hashlib.sha256()
digest.update(str(cpu.dtype).encode("ascii"))
digest.update(b"|")
digest.update(str(tuple(int(value) for value in cpu.shape)).encode("ascii"))
digest.update(b"|")
digest.update(payload)
return digest.hexdigest()
def _schedule_identity(contract: Mapping[str, Any]) -> str:
payload = {
key: value
for key, value in contract.items()
if key != "schedule_hash"
}
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _build_schedule(
source_sigmas: torch.Tensor,
*,
mode: str,
start_index: int,
end_index: int,
) -> RefineSchedule:
source = _validate_sigmas(source_sigmas, name="source_sigmas")
canonical_mode = str(mode).lower()
if canonical_mode not in MODES:
raise RefineScheduleError(f"unsupported refine schedule mode {mode!r}")
source_evaluations = int(source.numel()) - 1
if type(start_index) is not int or type(end_index) is not int:
raise RefineScheduleError("schedule indices must be integers")
if start_index < 0 or end_index <= start_index or end_index > source_evaluations:
raise RefineScheduleError(
f"schedule range {start_index}:{end_index} is outside 0:{source_evaluations}"
)
if start_index == 0 and end_index == source_evaluations:
effective = source
else:
effective = source[start_index : end_index + 1]
_validate_sigmas(effective, name="effective sigmas")
effective_hash = sigma_hash(effective)
source_hash = sigma_hash(source)
values = tuple(float(value) for value in effective.detach().to(device="cpu").tolist())
contract: dict[str, Any] = {
"magic": MAGIC,
"schema_version": SCHEMA_VERSION,
"mode": canonical_mode,
"sigma_values": values,
"sigma_hash": effective_hash,
"source_sigma_hash": source_hash,
"start_sigma": values[0],
"end_sigma": values[-1],
"evaluation_count": len(values) - 1,
"source_evaluation_count": source_evaluations,
"start_index": start_index,
"end_index": end_index,
"noise_mode": NOISE_MODE_VIDEO_RANDOM_AUDIO_ZERO,
"audio_policy": AUDIO_POLICY_LOCKED_PASSTHROUGH,
}
contract["schedule_hash"] = _schedule_identity(contract)
return RefineSchedule(
sigmas=effective,
contract=MappingProxyType(contract),
source_sigma_hash=source_hash,
)
def make_external_schedule(sigmas: torch.Tensor) -> RefineSchedule:
evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
return _build_schedule(
sigmas,
mode=MODE_EXTERNAL,
start_index=0,
end_index=evaluations,
)
def make_full_schedule(sigmas: torch.Tensor) -> RefineSchedule:
evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
return _build_schedule(
sigmas,
mode=MODE_FULL,
start_index=0,
end_index=evaluations,
)
def make_tail_schedule(
sigmas: torch.Tensor,
*,
evaluation_count: int,
) -> RefineSchedule:
source_evaluations = int(_validate_sigmas(sigmas, name="sigmas").numel()) - 1
if type(evaluation_count) is not int or not 1 <= evaluation_count <= source_evaluations:
raise RefineScheduleError(
f"tail evaluation_count must be between 1 and {source_evaluations}"
)
return _build_schedule(
sigmas,
mode=MODE_TAIL,
start_index=source_evaluations - evaluation_count,
end_index=source_evaluations,
)
def make_partial_schedule(
sigmas: torch.Tensor,
*,
start_index: int,
end_index: int,
) -> RefineSchedule:
return _build_schedule(
sigmas,
mode=MODE_PARTIAL,
start_index=start_index,
end_index=end_index,
)
def resolve_refine_schedule(
sigmas: torch.Tensor,
schedule: RefineSchedule | None = None,
) -> RefineSchedule:
"""Return External compatibility or validate an explicit internal schedule."""
source = _validate_sigmas(sigmas, name="sigmas")
if schedule is None:
return make_external_schedule(source)
if not isinstance(schedule, RefineSchedule):
raise RefineScheduleError("refine_schedule must be a RefineSchedule")
if sigma_hash(source) != schedule.source_sigma_hash:
raise RefineScheduleError("refine_schedule was derived from different source SIGMAS")
rebuilt = _build_schedule(
source,
mode=str(schedule.contract.get("mode", "")),
start_index=int(schedule.contract.get("start_index", -1)),
end_index=int(schedule.contract.get("end_index", -1)),
)
if rebuilt.contract["schedule_hash"] != schedule.contract.get("schedule_hash"):
raise RefineScheduleError("refine_schedule contract identity is inconsistent")
return schedule
def serializable_schedule_contract(schedule: RefineSchedule) -> dict[str, Any]:
"""Return a mutable JSON-safe copy for Assembly/diagnostic identity."""
if not isinstance(schedule, RefineSchedule):
raise RefineScheduleError("schedule must be a RefineSchedule")
return dict(schedule.contract)
+179
View File
@@ -0,0 +1,179 @@
"""Versioned physical-group selection for Experimental selective refinement."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any
MAGIC = "H3_CONTINUUM_REFINE_SCOPE"
SCHEMA_VERSION = 1
MODE_ALL = "all"
MODE_PHYSICAL_GROUP = "physical_group"
MODE_LOGICAL_CHUNK = "logical_chunk"
MODES = (MODE_ALL, MODE_PHYSICAL_GROUP, MODE_LOGICAL_CHUNK)
class RefineScopeError(ValueError):
"""Raised before preparation when a requested scope cannot be resolved."""
@dataclass(frozen=True)
class RefineScope:
"""One immutable scope resolution containing no Tensor or runtime object."""
mode: str
selected_group_indices: tuple[int, ...]
contract: Mapping[str, Any]
@property
def is_all(self) -> bool:
return self.mode == MODE_ALL
def _identity(payload: Mapping[str, Any]) -> str:
encoded = json.dumps(
dict(payload),
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _physical_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise RefineScopeError("assembly_plan must be a mapping")
second_pass = assembly_plan.get("second_pass_contract")
if not isinstance(second_pass, Mapping) or second_pass.get("version") != 1:
raise RefineScopeError("assembly_plan has no Second Pass contract version 1")
groups = second_pass.get("physical_groups")
if (
not isinstance(groups, Sequence)
or isinstance(groups, (str, bytes))
or not groups
or not all(isinstance(group, Mapping) for group in groups)
):
raise RefineScopeError("assembly_plan has no physical groups")
return list(groups)
def _logical_chunks(group: Mapping[str, Any], group_index: int) -> tuple[int, ...]:
values = group.get("logical_chunks")
if (
not isinstance(values, Sequence)
or isinstance(values, (str, bytes))
or not values
or any(type(value) is not int or value <= 0 for value in values)
):
raise RefineScopeError(
f"physical group {group_index + 1} has invalid logical chunk identity"
)
chunks = tuple(int(value) for value in values)
if len(set(chunks)) != len(chunks):
raise RefineScopeError(
f"physical group {group_index + 1} repeats a logical chunk"
)
return chunks
def resolve_refine_scope(
mode: str | None,
scope_index: int,
assembly_plan: Mapping[str, Any],
) -> RefineScope:
"""Resolve one public 1-based selection to complete physical groups.
Logical chunks never split a physical group. Selecting either logical half of
a Terminal Merge group therefore selects the complete terminal pair.
"""
if mode is None:
canonical_mode = MODE_ALL
elif not isinstance(mode, str):
raise RefineScopeError(f"unsupported refine scope mode {mode!r}")
else:
canonical_mode = mode.strip().lower()
if canonical_mode not in MODES:
raise RefineScopeError(f"unsupported refine scope mode {mode!r}")
if type(scope_index) is not int or scope_index <= 0:
raise RefineScopeError("refine scope index must be a positive integer")
groups = _physical_groups(assembly_plan)
group_chunks = [
_logical_chunks(group, group_index)
for group_index, group in enumerate(groups)
]
terminal_expanded = False
requested_index: int | None = None
if canonical_mode == MODE_ALL:
selected = tuple(range(len(groups)))
elif canonical_mode == MODE_PHYSICAL_GROUP:
requested_index = int(scope_index)
selected_index = requested_index - 1
if selected_index >= len(groups):
raise RefineScopeError(
f"physical group {requested_index} is outside 1..{len(groups)}"
)
selected = (selected_index,)
else:
requested_index = int(scope_index)
matches = [
group_index
for group_index, chunks in enumerate(group_chunks)
if requested_index in chunks
]
if len(matches) != 1:
if matches:
raise RefineScopeError(
f"logical chunk {requested_index} belongs to multiple physical groups"
)
raise RefineScopeError(
f"logical chunk {requested_index} is not present in the assembly plan"
)
selected = (matches[0],)
selected_group = groups[selected[0]]
selected_chunks = group_chunks[selected[0]]
terminal_expanded = bool(
selected_group.get("terminal_merged") and len(selected_chunks) > 1
)
selected_chunks = tuple(
chunk
for group_index in selected
for chunk in group_chunks[group_index]
)
contract: dict[str, Any] = {
"magic": MAGIC,
"schema_version": SCHEMA_VERSION,
"mode": canonical_mode,
"requested_index": requested_index,
"selected_physical_group_indices": tuple(selected),
"selected_physical_groups": tuple(index + 1 for index in selected),
"selected_logical_chunks": tuple(selected_chunks),
"terminal_expanded": terminal_expanded,
}
contract["scope_hash"] = _identity(contract)
return RefineScope(
mode=canonical_mode,
selected_group_indices=selected,
contract=MappingProxyType(contract),
)
def serializable_scope_contract(scope: RefineScope) -> dict[str, Any]:
if not isinstance(scope, RefineScope):
raise RefineScopeError("refine_scope must be a RefineScope")
return {
**dict(scope.contract),
"selected_physical_group_indices": list(
scope.contract["selected_physical_group_indices"]
),
"selected_physical_groups": list(scope.contract["selected_physical_groups"]),
"selected_logical_chunks": list(scope.contract["selected_logical_chunks"]),
}
+206
View File
@@ -0,0 +1,206 @@
"""Versioned stream-target policies for future Continuum refinement modes."""
from __future__ import annotations
import hashlib
import json
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any, Mapping
MAGIC = "H3_CONTINUUM_REFINE_TARGET"
SCHEMA_VERSION = 1
EXECUTION_MAGIC = "H3_CONTINUUM_TARGETED_REFINE_EXECUTION"
EXECUTION_SCHEMA_VERSION = 1
MODE_VIDEO_ONLY = "video_only"
MODE_AUDIO_ONLY = "audio_only"
MODE_VIDEO_AUDIO = "video_audio"
MODES = (MODE_VIDEO_ONLY, MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO)
NOISE_SEEDED_RANDOM = "seeded_random"
NOISE_ZERO = "zero"
OUTPUT_SAMPLED = "sampled"
OUTPUT_INPUT_PASSTHROUGH = "input_passthrough_bit_exact"
VIDEO_ONLY_SEED_NAMESPACE = "h3-continuum-refine-v1"
AUDIO_ONLY_SEED_NAMESPACE = "h3-continuum-refine-audio-v1"
VIDEO_AUDIO_SEED_NAMESPACE = "h3-continuum-refine-av-v1"
class RefineTargetError(ValueError):
"""Raised when a target policy or target execution contract is invalid."""
@dataclass(frozen=True)
class RefineTarget:
"""One immutable target policy; it contains no Tensor or runtime object."""
mode: str
contract: Mapping[str, Any]
def _identity(payload: Mapping[str, Any]) -> str:
encoded = json.dumps(
dict(payload),
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _stream_policy(*, noise: str, mask: int, output: str) -> dict[str, Any]:
return {
"noise": str(noise),
"mask": int(mask),
"output": str(output),
}
_POLICIES: dict[str, dict[str, Any]] = {
MODE_VIDEO_ONLY: {
"video": _stream_policy(
noise=NOISE_SEEDED_RANDOM,
mask=1,
output=OUTPUT_SAMPLED,
),
"audio": _stream_policy(
noise=NOISE_ZERO,
mask=0,
output=OUTPUT_INPUT_PASSTHROUGH,
),
"seed_namespace": VIDEO_ONLY_SEED_NAMESPACE,
},
MODE_AUDIO_ONLY: {
"video": _stream_policy(
noise=NOISE_ZERO,
mask=0,
output=OUTPUT_INPUT_PASSTHROUGH,
),
"audio": _stream_policy(
noise=NOISE_SEEDED_RANDOM,
mask=1,
output=OUTPUT_SAMPLED,
),
"seed_namespace": AUDIO_ONLY_SEED_NAMESPACE,
},
MODE_VIDEO_AUDIO: {
"video": _stream_policy(
noise=NOISE_SEEDED_RANDOM,
mask=1,
output=OUTPUT_SAMPLED,
),
"audio": _stream_policy(
noise=NOISE_SEEDED_RANDOM,
mask=1,
output=OUTPUT_SAMPLED,
),
"seed_namespace": VIDEO_AUDIO_SEED_NAMESPACE,
},
}
def make_refine_target(mode: str) -> RefineTarget:
canonical_mode = str(mode).strip().lower()
if canonical_mode not in MODES:
raise RefineTargetError(f"unsupported refine target mode {mode!r}")
policy = _POLICIES[canonical_mode]
contract: dict[str, Any] = {
"magic": MAGIC,
"schema_version": SCHEMA_VERSION,
"mode": canonical_mode,
"video": dict(policy["video"]),
"audio": dict(policy["audio"]),
"seed_namespace": str(policy["seed_namespace"]),
}
contract["target_hash"] = _identity(contract)
return RefineTarget(
mode=canonical_mode,
contract=MappingProxyType(contract),
)
def resolve_refine_target(value: RefineTarget | str | None) -> RefineTarget:
if value is None:
return make_refine_target(MODE_VIDEO_ONLY)
if isinstance(value, str):
return make_refine_target(value)
if not isinstance(value, RefineTarget):
raise RefineTargetError("refine_target must be a RefineTarget or mode string")
rebuilt = make_refine_target(value.mode)
if dict(rebuilt.contract) != dict(value.contract):
raise RefineTargetError("refine_target contract identity is inconsistent")
return value
def serializable_target_contract(target: RefineTarget) -> dict[str, Any]:
resolved = resolve_refine_target(target)
return {
**dict(resolved.contract),
"video": dict(resolved.contract["video"]),
"audio": dict(resolved.contract["audio"]),
}
def derive_target_refine_seed(
refine_seed: int,
physical_group_index: int,
target: RefineTarget | str,
) -> int:
if type(refine_seed) is not int or refine_seed < 0:
raise RefineTargetError("refine_seed must be a non-negative integer")
if type(physical_group_index) is not int or physical_group_index < 0:
raise RefineTargetError("physical_group_index must be a non-negative integer")
resolved = resolve_refine_target(target)
namespace = str(resolved.contract["seed_namespace"])
payload = f"{namespace}:{refine_seed}:{physical_group_index}".encode("ascii")
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
_SIGMA_RANGE_FIELDS = (
"schema_version",
"mode",
"source_sigma_hash",
"sigma_hash",
"start_sigma",
"end_sigma",
"evaluation_count",
"source_evaluation_count",
"start_index",
"end_index",
)
def build_target_execution_contract(
target: RefineTarget,
refine_schedule_contract: Mapping[str, Any],
) -> dict[str, Any]:
"""Combine target policy with sigma-range identity without mutating V1."""
resolved = resolve_refine_target(target)
if not isinstance(refine_schedule_contract, Mapping):
raise RefineTargetError("refine_schedule_contract must be a mapping")
missing = [
name for name in _SIGMA_RANGE_FIELDS if name not in refine_schedule_contract
]
if missing or not refine_schedule_contract.get("schedule_hash"):
details = ", ".join([*missing, "schedule_hash"] if not refine_schedule_contract.get("schedule_hash") else missing)
raise RefineTargetError(f"refine schedule range identity is incomplete: {details}")
sigma_range = {
name: refine_schedule_contract[name] for name in _SIGMA_RANGE_FIELDS
}
sigma_range["source_schedule_hash"] = str(
refine_schedule_contract["schedule_hash"]
)
contract: dict[str, Any] = {
"magic": EXECUTION_MAGIC,
"schema_version": EXECUTION_SCHEMA_VERSION,
"policy_authority": "refine_target_contract_v1",
"target": serializable_target_contract(resolved),
"sigma_range": sigma_range,
}
contract["execution_hash"] = _identity(contract)
return contract
+421
View File
@@ -0,0 +1,421 @@
"""Pure visible-timeline resolution for Experimental temporal refinement."""
from __future__ import annotations
import hashlib
import json
import math
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Any
from ..temporal import audio_latent_t, latent_slot_offsets, pixel_frames_for_latent_t
MAGIC = "H3_CONTINUUM_REFINE_WINDOW"
SCHEMA_VERSION = 1
MODE_TIME_WINDOW = "time_window"
class RefineWindowError(ValueError):
"""Raised before preparation when a visible time window is invalid."""
@dataclass(frozen=True)
class GroupTemporalWindow:
"""One physical group's immutable temporal mask ranges."""
group_index: int
logical_chunks: tuple[int, ...]
terminal_merged: bool
visible_frame_ranges: tuple[tuple[int, int], ...]
physical_frame_ranges: tuple[tuple[int, int], ...]
video_slot_ranges: tuple[tuple[int, int], ...]
audio_tick_ranges: tuple[tuple[int, int], ...]
@dataclass(frozen=True)
class RefineWindow:
"""One versioned visible-output window with no Tensor or runtime object."""
selected_group_indices: tuple[int, ...]
groups: tuple[GroupTemporalWindow, ...]
contract: Mapping[str, Any]
def group(self, group_index: int) -> GroupTemporalWindow | None:
for group in self.groups:
if group.group_index == int(group_index):
return group
return None
def _identity(payload: Mapping[str, Any]) -> str:
encoded = json.dumps(
dict(payload),
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _merge_ranges(values: Sequence[tuple[int, int]]) -> tuple[tuple[int, int], ...]:
ordered = sorted(
(int(start), int(stop))
for start, stop in values
if int(stop) > int(start)
)
merged: list[list[int]] = []
for start, stop in ordered:
if merged and start <= merged[-1][1]:
merged[-1][1] = max(merged[-1][1], stop)
else:
merged.append([start, stop])
return tuple((start, stop) for start, stop in merged)
def _indices_to_ranges(indices: Sequence[int]) -> tuple[tuple[int, int], ...]:
ordered = sorted(set(int(value) for value in indices))
if not ordered:
return ()
ranges: list[tuple[int, int]] = []
start = previous = ordered[0]
for value in ordered[1:]:
if value == previous + 1:
previous = value
continue
ranges.append((start, previous + 1))
start = previous = value
ranges.append((start, previous + 1))
return tuple(ranges)
def _overlaps(
start: int,
stop: int,
ranges: Sequence[tuple[int, int]],
) -> bool:
return any(start < range_stop and stop > range_start for range_start, range_stop in ranges)
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise RefineWindowError("assembly_plan must be a mapping")
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping) or contract.get("version") != 1:
raise RefineWindowError("assembly_plan has no Second Pass contract version 1")
groups = contract.get("physical_groups")
if (
not isinstance(groups, Sequence)
or isinstance(groups, (str, bytes))
or not groups
or not all(isinstance(group, Mapping) for group in groups)
):
raise RefineWindowError("assembly_plan has no physical groups")
return list(groups)
def _logical_chunks(group: Mapping[str, Any], group_index: int) -> tuple[int, ...]:
values = group.get("logical_chunks")
if (
not isinstance(values, Sequence)
or isinstance(values, (str, bytes))
or not values
or any(type(value) is not int or value <= 0 for value in values)
):
raise RefineWindowError(
f"physical group {group_index + 1} has invalid logical chunk identity"
)
chunks = tuple(int(value) for value in values)
if tuple(sorted(chunks)) != chunks or len(set(chunks)) != len(chunks):
raise RefineWindowError(
f"physical group {group_index + 1} logical chunks are not ordered and unique"
)
return chunks
def _visible_units(
assembly_plan: Mapping[str, Any],
groups: Sequence[Mapping[str, Any]],
) -> tuple[list[dict[str, int]], int]:
decode_groups = assembly_plan.get("decode_groups")
if isinstance(decode_groups, Sequence) and not isinstance(
decode_groups, (str, bytes)
):
if len(decode_groups) != len(groups) or not all(
isinstance(group, Mapping) for group in decode_groups
):
raise RefineWindowError(
"assembly_plan decode groups differ from Second Pass physical groups"
)
source_units: Sequence[Mapping[str, Any]] = decode_groups
else:
source_units = groups
cursor = 0
units: list[dict[str, int]] = []
for group_index, (group, source) in enumerate(zip(groups, source_units, strict=True)):
physical_frames = int(group.get("physical_frames", 0))
trim_frames = int(group.get("trim_prefix_frames", -1))
source_latent_t = int(group.get("source_latent_t", 0))
audio_shape = group.get("source_audio_shape")
if physical_frames <= 0 or trim_frames < 0 or trim_frames >= physical_frames:
raise RefineWindowError(
f"physical group {group_index + 1} frame geometry is invalid"
)
if source_latent_t <= 0 or pixel_frames_for_latent_t(source_latent_t) != physical_frames:
raise RefineWindowError(
f"physical group {group_index + 1} Video latent T differs from its frame grid"
)
if (
not isinstance(audio_shape, Sequence)
or isinstance(audio_shape, (str, bytes))
or len(audio_shape) != 4
or int(audio_shape[-1]) != audio_latent_t(physical_frames)
):
raise RefineWindowError(
f"physical group {group_index + 1} Audio latent T differs from its frame grid"
)
net_frames = physical_frames - trim_frames
source_total = int(source.get("total_frames", physical_frames))
source_trim = int(source.get("trim_frames", trim_frames))
source_net = int(source.get("net_frames", source_total - source_trim))
source_start = int(source.get("frame_start", cursor))
source_stop = int(source.get("frame_stop", source_start + source_net))
if (
source_total != physical_frames
or source_trim != trim_frames
or source_net != net_frames
or source_start != cursor
or source_stop != cursor + net_frames
):
raise RefineWindowError(
f"physical group {group_index + 1} visible range is inconsistent"
)
units.append(
{
"group_index": group_index,
"visible_start": source_start,
"visible_stop": source_stop,
"physical_frames": physical_frames,
"trim_frames": trim_frames,
"video_t": source_latent_t,
"audio_t": int(audio_shape[-1]),
}
)
cursor = source_stop
return units, cursor
def _output_to_natural_ranges(
output_start: int,
output_stop: int,
*,
natural_frames: int,
target_frames: int,
preserve_final_frame: bool,
) -> tuple[tuple[int, int], ...]:
mappings: list[tuple[int, int, int, bool]] = []
if target_frames >= natural_frames:
mappings.append((0, natural_frames, 0, False))
if target_frames > natural_frames:
mappings.append((natural_frames, target_frames, natural_frames - 1, True))
elif preserve_final_frame and target_frames >= 2:
mappings.extend(
(
(0, target_frames - 1, 0, False),
(target_frames - 1, target_frames, natural_frames - 1, True),
)
)
else:
mappings.append((0, target_frames, 0, False))
natural_ranges: list[tuple[int, int]] = []
for destination_start, destination_stop, source_start, repeated in mappings:
overlap_start = max(output_start, destination_start)
overlap_stop = min(output_stop, destination_stop)
if overlap_stop <= overlap_start:
continue
if repeated:
natural_ranges.append((source_start, source_start + 1))
else:
mapped_start = source_start + overlap_start - destination_start
mapped_stop = source_start + overlap_stop - destination_start
natural_ranges.append((mapped_start, mapped_stop))
return _merge_ranges(natural_ranges)
def _group_window(
group: Mapping[str, Any],
unit: Mapping[str, int],
natural_ranges: Sequence[tuple[int, int]],
) -> GroupTemporalWindow | None:
visible_start = int(unit["visible_start"])
visible_stop = int(unit["visible_stop"])
visible_ranges = _merge_ranges(
(
max(start, visible_start),
min(stop, visible_stop),
)
for start, stop in natural_ranges
if start < visible_stop and stop > visible_start
)
if not visible_ranges:
return None
trim_frames = int(unit["trim_frames"])
physical_ranges = _merge_ranges(
(
trim_frames + start - visible_start,
trim_frames + stop - visible_start,
)
for start, stop in visible_ranges
)
video_t = int(unit["video_t"])
offsets = latent_slot_offsets(video_t)
selected_slots = []
for slot, slot_start in enumerate(offsets):
slot_stop = offsets[slot + 1] if slot + 1 < video_t else int(unit["physical_frames"])
if _overlaps(slot_start, slot_stop, physical_ranges):
selected_slots.append(slot)
video_ranges = _indices_to_ranges(selected_slots)
if video_ranges and offsets[video_ranges[0][0]] < trim_frames:
raise RefineWindowError(
f"physical group {int(unit['group_index']) + 1} Video grid would expose "
"the protected continuation prefix"
)
audio_ranges = _merge_ranges(
(
max(0, min(int(unit["audio_t"]), audio_latent_t(start))),
max(0, min(int(unit["audio_t"]), audio_latent_t(stop))),
)
for start, stop in physical_ranges
)
protected_audio_stop = audio_latent_t(trim_frames)
if audio_ranges and audio_ranges[0][0] < protected_audio_stop:
raise RefineWindowError(
f"physical group {int(unit['group_index']) + 1} Audio grid would expose "
"the protected continuation prefix"
)
if not video_ranges or not audio_ranges:
raise RefineWindowError(
f"physical group {int(unit['group_index']) + 1} window resolves to an empty AV grid"
)
return GroupTemporalWindow(
group_index=int(unit["group_index"]),
logical_chunks=_logical_chunks(group, int(unit["group_index"])),
terminal_merged=bool(group.get("terminal_merged", False)),
visible_frame_ranges=visible_ranges,
physical_frame_ranges=physical_ranges,
video_slot_ranges=video_ranges,
audio_tick_ranges=audio_ranges,
)
def resolve_refine_window(
start_sec: float,
end_sec: float,
assembly_plan: Mapping[str, Any],
) -> RefineWindow:
"""Resolve final visible output time to protected physical AV ranges.
The requested interval is half-open. Fractions select every output frame
interval they overlap. Continuation trim prefixes are never selected.
"""
try:
requested_start = float(start_sec)
requested_end = float(end_sec)
except (TypeError, ValueError) as exc:
raise RefineWindowError("refine window seconds must be numeric") from exc
if not math.isfinite(requested_start) or not math.isfinite(requested_end):
raise RefineWindowError("refine window seconds must be finite")
if requested_start >= requested_end:
raise RefineWindowError("refine window start must be less than end")
groups = _contract_groups(assembly_plan)
units, natural_frames = _visible_units(assembly_plan, groups)
fps = int(assembly_plan.get("fps", 0))
target_frames = int(assembly_plan.get("target_frames", 0))
if fps <= 0 or target_frames <= 0 or natural_frames <= 0:
raise RefineWindowError("assembly_plan visible timeline is invalid")
clamped_start = min(max(requested_start, 0.0), target_frames / fps)
clamped_end = min(max(requested_end, 0.0), target_frames / fps)
output_start = max(0, min(target_frames, math.floor(clamped_start * fps + 1e-9)))
output_stop = max(0, min(target_frames, math.ceil(clamped_end * fps - 1e-9)))
if output_stop <= output_start:
raise RefineWindowError("refine window does not overlap the visible output")
natural_ranges = _output_to_natural_ranges(
output_start,
output_stop,
natural_frames=natural_frames,
target_frames=target_frames,
preserve_final_frame=bool(assembly_plan.get("preserve_final_frame", False)),
)
if not natural_ranges:
raise RefineWindowError("refine window has no natural timeline source")
resolved_groups = tuple(
group_window
for group, unit in zip(groups, units, strict=True)
if (
group_window := _group_window(
group,
unit,
natural_ranges,
)
)
is not None
)
if not resolved_groups:
raise RefineWindowError("refine window selects no physical group")
selected = tuple(group.group_index for group in resolved_groups)
group_contracts = [
{
"group_index": group.group_index,
"physical_group": group.group_index + 1,
"logical_chunks": list(group.logical_chunks),
"terminal_merged": group.terminal_merged,
"visible_frame_ranges": [list(value) for value in group.visible_frame_ranges],
"physical_frame_ranges": [list(value) for value in group.physical_frame_ranges],
"video_slot_ranges": [list(value) for value in group.video_slot_ranges],
"audio_tick_ranges": [list(value) for value in group.audio_tick_ranges],
}
for group in resolved_groups
]
contract: dict[str, Any] = {
"magic": MAGIC,
"schema_version": SCHEMA_VERSION,
"mode": MODE_TIME_WINDOW,
"timeline": "final_visible_output",
"interval": "half_open",
"requested_start_sec": requested_start,
"requested_end_sec": requested_end,
"effective_start_sec": output_start / fps,
"effective_end_sec": output_stop / fps,
"output_frame_range": [output_start, output_stop],
"target_frames": target_frames,
"natural_frames": natural_frames,
"natural_frame_ranges": [list(value) for value in natural_ranges],
"selected_physical_groups": [index + 1 for index in selected],
"groups": group_contracts,
"continuation_prefix_policy": "always_protected",
"outside_window_policy": "mask_zero_then_bit_exact_restore",
"physical_group_atomicity": "preserved",
"temporal_quantization": "intersecting_video_slots_and_audio_ticks",
}
contract["window_hash"] = _identity(contract)
return RefineWindow(
selected_group_indices=selected,
groups=resolved_groups,
contract=MappingProxyType(contract),
)
def serializable_window_contract(window: RefineWindow) -> dict[str, Any]:
if not isinstance(window, RefineWindow):
raise RefineWindowError("refine_window must be a RefineWindow")
return json.loads(json.dumps(dict(window.contract)))
+920
View File
@@ -0,0 +1,920 @@
"""Read-only V3.8 reliability diagnostics.
The helpers in this module inspect shapes and existing metadata only. They do
not resize, encode, hash, copy, or otherwise modify user tensors.
"""
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from ..compatibility import check_comfy_h3_runtime
from ..constants import (
DIAGNOSTICS_BASIC,
DIAGNOSTICS_FULL,
DIAGNOSTICS_OFF,
FPS,
normalize_diagnostics_mode,
)
from ..reference import resolve_reference_image_size
from ..reference_video import (
REFERENCE_VIDEO_SIZE_EFFICIENT,
resolve_reference_video_frame_count,
resolve_reference_video_size,
)
from ..temporal import (
align_frame_count_up,
audio_grid_offset,
audio_latent_t,
context_slots,
is_valid_frame_count,
video_latent_t,
)
VISUAL_LOAD_LOW_MAX = 0.75
VISUAL_LOAD_MEDIUM_MAX = 1.75
@dataclass(frozen=True, slots=True)
class DiagnosticSection:
basic: tuple[str, ...] = ()
detailed: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class V38Diagnostics:
basic: tuple[str, ...]
detailed: tuple[str, ...]
def _combine_sections(*sections: DiagnosticSection) -> V38Diagnostics:
return V38Diagnostics(
basic=tuple(line for section in sections for line in section.basic),
detailed=tuple(line for section in sections for line in section.detailed),
)
def _physical_groups(assembly_plan: Any) -> list[Any]:
if not isinstance(assembly_plan, Mapping):
return []
decode_groups = assembly_plan.get("decode_groups")
if isinstance(decode_groups, list) and decode_groups:
return list(decode_groups)
chunks = assembly_plan.get("chunks")
return list(chunks) if isinstance(chunks, list) else []
def _latent_outputs(value: Any) -> list[Any]:
return list(value) if isinstance(value, (list, tuple)) else []
def _latent_samples(value: Any) -> Any:
return value.get("samples") if isinstance(value, Mapping) else None
def _image_geometry(value: Any) -> tuple[int, int]:
shape = getattr(value, "shape", None)
if shape is None or len(shape) != 4:
raise ValueError("IMAGE must have shape [B,H,W,C]")
height = int(shape[1])
width = int(shape[2])
if height < 1 or width < 1:
raise ValueError("IMAGE geometry must be positive")
return width, height
def _video_guide_geometry(value: Any) -> tuple[int, int, int]:
shape = getattr(value, "shape", None)
if shape is None or len(shape) != 4:
raise ValueError("Video Guide must have shape [T,H,W,C]")
frames = int(shape[0])
height = int(shape[1])
width = int(shape[2])
if frames < 1 or height < 1 or width < 1:
raise ValueError("Video Guide geometry must be positive")
return frames, width, height
def _native_chunk_frames(chunk_seconds: float) -> int:
return align_frame_count_up(int(round(float(chunk_seconds) * FPS)))
def build_frame_alignment_diagnostics(
*,
video_latents: Any,
assembly_plan: Any,
video_guide: Any = None,
chunk_seconds: float,
) -> DiagnosticSection:
"""Inspect Video Guide alignment and physical video latent lengths."""
basic: list[str] = []
detailed: list[str] = []
if video_guide is not None:
try:
source_frames, _source_width, _source_height = _video_guide_geometry(
video_guide
)
aligned_frames = resolve_reference_video_frame_count(
source_frames,
_native_chunk_frames(chunk_seconds),
)
if source_frames < aligned_frames:
action = "final-frame pad"
elif source_frames > aligned_frames:
action = "trimmed to one-chunk H3 limit"
else:
action = "unchanged"
line = (
f"Frame alignment: Video Guide {source_frames} -> {aligned_frames} "
f"({action}, H3 17k+5)."
)
if action != "unchanged":
basic.append(line)
detailed.append(line)
except Exception as exc:
line = (
"Frame alignment advisory: Video Guide alignment could not be "
f"inspected ({type(exc).__name__}: {exc})."
)
basic.append(line)
detailed.append(line)
groups = _physical_groups(assembly_plan)
outputs = _latent_outputs(video_latents)
if not groups:
line = "Frame alignment advisory: physical group metadata is unavailable."
basic.append(line)
detailed.append(line)
return DiagnosticSection(tuple(basic), tuple(detailed))
if len(outputs) != len(groups):
line = (
"Frame alignment advisory: physical video output count "
f"{len(outputs)} does not match group count {len(groups)}."
)
basic.append(line)
detailed.append(line)
for index, group in enumerate(groups, start=1):
problems: list[str] = []
try:
if not isinstance(group, Mapping):
raise ValueError("group metadata is not a mapping")
frames = int(group["total_frames"])
trim = int(group["trim_frames"])
net = int(group["net_frames"])
metadata_t = int(group["expected_video_latent_t"])
if not is_valid_frame_count(frames):
problems.append("frames are not on H3 17k+5")
expected_t = None
else:
expected_t = video_latent_t(frames)
if metadata_t != expected_t:
problems.append("metadata T differs from native grid")
if frames - trim != net:
problems.append("total-trim differs from net")
samples = (
_latent_samples(outputs[index - 1])
if index <= len(outputs)
else None
)
if not torch.is_tensor(samples) or samples.ndim < 3:
actual_t = None
problems.append("video LATENT samples are missing or invalid")
else:
actual_t = int(samples.shape[2])
if actual_t != metadata_t:
problems.append("actual T differs from metadata")
if expected_t is not None and actual_t != expected_t:
problems.append("actual T differs from native grid")
result = "PASS" if not problems else "ADVISORY"
expected_text = "n/a" if expected_t is None else str(expected_t)
actual_text = "n/a" if actual_t is None else str(actual_t)
detailed.append(
f"Frame grid: group={index} frames={frames} trim={trim} net={net} "
f"video_t={actual_text} metadata={metadata_t} "
f"expected={expected_text} {result}."
)
except Exception as exc:
problems.append(f"{type(exc).__name__}: {exc}")
detailed.append(
f"Frame grid: group={index} unavailable ADVISORY "
f"({problems[-1]})."
)
if problems:
basic.append(
f"Frame alignment advisory: group {index}: "
+ "; ".join(problems)
+ "."
)
return DiagnosticSection(tuple(basic), tuple(detailed))
def validate_audio_latent_lengths(
*,
audio_latents: Any,
assembly_plan: Any,
) -> DiagnosticSection:
"""Validate physical audio latent shapes without reading tensor values."""
basic: list[str] = []
detailed: list[str] = []
groups = _physical_groups(assembly_plan)
outputs = _latent_outputs(audio_latents)
if not groups:
line = "Audio latent advisory: physical group metadata is unavailable."
return DiagnosticSection((line,), (line,))
if len(outputs) != len(groups):
line = (
f"Audio latent advisory: physical output count {len(outputs)} "
f"does not match group count {len(groups)}."
)
basic.append(line)
detailed.append(line)
for index, group in enumerate(groups, start=1):
problems: list[str] = []
frames = metadata_t = nominal_t = actual_t = None
phase = None
try:
if not isinstance(group, Mapping):
raise ValueError("group metadata is not a mapping")
frames = int(group["total_frames"])
metadata_t = int(group["expected_audio_latent_t"])
nominal_t = audio_latent_t(frames)
samples = (
_latent_samples(outputs[index - 1])
if index <= len(outputs)
else None
)
if not torch.is_tensor(samples):
problems.append("LATENT samples are missing or not a Tensor")
elif samples.ndim != 4:
problems.append(f"rank is {samples.ndim}, expected 4")
else:
if int(samples.shape[0]) < 1:
problems.append("batch dimension is empty")
if int(samples.shape[1]) != 32:
problems.append(f"channels={int(samples.shape[1])}, expected 32")
if int(samples.shape[2]) != 2:
problems.append(f"stereo={int(samples.shape[2])}, expected 2")
actual_t = int(samples.shape[-1])
if actual_t <= 0:
problems.append("T must be positive")
else:
phase = audio_grid_offset(frames, actual_t)
if actual_t != metadata_t:
problems.append(
f"metadata={metadata_t} actual={actual_t} "
f"delta={actual_t - metadata_t}"
)
if not -0.500001 <= phase <= 0.500001:
problems.append(f"phase={phase:.6f} is outside [-0.5,0.5]")
result = "PASS" if not problems else "ADVISORY"
detailed.append(
f"Audio latent check: group={index} frames={frames} "
f"metadata={metadata_t} actual="
f"{'n/a' if actual_t is None else actual_t} nominal={nominal_t} "
f"phase={'n/a' if phase is None else f'{phase:.6f}'} {result}."
)
except Exception as exc:
problems.append(f"{type(exc).__name__}: {exc}")
detailed.append(
f"Audio latent check: group={index} unavailable ADVISORY "
f"({problems[-1]})."
)
if problems:
basic.append(
f"Audio latent advisory: group {index}: "
+ "; ".join(problems)
+ "."
)
return DiagnosticSection(tuple(basic), tuple(detailed))
def collect_core_compatibility_advisory(
checker: Callable[[], list[str]] | None = None,
) -> DiagnosticSection:
"""Collect the existing feature-based Core advisory after successful generation."""
active_checker = checker or check_comfy_h3_runtime
try:
issues = tuple(dict.fromkeys(str(issue) for issue in active_checker() if issue))
except Exception as exc:
basic = (
"Core compatibility advisory unavailable; generation result was preserved.",
)
detailed = (
basic[0][:-1] + f" ({type(exc).__name__}: {exc}).",
)
return DiagnosticSection(basic, detailed)
if not issues:
return DiagnosticSection(
(),
("Core compatibility: native MiniMax H3 contracts detected.",),
)
return DiagnosticSection(
(
"Core compatibility advisory: "
f"{len(issues)} native H3 contract difference(s) detected; "
"generation completed; enable Detailed Report for details.",
),
tuple(f"Core compatibility advisory: {issue}." for issue in issues),
)
def _visual_load_class(total: float) -> str:
if total < VISUAL_LOAD_LOW_MAX:
return "Low"
if total < VISUAL_LOAD_MEDIUM_MAX:
return "Medium"
return "High"
def estimate_visual_conditioning_load(
*,
output_width: int,
output_height: int,
reference_images: Sequence[Any],
reference_size: str,
video_guide: Any,
video_guide_size: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
chunk_seconds: float,
still_guide_active: bool,
) -> DiagnosticSection:
"""Estimate peak per-group relative visual conditioning from geometry only."""
output_area = int(output_width) * int(output_height)
if output_area <= 0:
line = "Visual conditioning load advisory: output geometry is invalid."
return DiagnosticSection((line,), (line,))
components: list[tuple[str, float]] = []
problems: list[str] = []
for socket_index, image in enumerate(reference_images, start=1):
if image is None:
continue
try:
source_width, source_height = _image_geometry(image)
target_width, target_height = resolve_reference_image_size(
source_width,
source_height,
output_width=int(output_width),
output_height=int(output_height),
size_mode=str(reference_size),
)
load = (target_width * target_height) / float(output_area)
components.append(
(f"Ref{socket_index}={target_width}x{target_height}", load)
)
except Exception as exc:
problems.append(
f"Reference Image {socket_index} unavailable "
f"({type(exc).__name__}: {exc})"
)
if video_guide is not None:
try:
source_frames, source_width, source_height = _video_guide_geometry(
video_guide
)
native_frames = _native_chunk_frames(chunk_seconds)
aligned_frames = resolve_reference_video_frame_count(
source_frames,
native_frames,
)
target_width, target_height = resolve_reference_video_size(
source_width,
source_height,
output_width=int(output_width),
output_height=int(output_height),
size_mode=str(video_guide_size),
)
spatial_ratio = (target_width * target_height) / float(output_area)
temporal_ratio = video_latent_t(aligned_frames) / float(
video_latent_t(native_frames)
)
components.append(
(
f"Video Guide={target_width}x{target_height}/{aligned_frames}f",
spatial_ratio * temporal_ratio,
)
)
except Exception as exc:
problems.append(
f"Video Guide unavailable ({type(exc).__name__}: {exc})"
)
basic: list[str] = []
detailed: list[str] = []
if components:
total = sum(load for _name, load in components)
classification = _visual_load_class(total)
basic_line = (
f"Visual conditioning load: {classification} "
"(peak per physical group)."
)
if still_guide_active:
basic_line = basic_line[:-1] + "; Still Guide active."
basic.append(basic_line)
details = "; ".join(
f"{name}={load:.3f}" for name, load in components
)
detailed_line = (
f"Visual conditioning load: total={total:.3f} "
f"class={classification}; {details}"
)
if still_guide_active:
detailed_line += "; Still Guide=active"
detailed.append(detailed_line + ".")
elif still_guide_active:
basic.append("Still Guide: active.")
detailed.append("Still Guide: active (excluded from numeric visual load).")
for problem in problems:
line = f"Visual conditioning load advisory: {problem}."
basic.append(line)
detailed.append(line)
return DiagnosticSection(tuple(basic), tuple(detailed))
@dataclass(frozen=True, slots=True)
class _LogicalChunkGeometry:
index: int
total_frames: int | None
trim_frames: int | None
net_frames: int | None
derived_start: int
derived_stop: int
visible_start: int | None
visible_stop: int | None
problems: tuple[str, ...]
def _chunk_boundary_advisory(message: str) -> str:
return f"Chunk boundary advisory: {str(message).rstrip('.')}."
def _logical_chunk_geometries(
assembly_plan: Any,
) -> tuple[list[_LogicalChunkGeometry], list[str]]:
problems: list[str] = []
if not isinstance(assembly_plan, Mapping):
return [], ["Assembly Plan metadata is unavailable"]
raw_chunks = assembly_plan.get("chunks")
if not isinstance(raw_chunks, list) or not raw_chunks:
return [], ["logical chunk metadata is unavailable"]
geometries: list[_LogicalChunkGeometry] = []
cursor = 0
for position, chunk in enumerate(raw_chunks, start=1):
local: list[str] = []
total = trim = net = None
visible_start = visible_stop = None
index = position
if not isinstance(chunk, Mapping):
local.append("metadata is not a mapping")
else:
try:
index = int(chunk.get("chunk_index", chunk.get("sequence_index", position)))
if index != position:
local.append(
f"logical order has Chunk {index} at position {position}"
)
except Exception:
index = position
local.append("logical chunk index is invalid")
try:
total = int(chunk["total_frames"])
trim = int(chunk["trim_frames"])
net = int(chunk["net_frames"])
if total < 0:
local.append("total_frames is negative")
if trim < 0:
local.append("trim_frames is negative")
if trim > total:
local.append("trim_frames exceeds total_frames")
if net < 0:
local.append("net_frames is negative")
if total - trim != net:
local.append("total_frames - trim_frames differs from net_frames")
except Exception as exc:
local.append(
"required geometry is unavailable "
f"({type(exc).__name__}: {exc})"
)
derived_start = cursor
derived_stop = cursor + (net if net is not None and net >= 0 else 0)
if isinstance(chunk, Mapping) and "frame_start" in chunk:
try:
visible_start = int(chunk["frame_start"])
if visible_start != derived_start:
local.append(
f"frame_start={visible_start} differs from derived {derived_start}"
)
except Exception:
local.append("frame_start is invalid")
else:
visible_start = derived_start if net is not None else None
if isinstance(chunk, Mapping) and "frame_stop" in chunk:
try:
visible_stop = int(chunk["frame_stop"])
if visible_stop != derived_stop:
local.append(
f"frame_stop={visible_stop} differs from derived {derived_stop}"
)
except Exception:
local.append("frame_stop is invalid")
else:
visible_stop = derived_stop if net is not None else None
geometries.append(
_LogicalChunkGeometry(
index=index,
total_frames=total,
trim_frames=trim,
net_frames=net,
derived_start=derived_start,
derived_stop=derived_stop,
visible_start=visible_start,
visible_stop=visible_stop,
problems=tuple(local),
)
)
problems.extend(f"Chunk {position}: {problem}" for problem in local)
cursor = derived_stop
return geometries, problems
def _physical_boundary_mapping(
*,
assembly_plan: Any,
logical_count: int,
video_latents: Any,
audio_latents: Any,
) -> tuple[list[Mapping[str, Any]], list[tuple[int, ...]], dict[int, int], list[str]]:
problems: list[str] = []
if not isinstance(assembly_plan, Mapping):
return [], [], {}, ["physical group metadata is unavailable"]
if "decode_groups" in assembly_plan:
raw_groups = assembly_plan.get("decode_groups")
if not isinstance(raw_groups, list) or not raw_groups:
return [], [], {}, ["physical groups are not a non-empty list"]
groups = list(raw_groups)
fallback = False
else:
raw_chunks = assembly_plan.get("chunks")
if not isinstance(raw_chunks, list) or not raw_chunks:
return [], [], {}, ["physical group metadata is unavailable"]
groups = list(raw_chunks)
fallback = True
normalized_groups: list[Mapping[str, Any]] = []
logical_groups: list[tuple[int, ...]] = []
logical_to_group: dict[int, int] = {}
flattened: list[int] = []
for group_index, group in enumerate(groups, start=1):
if not isinstance(group, Mapping):
problems.append(f"physical group {group_index} metadata is not a mapping")
normalized_groups.append({})
logical_groups.append(())
continue
normalized_groups.append(group)
raw_logical = (
[group.get("chunk_index", group.get("sequence_index", group_index))]
if fallback
else group.get("logical_chunk_indices")
)
if not isinstance(raw_logical, (list, tuple)) or not raw_logical:
problems.append(f"physical group {group_index} has no logical chunks")
logical_groups.append(())
continue
try:
logical = tuple(int(value) for value in raw_logical)
except Exception:
problems.append(f"physical group {group_index} logical chunks are invalid")
logical_groups.append(())
continue
logical_groups.append(logical)
flattened.extend(logical)
if tuple(sorted(logical)) != logical:
problems.append(f"physical group {group_index} logical order is reversed")
terminal = bool(group.get("terminal_merged", False))
if terminal:
expected_terminal = (logical_count - 1, logical_count)
if len(logical) != 2 or logical != expected_terminal:
problems.append(
f"physical group {group_index} Terminal Merge mapping is invalid"
)
elif len(logical) != 1:
problems.append(
f"physical group {group_index} has multiple logical chunks without Terminal Merge"
)
for chunk_index in logical:
if chunk_index < 1 or chunk_index > logical_count:
problems.append(
f"physical group {group_index} references invalid Chunk {chunk_index}"
)
elif chunk_index in logical_to_group:
problems.append(f"logical Chunk {chunk_index} is mapped more than once")
else:
logical_to_group[chunk_index] = group_index
expected = list(range(1, logical_count + 1))
if flattened != expected:
missing = [value for value in expected if value not in flattened]
if missing:
problems.append(
"physical group mapping does not cover all logical chunks "
f"(missing={missing})"
)
if not missing and flattened != sorted(flattened):
problems.append("physical group logical chunk order is not contiguous")
video_count = len(_latent_outputs(video_latents))
audio_count = len(_latent_outputs(audio_latents))
if video_count != len(groups) or audio_count != len(groups):
problems.append(
"physical output/group counts differ "
f"(groups={len(groups)}, video={video_count}, audio={audio_count})"
)
return normalized_groups, logical_groups, logical_to_group, problems
def _second_pass_boundary_contract_problems(
*,
assembly_plan: Any,
groups: Sequence[Mapping[str, Any]],
logical_groups: Sequence[tuple[int, ...]],
) -> tuple[list[str], str | None]:
if not isinstance(assembly_plan, Mapping) or "second_pass_contract" not in assembly_plan:
return [], None
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
return ["Second Pass contract is not a mapping"], "Second Pass contract: UNAVAILABLE."
contract_groups = contract.get("physical_groups")
if not isinstance(contract_groups, list):
return ["Second Pass physical_groups is not a list"], "Second Pass contract: UNAVAILABLE."
problems: list[str] = []
if len(contract_groups) != len(groups):
problems.append(
"Second Pass physical group count differs from decode groups "
f"({len(contract_groups)} != {len(groups)})"
)
for index, (decode_group, logical) in enumerate(
zip(groups, logical_groups, strict=False)
):
if index >= len(contract_groups):
break
contract_group = contract_groups[index]
if not isinstance(contract_group, Mapping):
problems.append(f"Second Pass group {index} is not a mapping")
continue
raw_contract_logical = contract_group.get("logical_chunks")
contract_logical = (
tuple(raw_contract_logical)
if isinstance(raw_contract_logical, (list, tuple))
else ()
)
checks = (
("group_id", contract_group.get("group_id"), index),
("logical_chunks", contract_logical, logical),
(
"physical_frames",
contract_group.get("physical_frames"),
decode_group.get("total_frames"),
),
(
"trim_prefix_frames",
contract_group.get("trim_prefix_frames"),
decode_group.get("trim_frames"),
),
(
"terminal_merged",
bool(contract_group.get("terminal_merged", False)),
bool(decode_group.get("terminal_merged", False)),
),
)
for name, actual, expected in checks:
if actual != expected:
problems.append(
f"Second Pass group {index} {name} differs "
f"({actual!r} != {expected!r})"
)
result = "PASS" if not problems else "ADVISORY"
return problems, f"Second Pass contract: groups={len(contract_groups)} {result}."
def build_chunk_boundary_diagnostics(
*,
video_latents: Any,
audio_latents: Any,
assembly_plan: Any,
) -> DiagnosticSection:
"""Inspect logical/physical chunk boundaries using metadata and shapes only."""
detailed = [
"Chunk boundary contract: structural diagnostic only; "
"no image/audio similarity measurement."
]
basic: list[str] = []
geometries, logical_problems = _logical_chunk_geometries(assembly_plan)
groups, logical_groups, logical_to_group, mapping_problems = (
_physical_boundary_mapping(
assembly_plan=assembly_plan,
logical_count=len(geometries),
video_latents=video_latents,
audio_latents=audio_latents,
)
)
second_pass_problems, second_pass_line = _second_pass_boundary_contract_problems(
assembly_plan=assembly_plan,
groups=groups,
logical_groups=logical_groups,
)
global_problems = logical_problems + mapping_problems + second_pass_problems
for problem in dict.fromkeys(global_problems):
basic.append(_chunk_boundary_advisory(problem))
if second_pass_line is not None:
detailed.append(second_pass_line)
for boundary_index in range(1, len(geometries)):
left = geometries[boundary_index - 1]
right = geometries[boundary_index]
local = list(left.problems) + list(right.problems)
if left.visible_stop is None or right.visible_start is None:
gap = None
local.append("visible range is unavailable")
else:
gap = int(right.visible_start) - int(left.visible_stop)
if gap != 0:
local.append("non-contiguous visible range")
context_frames = right.trim_frames
context_video_t = context_audio_t = None
if context_frames is None:
local.append("incoming context is unavailable")
else:
try:
context_video_t = context_slots(context_frames)
context_audio_t = audio_latent_t(context_frames)
except Exception as exc:
local.append(
f"incoming context {context_frames}f is invalid "
f"({type(exc).__name__}: {exc})"
)
left_group = logical_to_group.get(left.index)
right_group = logical_to_group.get(right.index)
if left_group is None or right_group is None:
local.append("physical group mapping is unavailable")
classification = "UNAVAILABLE"
detailed.append(
f"Boundary {left.index}→{right.index}: physical mapping unavailable; "
f"UNAVAILABLE ({'; '.join(dict.fromkeys(local))})."
)
elif left_group == right_group:
group = groups[right_group - 1] if right_group <= len(groups) else {}
terminal = bool(group.get("terminal_merged", False))
expected_pair = (left.index, right.index)
actual_pair = (
logical_groups[right_group - 1]
if right_group <= len(logical_groups)
else ()
)
if not terminal:
local.append("shared physical group lacks terminal_merged=yes")
if actual_pair != expected_pair:
local.append("shared physical group does not match this boundary")
classification = "PASS" if not local else "ADVISORY"
detailed.append(
f"Boundary {left.index}→{right.index}: shared physical group={right_group}; "
f"terminal_merged={'yes' if terminal else 'no'}; "
"no external decode boundary; "
f"{classification}"
+ (f" ({'; '.join(dict.fromkeys(local))})" if local else "")
+ "."
)
else:
classification = "PASS" if not local else "ADVISORY"
visible_left = "n/a" if left.visible_stop is None else str(left.visible_stop)
visible_right = "n/a" if right.visible_start is None else str(right.visible_start)
gap_text = "n/a" if gap is None else str(gap)
context_text = "n/a" if context_frames is None else f"{context_frames}f"
video_text = "n/a" if context_video_t is None else f"{context_video_t}T"
audio_text = "n/a" if context_audio_t is None else f"{context_audio_t}T"
right_text = (
"right=unavailable"
if right.total_frames is None
or right.trim_frames is None
or right.net_frames is None
else (
f"right={right.total_frames}f trim={right.trim_frames} "
f"net={right.net_frames}"
)
)
detailed.append(
f"Boundary {left.index}→{right.index}: physical; "
f"visible={visible_left}→{visible_right} gap={gap_text}; "
f"context={context_text} video={video_text} audio={audio_text}; "
f"{right_text}; {classification}"
+ (f" ({'; '.join(dict.fromkeys(local))})" if local else "")
+ "."
)
if local:
if gap not in (None, 0):
message = (
f"boundary {left.index}→{right.index} has a non-contiguous "
"visible range"
)
elif classification == "UNAVAILABLE":
message = f"boundary {left.index}→{right.index} is unavailable"
else:
message = (
f"boundary {left.index}→{right.index}: "
+ "; ".join(dict.fromkeys(local))
)
advisory = _chunk_boundary_advisory(message)
if advisory not in basic:
basic.append(advisory)
return DiagnosticSection(tuple(basic), tuple(detailed))
def build_v38_diagnostics(
*,
video_latents: Any,
audio_latents: Any,
assembly_plan: Any,
output_width: int,
output_height: int,
chunk_seconds: float,
reference_images: Sequence[Any] = (),
reference_size: str = "Match Output",
video_guide: Any = None,
video_guide_size: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
still_guide_active: bool = False,
core_checker: Callable[[], list[str]] | None = None,
) -> V38Diagnostics:
return _combine_sections(
build_frame_alignment_diagnostics(
video_latents=video_latents,
assembly_plan=assembly_plan,
video_guide=video_guide,
chunk_seconds=float(chunk_seconds),
),
validate_audio_latent_lengths(
audio_latents=audio_latents,
assembly_plan=assembly_plan,
),
build_chunk_boundary_diagnostics(
video_latents=video_latents,
audio_latents=audio_latents,
assembly_plan=assembly_plan,
),
collect_core_compatibility_advisory(core_checker),
estimate_visual_conditioning_load(
output_width=int(output_width),
output_height=int(output_height),
reference_images=tuple(reference_images),
reference_size=str(reference_size),
video_guide=video_guide,
video_guide_size=str(video_guide_size),
chunk_seconds=float(chunk_seconds),
still_guide_active=bool(still_guide_active),
),
)
def format_v38_diagnostics(
diagnostics: V38Diagnostics,
*,
mode: str = DIAGNOSTICS_BASIC,
) -> str:
normalized = normalize_diagnostics_mode(str(mode))
if normalized == DIAGNOSTICS_OFF:
return ""
lines = diagnostics.detailed if normalized == DIAGNOSTICS_FULL else diagnostics.basic
if not lines:
return ""
return "V3.8 Reliability\n" + "\n".join(lines)
def append_v38_status(
status: Any,
diagnostics: V38Diagnostics,
*,
mode: str = DIAGNOSTICS_BASIC,
) -> str:
suffix = format_v38_diagnostics(diagnostics, mode=mode)
original = str(status)
return original if not suffix else original.rstrip() + "\n" + suffix
+248
View File
@@ -0,0 +1,248 @@
"""Shared H3 resolution presets for Easy and V3.8 sampler facades."""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any
H3_ASPECT_AUTO = "Auto from First Image"
H3_ASPECT_LANDSCAPE = "Landscape 16:9"
H3_ASPECT_PORTRAIT = "Portrait 9:16"
H3_ASPECT_SQUARE = "Square 1:1"
H3_ASPECT_OPTIONS = (
H3_ASPECT_AUTO,
H3_ASPECT_LANDSCAPE,
H3_ASPECT_PORTRAIT,
H3_ASPECT_SQUARE,
)
H3_SIZE_SOURCE_FIRST_IMAGE = "First Image"
H3_SIZE_SOURCE_MANUAL = "Manual"
H3_SIZE_SOURCE_LEGACY = "Legacy Aspect"
H3_SIZE_SOURCE_OPTIONS = (
H3_SIZE_SOURCE_FIRST_IMAGE,
H3_SIZE_SOURCE_MANUAL,
H3_SIZE_SOURCE_LEGACY,
)
H3_PRESET_DRAFT = "Draft — 0.30 MP"
H3_PRESET_BALANCED = "Balanced — 0.60 MP"
H3_PRESET_NATIVE = "Native 768"
H3_PRESET_CUSTOM = "Custom"
H3_PRESET_OPTIONS = (
H3_PRESET_DRAFT,
H3_PRESET_BALANCED,
H3_PRESET_NATIVE,
H3_PRESET_CUSTOM,
)
H3_DRAFT_MP = 0.30
H3_BALANCED_MP = 0.60
H3_CUSTOM_MP_DEFAULT = 0.30
H3_CUSTOM_MP_MIN = 0.10
H3_CUSTOM_MP_MAX = 16.0
H3_CUSTOM_MP_STRONG_WARNING = 2.50
H3_MP_PIXELS = 1_000_000.0
H3_CANVAS_MULTIPLE = 32
H3_NATIVE_SHORT_EDGE = 768
H3_NATIVE_LONG_EDGE_CAP = 1344
H3_MANUAL_WIDTH_DEFAULT = 736
H3_MANUAL_HEIGHT_DEFAULT = 416
H3_CANVAS_MIN = 32
H3_CANVAS_MAX = 16384
@dataclass(frozen=True)
class H3ResolutionPlan:
width: int
height: int
aspect_ratio: float
aspect_source: str
preset: str
target_mp: float | None
warnings: tuple[str, ...]
@property
def actual_mp(self) -> float:
return float(self.width * self.height) / H3_MP_PIXELS
def _round_canvas(value: float) -> int:
units = math.floor(float(value) / H3_CANVAS_MULTIPLE + 0.5)
return max(H3_CANVAS_MULTIPLE, int(units) * H3_CANVAS_MULTIPLE)
def _first_image_aspect(first_frame: Any) -> float | None:
shape = getattr(first_frame, "shape", None)
if shape is None or len(shape) != 4:
return None
height = int(shape[1])
width = int(shape[2])
if width < 1 or height < 1:
return None
return float(width) / float(height)
def _manual_dimension(value: Any, name: str) -> int:
if isinstance(value, bool):
raise TypeError(f"Manual {name} must be an integer")
resolved = int(value)
if float(value) != float(resolved):
raise ValueError(f"Manual {name} must be an integer")
if not H3_CANVAS_MIN <= resolved <= H3_CANVAS_MAX:
raise ValueError(
f"Manual {name} must be between {H3_CANVAS_MIN} and {H3_CANVAS_MAX}"
)
if resolved % H3_CANVAS_MULTIPLE:
raise ValueError(
f"Manual {name} must be aligned to {H3_CANVAS_MULTIPLE} pixels"
)
return resolved
def _manual_resolution(width: Any, height: Any, *, warning: str | None = None) -> H3ResolutionPlan:
resolved_width = _manual_dimension(width, "Width")
resolved_height = _manual_dimension(height, "Height")
return H3ResolutionPlan(
width=resolved_width,
height=resolved_height,
aspect_ratio=float(resolved_width) / float(resolved_height),
aspect_source="Manual Width / Height",
preset="Manual",
target_mp=None,
warnings=(warning,) if warning else (),
)
def resolve_h3_aspect(aspect: str, first_frame: Any = None) -> tuple[float, str]:
"""Resolve the existing Easy aspect policy without creating a second fallback."""
if aspect == H3_ASPECT_AUTO:
image_aspect = _first_image_aspect(first_frame)
if image_aspect is not None:
return image_aspect, "First Image"
return 16.0 / 9.0, "Landscape 16:9 fallback"
if aspect == H3_ASPECT_LANDSCAPE:
return 16.0 / 9.0, H3_ASPECT_LANDSCAPE
if aspect == H3_ASPECT_PORTRAIT:
return 9.0 / 16.0, H3_ASPECT_PORTRAIT
if aspect == H3_ASPECT_SQUARE:
return 1.0, H3_ASPECT_SQUARE
return 16.0 / 9.0, "Landscape 16:9 fallback"
def _area_resolution(aspect_ratio: float, megapixels: float) -> tuple[int, int]:
target_pixels = float(megapixels) * H3_MP_PIXELS
raw_width = math.sqrt(target_pixels * float(aspect_ratio))
raw_height = math.sqrt(target_pixels / float(aspect_ratio))
return _round_canvas(raw_width), _round_canvas(raw_height)
def _native_resolution(aspect_ratio: float) -> tuple[int, int]:
landscape = float(aspect_ratio) >= 1.0
major_ratio = float(aspect_ratio) if landscape else 1.0 / float(aspect_ratio)
short_edge = float(H3_NATIVE_SHORT_EDGE)
long_edge = short_edge * major_ratio
if long_edge > H3_NATIVE_LONG_EDGE_CAP:
long_edge = float(H3_NATIVE_LONG_EDGE_CAP)
short_edge = long_edge / major_ratio
long_px = min(H3_NATIVE_LONG_EDGE_CAP, _round_canvas(long_edge))
short_px = min(H3_NATIVE_SHORT_EDGE, _round_canvas(short_edge))
if landscape:
return long_px, short_px
return short_px, long_px
def resolve_h3_resolution(
*,
aspect: str,
preset: str,
custom_mp: float = H3_CUSTOM_MP_DEFAULT,
first_frame: Any = None,
) -> H3ResolutionPlan:
"""Resolve decimal-MP or Native 768 dimensions on the H3 32-pixel grid."""
aspect_ratio, aspect_source = resolve_h3_aspect(aspect, first_frame)
warnings: list[str] = []
target_mp: float | None
if preset == H3_PRESET_DRAFT:
target_mp = H3_DRAFT_MP
width, height = _area_resolution(aspect_ratio, target_mp)
elif preset == H3_PRESET_BALANCED:
target_mp = H3_BALANCED_MP
width, height = _area_resolution(aspect_ratio, target_mp)
elif preset == H3_PRESET_NATIVE:
target_mp = None
width, height = _native_resolution(aspect_ratio)
else:
preset = H3_PRESET_CUSTOM
if isinstance(custom_mp, bool):
raise TypeError("Custom MP must be numeric")
target_mp = float(custom_mp)
if not H3_CUSTOM_MP_MIN <= target_mp <= H3_CUSTOM_MP_MAX:
raise ValueError(
f"Custom MP must be between {H3_CUSTOM_MP_MIN:.2f} and "
f"{H3_CUSTOM_MP_MAX:.2f}"
)
width, height = _area_resolution(aspect_ratio, target_mp)
native_width, native_height = _native_resolution(aspect_ratio)
native_pixels = native_width * native_height
if width * height > native_pixels:
warnings.append(
"Above Native 768: processing time and memory usage may increase."
)
if target_mp > H3_CUSTOM_MP_STRONG_WARNING:
warnings.append(
"High Custom MP: substantial VRAM use and generation time are possible."
)
return H3ResolutionPlan(
width=width,
height=height,
aspect_ratio=aspect_ratio,
aspect_source=aspect_source,
preset=preset,
target_mp=target_mp,
warnings=tuple(warnings),
)
def resolve_h3_size_source(
*,
size_source: str,
width: Any = H3_MANUAL_WIDTH_DEFAULT,
height: Any = H3_MANUAL_HEIGHT_DEFAULT,
aspect: str = H3_ASPECT_AUTO,
preset: str = H3_PRESET_DRAFT,
custom_mp: float = H3_CUSTOM_MP_DEFAULT,
first_frame: Any = None,
) -> H3ResolutionPlan:
"""Resolve the V3.8 two-choice Size Source while retaining legacy API input."""
if size_source == H3_SIZE_SOURCE_MANUAL:
return _manual_resolution(width, height)
if size_source == H3_SIZE_SOURCE_FIRST_IMAGE:
if _first_image_aspect(first_frame) is None:
return _manual_resolution(
width,
height,
warning=(
"First Image is unavailable for Size Source = First Image; "
"the displayed Manual Width / Height were used."
),
)
return resolve_h3_resolution(
aspect=H3_ASPECT_AUTO,
preset=preset,
custom_mp=custom_mp,
first_frame=first_frame,
)
# Existing API prompts and saved workflows omit size_source. Preserve their
# exact V3.8 aspect resolver until the frontend migrates the visible widgets.
return resolve_h3_resolution(
aspect=aspect,
preset=preset,
custom_mp=custom_mp,
first_frame=first_frame,
)
+812
View File
@@ -0,0 +1,812 @@
"""Pure execution-policy resolvers for V3.8 chunk review.
This module deliberately owns no ComfyUI, torch, filesystem, Run Storage, or
sampling behavior. It turns already-validated storage facts into an execution
decision that later phases can apply to the existing Production engine.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
from ..branch_provenance import TAKE_ACTION_CONTINUE, TAKE_ACTION_USE
GENERATION_MODE_FULL_RUN = "Full Run"
GENERATION_MODE_REVIEW = "Review Each Chunk"
GENERATION_MODE_OPTIONS = (
GENERATION_MODE_FULL_RUN,
GENERATION_MODE_REVIEW,
)
REVIEW_ACTION_CONTINUE = "Continue / Next"
REVIEW_ACTION_REGENERATE_CURRENT = "Regenerate Current"
REVIEW_ACTION_FINISH_REMAINING = "Finish Remaining"
REVIEW_ACTION_OPTIONS = (
REVIEW_ACTION_CONTINUE,
REVIEW_ACTION_REGENERATE_CURRENT,
REVIEW_ACTION_FINISH_REMAINING,
)
RUN_STORAGE_OFF = "Off"
RUN_STORAGE_SAVE_AUTO_RESUME = "Save + Auto Resume"
REVISION_STATUS_IN_PROGRESS = "in_progress"
REVISION_STATUS_INTERRUPTED = "interrupted"
REVISION_STATUS_REVIEW_READY = "review_ready"
REVISION_STATUS_COMPLETE = "complete"
REVISION_STATUSES = (
REVISION_STATUS_IN_PROGRESS,
REVISION_STATUS_INTERRUPTED,
REVISION_STATUS_REVIEW_READY,
REVISION_STATUS_COMPLETE,
)
EXECUTION_MODE_FULL_RUN = "full_run"
EXECUTION_MODE_REVIEW_CONTINUE = "review_continue"
EXECUTION_MODE_REVIEW_REGENERATE = "review_regenerate_current"
EXECUTION_MODE_REVIEW_FINISH = "review_finish_remaining"
EXECUTION_MODE_REVIEW_USE_TAKE = "review_use_take"
EXECUTION_MODE_REVIEW_CONTINUE_FROM_TAKE = "review_continue_from_take"
NONCE_POLICY_INACTIVE = "inactive"
NONCE_POLICY_INHERIT = "inherit_branch"
NONCE_POLICY_ADVANCE = "advance_variation"
NONCE_POLICY_MANUAL = "manual"
REVIEW_PAUSE_REASON = "review_each_chunk"
class ReviewControlError(ValueError):
"""Raised when a requested review policy is structurally ambiguous."""
@dataclass(frozen=True)
class ReviewUnit:
"""One atomic physical sampling group expressed in logical chunk numbers."""
start: int
end: int
physical_group: int
def as_metadata(self) -> dict[str, int]:
return {
"start": self.start,
"end": self.end,
"physical_group": self.physical_group,
}
@dataclass(frozen=True)
class ReviewExecution:
"""Side-effect-free decision consumed by later execution phases."""
execution_mode: str
requires_run_storage: bool
effective_regenerate_from: int
max_new_physical_groups: int | None
next_review_unit_start: int | None
next_review_unit_end: int | None
next_review_physical_group: int | None
finish_remaining: bool
smart_regenerate: bool
one_shot_reset_required: bool
partial_review: bool
projected_prefix_count: int
nonce_policy: str
branch_regenerate_from: int
base_effective_nonce: int
requested_effective_nonce: int | None
status_hint: str
def _integer(value: Any, *, name: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ReviewControlError(f"{name} must be an integer")
if not minimum <= value <= maximum:
raise ReviewControlError(
f"{name} must be between {minimum} and {maximum}"
)
return value
def _manual_regenerate_from(value: Any, *, configured_chunks: int) -> int:
if value is None or value == "Auto" or value == 0:
return 0
if isinstance(value, str) and value.startswith("Chunk "):
try:
value = int(value[6:])
except ValueError as exc:
raise ReviewControlError(
"manual_regenerate_from must be Auto or a valid chunk"
) from exc
return _integer(
value,
name="manual_regenerate_from",
minimum=1,
maximum=configured_chunks,
)
def _terminal_pair_start(
*,
configured_chunks: int,
terminal_merge_enabled: bool,
terminal_pair_start: Any,
) -> int | None:
if type(terminal_merge_enabled) is not bool:
raise ReviewControlError("terminal_merge_enabled must be a boolean")
if not terminal_merge_enabled:
return None
if configured_chunks < 2:
raise ReviewControlError(
"Terminal Merge requires at least two configured chunks"
)
start = _integer(
terminal_pair_start,
name="terminal_pair_start",
minimum=1,
maximum=configured_chunks - 1,
)
if start != configured_chunks - 1:
raise ReviewControlError(
"Terminal Merge must cover the final two configured chunks"
)
return start
def _review_geometry(
*,
configured_chunks: Any,
validated_prefix_count: Any,
terminal_merge_enabled: bool,
terminal_pair_start: Any,
) -> tuple[int, int, int | None]:
chunks = _integer(
configured_chunks,
name="configured_chunks",
minimum=1,
maximum=0x7FFFFFFF,
)
prefix = _integer(
validated_prefix_count,
name="validated_prefix_count",
minimum=0,
maximum=chunks,
)
pair_start = _terminal_pair_start(
configured_chunks=chunks,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=terminal_pair_start,
)
if pair_start is not None and pair_start <= prefix < chunks:
raise ReviewControlError(
"validated prefix ends inside the atomic Terminal Merge pair"
)
return chunks, prefix, pair_start
def resolve_next_review_unit(
*,
configured_chunks: int,
validated_prefix_count: int,
terminal_merge_enabled: bool,
terminal_pair_start: int | None,
) -> ReviewUnit | None:
"""Return the next atomic physical group after a validated logical prefix."""
chunks, prefix, pair_start = _review_geometry(
configured_chunks=configured_chunks,
validated_prefix_count=validated_prefix_count,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=terminal_pair_start,
)
if prefix == chunks:
return None
start = prefix + 1
if pair_start is not None and start == pair_start:
return ReviewUnit(
start=pair_start,
end=chunks,
physical_group=pair_start,
)
return ReviewUnit(start=start, end=start, physical_group=start)
def _review_unit_for_chunk(
chunk: int,
*,
configured_chunks: int,
terminal_pair_start: int | None,
) -> ReviewUnit:
if terminal_pair_start is not None and chunk >= terminal_pair_start:
return ReviewUnit(
start=terminal_pair_start,
end=configured_chunks,
physical_group=terminal_pair_start,
)
return ReviewUnit(start=chunk, end=chunk, physical_group=chunk)
def _validated_review_unit(
value: Any,
*,
configured_chunks: int,
validated_prefix_count: int,
terminal_pair_start: int | None,
) -> ReviewUnit | None:
if value is None:
return None
if isinstance(value, ReviewUnit):
unit = value
elif isinstance(value, Mapping):
try:
unit = ReviewUnit(
start=value["start"],
end=value["end"],
physical_group=value["physical_group"],
)
except KeyError as exc:
raise ReviewControlError(
"latest_review_unit is missing required metadata"
) from exc
else:
raise ReviewControlError(
"latest_review_unit must be review metadata or None"
)
start = _integer(
unit.start,
name="latest_review_unit.start",
minimum=1,
maximum=configured_chunks,
)
end = _integer(
unit.end,
name="latest_review_unit.end",
minimum=start,
maximum=configured_chunks,
)
physical_group = _integer(
unit.physical_group,
name="latest_review_unit.physical_group",
minimum=1,
maximum=configured_chunks,
)
if physical_group != start:
raise ReviewControlError(
"latest_review_unit physical group must equal its first logical chunk"
)
if terminal_pair_start is not None and start == terminal_pair_start:
if end != configured_chunks:
raise ReviewControlError(
"latest_review_unit splits the atomic Terminal Merge pair"
)
elif end != start:
raise ReviewControlError(
"a non-terminal review unit must contain exactly one logical chunk"
)
if terminal_pair_start is not None and start > terminal_pair_start:
raise ReviewControlError(
"latest_review_unit starts inside the atomic Terminal Merge pair"
)
if end != validated_prefix_count:
raise ReviewControlError(
"latest_review_unit is not the head of the validated prefix"
)
return ReviewUnit(start=start, end=end, physical_group=physical_group)
def make_review_pause_metadata(review_unit: ReviewUnit) -> dict[str, Any]:
"""Build non-identity manifest metadata for a successful review pause."""
if not isinstance(review_unit, ReviewUnit):
raise ReviewControlError("review_unit must be a ReviewUnit")
start = _integer(
review_unit.start,
name="review_unit.start",
minimum=1,
maximum=0x7FFFFFFF,
)
end = _integer(
review_unit.end,
name="review_unit.end",
minimum=start,
maximum=0x7FFFFFFF,
)
physical_group = _integer(
review_unit.physical_group,
name="review_unit.physical_group",
minimum=1,
maximum=0x7FFFFFFF,
)
if physical_group != start:
raise ReviewControlError(
"review_unit physical group must equal its first logical chunk"
)
return {
"review_unit": {
"start": start,
"end": end,
"physical_group": physical_group,
},
"review_pause_reason": REVIEW_PAUSE_REASON,
}
def _revision_status(value: Any, *, prefix: int, chunks: int) -> str | None:
if value is None or value == "":
if prefix:
raise ReviewControlError(
"a non-empty validated prefix requires a revision status"
)
return None
status = str(value)
if status not in REVISION_STATUSES:
raise ReviewControlError(f"unknown latest revision status: {status!r}")
if status == REVISION_STATUS_COMPLETE and prefix != chunks:
raise ReviewControlError(
"a complete revision must contain every configured chunk"
)
if status == REVISION_STATUS_REVIEW_READY and not 0 < prefix < chunks:
raise ReviewControlError(
"review_ready requires a non-empty incomplete prefix"
)
return status
def _branch_contract(
*,
latest_effective_nonce: Any,
latest_branch_regenerate_from: Any,
validated_prefix_count: int,
configured_chunks: int,
latest_revision_status: str | None,
) -> tuple[int, int]:
nonce = _integer(
latest_effective_nonce,
name="latest_effective_nonce",
minimum=0,
maximum=0xFFFFFFFF,
)
branch_start = _manual_regenerate_from(
latest_branch_regenerate_from,
configured_chunks=configured_chunks,
)
if nonce == 0 and branch_start != 0:
raise ReviewControlError(
"a regenerate branch requires a positive effective nonce"
)
if nonce > 0 and branch_start == 0:
raise ReviewControlError(
"a positive effective nonce requires its regenerate branch boundary"
)
if branch_start > validated_prefix_count + 1:
raise ReviewControlError(
"regenerate branch boundary skips the validated prefix head"
)
if (
latest_revision_status
in (REVISION_STATUS_REVIEW_READY, REVISION_STATUS_COMPLETE)
and branch_start > validated_prefix_count
):
raise ReviewControlError(
"a ready regenerate branch has not committed its boundary chunk"
)
return nonce, branch_start
def validate_review_prefix_metadata(
*,
configured_chunks: int,
validated_prefix_count: int,
terminal_merge_enabled: bool,
terminal_pair_start: int | None,
latest_review_unit: ReviewUnit | Mapping[str, Any] | None,
latest_revision_status: str | None,
latest_effective_nonce: int,
latest_branch_regenerate_from: Any = 0,
) -> ReviewUnit | None:
"""Validate persisted Review metadata without creating an execution decision.
Repository reads and finalization use this boundary after raw prefix
validation. It deliberately returns no ``ReviewExecution`` so those paths
cannot become a second policy owner.
"""
chunks, prefix, pair_start = _review_geometry(
configured_chunks=configured_chunks,
validated_prefix_count=validated_prefix_count,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=terminal_pair_start,
)
status = _revision_status(
latest_revision_status,
prefix=prefix,
chunks=chunks,
)
unit = _validated_review_unit(
latest_review_unit,
configured_chunks=chunks,
validated_prefix_count=prefix,
terminal_pair_start=pair_start,
)
_branch_contract(
latest_effective_nonce=latest_effective_nonce,
latest_branch_regenerate_from=latest_branch_regenerate_from,
validated_prefix_count=prefix,
configured_chunks=chunks,
latest_revision_status=status,
)
return unit
def _nonce_decision(
*,
manual_regenerate_from: int,
smart_regenerate: bool,
latest_effective_nonce: int,
latest_branch_regenerate_from: int,
smart_target: int = 0,
) -> tuple[str, int, int | None]:
if smart_regenerate:
if latest_effective_nonce >= 0xFFFFFFFF:
raise ReviewControlError(
"effective reroll nonce variation space is exhausted"
)
return (
NONCE_POLICY_ADVANCE,
smart_target,
latest_effective_nonce + 1,
)
if manual_regenerate_from:
return NONCE_POLICY_MANUAL, manual_regenerate_from, None
if latest_effective_nonce:
return (
NONCE_POLICY_INHERIT,
latest_branch_regenerate_from,
latest_effective_nonce,
)
return NONCE_POLICY_INACTIVE, 0, 0
def _unit_status(unit: ReviewUnit, *, chunks: int, verb: str) -> str:
if unit.start == unit.end:
return f"{verb}: Chunk {unit.start} / {chunks}."
return (
f"{verb}: Chunks {unit.start}-{unit.end} / {chunks}; "
"Terminal Merge atomic group."
)
def resolve_review_execution(
*,
generation_mode: str,
review_action: str,
configured_chunks: int,
validated_prefix_count: int,
terminal_merge_enabled: bool,
terminal_pair_start: int | None,
manual_regenerate_from: Any,
run_storage_mode: str,
latest_review_unit: ReviewUnit | Mapping[str, Any] | None,
latest_revision_status: str | None,
latest_effective_nonce: int,
latest_branch_regenerate_from: Any = 0,
) -> ReviewExecution:
"""Resolve V3.8 review intent without touching Production or storage state."""
if generation_mode not in GENERATION_MODE_OPTIONS:
raise ReviewControlError(f"unknown Generation Mode: {generation_mode!r}")
if review_action not in REVIEW_ACTION_OPTIONS:
raise ReviewControlError(f"unknown Review Action: {review_action!r}")
chunks, prefix, pair_start = _review_geometry(
configured_chunks=configured_chunks,
validated_prefix_count=validated_prefix_count,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=terminal_pair_start,
)
status = _revision_status(
latest_revision_status,
prefix=prefix,
chunks=chunks,
)
manual_from = _manual_regenerate_from(
manual_regenerate_from,
configured_chunks=chunks,
)
nonce, existing_branch_start = _branch_contract(
latest_effective_nonce=latest_effective_nonce,
latest_branch_regenerate_from=latest_branch_regenerate_from,
validated_prefix_count=prefix,
configured_chunks=chunks,
latest_revision_status=status,
)
if generation_mode == GENERATION_MODE_FULL_RUN:
nonce_policy, branch_start, requested_nonce = _nonce_decision(
manual_regenerate_from=manual_from,
smart_regenerate=False,
latest_effective_nonce=nonce,
latest_branch_regenerate_from=existing_branch_start,
)
return ReviewExecution(
execution_mode=EXECUTION_MODE_FULL_RUN,
requires_run_storage=False,
effective_regenerate_from=(manual_from or branch_start),
max_new_physical_groups=None,
next_review_unit_start=None,
next_review_unit_end=None,
next_review_physical_group=None,
finish_remaining=False,
smart_regenerate=False,
one_shot_reset_required=False,
partial_review=False,
projected_prefix_count=chunks,
nonce_policy=nonce_policy,
branch_regenerate_from=branch_start,
base_effective_nonce=nonce,
requested_effective_nonce=requested_nonce,
status_hint="Full Run: Review execution limit disabled.",
)
if run_storage_mode != RUN_STORAGE_SAVE_AUTO_RESUME:
raise ReviewControlError(
"Review Each Chunk requires Run Storage = Save + Auto Resume."
)
latest_unit = _validated_review_unit(
latest_review_unit,
configured_chunks=chunks,
validated_prefix_count=prefix,
terminal_pair_start=pair_start,
)
if review_action == REVIEW_ACTION_REGENERATE_CURRENT:
if manual_from:
raise ReviewControlError(
"Regenerate Current cannot be combined with manual Regenerate From. "
"Set Regenerate From to Auto."
)
if prefix == 0:
raise ReviewControlError(
"Regenerate Current requires an existing reviewed unit"
)
if latest_unit is None:
raise ReviewControlError(
"Regenerate Current requires latest review_unit metadata"
)
if status not in (
REVISION_STATUS_REVIEW_READY,
REVISION_STATUS_COMPLETE,
):
raise ReviewControlError(
"Regenerate Current requires a review_ready revision or a complete "
"revision with review_unit metadata"
)
nonce_policy, branch_start, requested_nonce = _nonce_decision(
manual_regenerate_from=0,
smart_regenerate=True,
latest_effective_nonce=nonce,
latest_branch_regenerate_from=existing_branch_start,
smart_target=latest_unit.start,
)
projected = latest_unit.end
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_REGENERATE,
requires_run_storage=True,
effective_regenerate_from=latest_unit.start,
max_new_physical_groups=1,
next_review_unit_start=latest_unit.start,
next_review_unit_end=latest_unit.end,
next_review_physical_group=latest_unit.physical_group,
finish_remaining=False,
smart_regenerate=True,
one_shot_reset_required=True,
partial_review=projected < chunks,
projected_prefix_count=projected,
nonce_policy=nonce_policy,
branch_regenerate_from=branch_start,
base_effective_nonce=nonce,
requested_effective_nonce=requested_nonce,
status_hint=_unit_status(
latest_unit,
chunks=chunks,
verb="Smart Regenerate",
),
)
nonce_policy, branch_start, requested_nonce = _nonce_decision(
manual_regenerate_from=manual_from,
smart_regenerate=False,
latest_effective_nonce=nonce,
latest_branch_regenerate_from=existing_branch_start,
)
if review_action == REVIEW_ACTION_FINISH_REMAINING:
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_FINISH,
requires_run_storage=True,
effective_regenerate_from=(manual_from or branch_start),
max_new_physical_groups=None,
next_review_unit_start=None,
next_review_unit_end=None,
next_review_physical_group=None,
finish_remaining=True,
smart_regenerate=False,
one_shot_reset_required=True,
partial_review=False,
projected_prefix_count=chunks,
nonce_policy=nonce_policy,
branch_regenerate_from=branch_start,
base_effective_nonce=nonce,
requested_effective_nonce=requested_nonce,
status_hint=(
f"Review Mode: finish remaining after accepted prefix "
f"{prefix} / {chunks}."
),
)
if manual_from:
next_unit = _review_unit_for_chunk(
manual_from,
configured_chunks=chunks,
terminal_pair_start=pair_start,
)
else:
next_unit = resolve_next_review_unit(
configured_chunks=chunks,
validated_prefix_count=prefix,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=pair_start,
)
if next_unit is None:
projected = chunks
status_hint = "Review Mode: Sequence complete."
unit_start = unit_end = physical_group = None
else:
projected = next_unit.end
status_hint = _unit_status(
next_unit,
chunks=chunks,
verb="Review next unit",
)
unit_start = next_unit.start
unit_end = next_unit.end
physical_group = next_unit.physical_group
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_CONTINUE,
requires_run_storage=True,
effective_regenerate_from=(manual_from or branch_start),
max_new_physical_groups=1,
next_review_unit_start=unit_start,
next_review_unit_end=unit_end,
next_review_physical_group=physical_group,
finish_remaining=False,
smart_regenerate=False,
one_shot_reset_required=False,
partial_review=projected < chunks,
projected_prefix_count=projected,
nonce_policy=nonce_policy,
branch_regenerate_from=branch_start,
base_effective_nonce=nonce,
requested_effective_nonce=requested_nonce,
status_hint=status_hint,
)
def resolve_take_execution(
*,
take_action: str,
configured_chunks: int,
selected_prefix_count: int,
terminal_merge_enabled: bool,
terminal_pair_start: int | None,
selected_review_unit: ReviewUnit | Mapping[str, Any],
selected_effective_nonce: int,
selected_branch_regenerate_from: int,
next_effective_nonce: int,
) -> ReviewExecution:
"""Resolve an already-validated immutable Take selection.
The selected prefix is a Run Storage fact. Continuing from it starts a
fresh downstream nonce boundary without changing the Sampling Contract.
"""
if take_action not in (TAKE_ACTION_USE, TAKE_ACTION_CONTINUE):
raise ReviewControlError(f"unknown Take action: {take_action!r}")
chunks, prefix, pair_start = _review_geometry(
configured_chunks=configured_chunks,
validated_prefix_count=selected_prefix_count,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=terminal_pair_start,
)
unit = _validated_review_unit(
selected_review_unit,
configured_chunks=chunks,
validated_prefix_count=prefix,
terminal_pair_start=pair_start,
)
if unit is None:
raise ReviewControlError("selected Take is missing its physical review unit")
selected_nonce, selected_branch = _branch_contract(
latest_effective_nonce=selected_effective_nonce,
latest_branch_regenerate_from=selected_branch_regenerate_from,
validated_prefix_count=prefix,
configured_chunks=chunks,
latest_revision_status=(
REVISION_STATUS_COMPLETE if prefix == chunks else REVISION_STATUS_REVIEW_READY
),
)
if take_action == TAKE_ACTION_USE:
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_USE_TAKE,
requires_run_storage=True,
effective_regenerate_from=selected_branch,
max_new_physical_groups=0,
next_review_unit_start=None,
next_review_unit_end=None,
next_review_physical_group=None,
finish_remaining=False,
smart_regenerate=False,
one_shot_reset_required=True,
partial_review=prefix < chunks,
projected_prefix_count=prefix,
nonce_policy=NONCE_POLICY_INHERIT if selected_nonce else NONCE_POLICY_INACTIVE,
branch_regenerate_from=selected_branch,
base_effective_nonce=selected_nonce,
requested_effective_nonce=selected_nonce,
status_hint=_unit_status(unit, chunks=chunks, verb="Use This Take"),
)
next_unit = resolve_next_review_unit(
configured_chunks=chunks,
validated_prefix_count=prefix,
terminal_merge_enabled=terminal_merge_enabled,
terminal_pair_start=pair_start,
)
if next_unit is None:
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_USE_TAKE,
requires_run_storage=True,
effective_regenerate_from=selected_branch,
max_new_physical_groups=0,
next_review_unit_start=None,
next_review_unit_end=None,
next_review_physical_group=None,
finish_remaining=False,
smart_regenerate=False,
one_shot_reset_required=True,
partial_review=False,
projected_prefix_count=chunks,
nonce_policy=NONCE_POLICY_INHERIT if selected_nonce else NONCE_POLICY_INACTIVE,
branch_regenerate_from=selected_branch,
base_effective_nonce=selected_nonce,
requested_effective_nonce=selected_nonce,
status_hint="Continue From Here: selected Take is already complete.",
)
nonce = _integer(
next_effective_nonce,
name="next_effective_nonce",
minimum=1,
maximum=0xFFFFFFFF,
)
return ReviewExecution(
execution_mode=EXECUTION_MODE_REVIEW_CONTINUE_FROM_TAKE,
requires_run_storage=True,
effective_regenerate_from=next_unit.start,
max_new_physical_groups=1,
next_review_unit_start=next_unit.start,
next_review_unit_end=next_unit.end,
next_review_physical_group=next_unit.physical_group,
finish_remaining=False,
smart_regenerate=False,
one_shot_reset_required=True,
partial_review=next_unit.end < chunks,
projected_prefix_count=next_unit.end,
nonce_policy=NONCE_POLICY_ADVANCE,
branch_regenerate_from=next_unit.start,
base_effective_nonce=selected_nonce,
requested_effective_nonce=nonce,
status_hint=_unit_status(next_unit, chunks=chunks, verb="Continue From Here"),
)
+223
View File
@@ -0,0 +1,223 @@
"""Internal owner of Continuum runtime continuation-source priority.
The pure execution planner receives only the source selected here. Runtime
payload validation remains outside the planner so Run Storage, explicit
Session, and initial State can never become competing planner inputs.
"""
from __future__ import annotations
from dataclasses import dataclass
import logging
from typing import Any
from ..branch_provenance import physical_groups
from ..run_storage import RunStorageError
from ..state import validate_state
from ..v2.session import SessionValidationError, entry_to_state
from .planning_types import ContinuationSourceFacts
LOG = logging.getLogger("h3_continuum_join")
class RuntimeCoordinatorError(ValueError):
"""Raised when the characterized runtime source contract is invalid."""
@dataclass(slots=True)
class RuntimeContinuationSelection:
"""Selected runtime payload plus its single planning projection."""
entries: list[dict[str, Any]]
previous_state: dict[str, Any] | None
continuation_source_facts: ContinuationSourceFacts
notes: tuple[str, ...]
def project_selected_continuation_source(
*,
kind: str,
selected_source: dict[str, Any] | None,
) -> ContinuationSourceFacts:
"""Project one already-selected runtime source into planning-only facts."""
normalized_kind = str(kind)
if normalized_kind == "none":
if selected_source is not None:
raise RuntimeCoordinatorError("the none continuation source has no payload")
return ContinuationSourceFacts(
kind="none",
accepted_chunks=0,
reroll_from_chunk=0,
physical_group_boundaries=(),
)
if normalized_kind not in {
"run_storage",
"explicit_session",
"initial_state",
}:
raise RuntimeCoordinatorError(
f"unknown selected continuation source: {normalized_kind!r}"
)
if not isinstance(selected_source, dict):
raise RuntimeCoordinatorError(
"the selected continuation source requires normalized runtime facts"
)
try:
accepted_chunks = int(selected_source["accepted_chunks"])
reroll_from_chunk = int(selected_source["reroll_from_chunk"])
boundaries = tuple(
(int(start), int(end))
for start, end in selected_source["physical_group_boundaries"]
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeCoordinatorError(
"selected continuation source facts are invalid"
) from exc
return ContinuationSourceFacts(
kind=normalized_kind,
accepted_chunks=accepted_chunks,
reroll_from_chunk=reroll_from_chunk,
physical_group_boundaries=boundaries,
)
class InternalRuntimeCoordinator:
"""Own the established Storage / Session / State priority contract."""
def __init__(
self,
*,
storage_controller: Any,
input_session: dict[str, Any] | None,
initial_state: dict[str, Any] | None,
) -> None:
self.storage_controller = storage_controller
self.input_session = input_session
self.initial_state = initial_state
self.session_present = input_session is not None
@property
def storage_enabled(self) -> bool:
return self.storage_controller is not None
def prepare_inputs(
self,
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
"""Suppress State as soon as an explicit Session is present."""
if self.session_present and self.initial_state is not None:
LOG.warning(
"Both session and initial_state were supplied; using the session "
"and ignoring initial_state"
)
self.initial_state = None
return self.input_session, self.initial_state
def assert_storage_session_compatible(self) -> None:
"""Preserve the existing Run Storage plus explicit Session hard error."""
if self.storage_enabled and self.session_present:
raise RunStorageError(
"Run Storage cannot be combined with an explicit Session"
)
def validate_initial_state_reroll(self, reroll_from_chunk: int) -> None:
"""Apply the existing reroll restriction to the effective State only."""
if self.initial_state is not None and int(reroll_from_chunk) not in (0, 1):
raise RuntimeCoordinatorError(
"with initial_state, reroll_from_chunk can only be 0 or 1"
)
def select_continuation_source(
self,
*,
preserved: list[dict[str, Any]],
initial_state: dict[str, Any] | None,
chunks: int,
width: int,
height: int,
reroll_from_chunk: int,
effective_reroll_from_chunk: int,
terminal_merge_enabled: bool,
) -> RuntimeContinuationSelection:
"""Select exactly one runtime continuation source, then project it."""
entries = preserved[:]
previous_state = None
notes: list[str] = []
if entries:
try:
previous_state = entry_to_state(entries[-1])
except (SessionValidationError, ValueError) as exc:
notes.append(
"saved continuation state was rejected; generated a fresh run "
f"({exc})"
)
entries = []
previous_state = None
# An initial State is considered only if no usable Storage/Session prefix
# survived. Explicit Session presence already suppressed it in
# prepare_inputs(), before Session validity was known.
if not entries and initial_state is not None:
try:
candidate = validate_state(initial_state)
if int(candidate["width"]) != int(width) or int(
candidate["height"]
) != int(height):
notes.append(
"initial_state resolution differs; generated a fresh run"
)
else:
previous_state = candidate
except ValueError as exc:
notes.append(
f"initial_state was rejected; generated a fresh run ({exc})"
)
if entries:
selected_kind = (
"run_storage" if self.storage_enabled else "explicit_session"
)
source_reroll = (
int(reroll_from_chunk)
if self.storage_enabled
else int(effective_reroll_from_chunk)
)
accepted_boundaries = tuple(
(int(group.start), int(group.end))
for group in physical_groups(
chunks=int(chunks),
terminal_merge_enabled=bool(terminal_merge_enabled),
)
if int(group.end) <= len(entries)
)
selected_source = {
"accepted_chunks": len(entries),
"reroll_from_chunk": source_reroll,
"physical_group_boundaries": accepted_boundaries,
}
elif previous_state is not None and initial_state is not None:
selected_kind = "initial_state"
selected_source = {
"accepted_chunks": 0,
"reroll_from_chunk": int(reroll_from_chunk),
"physical_group_boundaries": (),
}
else:
selected_kind = "none"
selected_source = None
return RuntimeContinuationSelection(
entries=entries,
previous_state=previous_state,
continuation_source_facts=project_selected_continuation_source(
kind=selected_kind,
selected_source=selected_source,
),
notes=tuple(notes),
)
+71
View File
@@ -0,0 +1,71 @@
"""One-physical-group Sampling Engine for the Continuum runtime."""
from __future__ import annotations
from typing import Any, Callable
def sample_physical_group(
*,
sample_callable: Callable[..., Any],
model: Any,
conditioning: Any,
latent: Any,
sampler: Any,
sigmas: Any,
seed: int,
enable_preview: bool,
physical_group: int,
logical_chunks: tuple[int, ...],
terminal_atomic: bool,
layout_validation_profiler: Any = None,
packed_row_planner: Any = None,
planner_event: Any = None,
diagnostic_policy: Any = None,
context_frames: int = 0,
) -> tuple[Any, Any]:
"""Run exactly one physical sample with unchanged observer ordering."""
normalized_chunks = tuple(int(value) for value in logical_chunks)
layout_profile_token = (
layout_validation_profiler.begin_sampling_group(
physical_group=int(physical_group),
logical_chunks=normalized_chunks,
sampling_steps=max(0, int(sigmas.shape[-1]) - 1),
)
if layout_validation_profiler is not None
else None
)
diagnostic_sample_token = None
if diagnostic_policy is not None:
diagnostic_sample_token = diagnostic_policy.before_sampling(
physical_group=int(physical_group),
logical_chunks=normalized_chunks,
context_frames=int(context_frames),
latent=latent,
conditioning=conditioning,
seed=int(seed),
sigmas=sigmas,
)
if planner_event is not None:
planner_event(
packed_row_planner,
"begin_group",
physical_group=int(physical_group),
logical_chunks=normalized_chunks,
terminal_atomic=bool(terminal_atomic),
)
sampled = sample_callable(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
)
if planner_event is not None:
planner_event(packed_row_planner, "finish_group")
if layout_validation_profiler is not None and layout_profile_token is not None:
layout_validation_profiler.finish_sampling_group(layout_profile_token)
return sampled, diagnostic_sample_token
+644
View File
@@ -0,0 +1,644 @@
"""V3.5 Second Pass contracts and the physical-group sampling runtime."""
from __future__ import annotations
import copy
import hashlib
from collections.abc import Mapping, Sequence
from typing import Any
import torch
from ..model_patch import clone_model_for_chunk
from ..state import extract_av_streams
from ..v2.h3_builder import encode_prompt_conditioning
from ..v2.sampling import latent_from_cpu
from .plan import SECOND_PASS_CONTRACT_VERSION
from .refine_context import (
MAGIC as REFINE_CONTEXT_MAGIC,
OUTPUT_SCOPE_KEY,
RefineConditioningAdaptationError,
RefineContextError,
adapt_group_conditioning,
validate_refine_context,
)
from .refine_schedule import (
RefineSchedule,
RefineScheduleError,
resolve_refine_schedule,
serializable_schedule_contract,
)
from .refine_sampling import sample_refine_chunk
class SecondPassContractError(ValueError):
"""Raised only when a Second Pass structural contract is impossible."""
def derive_refine_seed(refine_seed: int, physical_group_index: int) -> int:
"""Derive one deterministic seed per physical group in an independent namespace."""
if type(refine_seed) is not int or refine_seed < 0:
raise SecondPassContractError("refine_seed must be a non-negative integer")
if type(physical_group_index) is not int or physical_group_index < 0:
raise SecondPassContractError("physical_group_index must be a non-negative integer")
payload = f"h3-continuum-refine-v1:{refine_seed}:{physical_group_index}".encode("ascii")
return int.from_bytes(hashlib.blake2b(payload, digest_size=8).digest(), "big")
def _latent_samples(value: Any, *, name: str) -> torch.Tensor:
if not isinstance(value, Mapping) or "samples" not in value:
raise SecondPassContractError(f"{name} must be a LATENT mapping with samples")
samples = value["samples"]
if not isinstance(samples, torch.Tensor):
raise SecondPassContractError(f"{name}.samples must be a tensor")
return samples
def _require_finite_samples(samples: torch.Tensor, *, name: str) -> None:
"""Reject unusable processor output before it reaches Second Pass Sampling."""
if not bool(torch.isfinite(samples).all().item()):
raise SecondPassContractError(f"{name}.samples contains NaN or Inf")
def _contract_groups(assembly_plan: Mapping[str, Any]) -> list[Mapping[str, Any]]:
if not isinstance(assembly_plan, Mapping):
raise SecondPassContractError("assembly_plan must be a mapping")
contract = assembly_plan.get("second_pass_contract")
if not isinstance(contract, Mapping):
raise SecondPassContractError("assembly_plan has no second_pass_contract")
if contract.get("version") != SECOND_PASS_CONTRACT_VERSION:
raise SecondPassContractError("unsupported second_pass_contract version")
groups = contract.get("physical_groups")
if not isinstance(groups, list) or not groups or not all(isinstance(g, Mapping) for g in groups):
raise SecondPassContractError("second_pass_contract.physical_groups must be non-empty")
return groups
def _validate_video_groups(
video_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
) -> tuple[list[Mapping[str, Any]], tuple[int, int]]:
groups = _contract_groups(assembly_plan)
if not isinstance(video_latents, Sequence) or isinstance(video_latents, (str, bytes)):
raise SecondPassContractError("video_latents must be a sequence")
if len(video_latents) != len(groups):
raise SecondPassContractError("video latent group count changed")
target_geometry: tuple[int, int] | None = None
for position, (latent, group) in enumerate(zip(video_latents, groups, strict=True)):
if group.get("group_id") != position:
raise SecondPassContractError("physical group order or group_id changed")
samples = _latent_samples(latent, name=f"video_latents[{position}]")
if samples.ndim != 5:
raise SecondPassContractError("video latent must have shape [B,C,T,H,W]")
batch, channels, temporal, height, width = (int(value) for value in samples.shape)
if channels != 24:
raise SecondPassContractError("MiniMax H3 video latent must have 24 channels")
if batch != int(group.get("source_batch", -1)):
raise SecondPassContractError("video latent batch changed")
if temporal != int(group.get("source_latent_t", -1)):
raise SecondPassContractError("video latent temporal length changed")
source_h = int(group.get("source_latent_h", -1))
source_w = int(group.get("source_latent_w", -1))
if source_h < 1 or source_w < 1 or height < source_h or width < source_w:
raise SecondPassContractError("video latent spatial size must be preserved or enlarged")
_require_finite_samples(samples, name=f"video_latents[{position}]")
if target_geometry is None:
target_geometry = (height, width)
elif target_geometry != (height, width):
raise SecondPassContractError("all physical groups must use the same target H/W")
assert target_geometry is not None
return groups, target_geometry
def validate_second_pass_inputs(
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
) -> dict[str, Any]:
"""Validate only structural invariants required by a future Second Pass node."""
groups, target_geometry = _validate_video_groups(video_latents, assembly_plan)
if not isinstance(audio_latents, Sequence) or isinstance(audio_latents, (str, bytes)):
raise SecondPassContractError("audio_latents must be a sequence")
if len(audio_latents) != len(groups):
raise SecondPassContractError("audio latent group count changed")
for position, (latent, group) in enumerate(zip(audio_latents, groups, strict=True)):
samples = _latent_samples(latent, name=f"audio_latents[{position}]")
expected_shape = group.get("source_audio_shape")
if not isinstance(expected_shape, list) or tuple(expected_shape) != tuple(samples.shape):
raise SecondPassContractError("audio latent shape or physical group order changed")
_require_finite_samples(samples, name=f"audio_latents[{position}]")
return {
"physical_group_count": len(groups),
"target_latent_h": target_geometry[0],
"target_latent_w": target_geometry[1],
"audio_passthrough": True,
}
def passthrough_audio_latents(audio_latents: Sequence[Any]) -> list[Any]:
"""Return original LATENT objects; temporary refined audio is not adopted in V1."""
return list(audio_latents)
def _normalize_selected_group_indices(
selected_group_indices: Sequence[int] | None,
physical_group_count: int,
) -> tuple[tuple[int, ...], bool]:
"""Validate an optional internal physical-group selection."""
if selected_group_indices is None:
return tuple(range(physical_group_count)), False
if (
not isinstance(selected_group_indices, Sequence)
or isinstance(selected_group_indices, (str, bytes))
or not selected_group_indices
):
raise SecondPassContractError(
"selected physical groups must be a non-empty integer sequence"
)
selected = tuple(selected_group_indices)
if any(type(index) is not int for index in selected):
raise SecondPassContractError(
"selected physical group indices must be integers"
)
if len(set(selected)) != len(selected):
raise SecondPassContractError("selected physical groups contain duplicates")
if tuple(sorted(selected)) != selected:
raise SecondPassContractError(
"selected physical groups must follow physical execution order"
)
if selected[0] < 0 or selected[-1] >= physical_group_count:
raise SecondPassContractError("selected physical group index is out of range")
return selected, True
def update_second_pass_geometry(
assembly_plan: Mapping[str, Any],
video_latents: Sequence[Any],
) -> dict[str, Any]:
"""Return a copied plan whose output geometry follows the upscaled physical latents."""
groups, (target_h, target_w) = _validate_video_groups(video_latents, assembly_plan)
scale_h: int | None = None
scale_w: int | None = None
for group in groups:
source_h = int(group["source_latent_h"])
source_w = int(group["source_latent_w"])
source_height = int(group["source_height"])
source_width = int(group["source_width"])
if source_height % source_h or source_width % source_w:
raise SecondPassContractError("source pixel/latent geometry is not integral")
group_scale_h = source_height // source_h
group_scale_w = source_width // source_w
if scale_h is None:
scale_h, scale_w = group_scale_h, group_scale_w
elif (scale_h, scale_w) != (group_scale_h, group_scale_w):
raise SecondPassContractError("source geometry scale differs between groups")
assert scale_h is not None and scale_w is not None
target_height = target_h * scale_h
target_width = target_w * scale_w
updated = copy.deepcopy(dict(assembly_plan))
updated["width"] = target_width
updated["height"] = target_height
contract = updated["second_pass_contract"]
contract["target_width"] = target_width
contract["target_height"] = target_height
for group in contract["physical_groups"]:
group["target_latent_h"] = target_h
group["target_latent_w"] = target_w
group["target_width"] = target_width
group["target_height"] = target_height
return updated
def prepare_physical_refine_groups(
*,
model: Any,
clip: Any,
video_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
refine_context: Any = None,
video_vae: Any = None,
conditioning_upscale_method: str = "bilinear",
encode_prompt_fn=None,
clone_model_fn=None,
validate_refine_context_fn=None,
adapt_group_conditioning_fn=None,
group_consumer_fn=None,
retain_group_outputs: bool = True,
selected_group_indices: Sequence[int] | None = None,
) -> dict[str, Any]:
"""Prepare one MODEL and one complete CONDITIONING per physical group.
This is the shared, sampling-independent contract used by the built-in
Second Pass and the public Advanced Conditioning Bridge. The returned
``conditioning`` value is an outer physical-group list; every item inside
it remains one complete ComfyUI CONDITIONING object.
"""
groups, (target_h, target_w) = _validate_video_groups(
video_latents,
assembly_plan,
)
selected_indices, selection_active = _normalize_selected_group_indices(
selected_group_indices,
len(groups),
)
selected_index_set = set(selected_indices)
encode_prompt_fn = encode_prompt_fn or encode_prompt_conditioning
clone_model_fn = clone_model_fn or clone_model_for_chunk
validate_refine_context_fn = validate_refine_context_fn or validate_refine_context
adapt_group_conditioning_fn = (
adapt_group_conditioning_fn or adapt_group_conditioning
)
captured_groups: Sequence[Mapping[str, Any]] | None = None
warnings: list[str] = []
if refine_context is None:
warnings.append(
"WARNING: refine_context is not connected; using legacy prompt-only "
"conditioning."
)
else:
typed_context = (
isinstance(refine_context, Mapping)
and refine_context.get("magic") == REFINE_CONTEXT_MAGIC
)
try:
validated_context = validate_refine_context_fn(
refine_context,
assembly_plan=assembly_plan,
)
except RefineContextError as exc:
if typed_context:
raise SecondPassContractError(
f"refine_context contract is invalid: {exc}"
) from exc
warnings.append(
"WARNING: optional refine_context was not a typed Continuum context; "
f"using legacy prompt-only conditioning ({exc})."
)
else:
if (bool(validated_context.get("complete"))
or validated_context.get(OUTPUT_SCOPE_KEY) is not None):
captured_groups = validated_context["groups"]
else:
warnings.append(
"WARNING: refine_context capture is incomplete; using legacy "
"prompt-only conditioning."
)
warnings.extend(str(note) for note in validated_context.get("notes", ())
if str(note).startswith("Output scope unavailable:"))
group_models: list[Any] = []
group_conditioning: list[list[Any]] = []
details: list[dict[str, Any]] = []
for group_index, group in enumerate(groups):
if group_index not in selected_index_set:
continue
captured_group = (
captured_groups[group_index] if captured_groups is not None else None
)
model_group = captured_group if captured_group is not None else group
adaptation_stats: Mapping[str, Any] | None = None
context_conditioning_available = False
if captured_group is not None:
try:
conditioning, adaptation_stats = adapt_group_conditioning_fn(
captured_group,
target_latent_h=target_h,
target_latent_w=target_w,
video_vae=video_vae,
upscale_method=str(conditioning_upscale_method),
)
conditioning_source = "refine_context"
context_conditioning_available = True
except RefineConditioningAdaptationError as exc:
warnings.append(
f"WARNING: group {group_index + 1} refine_context target "
f"adaptation was unavailable; using legacy prompt-only "
f"conditioning ({exc})."
)
except RefineContextError as exc:
raise SecondPassContractError(
f"refine_context group {group_index} is invalid: {exc}"
) from exc
if not context_conditioning_available:
prompt = str(group.get("physical_prompt", ""))
conditioning = encode_prompt_fn(
clip,
prompt,
first_image=None,
last_image=None,
reference_audio_assets=None,
timeline_video_assets=None,
)
conditioning_source = "prompt_only_fallback"
if not isinstance(conditioning, list):
raise SecondPassContractError(
f"physical group {group_index} conditioning is not a complete "
"ComfyUI CONDITIONING object"
)
logical_chunks = model_group.get("logical_chunks")
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
logical_chunks = group.get("logical_chunks")
if not isinstance(logical_chunks, Sequence) or not logical_chunks:
raise SecondPassContractError(
f"physical group {group_index} has no logical chunk identity"
)
physical_clip_index = int(
model_group.get("physical_clip_index", logical_chunks[0])
)
context_frames = int(
model_group.get(
"context_frames",
group.get("trim_prefix_frames", 0),
)
)
if physical_clip_index <= 0 or context_frames < 0:
raise SecondPassContractError(
f"physical group {group_index} MODEL context is invalid"
)
chunk_model = clone_model_fn(
model,
strict=False,
debug=False,
chunk_index=physical_clip_index,
context_frames=context_frames if context_frames > 0 else None,
)
detail = {
"group_index": group_index,
"group": group,
"conditioning_source": conditioning_source,
"adaptation_stats": adaptation_stats,
"physical_clip_index": physical_clip_index,
"context_frames": context_frames,
"reference_routing_v1": (
captured_group.get("reference_routing_v1")
if captured_group is not None else None
),
}
if group_consumer_fn is not None:
group_consumer_fn(
group_index,
video_latents[group_index],
group,
chunk_model,
conditioning,
detail,
)
if retain_group_outputs:
group_models.append(chunk_model)
# Append, never extend: CONDITIONING's internal entries belong together.
group_conditioning.append(conditioning)
details.append(detail)
physical_group_count = len(groups)
selected_group_count = len(selected_indices)
if retain_group_outputs and not (
len(group_models)
== len(group_conditioning)
== len(details)
== selected_group_count
):
raise SecondPassContractError(
"prepared MODEL/CONDITIONING physical group counts differ"
)
if not retain_group_outputs and group_consumer_fn is None:
raise SecondPassContractError(
"non-retained physical group preparation requires a consumer"
)
return {
"group_models": group_models,
"conditioning": group_conditioning,
"updated_assembly_plan": update_second_pass_geometry(
assembly_plan,
video_latents,
),
"details": details,
"warnings": warnings,
"physical_group_count": physical_group_count,
"selected_group_count": selected_group_count,
"selected_group_indices": list(selected_indices),
"selection_active": selection_active,
"target_latent_h": target_h,
"target_latent_w": target_w,
}
def run_second_pass_groups(
*,
model: Any,
clip: Any,
sampler: Any,
sigmas: torch.Tensor,
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
refine_seed: int,
refine_context: Any = None,
video_vae: Any = None,
conditioning_upscale_method: str = "bilinear",
enable_preview: bool = True,
encode_prompt_fn=None,
latent_builder=None,
sample_fn=None,
stream_extractor=None,
clone_model_fn=None,
validate_refine_context_fn=None,
adapt_group_conditioning_fn=None,
refine_schedule: RefineSchedule | None = None,
selected_group_indices: Sequence[int] | None = None,
temporal_windows_by_group: Mapping[int, Any] | None = None,
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
"""Refine complete physical groups while preserving first-pass audio exactly."""
validation = validate_second_pass_inputs(video_latents, audio_latents, assembly_plan)
try:
resolved_schedule = resolve_refine_schedule(sigmas, refine_schedule)
except RefineScheduleError as exc:
raise SecondPassContractError(f"refine schedule is invalid: {exc}") from exc
effective_sigmas = resolved_schedule.sigmas
schedule_contract = serializable_schedule_contract(resolved_schedule)
groups = _contract_groups(assembly_plan)
selected_indices, selection_active = _normalize_selected_group_indices(
selected_group_indices,
len(groups),
)
latent_builder = latent_builder or latent_from_cpu
sample_fn = sample_fn or sample_refine_chunk
stream_extractor = stream_extractor or extract_av_streams
refined_videos: list[dict[str, Any]] = (
list(video_latents) if selection_active else []
)
group_seeds: list[int] = []
conditioning_sources: list[str] = []
reference_inheritance: list[dict[str, Any]] = []
group_report_lines: list[str] = []
routed_first_pass = isinstance(assembly_plan.get("reference_routing_v1"), Mapping)
def sample_prepared_group(
group_index,
video_latent,
group,
chunk_model,
conditioning,
detail,
):
audio_latent = audio_latents[group_index]
video_samples = _latent_samples(video_latent, name=f"video_latents[{group_index}]")
audio_samples = _latent_samples(audio_latent, name=f"audio_latents[{group_index}]")
conditioning_source = str(detail["conditioning_source"])
adaptation_stats = detail["adaptation_stats"]
physical_clip_index = int(detail["physical_clip_index"])
context_frames = int(detail["context_frames"])
physical_seed = derive_refine_seed(int(refine_seed), group_index)
group_seeds.append(physical_seed)
conditioning_sources.append(conditioning_source)
route_record = detail.get("reference_routing_v1")
inheritance = (
"verified" if conditioning_source == "refine_context" and route_record is not None
else "not_inherited" if routed_first_pass
else "not_applicable"
)
reference_inheritance.append({
"physical_group": group_index + 1,
"status": inheritance,
"contract_sha256": route_record.get("sha256") if route_record is not None else None,
})
nested_latent = latent_builder(video_samples, audio_samples)
sample_arguments = dict(
model=chunk_model,
conditioning=conditioning,
latent=nested_latent,
sampler=sampler,
sigmas=effective_sigmas,
seed=physical_seed,
enable_preview=bool(enable_preview),
)
if temporal_windows_by_group is not None:
temporal_window = temporal_windows_by_group.get(group_index)
if temporal_window is None:
raise SecondPassContractError(
f"selected physical group {group_index + 1} has no temporal window"
)
sample_arguments["temporal_window"] = temporal_window
sampled = sample_fn(**sample_arguments)
refined_video, _temporary_audio = stream_extractor(sampled)
if tuple(refined_video.shape) != tuple(video_samples.shape):
raise SecondPassContractError(
f"refined video group {group_index} changed B/C/T/H/W geometry"
)
if not bool(torch.isfinite(refined_video.float()).all().item()):
raise SecondPassContractError(
f"refined video group {group_index} contains NaN or Inf"
)
output_latent = dict(video_latent)
output_latent["samples"] = refined_video
if selection_active:
refined_videos[group_index] = output_latent
else:
refined_videos.append(output_latent)
group_report_lines.append(
"group "
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
f"conditioning_source={conditioning_source}, "
f"reference_inheritance={inheritance}, "
f"physical_clip_index={physical_clip_index}, "
f"context_frames={context_frames}, "
f"refine_seed={physical_seed}, shape={tuple(refined_video.shape)}, "
"sampling_passes=1, temporary_audio_discarded=true"
)
if adaptation_stats is not None:
group_report_lines.append(
"group "
f"{group_index + 1} target conditioning: "
f"keyframes_resized={int(adaptation_stats.get('keyframes_resized', 0))}, "
f"keyframes_reencoded={int(adaptation_stats.get('keyframes_reencoded', 0))}, "
f"guide_keyframes_reencoded={int(adaptation_stats.get('guide_keyframes_reencoded', 0))}, "
f"context_refs_resized={int(adaptation_stats.get('context_refs_resized', 0))}"
)
group_report_lines.extend(
f"WARNING: {warning}"
for warning in adaptation_stats.get("warnings", ())
)
prepared = prepare_physical_refine_groups(
model=model,
clip=clip,
video_latents=video_latents,
assembly_plan=assembly_plan,
refine_context=refine_context,
video_vae=video_vae,
conditioning_upscale_method=conditioning_upscale_method,
encode_prompt_fn=encode_prompt_fn,
clone_model_fn=clone_model_fn,
validate_refine_context_fn=validate_refine_context_fn,
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
group_consumer_fn=sample_prepared_group,
retain_group_outputs=False,
selected_group_indices=(selected_indices if selection_active else None),
)
if len(group_seeds) != len(selected_indices):
raise SecondPassContractError(
"Video refine Sampling count differs from selected physical group count"
)
report_lines = [
"H3 Continuum Second Pass V3.5",
"Mode: context-aware physical-group refine when a complete First Pass "
"refine_context is available.",
f"Physical groups: {validation['physical_group_count']}.",
"Sampling contract: video random noise/mask=1; audio zero noise/mask=0; "
"the resolved RefineSchedule SIGMAS are applied directly by Core.",
"RefineSchedule: "
f"schema={schedule_contract['schema_version']}, "
f"mode={schedule_contract['mode']}, "
f"evaluations={schedule_contract['evaluation_count']}, "
f"start_sigma={schedule_contract['start_sigma']}, "
f"end_sigma={schedule_contract['end_sigma']}, "
f"sigma_hash={schedule_contract['sigma_hash']}.",
*prepared["warnings"],
*group_report_lines,
]
if selection_active:
report_lines.insert(
3,
"Selected physical groups: "
+ ", ".join(str(index + 1) for index in selected_indices)
+ "; non-selected groups were exact object passthrough with no "
"conditioning, MODEL clone, Sampling, or seed derivation.",
)
passthrough_audio = passthrough_audio_latents(audio_latents)
updated_plan = prepared["updated_assembly_plan"]
contract = updated_plan["second_pass_contract"]
if conditioning_sources and all(
source == "refine_context" for source in conditioning_sources
):
contract["execution"] = "context_aware_physical_groups_audio_locked_v3"
elif any(source == "refine_context" for source in conditioning_sources):
contract["execution"] = "mixed_context_physical_groups_audio_locked_v3"
else:
contract["execution"] = "t2va_physical_groups_audio_locked_v2"
contract["conditioning_sources"] = conditioning_sources
if any(item["status"] != "not_applicable" for item in reference_inheritance):
contract["reference_inheritance"] = reference_inheritance
contract["refine_seed_base"] = int(refine_seed)
contract["refine_group_seeds"] = group_seeds
contract["audio_output"] = "bit_exact_first_pass_passthrough"
contract["audio_sampling"] = "zero_noise_mask_locked"
contract["refine_schedule"] = schedule_contract
contract["refine_schedule_identity"] = schedule_contract["schedule_hash"]
report_lines.append(
"Audio output: first-pass physical audio LATENT objects returned bit-exact; "
"temporary refined audio discarded."
)
return refined_videos, passthrough_audio, updated_plan, "\n".join(report_lines)
+385
View File
@@ -0,0 +1,385 @@
"""V3.5 Second Pass public node."""
from __future__ import annotations
from typing import Any
from .refine_target import MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO, MODE_VIDEO_ONLY
from .refine_scope import MODE_ALL, MODE_LOGICAL_CHUNK, MODE_PHYSICAL_GROUP
from .refine_window import MODE_TIME_WINDOW
from .second_pass import run_second_pass_groups
from .targeted_second_pass import run_targeted_second_pass_groups
REFINE_TARGET_VIDEO_ONLY = "Video Only"
REFINE_TARGET_AUDIO_ONLY = "Audio Only"
REFINE_TARGET_VIDEO_AUDIO = "Video + Audio"
REFINE_TARGET_OPTIONS = (
REFINE_TARGET_VIDEO_ONLY,
REFINE_TARGET_AUDIO_ONLY,
REFINE_TARGET_VIDEO_AUDIO,
)
_REFINE_TARGET_MODES = {
REFINE_TARGET_VIDEO_ONLY: MODE_VIDEO_ONLY,
REFINE_TARGET_AUDIO_ONLY: MODE_AUDIO_ONLY,
REFINE_TARGET_VIDEO_AUDIO: MODE_VIDEO_AUDIO,
}
REFINE_SCOPE_ALL = "All"
REFINE_SCOPE_PHYSICAL_GROUP = "Physical Group"
REFINE_SCOPE_LOGICAL_CHUNK = "Logical Chunk"
REFINE_SCOPE_TIME_WINDOW = "Time Window"
REFINE_SCOPE_OPTIONS = (
REFINE_SCOPE_ALL,
REFINE_SCOPE_PHYSICAL_GROUP,
REFINE_SCOPE_LOGICAL_CHUNK,
REFINE_SCOPE_TIME_WINDOW,
)
_REFINE_SCOPE_MODES = {
REFINE_SCOPE_ALL: MODE_ALL,
REFINE_SCOPE_PHYSICAL_GROUP: MODE_PHYSICAL_GROUP,
REFINE_SCOPE_LOGICAL_CHUNK: MODE_LOGICAL_CHUNK,
REFINE_SCOPE_TIME_WINDOW: MODE_TIME_WINDOW,
}
def _single_list_input(name: str, value: Any) -> Any:
if not isinstance(value, list) or len(value) != 1:
raise ValueError(f"{name} must contain exactly one value")
return value[0]
class H3ContinuumSecondPassV35:
"""Low-sigma refinement over complete physical decode groups."""
DEPRECATED = False
DESCRIPTION = (
"V3.5 context-aware Second Pass. Connect externally upscaled H3 video "
"latents, the matching first-pass audio latents, and the original assembly plan. "
"Connect the optional Continuum refine context to preserve the physical group's "
"First/Last/Reference conditioning. "
"SIGMAS controls the low-denoise refine schedule; audio output is preserved "
"bit-exact from the first pass."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"refine_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"tooltip": "Base seed independently derived once per physical group.",
},
),
},
"optional": {
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
"video_vae": ("VAE",),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = (
"refined_video_latents",
"audio_latents",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "refine"
CATEGORY = "MiniMax H3/Continuum/Advanced"
def refine(
self,
model,
clip,
sampler,
sigmas,
video_latents,
audio_latents,
assembly_plan,
refine_seed,
refine_context=None,
video_vae=None,
refine_schedule=None,
):
return run_second_pass_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
sampler=_single_list_input("sampler", sampler),
sigmas=_single_list_input("sigmas", sigmas),
video_latents=video_latents,
audio_latents=audio_latents,
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
refine_context=(
None
if refine_context is None
else _single_list_input("refine_context", refine_context)
),
video_vae=(
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
),
refine_schedule=refine_schedule,
)
def resolve_selective_refine_target(value: str) -> str:
"""Resolve one public UI label to the versioned internal target mode."""
try:
return _REFINE_TARGET_MODES[value]
except (KeyError, TypeError) as exc:
raise ValueError(f"unsupported selective refine target {value!r}") from exc
def resolve_selective_refine_scope(value: str) -> str:
"""Resolve one public UI label to the versioned internal scope mode."""
try:
return _REFINE_SCOPE_MODES[value]
except (KeyError, TypeError) as exc:
raise ValueError(f"unsupported selective refine scope {value!r}") from exc
def _selective_status_prefix(
label: str,
physical_group_count: int,
scope_label: str,
scope_index: int,
window_start_sec: float,
window_end_sec: float,
) -> str:
if label == REFINE_TARGET_VIDEO_ONLY:
detail = "Video=refined; Audio=bit-exact passthrough"
elif label == REFINE_TARGET_AUDIO_ONLY:
detail = "Experimental / Production HOLD; Video=locked; Audio=refined"
elif label == REFINE_TARGET_VIDEO_AUDIO:
detail = (
"GPU Experimental PASS / Production HOLD; "
"Video=refined; Audio=refined"
)
else:
raise ValueError(f"unsupported selective refine target {label!r}")
if scope_label == REFINE_SCOPE_ALL:
scope_detail = "."
elif scope_label == REFINE_SCOPE_TIME_WINDOW:
scope_detail = f" [{window_start_sec:g}, {window_end_sec:g}) sec."
else:
scope_detail = f" {scope_index}."
return (
f"Selective Refine: {label}; {detail}; "
f"physical groups={physical_group_count}; scope={scope_label}"
+ scope_detail
)
class H3ContinuumSelectiveSecondPassExperimental:
"""Experimental public adapter for versioned selective-refine targets."""
DEPRECATED = False
DESCRIPTION = (
"Experimental Selective Second Pass for MiniMax H3. Video Only preserves "
"compatibility with the existing Continuum Second Pass. Audio Only keeps "
"the input video locked and refines audio. Video + Audio refines both AV "
"streams. Audio-targeted modes remain Experimental while upstream H3 "
"denoise-mask correctness is under review. Chunk scope skips complete "
"physical groups without conditioning, model clone, Sampling, or seed use; "
"a logical chunk inside Terminal Merge expands to the complete pair. Time "
"Window uses the final visible output timeline, protects continuation "
"prefixes, and restores every out-of-window latent position bit-exactly."
)
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"sampler": ("SAMPLER",),
"sigmas": ("SIGMAS",),
"video_latents": ("LATENT",),
"audio_latents": ("LATENT",),
"assembly_plan": ("H3_CONTINUUM_ASSEMBLY_PLAN",),
"refine_seed": (
"INT",
{
"default": 0,
"min": 0,
"max": 0xFFFFFFFFFFFFFFFF,
"control_after_generate": True,
"tooltip": "Base seed independently derived once per physical group.",
},
),
"refine_target": (
REFINE_TARGET_OPTIONS,
{"default": REFINE_TARGET_VIDEO_ONLY},
),
"refine_scope": (
REFINE_SCOPE_OPTIONS,
{"default": REFINE_SCOPE_ALL},
),
"refine_scope_index": (
"INT",
{
"default": 1,
"min": 1,
"max": 0x7FFFFFFF,
"tooltip": (
"1-based physical-group or logical-chunk number. "
"Ignored when Refine Scope is All or Time Window."
),
},
),
"window_start_sec": (
"FLOAT",
{
"default": 0.0,
"min": 0.0,
"max": 86400.0,
"step": 0.01,
"tooltip": (
"Time Window start on the final visible output timeline. "
"Ignored by other Refine Scope modes."
),
},
),
"window_end_sec": (
"FLOAT",
{
"default": 5.0,
"min": 0.01,
"max": 86400.0,
"step": 0.01,
"tooltip": (
"Exclusive Time Window end on the final visible output "
"timeline. Ignored by other Refine Scope modes."
),
},
),
},
"optional": {
"refine_context": ("H3_CONTINUUM_REFINE_CONTEXT",),
"video_vae": ("VAE",),
},
}
INPUT_IS_LIST = True
RETURN_TYPES = ("LATENT", "LATENT", "H3_CONTINUUM_ASSEMBLY_PLAN", "STRING")
RETURN_NAMES = (
"refined_video_latents",
"audio_latents",
"updated_assembly_plan",
"status",
)
OUTPUT_IS_LIST = (True, True, False, False)
FUNCTION = "refine"
CATEGORY = "MiniMax H3/Continuum/Advanced"
def refine(
self,
model,
clip,
sampler,
sigmas,
video_latents,
audio_latents,
assembly_plan,
refine_seed,
refine_target,
refine_scope=None,
refine_scope_index=None,
window_start_sec=None,
window_end_sec=None,
refine_context=None,
video_vae=None,
refine_schedule=None,
):
label = _single_list_input("refine_target", refine_target)
target_mode = resolve_selective_refine_target(label)
scope_label = (
REFINE_SCOPE_ALL
if refine_scope is None
else _single_list_input("refine_scope", refine_scope)
)
scope_mode = resolve_selective_refine_scope(scope_label)
resolved_scope_index = (
1
if refine_scope_index is None
else int(_single_list_input("refine_scope_index", refine_scope_index))
)
resolved_window_start = (
0.0
if window_start_sec is None
else float(_single_list_input("window_start_sec", window_start_sec))
)
resolved_window_end = (
5.0
if window_end_sec is None
else float(_single_list_input("window_end_sec", window_end_sec))
)
videos, audios, updated_plan, status = run_targeted_second_pass_groups(
model=_single_list_input("model", model),
clip=_single_list_input("clip", clip),
sampler=_single_list_input("sampler", sampler),
sigmas=_single_list_input("sigmas", sigmas),
video_latents=video_latents,
audio_latents=audio_latents,
assembly_plan=_single_list_input("assembly_plan", assembly_plan),
refine_seed=int(_single_list_input("refine_seed", refine_seed)),
refine_target=target_mode,
refine_scope=scope_mode,
refine_scope_index=resolved_scope_index,
window_start_sec=resolved_window_start,
window_end_sec=resolved_window_end,
refine_context=(
None
if refine_context is None
else _single_list_input("refine_context", refine_context)
),
video_vae=(
None
if video_vae is None
else _single_list_input("video_vae", video_vae)
),
refine_schedule=refine_schedule,
)
prefix = _selective_status_prefix(
label,
len(video_latents),
scope_label,
resolved_scope_index,
resolved_window_start,
resolved_window_end,
)
return videos, audios, updated_plan, f"{prefix}\n{status}"
NODE_CLASS_MAPPINGS = {
"H3ContinuumSecondPassV35": H3ContinuumSecondPassV35,
"H3ContinuumSelectiveSecondPassExperimental": (
H3ContinuumSelectiveSecondPassExperimental
),
}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumSecondPassV35": "H3 Continuum Second Pass",
"H3ContinuumSelectiveSecondPassExperimental": (
"H3 Continuum Selective Second Pass (Experimental)"
),
}
+496
View File
@@ -0,0 +1,496 @@
"""Target-aware refinement sampling for frozen Video and Audio policies."""
from __future__ import annotations
from typing import Any, Callable
import torch
from ..state import extract_av_streams
from .refine_sampling import (
RefineSamplingRuntimeError,
_load_runtime_modules,
_make_basic_guider,
sample_refine_chunk,
)
from .refine_target import (
MODE_AUDIO_ONLY,
MODE_VIDEO_AUDIO,
MODE_VIDEO_ONLY,
RefineTarget,
RefineTargetError,
resolve_refine_target,
)
from .refine_window import GroupTemporalWindow
class TargetedRefineExecutionUnavailable(RefineTargetError):
"""Raised before Sampling when a future target mode is not implemented."""
def _validate_ranges(
ranges: tuple[tuple[int, int], ...],
length: int,
*,
name: str,
) -> tuple[tuple[int, int], ...]:
if not ranges:
raise RefineSamplingRuntimeError(f"{name} temporal window is empty")
previous_stop = 0
for start, stop in ranges:
if start < previous_stop or stop <= start or stop > int(length):
raise RefineSamplingRuntimeError(
f"{name} temporal window is outside its latent length"
)
previous_stop = stop
return ranges
def _window_masks(
video: torch.Tensor,
audio: torch.Tensor,
temporal_window: GroupTemporalWindow,
) -> tuple[torch.Tensor, torch.Tensor]:
if not isinstance(temporal_window, GroupTemporalWindow):
raise RefineSamplingRuntimeError(
"temporal window must be a resolved GroupTemporalWindow"
)
video_ranges = _validate_ranges(
temporal_window.video_slot_ranges,
int(video.shape[2]),
name="Video",
)
audio_ranges = _validate_ranges(
temporal_window.audio_tick_ranges,
int(audio.shape[-1]),
name="Audio",
)
video_mask = torch.zeros(
(
int(video.shape[0]),
1,
int(video.shape[2]),
int(video.shape[3]),
int(video.shape[4]),
),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.zeros(
(
int(audio.shape[0]),
1,
int(audio.shape[2]),
int(audio.shape[3]),
),
dtype=torch.float32,
device="cpu",
)
for start, stop in video_ranges:
video_mask[:, :, start:stop] = 1.0
for start, stop in audio_ranges:
audio_mask[..., start:stop] = 1.0
return video_mask, audio_mask
def _prepare_video_only_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow,
) -> tuple[Any, Any]:
"""Build windowed Video noise/mask while locking Audio completely."""
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video Only noise geometry"
)
video_mask, _audio_window_mask = _window_masks(video, audio, temporal_window)
audio_noise = torch.zeros_like(audio, device="cpu")
audio_mask = torch.zeros(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
return (
nested_builder((video_noise, audio_noise)),
nested_builder((video_mask, audio_mask)),
)
def require_video_only_execution(
refine_target: RefineTarget | str | None,
) -> RefineTarget:
target = resolve_refine_target(refine_target)
if target.mode != MODE_VIDEO_ONLY:
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is contract-only in Phase A1"
)
return target
def _prepare_audio_only_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow | None = None,
) -> tuple[Any, Any]:
"""Build zero/locked Video and seeded-random/sampled Audio streams."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Audio Only noise geometry"
)
video_noise = torch.zeros_like(video, device="cpu")
if temporal_window is None:
video_mask = torch.zeros(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.ones(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
else:
_video_window_mask, audio_mask = _window_masks(
video,
audio,
temporal_window,
)
video_mask = torch.zeros(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def _prepare_video_audio_noise_and_mask(
video: torch.Tensor,
audio: torch.Tensor,
*,
seed: int,
batch_inds: Any = None,
prepare_noise_fn: Callable[[torch.Tensor, int, Any], torch.Tensor],
nested_builder: Callable[[tuple[torch.Tensor, torch.Tensor]], Any],
temporal_window: GroupTemporalWindow | None = None,
) -> tuple[Any, Any]:
"""Build seeded-random noise and sampled masks for both AV streams."""
if video.ndim != 5 or audio.ndim != 4:
raise RefineSamplingRuntimeError(
"targeted latent must contain video [B,C,T,H,W] and audio [B,C,2,T]"
)
if int(video.shape[0]) != int(audio.shape[0]):
raise RefineSamplingRuntimeError("targeted video/audio batch sizes differ")
video_noise = prepare_noise_fn(video, int(seed), batch_inds)
audio_noise = prepare_noise_fn(audio, int(seed), batch_inds)
if not torch.is_tensor(video_noise) or tuple(video_noise.shape) != tuple(video.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video + Audio video noise geometry"
)
if not torch.is_tensor(audio_noise) or tuple(audio_noise.shape) != tuple(audio.shape):
raise RefineSamplingRuntimeError(
"ComfyUI returned invalid Video + Audio audio noise geometry"
)
if temporal_window is None:
video_mask = torch.ones(
(int(video.shape[0]), 1, 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
audio_mask = torch.ones(
(int(audio.shape[0]), 1, 1, 1),
dtype=torch.float32,
device="cpu",
)
else:
video_mask, audio_mask = _window_masks(video, audio, temporal_window)
noise = nested_builder((video_noise, audio_noise))
noise_mask = nested_builder((video_mask, audio_mask))
return noise, noise_mask
def _sample_with_target_noise_and_mask(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
enable_preview: bool,
noise_mask_builder: Callable[..., tuple[Any, Any]],
temporal_window: GroupTemporalWindow | None = None,
restore_video_outside_window: bool = False,
restore_audio_outside_window: bool = False,
) -> dict[str, Any]:
"""Run Core once after constructing one explicit AV target policy."""
if not isinstance(latent, dict) or "samples" not in latent:
raise RefineSamplingRuntimeError("latent must be a ComfyUI LATENT dictionary")
if not torch.is_tensor(sigmas) or sigmas.ndim != 1 or sigmas.numel() < 2:
raise RefineSamplingRuntimeError("sigmas must contain at least two values")
model_management, nested_tensor, comfy_sample, comfy_utils, preview = (
_load_runtime_modules()
)
working = latent.copy()
latent_image = comfy_sample.fix_empty_latent_channels(
model,
working["samples"],
working.get("downscale_ratio_spacial"),
working.get("downscale_ratio_temporal"),
)
working["samples"] = latent_image
video, audio = extract_av_streams(working)
noise, noise_mask = noise_mask_builder(
video,
audio,
seed=int(seed),
batch_inds=working.get("batch_index"),
prepare_noise_fn=comfy_sample.prepare_noise,
nested_builder=nested_tensor.NestedTensor,
temporal_window=temporal_window,
)
guider = _make_basic_guider(model, conditioning)
x0_output: dict[str, Any] = {}
callback = None
if enable_preview:
callback = preview.prepare_callback(
model,
int(sigmas.shape[-1]) - 1,
x0_output,
)
samples = guider.sample(
noise,
latent_image,
sampler,
sigmas,
denoise_mask=noise_mask,
callback=callback,
disable_pbar=not comfy_utils.PROGRESS_BAR_ENABLED,
seed=int(seed),
)
samples = samples.to(model_management.intermediate_device())
if temporal_window is not None:
sampled_output = {"samples": samples}
sampled_video, sampled_audio = extract_av_streams(sampled_output)
if restore_video_outside_window:
sampled_video = _restore_outside_temporal_ranges(
sampled_video,
video,
temporal_window.video_slot_ranges,
time_dim=2,
name="Video",
)
if restore_audio_outside_window:
sampled_audio = _restore_outside_temporal_ranges(
sampled_audio,
audio,
temporal_window.audio_tick_ranges,
time_dim=-1,
name="Audio",
)
samples = nested_tensor.NestedTensor((sampled_video, sampled_audio))
output = working.copy()
output.pop("downscale_ratio_spacial", None)
output.pop("downscale_ratio_temporal", None)
output["samples"] = samples
extract_av_streams(output)
return output
def _restore_outside_temporal_ranges(
sampled: torch.Tensor,
source: torch.Tensor,
ranges: tuple[tuple[int, int], ...],
*,
time_dim: int,
name: str,
) -> torch.Tensor:
"""Restore every inactive temporal position from the input bit-exactly."""
if not torch.is_tensor(sampled) or not torch.is_tensor(source):
raise RefineSamplingRuntimeError(f"{name} temporal restore requires Tensors")
if tuple(sampled.shape) != tuple(source.shape):
raise RefineSamplingRuntimeError(
f"{name} temporal restore input/output shapes differ"
)
if sampled.dtype != source.dtype:
raise RefineSamplingRuntimeError(
f"{name} temporal restore input/output dtypes differ"
)
dim = int(time_dim)
if dim < 0:
dim += sampled.ndim
if dim < 0 or dim >= sampled.ndim:
raise RefineSamplingRuntimeError(f"{name} temporal restore dimension is invalid")
validated = _validate_ranges(ranges, int(sampled.shape[dim]), name=name)
restored = sampled.clone()
original = source.to(device=restored.device)
cursor = 0
for start, stop in validated:
if start > cursor:
restored.narrow(dim, cursor, start - cursor).copy_(
original.narrow(dim, cursor, start - cursor)
)
cursor = stop
if cursor < int(restored.shape[dim]):
restored.narrow(dim, cursor, int(restored.shape[dim]) - cursor).copy_(
original.narrow(dim, cursor, int(restored.shape[dim]) - cursor)
)
return restored
def sample_video_only_window_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Refine only selected Video slots and restore all other AV positions."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_video_only_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=True,
restore_audio_outside_window=True,
)
def sample_audio_only_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Sample Audio while Core carries Video with zero noise and mask zero."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_audio_only_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=temporal_window is not None,
restore_audio_outside_window=temporal_window is not None,
)
def sample_video_audio_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
) -> dict[str, Any]:
"""Sample and adopt both Video and Audio using mask one for each stream."""
return _sample_with_target_noise_and_mask(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
noise_mask_builder=_prepare_video_audio_noise_and_mask,
temporal_window=temporal_window,
restore_video_outside_window=temporal_window is not None,
restore_audio_outside_window=temporal_window is not None,
)
def sample_targeted_refine_chunk(
*,
model: Any,
conditioning: list,
latent: dict[str, Any],
sampler: Any,
sigmas: torch.Tensor,
seed: int,
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
temporal_window: GroupTemporalWindow | None = None,
enable_preview: bool = True,
legacy_sample_fn: Callable[..., dict[str, Any]] | None = None,
video_only_window_sample_fn: Callable[..., dict[str, Any]] | None = None,
audio_only_sample_fn: Callable[..., dict[str, Any]] | None = None,
video_audio_sample_fn: Callable[..., dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Dispatch an implemented target without changing its stream policy."""
target = resolve_refine_target(refine_target)
if target.mode == MODE_VIDEO_ONLY and temporal_window is not None:
sample_fn = video_only_window_sample_fn or sample_video_only_window_refine_chunk
elif target.mode == MODE_VIDEO_ONLY:
sample_fn = legacy_sample_fn or sample_refine_chunk
elif target.mode == MODE_AUDIO_ONLY:
sample_fn = audio_only_sample_fn or sample_audio_only_refine_chunk
elif target.mode == MODE_VIDEO_AUDIO:
sample_fn = video_audio_sample_fn or sample_video_audio_refine_chunk
else: # pragma: no cover - resolve_refine_target validates the mode.
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is unavailable"
)
arguments = dict(
model=model,
conditioning=conditioning,
latent=latent,
sampler=sampler,
sigmas=sigmas,
seed=int(seed),
enable_preview=bool(enable_preview),
)
if temporal_window is not None:
arguments["temporal_window"] = temporal_window
return sample_fn(**arguments)
+491
View File
@@ -0,0 +1,491 @@
"""Target-aware Second Pass orchestration over complete physical groups."""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import Any
import torch
from ..state import extract_av_streams
from ..v2.sampling import latent_from_cpu
from .refine_schedule import (
RefineScheduleError,
resolve_refine_schedule,
serializable_schedule_contract,
)
from .refine_scope import (
MODE_ALL as SCOPE_ALL,
resolve_refine_scope,
)
from .refine_target import (
MODE_AUDIO_ONLY,
MODE_VIDEO_AUDIO,
MODE_VIDEO_ONLY,
RefineTarget,
build_target_execution_contract,
derive_target_refine_seed,
resolve_refine_target,
serializable_target_contract,
)
from .refine_window import (
MODE_TIME_WINDOW,
GroupTemporalWindow,
RefineWindow,
resolve_refine_window,
)
from .second_pass import (
SecondPassContractError,
_normalize_selected_group_indices,
_latent_samples,
prepare_physical_refine_groups,
run_second_pass_groups,
validate_second_pass_inputs,
)
from .targeted_refine_sampling import (
TargetedRefineExecutionUnavailable,
sample_targeted_refine_chunk,
)
def _attach_target_contract(
updated_plan: dict[str, Any],
target: RefineTarget,
) -> None:
contract = updated_plan.get("second_pass_contract")
if not isinstance(contract, dict):
raise SecondPassContractError(
"targeted Second Pass output has no mutable contract"
)
if contract.get("version") != 1:
raise SecondPassContractError(
"targeted Second Pass must preserve contract version 1"
)
schedule_contract = contract.get("refine_schedule")
contract["refine_target_contract"] = serializable_target_contract(target)
contract["refine_execution_contract"] = build_target_execution_contract(
target,
schedule_contract,
)
def _run_video_only_groups(
*,
target: RefineTarget,
sample_fn,
arguments: dict[str, Any],
selected_group_indices: Sequence[int] | None = None,
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
def targeted_sample(**kwargs):
return sample_targeted_refine_chunk(
refine_target=target,
legacy_sample_fn=sample_fn,
video_only_window_sample_fn=sample_fn,
**kwargs,
)
videos, audios, updated_plan, status = run_second_pass_groups(
**arguments,
sample_fn=targeted_sample,
selected_group_indices=selected_group_indices,
temporal_windows_by_group=temporal_windows_by_group,
)
_attach_target_contract(updated_plan, target)
return videos, audios, updated_plan, status
def _run_sampled_target_groups(
*,
target: RefineTarget,
model: Any,
clip: Any,
sampler: Any,
sigmas: torch.Tensor,
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
refine_seed: int,
refine_context: Any,
video_vae: Any,
conditioning_upscale_method: str,
enable_preview: bool,
encode_prompt_fn,
latent_builder,
sample_fn,
stream_extractor,
clone_model_fn,
validate_refine_context_fn,
adapt_group_conditioning_fn,
refine_schedule,
selected_group_indices: Sequence[int] | None = None,
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
) -> tuple[list[Any], list[dict[str, Any]], dict[str, Any], str]:
"""Sample one non-legacy target and adopt only its configured outputs."""
if target.mode not in (MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO):
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is not a sampled-stream target"
)
adopt_sampled_video = target.mode == MODE_VIDEO_AUDIO
validation = validate_second_pass_inputs(
video_latents,
audio_latents,
assembly_plan,
)
selected_indices, selection_active = _normalize_selected_group_indices(
selected_group_indices,
int(validation["physical_group_count"]),
)
try:
resolved_schedule = resolve_refine_schedule(sigmas, refine_schedule)
except RefineScheduleError as exc:
raise SecondPassContractError(
f"refine schedule is invalid: {exc}"
) from exc
effective_sigmas = resolved_schedule.sigmas
schedule_contract = serializable_schedule_contract(resolved_schedule)
latent_builder = latent_builder or latent_from_cpu
stream_extractor = stream_extractor or extract_av_streams
refined_videos: list[dict[str, Any]] = (
list(video_latents) if selection_active else []
)
refined_audios: list[dict[str, Any]] = (
list(audio_latents) if selection_active else []
)
group_seeds: list[int] = []
conditioning_sources: list[str] = []
group_report_lines: list[str] = []
def sample_prepared_group(
group_index,
video_latent,
group,
chunk_model,
conditioning,
detail,
):
audio_latent = audio_latents[group_index]
video_samples = _latent_samples(
video_latent,
name=f"video_latents[{group_index}]",
)
audio_samples = _latent_samples(
audio_latent,
name=f"audio_latents[{group_index}]",
)
physical_seed = derive_target_refine_seed(
int(refine_seed),
group_index,
target,
)
group_seeds.append(physical_seed)
conditioning_source = str(detail["conditioning_source"])
conditioning_sources.append(conditioning_source)
nested_latent = latent_builder(video_samples, audio_samples)
sample_arguments = dict(
model=chunk_model,
conditioning=conditioning,
latent=nested_latent,
sampler=sampler,
sigmas=effective_sigmas,
seed=physical_seed,
refine_target=target,
enable_preview=bool(enable_preview),
audio_only_sample_fn=sample_fn,
video_audio_sample_fn=sample_fn,
)
if temporal_windows_by_group is not None:
temporal_window = temporal_windows_by_group.get(group_index)
if temporal_window is None:
raise SecondPassContractError(
f"selected physical group {group_index + 1} has no temporal window"
)
sample_arguments["temporal_window"] = temporal_window
sampled = sample_targeted_refine_chunk(**sample_arguments)
sampled_video, sampled_audio = stream_extractor(sampled)
if tuple(sampled_video.shape) != tuple(video_samples.shape):
raise SecondPassContractError(
f"sampled video group {group_index} changed B/C/T/H/W geometry"
)
if tuple(sampled_audio.shape) != tuple(audio_samples.shape):
raise SecondPassContractError(
f"refined audio group {group_index} changed B/C/stereo/T geometry"
)
if not bool(torch.isfinite(sampled_audio.float()).all().item()):
raise SecondPassContractError(
f"refined audio group {group_index} contains NaN or Inf"
)
if adopt_sampled_video:
if not bool(torch.isfinite(sampled_video.float()).all().item()):
raise SecondPassContractError(
f"refined video group {group_index} contains NaN or Inf"
)
output_video = dict(video_latent)
output_video["samples"] = sampled_video
if selection_active:
refined_videos[group_index] = output_video
else:
refined_videos.append(output_video)
output_audio = dict(audio_latent)
output_audio["samples"] = sampled_audio
if selection_active:
refined_audios[group_index] = output_audio
else:
refined_audios.append(output_audio)
adoption = (
f"video_shape={tuple(sampled_video.shape)}, "
"sampled_video_adopted=true, sampled_audio_adopted=true"
if adopt_sampled_video
else "temporary_video_discarded=true"
)
group_report_lines.append(
"group "
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
f"conditioning_source={conditioning_source}, "
f"refine_seed={physical_seed}, "
f"audio_shape={tuple(sampled_audio.shape)}, sampling_passes=1, "
f"{adoption}"
)
prepared = prepare_physical_refine_groups(
model=model,
clip=clip,
video_latents=video_latents,
assembly_plan=assembly_plan,
refine_context=refine_context,
video_vae=video_vae,
conditioning_upscale_method=conditioning_upscale_method,
encode_prompt_fn=encode_prompt_fn,
clone_model_fn=clone_model_fn,
validate_refine_context_fn=validate_refine_context_fn,
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
group_consumer_fn=sample_prepared_group,
retain_group_outputs=False,
selected_group_indices=(selected_indices if selection_active else None),
)
expected_groups = int(validation["physical_group_count"])
expected_sampled_groups = len(selected_indices)
if len(group_seeds) != expected_sampled_groups:
raise SecondPassContractError(
"targeted Sampling count differs from selected physical group count"
)
if len(refined_audios) != expected_groups:
raise SecondPassContractError(
"targeted Audio output count differs from physical group count"
)
if adopt_sampled_video and len(refined_videos) != expected_groups:
raise SecondPassContractError(
"targeted Video output count differs from physical group count"
)
output_videos = refined_videos if adopt_sampled_video else list(video_latents)
updated_plan = prepared["updated_assembly_plan"]
contract = updated_plan["second_pass_contract"]
execution_target = "both_sampled" if adopt_sampled_video else "video_locked_audio"
if conditioning_sources and all(
source == "refine_context" for source in conditioning_sources
):
contract["execution"] = (
f"context_aware_physical_groups_{execution_target}_v1"
)
elif any(source == "refine_context" for source in conditioning_sources):
contract["execution"] = f"mixed_context_physical_groups_{execution_target}_v1"
else:
contract["execution"] = f"t2va_physical_groups_{execution_target}_v1"
contract["conditioning_sources"] = conditioning_sources
contract["refine_seed_base"] = int(refine_seed)
contract["refine_group_seeds"] = group_seeds
contract["video_output"] = (
"sampled" if adopt_sampled_video else "bit_exact_first_pass_passthrough"
)
contract["video_sampling"] = (
"seeded_random_mask_1" if adopt_sampled_video else "zero_noise_mask_locked"
)
contract["audio_output"] = "sampled"
contract["audio_sampling"] = "seeded_random_mask_1"
contract["refine_schedule"] = schedule_contract
contract["refine_schedule_identity"] = schedule_contract["schedule_hash"]
_attach_target_contract(updated_plan, target)
if adopt_sampled_video:
target_line = (
"Target: Video + Audio "
"(GPU Experimental PASS / Production HOLD)."
)
sampling_line = (
"Sampling contract: video seeded random noise/mask=1 and adopted "
"sampled output; audio seeded random noise/mask=1 and adopted "
"sampled output."
)
output_line = (
"Outputs: sampled Video and sampled Audio adopted for every complete "
"physical group."
)
else:
target_line = "Target: Audio Only (Experimental / Production HOLD)."
sampling_line = (
"Sampling contract: video zero noise/mask=0 and discarded sampled "
"output; audio seeded random noise/mask=1 and adopted sampled output."
)
output_line = (
"Video output: original first-pass physical Video LATENT objects "
"returned bit-exact; temporary sampled Video discarded."
)
report_lines = [
"H3 Continuum Targeted Second Pass",
target_line,
f"Physical groups: {validation['physical_group_count']}.",
sampling_line,
"RefineSchedule: "
f"schema={schedule_contract['schema_version']}, "
f"mode={schedule_contract['mode']}, "
f"evaluations={schedule_contract['evaluation_count']}, "
f"sigma_hash={schedule_contract['sigma_hash']}.",
*prepared["warnings"],
*group_report_lines,
output_line,
]
if selection_active:
report_lines.insert(
3,
"Selected physical groups: "
+ ", ".join(str(index + 1) for index in selected_indices)
+ "; non-selected groups were exact object passthrough with no "
"conditioning, MODEL clone, Sampling, or seed derivation.",
)
return output_videos, refined_audios, updated_plan, "\n".join(report_lines)
def run_targeted_second_pass_groups(
*,
model: Any,
clip: Any,
sampler: Any,
sigmas: torch.Tensor,
video_latents: Sequence[Any],
audio_latents: Sequence[Any],
assembly_plan: Mapping[str, Any],
refine_seed: int,
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
refine_context: Any = None,
video_vae: Any = None,
conditioning_upscale_method: str = "bilinear",
enable_preview: bool = True,
encode_prompt_fn=None,
latent_builder=None,
sample_fn=None,
stream_extractor=None,
clone_model_fn=None,
validate_refine_context_fn=None,
adapt_group_conditioning_fn=None,
refine_schedule=None,
refine_scope: str | None = SCOPE_ALL,
refine_scope_index: int = 1,
window_start_sec: float = 0.0,
window_end_sec: float = 5.0,
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
"""Run an implemented target without changing the V3.5 public node."""
target = resolve_refine_target(refine_target)
window: RefineWindow | None = None
if isinstance(refine_scope, str) and refine_scope.strip().lower() == MODE_TIME_WINDOW:
scope = None
window = resolve_refine_window(
window_start_sec,
window_end_sec,
assembly_plan,
)
selected_group_indices = window.selected_group_indices
temporal_windows_by_group = {
group.group_index: group for group in window.groups
}
else:
scope = resolve_refine_scope(
refine_scope,
refine_scope_index,
assembly_plan,
)
selected_group_indices = (
None if scope.is_all else scope.selected_group_indices
)
temporal_windows_by_group = None
arguments = dict(
model=model,
clip=clip,
sampler=sampler,
sigmas=sigmas,
video_latents=video_latents,
audio_latents=audio_latents,
assembly_plan=assembly_plan,
refine_seed=int(refine_seed),
refine_context=refine_context,
video_vae=video_vae,
conditioning_upscale_method=str(conditioning_upscale_method),
enable_preview=bool(enable_preview),
encode_prompt_fn=encode_prompt_fn,
latent_builder=latent_builder,
stream_extractor=stream_extractor,
clone_model_fn=clone_model_fn,
validate_refine_context_fn=validate_refine_context_fn,
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
refine_schedule=refine_schedule,
)
if target.mode == MODE_VIDEO_ONLY:
result = _run_video_only_groups(
target=target,
sample_fn=sample_fn,
arguments=arguments,
selected_group_indices=selected_group_indices,
temporal_windows_by_group=temporal_windows_by_group,
)
elif target.mode == MODE_AUDIO_ONLY:
result = _run_sampled_target_groups(
target=target,
sample_fn=sample_fn,
selected_group_indices=selected_group_indices,
temporal_windows_by_group=temporal_windows_by_group,
**arguments,
)
elif target.mode == MODE_VIDEO_AUDIO:
result = _run_sampled_target_groups(
target=target,
sample_fn=sample_fn,
selected_group_indices=selected_group_indices,
temporal_windows_by_group=temporal_windows_by_group,
**arguments,
)
else:
raise TargetedRefineExecutionUnavailable(
f"refine target {target.mode!r} is unavailable"
)
videos, audios, updated_plan, status = result
if window is not None:
contract = window.contract
window_line = (
"Refine Window: timeline=final_visible_output, interval=["
f"{contract['effective_start_sec']:.6f}, "
f"{contract['effective_end_sec']:.6f}) sec, output_frames="
f"{contract['output_frame_range']}, selected_physical_groups="
f"{contract['selected_physical_groups']}, continuation_prefix=protected, "
"outside_window=mask_zero_then_bit_exact_restore, "
"physical_group_atomicity=preserved."
)
return videos, audios, updated_plan, f"{status}\n{window_line}"
assert scope is not None
if scope.is_all:
return videos, audios, updated_plan, status
scope_contract = scope.contract
scope_line = (
"Refine Scope: "
f"mode={scope.mode}, requested_index={scope_contract['requested_index']}, "
"selected_physical_groups="
f"{list(scope_contract['selected_physical_groups'])}, "
"selected_logical_chunks="
f"{list(scope_contract['selected_logical_chunks'])}, "
f"terminal_expanded={str(scope_contract['terminal_expanded']).lower()}."
)
return videos, audios, updated_plan, f"{status}\n{scope_line}"
+56
View File
@@ -0,0 +1,56 @@
"""Core VIDEO to H3 frame batches, with no file/upload/Enable frontend."""
from __future__ import annotations
from fractions import Fraction
from .easy_video_nodes import ExecutionBlocker, _resample_video_frames, io
class H3ContinuumVideoAdapter(io.ComfyNode):
"""Reuse the established frame-rate conversion; audio is passed unchanged."""
@classmethod
def define_schema(cls):
return io.Schema(
node_id="H3ContinuumVideoAdapter",
display_name="H3 Continuum Video Adapter",
category="MiniMax H3/Continuum/Input",
description=(
"Converts Core Load Video's VIDEO to IMAGE frames and AUDIO. "
"Use Force Rate 24 for H3 Video Guide. No file selector or Enable. "
"Disable the whole video-input group, or bypass this adapter, "
"to disconnect its optional Sampler inputs."
),
inputs=[
io.Video.Input("video"),
io.Float.Input(
"force_rate", default=24.0, min=0.0, max=120.0, step=0.01,
display_name="Force Rate",
tooltip=(
"24 resamples to H3's expected FPS without changing nominal "
"duration or audio. 0 keeps source frames (not normally for H3)."
),
),
],
outputs=[
io.Image.Output(display_name="images"),
io.Audio.Output(display_name="audio"),
],
)
@classmethod
def execute(cls, video, force_rate: float = 24.0):
components = video.get_components()
images = _resample_video_frames(
components.images, Fraction(components.frame_rate), float(force_rate)
)
audio = components.audio
if audio is None:
audio = ExecutionBlocker(None)
return io.NodeOutput(images, audio)
NODE_CLASS_MAPPINGS = {"H3ContinuumVideoAdapter": H3ContinuumVideoAdapter}
NODE_DISPLAY_NAME_MAPPINGS = {
"H3ContinuumVideoAdapter": "H3 Continuum Video Adapter",
}
+532
View File
@@ -0,0 +1,532 @@
"""Read-only decoded video-boundary analysis for H3 Continuum V3."""
from __future__ import annotations
from dataclasses import dataclass
import math
from typing import Any
import torch
import torch.nn.functional as F
from .plan import validate_assembly_plan
COMPARE_LONG_SIDE = 192
MAX_REWIND_FRAMES = 2
FLASH_WINDOW_FRAMES = 6
FLASH_TILE_GRID = 4
MIN_FLASH_LUMA_SHIFT = 0.01
MIN_FLASH_REVERSAL = 0.35
MIN_FLASH_GLOBAL_FRACTION = 0.75
MIN_ANOMALY_RATIO = 1.35
MIN_RELATIVE_IMPROVEMENT = 0.06
MIN_AUTO_FLASH_REVERSAL = 0.70
MIN_AUTO_FLASH_IMPROVEMENT = 0.70
SMALLER_SHIFT_TOLERANCE = 0.03
MIN_MICRO_FLASH_LUMA_SHIFT = 0.003
MIN_MICRO_FLASH_REVERSAL = 0.25
MIN_MICRO_FLASH_GLOBAL_FRACTION = 0.70
MAX_EXPOSURE_RAMP_REVERSAL = 0.25
MIN_MOTION_CROSS_DELTA = 0.006
MIN_MOTION_JUMP_RATIO = 1.60
@dataclass(frozen=True)
class VideoBoundaryAnalysis:
boundary_index: int
nominal_error: float
baseline_error: float
anomaly_ratio: float
recommended_rewind: int
best_error: float
improvement: float
luma_delta: float
chroma_delta: float
edge_delta: float
scene_cut_score: float
scene_cut: bool
flash_luma_shift: float
flash_reversal: float
flash_global_fraction: float
transient_flash_candidate: bool
cross_frame_delta: float
motion_baseline_delta: float
motion_jump_ratio: float
micro_flash_candidate: bool
exposure_ramp_candidate: bool
motion_hitch_candidate: bool
classification: str
reason: str
def _validate_frames(frames: Any, label: str) -> None:
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
if int(frames.shape[0]) < 1 or int(frames.shape[1]) < 1 or int(frames.shape[2]) < 1:
raise ValueError(f"{label} must not be empty")
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
_validate_frames(frames, "video seam frames")
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
rgb = rgb.permute(0, 3, 1, 2)
height, width = int(rgb.shape[-2]), int(rgb.shape[-1])
scale = min(1.0, COMPARE_LONG_SIDE / float(max(height, width)))
target_height = max(1, int(round(height * scale)))
target_width = max(1, int(round(width * scale)))
if (target_height, target_width) != (height, width):
rgb = F.interpolate(
rgb,
size=(target_height, target_width),
mode="bilinear",
align_corners=False,
)
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
if not bool(torch.isfinite(rgb).all().item()):
raise ValueError("video seam comparison contains NaN or Inf")
return rgb
def _luma(frame: torch.Tensor) -> torch.Tensor:
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
return torch.sum(frame * weights, dim=-1)
def _frame_delta(left: torch.Tensor, right: torch.Tensor) -> float:
return float(torch.mean(torch.abs(left - right)).item())
def _prediction_error(history: torch.Tensor, candidate: torch.Tensor) -> float:
if int(history.shape[0]) < 3:
raise ValueError("temporal prediction requires three history frames")
recent_velocity = history[-1] - history[-2]
prior_velocity = history[-2] - history[-3]
predicted = (
history[-1] + recent_velocity * 0.65 + prior_velocity * 0.35
).clamp(0.0, 1.0)
motion = torch.mean(torch.abs(recent_velocity), dim=-1)
motion_scale = motion / motion.mean().clamp_min(1.0e-6)
weights = (1.0 + motion_scale).clamp(1.0, 4.0).unsqueeze(-1)
value = float(torch.mean((predicted - candidate).square() * weights).item())
if not math.isfinite(value):
raise ValueError("video seam prediction score is not finite")
return max(0.0, value)
def _baseline_prediction_error(previous: torch.Tensor) -> float:
recent = previous[-min(8, int(previous.shape[0])) :]
errors = [
_prediction_error(recent[:index], recent[index])
for index in range(3, int(recent.shape[0]))
]
if not errors:
return 0.0
return float(torch.tensor(errors, dtype=torch.float64).median().item())
def _edge_delta(left: torch.Tensor, right: torch.Tensor) -> float:
left_y = _luma(left)
right_y = _luma(right)
left_x = left_y[:, 1:] - left_y[:, :-1]
right_x = right_y[:, 1:] - right_y[:, :-1]
left_v = left_y[1:, :] - left_y[:-1, :]
right_v = right_y[1:, :] - right_y[:-1, :]
horizontal = torch.mean(torch.abs(left_x - right_x)) if left_x.numel() else left_y.new_tensor(0.0)
vertical = torch.mean(torch.abs(left_v - right_v)) if left_v.numel() else left_y.new_tensor(0.0)
return float(((horizontal + vertical) * 0.5).item())
def _scene_cut_metrics(previous: torch.Tensor, current: torch.Tensor) -> tuple[float, bool]:
cross_delta = _frame_delta(previous[-1], current[0])
previous_internal = _frame_delta(previous[-2], previous[-1])
if int(current.shape[0]) >= 3:
current_internal = 0.5 * (
_frame_delta(current[0], current[1])
+ _frame_delta(current[1], current[2])
)
elif int(current.shape[0]) == 2:
current_internal = _frame_delta(current[0], current[1])
else:
current_internal = cross_delta
denominator = cross_delta + previous_internal + current_internal + 1.0e-8
score = cross_delta / denominator
persistent_new_scene = (
int(current.shape[0]) >= 3
and cross_delta >= 0.12
and current_internal <= cross_delta * 0.35
and cross_delta >= max(0.12, previous_internal * 3.0)
)
return max(0.0, min(1.0, score)), persistent_new_scene
def _transient_flash_metrics(
previous: torch.Tensor,
current: torch.Tensor,
) -> tuple[float, float, float]:
"""Measure a broad boundary exposure pulse that quickly returns."""
previous_y = _luma(previous[-1])
current_y = _luma(current[:FLASH_WINDOW_FRAMES])
shifts = torch.mean(current_y - previous_y.unsqueeze(0), dim=(1, 2))
initial_shift = float(torch.abs(shifts[0]).item())
if int(shifts.shape[0]) > 1 and initial_shift > 1.0e-8:
closest_later = float(torch.min(torch.abs(shifts[1:])).item())
reversal = max(0.0, min(1.0, 1.0 - closest_later / initial_shift))
else:
reversal = 0.0
height, width = int(previous_y.shape[0]), int(previous_y.shape[1])
tile_height = max(1, height // FLASH_TILE_GRID)
tile_width = max(1, width // FLASH_TILE_GRID)
delta = current_y[0] - previous_y
sign = 1.0 if float(shifts[0].item()) >= 0.0 else -1.0
tile_matches = []
for top in range(0, height, tile_height):
for left in range(0, width, tile_width):
tile = delta[
top : min(height, top + tile_height),
left : min(width, left + tile_width),
]
tile_shift = float(torch.mean(tile).item())
tile_matches.append(
tile_shift * sign >= min(MIN_FLASH_LUMA_SHIFT * 0.5, initial_shift * 0.5)
)
global_fraction = sum(tile_matches) / max(1, len(tile_matches))
return initial_shift, reversal, float(global_fraction)
def _temporal_motion_metrics(
previous: torch.Tensor,
current: torch.Tensor,
) -> tuple[float, float, float]:
"""Compare the boundary transition with nearby native frame motion."""
cross_delta = _frame_delta(previous[-1], current[0])
internal_deltas: list[float] = []
previous_start = max(1, int(previous.shape[0]) - 5)
for index in range(previous_start, int(previous.shape[0])):
internal_deltas.append(_frame_delta(previous[index - 1], previous[index]))
current_limit = min(5, int(current.shape[0]))
for index in range(1, current_limit):
internal_deltas.append(_frame_delta(current[index - 1], current[index]))
if internal_deltas:
baseline = float(
torch.tensor(internal_deltas, dtype=torch.float64).median().item()
)
else:
baseline = 0.0
if cross_delta <= 1.0e-8:
ratio = 0.0
else:
ratio = cross_delta / max(baseline, 1.0e-5)
return cross_delta, baseline, ratio
def analyze_video_boundary(
previous_images: torch.Tensor,
current_raw_images: torch.Tensor,
*,
trim_frames: int,
boundary_index: int = 1,
) -> VideoBoundaryAnalysis:
"""Analyze one boundary without modifying either input tensor."""
_validate_frames(previous_images, "previous decoded image chunk")
_validate_frames(current_raw_images, "current decoded image chunk")
trim_frames = int(trim_frames)
if int(previous_images.shape[0]) < 4:
raise ValueError("video seam analysis requires four previous frames")
if trim_frames < 0 or trim_frames >= int(current_raw_images.shape[0]):
raise ValueError("video seam trim position is outside the current chunk")
if trim_frames + 3 > int(current_raw_images.shape[0]):
raise ValueError("video seam analysis requires three retained current frames")
previous = _downsample_rgb(previous_images[-8:])
candidate_start = max(0, trim_frames - MAX_REWIND_FRAMES)
current = _downsample_rgb(
current_raw_images[
candidate_start : trim_frames + FLASH_WINDOW_FRAMES
]
)
nominal_index = trim_frames - candidate_start
nominal = current[nominal_index]
candidate_errors: dict[int, float] = {}
maximum_rewind = min(MAX_REWIND_FRAMES, trim_frames)
for rewind in range(maximum_rewind + 1):
candidate = current[nominal_index - rewind]
candidate_errors[rewind] = _prediction_error(previous, candidate)
nominal_error = candidate_errors[0]
minimum_error = min(candidate_errors.values())
tolerance = minimum_error * SMALLER_SHIFT_TOLERANCE + 1.0e-10
best_rewind = min(
rewind
for rewind, error in candidate_errors.items()
if error <= minimum_error + tolerance
)
best_error = candidate_errors[best_rewind]
baseline_error = _baseline_prediction_error(previous)
anomaly_ratio = nominal_error / max(baseline_error, 1.0e-5)
improvement = (nominal_error - best_error) / max(nominal_error, 1.0e-12)
previous_edge = previous[-1]
previous_y = _luma(previous_edge)
nominal_y = _luma(nominal)
luma_delta = float(torch.mean(torch.abs(previous_y - nominal_y)).item())
previous_chroma = previous_edge - previous_y.unsqueeze(-1)
nominal_chroma = nominal - nominal_y.unsqueeze(-1)
chroma_delta = float(
torch.mean(torch.abs(previous_chroma - nominal_chroma)).item()
)
edge_delta = _edge_delta(previous_edge, nominal)
scene_score, scene_cut = _scene_cut_metrics(
previous,
current[nominal_index : nominal_index + 3],
)
flash_luma_shift, flash_reversal, flash_global_fraction = (
_transient_flash_metrics(previous, current[nominal_index:])
)
cross_frame_delta, motion_baseline_delta, motion_jump_ratio = (
_temporal_motion_metrics(previous, current[nominal_index:])
)
transient_flash_candidate = (
not scene_cut
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
and flash_reversal >= MIN_FLASH_REVERSAL
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
)
micro_flash_candidate = (
not scene_cut
and not transient_flash_candidate
and flash_luma_shift >= MIN_MICRO_FLASH_LUMA_SHIFT
and flash_reversal >= MIN_MICRO_FLASH_REVERSAL
and flash_global_fraction >= MIN_MICRO_FLASH_GLOBAL_FRACTION
)
exposure_ramp_candidate = (
not scene_cut
and not transient_flash_candidate
and not micro_flash_candidate
and anomaly_ratio >= MIN_ANOMALY_RATIO
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
and flash_reversal < MAX_EXPOSURE_RAMP_REVERSAL
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
)
motion_hitch_candidate = (
not scene_cut
and not transient_flash_candidate
and not micro_flash_candidate
and not exposure_ramp_candidate
and cross_frame_delta >= MIN_MOTION_CROSS_DELTA
and motion_jump_ratio >= MIN_MOTION_JUMP_RATIO
)
if scene_cut:
classification = "scene_cut"
elif transient_flash_candidate:
classification = "transient_flash"
elif micro_flash_candidate:
classification = "micro_flash"
elif exposure_ramp_candidate:
classification = "exposure_ramp"
elif motion_hitch_candidate:
classification = "motion_hitch"
else:
classification = "clean_boundary"
recommended_rewind = 0
if scene_cut:
reason = "kept nominal boundary; persistent content change resembles a scene cut"
elif best_rewind == 0:
reason = "kept nominal boundary; it has the lowest prediction error"
elif (
transient_flash_candidate
and flash_reversal >= MIN_AUTO_FLASH_REVERSAL
and improvement >= MIN_AUTO_FLASH_IMPROVEMENT
):
recommended_rewind = best_rewind
reason = f"mild transient flash recommends correcting {best_rewind} frame(s)"
elif anomaly_ratio < MIN_ANOMALY_RATIO:
reason = "kept nominal boundary; anomaly ratio is below the analysis threshold"
elif improvement < MIN_RELATIVE_IMPROVEMENT:
reason = "kept nominal boundary; candidate improvement is below 6%"
else:
recommended_rewind = best_rewind
reason = f"analysis recommends shift {-best_rewind:+d} frame(s)"
return VideoBoundaryAnalysis(
boundary_index=int(boundary_index),
nominal_error=nominal_error,
baseline_error=baseline_error,
anomaly_ratio=anomaly_ratio,
recommended_rewind=recommended_rewind,
best_error=best_error,
improvement=improvement,
luma_delta=luma_delta,
chroma_delta=chroma_delta,
edge_delta=edge_delta,
scene_cut_score=scene_score,
scene_cut=scene_cut,
flash_luma_shift=flash_luma_shift,
flash_reversal=flash_reversal,
flash_global_fraction=flash_global_fraction,
transient_flash_candidate=transient_flash_candidate,
cross_frame_delta=cross_frame_delta,
motion_baseline_delta=motion_baseline_delta,
motion_jump_ratio=motion_jump_ratio,
micro_flash_candidate=micro_flash_candidate,
exposure_ramp_candidate=exposure_ramp_candidate,
motion_hitch_candidate=motion_hitch_candidate,
classification=classification,
reason=reason,
)
def analyze_decoded_boundaries(
*,
images: list[Any],
assembly_plan: dict[str, Any],
) -> list[VideoBoundaryAnalysis]:
"""Analyze every decoded chunk boundary without changing decoded content."""
plan = validate_assembly_plan(assembly_plan)
chunks = list(
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
)
if len(images) != len(chunks):
raise ValueError("decoded image count does not match the assembly plan")
analyses: list[VideoBoundaryAnalysis] = []
for index in range(1, len(chunks)):
previous_raw = images[index - 1]
current_raw = images[index]
previous_plan = chunks[index - 1]
current_plan = chunks[index]
_validate_frames(previous_raw, f"decoded image chunk {index}")
_validate_frames(current_raw, f"decoded image chunk {index + 1}")
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
previous_segment = previous_raw[:previous_total][previous_trim:]
current_segment = current_raw[:current_total]
analyses.append(
analyze_video_boundary(
previous_segment,
current_segment,
trim_frames=current_trim,
boundary_index=index,
)
)
return analyses
def correct_decoded_boundaries(
*,
images: list[Any],
assembly_plan: dict[str, Any],
analyses: list[VideoBoundaryAnalysis],
enable_exposure_ramp: bool = False,
) -> tuple[list[Any], dict[int, str]]:
"""Normalize qualified transient exposure shifts without changing motion."""
plan = validate_assembly_plan(assembly_plan)
chunks = list(
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
)
if len(images) != len(chunks):
raise ValueError("decoded image count does not match the assembly plan")
corrected = list(images)
actions: dict[int, str] = {}
for analysis in analyses:
boundary = int(analysis.boundary_index)
correction_kind = "transient flash"
replace_frames = int(analysis.recommended_rewind)
if analysis.micro_flash_candidate:
correction_kind = "micro-flash"
replace_frames = 1
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
correction_kind = "exposure ramp"
replace_frames = 4
if replace_frames <= 0:
actions[boundary] = "kept native boundary"
continue
if analysis.scene_cut or not (
analysis.transient_flash_candidate
or analysis.micro_flash_candidate
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
):
actions[boundary] = "kept native boundary; correction guard rejected candidate"
continue
previous_index = boundary - 1
current_index = boundary
if previous_index < 0 or current_index >= len(images):
raise ValueError("video seam boundary index is outside decoded chunks")
previous_raw = images[previous_index]
current_raw = images[current_index]
_validate_frames(previous_raw, f"decoded image chunk {previous_index + 1}")
_validate_frames(current_raw, f"decoded image chunk {current_index + 1}")
previous_plan = chunks[previous_index]
current_plan = chunks[current_index]
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
if current_trim + replace_frames >= current_total:
raise ValueError("video seam correction has no retained recovery frame")
previous_segment = previous_raw[:previous_total][previous_trim:]
if int(previous_segment.shape[0]) < 1:
raise ValueError("video seam correction has no previous retained frame")
current_cpu = current_raw.detach().to(device="cpu").clone()
previous_anchor = previous_segment[-1].detach().to(
device="cpu", dtype=torch.float32
)
recovery_anchor = current_cpu[current_trim + replace_frames].to(
dtype=torch.float32
)
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
raise ValueError("decoded image geometry changed at video seam")
for offset in range(replace_frames):
alpha = float(offset + 1) / float(replace_frames + 1)
original = current_cpu[current_trim + offset].to(dtype=torch.float32)
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
original_mean = original.mean(dim=(0, 1), keepdim=True)
target_mean = target.mean(dim=(0, 1), keepdim=True)
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
current_cpu[current_trim + offset].copy_(
normalized.to(dtype=current_cpu.dtype)
)
corrected[current_index] = current_cpu
actions[boundary] = (
f"normalized exposure/color on {replace_frames} {correction_kind} "
"boundary frame(s)"
)
return corrected, actions
def format_video_boundary_analysis(
analysis: VideoBoundaryAnalysis,
*,
action: str = "analysis only",
) -> str:
shift = -int(analysis.recommended_rewind)
return (
f"video seam {analysis.boundary_index}->{analysis.boundary_index + 1}: "
f"anomaly={analysis.anomaly_ratio:.3f}, "
f"prediction={analysis.nominal_error:.6f}->{analysis.best_error:.6f}, "
f"improvement={analysis.improvement:.1%}, shift={shift:+d}, "
f"luma={analysis.luma_delta:.6f}, chroma={analysis.chroma_delta:.6f}, "
f"edge={analysis.edge_delta:.6f}, "
f"scene_cut={'yes' if analysis.scene_cut else 'no'} "
f"flash_shift={analysis.flash_luma_shift:.6f}, "
f"flash_reversal={analysis.flash_reversal:.1%}, "
f"flash_global={analysis.flash_global_fraction:.1%}, "
f"flash_candidate={'yes' if analysis.transient_flash_candidate else 'no'} "
f"class={analysis.classification}, "
f"cross={analysis.cross_frame_delta:.6f}, "
f"motion_base={analysis.motion_baseline_delta:.6f}, "
f"motion_jump={analysis.motion_jump_ratio:.2f}x, "
f"micro_flash={'yes' if analysis.micro_flash_candidate else 'no'}, "
f"exposure_ramp={'yes' if analysis.exposure_ramp_candidate else 'no'}, "
f"motion_hitch={'yes' if analysis.motion_hitch_candidate else 'no'} "
f"({analysis.reason}); action={action}"
)
+131
View File
@@ -0,0 +1,131 @@
"""Small decoded-boundary patches for the V3.5 direct-write assembler.
The established V3.4 seam analyzer remains the source of all boundary
decisions. This module reproduces only the correction step while avoiding the
full physical-group clone performed by ``correct_decoded_boundaries``.
"""
from __future__ import annotations
from typing import Any
import torch
from .plan import validate_assembly_plan
from .video_seam import VideoBoundaryAnalysis, _validate_frames
def build_boundary_patches(
images: list[Any],
assembly_plan: dict[str, Any],
analyses: list[VideoBoundaryAnalysis],
*,
enable_exposure_ramp: bool = False,
) -> tuple[dict[int, torch.Tensor], dict[int, str]]:
"""Return qualified 1--4 frame corrections without changing ``images``.
Patch keys are the existing one-based ``boundary_index`` values. They are
also the zero-based index of the current physical decode group, so the
direct writer can add its natural-frame cursor without this module knowing
the final buffer layout.
The result is transactional: all patches and actions are built in local
containers and are returned only after every requested boundary succeeds.
Any exception therefore leaves the caller with no partial correction set.
"""
plan = validate_assembly_plan(assembly_plan)
groups = list(
plan["decode_groups"] if "decode_groups" in plan else plan["chunks"]
)
if len(images) != len(groups):
raise ValueError("decoded image count does not match the assembly plan")
patches: dict[int, torch.Tensor] = {}
actions: dict[int, str] = {}
for analysis in analyses:
boundary = int(analysis.boundary_index)
correction_kind = "transient flash"
replace_frames = int(analysis.recommended_rewind)
if analysis.micro_flash_candidate:
correction_kind = "micro-flash"
replace_frames = 1
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
correction_kind = "exposure ramp"
replace_frames = 4
if replace_frames <= 0:
actions[boundary] = "kept native boundary"
continue
if analysis.scene_cut or not (
analysis.transient_flash_candidate
or analysis.micro_flash_candidate
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
):
actions[boundary] = (
"kept native boundary; correction guard rejected candidate"
)
continue
previous_index = boundary - 1
current_index = boundary
if previous_index < 0 or current_index >= len(images):
raise ValueError("video seam boundary index is outside decoded chunks")
previous_raw = images[previous_index]
current_raw = images[current_index]
_validate_frames(
previous_raw,
f"decoded image chunk {previous_index + 1}",
)
_validate_frames(
current_raw,
f"decoded image chunk {current_index + 1}",
)
previous_plan = groups[previous_index]
current_plan = groups[current_index]
previous_total = int(previous_plan["total_frames"])
previous_trim = int(previous_plan["trim_frames"])
current_total = int(current_plan["total_frames"])
current_trim = int(current_plan["trim_frames"])
if current_trim + replace_frames >= current_total:
raise ValueError("video seam correction has no retained recovery frame")
previous_segment = previous_raw[:previous_total][previous_trim:]
if int(previous_segment.shape[0]) < 1:
raise ValueError("video seam correction has no previous retained frame")
# Only the frames that may be changed are cloned. The recovery frame
# and previous anchor remain read-only views of their physical groups.
patch = current_raw[
current_trim : current_trim + replace_frames
].detach().to(device="cpu").clone()
if int(patch.shape[0]) != replace_frames:
raise ValueError("video seam correction frames are outside the current chunk")
previous_anchor = previous_segment[-1].detach().to(
device="cpu",
dtype=torch.float32,
)
recovery_anchor = current_raw[current_trim + replace_frames].detach().to(
device="cpu",
dtype=torch.float32,
)
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
raise ValueError("decoded image geometry changed at video seam")
for offset in range(replace_frames):
alpha = float(offset + 1) / float(replace_frames + 1)
original = patch[offset].to(dtype=torch.float32)
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
original_mean = original.mean(dim=(0, 1), keepdim=True)
target_mean = target.mean(dim=(0, 1), keepdim=True)
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
patch[offset].copy_(normalized.to(dtype=patch.dtype))
patches[boundary] = patch
actions[boundary] = (
f"normalized exposure/color on {replace_frames} {correction_kind} "
"boundary frame(s)"
)
return patches, actions