Release H3 Continuum V3.3.0
This commit is contained in:
+132
-2
@@ -20,6 +20,16 @@ from .plan import FPS, validate_assembly_plan
|
||||
AUDIO_SEAM_OFF = "Off"
|
||||
AUDIO_SEAM_AUTO = "Auto"
|
||||
AUDIO_SEAM_OPTIONS = (AUDIO_SEAM_OFF, AUDIO_SEAM_AUTO)
|
||||
VIDEO_SEAM_OFF = "Off"
|
||||
VIDEO_SEAM_ANALYZE = "Analyze Only"
|
||||
VIDEO_SEAM_AUTO = "Auto"
|
||||
VIDEO_SEAM_AUTO_2 = "Auto 2"
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS = (
|
||||
VIDEO_SEAM_AUTO,
|
||||
VIDEO_SEAM_AUTO_2,
|
||||
VIDEO_SEAM_ANALYZE,
|
||||
VIDEO_SEAM_OFF,
|
||||
)
|
||||
|
||||
|
||||
def _singleton(value: Any, name: str) -> Any:
|
||||
@@ -224,7 +234,7 @@ class H3ContinuumAssembleV3:
|
||||
"audio_seam": (
|
||||
AUDIO_SEAM_OPTIONS,
|
||||
{
|
||||
"default": AUDIO_SEAM_OFF,
|
||||
"default": AUDIO_SEAM_AUTO,
|
||||
"display_name": "Audio Seam",
|
||||
"tooltip": (
|
||||
"Auto corrects only decoded audio boundaries; "
|
||||
@@ -274,4 +284,124 @@ _assemble_decoded_chunks_v300 = assemble_decoded_chunks
|
||||
|
||||
|
||||
def assemble_decoded_chunks(*args, **kwargs):
|
||||
return _assemble_with_hardening(_assemble_decoded_chunks_v300, args, kwargs)
|
||||
return _assemble_with_hardening(_assemble_decoded_chunks_v300, args, kwargs)
|
||||
|
||||
|
||||
class H3ContinuumAssembleSeamExperimental(H3ContinuumAssembleV3):
|
||||
DESCRIPTION = (
|
||||
"Analyze decoded chunk boundaries and apply guarded video seam correction."
|
||||
)
|
||||
SEARCH_ALIASES = ["H3 seam analysis", "H3 boundary flicker analysis"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
schema = super().INPUT_TYPES()
|
||||
required = dict(schema["required"])
|
||||
diagnostics = required.pop("diagnostics")
|
||||
required["video_seam"] = (
|
||||
VIDEO_SEAM_ANALYSIS_OPTIONS,
|
||||
{
|
||||
"default": VIDEO_SEAM_AUTO,
|
||||
"display_name": "Video Seam",
|
||||
"tooltip": (
|
||||
"Analyze Only reports decoded video boundaries without changing them. "
|
||||
"Auto applies the validated transient and micro-flash correction. "
|
||||
"Auto 2 experimentally adds qualified exposure-ramp smoothing."
|
||||
),
|
||||
},
|
||||
)
|
||||
required["diagnostics"] = diagnostics
|
||||
schema["required"] = required
|
||||
return schema
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
video_seam,
|
||||
diagnostics,
|
||||
):
|
||||
mode = str(_singleton(video_seam, "video_seam"))
|
||||
if mode not in VIDEO_SEAM_ANALYSIS_OPTIONS:
|
||||
raise ValueError(f"unknown Video Seam mode: {mode!r}")
|
||||
if mode == VIDEO_SEAM_OFF:
|
||||
return super().assemble(
|
||||
images,
|
||||
audio,
|
||||
assembly_plan,
|
||||
exact_total_duration,
|
||||
audio_seam,
|
||||
diagnostics,
|
||||
)
|
||||
|
||||
image_chunks = _chunk_list(images)
|
||||
plan = _singleton(assembly_plan, "assembly_plan")
|
||||
analyses = []
|
||||
analysis_error = None
|
||||
actions = {}
|
||||
try:
|
||||
from .video_seam import (
|
||||
analyze_decoded_boundaries,
|
||||
correct_decoded_boundaries,
|
||||
format_video_boundary_analysis,
|
||||
)
|
||||
|
||||
analyses = analyze_decoded_boundaries(
|
||||
images=image_chunks,
|
||||
assembly_plan=plan,
|
||||
)
|
||||
if mode in (VIDEO_SEAM_AUTO, VIDEO_SEAM_AUTO_2):
|
||||
image_chunks, actions = correct_decoded_boundaries(
|
||||
images=image_chunks,
|
||||
assembly_plan=plan,
|
||||
analyses=analyses,
|
||||
enable_exposure_ramp=mode == VIDEO_SEAM_AUTO_2,
|
||||
)
|
||||
except Exception as exc:
|
||||
analysis_error = exc
|
||||
|
||||
result_images, result_audio, report = assemble_decoded_chunks(
|
||||
images=image_chunks,
|
||||
audio=_chunk_list(audio),
|
||||
assembly_plan=plan,
|
||||
exact_total_duration=bool(
|
||||
_singleton(exact_total_duration, "exact_total_duration")
|
||||
),
|
||||
audio_seam=str(_singleton(audio_seam, "audio_seam")),
|
||||
diagnostics=str(_singleton(diagnostics, "diagnostics")),
|
||||
)
|
||||
if mode == VIDEO_SEAM_ANALYZE:
|
||||
status = "Video Seam: Analyze Only; decoded frames and audio are unchanged."
|
||||
elif mode == VIDEO_SEAM_AUTO:
|
||||
status = (
|
||||
"Video Seam: Auto; guarded transient and micro-flash correction enabled."
|
||||
)
|
||||
else:
|
||||
status = (
|
||||
"Video Seam: Auto 2 (Experimental); guarded transient, micro-flash, "
|
||||
"and exposure-ramp correction enabled."
|
||||
)
|
||||
report = report.replace("Video seam correction is disabled.", status, 1)
|
||||
if analysis_error is None:
|
||||
lines = [
|
||||
format_video_boundary_analysis(
|
||||
item,
|
||||
action=(
|
||||
"analysis only"
|
||||
if mode == VIDEO_SEAM_ANALYZE
|
||||
else actions.get(item.boundary_index, "kept native boundary")
|
||||
),
|
||||
)
|
||||
for item in analyses
|
||||
]
|
||||
if not lines:
|
||||
lines = [f"Video Seam {mode}: no decoded chunk boundary to analyze."]
|
||||
else:
|
||||
lines = [
|
||||
f"Video Seam {mode}: native output preserved; analysis unavailable "
|
||||
f"({type(analysis_error).__name__}: {analysis_error})"
|
||||
]
|
||||
return result_images, result_audio, report.rstrip() + "\n" + "\n".join(lines)
|
||||
|
||||
+112
-24
@@ -12,8 +12,9 @@ from ..v2.nodes import (
|
||||
V2_CONTINUITY_OPTIONS,
|
||||
validate_sparse_prompt_overrides,
|
||||
)
|
||||
from .assembly import H3ContinuumAssembleV3
|
||||
from .assembly import H3ContinuumAssembleSeamExperimental, H3ContinuumAssembleV3
|
||||
from .plan import make_assembly_plan
|
||||
from ..timeline_video import TIMELINE_VIDEO_SIZE_OPTIONS
|
||||
|
||||
|
||||
REGENERATE_AUTO = "Auto"
|
||||
@@ -53,24 +54,12 @@ def _validate_regenerate_storage(run_storage: str, regenerate_from: int) -> None
|
||||
def _validate_reference_checkpoint(
|
||||
prompt, unique_id, *, strict_compatibility: bool
|
||||
) -> str:
|
||||
from ..graph_contract import (
|
||||
CHECKPOINT_FL2VA,
|
||||
CHECKPOINT_REF2VA,
|
||||
classify_h3_checkpoint,
|
||||
)
|
||||
from ..graph_contract import classify_h3_checkpoint
|
||||
|
||||
classification = classify_h3_checkpoint(prompt, unique_id)
|
||||
if strict_compatibility and classification != CHECKPOINT_REF2VA:
|
||||
if classification == CHECKPOINT_FL2VA:
|
||||
detail = "an FL2VA checkpoint is connected"
|
||||
else:
|
||||
detail = "the base checkpoint cannot be verified as Ref2VA"
|
||||
raise ValueError(
|
||||
"Reference Images require a verified MiniMax H3 Ref2VA MODEL when "
|
||||
f"Strict Compatibility is enabled; {detail}. Disable Strict "
|
||||
"Compatibility only for experimental FL2VA/unknown combinations."
|
||||
)
|
||||
return classification
|
||||
# Checkpoint choice is diagnostic only. Strict Compatibility remains
|
||||
# reserved for H3 contracts that are actually unsafe or unsupported.
|
||||
_ = strict_compatibility
|
||||
return classify_h3_checkpoint(prompt, unique_id)
|
||||
|
||||
|
||||
class H3ContinuumAdvancedV3:
|
||||
@@ -269,6 +258,9 @@ class H3ContinuumSamplerV3:
|
||||
prompt_overrides=None,
|
||||
advanced=None,
|
||||
reference_assets=None,
|
||||
reference_audio_source=None,
|
||||
reference_audio_vae=None,
|
||||
timeline_video_source=None,
|
||||
):
|
||||
if prompt_overrides is not None and not isinstance(prompt_overrides, dict):
|
||||
raise TypeError(
|
||||
@@ -339,6 +331,9 @@ class H3ContinuumSamplerV3:
|
||||
show_preview=advanced_values["show_preview"],
|
||||
latent_only=True,
|
||||
reference_assets=reference_assets,
|
||||
reference_audio_source=reference_audio_source,
|
||||
reference_audio_vae=reference_audio_vae,
|
||||
timeline_video_source=timeline_video_source,
|
||||
**clip_prompt_inputs,
|
||||
)
|
||||
|
||||
@@ -539,6 +534,9 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
"last_frame": ("IMAGE",),
|
||||
"reference_image_1": ("IMAGE",),
|
||||
"reference_image_2": ("IMAGE",),
|
||||
"reference_image_3": ("IMAGE",),
|
||||
"reference_audio_1": ("AUDIO",),
|
||||
"reference_audio_vae": ("VAE",),
|
||||
},
|
||||
"hidden": {
|
||||
"prompt": "PROMPT",
|
||||
@@ -579,8 +577,13 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
prompt_overrides=None,
|
||||
prompt=None,
|
||||
unique_id=None,
|
||||
reference_image_3=None,
|
||||
reference_audio_1=None,
|
||||
reference_audio_vae=None,
|
||||
timeline_video_source=None,
|
||||
):
|
||||
from ..reference import prepare_reference_assets
|
||||
from ..reference_audio import prepare_reference_audio_source
|
||||
regenerate_from = _regenerate_from_value(
|
||||
reroll_from_chunk,
|
||||
chunks=int(chunks),
|
||||
@@ -591,13 +594,18 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
output_width=int(width),
|
||||
output_height=int(height),
|
||||
size_mode=reference_size,
|
||||
reference_image_3=reference_image_3,
|
||||
)
|
||||
reference_audio_source = prepare_reference_audio_source(
|
||||
reference_audio_1,
|
||||
reference_audio_vae,
|
||||
)
|
||||
if reference_assets is not None and (first_frame is not None or last_frame is not None):
|
||||
raise ValueError(
|
||||
"V3.2 Reference Images cannot be combined with First Frame or Last Frame"
|
||||
"Reference Images cannot be combined with First Frame or Last Frame"
|
||||
)
|
||||
reference_checkpoint = None
|
||||
if reference_assets is not None:
|
||||
if reference_assets is not None or timeline_video_source is not None:
|
||||
reference_checkpoint = _validate_reference_checkpoint(
|
||||
prompt,
|
||||
unique_id,
|
||||
@@ -609,11 +617,10 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
return str(report)
|
||||
if reference_checkpoint == "ref2va":
|
||||
status = "Ref2VA verified."
|
||||
elif reference_checkpoint == "fl2va":
|
||||
status = "FL2VA; allowed, but reference fidelity may differ from Ref2VA."
|
||||
else:
|
||||
status = (
|
||||
f"{reference_checkpoint}; Experimental because Strict "
|
||||
"Compatibility is disabled."
|
||||
)
|
||||
status = "unverified; execution allowed, Core contract errors remain fatal."
|
||||
return str(report) + "\nReference MODEL: " + status
|
||||
|
||||
def execute():
|
||||
@@ -634,6 +641,9 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
first_frame=first_frame,
|
||||
prompt_overrides=prompt_overrides,
|
||||
reference_assets=reference_assets,
|
||||
reference_audio_source=reference_audio_source,
|
||||
reference_audio_vae=reference_audio_vae,
|
||||
timeline_video_source=timeline_video_source,
|
||||
advanced={
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"diagnostics": diagnostics,
|
||||
@@ -680,16 +690,94 @@ class H3ContinuumSamplerProduction(H3ContinuumSamplerV3):
|
||||
return video_latents, audio_latents, assembly_plan, report
|
||||
|
||||
|
||||
class H3ContinuumSamplerTimelineVideo(H3ContinuumSamplerProduction):
|
||||
"""Chunk-local Timeline Video facade over the stable engine."""
|
||||
|
||||
DEPRECATED = False
|
||||
DESCRIPTION = (
|
||||
"Timeline Video sampler. It slices one Core VIDEO per chunk, "
|
||||
"resizes only that slice, and reuses the stable Continuum sampling engine."
|
||||
)
|
||||
SEARCH_ALIASES = [
|
||||
"H3 Continuum Timeline Video",
|
||||
"MiniMax H3 video reference timeline",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
schema = super().INPUT_TYPES()
|
||||
required = {}
|
||||
for name, definition in schema["required"].items():
|
||||
required[name] = definition
|
||||
if name == "reference_size":
|
||||
required["timeline_video_size"] = (
|
||||
TIMELINE_VIDEO_SIZE_OPTIONS,
|
||||
{
|
||||
"default": TIMELINE_VIDEO_SIZE_OPTIONS[0],
|
||||
"display_name": "Timeline Video Size",
|
||||
"advanced": True,
|
||||
"tooltip": (
|
||||
"Efficient limits each chunk-local reference slice to about 0.4 MP. "
|
||||
"Match Output uses the output pixel area and may be substantially heavier."
|
||||
),
|
||||
},
|
||||
)
|
||||
schema["required"] = required
|
||||
optional = dict(schema.get("optional", {}))
|
||||
optional["timeline_video"] = (
|
||||
"VIDEO",
|
||||
{
|
||||
"tooltip": (
|
||||
"Optional. One continuous Core VIDEO covering every configured chunk. "
|
||||
"When omitted or bypassed, the node uses the standard conditioning path. "
|
||||
"Its audio is ignored; use the normal audio inputs for generated audio."
|
||||
)
|
||||
},
|
||||
)
|
||||
schema["optional"] = optional
|
||||
return schema
|
||||
|
||||
def run(
|
||||
self,
|
||||
timeline_video=None,
|
||||
timeline_video_size=TIMELINE_VIDEO_SIZE_OPTIONS[0],
|
||||
**kwargs,
|
||||
):
|
||||
if timeline_video is None:
|
||||
return super().run(**kwargs)
|
||||
|
||||
from ..timeline_video import prepare_timeline_video_source
|
||||
|
||||
source = prepare_timeline_video_source(
|
||||
timeline_video,
|
||||
chunks=int(kwargs["chunks"]),
|
||||
chunk_seconds=float(kwargs["chunk_seconds"]),
|
||||
output_width=int(kwargs["width"]),
|
||||
output_height=int(kwargs["height"]),
|
||||
size_mode=str(timeline_video_size),
|
||||
)
|
||||
return super().run(timeline_video_source=source, **kwargs)
|
||||
|
||||
|
||||
from .seam_test_source import H3ContinuumSeamTestSource
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"H3ContinuumSamplerProduction": H3ContinuumSamplerProduction,
|
||||
"H3ContinuumSamplerTimelineVideo": H3ContinuumSamplerTimelineVideo,
|
||||
"H3ContinuumSamplerV3": H3ContinuumSamplerV3,
|
||||
"H3ContinuumAdvancedV3": H3ContinuumAdvancedV3,
|
||||
"H3ContinuumAssembleV3": H3ContinuumAssembleV3,
|
||||
"H3ContinuumAssembleSeamExperimental": H3ContinuumAssembleSeamExperimental,
|
||||
"H3ContinuumSeamTestSource": H3ContinuumSeamTestSource,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"H3ContinuumSamplerProduction": "H3 Continuum Sampler V3.2",
|
||||
"H3ContinuumSamplerTimelineVideo": "H3 Continuum Timeline Video",
|
||||
"H3ContinuumSamplerV3": "H3 Continuum Sampler V3",
|
||||
"H3ContinuumAdvancedV3": "H3 Continuum Advanced V3",
|
||||
"H3ContinuumAssembleV3": "H3 Continuum Assemble V3",
|
||||
"H3ContinuumAssembleSeamExperimental": "H3 Continuum Assemble + Seam",
|
||||
"H3ContinuumSeamTestSource": "H3 Continuum Seam Test Source (Experimental)",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Deterministic decoded-chunk source for video seam validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from ..hardening import enrich_assembly_plan
|
||||
from .plan import (
|
||||
ASSEMBLY_PLAN_MAGIC,
|
||||
ASSEMBLY_PLAN_SCHEMA_VERSION,
|
||||
FPS,
|
||||
validate_assembly_plan,
|
||||
)
|
||||
|
||||
|
||||
TEST_SIZE_256 = "256 px (Fast)"
|
||||
TEST_SIZE_512 = "512 px"
|
||||
TEST_SIZE_INPUT = "Input Size"
|
||||
TEST_SIZE_OPTIONS = (TEST_SIZE_256, TEST_SIZE_512, TEST_SIZE_INPUT)
|
||||
|
||||
|
||||
def _prepare_image(image: Any, size_mode: str) -> torch.Tensor:
|
||||
if not torch.is_tensor(image) or image.ndim != 4 or int(image.shape[-1]) < 3:
|
||||
raise ValueError("image must be an IMAGE tensor [B,H,W,C]")
|
||||
if int(image.shape[0]) != 1:
|
||||
raise ValueError("Seam Test Source requires exactly one input image")
|
||||
|
||||
frame = image[0, ..., :3].detach().to(device="cpu", dtype=torch.float32)
|
||||
if not bool(torch.isfinite(frame).all().item()):
|
||||
raise ValueError("input image contains NaN or Inf")
|
||||
frame = frame.clamp(0.0, 1.0)
|
||||
if size_mode == TEST_SIZE_INPUT:
|
||||
return frame.contiguous()
|
||||
|
||||
limits = {TEST_SIZE_256: 256, TEST_SIZE_512: 512}
|
||||
if size_mode not in limits:
|
||||
raise ValueError(f"unknown Test Resolution: {size_mode!r}")
|
||||
height, width = int(frame.shape[0]), int(frame.shape[1])
|
||||
scale = min(1.0, float(limits[size_mode]) / float(max(height, width)))
|
||||
target_height = max(1, int(round(height * scale)))
|
||||
target_width = max(1, int(round(width * scale)))
|
||||
if (target_height, target_width) == (height, width):
|
||||
return frame.contiguous()
|
||||
resized = F.interpolate(
|
||||
frame.permute(2, 0, 1).unsqueeze(0),
|
||||
size=(target_height, target_width),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
return resized[0].permute(1, 2, 0).contiguous()
|
||||
|
||||
|
||||
def _repeat_frame(frame: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return frame.unsqueeze(0).expand(int(count), -1, -1, -1).clone()
|
||||
|
||||
|
||||
def _ramp_chunk(
|
||||
previous_anchor: torch.Tensor,
|
||||
*,
|
||||
context_frames: int,
|
||||
net_frames: int,
|
||||
exposure_change: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
context = _repeat_frame(previous_anchor, context_frames)
|
||||
ramp_count = 5
|
||||
ramp_factors = torch.linspace(
|
||||
1.0 - exposure_change,
|
||||
1.0 - exposure_change * 2.0,
|
||||
steps=ramp_count,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
ramp = (
|
||||
previous_anchor.unsqueeze(0)
|
||||
* ramp_factors.reshape(-1, 1, 1, 1)
|
||||
).clamp(0.0, 1.0)
|
||||
next_anchor = ramp[-1].contiguous()
|
||||
tail = _repeat_frame(next_anchor, net_frames - ramp_count)
|
||||
return torch.cat((context, ramp, tail), dim=0).contiguous(), next_anchor
|
||||
|
||||
|
||||
def _silent_audio(total_frames: int, sample_rate: int) -> dict[str, Any]:
|
||||
samples = int(math.ceil(total_frames / FPS * sample_rate)) + 8
|
||||
return {
|
||||
"waveform": torch.zeros((1, 2, samples), dtype=torch.float32),
|
||||
"sample_rate": int(sample_rate),
|
||||
}
|
||||
|
||||
|
||||
def _assembly_plan(width: int, height: int) -> dict[str, Any]:
|
||||
chunks = [
|
||||
{
|
||||
"sequence_index": 1,
|
||||
"chunk_index": 1,
|
||||
"total_frames": 124,
|
||||
"trim_frames": 0,
|
||||
"net_frames": 124,
|
||||
"context_frames": 0,
|
||||
"expected_video_latent_t": 32,
|
||||
"expected_audio_latent_t": 207,
|
||||
},
|
||||
{
|
||||
"sequence_index": 2,
|
||||
"chunk_index": 2,
|
||||
"total_frames": 141,
|
||||
"trim_frames": 22,
|
||||
"net_frames": 119,
|
||||
"context_frames": 22,
|
||||
"expected_video_latent_t": 36,
|
||||
"expected_audio_latent_t": 235,
|
||||
},
|
||||
{
|
||||
"sequence_index": 3,
|
||||
"chunk_index": 3,
|
||||
"total_frames": 141,
|
||||
"trim_frames": 22,
|
||||
"net_frames": 119,
|
||||
"context_frames": 22,
|
||||
"expected_video_latent_t": 36,
|
||||
"expected_audio_latent_t": 235,
|
||||
},
|
||||
]
|
||||
plan = {
|
||||
"magic": ASSEMBLY_PLAN_MAGIC,
|
||||
"schema_version": ASSEMBLY_PLAN_SCHEMA_VERSION,
|
||||
"fps": FPS,
|
||||
"width": int(width),
|
||||
"height": int(height),
|
||||
"chunk_seconds": 5.0,
|
||||
"target_frames": 360,
|
||||
"preserve_final_frame": True,
|
||||
"chunks": chunks,
|
||||
}
|
||||
enrich_assembly_plan(plan)
|
||||
return validate_assembly_plan(plan)
|
||||
|
||||
|
||||
class H3ContinuumSeamTestSource:
|
||||
DESCRIPTION = (
|
||||
"Create deterministic decoded chunks for testing Video Seam modes. "
|
||||
"Chunk 2 and Chunk 3 each begin with a controlled exposure ramp. "
|
||||
"This node performs no H3 sampling."
|
||||
)
|
||||
SEARCH_ALIASES = ["H3 seam test", "H3 exposure ramp test"]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"exposure_change": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.08,
|
||||
"min": 0.01,
|
||||
"max": 0.20,
|
||||
"step": 0.01,
|
||||
"display_name": "Exposure Change",
|
||||
"tooltip": (
|
||||
"8% is the controlled default. Each new chunk ramps "
|
||||
"from one step darker to two steps darker."
|
||||
),
|
||||
},
|
||||
),
|
||||
"test_resolution": (
|
||||
TEST_SIZE_OPTIONS,
|
||||
{
|
||||
"default": TEST_SIZE_256,
|
||||
"display_name": "Test Resolution",
|
||||
"tooltip": (
|
||||
"256 px is recommended for fast seam testing. "
|
||||
"Input Size can require several GiB of RAM."
|
||||
),
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (
|
||||
"IMAGE",
|
||||
"AUDIO",
|
||||
"H3_CONTINUUM_ASSEMBLY_PLAN",
|
||||
"STRING",
|
||||
)
|
||||
RETURN_NAMES = ("images", "audio", "assembly_plan", "instructions")
|
||||
OUTPUT_IS_LIST = (True, True, False, False)
|
||||
FUNCTION = "create"
|
||||
CATEGORY = "MiniMax H3/Continuum/Testing"
|
||||
EXPERIMENTAL = True
|
||||
|
||||
def create(
|
||||
self,
|
||||
image,
|
||||
exposure_change=0.08,
|
||||
test_resolution=TEST_SIZE_256,
|
||||
):
|
||||
change = float(exposure_change)
|
||||
if not math.isfinite(change) or not 0.01 <= change <= 0.20:
|
||||
raise ValueError("Exposure Change must be between 0.01 and 0.20")
|
||||
base = _prepare_image(image, str(test_resolution))
|
||||
chunk_1 = _repeat_frame(base, 124)
|
||||
chunk_2, anchor_2 = _ramp_chunk(
|
||||
base,
|
||||
context_frames=22,
|
||||
net_frames=119,
|
||||
exposure_change=change,
|
||||
)
|
||||
chunk_3, _anchor_3 = _ramp_chunk(
|
||||
anchor_2,
|
||||
context_frames=22,
|
||||
net_frames=119,
|
||||
exposure_change=change,
|
||||
)
|
||||
images = [chunk_1, chunk_2, chunk_3]
|
||||
audio = [
|
||||
_silent_audio(124, 32000),
|
||||
_silent_audio(141, 32000),
|
||||
_silent_audio(141, 32000),
|
||||
]
|
||||
plan = _assembly_plan(int(base.shape[1]), int(base.shape[0]))
|
||||
instructions = (
|
||||
"H3 Continuum Seam Test Source\n"
|
||||
"1. Connect images, audio, and assembly_plan to the matching inputs "
|
||||
"of H3 Continuum Assemble + Seam.\n"
|
||||
"2. Set Audio Seam=Off and exact_total_duration=true.\n"
|
||||
"3. Run Video Seam=Analyze Only, then Auto, then Auto 2.\n"
|
||||
"4. Analyze Only should report class=exposure_ramp at both boundaries.\n"
|
||||
"5. Auto should keep both native boundaries. Auto 2 should report "
|
||||
"normalized exposure/color on 4 exposure ramp boundary frame(s).\n"
|
||||
f"Source: 3 x 5 sec, 24 fps, {base.shape[1]}x{base.shape[0]}, "
|
||||
f"exposure change={change:.0%}."
|
||||
)
|
||||
return images, audio, plan, instructions
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Read-only decoded video-boundary analysis for H3 Continuum V3."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .plan import validate_assembly_plan
|
||||
|
||||
|
||||
COMPARE_LONG_SIDE = 192
|
||||
MAX_REWIND_FRAMES = 2
|
||||
FLASH_WINDOW_FRAMES = 6
|
||||
FLASH_TILE_GRID = 4
|
||||
MIN_FLASH_LUMA_SHIFT = 0.01
|
||||
MIN_FLASH_REVERSAL = 0.35
|
||||
MIN_FLASH_GLOBAL_FRACTION = 0.75
|
||||
MIN_ANOMALY_RATIO = 1.35
|
||||
MIN_RELATIVE_IMPROVEMENT = 0.06
|
||||
MIN_AUTO_FLASH_REVERSAL = 0.70
|
||||
MIN_AUTO_FLASH_IMPROVEMENT = 0.70
|
||||
SMALLER_SHIFT_TOLERANCE = 0.03
|
||||
MIN_MICRO_FLASH_LUMA_SHIFT = 0.003
|
||||
MIN_MICRO_FLASH_REVERSAL = 0.25
|
||||
MIN_MICRO_FLASH_GLOBAL_FRACTION = 0.70
|
||||
MAX_EXPOSURE_RAMP_REVERSAL = 0.25
|
||||
MIN_MOTION_CROSS_DELTA = 0.006
|
||||
MIN_MOTION_JUMP_RATIO = 1.60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VideoBoundaryAnalysis:
|
||||
boundary_index: int
|
||||
nominal_error: float
|
||||
baseline_error: float
|
||||
anomaly_ratio: float
|
||||
recommended_rewind: int
|
||||
best_error: float
|
||||
improvement: float
|
||||
luma_delta: float
|
||||
chroma_delta: float
|
||||
edge_delta: float
|
||||
scene_cut_score: float
|
||||
scene_cut: bool
|
||||
flash_luma_shift: float
|
||||
flash_reversal: float
|
||||
flash_global_fraction: float
|
||||
transient_flash_candidate: bool
|
||||
cross_frame_delta: float
|
||||
motion_baseline_delta: float
|
||||
motion_jump_ratio: float
|
||||
micro_flash_candidate: bool
|
||||
exposure_ramp_candidate: bool
|
||||
motion_hitch_candidate: bool
|
||||
classification: str
|
||||
reason: str
|
||||
|
||||
|
||||
def _validate_frames(frames: Any, label: str) -> None:
|
||||
if not torch.is_tensor(frames) or frames.ndim != 4 or frames.shape[-1] < 3:
|
||||
raise ValueError(f"{label} must be an IMAGE tensor [T,H,W,C]")
|
||||
if int(frames.shape[0]) < 1 or int(frames.shape[1]) < 1 or int(frames.shape[2]) < 1:
|
||||
raise ValueError(f"{label} must not be empty")
|
||||
|
||||
|
||||
def _downsample_rgb(frames: torch.Tensor) -> torch.Tensor:
|
||||
_validate_frames(frames, "video seam frames")
|
||||
rgb = frames[..., :3].detach().to(device="cpu", dtype=torch.float32)
|
||||
rgb = rgb.permute(0, 3, 1, 2)
|
||||
height, width = int(rgb.shape[-2]), int(rgb.shape[-1])
|
||||
scale = min(1.0, COMPARE_LONG_SIDE / float(max(height, width)))
|
||||
target_height = max(1, int(round(height * scale)))
|
||||
target_width = max(1, int(round(width * scale)))
|
||||
if (target_height, target_width) != (height, width):
|
||||
rgb = F.interpolate(
|
||||
rgb,
|
||||
size=(target_height, target_width),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
rgb = rgb.permute(0, 2, 3, 1).clamp(0.0, 1.0)
|
||||
if not bool(torch.isfinite(rgb).all().item()):
|
||||
raise ValueError("video seam comparison contains NaN or Inf")
|
||||
return rgb
|
||||
|
||||
|
||||
def _luma(frame: torch.Tensor) -> torch.Tensor:
|
||||
weights = frame.new_tensor((0.2126, 0.7152, 0.0722))
|
||||
return torch.sum(frame * weights, dim=-1)
|
||||
|
||||
|
||||
def _frame_delta(left: torch.Tensor, right: torch.Tensor) -> float:
|
||||
return float(torch.mean(torch.abs(left - right)).item())
|
||||
|
||||
|
||||
def _prediction_error(history: torch.Tensor, candidate: torch.Tensor) -> float:
|
||||
if int(history.shape[0]) < 3:
|
||||
raise ValueError("temporal prediction requires three history frames")
|
||||
recent_velocity = history[-1] - history[-2]
|
||||
prior_velocity = history[-2] - history[-3]
|
||||
predicted = (
|
||||
history[-1] + recent_velocity * 0.65 + prior_velocity * 0.35
|
||||
).clamp(0.0, 1.0)
|
||||
motion = torch.mean(torch.abs(recent_velocity), dim=-1)
|
||||
motion_scale = motion / motion.mean().clamp_min(1.0e-6)
|
||||
weights = (1.0 + motion_scale).clamp(1.0, 4.0).unsqueeze(-1)
|
||||
value = float(torch.mean((predicted - candidate).square() * weights).item())
|
||||
if not math.isfinite(value):
|
||||
raise ValueError("video seam prediction score is not finite")
|
||||
return max(0.0, value)
|
||||
|
||||
|
||||
def _baseline_prediction_error(previous: torch.Tensor) -> float:
|
||||
recent = previous[-min(8, int(previous.shape[0])) :]
|
||||
errors = [
|
||||
_prediction_error(recent[:index], recent[index])
|
||||
for index in range(3, int(recent.shape[0]))
|
||||
]
|
||||
if not errors:
|
||||
return 0.0
|
||||
return float(torch.tensor(errors, dtype=torch.float64).median().item())
|
||||
|
||||
|
||||
def _edge_delta(left: torch.Tensor, right: torch.Tensor) -> float:
|
||||
left_y = _luma(left)
|
||||
right_y = _luma(right)
|
||||
left_x = left_y[:, 1:] - left_y[:, :-1]
|
||||
right_x = right_y[:, 1:] - right_y[:, :-1]
|
||||
left_v = left_y[1:, :] - left_y[:-1, :]
|
||||
right_v = right_y[1:, :] - right_y[:-1, :]
|
||||
horizontal = torch.mean(torch.abs(left_x - right_x)) if left_x.numel() else left_y.new_tensor(0.0)
|
||||
vertical = torch.mean(torch.abs(left_v - right_v)) if left_v.numel() else left_y.new_tensor(0.0)
|
||||
return float(((horizontal + vertical) * 0.5).item())
|
||||
|
||||
|
||||
def _scene_cut_metrics(previous: torch.Tensor, current: torch.Tensor) -> tuple[float, bool]:
|
||||
cross_delta = _frame_delta(previous[-1], current[0])
|
||||
previous_internal = _frame_delta(previous[-2], previous[-1])
|
||||
if int(current.shape[0]) >= 3:
|
||||
current_internal = 0.5 * (
|
||||
_frame_delta(current[0], current[1])
|
||||
+ _frame_delta(current[1], current[2])
|
||||
)
|
||||
elif int(current.shape[0]) == 2:
|
||||
current_internal = _frame_delta(current[0], current[1])
|
||||
else:
|
||||
current_internal = cross_delta
|
||||
denominator = cross_delta + previous_internal + current_internal + 1.0e-8
|
||||
score = cross_delta / denominator
|
||||
persistent_new_scene = (
|
||||
int(current.shape[0]) >= 3
|
||||
and cross_delta >= 0.12
|
||||
and current_internal <= cross_delta * 0.35
|
||||
and cross_delta >= max(0.12, previous_internal * 3.0)
|
||||
)
|
||||
return max(0.0, min(1.0, score)), persistent_new_scene
|
||||
|
||||
|
||||
def _transient_flash_metrics(
|
||||
previous: torch.Tensor,
|
||||
current: torch.Tensor,
|
||||
) -> tuple[float, float, float]:
|
||||
"""Measure a broad boundary exposure pulse that quickly returns."""
|
||||
|
||||
previous_y = _luma(previous[-1])
|
||||
current_y = _luma(current[:FLASH_WINDOW_FRAMES])
|
||||
shifts = torch.mean(current_y - previous_y.unsqueeze(0), dim=(1, 2))
|
||||
initial_shift = float(torch.abs(shifts[0]).item())
|
||||
if int(shifts.shape[0]) > 1 and initial_shift > 1.0e-8:
|
||||
closest_later = float(torch.min(torch.abs(shifts[1:])).item())
|
||||
reversal = max(0.0, min(1.0, 1.0 - closest_later / initial_shift))
|
||||
else:
|
||||
reversal = 0.0
|
||||
|
||||
height, width = int(previous_y.shape[0]), int(previous_y.shape[1])
|
||||
tile_height = max(1, height // FLASH_TILE_GRID)
|
||||
tile_width = max(1, width // FLASH_TILE_GRID)
|
||||
delta = current_y[0] - previous_y
|
||||
sign = 1.0 if float(shifts[0].item()) >= 0.0 else -1.0
|
||||
tile_matches = []
|
||||
for top in range(0, height, tile_height):
|
||||
for left in range(0, width, tile_width):
|
||||
tile = delta[
|
||||
top : min(height, top + tile_height),
|
||||
left : min(width, left + tile_width),
|
||||
]
|
||||
tile_shift = float(torch.mean(tile).item())
|
||||
tile_matches.append(
|
||||
tile_shift * sign >= min(MIN_FLASH_LUMA_SHIFT * 0.5, initial_shift * 0.5)
|
||||
)
|
||||
global_fraction = sum(tile_matches) / max(1, len(tile_matches))
|
||||
return initial_shift, reversal, float(global_fraction)
|
||||
|
||||
|
||||
def _temporal_motion_metrics(
|
||||
previous: torch.Tensor,
|
||||
current: torch.Tensor,
|
||||
) -> tuple[float, float, float]:
|
||||
"""Compare the boundary transition with nearby native frame motion."""
|
||||
|
||||
cross_delta = _frame_delta(previous[-1], current[0])
|
||||
internal_deltas: list[float] = []
|
||||
previous_start = max(1, int(previous.shape[0]) - 5)
|
||||
for index in range(previous_start, int(previous.shape[0])):
|
||||
internal_deltas.append(_frame_delta(previous[index - 1], previous[index]))
|
||||
current_limit = min(5, int(current.shape[0]))
|
||||
for index in range(1, current_limit):
|
||||
internal_deltas.append(_frame_delta(current[index - 1], current[index]))
|
||||
if internal_deltas:
|
||||
baseline = float(
|
||||
torch.tensor(internal_deltas, dtype=torch.float64).median().item()
|
||||
)
|
||||
else:
|
||||
baseline = 0.0
|
||||
if cross_delta <= 1.0e-8:
|
||||
ratio = 0.0
|
||||
else:
|
||||
ratio = cross_delta / max(baseline, 1.0e-5)
|
||||
return cross_delta, baseline, ratio
|
||||
|
||||
|
||||
def analyze_video_boundary(
|
||||
previous_images: torch.Tensor,
|
||||
current_raw_images: torch.Tensor,
|
||||
*,
|
||||
trim_frames: int,
|
||||
boundary_index: int = 1,
|
||||
) -> VideoBoundaryAnalysis:
|
||||
"""Analyze one boundary without modifying either input tensor."""
|
||||
|
||||
_validate_frames(previous_images, "previous decoded image chunk")
|
||||
_validate_frames(current_raw_images, "current decoded image chunk")
|
||||
trim_frames = int(trim_frames)
|
||||
if int(previous_images.shape[0]) < 4:
|
||||
raise ValueError("video seam analysis requires four previous frames")
|
||||
if trim_frames < 0 or trim_frames >= int(current_raw_images.shape[0]):
|
||||
raise ValueError("video seam trim position is outside the current chunk")
|
||||
if trim_frames + 3 > int(current_raw_images.shape[0]):
|
||||
raise ValueError("video seam analysis requires three retained current frames")
|
||||
|
||||
previous = _downsample_rgb(previous_images[-8:])
|
||||
candidate_start = max(0, trim_frames - MAX_REWIND_FRAMES)
|
||||
current = _downsample_rgb(
|
||||
current_raw_images[
|
||||
candidate_start : trim_frames + FLASH_WINDOW_FRAMES
|
||||
]
|
||||
)
|
||||
nominal_index = trim_frames - candidate_start
|
||||
nominal = current[nominal_index]
|
||||
|
||||
candidate_errors: dict[int, float] = {}
|
||||
maximum_rewind = min(MAX_REWIND_FRAMES, trim_frames)
|
||||
for rewind in range(maximum_rewind + 1):
|
||||
candidate = current[nominal_index - rewind]
|
||||
candidate_errors[rewind] = _prediction_error(previous, candidate)
|
||||
|
||||
nominal_error = candidate_errors[0]
|
||||
minimum_error = min(candidate_errors.values())
|
||||
tolerance = minimum_error * SMALLER_SHIFT_TOLERANCE + 1.0e-10
|
||||
best_rewind = min(
|
||||
rewind
|
||||
for rewind, error in candidate_errors.items()
|
||||
if error <= minimum_error + tolerance
|
||||
)
|
||||
best_error = candidate_errors[best_rewind]
|
||||
baseline_error = _baseline_prediction_error(previous)
|
||||
anomaly_ratio = nominal_error / max(baseline_error, 1.0e-5)
|
||||
improvement = (nominal_error - best_error) / max(nominal_error, 1.0e-12)
|
||||
|
||||
previous_edge = previous[-1]
|
||||
previous_y = _luma(previous_edge)
|
||||
nominal_y = _luma(nominal)
|
||||
luma_delta = float(torch.mean(torch.abs(previous_y - nominal_y)).item())
|
||||
previous_chroma = previous_edge - previous_y.unsqueeze(-1)
|
||||
nominal_chroma = nominal - nominal_y.unsqueeze(-1)
|
||||
chroma_delta = float(
|
||||
torch.mean(torch.abs(previous_chroma - nominal_chroma)).item()
|
||||
)
|
||||
edge_delta = _edge_delta(previous_edge, nominal)
|
||||
scene_score, scene_cut = _scene_cut_metrics(
|
||||
previous,
|
||||
current[nominal_index : nominal_index + 3],
|
||||
)
|
||||
flash_luma_shift, flash_reversal, flash_global_fraction = (
|
||||
_transient_flash_metrics(previous, current[nominal_index:])
|
||||
)
|
||||
cross_frame_delta, motion_baseline_delta, motion_jump_ratio = (
|
||||
_temporal_motion_metrics(previous, current[nominal_index:])
|
||||
)
|
||||
transient_flash_candidate = (
|
||||
not scene_cut
|
||||
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
|
||||
and flash_reversal >= MIN_FLASH_REVERSAL
|
||||
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
micro_flash_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and flash_luma_shift >= MIN_MICRO_FLASH_LUMA_SHIFT
|
||||
and flash_reversal >= MIN_MICRO_FLASH_REVERSAL
|
||||
and flash_global_fraction >= MIN_MICRO_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
exposure_ramp_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and not micro_flash_candidate
|
||||
and anomaly_ratio >= MIN_ANOMALY_RATIO
|
||||
and flash_luma_shift >= MIN_FLASH_LUMA_SHIFT
|
||||
and flash_reversal < MAX_EXPOSURE_RAMP_REVERSAL
|
||||
and flash_global_fraction >= MIN_FLASH_GLOBAL_FRACTION
|
||||
)
|
||||
motion_hitch_candidate = (
|
||||
not scene_cut
|
||||
and not transient_flash_candidate
|
||||
and not micro_flash_candidate
|
||||
and not exposure_ramp_candidate
|
||||
and cross_frame_delta >= MIN_MOTION_CROSS_DELTA
|
||||
and motion_jump_ratio >= MIN_MOTION_JUMP_RATIO
|
||||
)
|
||||
if scene_cut:
|
||||
classification = "scene_cut"
|
||||
elif transient_flash_candidate:
|
||||
classification = "transient_flash"
|
||||
elif micro_flash_candidate:
|
||||
classification = "micro_flash"
|
||||
elif exposure_ramp_candidate:
|
||||
classification = "exposure_ramp"
|
||||
elif motion_hitch_candidate:
|
||||
classification = "motion_hitch"
|
||||
else:
|
||||
classification = "clean_boundary"
|
||||
|
||||
recommended_rewind = 0
|
||||
if scene_cut:
|
||||
reason = "kept nominal boundary; persistent content change resembles a scene cut"
|
||||
elif best_rewind == 0:
|
||||
reason = "kept nominal boundary; it has the lowest prediction error"
|
||||
elif (
|
||||
transient_flash_candidate
|
||||
and flash_reversal >= MIN_AUTO_FLASH_REVERSAL
|
||||
and improvement >= MIN_AUTO_FLASH_IMPROVEMENT
|
||||
):
|
||||
recommended_rewind = best_rewind
|
||||
reason = f"mild transient flash recommends correcting {best_rewind} frame(s)"
|
||||
elif anomaly_ratio < MIN_ANOMALY_RATIO:
|
||||
reason = "kept nominal boundary; anomaly ratio is below the analysis threshold"
|
||||
elif improvement < MIN_RELATIVE_IMPROVEMENT:
|
||||
reason = "kept nominal boundary; candidate improvement is below 6%"
|
||||
else:
|
||||
recommended_rewind = best_rewind
|
||||
reason = f"analysis recommends shift {-best_rewind:+d} frame(s)"
|
||||
|
||||
return VideoBoundaryAnalysis(
|
||||
boundary_index=int(boundary_index),
|
||||
nominal_error=nominal_error,
|
||||
baseline_error=baseline_error,
|
||||
anomaly_ratio=anomaly_ratio,
|
||||
recommended_rewind=recommended_rewind,
|
||||
best_error=best_error,
|
||||
improvement=improvement,
|
||||
luma_delta=luma_delta,
|
||||
chroma_delta=chroma_delta,
|
||||
edge_delta=edge_delta,
|
||||
scene_cut_score=scene_score,
|
||||
scene_cut=scene_cut,
|
||||
flash_luma_shift=flash_luma_shift,
|
||||
flash_reversal=flash_reversal,
|
||||
flash_global_fraction=flash_global_fraction,
|
||||
transient_flash_candidate=transient_flash_candidate,
|
||||
cross_frame_delta=cross_frame_delta,
|
||||
motion_baseline_delta=motion_baseline_delta,
|
||||
motion_jump_ratio=motion_jump_ratio,
|
||||
micro_flash_candidate=micro_flash_candidate,
|
||||
exposure_ramp_candidate=exposure_ramp_candidate,
|
||||
motion_hitch_candidate=motion_hitch_candidate,
|
||||
classification=classification,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def analyze_decoded_boundaries(
|
||||
*,
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
) -> list[VideoBoundaryAnalysis]:
|
||||
"""Analyze every decoded chunk boundary without changing decoded content."""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
chunks = list(plan["chunks"])
|
||||
if len(images) != len(chunks):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
analyses: list[VideoBoundaryAnalysis] = []
|
||||
for index in range(1, len(chunks)):
|
||||
previous_raw = images[index - 1]
|
||||
current_raw = images[index]
|
||||
previous_plan = chunks[index - 1]
|
||||
current_plan = chunks[index]
|
||||
_validate_frames(previous_raw, f"decoded image chunk {index}")
|
||||
_validate_frames(current_raw, f"decoded image chunk {index + 1}")
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
current_segment = current_raw[:current_total]
|
||||
analyses.append(
|
||||
analyze_video_boundary(
|
||||
previous_segment,
|
||||
current_segment,
|
||||
trim_frames=current_trim,
|
||||
boundary_index=index,
|
||||
)
|
||||
)
|
||||
return analyses
|
||||
|
||||
|
||||
def correct_decoded_boundaries(
|
||||
*,
|
||||
images: list[Any],
|
||||
assembly_plan: dict[str, Any],
|
||||
analyses: list[VideoBoundaryAnalysis],
|
||||
enable_exposure_ramp: bool = False,
|
||||
) -> tuple[list[Any], dict[int, str]]:
|
||||
"""Normalize qualified transient exposure shifts without changing motion."""
|
||||
|
||||
plan = validate_assembly_plan(assembly_plan)
|
||||
chunks = list(plan["chunks"])
|
||||
if len(images) != len(chunks):
|
||||
raise ValueError("decoded image count does not match the assembly plan")
|
||||
corrected = list(images)
|
||||
actions: dict[int, str] = {}
|
||||
for analysis in analyses:
|
||||
boundary = int(analysis.boundary_index)
|
||||
correction_kind = "transient flash"
|
||||
replace_frames = int(analysis.recommended_rewind)
|
||||
if analysis.micro_flash_candidate:
|
||||
correction_kind = "micro-flash"
|
||||
replace_frames = 1
|
||||
elif enable_exposure_ramp and analysis.exposure_ramp_candidate:
|
||||
correction_kind = "exposure ramp"
|
||||
replace_frames = 4
|
||||
if replace_frames <= 0:
|
||||
actions[boundary] = "kept native boundary"
|
||||
continue
|
||||
if analysis.scene_cut or not (
|
||||
analysis.transient_flash_candidate
|
||||
or analysis.micro_flash_candidate
|
||||
or (enable_exposure_ramp and analysis.exposure_ramp_candidate)
|
||||
):
|
||||
actions[boundary] = "kept native boundary; correction guard rejected candidate"
|
||||
continue
|
||||
previous_index = boundary - 1
|
||||
current_index = boundary
|
||||
if previous_index < 0 or current_index >= len(images):
|
||||
raise ValueError("video seam boundary index is outside decoded chunks")
|
||||
previous_raw = images[previous_index]
|
||||
current_raw = images[current_index]
|
||||
_validate_frames(previous_raw, f"decoded image chunk {previous_index + 1}")
|
||||
_validate_frames(current_raw, f"decoded image chunk {current_index + 1}")
|
||||
previous_plan = chunks[previous_index]
|
||||
current_plan = chunks[current_index]
|
||||
previous_total = int(previous_plan["total_frames"])
|
||||
previous_trim = int(previous_plan["trim_frames"])
|
||||
current_total = int(current_plan["total_frames"])
|
||||
current_trim = int(current_plan["trim_frames"])
|
||||
if current_trim + replace_frames >= current_total:
|
||||
raise ValueError("video seam correction has no retained recovery frame")
|
||||
previous_segment = previous_raw[:previous_total][previous_trim:]
|
||||
if int(previous_segment.shape[0]) < 1:
|
||||
raise ValueError("video seam correction has no previous retained frame")
|
||||
current_cpu = current_raw.detach().to(device="cpu").clone()
|
||||
previous_anchor = previous_segment[-1].detach().to(
|
||||
device="cpu", dtype=torch.float32
|
||||
)
|
||||
recovery_anchor = current_cpu[current_trim + replace_frames].to(
|
||||
dtype=torch.float32
|
||||
)
|
||||
if tuple(previous_anchor.shape) != tuple(recovery_anchor.shape):
|
||||
raise ValueError("decoded image geometry changed at video seam")
|
||||
for offset in range(replace_frames):
|
||||
alpha = float(offset + 1) / float(replace_frames + 1)
|
||||
original = current_cpu[current_trim + offset].to(dtype=torch.float32)
|
||||
target = torch.lerp(previous_anchor, recovery_anchor, alpha)
|
||||
original_mean = original.mean(dim=(0, 1), keepdim=True)
|
||||
target_mean = target.mean(dim=(0, 1), keepdim=True)
|
||||
normalized = (original + target_mean - original_mean).clamp(0.0, 1.0)
|
||||
current_cpu[current_trim + offset].copy_(
|
||||
normalized.to(dtype=current_cpu.dtype)
|
||||
)
|
||||
corrected[current_index] = current_cpu
|
||||
actions[boundary] = (
|
||||
f"normalized exposure/color on {replace_frames} {correction_kind} "
|
||||
"boundary frame(s)"
|
||||
)
|
||||
return corrected, actions
|
||||
|
||||
|
||||
def format_video_boundary_analysis(
|
||||
analysis: VideoBoundaryAnalysis,
|
||||
*,
|
||||
action: str = "analysis only",
|
||||
) -> str:
|
||||
shift = -int(analysis.recommended_rewind)
|
||||
return (
|
||||
f"video seam {analysis.boundary_index}->{analysis.boundary_index + 1}: "
|
||||
f"anomaly={analysis.anomaly_ratio:.3f}, "
|
||||
f"prediction={analysis.nominal_error:.6f}->{analysis.best_error:.6f}, "
|
||||
f"improvement={analysis.improvement:.1%}, shift={shift:+d}, "
|
||||
f"luma={analysis.luma_delta:.6f}, chroma={analysis.chroma_delta:.6f}, "
|
||||
f"edge={analysis.edge_delta:.6f}, "
|
||||
f"scene_cut={'yes' if analysis.scene_cut else 'no'} "
|
||||
f"flash_shift={analysis.flash_luma_shift:.6f}, "
|
||||
f"flash_reversal={analysis.flash_reversal:.1%}, "
|
||||
f"flash_global={analysis.flash_global_fraction:.1%}, "
|
||||
f"flash_candidate={'yes' if analysis.transient_flash_candidate else 'no'} "
|
||||
f"class={analysis.classification}, "
|
||||
f"cross={analysis.cross_frame_delta:.6f}, "
|
||||
f"motion_base={analysis.motion_baseline_delta:.6f}, "
|
||||
f"motion_jump={analysis.motion_jump_ratio:.2f}x, "
|
||||
f"micro_flash={'yes' if analysis.micro_flash_candidate else 'no'}, "
|
||||
f"exposure_ramp={'yes' if analysis.exposure_ramp_candidate else 'no'}, "
|
||||
f"motion_hitch={'yes' if analysis.motion_hitch_candidate else 'no'} "
|
||||
f"({analysis.reason}); action={action}"
|
||||
)
|
||||
Reference in New Issue
Block a user