Make Auto seam correction native-first
This commit is contained in:
+7
-1
@@ -65,8 +65,14 @@ PROMPT_FORMAT_OPTIONS = (
|
||||
|
||||
DIAGNOSTICS_OFF = "Off"
|
||||
DIAGNOSTICS_BASIC = "Basic"
|
||||
DIAGNOSTICS_FULL = "Full"
|
||||
DIAGNOSTICS_FULL = "Detailed Report"
|
||||
DIAGNOSTICS_LEGACY_FULL = "Full"
|
||||
DIAGNOSTICS_OPTIONS = (DIAGNOSTICS_BASIC, DIAGNOSTICS_FULL, DIAGNOSTICS_OFF)
|
||||
DIAGNOSTICS_ACCEPTED_OPTIONS = DIAGNOSTICS_OPTIONS + (DIAGNOSTICS_LEGACY_FULL,)
|
||||
|
||||
|
||||
def normalize_diagnostics_mode(value: str) -> str:
|
||||
return DIAGNOSTICS_FULL if value == DIAGNOSTICS_LEGACY_FULL else value
|
||||
|
||||
SEAM_CORRECTION_AUTO = "Auto"
|
||||
SEAM_CORRECTION_OFF = "Off"
|
||||
|
||||
+2
-2
@@ -27,5 +27,5 @@ def av_duration_error_samples(images,audio):
|
||||
def format_join_metrics(index,metrics): return f"join {index-1}->{index}: video MAE={metrics['video_overlap_mae']:.6f}, PSNR={metrics['video_overlap_psnr']:.2f}dB, audio corr={metrics['audio_overlap_correlation']:.4f}, boundary jump={metrics['audio_boundary_jump']:.6f}"
|
||||
def format_seam_decision(index,decision:SeamDecision,*,full:bool):
|
||||
fallback=decision.fallback_reason or "none"
|
||||
if not full:return f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, video blend={decision.video_blend_frames}f, audio offset={decision.audio_metrics.offset_samples:+d} samples, fallback={fallback}"
|
||||
return (f"Seam {index-1}->{index}\n Legacy score : {decision.legacy_score:.6f}\n Selected cut : -{decision.cut_rewind_frames} frames\n Cut score : {decision.cut_score:.6f}\n Corrected score : {decision.corrected_score:.6f}\n Improvement : {decision.improvement*100.0:.2f}%\n Luminance gain : {decision.luma_gain:.4f}\n Chroma bias : {decision.chroma_bias}\n Video blend : {decision.video_blend_frames} frames\n Audio correlation : {decision.audio_metrics.correlation_before:.4f}->{decision.audio_metrics.correlation_after:.4f}\n Audio jump : {decision.audio_metrics.boundary_jump_before:.6f}->{decision.audio_metrics.boundary_jump_after:.6f}\n Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n Audio crossfade : {decision.audio_crossfade_samples} samples\n Fallback : {fallback}")
|
||||
if not full:return f"seam {index-1}->{index}: cut=-{decision.cut_rewind_frames}f, native score {decision.legacy_score:.4f}->{decision.corrected_score:.4f}, video blend={decision.video_blend_frames}f, audio offset={decision.audio_metrics.offset_samples:+d} samples, fallback={fallback}"
|
||||
return (f"Seam {index-1}->{index}\n Native score : {decision.legacy_score:.6f}\n Selected cut : -{decision.cut_rewind_frames} frames\n Cut score : {decision.cut_score:.6f}\n Candidate score : {decision.candidate_score:.6f}\n Final score : {decision.corrected_score:.6f}\n Improvement : {decision.improvement*100.0:.2f}%\n Native 5f peak : {decision.native_window_peak:.6f}\n Candidate 5f peak : {decision.candidate_window_peak:.6f}\n Luminance gain : {decision.luma_gain:.4f}\n Chroma bias : {decision.chroma_bias}\n Video blend : {decision.video_blend_frames} frames\n Audio correlation : {decision.audio_metrics.correlation_before:.4f}->{decision.audio_metrics.correlation_after:.4f}\n Audio jump : {decision.audio_metrics.boundary_jump_before:.6f}->{decision.audio_metrics.boundary_jump_after:.6f}\n Audio offset : {decision.audio_metrics.offset_samples:+d} samples\n Audio crossfade : {decision.audio_crossfade_samples} samples\n Fallback : {fallback}")
|
||||
|
||||
+41
-10
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_ACCEPTED_OPTIONS,
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
PROMPT_FORMAT_OPTIONS,
|
||||
PROMPT_MODE_OPTIONS,
|
||||
@@ -12,6 +13,7 @@ from ..constants import (
|
||||
SEAM_CORRECTION_OFF,
|
||||
SEAM_CORRECTION_OPTIONS,
|
||||
V2_CONTINUITY_OPTIONS,
|
||||
normalize_diagnostics_mode,
|
||||
)
|
||||
from .prompts import (
|
||||
apply_prompt_overrides,
|
||||
@@ -56,7 +58,7 @@ def _repair_v200_example_widget_shift(
|
||||
back losslessly.
|
||||
"""
|
||||
|
||||
if isinstance(exact_total_duration, str) and exact_total_duration in DIAGNOSTICS_OPTIONS:
|
||||
if isinstance(exact_total_duration, str) and exact_total_duration in DIAGNOSTICS_ACCEPTED_OPTIONS:
|
||||
if isinstance(diagnostics, (int, float, bool)) and not isinstance(diagnostics, str):
|
||||
LOG.warning(
|
||||
"H3 Continuum V2 detected the malformed 2.0.0 bundled workflow widget "
|
||||
@@ -66,7 +68,7 @@ def _repair_v200_example_widget_shift(
|
||||
return (
|
||||
bool(audio_continuity),
|
||||
True,
|
||||
exact_total_duration,
|
||||
normalize_diagnostics_mode(exact_total_duration),
|
||||
int(diagnostics),
|
||||
int(reroll_from_chunk),
|
||||
bool(reroll_nonce),
|
||||
@@ -75,7 +77,7 @@ def _repair_v200_example_widget_shift(
|
||||
return (
|
||||
bool(audio_continuity),
|
||||
bool(exact_total_duration),
|
||||
diagnostics,
|
||||
normalize_diagnostics_mode(diagnostics),
|
||||
int(reroll_from_chunk),
|
||||
int(reroll_nonce),
|
||||
bool(strict_compatibility),
|
||||
@@ -95,6 +97,10 @@ class H3ContinuumSamplerV2:
|
||||
"H3 Continuum V2",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, diagnostics):
|
||||
return True if diagnostics in DIAGNOSTICS_ACCEPTED_OPTIONS else f"unknown report detail: {diagnostics!r}"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
@@ -170,7 +176,14 @@ class H3ContinuumSamplerV2:
|
||||
"tooltip": "Trim or minimally pad the assembled result to chunks × seconds × 24fps.",
|
||||
},
|
||||
),
|
||||
"diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}),
|
||||
"diagnostics": (
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
{
|
||||
"default": DIAGNOSTICS_OPTIONS[0],
|
||||
"display_name": "Report Detail",
|
||||
"tooltip": "Controls report verbosity only; it does not change generation or seam correction.",
|
||||
},
|
||||
),
|
||||
"reroll_from_chunk": (
|
||||
"INT",
|
||||
{
|
||||
@@ -201,9 +214,10 @@ class H3ContinuumSamplerV2:
|
||||
{
|
||||
"default": SEAM_CORRECTION_AUTO,
|
||||
"tooltip": (
|
||||
"Auto keeps the legacy video boundary unless the guarded candidate "
|
||||
"improves it. Off preserves the V2.0.2 decode path. Basic always runs "
|
||||
"the conservative decoded A/V boundary correction."
|
||||
"Auto keeps native video unless a no-blend candidate improves the boundary "
|
||||
"and five-frame transition without component regression; audio is guarded "
|
||||
"independently. Off preserves the decode path. Basic applies the decoded "
|
||||
"A/V correction including conservative video blend."
|
||||
),
|
||||
},
|
||||
),
|
||||
@@ -512,13 +526,24 @@ class H3ContinuumClipOverrides:
|
||||
class H3ContinuumAdvanced:
|
||||
"""Package optional continuation inputs and infrequent sampler settings."""
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, diagnostics):
|
||||
return True if diagnostics in DIAGNOSTICS_ACCEPTED_OPTIONS else f"unknown report detail: {diagnostics!r}"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio_continuity": ("BOOLEAN", {"default": True}),
|
||||
"exact_total_duration": ("BOOLEAN", {"default": True}),
|
||||
"diagnostics": (DIAGNOSTICS_OPTIONS, {"default": DIAGNOSTICS_OPTIONS[0]}),
|
||||
"diagnostics": (
|
||||
DIAGNOSTICS_OPTIONS,
|
||||
{
|
||||
"default": DIAGNOSTICS_OPTIONS[0],
|
||||
"display_name": "Report Detail",
|
||||
"tooltip": "Controls report verbosity only; it does not change generation or seam correction.",
|
||||
},
|
||||
),
|
||||
"reroll_from_chunk": (
|
||||
"INT",
|
||||
{"default": 0, "min": 0, "max": 16, "step": 1},
|
||||
@@ -563,7 +588,7 @@ class H3ContinuumAdvanced:
|
||||
{
|
||||
"audio_continuity": bool(audio_continuity),
|
||||
"exact_total_duration": bool(exact_total_duration),
|
||||
"diagnostics": diagnostics,
|
||||
"diagnostics": normalize_diagnostics_mode(diagnostics),
|
||||
"reroll_from_chunk": int(reroll_from_chunk),
|
||||
"reroll_nonce": int(reroll_nonce),
|
||||
"strict_compatibility": bool(strict_compatibility),
|
||||
@@ -633,7 +658,13 @@ class H3ContinuumSampler:
|
||||
),
|
||||
"seam_correction": (
|
||||
SEAM_CORRECTION_OPTIONS,
|
||||
{"default": SEAM_CORRECTION_AUTO},
|
||||
{
|
||||
"default": SEAM_CORRECTION_AUTO,
|
||||
"tooltip": (
|
||||
"Auto is native-first: video correction is adopted only when the boundary "
|
||||
"and five-frame transition clearly improve; audio is guarded independently."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
|
||||
+28
-9
@@ -5,6 +5,8 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from .seam_types import AudioSeamMetrics, SeamDecision, VideoSeamMetrics
|
||||
COMPARE_SIZE=96; MAX_CUT_REWIND=3; MIN_RELATIVE_IMPROVEMENT=0.10; MIN_ABSOLUTE_IMPROVEMENT=0.003; SAFE_SCORE_LIMIT=0.35
|
||||
AUTO_MIN_RELATIVE_IMPROVEMENT=0.20; AUTO_WINDOW_FRAMES=5; AUTO_MIN_WINDOW_RELATIVE_IMPROVEMENT=0.05; AUTO_MIN_WINDOW_ABSOLUTE_IMPROVEMENT=0.0005
|
||||
AUTO_COMPONENT_RELATIVE_TOLERANCE=0.01; AUTO_COMPONENT_ABSOLUTE_TOLERANCE=0.0001
|
||||
WEIGHT_PIXEL=0.35; WEIGHT_LUMA=0.20; WEIGHT_CHROMA=0.10; WEIGHT_EDGE=0.15; WEIGHT_TEMPORAL=0.20
|
||||
|
||||
def _video_shape(frames,label):
|
||||
@@ -67,20 +69,37 @@ def apply_video_blend(previous_frames,aligned_next_frames,*,blend_frames):
|
||||
for index in range(blend_frames):
|
||||
progress=float(index+1)/float(blend_frames+1); alpha=0.5-0.5*math.cos(math.pi*progress); previous[index].mul_(1.0-alpha).add_(current[index],alpha=alpha)
|
||||
return previous
|
||||
def _seam_window_peak(previous_images,next_images,*,window_frames=AUTO_WINDOW_FRAMES):
|
||||
if previous_images.shape[0]<1 or next_images.shape[0]<2: raise ValueError("video seam window scoring needs one previous and two next frames")
|
||||
count=min(max(2,int(window_frames)),int(next_images.shape[0])); sequence=torch.cat((previous_images[-1:],next_images[:count]),dim=0); rgb=_downsample_rgb(sequence)
|
||||
pixel_delta=torch.mean(torch.abs(rgb[1:]-rgb[:-1]),dim=(1,2,3)); mean_luma=_luma(rgb).mean(dim=(1,2)); luma_delta=torch.abs(mean_luma[1:]-mean_luma[:-1]); combined=0.75*pixel_delta+0.25*luma_delta
|
||||
peak=float(torch.max(combined).item())
|
||||
if not math.isfinite(peak): raise ValueError("video seam window score is not finite")
|
||||
return max(0.0,min(1.0,peak))
|
||||
def _auto_rejection_reason(legacy,candidate,*,improvement,absolute_improvement,native_window_peak,candidate_window_peak):
|
||||
if candidate.score>SAFE_SCORE_LIMIT: return "Auto kept native video; candidate score exceeded the safe limit"
|
||||
if improvement<AUTO_MIN_RELATIVE_IMPROVEMENT or absolute_improvement<MIN_ABSOLUTE_IMPROVEMENT: return "Auto kept native video; candidate improvement was below the conservative threshold"
|
||||
for label in ("pixel_mae_norm","luma_norm","chroma_norm","edge_norm","temporal_norm","motion"):
|
||||
before=float(getattr(legacy,label)); after=float(getattr(candidate,label)); limit=before*(1.0+AUTO_COMPONENT_RELATIVE_TOLERANCE)+AUTO_COMPONENT_ABSOLUTE_TOLERANCE
|
||||
if after>limit: return f"Auto kept native video; {label} regressed"
|
||||
window_absolute=native_window_peak-candidate_window_peak; window_relative=window_absolute/max(native_window_peak,1e-12)
|
||||
if window_relative<AUTO_MIN_WINDOW_RELATIVE_IMPROVEMENT or window_absolute<AUTO_MIN_WINDOW_ABSOLUTE_IMPROVEMENT: return "Auto kept native video; the five-frame transition peak did not clearly improve"
|
||||
return None
|
||||
def correct_seam(previous_images,next_raw_images,*,context_frames,automatic=False):
|
||||
rewind,legacy,selected=choose_adaptive_cut(previous_images,next_raw_images,context_frames=context_frames); cut_index=int(context_frames)-rewind; previous_stop=int(previous_images.shape[0])-rewind
|
||||
native_current=next_raw_images[int(context_frames):]; native_window_peak=_seam_window_peak(previous_images,native_current); rewind,legacy,selected=choose_adaptive_cut(previous_images,next_raw_images,context_frames=context_frames); cut_index=int(context_frames)-rewind; previous_stop=int(previous_images.shape[0])-rewind
|
||||
if previous_stop<2 or cut_index<0: raise ValueError("adaptive cut would exhaust a seam side")
|
||||
current=next_raw_images[cut_index:].clone().contiguous(); kept_previous=previous_images[:previous_stop]; current,gain,bias,corrected=apply_color_guard(kept_previous,current)
|
||||
blend_frames=min(choose_video_blend(selected),cut_index,previous_stop); video_patch=None
|
||||
blend_frames=0 if automatic else min(choose_video_blend(selected),cut_index,previous_stop); video_patch=None; score_previous=kept_previous[-2:].clone()
|
||||
if blend_frames>0:
|
||||
video_patch=apply_video_blend(kept_previous[-blend_frames:],next_raw_images[cut_index-blend_frames:cut_index],blend_frames=blend_frames); score_previous=kept_previous[-2:].clone(); replaced=min(blend_frames,2); score_previous[-replaced:].copy_(video_patch[-replaced:]); corrected=_boundary_metrics(score_previous,current[:2])
|
||||
improvement=(legacy.score-corrected.score)/max(legacy.score,1e-12)
|
||||
video_patch=apply_video_blend(kept_previous[-blend_frames:],next_raw_images[cut_index-blend_frames:cut_index],blend_frames=blend_frames); replaced=min(blend_frames,2); score_previous[-replaced:].copy_(video_patch[-replaced:]); corrected=_boundary_metrics(score_previous,current[:2])
|
||||
candidate_window_peak=_seam_window_peak(score_previous,current); improvement=(legacy.score-corrected.score)/max(legacy.score,1e-12); absolute_improvement=legacy.score-corrected.score
|
||||
if corrected.score>legacy.score*1.01:
|
||||
return next_raw_images[int(context_frames):].clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=legacy.score,corrected_score=legacy.score,fallback_reason="corrected score was worse than legacy"),None
|
||||
absolute_improvement=legacy.score-corrected.score
|
||||
if automatic and (improvement<MIN_RELATIVE_IMPROVEMENT or absolute_improvement<MIN_ABSOLUTE_IMPROVEMENT):
|
||||
return next_raw_images[int(context_frames):].clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=selected.score,corrected_score=legacy.score,fallback_reason="Auto kept the legacy video seam; improvement was below threshold"),None
|
||||
return current,SeamDecision(cut_rewind_frames=rewind,legacy_score=legacy.score,cut_score=selected.score,corrected_score=corrected.score,improvement=improvement,video_blend_frames=blend_frames,luma_gain=gain,chroma_bias=bias),video_patch
|
||||
return native_current.clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=selected.score,candidate_score=corrected.score,corrected_score=legacy.score,native_window_peak=native_window_peak,candidate_window_peak=candidate_window_peak,fallback_reason="corrected score was worse than native video"),None
|
||||
if automatic:
|
||||
fallback_reason=_auto_rejection_reason(legacy,corrected,improvement=improvement,absolute_improvement=absolute_improvement,native_window_peak=native_window_peak,candidate_window_peak=candidate_window_peak)
|
||||
if fallback_reason is not None:
|
||||
return native_current.clone().contiguous(),SeamDecision(legacy_score=legacy.score,cut_score=selected.score,candidate_score=corrected.score,corrected_score=legacy.score,improvement=improvement,native_window_peak=native_window_peak,candidate_window_peak=candidate_window_peak,fallback_reason=fallback_reason),None
|
||||
return current,SeamDecision(cut_rewind_frames=rewind,legacy_score=legacy.score,cut_score=selected.score,candidate_score=corrected.score,corrected_score=corrected.score,improvement=improvement,native_window_peak=native_window_peak,candidate_window_peak=candidate_window_peak,video_blend_frames=blend_frames,luma_gain=gain,chroma_bias=bias),video_patch
|
||||
|
||||
def _audio_mid(waveform):
|
||||
if not torch.is_tensor(waveform) or waveform.ndim!=3: raise ValueError("audio seam waveform must be [B,C,S]")
|
||||
|
||||
@@ -22,8 +22,11 @@ class SeamDecision:
|
||||
cut_rewind_frames: int = 0
|
||||
legacy_score: float = 0.0
|
||||
cut_score: float = 0.0
|
||||
candidate_score: float = 0.0
|
||||
corrected_score: float = 0.0
|
||||
improvement: float = 0.0
|
||||
native_window_peak: float = 0.0
|
||||
candidate_window_peak: float = 0.0
|
||||
video_blend_frames: int = 0
|
||||
luma_gain: float = 1.0
|
||||
chroma_bias: tuple[float,float,float] = (0.0,0.0,0.0)
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ from ..compatibility import accelerator_summary, check_comfy_h3_runtime
|
||||
from ..constants import (
|
||||
DIAGNOSTICS_FULL, DIAGNOSTICS_OFF, DIAGNOSTICS_OPTIONS, FPS,
|
||||
CONTINUUM_ACTUAL_PREFIX_STEPS, SEAM_CORRECTION_AUTO, SEAM_CORRECTION_OFF,
|
||||
SEAM_CORRECTION_OPTIONS, V2_CONTINUITY_AUTO,
|
||||
SEAM_CORRECTION_OPTIONS, V2_CONTINUITY_AUTO, normalize_diagnostics_mode,
|
||||
)
|
||||
from ..continuation import POLICY_REPLACE, prepare_conditioning
|
||||
from ..model_patch import clone_model_for_chunk, patch_model
|
||||
@@ -76,7 +76,7 @@ def _conditioning_cache(*,clip,prompts,assets,final_has_last_frame):
|
||||
return cache
|
||||
|
||||
def run_sequence(*,model:Any,clip:Any,video_vae:Any,audio_vae:Any,sampler:Any,sigmas:torch.Tensor,first_frame:torch.Tensor|None,last_frame:torch.Tensor|None,prompt_plan:dict[str,Any],width:int,height:int,continuity:str,base_seed:int,audio_continuity:bool,exact_total_duration:bool,diagnostics_mode:str,reroll_from_chunk:int,reroll_nonce:int,strict_compatibility:bool,debug:bool,seam_correction:str=SEAM_CORRECTION_OFF,enable_preview:bool=True,session:dict[str,Any]|None=None,initial_state:dict[str,Any]|None=None):
|
||||
plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height)
|
||||
diagnostics_mode=normalize_diagnostics_mode(diagnostics_mode); plan=validate_prompt_plan(prompt_plan); chunks=int(plan["chunks"]); chunk_seconds=float(plan["chunk_seconds"]); prompts=list(plan["prompts"]); prompt_hashes=list(plan["hashes"]); width,height=int(width),int(height)
|
||||
if width<=0 or height<=0 or width%32 or height%32: raise SequenceRuntimeError("width and height must be positive multiples of 32")
|
||||
if session is not None and initial_state is not None: raise SequenceRuntimeError("connect either a session or an initial_state, not both")
|
||||
if diagnostics_mode not in DIAGNOSTICS_OPTIONS: raise SequenceRuntimeError(f"unknown diagnostics mode: {diagnostics_mode!r}")
|
||||
|
||||
Reference in New Issue
Block a user