git-subtree-dir: vendor/dumas_h3_continuum git-subtree-split: d07ed31b442d283d6a1740f875a843b9feda4c91
492 lines
18 KiB
Python
492 lines
18 KiB
Python
"""Target-aware Second Pass orchestration over complete physical groups."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..state import extract_av_streams
|
|
from ..v2.sampling import latent_from_cpu
|
|
from .refine_schedule import (
|
|
RefineScheduleError,
|
|
resolve_refine_schedule,
|
|
serializable_schedule_contract,
|
|
)
|
|
from .refine_scope import (
|
|
MODE_ALL as SCOPE_ALL,
|
|
resolve_refine_scope,
|
|
)
|
|
from .refine_target import (
|
|
MODE_AUDIO_ONLY,
|
|
MODE_VIDEO_AUDIO,
|
|
MODE_VIDEO_ONLY,
|
|
RefineTarget,
|
|
build_target_execution_contract,
|
|
derive_target_refine_seed,
|
|
resolve_refine_target,
|
|
serializable_target_contract,
|
|
)
|
|
from .refine_window import (
|
|
MODE_TIME_WINDOW,
|
|
GroupTemporalWindow,
|
|
RefineWindow,
|
|
resolve_refine_window,
|
|
)
|
|
from .second_pass import (
|
|
SecondPassContractError,
|
|
_normalize_selected_group_indices,
|
|
_latent_samples,
|
|
prepare_physical_refine_groups,
|
|
run_second_pass_groups,
|
|
validate_second_pass_inputs,
|
|
)
|
|
from .targeted_refine_sampling import (
|
|
TargetedRefineExecutionUnavailable,
|
|
sample_targeted_refine_chunk,
|
|
)
|
|
|
|
|
|
def _attach_target_contract(
|
|
updated_plan: dict[str, Any],
|
|
target: RefineTarget,
|
|
) -> None:
|
|
contract = updated_plan.get("second_pass_contract")
|
|
if not isinstance(contract, dict):
|
|
raise SecondPassContractError(
|
|
"targeted Second Pass output has no mutable contract"
|
|
)
|
|
if contract.get("version") != 1:
|
|
raise SecondPassContractError(
|
|
"targeted Second Pass must preserve contract version 1"
|
|
)
|
|
schedule_contract = contract.get("refine_schedule")
|
|
contract["refine_target_contract"] = serializable_target_contract(target)
|
|
contract["refine_execution_contract"] = build_target_execution_contract(
|
|
target,
|
|
schedule_contract,
|
|
)
|
|
|
|
|
|
def _run_video_only_groups(
|
|
*,
|
|
target: RefineTarget,
|
|
sample_fn,
|
|
arguments: dict[str, Any],
|
|
selected_group_indices: Sequence[int] | None = None,
|
|
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
|
|
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
|
def targeted_sample(**kwargs):
|
|
return sample_targeted_refine_chunk(
|
|
refine_target=target,
|
|
legacy_sample_fn=sample_fn,
|
|
video_only_window_sample_fn=sample_fn,
|
|
**kwargs,
|
|
)
|
|
|
|
videos, audios, updated_plan, status = run_second_pass_groups(
|
|
**arguments,
|
|
sample_fn=targeted_sample,
|
|
selected_group_indices=selected_group_indices,
|
|
temporal_windows_by_group=temporal_windows_by_group,
|
|
)
|
|
_attach_target_contract(updated_plan, target)
|
|
return videos, audios, updated_plan, status
|
|
|
|
|
|
def _run_sampled_target_groups(
|
|
*,
|
|
target: RefineTarget,
|
|
model: Any,
|
|
clip: Any,
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
video_latents: Sequence[Any],
|
|
audio_latents: Sequence[Any],
|
|
assembly_plan: Mapping[str, Any],
|
|
refine_seed: int,
|
|
refine_context: Any,
|
|
video_vae: Any,
|
|
conditioning_upscale_method: str,
|
|
enable_preview: bool,
|
|
encode_prompt_fn,
|
|
latent_builder,
|
|
sample_fn,
|
|
stream_extractor,
|
|
clone_model_fn,
|
|
validate_refine_context_fn,
|
|
adapt_group_conditioning_fn,
|
|
refine_schedule,
|
|
selected_group_indices: Sequence[int] | None = None,
|
|
temporal_windows_by_group: Mapping[int, GroupTemporalWindow] | None = None,
|
|
) -> tuple[list[Any], list[dict[str, Any]], dict[str, Any], str]:
|
|
"""Sample one non-legacy target and adopt only its configured outputs."""
|
|
|
|
if target.mode not in (MODE_AUDIO_ONLY, MODE_VIDEO_AUDIO):
|
|
raise TargetedRefineExecutionUnavailable(
|
|
f"refine target {target.mode!r} is not a sampled-stream target"
|
|
)
|
|
adopt_sampled_video = target.mode == MODE_VIDEO_AUDIO
|
|
|
|
validation = validate_second_pass_inputs(
|
|
video_latents,
|
|
audio_latents,
|
|
assembly_plan,
|
|
)
|
|
selected_indices, selection_active = _normalize_selected_group_indices(
|
|
selected_group_indices,
|
|
int(validation["physical_group_count"]),
|
|
)
|
|
try:
|
|
resolved_schedule = resolve_refine_schedule(sigmas, refine_schedule)
|
|
except RefineScheduleError as exc:
|
|
raise SecondPassContractError(
|
|
f"refine schedule is invalid: {exc}"
|
|
) from exc
|
|
effective_sigmas = resolved_schedule.sigmas
|
|
schedule_contract = serializable_schedule_contract(resolved_schedule)
|
|
latent_builder = latent_builder or latent_from_cpu
|
|
stream_extractor = stream_extractor or extract_av_streams
|
|
refined_videos: list[dict[str, Any]] = (
|
|
list(video_latents) if selection_active else []
|
|
)
|
|
refined_audios: list[dict[str, Any]] = (
|
|
list(audio_latents) if selection_active else []
|
|
)
|
|
group_seeds: list[int] = []
|
|
conditioning_sources: list[str] = []
|
|
group_report_lines: list[str] = []
|
|
|
|
def sample_prepared_group(
|
|
group_index,
|
|
video_latent,
|
|
group,
|
|
chunk_model,
|
|
conditioning,
|
|
detail,
|
|
):
|
|
audio_latent = audio_latents[group_index]
|
|
video_samples = _latent_samples(
|
|
video_latent,
|
|
name=f"video_latents[{group_index}]",
|
|
)
|
|
audio_samples = _latent_samples(
|
|
audio_latent,
|
|
name=f"audio_latents[{group_index}]",
|
|
)
|
|
physical_seed = derive_target_refine_seed(
|
|
int(refine_seed),
|
|
group_index,
|
|
target,
|
|
)
|
|
group_seeds.append(physical_seed)
|
|
conditioning_source = str(detail["conditioning_source"])
|
|
conditioning_sources.append(conditioning_source)
|
|
nested_latent = latent_builder(video_samples, audio_samples)
|
|
sample_arguments = dict(
|
|
model=chunk_model,
|
|
conditioning=conditioning,
|
|
latent=nested_latent,
|
|
sampler=sampler,
|
|
sigmas=effective_sigmas,
|
|
seed=physical_seed,
|
|
refine_target=target,
|
|
enable_preview=bool(enable_preview),
|
|
audio_only_sample_fn=sample_fn,
|
|
video_audio_sample_fn=sample_fn,
|
|
)
|
|
if temporal_windows_by_group is not None:
|
|
temporal_window = temporal_windows_by_group.get(group_index)
|
|
if temporal_window is None:
|
|
raise SecondPassContractError(
|
|
f"selected physical group {group_index + 1} has no temporal window"
|
|
)
|
|
sample_arguments["temporal_window"] = temporal_window
|
|
sampled = sample_targeted_refine_chunk(**sample_arguments)
|
|
sampled_video, sampled_audio = stream_extractor(sampled)
|
|
if tuple(sampled_video.shape) != tuple(video_samples.shape):
|
|
raise SecondPassContractError(
|
|
f"sampled video group {group_index} changed B/C/T/H/W geometry"
|
|
)
|
|
if tuple(sampled_audio.shape) != tuple(audio_samples.shape):
|
|
raise SecondPassContractError(
|
|
f"refined audio group {group_index} changed B/C/stereo/T geometry"
|
|
)
|
|
if not bool(torch.isfinite(sampled_audio.float()).all().item()):
|
|
raise SecondPassContractError(
|
|
f"refined audio group {group_index} contains NaN or Inf"
|
|
)
|
|
if adopt_sampled_video:
|
|
if not bool(torch.isfinite(sampled_video.float()).all().item()):
|
|
raise SecondPassContractError(
|
|
f"refined video group {group_index} contains NaN or Inf"
|
|
)
|
|
output_video = dict(video_latent)
|
|
output_video["samples"] = sampled_video
|
|
if selection_active:
|
|
refined_videos[group_index] = output_video
|
|
else:
|
|
refined_videos.append(output_video)
|
|
output_audio = dict(audio_latent)
|
|
output_audio["samples"] = sampled_audio
|
|
if selection_active:
|
|
refined_audios[group_index] = output_audio
|
|
else:
|
|
refined_audios.append(output_audio)
|
|
adoption = (
|
|
f"video_shape={tuple(sampled_video.shape)}, "
|
|
"sampled_video_adopted=true, sampled_audio_adopted=true"
|
|
if adopt_sampled_video
|
|
else "temporary_video_discarded=true"
|
|
)
|
|
group_report_lines.append(
|
|
"group "
|
|
f"{group_index + 1}: logical_chunks={group.get('logical_chunks')}, "
|
|
f"prompt_policy={group.get('prompt_policy', 'unknown')}, "
|
|
f"conditioning_source={conditioning_source}, "
|
|
f"refine_seed={physical_seed}, "
|
|
f"audio_shape={tuple(sampled_audio.shape)}, sampling_passes=1, "
|
|
f"{adoption}"
|
|
)
|
|
|
|
prepared = prepare_physical_refine_groups(
|
|
model=model,
|
|
clip=clip,
|
|
video_latents=video_latents,
|
|
assembly_plan=assembly_plan,
|
|
refine_context=refine_context,
|
|
video_vae=video_vae,
|
|
conditioning_upscale_method=conditioning_upscale_method,
|
|
encode_prompt_fn=encode_prompt_fn,
|
|
clone_model_fn=clone_model_fn,
|
|
validate_refine_context_fn=validate_refine_context_fn,
|
|
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
|
|
group_consumer_fn=sample_prepared_group,
|
|
retain_group_outputs=False,
|
|
selected_group_indices=(selected_indices if selection_active else None),
|
|
)
|
|
expected_groups = int(validation["physical_group_count"])
|
|
expected_sampled_groups = len(selected_indices)
|
|
if len(group_seeds) != expected_sampled_groups:
|
|
raise SecondPassContractError(
|
|
"targeted Sampling count differs from selected physical group count"
|
|
)
|
|
if len(refined_audios) != expected_groups:
|
|
raise SecondPassContractError(
|
|
"targeted Audio output count differs from physical group count"
|
|
)
|
|
if adopt_sampled_video and len(refined_videos) != expected_groups:
|
|
raise SecondPassContractError(
|
|
"targeted Video output count differs from physical group count"
|
|
)
|
|
|
|
output_videos = refined_videos if adopt_sampled_video else list(video_latents)
|
|
updated_plan = prepared["updated_assembly_plan"]
|
|
contract = updated_plan["second_pass_contract"]
|
|
execution_target = "both_sampled" if adopt_sampled_video else "video_locked_audio"
|
|
if conditioning_sources and all(
|
|
source == "refine_context" for source in conditioning_sources
|
|
):
|
|
contract["execution"] = (
|
|
f"context_aware_physical_groups_{execution_target}_v1"
|
|
)
|
|
elif any(source == "refine_context" for source in conditioning_sources):
|
|
contract["execution"] = f"mixed_context_physical_groups_{execution_target}_v1"
|
|
else:
|
|
contract["execution"] = f"t2va_physical_groups_{execution_target}_v1"
|
|
contract["conditioning_sources"] = conditioning_sources
|
|
contract["refine_seed_base"] = int(refine_seed)
|
|
contract["refine_group_seeds"] = group_seeds
|
|
contract["video_output"] = (
|
|
"sampled" if adopt_sampled_video else "bit_exact_first_pass_passthrough"
|
|
)
|
|
contract["video_sampling"] = (
|
|
"seeded_random_mask_1" if adopt_sampled_video else "zero_noise_mask_locked"
|
|
)
|
|
contract["audio_output"] = "sampled"
|
|
contract["audio_sampling"] = "seeded_random_mask_1"
|
|
contract["refine_schedule"] = schedule_contract
|
|
contract["refine_schedule_identity"] = schedule_contract["schedule_hash"]
|
|
_attach_target_contract(updated_plan, target)
|
|
|
|
if adopt_sampled_video:
|
|
target_line = (
|
|
"Target: Video + Audio "
|
|
"(GPU Experimental PASS / Production HOLD)."
|
|
)
|
|
sampling_line = (
|
|
"Sampling contract: video seeded random noise/mask=1 and adopted "
|
|
"sampled output; audio seeded random noise/mask=1 and adopted "
|
|
"sampled output."
|
|
)
|
|
output_line = (
|
|
"Outputs: sampled Video and sampled Audio adopted for every complete "
|
|
"physical group."
|
|
)
|
|
else:
|
|
target_line = "Target: Audio Only (Experimental / Production HOLD)."
|
|
sampling_line = (
|
|
"Sampling contract: video zero noise/mask=0 and discarded sampled "
|
|
"output; audio seeded random noise/mask=1 and adopted sampled output."
|
|
)
|
|
output_line = (
|
|
"Video output: original first-pass physical Video LATENT objects "
|
|
"returned bit-exact; temporary sampled Video discarded."
|
|
)
|
|
report_lines = [
|
|
"H3 Continuum Targeted Second Pass",
|
|
target_line,
|
|
f"Physical groups: {validation['physical_group_count']}.",
|
|
sampling_line,
|
|
"RefineSchedule: "
|
|
f"schema={schedule_contract['schema_version']}, "
|
|
f"mode={schedule_contract['mode']}, "
|
|
f"evaluations={schedule_contract['evaluation_count']}, "
|
|
f"sigma_hash={schedule_contract['sigma_hash']}.",
|
|
*prepared["warnings"],
|
|
*group_report_lines,
|
|
output_line,
|
|
]
|
|
if selection_active:
|
|
report_lines.insert(
|
|
3,
|
|
"Selected physical groups: "
|
|
+ ", ".join(str(index + 1) for index in selected_indices)
|
|
+ "; non-selected groups were exact object passthrough with no "
|
|
"conditioning, MODEL clone, Sampling, or seed derivation.",
|
|
)
|
|
return output_videos, refined_audios, updated_plan, "\n".join(report_lines)
|
|
|
|
|
|
def run_targeted_second_pass_groups(
|
|
*,
|
|
model: Any,
|
|
clip: Any,
|
|
sampler: Any,
|
|
sigmas: torch.Tensor,
|
|
video_latents: Sequence[Any],
|
|
audio_latents: Sequence[Any],
|
|
assembly_plan: Mapping[str, Any],
|
|
refine_seed: int,
|
|
refine_target: RefineTarget | str | None = MODE_VIDEO_ONLY,
|
|
refine_context: Any = None,
|
|
video_vae: Any = None,
|
|
conditioning_upscale_method: str = "bilinear",
|
|
enable_preview: bool = True,
|
|
encode_prompt_fn=None,
|
|
latent_builder=None,
|
|
sample_fn=None,
|
|
stream_extractor=None,
|
|
clone_model_fn=None,
|
|
validate_refine_context_fn=None,
|
|
adapt_group_conditioning_fn=None,
|
|
refine_schedule=None,
|
|
refine_scope: str | None = SCOPE_ALL,
|
|
refine_scope_index: int = 1,
|
|
window_start_sec: float = 0.0,
|
|
window_end_sec: float = 5.0,
|
|
) -> tuple[list[dict[str, Any]], list[Any], dict[str, Any], str]:
|
|
"""Run an implemented target without changing the V3.5 public node."""
|
|
|
|
target = resolve_refine_target(refine_target)
|
|
window: RefineWindow | None = None
|
|
if isinstance(refine_scope, str) and refine_scope.strip().lower() == MODE_TIME_WINDOW:
|
|
scope = None
|
|
window = resolve_refine_window(
|
|
window_start_sec,
|
|
window_end_sec,
|
|
assembly_plan,
|
|
)
|
|
selected_group_indices = window.selected_group_indices
|
|
temporal_windows_by_group = {
|
|
group.group_index: group for group in window.groups
|
|
}
|
|
else:
|
|
scope = resolve_refine_scope(
|
|
refine_scope,
|
|
refine_scope_index,
|
|
assembly_plan,
|
|
)
|
|
selected_group_indices = (
|
|
None if scope.is_all else scope.selected_group_indices
|
|
)
|
|
temporal_windows_by_group = None
|
|
arguments = dict(
|
|
model=model,
|
|
clip=clip,
|
|
sampler=sampler,
|
|
sigmas=sigmas,
|
|
video_latents=video_latents,
|
|
audio_latents=audio_latents,
|
|
assembly_plan=assembly_plan,
|
|
refine_seed=int(refine_seed),
|
|
refine_context=refine_context,
|
|
video_vae=video_vae,
|
|
conditioning_upscale_method=str(conditioning_upscale_method),
|
|
enable_preview=bool(enable_preview),
|
|
encode_prompt_fn=encode_prompt_fn,
|
|
latent_builder=latent_builder,
|
|
stream_extractor=stream_extractor,
|
|
clone_model_fn=clone_model_fn,
|
|
validate_refine_context_fn=validate_refine_context_fn,
|
|
adapt_group_conditioning_fn=adapt_group_conditioning_fn,
|
|
refine_schedule=refine_schedule,
|
|
)
|
|
if target.mode == MODE_VIDEO_ONLY:
|
|
result = _run_video_only_groups(
|
|
target=target,
|
|
sample_fn=sample_fn,
|
|
arguments=arguments,
|
|
selected_group_indices=selected_group_indices,
|
|
temporal_windows_by_group=temporal_windows_by_group,
|
|
)
|
|
elif target.mode == MODE_AUDIO_ONLY:
|
|
result = _run_sampled_target_groups(
|
|
target=target,
|
|
sample_fn=sample_fn,
|
|
selected_group_indices=selected_group_indices,
|
|
temporal_windows_by_group=temporal_windows_by_group,
|
|
**arguments,
|
|
)
|
|
elif target.mode == MODE_VIDEO_AUDIO:
|
|
result = _run_sampled_target_groups(
|
|
target=target,
|
|
sample_fn=sample_fn,
|
|
selected_group_indices=selected_group_indices,
|
|
temporal_windows_by_group=temporal_windows_by_group,
|
|
**arguments,
|
|
)
|
|
else:
|
|
raise TargetedRefineExecutionUnavailable(
|
|
f"refine target {target.mode!r} is unavailable"
|
|
)
|
|
|
|
videos, audios, updated_plan, status = result
|
|
if window is not None:
|
|
contract = window.contract
|
|
window_line = (
|
|
"Refine Window: timeline=final_visible_output, interval=["
|
|
f"{contract['effective_start_sec']:.6f}, "
|
|
f"{contract['effective_end_sec']:.6f}) sec, output_frames="
|
|
f"{contract['output_frame_range']}, selected_physical_groups="
|
|
f"{contract['selected_physical_groups']}, continuation_prefix=protected, "
|
|
"outside_window=mask_zero_then_bit_exact_restore, "
|
|
"physical_group_atomicity=preserved."
|
|
)
|
|
return videos, audios, updated_plan, f"{status}\n{window_line}"
|
|
assert scope is not None
|
|
if scope.is_all:
|
|
return videos, audios, updated_plan, status
|
|
scope_contract = scope.contract
|
|
scope_line = (
|
|
"Refine Scope: "
|
|
f"mode={scope.mode}, requested_index={scope_contract['requested_index']}, "
|
|
"selected_physical_groups="
|
|
f"{list(scope_contract['selected_physical_groups'])}, "
|
|
"selected_logical_chunks="
|
|
f"{list(scope_contract['selected_logical_chunks'])}, "
|
|
f"terminal_expanded={str(scope_contract['terminal_expanded']).lower()}."
|
|
)
|
|
return videos, audios, updated_plan, f"{status}\n{scope_line}"
|