Files
ComfyUI-H3-Continuum-Tiled/reference_video.py
T

382 lines
12 KiB
Python

"""Persistent MiniMax H3 video-reference conditioning for the V3.4 sampler."""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
import math
from typing import Any
import torch
from .temporal import align_frame_count_up
REFERENCE_VIDEO_CONTRACT_VERSION = 1
REFERENCE_VIDEO_PREPROCESS_VERSION = 3
REFERENCE_VIDEO_SIZE_EFFICIENT = "Efficient - 0.4 MP"
REFERENCE_VIDEO_SIZE_BALANCED = "Balanced - 0.6 MP"
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT = "Match Output"
REFERENCE_VIDEO_SIZE_OPTIONS = (
REFERENCE_VIDEO_SIZE_EFFICIENT,
REFERENCE_VIDEO_SIZE_BALANCED,
REFERENCE_VIDEO_SIZE_MATCH_OUTPUT,
)
_CANVAS_MULTIPLE = 32
_EFFICIENT_AREA = 400_000
_BALANCED_AREA = 600_000
_SOURCE_FPS = 24
_HASH_CHUNK_FRAMES = 8
class ReferenceVideoError(ValueError):
pass
def _canonical_hash(value: Any) -> str:
payload = json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def _tensor_hash(value: torch.Tensor) -> str:
tensor = value.detach()
digest = hashlib.sha256()
digest.update(str(tuple(int(v) for v in tensor.shape)).encode("ascii"))
digest.update(str(tensor.dtype).encode("ascii"))
for start in range(0, int(tensor.shape[0]), _HASH_CHUNK_FRAMES):
chunk = tensor[start : start + _HASH_CHUNK_FRAMES].to(
device="cpu"
).contiguous()
digest.update(chunk.numpy().tobytes(order="C"))
return digest.hexdigest()
def _round_canvas(value: float) -> int:
return max(
_CANVAS_MULTIPLE,
int(round(float(value) / _CANVAS_MULTIPLE)) * _CANVAS_MULTIPLE,
)
def resolve_reference_video_size(
source_width: int,
source_height: int,
*,
output_width: int,
output_height: int,
size_mode: str,
) -> tuple[int, int]:
if size_mode == REFERENCE_VIDEO_SIZE_MATCH_OUTPUT:
target_area = int(output_width) * int(output_height)
elif size_mode == REFERENCE_VIDEO_SIZE_BALANCED:
target_area = _BALANCED_AREA
else:
target_area = _EFFICIENT_AREA
source_area = int(source_width) * int(source_height)
scale = min(1.0, math.sqrt(float(target_area) / float(source_area)))
target_width = _round_canvas(int(source_width) * scale)
target_height = _round_canvas(int(source_height) * scale)
if source_width >= _CANVAS_MULTIPLE:
target_width = min(
target_width,
max(_CANVAS_MULTIPLE, (int(source_width) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
)
if source_height >= _CANVAS_MULTIPLE:
target_height = min(
target_height,
max(_CANVAS_MULTIPLE, (int(source_height) // _CANVAS_MULTIPLE) * _CANVAS_MULTIPLE),
)
return target_width, target_height
# Preserve the established private helper for existing internal callers/tests.
_resolved_size = resolve_reference_video_size
@dataclass(frozen=True)
class ReferenceVideoSource:
frames: torch.Tensor
source_shape: tuple[int, ...]
source_dtype: str
source_sha256: str
size_mode: str
target_width: int
target_height: int
frame_count: int
combined_hash: str
@property
def contract(self) -> dict[str, Any]:
return {
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
"source_shape": list(self.source_shape),
"source_dtype": self.source_dtype,
"source_sha256": self.source_sha256,
"source_fps": _SOURCE_FPS,
"size_mode": self.size_mode,
"target_width": self.target_width,
"target_height": self.target_height,
"frame_count": self.frame_count,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
"combined_hash": self.combined_hash,
}
@dataclass(frozen=True)
class ReferenceVideoAssets:
source: ReferenceVideoSource
item: dict[str, Any]
block: dict[str, Any]
def resolve_reference_video_frame_count(
source_frames: int,
target_frames: int,
) -> int:
"""Resolve the used Video Guide frame count without touching frame data."""
source_count = int(source_frames)
target_count = int(target_frames)
if source_count < 5 or target_count < 5:
raise ReferenceVideoError("Video Reference requires at least 5 frames")
target_cap = align_frame_count_up(target_count)
available_count = min(source_count, target_cap)
return min(align_frame_count_up(available_count), target_cap)
# Preserve the established private name while exposing one shared pure resolver.
_resolve_reference_frame_count = resolve_reference_video_frame_count
def _fit_reference_frames(frames: torch.Tensor, frame_count: int) -> torch.Tensor:
requested_count = int(frame_count)
current_count = int(frames.shape[0])
if current_count >= requested_count:
return frames[:requested_count].contiguous()
if current_count < 1:
raise ReferenceVideoError("Video Reference contains no frames")
padding = frames[-1:].expand(requested_count - current_count, -1, -1, -1)
return torch.cat((frames, padding), dim=0).contiguous()
def _prepare_reference_frames(
value: torch.Tensor,
*,
frame_count: int,
) -> tuple[torch.Tensor, tuple[int, ...], str, str]:
"""Hash the full normalized source while retaining only its used prefix."""
source_shape = (
int(value.shape[0]),
int(value.shape[1]),
int(value.shape[2]),
3,
)
source_dtype = str(torch.float32)
retained_count = min(int(source_shape[0]), int(frame_count))
retained = torch.empty(
(retained_count, *source_shape[1:]),
dtype=torch.float32,
device="cpu",
)
digest = hashlib.sha256()
digest.update(str(source_shape).encode("ascii"))
digest.update(source_dtype.encode("ascii"))
for start in range(0, source_shape[0], _HASH_CHUNK_FRAMES):
stop = min(start + _HASH_CHUNK_FRAMES, source_shape[0])
chunk = value[start:stop, :, :, :3].detach().to(
device="cpu", dtype=torch.float32
).contiguous()
if not bool(torch.isfinite(chunk).all().item()):
raise ReferenceVideoError("Video Reference contains NaN or infinity")
digest.update(chunk.numpy().tobytes(order="C"))
if start < retained_count:
retained_stop = min(stop, retained_count)
retained[start:retained_stop].copy_(chunk[: retained_stop - start])
return retained, source_shape, source_dtype, digest.hexdigest()
def prepare_reference_video_source(
reference_video_1: Any,
*,
target_frames: int,
output_width: int,
output_height: int,
size_mode: str = REFERENCE_VIDEO_SIZE_EFFICIENT,
) -> ReferenceVideoSource | None:
if reference_video_1 is None:
return None
if not torch.is_tensor(reference_video_1) or reference_video_1.ndim != 4:
raise ReferenceVideoError(
"Video Reference must be a ComfyUI IMAGE batch with shape [T, H, W, C]"
)
if int(reference_video_1.shape[-1]) < 3:
raise ReferenceVideoError("Video Reference must contain RGB channels")
frame_count = resolve_reference_video_frame_count(
int(reference_video_1.shape[0]),
int(target_frames),
)
frames, source_shape, source_dtype, source_sha256 = _prepare_reference_frames(
reference_video_1,
frame_count=frame_count,
)
normalized_size_mode = (
str(size_mode)
if str(size_mode) in REFERENCE_VIDEO_SIZE_OPTIONS
else REFERENCE_VIDEO_SIZE_EFFICIENT
)
target_width, target_height = resolve_reference_video_size(
int(source_shape[2]),
int(source_shape[1]),
output_width=int(output_width),
output_height=int(output_height),
size_mode=normalized_size_mode,
)
contract_base = {
"reference_video_contract_version": REFERENCE_VIDEO_CONTRACT_VERSION,
"source_shape": list(source_shape),
"source_dtype": source_dtype,
"source_sha256": source_sha256,
"source_fps": _SOURCE_FPS,
"size_mode": normalized_size_mode,
"target_width": target_width,
"target_height": target_height,
"frame_count": frame_count,
"preprocess_version": REFERENCE_VIDEO_PREPROCESS_VERSION,
}
return ReferenceVideoSource(
frames=frames,
source_shape=source_shape,
source_dtype=source_dtype,
source_sha256=source_sha256,
size_mode=normalized_size_mode,
target_width=target_width,
target_height=target_height,
frame_count=frame_count,
combined_hash=_canonical_hash(contract_base),
)
def _prepare_reference_video_encode_input(
source: ReferenceVideoSource,
) -> tuple[torch.Tensor, dict[str, Any]]:
available_count = min(int(source.frames.shape[0]), int(source.frame_count))
frames = source.frames[:available_count]
if (
int(frames.shape[2]) != source.target_width
or int(frames.shape[1]) != source.target_height
):
try:
import comfy.utils
except Exception as exc:
raise ReferenceVideoError(
"ComfyUI Core image resize support is unavailable"
) from exc
frames = comfy.utils.common_upscale(
frames.movedim(-1, 1),
source.target_width,
source.target_height,
"lanczos",
"disabled",
).movedim(1, -1).contiguous()
frames = _fit_reference_frames(frames, source.frame_count)
qwen_frames = frames[::_SOURCE_FPS // 2].contiguous()
item = {
"type": "video",
"data": qwen_frames,
"timestamps": [index / 2.0 for index in range(int(qwen_frames.shape[0]))],
}
return frames, item
def _build_reference_video_assets(
source: ReferenceVideoSource,
item: dict[str, Any],
latent: torch.Tensor,
) -> ReferenceVideoAssets:
block = {
"kind": "video",
"latent_t": int(latent.shape[2]),
"latent_h": source.target_height // 16,
"latent_w": source.target_width // 16,
"ref_audio_t": 0,
"latent": latent,
"audio_latent": None,
}
return ReferenceVideoAssets(source=source, item=item, block=block)
def encode_reference_video(
video_vae: Any,
source: ReferenceVideoSource,
) -> ReferenceVideoAssets:
frames, item = _prepare_reference_video_encode_input(source)
try:
latent = video_vae.encode(frames).detach().to("cpu").contiguous()
except Exception as exc:
raise ReferenceVideoError("Video Reference VAE Encode failed") from exc
return _build_reference_video_assets(source, item, latent)
def encode_reference_video_cached(
video_vae: Any,
source: ReferenceVideoSource,
*,
cache_event=None,
) -> ReferenceVideoAssets:
"""Encode only the V3.8 Video Guide latent through the shared cache."""
from .v3.ref_encode_cache import (
get_ref_encode_cache,
make_ref_encode_cache_key,
)
frames, item = _prepare_reference_video_encode_input(source)
cache = get_ref_encode_cache()
key = make_ref_encode_cache_key(
"video_guide",
REFERENCE_VIDEO_PREPROCESS_VERSION,
source.combined_hash,
)
latent = cache.lookup(video_vae, key, event_sink=cache_event)
if latent is None:
try:
latent = video_vae.encode(frames).detach().to("cpu").contiguous()
except Exception as exc:
raise ReferenceVideoError("Video Reference VAE Encode failed") from exc
if cache.supports_vae(video_vae):
cache.store(video_vae, key, latent, event_sink=cache_event)
return _build_reference_video_assets(source, item, latent)
def combine_reference_video_identity(
visual_identity_hash: str,
source: ReferenceVideoSource | None,
) -> str:
if source is None:
return str(visual_identity_hash)
return _canonical_hash(
{
"reference_video_identity_version": 1,
"visual_identity_hash": str(visual_identity_hash),
"reference_video_hash": source.combined_hash,
}
)
def validate_reference_video_prompts(
prompts: list[str], source: ReferenceVideoSource | None
) -> str:
if source is None or any("<Video 1>" in str(prompt) for prompt in prompts):
return ""
return (
"H3C-P103 Warning: Video Reference is connected but the prompt contains no "
"<Video 1> tag; video still conditions generation, but an explicit tag is "
"recommended."
)