Pass nested AV latents to H3 tiled upscale
This commit is contained in:
+39
-13
@@ -33,18 +33,26 @@ def _continuum_helper(name: str):
|
||||
"""Resolve a helper from the installed upstream H3 Continuum package."""
|
||||
import sys
|
||||
for module_name, module in list(sys.modules.items()):
|
||||
if module is not None and module_name.endswith(".v3.second_pass"):
|
||||
if module is not None and (
|
||||
module_name.endswith(".v3.second_pass")
|
||||
or module_name.endswith(".v2.sampling")
|
||||
or module_name.endswith(".state")
|
||||
):
|
||||
helper = getattr(module, name, None)
|
||||
if helper is not None:
|
||||
return helper
|
||||
try:
|
||||
import importlib
|
||||
return getattr(importlib.import_module("v3.second_pass"), name)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"DUMAS H3 per-clip upscale requires the original "
|
||||
"ComfyUI-H3-Continuum second_pass module."
|
||||
) from exc
|
||||
import importlib
|
||||
for module_name in ("v3.second_pass", "v2.sampling", "state"):
|
||||
try:
|
||||
helper = getattr(importlib.import_module(module_name), name, None)
|
||||
if helper is not None:
|
||||
return helper
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(
|
||||
"DUMAS H3 per-clip upscale requires the original "
|
||||
"ComfyUI-H3-Continuum AV latent helpers."
|
||||
)
|
||||
|
||||
|
||||
def _native_node(name: str):
|
||||
@@ -308,12 +316,27 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
||||
|
||||
outputs = []
|
||||
reports = []
|
||||
build_av_latent = _continuum_helper("latent_from_cpu")
|
||||
extract_av_streams = _continuum_helper("extract_av_streams")
|
||||
audio_values = _clips(audio_latents)
|
||||
if len(audio_values) != len(latents):
|
||||
raise ValueError(
|
||||
"audio_latents must contain one first-pass audio latent per physical clip"
|
||||
)
|
||||
for index, (clip_latent, clip_model, clip_conditioning) in enumerate(
|
||||
zip(latents, models, conditionings), start=1
|
||||
):
|
||||
samples = clip_latent.get("samples") if isinstance(clip_latent, dict) else None
|
||||
if samples is None or getattr(samples, "ndim", None) != 5:
|
||||
raise ValueError(f"clip {index} is not a 5-D H3 LATENT")
|
||||
audio_latent = audio_values[index - 1]
|
||||
audio_samples = (
|
||||
audio_latent.get("samples")
|
||||
if isinstance(audio_latent, dict) else None
|
||||
)
|
||||
if audio_samples is None:
|
||||
raise ValueError(f"audio clip {index} is not a LATENT mapping")
|
||||
nested_latent = build_av_latent(samples, audio_samples)
|
||||
noise = _unwrap_single_output(
|
||||
random_noise().get_noise(seed + index - 1), "RandomNoise"
|
||||
)
|
||||
@@ -331,7 +354,7 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
||||
kwargs = {
|
||||
"model": clip_model,
|
||||
"conditioning": clip_conditioning,
|
||||
"latent": clip_latent,
|
||||
"latent": nested_latent,
|
||||
"noise": noise,
|
||||
"sampler": sampler,
|
||||
"sigmas": sigmas,
|
||||
@@ -346,9 +369,12 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
||||
if not isinstance(result, (tuple, list)) or not result:
|
||||
raise RuntimeError(f"native upscale returned no output for clip {index}")
|
||||
output = result[0]
|
||||
output_samples = output.get("samples") if isinstance(output, dict) else None
|
||||
if output_samples is None or output_samples.ndim != 5:
|
||||
raise RuntimeError(f"native upscale returned an invalid clip {index}")
|
||||
try:
|
||||
output_samples, _ = extract_av_streams(output)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"native upscale returned an invalid AV latent for clip {index}: {exc}"
|
||||
) from exc
|
||||
if output_samples.shape[2] != samples.shape[2]:
|
||||
raise RuntimeError(f"tiled upscale changed temporal length for clip {index}")
|
||||
outputs.append(output)
|
||||
|
||||
@@ -98,15 +98,19 @@ class H3NodeTests(unittest.TestCase):
|
||||
"KSamplerSelect": FakeSampler,
|
||||
"BasicScheduler": FakeScheduler,
|
||||
}.get(name)
|
||||
UPSCALE._continuum_helper = lambda name: (
|
||||
lambda plan, latents: {**plan, "updated": True}
|
||||
)
|
||||
def fake_helper(name):
|
||||
if name == "latent_from_cpu":
|
||||
return lambda video, audio: {"samples": video}
|
||||
if name == "extract_av_streams":
|
||||
return lambda latent: (latent["samples"], None)
|
||||
return lambda plan, latents: {**plan, "updated": True}
|
||||
UPSCALE._continuum_helper = fake_helper
|
||||
try:
|
||||
node = UPSCALE.DumasH3ContinuumTiledLatentUpscalePerClip()
|
||||
outputs, audio, plan, report = node.upscale(
|
||||
model=["model"], clip=["clip"],
|
||||
video_latents=[{"samples": FakeSamples((1, 16, 85, 48, 84))}],
|
||||
audio_latents=[{"audio": "first-pass"}],
|
||||
audio_latents=[{"samples": "first-pass"}],
|
||||
assembly_plan=[{"plan": True}], refine_context=["context"],
|
||||
latent_upscale_param=["upscale"], spatial_split_param=["spatial"],
|
||||
upscale_sampler=["euler"], upscale_scheduler=["simple"],
|
||||
@@ -115,7 +119,7 @@ class H3NodeTests(unittest.TestCase):
|
||||
upscale_sigmas=["connected-sigmas"],
|
||||
)
|
||||
self.assertEqual(len(outputs), 1)
|
||||
self.assertEqual(audio, [{"audio": "first-pass"}])
|
||||
self.assertEqual(audio, [{"samples": "first-pass"}])
|
||||
self.assertTrue(plan["updated"])
|
||||
self.assertEqual(FakeUpscale.calls[0]["sigmas"], "connected-sigmas")
|
||||
self.assertIn("clips=1", report)
|
||||
|
||||
Reference in New Issue
Block a user