Pass nested AV latents to H3 tiled upscale
This commit is contained in:
+39
-13
@@ -33,18 +33,26 @@ def _continuum_helper(name: str):
|
|||||||
"""Resolve a helper from the installed upstream H3 Continuum package."""
|
"""Resolve a helper from the installed upstream H3 Continuum package."""
|
||||||
import sys
|
import sys
|
||||||
for module_name, module in list(sys.modules.items()):
|
for module_name, module in list(sys.modules.items()):
|
||||||
if module is not None and module_name.endswith(".v3.second_pass"):
|
if module is not None and (
|
||||||
|
module_name.endswith(".v3.second_pass")
|
||||||
|
or module_name.endswith(".v2.sampling")
|
||||||
|
or module_name.endswith(".state")
|
||||||
|
):
|
||||||
helper = getattr(module, name, None)
|
helper = getattr(module, name, None)
|
||||||
if helper is not None:
|
if helper is not None:
|
||||||
return helper
|
return helper
|
||||||
try:
|
import importlib
|
||||||
import importlib
|
for module_name in ("v3.second_pass", "v2.sampling", "state"):
|
||||||
return getattr(importlib.import_module("v3.second_pass"), name)
|
try:
|
||||||
except Exception as exc:
|
helper = getattr(importlib.import_module(module_name), name, None)
|
||||||
raise RuntimeError(
|
if helper is not None:
|
||||||
"DUMAS H3 per-clip upscale requires the original "
|
return helper
|
||||||
"ComfyUI-H3-Continuum second_pass module."
|
except Exception:
|
||||||
) from exc
|
pass
|
||||||
|
raise RuntimeError(
|
||||||
|
"DUMAS H3 per-clip upscale requires the original "
|
||||||
|
"ComfyUI-H3-Continuum AV latent helpers."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _native_node(name: str):
|
def _native_node(name: str):
|
||||||
@@ -308,12 +316,27 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
|||||||
|
|
||||||
outputs = []
|
outputs = []
|
||||||
reports = []
|
reports = []
|
||||||
|
build_av_latent = _continuum_helper("latent_from_cpu")
|
||||||
|
extract_av_streams = _continuum_helper("extract_av_streams")
|
||||||
|
audio_values = _clips(audio_latents)
|
||||||
|
if len(audio_values) != len(latents):
|
||||||
|
raise ValueError(
|
||||||
|
"audio_latents must contain one first-pass audio latent per physical clip"
|
||||||
|
)
|
||||||
for index, (clip_latent, clip_model, clip_conditioning) in enumerate(
|
for index, (clip_latent, clip_model, clip_conditioning) in enumerate(
|
||||||
zip(latents, models, conditionings), start=1
|
zip(latents, models, conditionings), start=1
|
||||||
):
|
):
|
||||||
samples = clip_latent.get("samples") if isinstance(clip_latent, dict) else None
|
samples = clip_latent.get("samples") if isinstance(clip_latent, dict) else None
|
||||||
if samples is None or getattr(samples, "ndim", None) != 5:
|
if samples is None or getattr(samples, "ndim", None) != 5:
|
||||||
raise ValueError(f"clip {index} is not a 5-D H3 LATENT")
|
raise ValueError(f"clip {index} is not a 5-D H3 LATENT")
|
||||||
|
audio_latent = audio_values[index - 1]
|
||||||
|
audio_samples = (
|
||||||
|
audio_latent.get("samples")
|
||||||
|
if isinstance(audio_latent, dict) else None
|
||||||
|
)
|
||||||
|
if audio_samples is None:
|
||||||
|
raise ValueError(f"audio clip {index} is not a LATENT mapping")
|
||||||
|
nested_latent = build_av_latent(samples, audio_samples)
|
||||||
noise = _unwrap_single_output(
|
noise = _unwrap_single_output(
|
||||||
random_noise().get_noise(seed + index - 1), "RandomNoise"
|
random_noise().get_noise(seed + index - 1), "RandomNoise"
|
||||||
)
|
)
|
||||||
@@ -331,7 +354,7 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
|||||||
kwargs = {
|
kwargs = {
|
||||||
"model": clip_model,
|
"model": clip_model,
|
||||||
"conditioning": clip_conditioning,
|
"conditioning": clip_conditioning,
|
||||||
"latent": clip_latent,
|
"latent": nested_latent,
|
||||||
"noise": noise,
|
"noise": noise,
|
||||||
"sampler": sampler,
|
"sampler": sampler,
|
||||||
"sigmas": sigmas,
|
"sigmas": sigmas,
|
||||||
@@ -346,9 +369,12 @@ class DumasH3ContinuumTiledLatentUpscalePerClip:
|
|||||||
if not isinstance(result, (tuple, list)) or not result:
|
if not isinstance(result, (tuple, list)) or not result:
|
||||||
raise RuntimeError(f"native upscale returned no output for clip {index}")
|
raise RuntimeError(f"native upscale returned no output for clip {index}")
|
||||||
output = result[0]
|
output = result[0]
|
||||||
output_samples = output.get("samples") if isinstance(output, dict) else None
|
try:
|
||||||
if output_samples is None or output_samples.ndim != 5:
|
output_samples, _ = extract_av_streams(output)
|
||||||
raise RuntimeError(f"native upscale returned an invalid clip {index}")
|
except Exception as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"native upscale returned an invalid AV latent for clip {index}: {exc}"
|
||||||
|
) from exc
|
||||||
if output_samples.shape[2] != samples.shape[2]:
|
if output_samples.shape[2] != samples.shape[2]:
|
||||||
raise RuntimeError(f"tiled upscale changed temporal length for clip {index}")
|
raise RuntimeError(f"tiled upscale changed temporal length for clip {index}")
|
||||||
outputs.append(output)
|
outputs.append(output)
|
||||||
|
|||||||
@@ -98,15 +98,19 @@ class H3NodeTests(unittest.TestCase):
|
|||||||
"KSamplerSelect": FakeSampler,
|
"KSamplerSelect": FakeSampler,
|
||||||
"BasicScheduler": FakeScheduler,
|
"BasicScheduler": FakeScheduler,
|
||||||
}.get(name)
|
}.get(name)
|
||||||
UPSCALE._continuum_helper = lambda name: (
|
def fake_helper(name):
|
||||||
lambda plan, latents: {**plan, "updated": True}
|
if name == "latent_from_cpu":
|
||||||
)
|
return lambda video, audio: {"samples": video}
|
||||||
|
if name == "extract_av_streams":
|
||||||
|
return lambda latent: (latent["samples"], None)
|
||||||
|
return lambda plan, latents: {**plan, "updated": True}
|
||||||
|
UPSCALE._continuum_helper = fake_helper
|
||||||
try:
|
try:
|
||||||
node = UPSCALE.DumasH3ContinuumTiledLatentUpscalePerClip()
|
node = UPSCALE.DumasH3ContinuumTiledLatentUpscalePerClip()
|
||||||
outputs, audio, plan, report = node.upscale(
|
outputs, audio, plan, report = node.upscale(
|
||||||
model=["model"], clip=["clip"],
|
model=["model"], clip=["clip"],
|
||||||
video_latents=[{"samples": FakeSamples((1, 16, 85, 48, 84))}],
|
video_latents=[{"samples": FakeSamples((1, 16, 85, 48, 84))}],
|
||||||
audio_latents=[{"audio": "first-pass"}],
|
audio_latents=[{"samples": "first-pass"}],
|
||||||
assembly_plan=[{"plan": True}], refine_context=["context"],
|
assembly_plan=[{"plan": True}], refine_context=["context"],
|
||||||
latent_upscale_param=["upscale"], spatial_split_param=["spatial"],
|
latent_upscale_param=["upscale"], spatial_split_param=["spatial"],
|
||||||
upscale_sampler=["euler"], upscale_scheduler=["simple"],
|
upscale_sampler=["euler"], upscale_scheduler=["simple"],
|
||||||
@@ -115,7 +119,7 @@ class H3NodeTests(unittest.TestCase):
|
|||||||
upscale_sigmas=["connected-sigmas"],
|
upscale_sigmas=["connected-sigmas"],
|
||||||
)
|
)
|
||||||
self.assertEqual(len(outputs), 1)
|
self.assertEqual(len(outputs), 1)
|
||||||
self.assertEqual(audio, [{"audio": "first-pass"}])
|
self.assertEqual(audio, [{"samples": "first-pass"}])
|
||||||
self.assertTrue(plan["updated"])
|
self.assertTrue(plan["updated"])
|
||||||
self.assertEqual(FakeUpscale.calls[0]["sigmas"], "connected-sigmas")
|
self.assertEqual(FakeUpscale.calls[0]["sigmas"], "connected-sigmas")
|
||||||
self.assertIn("clips=1", report)
|
self.assertIn("clips=1", report)
|
||||||
|
|||||||
Reference in New Issue
Block a user