Prioritize temporal fallback for latent upscale OOM
This commit is contained in:
@@ -72,6 +72,10 @@ def _is_oom_error(exc):
|
|||||||
return "out of memory" in text or "exhausted its gpu spatial fallbacks" in text
|
return "out of memory" in text or "exhausted its gpu spatial fallbacks" in text
|
||||||
|
|
||||||
|
|
||||||
|
def _should_retry_temporal_before_spatial(param):
|
||||||
|
return _uses_cuda_model_upscale(param) and int(param.get("chunk_length", 0) or 0) > CUDA_MODEL_CHUNK_LENGTH
|
||||||
|
|
||||||
|
|
||||||
def _retry_with_smaller_temporal(video, param, upscaler, exc):
|
def _retry_with_smaller_temporal(video, param, upscaler, exc):
|
||||||
if not _is_oom_error(exc):
|
if not _is_oom_error(exc):
|
||||||
raise exc
|
raise exc
|
||||||
@@ -665,6 +669,11 @@ def upscale_video_model(video, param):
|
|||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
if "out of memory" not in str(exc).lower():
|
if "out of memory" not in str(exc).lower():
|
||||||
raise
|
raise
|
||||||
|
if _should_retry_temporal_before_spatial(param):
|
||||||
|
raise RuntimeError(
|
||||||
|
"out of memory: retry H3 latent upscale with a smaller temporal chunk "
|
||||||
|
"before spatial fallback"
|
||||||
|
) from exc
|
||||||
smaller = _shrink_model_tile_param(param)
|
smaller = _shrink_model_tile_param(param)
|
||||||
if smaller is None:
|
if smaller is None:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
|
|||||||
@@ -1290,6 +1290,40 @@ class DumasH3LongVideosHelperTests(unittest.TestCase):
|
|||||||
self.assertEqual(chunk_length, 85)
|
self.assertEqual(chunk_length, 85)
|
||||||
self.assertEqual(temporal_overlap, 17)
|
self.assertEqual(temporal_overlap, 17)
|
||||||
|
|
||||||
|
def test_cuda_model_oom_retries_temporal_before_spatial_fallback(self):
|
||||||
|
latent = importlib.import_module("dumas_h3_latent_upscale")
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
original_tiled = latent._upscale_video_model_tiled
|
||||||
|
original_shrink_model = latent._shrink_model_tile_param
|
||||||
|
try:
|
||||||
|
def tiled(_video, param):
|
||||||
|
calls.append(("tiled", param.get("chunk_length"), param.get("tile_size_mode")))
|
||||||
|
raise RuntimeError("out of memory")
|
||||||
|
|
||||||
|
latent._upscale_video_model_tiled = tiled
|
||||||
|
latent._shrink_model_tile_param = (
|
||||||
|
lambda param: calls.append(("shrink_spatial", param.get("tile_size_mode"))) or None
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "smaller temporal chunk"):
|
||||||
|
latent.upscale_video_model(
|
||||||
|
"video",
|
||||||
|
{
|
||||||
|
"mode": "model",
|
||||||
|
"device": "cuda",
|
||||||
|
"model_name": "upscale.safetensors",
|
||||||
|
"chunk_length": 85,
|
||||||
|
"temporal_overlap": 17,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(calls[0], ("tiled", 85, None))
|
||||||
|
self.assertNotIn(("shrink_spatial", None), calls)
|
||||||
|
finally:
|
||||||
|
latent._upscale_video_model_tiled = original_tiled
|
||||||
|
latent._shrink_model_tile_param = original_shrink_model
|
||||||
|
|
||||||
def test_interp_temporal_params_preserve_upstream_defaults(self):
|
def test_interp_temporal_params_preserve_upstream_defaults(self):
|
||||||
latent = importlib.import_module("dumas_h3_latent_upscale")
|
latent = importlib.import_module("dumas_h3_latent_upscale")
|
||||||
chunk_length, temporal_overlap = latent._effective_temporal_params({
|
chunk_length, temporal_overlap = latent._effective_temporal_params({
|
||||||
|
|||||||
Reference in New Issue
Block a user