From 64a7b3d371997fa5f125018563fa0e0d205b8306 Mon Sep 17 00:00:00 2001
From: OpenClaw Agent <openclaw-agent@local>
Date: Fri, 10 Jul 2026 10:56:00 +0000
Subject: [PATCH] Auto-select Ollama vision model for analysis

---
 ltx_director.py | 73 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 63 insertions(+), 10 deletions(-)

diff --git a/ltx_director.py b/ltx_director.py
index 4581aee..cf39beb 100644
--- a/ltx_director.py
+++ b/ltx_director.py
@@ -432,11 +432,24 @@ async def ltx_director_check_file(request):
 
 
 # --- Provider defaults shared by the analyze + unload endpoints ---
-_PROVIDER_DEFAULTS = {
-    "ollama": {"url": "http://127.0.0.1:11434", "model": "huihui_ai/qwen3.5-abliterated:2b"},
-    "lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
-    "custom": {"url": "", "model": ""},
-}
+_PROVIDER_DEFAULTS = {
+    "ollama": {"url": "http://127.0.0.1:11434", "model": "huihui_ai/qwen3.5-abliterated:2b"},
+    "lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
+    "custom": {"url": "", "model": ""},
+}
+_OLLAMA_VISION_MODEL_HINTS = (
+    "qwen2.5vl",
+    "qwen2.5-vl",
+    "qwen2vl",
+    "qwen2-vl",
+    "llava",
+    "bakllava",
+    "minicpm-v",
+    "minicpm",
+    "moondream",
+    "gemma3",
+    "internvl",
+)
 
 _ANALYZE_PROMPT = (
     "Describe only the visible character in exactly two complete sentences. "
@@ -582,6 +595,13 @@ def _extract_json_description(text: str) -> str:
     except Exception:
         return ""
     return ""
+
+
+def _is_likely_vision_model_name(model_name: str) -> bool:
+    lower = (model_name or "").strip().lower()
+    if not lower:
+        return False
+    return any(hint in lower for hint in _OLLAMA_VISION_MODEL_HINTS)
 
 
 def _resolve_provider(data):
@@ -623,12 +643,45 @@ async def analyze_character_endpoint(request):
                 "message": f"No model name set for {provider}. Open the gear menu and enter your loaded model's name.",
             })
 
-        log.info("[LTXDirector] Analyzing Character %d via %s (%s, model '%s')...",
-                 char_index + 1, provider, base_url, model_name)
-
-        try:
-            async with aiohttp.ClientSession() as session:
+        log.info("[LTXDirector] Analyzing Character %d via %s (%s, model '%s')...",
+                 char_index + 1, provider, base_url, model_name)
+
+        try:
+            async with aiohttp.ClientSession() as session:
                 if provider == "ollama":
+                    requested_model_name = model_name
+                    if not _is_likely_vision_model_name(model_name):
+                        try:
+                            async with session.get(f"{base_url}/api/tags", timeout=20) as tags_response:
+                                if tags_response.status == 200:
+                                    tags_json = await tags_response.json()
+                                    models = tags_json.get("models") or []
+                                    candidate_names = []
+                                    for item in models:
+                                        if isinstance(item, dict):
+                                            name = item.get("name") or item.get("model")
+                                            if isinstance(name, str):
+                                                candidate_names.append(name)
+                                    vision_candidates = [name for name in candidate_names if _is_likely_vision_model_name(name)]
+                                    if vision_candidates:
+                                        model_name = vision_candidates[0]
+                                        log.info(
+                                            "[LTXDirector] Auto-selected Ollama vision model '%s' instead of non-vision default '%s'.",
+                                            model_name, requested_model_name,
+                                        )
+                        except Exception:
+                            pass
+
+                    if not _is_likely_vision_model_name(model_name):
+                        return web.json_response({
+                            "status": "error",
+                            "message": (
+                                "No Ollama vision model is configured. The current default "
+                                f"'{requested_model_name}' is not a vision model. Install/select a vision model "
+                                "such as qwen2.5vl, llava, moondream, minicpm-v, or gemma3."
+                            ),
+                        })
+
                     generated_text = ""
                     payload = {
                         "model": model_name,