Auto-select Ollama vision model for analysis

This commit is contained in:
OpenClaw Agent
2026-07-10 10:56:00 +00:00
parent 6551b4f6c3
commit 64a7b3d371

View File

@@ -437,6 +437,19 @@ _PROVIDER_DEFAULTS = {
"lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
"custom": {"url": "", "model": ""},
}
_OLLAMA_VISION_MODEL_HINTS = (
"qwen2.5vl",
"qwen2.5-vl",
"qwen2vl",
"qwen2-vl",
"llava",
"bakllava",
"minicpm-v",
"minicpm",
"moondream",
"gemma3",
"internvl",
)
_ANALYZE_PROMPT = (
"Describe only the visible character in exactly two complete sentences. "
@@ -584,6 +597,13 @@ def _extract_json_description(text: str) -> str:
return ""
def _is_likely_vision_model_name(model_name: str) -> bool:
lower = (model_name or "").strip().lower()
if not lower:
return False
return any(hint in lower for hint in _OLLAMA_VISION_MODEL_HINTS)
def _resolve_provider(data):
provider = (data.get("provider") or "ollama").lower()
defs = _PROVIDER_DEFAULTS.get(provider, _PROVIDER_DEFAULTS["ollama"])
@@ -629,6 +649,39 @@ async def analyze_character_endpoint(request):
try:
async with aiohttp.ClientSession() as session:
if provider == "ollama":
requested_model_name = model_name
if not _is_likely_vision_model_name(model_name):
try:
async with session.get(f"{base_url}/api/tags", timeout=20) as tags_response:
if tags_response.status == 200:
tags_json = await tags_response.json()
models = tags_json.get("models") or []
candidate_names = []
for item in models:
if isinstance(item, dict):
name = item.get("name") or item.get("model")
if isinstance(name, str):
candidate_names.append(name)
vision_candidates = [name for name in candidate_names if _is_likely_vision_model_name(name)]
if vision_candidates:
model_name = vision_candidates[0]
log.info(
"[LTXDirector] Auto-selected Ollama vision model '%s' instead of non-vision default '%s'.",
model_name, requested_model_name,
)
except Exception:
pass
if not _is_likely_vision_model_name(model_name):
return web.json_response({
"status": "error",
"message": (
"No Ollama vision model is configured. The current default "
f"'{requested_model_name}' is not a vision model. Install/select a vision model "
"such as qwen2.5vl, llava, moondream, minicpm-v, or gemma3."
),
})
generated_text = ""
payload = {
"model": model_name,