chris.dumas/WhatDreamsCost-Dumas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restore legacy Ollama character analysis flow

Browse Source
This commit is contained in:
OpenClaw Agent
2026-07-10 11:01:31 +00:00
parent 64a7b3d371
commit c20ded3319
1 changed files with 24 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
ltx_director.py
View File

@@ -437,124 +437,14 @@ _PROVIDER_DEFAULTS = {
"lmstudio": {"url": "http://127.0.0.1:1234", "model": ""}, "lmstudio": {"url": "http://127.0.0.1:1234", "model": ""},
"custom": {"url": "", "model": ""}, "custom": {"url": "", "model": ""},
} }
_OLLAMA_VISION_MODEL_HINTS = (
"qwen2.5vl",
"qwen2.5-vl",
"qwen2vl",
"qwen2-vl",
"llava",
"bakllava",
"minicpm-v",
"minicpm",
"moondream",
"gemma3",
"internvl",
)
_ANALYZE_PROMPT = ( _ANALYZE_PROMPT = (
"Describe only the visible character in exactly two complete sentences. " "Describe the character's physical appearance in two concise sentences. "
"Sentence 1 must cover hair, face, age impression, and any standout physical traits. " "Specify their hair color/style, face details, and their clothing type/color. "
"Sentence 2 must cover clothing, accessories, and overall silhouette. " "Keep the entire response very brief."
"Do not start with fragments like 'The'. Do not mention image quality, background, framing, or emotions."
) )
def _extract_ollama_generated_text(resp_json: dict) -> str:
"""Handle Ollama generate/chat variants and reasoning-capable models."""
if not isinstance(resp_json, dict):
return ""
candidates = []
candidates.append(resp_json.get("response"))
candidates.append(resp_json.get("thinking"))
candidates.append(resp_json.get("content"))
message = resp_json.get("message")
if isinstance(message, dict):
candidates.append(message.get("content"))
candidates.append(message.get("reasoning_content"))
candidates.append(message.get("thinking"))
for candidate in candidates:
if isinstance(candidate, str) and candidate.strip():
text = candidate.strip()
if "<think>" in text:
text = text.split("</think>")[-1].strip()
if text:
return text
return ""
def _analysis_text_is_usable(text: str) -> bool:
text = (text or "").strip()
if not text:
return False
if len(text) < 24:
return False
if len(text.split()) < 6:
return False
return True
def _sanitize_analysis_text(text: str) -> str:
"""Strip prompt-echo / reasoning junk and keep the shortest usable visual description."""
text = (text or "").strip()
if not text:
return ""
if "<think>" in text:
text = text.split("</think>")[-1].strip()
# Drop markdown emphasis and common prompt-echo scaffolding.
lines = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line:
continue
lower = line.lower()
if lower.startswith("the user wants"):
continue
if lower.startswith("sentence 1 requirements"):
continue
if lower.startswith("sentence 2 requirements"):
continue
if lower.startswith("requirements:"):
continue
if lower.startswith("let's interpret"):
continue
if lower.startswith("wait, the prompt says"):
continue
if lower.startswith("do not "):
continue
if line.startswith("- ") or line.startswith("* "):
continue
cleaned = line.replace("**", "").strip()
if cleaned:
lines.append(cleaned)
text = " ".join(lines).strip()
text = re.sub(r"\s+", " ", text)
if not text:
return ""
# If the model echoed instructions and then described the subject, keep the last descriptive sentences.
sentences = re.split(r"(?<=[.!?])\s+", text)
descriptive = []
for sentence in sentences:
sentence = sentence.strip()
if not sentence:
continue
lower = sentence.lower()
if "prompt says" in lower or "requirements" in lower or "the user wants" in lower:
continue
descriptive.append(sentence)
if descriptive:
text = " ".join(descriptive[-2:]).strip()
return text
def _compress_analysis_image_b64(b64_payload: str, max_dim: int = 768, quality: int = 82) -> str: def _compress_analysis_image_b64(b64_payload: str, max_dim: int = 768, quality: int = 82) -> str:
"""Shrink analysis images so multimodal providers do not burn their full context on pixels.""" """Shrink analysis images so multimodal providers do not burn their full context on pixels."""
try: try:
@@ -579,31 +469,6 @@ def _prepare_analysis_images(cleaned_b64_list: list[str], max_dim: int = 768, qu
] ]
def _extract_json_description(text: str) -> str:
text = (text or "").strip()
if not text:
return ""
try:
start = text.find("{")
end = text.rfind("}")
if start != -1 and end != -1 and end > start:
payload = json.loads(text[start:end + 1])
if isinstance(payload, dict):
description = payload.get("description")
if isinstance(description, str):
return description.strip()
except Exception:
return ""
return ""
def _is_likely_vision_model_name(model_name: str) -> bool:
lower = (model_name or "").strip().lower()
if not lower:
return False
return any(hint in lower for hint in _OLLAMA_VISION_MODEL_HINTS)
def _resolve_provider(data): def _resolve_provider(data):
provider = (data.get("provider") or "ollama").lower() provider = (data.get("provider") or "ollama").lower()
defs = _PROVIDER_DEFAULTS.get(provider, _PROVIDER_DEFAULTS["ollama"]) defs = _PROVIDER_DEFAULTS.get(provider, _PROVIDER_DEFAULTS["ollama"])
@@ -635,8 +500,6 @@ async def analyze_character_endpoint(request):
cleaned_b64_list.append(b64) cleaned_b64_list.append(b64)
if not cleaned_b64_list: if not cleaned_b64_list:
return web.json_response({"status": "error", "message": "No valid base64 images decoded."}) return web.json_response({"status": "error", "message": "No valid base64 images decoded."})
analysis_images = _prepare_analysis_images(cleaned_b64_list, max_dim=768, quality=82)
if provider in ("lmstudio", "custom") and not model_name: if provider in ("lmstudio", "custom") and not model_name:
return web.json_response({ return web.json_response({
"status": "error", "status": "error",
@@ -649,138 +512,35 @@ async def analyze_character_endpoint(request):
try: try:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
if provider == "ollama": if provider == "ollama":
requested_model_name = model_name analysis_images = cleaned_b64_list
if not _is_likely_vision_model_name(model_name):
try:
async with session.get(f"{base_url}/api/tags", timeout=20) as tags_response:
if tags_response.status == 200:
tags_json = await tags_response.json()
models = tags_json.get("models") or []
candidate_names = []
for item in models:
if isinstance(item, dict):
name = item.get("name") or item.get("model")
if isinstance(name, str):
candidate_names.append(name)
vision_candidates = [name for name in candidate_names if _is_likely_vision_model_name(name)]
if vision_candidates:
model_name = vision_candidates[0]
log.info(
"[LTXDirector] Auto-selected Ollama vision model '%s' instead of non-vision default '%s'.",
model_name, requested_model_name,
)
except Exception:
pass
if not _is_likely_vision_model_name(model_name):
return web.json_response({
"status": "error",
"message": (
"No Ollama vision model is configured. The current default "
f"'{requested_model_name}' is not a vision model. Install/select a vision model "
"such as qwen2.5vl, llava, moondream, minicpm-v, or gemma3."
),
})
generated_text = ""
payload = { payload = {
"model": model_name, "model": model_name,
"prompt": _ANALYZE_PROMPT, "prompt": _ANALYZE_PROMPT,
"images": analysis_images, "images": analysis_images,
"stream": False, "stream": False,
"keep_alive": 0, "keep_alive": 0,
"options": {
"temperature": 0.2,
"num_predict": 120,
},
} }
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as response: async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as response:
if response.status != 200: if response.status != 200:
err_txt = await response.text() err_txt = await response.text()
if response.status == 400 and "exceeds the available context size" in err_txt: if response.status == 400 and "exceeds the available context size" in err_txt:
analysis_images = _prepare_analysis_images(cleaned_b64_list, max_dim=512, quality=70) analysis_images = _prepare_analysis_images(cleaned_b64_list, max_dim=512, quality=70)
payload["images"] = analysis_images
async with session.post(f"{base_url}/api/generate", json=payload, timeout=300) as retry_response:
if retry_response.status != 200:
retry_err = await retry_response.text()
return web.json_response({"status": "error", "message": f"Ollama HTTP {retry_response.status}: {retry_err}"})
resp_json = await retry_response.json()
generated_text = (resp_json.get("response") or "").strip()
else: else:
return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"}) return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"})
else: else:
resp_json = await response.json() resp_json = await response.json()
generated_text = _extract_ollama_generated_text(resp_json) generated_text = (resp_json.get("response") or "").strip()
if not generated_text:
async with session.post(f"{base_url}/api/generate", json={**payload, "images": analysis_images}, timeout=300) as response:
if response.status != 200:
err_txt = await response.text()
return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"})
resp_json = await response.json()
generated_text = _extract_ollama_generated_text(resp_json)
# Some Ollama model variants return an empty or truncated response from
# `/api/generate` even though the same request succeeds via the chat endpoint.
if not _analysis_text_is_usable(generated_text):
chat_payload = {
"model": model_name,
"messages": [{
"role": "user",
"content": _ANALYZE_PROMPT,
"images": analysis_images,
}],
"stream": False,
"keep_alive": 0,
"options": {
"temperature": 0.2,
"num_predict": 120,
},
}
async with session.post(f"{base_url}/api/chat", json=chat_payload, timeout=300) as response:
if response.status == 200:
resp_json = await response.json()
generated_text = _extract_ollama_generated_text(resp_json)
else:
err_txt = await response.text()
if response.status == 400 and "exceeds the available context size" in err_txt:
smaller_images = _prepare_analysis_images(cleaned_b64_list, max_dim=384, quality=60)
retry_payload = {
**chat_payload,
"messages": [{
"role": "user",
"content": _ANALYZE_PROMPT,
"images": smaller_images,
}],
}
async with session.post(f"{base_url}/api/chat", json=retry_payload, timeout=300) as retry_response:
if retry_response.status == 200:
resp_json = await retry_response.json()
generated_text = _extract_ollama_generated_text(resp_json)
else:
err_txt = await retry_response.text()
return web.json_response({"status": "error", "message": f"Ollama HTTP {retry_response.status}: {err_txt}"})
else:
return web.json_response({"status": "error", "message": f"Ollama HTTP {response.status}: {err_txt}"})
if not _analysis_text_is_usable(generated_text) and analysis_images:
json_prompt = (
'Return JSON only with this exact shape: '
'{"description":"two concise sentences describing only the visible character\'s appearance"}'
)
single_image = [analysis_images[0]]
json_payload = {
"model": model_name,
"prompt": json_prompt,
"images": single_image,
"stream": False,
"format": "json",
"keep_alive": 0,
"options": {
"temperature": 0.1,
"num_predict": 120,
},
}
async with session.post(f"{base_url}/api/generate", json=json_payload, timeout=300) as response:
if response.status == 200:
resp_json = await response.json()
raw_text = _extract_ollama_generated_text(resp_json)
generated_text = _extract_json_description(raw_text) or raw_text
else: else:
# OpenAI-compatible vision chat (LM Studio / Custom). # OpenAI-compatible vision chat (LM Studio / Custom).
content = [{"type": "text", "text": _ANALYZE_PROMPT}] content = [{"type": "text", "text": _ANALYZE_PROMPT}]
for b64 in analysis_images: for b64 in cleaned_b64_list:
content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}) content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}})
payload = { payload = {
"model": model_name, "model": model_name,
@@ -807,12 +567,8 @@ async def analyze_character_endpoint(request):
"message": f"Could not connect to {provider} at {base_url}. Make sure the server is running and reachable.", "message": f"Could not connect to {provider} at {base_url}. Make sure the server is running and reachable.",
}) })
generated_text = _sanitize_analysis_text(generated_text) if "<think>" in generated_text:
if not _analysis_text_is_usable(generated_text): generated_text = generated_text.split("</think>")[-1].strip()
return web.json_response({
"status": "error",
"message": f"{provider} returned an empty or truncated analysis response.",
})
log.info("[LTXDirector] Analysis complete: %s", generated_text) log.info("[LTXDirector] Analysis complete: %s", generated_text)
return web.json_response({"status": "success", "description": generated_text}) return web.json_response({"status": "success", "description": generated_text})