chore: add vision tool and Claude Code skill for UI screenshot review
- scripts/vision.py: call MiMo v2.5 via Anthropic-format API for image analysis - scripts/screenshot_mission_list.py: Playwright screenshot of /missions page - .claude/skills/vision.md: Claude Code skill for vision task delegation Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,45 @@
|
|||||||
|
---
|
||||||
|
name: vision
|
||||||
|
description: Use when you need to visually inspect a screenshot, UI render, or any image — the current model has no multimodal capability, so vision tasks MUST be delegated to this tool.
|
||||||
|
---
|
||||||
|
|
||||||
|
# Vision Tool
|
||||||
|
|
||||||
|
When you need to look at an image (screenshot, UI render, diagram, photo), call the MiMo v2.5 multimodal model via this script.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python scripts/vision.py "<image_path>" "<question>"
|
||||||
|
```
|
||||||
|
|
||||||
|
## When to Call
|
||||||
|
|
||||||
|
You MUST use this for any of the following:
|
||||||
|
- Checking a UI screenshot for layout issues (overlapping elements, overflow, alignment)
|
||||||
|
- Verifying a page render looks correct after a change
|
||||||
|
- Reading text or data from a screenshot
|
||||||
|
- Analyzing any image the user provides
|
||||||
|
- Any task where you think "I wish I could see this"
|
||||||
|
|
||||||
|
## Examples
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Basic UI inspection
|
||||||
|
python scripts/vision.py "demo/test_screenshots/timeline_page.png" \
|
||||||
|
"Check this timeline page for any UI issues: overlapping elements, clipped text, misaligned segments. Report each issue with its location."
|
||||||
|
|
||||||
|
# Comparing two screenshots
|
||||||
|
python scripts/vision.py "demo/test_screenshots/docking_modal.png" \
|
||||||
|
"Is the docking modal properly centered? Are all buttons visible and correctly labeled? Is the datetime input functional-looking?"
|
||||||
|
|
||||||
|
# Reading wiki page render
|
||||||
|
python scripts/vision.py "data/KSP/Vulture Shuttle Full Stack at Launch Pad.png" \
|
||||||
|
"Describe what you see in this image in detail."
|
||||||
|
```
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Images over ~5 MB may be slow; consider resizing large screenshots first
|
||||||
|
- The MiMo model runs remotely — no local GPU needed
|
||||||
|
- Response is text-only; the model describes what it sees
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
"""Take screenshot of mission list page for review."""
|
||||||
|
import sys
|
||||||
|
sys.path.insert(0, "d:/My Coding Project/KSP_Project")
|
||||||
|
from playwright.sync_api import sync_playwright
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch()
|
||||||
|
page = browser.new_page(viewport={"width": 1440, "height": 900})
|
||||||
|
page.goto("http://127.0.0.1:9000/missions", timeout=15000)
|
||||||
|
page.wait_for_load_state("networkidle")
|
||||||
|
page.wait_for_timeout(1000) # extra render time
|
||||||
|
page.screenshot(path="scripts/mission_list_screenshot.png", full_page=True)
|
||||||
|
browser.close()
|
||||||
|
print("Screenshot saved: scripts/mission_list_screenshot.png")
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Vision tool — call MiMo v2.5 multimodal model via Anthropic-format API."""
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
|
||||||
|
|
||||||
|
ENDPOINT = "https://api.xiaomimimo.com/anthropic/v1/messages"
|
||||||
|
MODEL = "mimo-v2.5"
|
||||||
|
|
||||||
|
def load_env():
|
||||||
|
"""Load .env from the project root (same dir as this script or CWD)."""
|
||||||
|
candidates = [
|
||||||
|
Path(__file__).resolve().parent.parent / ".env",
|
||||||
|
Path.cwd() / ".env",
|
||||||
|
]
|
||||||
|
for p in candidates:
|
||||||
|
if not p.exists():
|
||||||
|
continue
|
||||||
|
for line in p.read_text(encoding="utf-8").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
k = k.strip()
|
||||||
|
v = v.strip().strip('"').strip("'")
|
||||||
|
if k not in os.environ:
|
||||||
|
os.environ[k] = v
|
||||||
|
|
||||||
|
|
||||||
|
def image_to_base64(path: str) -> tuple[str, str]:
|
||||||
|
"""Read image file, return (base64_string, media_type)."""
|
||||||
|
p = Path(path)
|
||||||
|
if not p.exists():
|
||||||
|
raise FileNotFoundError(f"Image not found: {path}")
|
||||||
|
data = p.read_bytes()
|
||||||
|
suffix = p.suffix.lower()
|
||||||
|
media_map = {
|
||||||
|
".png": "image/png",
|
||||||
|
".jpg": "image/jpeg",
|
||||||
|
".jpeg": "image/jpeg",
|
||||||
|
".gif": "image/gif",
|
||||||
|
".webp": "image/webp",
|
||||||
|
}
|
||||||
|
media_type = media_map.get(suffix, "image/png")
|
||||||
|
return base64.b64encode(data).decode("ascii"), media_type
|
||||||
|
|
||||||
|
|
||||||
|
def call_vision(image_path: str, prompt: str, max_tokens: int = 2048) -> str:
|
||||||
|
"""Send image + prompt to MiMo, return text response."""
|
||||||
|
load_env()
|
||||||
|
api_key = os.environ.get("XIAOMI_MIMO_API_KEY")
|
||||||
|
if not api_key:
|
||||||
|
raise RuntimeError("XIAOMI_MIMO_API_KEY not set in .env")
|
||||||
|
|
||||||
|
b64, media_type = image_to_base64(image_path)
|
||||||
|
|
||||||
|
body = {
|
||||||
|
"model": MODEL,
|
||||||
|
"max_tokens": max_tokens,
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "image",
|
||||||
|
"source": {
|
||||||
|
"type": "base64",
|
||||||
|
"media_type": media_type,
|
||||||
|
"data": b64,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{"type": "text", "text": prompt},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
req = Request(
|
||||||
|
ENDPOINT,
|
||||||
|
data=json.dumps(body).encode("utf-8"),
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"x-api-key": api_key,
|
||||||
|
"anthropic-version": "2023-06-01",
|
||||||
|
},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urlopen(req, timeout=120) as resp:
|
||||||
|
result = json.loads(resp.read().decode("utf-8"))
|
||||||
|
except HTTPError as exc:
|
||||||
|
body_text = exc.read().decode("utf-8", errors="replace")
|
||||||
|
return f"HTTP {exc.code}: {body_text[:2000]}"
|
||||||
|
except URLError as exc:
|
||||||
|
return f"Transport error: {exc}"
|
||||||
|
|
||||||
|
# Anthropic Messages API response format
|
||||||
|
content = result.get("content", [])
|
||||||
|
texts = []
|
||||||
|
for block in content:
|
||||||
|
if block.get("type") == "text":
|
||||||
|
texts.append(block["text"])
|
||||||
|
return "\n".join(texts) if texts else json.dumps(result, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
print("Usage: python vision.py <image_path> <prompt>")
|
||||||
|
print("Example: python vision.py screenshot.png 'Are there any overlapping elements?'")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
image_path = sys.argv[1]
|
||||||
|
prompt = sys.argv[2]
|
||||||
|
result = call_vision(image_path, prompt)
|
||||||
|
print(result)
|
||||||
Reference in New Issue
Block a user