mirror of
https://github.com/open-webui/computer.git
synced 2026-11-19 04:11:57 +03:00
refac
This commit is contained in:
@@ -38,6 +38,7 @@
|
||||
import AssistantMessage from './AssistantMessage.svelte';
|
||||
import ChatHistory from './ChatHistory.svelte';
|
||||
import Spinner from '../common/Spinner.svelte';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
interface Props {
|
||||
workspace: string;
|
||||
@@ -307,6 +308,7 @@
|
||||
delta?: string;
|
||||
output?: any;
|
||||
done?: boolean;
|
||||
error?: string;
|
||||
queue_processed?: boolean;
|
||||
title?: string;
|
||||
}) {
|
||||
@@ -375,6 +377,9 @@
|
||||
}
|
||||
allMessages = [...allMessages];
|
||||
}
|
||||
if (data.error) {
|
||||
toast.error(data.error, { duration: 8000 });
|
||||
}
|
||||
if (data.done) {
|
||||
// Clear streaming indicator for this tab
|
||||
if (tabId) {
|
||||
|
||||
+42
-4
@@ -187,6 +187,24 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]:
|
||||
content = formatted_content
|
||||
if role == "tool":
|
||||
# tool result → Anthropic tool_result block
|
||||
# Content may be a string or a list of blocks (multimodal image results)
|
||||
if isinstance(content, list):
|
||||
# Multimodal tool result — convert blocks to Anthropic format
|
||||
tool_content = []
|
||||
for block in content:
|
||||
if block.get("type") == "text":
|
||||
tool_content.append({"type": "text", "text": block.get("text", "")})
|
||||
elif block.get("type") == "image":
|
||||
tool_content.append({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": block.get("media_type", "image/jpeg"),
|
||||
"data": block.get("base64", ""),
|
||||
}
|
||||
})
|
||||
else:
|
||||
tool_content = content
|
||||
result.append(
|
||||
{
|
||||
"role": "user",
|
||||
@@ -194,7 +212,7 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]:
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": m.get("tool_call_id", ""),
|
||||
"content": content,
|
||||
"content": tool_content,
|
||||
}
|
||||
],
|
||||
}
|
||||
@@ -338,6 +356,9 @@ def _to_openai_messages(messages: list[dict], instructions: str) -> list[dict]:
|
||||
for block in content:
|
||||
if block.get("type") == "text":
|
||||
formatted_content.append({"type": "text", "text": block.get("text", "")})
|
||||
elif block.get("type") == "image_url":
|
||||
# Already in OpenAI-native format (e.g. from image extraction)
|
||||
formatted_content.append(block)
|
||||
elif block.get("type") == "image":
|
||||
data_uri = f"data:{block.get('media_type', 'image/jpeg')};base64,{block.get('base64', '')}"
|
||||
formatted_content.append({
|
||||
@@ -460,11 +481,20 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]:
|
||||
if role == "system":
|
||||
continue
|
||||
if role == "tool":
|
||||
content = m.get("content", "")
|
||||
if isinstance(content, list):
|
||||
# Multimodal tool content — extract text only for output
|
||||
# (images are handled in the agentic loop)
|
||||
text_parts = []
|
||||
for block in content:
|
||||
if block.get("type") == "text":
|
||||
text_parts.append(block.get("text", ""))
|
||||
content = "\n".join(text_parts)
|
||||
items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": m.get("tool_call_id", ""),
|
||||
"output": m.get("content", ""),
|
||||
"output": content,
|
||||
}
|
||||
)
|
||||
elif role == "assistant" and m.get("tool_calls"):
|
||||
@@ -472,10 +502,11 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]:
|
||||
items.append(
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": tc.get("id", ""),
|
||||
"id": tc.get("fc_id", tc.get("id", "")),
|
||||
"call_id": tc.get("id", ""),
|
||||
"name": tc["function"]["name"],
|
||||
"arguments": tc["function"].get("arguments", "{}"),
|
||||
"status": "completed",
|
||||
}
|
||||
)
|
||||
else:
|
||||
@@ -529,12 +560,18 @@ async def stream_openai_responses(
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client:
|
||||
logger.info(
|
||||
"[stream] openai responses POST %s/responses model=%s", url, form_data.model
|
||||
"[stream] openai responses POST %s/responses model=%s input_items=%d types=%s",
|
||||
url, form_data.model,
|
||||
len(body.get("input", [])),
|
||||
[i.get("type", i.get("role", "?")) for i in body.get("input", [])],
|
||||
)
|
||||
async with client.stream(
|
||||
"POST", f"{url}/responses", json=body, headers=headers
|
||||
) as resp:
|
||||
logger.info("[stream] openai responses status=%s", resp.status_code)
|
||||
if resp.status_code >= 400:
|
||||
error_body = await resp.aread()
|
||||
logger.error("[stream] openai responses error body: %s", error_body.decode(errors="replace"))
|
||||
resp.raise_for_status()
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
@@ -552,6 +589,7 @@ async def stream_openai_responses(
|
||||
emitted = True
|
||||
yield {
|
||||
"type": "tool_call",
|
||||
"id": item.get("id", ""),
|
||||
"call_id": item["call_id"],
|
||||
"name": item["name"],
|
||||
"arguments": json.loads(item["arguments"]),
|
||||
|
||||
+118
-26
@@ -560,7 +560,7 @@ async def _load_message_history(
|
||||
|
||||
text_content = entry["content"]
|
||||
|
||||
# Append file:// references so the AI can read them with view_file
|
||||
# Append file:// references so the AI can read them with read_file
|
||||
if non_images:
|
||||
from cptr.utils.storage import UPLOADS_DIR
|
||||
file_refs = []
|
||||
@@ -600,16 +600,18 @@ async def _load_message_history(
|
||||
tool_calls = []
|
||||
for item in m.output:
|
||||
if item.get("type") == "function_call" and item.get("status") == "completed":
|
||||
tool_calls.append(
|
||||
{
|
||||
"id": item["call_id"],
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": item["name"],
|
||||
"arguments": json.dumps(item.get("arguments", {})),
|
||||
},
|
||||
}
|
||||
)
|
||||
tc = {
|
||||
"id": item["call_id"],
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": item["name"],
|
||||
"arguments": json.dumps(item.get("arguments", {})),
|
||||
},
|
||||
}
|
||||
# Preserve Responses API fc_ ID for round-tripping
|
||||
if item.get("fc_id"):
|
||||
tc["fc_id"] = item["fc_id"]
|
||||
tool_calls.append(tc)
|
||||
if tool_calls:
|
||||
entry["tool_calls"] = tool_calls
|
||||
|
||||
@@ -627,12 +629,32 @@ async def _load_message_history(
|
||||
return result, existing_summary
|
||||
|
||||
|
||||
def _parse_image_data_uri(result: str) -> tuple[str, str] | None:
|
||||
"""Check if a tool result is a data URI image (from read_file on image files).
|
||||
|
||||
Returns (media_type, base64_data) if it's a data URI image, else None.
|
||||
"""
|
||||
if not result.startswith("data:image/"):
|
||||
return None
|
||||
# data:image/png;base64,iVBOR...
|
||||
try:
|
||||
header, b64_data = result.split(",", 1)
|
||||
media_type = header.split(";")[0].replace("data:", "")
|
||||
return media_type, b64_data
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _append_tool_to_messages(messages: list[dict], event: dict, result: str, provider: str):
|
||||
"""Append a tool call + result to the message history for the next API call."""
|
||||
# Guard against oversized tool outputs
|
||||
if len(result) > CHAT_TOOL_MAX_CHARS:
|
||||
half = CHAT_TOOL_MAX_CHARS // 2
|
||||
result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:]
|
||||
# Check for image result before truncation (data URI is large but needed)
|
||||
image = _parse_image_data_uri(result)
|
||||
|
||||
if not image:
|
||||
# Guard against oversized tool outputs (skip for images, handled above)
|
||||
if len(result) > CHAT_TOOL_MAX_CHARS:
|
||||
half = CHAT_TOOL_MAX_CHARS // 2
|
||||
result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:]
|
||||
|
||||
# Add assistant message with tool_call
|
||||
messages.append(
|
||||
@@ -642,6 +664,7 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": event["call_id"],
|
||||
"fc_id": event.get("id", ""),
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": event["name"],
|
||||
@@ -651,14 +674,35 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro
|
||||
],
|
||||
}
|
||||
)
|
||||
# Add tool result
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": event["call_id"],
|
||||
"content": result,
|
||||
}
|
||||
)
|
||||
|
||||
if image:
|
||||
# Structured multimodal content — provider converters handle the
|
||||
# "image" block type appropriately for each API.
|
||||
media_type, b64_data = image
|
||||
path = event["arguments"].get("path", "image")
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": event["call_id"],
|
||||
"content": [
|
||||
{"type": "text", "text": f"Image file: {path}"},
|
||||
{
|
||||
"type": "image",
|
||||
"media_type": media_type,
|
||||
"base64": b64_data,
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Plain text tool result
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": event["call_id"],
|
||||
"content": result,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _find_safe_split(messages: list[dict], target_keep: int) -> int:
|
||||
@@ -902,9 +946,36 @@ async def run_chat_task(
|
||||
message_id[:8], len(drop_zone), len(keep_zone), len(summary),
|
||||
)
|
||||
|
||||
# Anthropic supports images natively in tool_result content blocks.
|
||||
# Chat Completions and Responses API don't support multimodal tool messages,
|
||||
# so extract images into a follow-up user message.
|
||||
api_messages = messages
|
||||
if provider != "anthropic":
|
||||
image_blocks = []
|
||||
api_messages = []
|
||||
for m in messages:
|
||||
if m.get("role") == "tool" and isinstance(m.get("content"), list):
|
||||
text_parts = []
|
||||
for part in m["content"]:
|
||||
if part.get("type") == "text":
|
||||
text_parts.append(part.get("text", ""))
|
||||
elif part.get("type") == "image":
|
||||
image_blocks.append(part)
|
||||
api_messages.append({**m, "content": "\n".join(text_parts)})
|
||||
else:
|
||||
api_messages.append(m)
|
||||
if image_blocks:
|
||||
api_messages.append({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Here are the images from the tool results above."},
|
||||
*image_blocks,
|
||||
],
|
||||
})
|
||||
|
||||
form_data = ChatCompletionForm(
|
||||
model=model,
|
||||
messages=messages,
|
||||
messages=api_messages,
|
||||
instructions=system,
|
||||
tools=tools,
|
||||
)
|
||||
@@ -935,6 +1006,7 @@ async def run_chat_task(
|
||||
"type": "function_call",
|
||||
"id": str(uuid.uuid4()),
|
||||
"call_id": event["call_id"],
|
||||
"fc_id": event.get("id", ""),
|
||||
"name": name,
|
||||
"arguments": event["arguments"],
|
||||
}
|
||||
@@ -1073,15 +1145,35 @@ async def run_chat_task(
|
||||
except Exception as e:
|
||||
logger.exception(f"Chat task error for message {message_id}")
|
||||
_flush_text()
|
||||
error_msg = str(e)
|
||||
# Try to extract API error body for more detail
|
||||
if hasattr(e, 'response'):
|
||||
try:
|
||||
body = e.response.text or ""
|
||||
if body:
|
||||
import json as _json
|
||||
err_data = _json.loads(body)
|
||||
api_msg = err_data.get("error", {}).get("message", "")
|
||||
if api_msg:
|
||||
error_msg = api_msg
|
||||
except Exception:
|
||||
pass
|
||||
# Append error to content so it's visible in the chat
|
||||
error_block = f"\n\n> **Error:** {error_msg}"
|
||||
content += error_block
|
||||
text_buffer += error_block
|
||||
flushed_item = _flush_text()
|
||||
if flushed_item:
|
||||
await emit(output=flushed_item)
|
||||
await ChatMessage.update(
|
||||
message_id,
|
||||
content=content,
|
||||
output=output_items,
|
||||
done=True,
|
||||
meta={"error": str(e)},
|
||||
meta={"error": error_msg},
|
||||
)
|
||||
_task_state.pop(message_id, None)
|
||||
await _emit_done()
|
||||
await emit(done=True, error=error_msg)
|
||||
finally:
|
||||
_tasks.pop(message_id, None)
|
||||
_task_state.pop(message_id, None)
|
||||
|
||||
@@ -78,6 +78,84 @@ def _truncate_output(text: str, max_chars: int = 80_000) -> str:
|
||||
return text[:half] + "\n\n... (truncated) ...\n\n" + text[-half:]
|
||||
|
||||
|
||||
# ── Image support ───────────────────────────────────────────
|
||||
|
||||
IMAGE_EXTENSIONS = {
|
||||
".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif",
|
||||
}
|
||||
|
||||
_IMAGE_MAX_BYTES = 5 * 1024 * 1024 # 5 MB target for API payload
|
||||
|
||||
_IMAGE_MIME = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
".tiff": "image/tiff",
|
||||
".tif": "image/tiff",
|
||||
}
|
||||
|
||||
|
||||
def _read_image_file(full: Path, path: str) -> str:
|
||||
"""Read an image file and return a data URI string.
|
||||
|
||||
If the file exceeds _IMAGE_MAX_BYTES, attempts to resize it down
|
||||
using Pillow. Falls back to a text error if Pillow is unavailable
|
||||
and the file is too large.
|
||||
"""
|
||||
import base64
|
||||
|
||||
size = full.stat().st_size
|
||||
ext = full.suffix.lower()
|
||||
media_type = _IMAGE_MIME.get(ext, "image/png")
|
||||
data = full.read_bytes()
|
||||
|
||||
if size > _IMAGE_MAX_BYTES:
|
||||
try:
|
||||
from PIL import Image
|
||||
import io
|
||||
|
||||
img = Image.open(io.BytesIO(data))
|
||||
# Progressively scale down until under limit
|
||||
# Use JPEG for lossy formats, PNG for lossless
|
||||
out_format = "JPEG" if ext in (".jpg", ".jpeg", ".bmp", ".tiff", ".tif") else "PNG"
|
||||
if out_format == "JPEG":
|
||||
media_type = "image/jpeg"
|
||||
# Convert RGBA to RGB for JPEG
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
else:
|
||||
media_type = "image/png"
|
||||
|
||||
scale = 0.8 # start at 80%
|
||||
for _ in range(10):
|
||||
new_w = int(img.width * scale)
|
||||
new_h = int(img.height * scale)
|
||||
if new_w < 100 or new_h < 100:
|
||||
break
|
||||
resized = img.resize((new_w, new_h), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
save_kwargs = {"quality": 85} if out_format == "JPEG" else {}
|
||||
resized.save(buf, format=out_format, **save_kwargs)
|
||||
if buf.tell() <= _IMAGE_MAX_BYTES:
|
||||
data = buf.getvalue()
|
||||
size = len(data)
|
||||
break
|
||||
scale *= 0.7 # more aggressive on each pass
|
||||
else:
|
||||
return f"Error: image too large ({_human_size(full.stat().st_size)}) and could not be resized below 5MB."
|
||||
except ImportError:
|
||||
return (
|
||||
f"Error: image file is too large ({_human_size(size)}). "
|
||||
f"Install Pillow (`pip install Pillow`) to enable automatic resizing."
|
||||
)
|
||||
|
||||
b64 = base64.b64encode(data).decode("ascii")
|
||||
return f"data:{media_type};base64,{b64}"
|
||||
|
||||
|
||||
# ── Tool functions ──────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -100,6 +178,10 @@ async def read_file(
|
||||
if not full.is_file():
|
||||
return f"Error: file not found: {path}"
|
||||
|
||||
# Image files: return base64 JSON instead of garbled text
|
||||
if full.suffix.lower() in IMAGE_EXTENSIONS:
|
||||
return await asyncio.to_thread(_read_image_file, full, path)
|
||||
|
||||
def _read():
|
||||
size = full.stat().st_size
|
||||
if size > 500_000:
|
||||
|
||||
Reference in New Issue
Block a user