This commit is contained in:
Timothy Jaeryang Baek
2026-06-12 17:21:21 +01:00
Unverified
parent 73d2fe054f
commit d53b94c210
4 changed files with 247 additions and 30 deletions
@@ -38,6 +38,7 @@
import AssistantMessage from './AssistantMessage.svelte';
import ChatHistory from './ChatHistory.svelte';
import Spinner from '../common/Spinner.svelte';
import { toast } from 'svelte-sonner';
interface Props {
workspace: string;
@@ -307,6 +308,7 @@
delta?: string;
output?: any;
done?: boolean;
error?: string;
queue_processed?: boolean;
title?: string;
}) {
@@ -375,6 +377,9 @@
}
allMessages = [...allMessages];
}
if (data.error) {
toast.error(data.error, { duration: 8000 });
}
if (data.done) {
// Clear streaming indicator for this tab
if (tabId) {
+42 -4
View File
@@ -187,6 +187,24 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]:
content = formatted_content
if role == "tool":
# tool result → Anthropic tool_result block
# Content may be a string or a list of blocks (multimodal image results)
if isinstance(content, list):
# Multimodal tool result — convert blocks to Anthropic format
tool_content = []
for block in content:
if block.get("type") == "text":
tool_content.append({"type": "text", "text": block.get("text", "")})
elif block.get("type") == "image":
tool_content.append({
"type": "image",
"source": {
"type": "base64",
"media_type": block.get("media_type", "image/jpeg"),
"data": block.get("base64", ""),
}
})
else:
tool_content = content
result.append(
{
"role": "user",
@@ -194,7 +212,7 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]:
{
"type": "tool_result",
"tool_use_id": m.get("tool_call_id", ""),
"content": content,
"content": tool_content,
}
],
}
@@ -338,6 +356,9 @@ def _to_openai_messages(messages: list[dict], instructions: str) -> list[dict]:
for block in content:
if block.get("type") == "text":
formatted_content.append({"type": "text", "text": block.get("text", "")})
elif block.get("type") == "image_url":
# Already in OpenAI-native format (e.g. from image extraction)
formatted_content.append(block)
elif block.get("type") == "image":
data_uri = f"data:{block.get('media_type', 'image/jpeg')};base64,{block.get('base64', '')}"
formatted_content.append({
@@ -460,11 +481,20 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]:
if role == "system":
continue
if role == "tool":
content = m.get("content", "")
if isinstance(content, list):
# Multimodal tool content — extract text only for output
# (images are handled in the agentic loop)
text_parts = []
for block in content:
if block.get("type") == "text":
text_parts.append(block.get("text", ""))
content = "\n".join(text_parts)
items.append(
{
"type": "function_call_output",
"call_id": m.get("tool_call_id", ""),
"output": m.get("content", ""),
"output": content,
}
)
elif role == "assistant" and m.get("tool_calls"):
@@ -472,10 +502,11 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]:
items.append(
{
"type": "function_call",
"id": tc.get("id", ""),
"id": tc.get("fc_id", tc.get("id", "")),
"call_id": tc.get("id", ""),
"name": tc["function"]["name"],
"arguments": tc["function"].get("arguments", "{}"),
"status": "completed",
}
)
else:
@@ -529,12 +560,18 @@ async def stream_openai_responses(
try:
async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client:
logger.info(
"[stream] openai responses POST %s/responses model=%s", url, form_data.model
"[stream] openai responses POST %s/responses model=%s input_items=%d types=%s",
url, form_data.model,
len(body.get("input", [])),
[i.get("type", i.get("role", "?")) for i in body.get("input", [])],
)
async with client.stream(
"POST", f"{url}/responses", json=body, headers=headers
) as resp:
logger.info("[stream] openai responses status=%s", resp.status_code)
if resp.status_code >= 400:
error_body = await resp.aread()
logger.error("[stream] openai responses error body: %s", error_body.decode(errors="replace"))
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: "):
@@ -552,6 +589,7 @@ async def stream_openai_responses(
emitted = True
yield {
"type": "tool_call",
"id": item.get("id", ""),
"call_id": item["call_id"],
"name": item["name"],
"arguments": json.loads(item["arguments"]),
+118 -26
View File
@@ -560,7 +560,7 @@ async def _load_message_history(
text_content = entry["content"]
# Append file:// references so the AI can read them with view_file
# Append file:// references so the AI can read them with read_file
if non_images:
from cptr.utils.storage import UPLOADS_DIR
file_refs = []
@@ -600,16 +600,18 @@ async def _load_message_history(
tool_calls = []
for item in m.output:
if item.get("type") == "function_call" and item.get("status") == "completed":
tool_calls.append(
{
"id": item["call_id"],
"type": "function",
"function": {
"name": item["name"],
"arguments": json.dumps(item.get("arguments", {})),
},
}
)
tc = {
"id": item["call_id"],
"type": "function",
"function": {
"name": item["name"],
"arguments": json.dumps(item.get("arguments", {})),
},
}
# Preserve Responses API fc_ ID for round-tripping
if item.get("fc_id"):
tc["fc_id"] = item["fc_id"]
tool_calls.append(tc)
if tool_calls:
entry["tool_calls"] = tool_calls
@@ -627,12 +629,32 @@ async def _load_message_history(
return result, existing_summary
def _parse_image_data_uri(result: str) -> tuple[str, str] | None:
"""Check if a tool result is a data URI image (from read_file on image files).
Returns (media_type, base64_data) if it's a data URI image, else None.
"""
if not result.startswith("data:image/"):
return None
# data:image/png;base64,iVBOR...
try:
header, b64_data = result.split(",", 1)
media_type = header.split(";")[0].replace("data:", "")
return media_type, b64_data
except (ValueError, IndexError):
return None
def _append_tool_to_messages(messages: list[dict], event: dict, result: str, provider: str):
"""Append a tool call + result to the message history for the next API call."""
# Guard against oversized tool outputs
if len(result) > CHAT_TOOL_MAX_CHARS:
half = CHAT_TOOL_MAX_CHARS // 2
result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:]
# Check for image result before truncation (data URI is large but needed)
image = _parse_image_data_uri(result)
if not image:
# Guard against oversized tool outputs (skip for images, handled above)
if len(result) > CHAT_TOOL_MAX_CHARS:
half = CHAT_TOOL_MAX_CHARS // 2
result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:]
# Add assistant message with tool_call
messages.append(
@@ -642,6 +664,7 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro
"tool_calls": [
{
"id": event["call_id"],
"fc_id": event.get("id", ""),
"type": "function",
"function": {
"name": event["name"],
@@ -651,14 +674,35 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro
],
}
)
# Add tool result
messages.append(
{
"role": "tool",
"tool_call_id": event["call_id"],
"content": result,
}
)
if image:
# Structured multimodal content — provider converters handle the
# "image" block type appropriately for each API.
media_type, b64_data = image
path = event["arguments"].get("path", "image")
messages.append(
{
"role": "tool",
"tool_call_id": event["call_id"],
"content": [
{"type": "text", "text": f"Image file: {path}"},
{
"type": "image",
"media_type": media_type,
"base64": b64_data,
},
],
}
)
else:
# Plain text tool result
messages.append(
{
"role": "tool",
"tool_call_id": event["call_id"],
"content": result,
}
)
def _find_safe_split(messages: list[dict], target_keep: int) -> int:
@@ -902,9 +946,36 @@ async def run_chat_task(
message_id[:8], len(drop_zone), len(keep_zone), len(summary),
)
# Anthropic supports images natively in tool_result content blocks.
# Chat Completions and Responses API don't support multimodal tool messages,
# so extract images into a follow-up user message.
api_messages = messages
if provider != "anthropic":
image_blocks = []
api_messages = []
for m in messages:
if m.get("role") == "tool" and isinstance(m.get("content"), list):
text_parts = []
for part in m["content"]:
if part.get("type") == "text":
text_parts.append(part.get("text", ""))
elif part.get("type") == "image":
image_blocks.append(part)
api_messages.append({**m, "content": "\n".join(text_parts)})
else:
api_messages.append(m)
if image_blocks:
api_messages.append({
"role": "user",
"content": [
{"type": "text", "text": "Here are the images from the tool results above."},
*image_blocks,
],
})
form_data = ChatCompletionForm(
model=model,
messages=messages,
messages=api_messages,
instructions=system,
tools=tools,
)
@@ -935,6 +1006,7 @@ async def run_chat_task(
"type": "function_call",
"id": str(uuid.uuid4()),
"call_id": event["call_id"],
"fc_id": event.get("id", ""),
"name": name,
"arguments": event["arguments"],
}
@@ -1073,15 +1145,35 @@ async def run_chat_task(
except Exception as e:
logger.exception(f"Chat task error for message {message_id}")
_flush_text()
error_msg = str(e)
# Try to extract API error body for more detail
if hasattr(e, 'response'):
try:
body = e.response.text or ""
if body:
import json as _json
err_data = _json.loads(body)
api_msg = err_data.get("error", {}).get("message", "")
if api_msg:
error_msg = api_msg
except Exception:
pass
# Append error to content so it's visible in the chat
error_block = f"\n\n> **Error:** {error_msg}"
content += error_block
text_buffer += error_block
flushed_item = _flush_text()
if flushed_item:
await emit(output=flushed_item)
await ChatMessage.update(
message_id,
content=content,
output=output_items,
done=True,
meta={"error": str(e)},
meta={"error": error_msg},
)
_task_state.pop(message_id, None)
await _emit_done()
await emit(done=True, error=error_msg)
finally:
_tasks.pop(message_id, None)
_task_state.pop(message_id, None)
+82
View File
@@ -78,6 +78,84 @@ def _truncate_output(text: str, max_chars: int = 80_000) -> str:
return text[:half] + "\n\n... (truncated) ...\n\n" + text[-half:]
# ── Image support ───────────────────────────────────────────
IMAGE_EXTENSIONS = {
".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif",
}
_IMAGE_MAX_BYTES = 5 * 1024 * 1024 # 5 MB target for API payload
_IMAGE_MIME = {
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".gif": "image/gif",
".webp": "image/webp",
".bmp": "image/bmp",
".tiff": "image/tiff",
".tif": "image/tiff",
}
def _read_image_file(full: Path, path: str) -> str:
"""Read an image file and return a data URI string.
If the file exceeds _IMAGE_MAX_BYTES, attempts to resize it down
using Pillow. Falls back to a text error if Pillow is unavailable
and the file is too large.
"""
import base64
size = full.stat().st_size
ext = full.suffix.lower()
media_type = _IMAGE_MIME.get(ext, "image/png")
data = full.read_bytes()
if size > _IMAGE_MAX_BYTES:
try:
from PIL import Image
import io
img = Image.open(io.BytesIO(data))
# Progressively scale down until under limit
# Use JPEG for lossy formats, PNG for lossless
out_format = "JPEG" if ext in (".jpg", ".jpeg", ".bmp", ".tiff", ".tif") else "PNG"
if out_format == "JPEG":
media_type = "image/jpeg"
# Convert RGBA to RGB for JPEG
if img.mode in ("RGBA", "P"):
img = img.convert("RGB")
else:
media_type = "image/png"
scale = 0.8 # start at 80%
for _ in range(10):
new_w = int(img.width * scale)
new_h = int(img.height * scale)
if new_w < 100 or new_h < 100:
break
resized = img.resize((new_w, new_h), Image.LANCZOS)
buf = io.BytesIO()
save_kwargs = {"quality": 85} if out_format == "JPEG" else {}
resized.save(buf, format=out_format, **save_kwargs)
if buf.tell() <= _IMAGE_MAX_BYTES:
data = buf.getvalue()
size = len(data)
break
scale *= 0.7 # more aggressive on each pass
else:
return f"Error: image too large ({_human_size(full.stat().st_size)}) and could not be resized below 5MB."
except ImportError:
return (
f"Error: image file is too large ({_human_size(size)}). "
f"Install Pillow (`pip install Pillow`) to enable automatic resizing."
)
b64 = base64.b64encode(data).decode("ascii")
return f"data:{media_type};base64,{b64}"
# ── Tool functions ──────────────────────────────────────────
@@ -100,6 +178,10 @@ async def read_file(
if not full.is_file():
return f"Error: file not found: {path}"
# Image files: return base64 JSON instead of garbled text
if full.suffix.lower() in IMAGE_EXTENSIONS:
return await asyncio.to_thread(_read_image_file, full, path)
def _read():
size = full.stat().st_size
if size > 500_000: