diff --git a/cptr/frontend/src/lib/components/chat/ChatPanel.svelte b/cptr/frontend/src/lib/components/chat/ChatPanel.svelte index 662950d..e0be5c1 100644 --- a/cptr/frontend/src/lib/components/chat/ChatPanel.svelte +++ b/cptr/frontend/src/lib/components/chat/ChatPanel.svelte @@ -38,6 +38,7 @@ import AssistantMessage from './AssistantMessage.svelte'; import ChatHistory from './ChatHistory.svelte'; import Spinner from '../common/Spinner.svelte'; + import { toast } from 'svelte-sonner'; interface Props { workspace: string; @@ -307,6 +308,7 @@ delta?: string; output?: any; done?: boolean; + error?: string; queue_processed?: boolean; title?: string; }) { @@ -375,6 +377,9 @@ } allMessages = [...allMessages]; } + if (data.error) { + toast.error(data.error, { duration: 8000 }); + } if (data.done) { // Clear streaming indicator for this tab if (tabId) { diff --git a/cptr/utils/ai.py b/cptr/utils/ai.py index e162836..1577b86 100644 --- a/cptr/utils/ai.py +++ b/cptr/utils/ai.py @@ -187,6 +187,24 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]: content = formatted_content if role == "tool": # tool result → Anthropic tool_result block + # Content may be a string or a list of blocks (multimodal image results) + if isinstance(content, list): + # Multimodal tool result — convert blocks to Anthropic format + tool_content = [] + for block in content: + if block.get("type") == "text": + tool_content.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image": + tool_content.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": block.get("media_type", "image/jpeg"), + "data": block.get("base64", ""), + } + }) + else: + tool_content = content result.append( { "role": "user", @@ -194,7 +212,7 @@ def _to_anthropic_messages(messages: list[dict]) -> list[dict]: { "type": "tool_result", "tool_use_id": m.get("tool_call_id", ""), - "content": content, + "content": tool_content, } ], } @@ -338,6 +356,9 @@ def _to_openai_messages(messages: list[dict], instructions: str) -> list[dict]: for block in content: if block.get("type") == "text": formatted_content.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image_url": + # Already in OpenAI-native format (e.g. from image extraction) + formatted_content.append(block) elif block.get("type") == "image": data_uri = f"data:{block.get('media_type', 'image/jpeg')};base64,{block.get('base64', '')}" formatted_content.append({ @@ -460,11 +481,20 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]: if role == "system": continue if role == "tool": + content = m.get("content", "") + if isinstance(content, list): + # Multimodal tool content — extract text only for output + # (images are handled in the agentic loop) + text_parts = [] + for block in content: + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + content = "\n".join(text_parts) items.append( { "type": "function_call_output", "call_id": m.get("tool_call_id", ""), - "output": m.get("content", ""), + "output": content, } ) elif role == "assistant" and m.get("tool_calls"): @@ -472,10 +502,11 @@ def _to_responses_input(messages: list[dict], instructions: str) -> list[dict]: items.append( { "type": "function_call", - "id": tc.get("id", ""), + "id": tc.get("fc_id", tc.get("id", "")), "call_id": tc.get("id", ""), "name": tc["function"]["name"], "arguments": tc["function"].get("arguments", "{}"), + "status": "completed", } ) else: @@ -529,12 +560,18 @@ async def stream_openai_responses( try: async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client: logger.info( - "[stream] openai responses POST %s/responses model=%s", url, form_data.model + "[stream] openai responses POST %s/responses model=%s input_items=%d types=%s", + url, form_data.model, + len(body.get("input", [])), + [i.get("type", i.get("role", "?")) for i in body.get("input", [])], ) async with client.stream( "POST", f"{url}/responses", json=body, headers=headers ) as resp: logger.info("[stream] openai responses status=%s", resp.status_code) + if resp.status_code >= 400: + error_body = await resp.aread() + logger.error("[stream] openai responses error body: %s", error_body.decode(errors="replace")) resp.raise_for_status() async for line in resp.aiter_lines(): if not line.startswith("data: "): @@ -552,6 +589,7 @@ async def stream_openai_responses( emitted = True yield { "type": "tool_call", + "id": item.get("id", ""), "call_id": item["call_id"], "name": item["name"], "arguments": json.loads(item["arguments"]), diff --git a/cptr/utils/chat_task.py b/cptr/utils/chat_task.py index e45930d..93fdfe0 100644 --- a/cptr/utils/chat_task.py +++ b/cptr/utils/chat_task.py @@ -560,7 +560,7 @@ async def _load_message_history( text_content = entry["content"] - # Append file:// references so the AI can read them with view_file + # Append file:// references so the AI can read them with read_file if non_images: from cptr.utils.storage import UPLOADS_DIR file_refs = [] @@ -600,16 +600,18 @@ async def _load_message_history( tool_calls = [] for item in m.output: if item.get("type") == "function_call" and item.get("status") == "completed": - tool_calls.append( - { - "id": item["call_id"], - "type": "function", - "function": { - "name": item["name"], - "arguments": json.dumps(item.get("arguments", {})), - }, - } - ) + tc = { + "id": item["call_id"], + "type": "function", + "function": { + "name": item["name"], + "arguments": json.dumps(item.get("arguments", {})), + }, + } + # Preserve Responses API fc_ ID for round-tripping + if item.get("fc_id"): + tc["fc_id"] = item["fc_id"] + tool_calls.append(tc) if tool_calls: entry["tool_calls"] = tool_calls @@ -627,12 +629,32 @@ async def _load_message_history( return result, existing_summary +def _parse_image_data_uri(result: str) -> tuple[str, str] | None: + """Check if a tool result is a data URI image (from read_file on image files). + + Returns (media_type, base64_data) if it's a data URI image, else None. + """ + if not result.startswith("data:image/"): + return None + # data:image/png;base64,iVBOR... + try: + header, b64_data = result.split(",", 1) + media_type = header.split(";")[0].replace("data:", "") + return media_type, b64_data + except (ValueError, IndexError): + return None + + def _append_tool_to_messages(messages: list[dict], event: dict, result: str, provider: str): """Append a tool call + result to the message history for the next API call.""" - # Guard against oversized tool outputs - if len(result) > CHAT_TOOL_MAX_CHARS: - half = CHAT_TOOL_MAX_CHARS // 2 - result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:] + # Check for image result before truncation (data URI is large but needed) + image = _parse_image_data_uri(result) + + if not image: + # Guard against oversized tool outputs (skip for images, handled above) + if len(result) > CHAT_TOOL_MAX_CHARS: + half = CHAT_TOOL_MAX_CHARS // 2 + result = result[:half] + "\n\n...(truncated)...\n\n" + result[-half:] # Add assistant message with tool_call messages.append( @@ -642,6 +664,7 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro "tool_calls": [ { "id": event["call_id"], + "fc_id": event.get("id", ""), "type": "function", "function": { "name": event["name"], @@ -651,14 +674,35 @@ def _append_tool_to_messages(messages: list[dict], event: dict, result: str, pro ], } ) - # Add tool result - messages.append( - { - "role": "tool", - "tool_call_id": event["call_id"], - "content": result, - } - ) + + if image: + # Structured multimodal content — provider converters handle the + # "image" block type appropriately for each API. + media_type, b64_data = image + path = event["arguments"].get("path", "image") + messages.append( + { + "role": "tool", + "tool_call_id": event["call_id"], + "content": [ + {"type": "text", "text": f"Image file: {path}"}, + { + "type": "image", + "media_type": media_type, + "base64": b64_data, + }, + ], + } + ) + else: + # Plain text tool result + messages.append( + { + "role": "tool", + "tool_call_id": event["call_id"], + "content": result, + } + ) def _find_safe_split(messages: list[dict], target_keep: int) -> int: @@ -902,9 +946,36 @@ async def run_chat_task( message_id[:8], len(drop_zone), len(keep_zone), len(summary), ) + # Anthropic supports images natively in tool_result content blocks. + # Chat Completions and Responses API don't support multimodal tool messages, + # so extract images into a follow-up user message. + api_messages = messages + if provider != "anthropic": + image_blocks = [] + api_messages = [] + for m in messages: + if m.get("role") == "tool" and isinstance(m.get("content"), list): + text_parts = [] + for part in m["content"]: + if part.get("type") == "text": + text_parts.append(part.get("text", "")) + elif part.get("type") == "image": + image_blocks.append(part) + api_messages.append({**m, "content": "\n".join(text_parts)}) + else: + api_messages.append(m) + if image_blocks: + api_messages.append({ + "role": "user", + "content": [ + {"type": "text", "text": "Here are the images from the tool results above."}, + *image_blocks, + ], + }) + form_data = ChatCompletionForm( model=model, - messages=messages, + messages=api_messages, instructions=system, tools=tools, ) @@ -935,6 +1006,7 @@ async def run_chat_task( "type": "function_call", "id": str(uuid.uuid4()), "call_id": event["call_id"], + "fc_id": event.get("id", ""), "name": name, "arguments": event["arguments"], } @@ -1073,15 +1145,35 @@ async def run_chat_task( except Exception as e: logger.exception(f"Chat task error for message {message_id}") _flush_text() + error_msg = str(e) + # Try to extract API error body for more detail + if hasattr(e, 'response'): + try: + body = e.response.text or "" + if body: + import json as _json + err_data = _json.loads(body) + api_msg = err_data.get("error", {}).get("message", "") + if api_msg: + error_msg = api_msg + except Exception: + pass + # Append error to content so it's visible in the chat + error_block = f"\n\n> **Error:** {error_msg}" + content += error_block + text_buffer += error_block + flushed_item = _flush_text() + if flushed_item: + await emit(output=flushed_item) await ChatMessage.update( message_id, content=content, output=output_items, done=True, - meta={"error": str(e)}, + meta={"error": error_msg}, ) _task_state.pop(message_id, None) - await _emit_done() + await emit(done=True, error=error_msg) finally: _tasks.pop(message_id, None) _task_state.pop(message_id, None) diff --git a/cptr/utils/tools.py b/cptr/utils/tools.py index eb2e03f..7ca14c1 100644 --- a/cptr/utils/tools.py +++ b/cptr/utils/tools.py @@ -78,6 +78,84 @@ def _truncate_output(text: str, max_chars: int = 80_000) -> str: return text[:half] + "\n\n... (truncated) ...\n\n" + text[-half:] +# ── Image support ─────────────────────────────────────────── + +IMAGE_EXTENSIONS = { + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif", +} + +_IMAGE_MAX_BYTES = 5 * 1024 * 1024 # 5 MB target for API payload + +_IMAGE_MIME = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + ".tiff": "image/tiff", + ".tif": "image/tiff", +} + + +def _read_image_file(full: Path, path: str) -> str: + """Read an image file and return a data URI string. + + If the file exceeds _IMAGE_MAX_BYTES, attempts to resize it down + using Pillow. Falls back to a text error if Pillow is unavailable + and the file is too large. + """ + import base64 + + size = full.stat().st_size + ext = full.suffix.lower() + media_type = _IMAGE_MIME.get(ext, "image/png") + data = full.read_bytes() + + if size > _IMAGE_MAX_BYTES: + try: + from PIL import Image + import io + + img = Image.open(io.BytesIO(data)) + # Progressively scale down until under limit + # Use JPEG for lossy formats, PNG for lossless + out_format = "JPEG" if ext in (".jpg", ".jpeg", ".bmp", ".tiff", ".tif") else "PNG" + if out_format == "JPEG": + media_type = "image/jpeg" + # Convert RGBA to RGB for JPEG + if img.mode in ("RGBA", "P"): + img = img.convert("RGB") + else: + media_type = "image/png" + + scale = 0.8 # start at 80% + for _ in range(10): + new_w = int(img.width * scale) + new_h = int(img.height * scale) + if new_w < 100 or new_h < 100: + break + resized = img.resize((new_w, new_h), Image.LANCZOS) + buf = io.BytesIO() + save_kwargs = {"quality": 85} if out_format == "JPEG" else {} + resized.save(buf, format=out_format, **save_kwargs) + if buf.tell() <= _IMAGE_MAX_BYTES: + data = buf.getvalue() + size = len(data) + break + scale *= 0.7 # more aggressive on each pass + else: + return f"Error: image too large ({_human_size(full.stat().st_size)}) and could not be resized below 5MB." + except ImportError: + return ( + f"Error: image file is too large ({_human_size(size)}). " + f"Install Pillow (`pip install Pillow`) to enable automatic resizing." + ) + + b64 = base64.b64encode(data).decode("ascii") + return f"data:{media_type};base64,{b64}" + + # ── Tool functions ────────────────────────────────────────── @@ -100,6 +178,10 @@ async def read_file( if not full.is_file(): return f"Error: file not found: {path}" + # Image files: return base64 JSON instead of garbled text + if full.suffix.lower() in IMAGE_EXTENSIONS: + return await asyncio.to_thread(_read_image_file, full, path) + def _read(): size = full.stat().st_size if size > 500_000: