diff --git a/cptr/env.py b/cptr/env.py index 75a8d9e..253b269 100644 --- a/cptr/env.py +++ b/cptr/env.py @@ -25,3 +25,8 @@ CHAT_MAX_ITERATIONS = int(os.environ.get("CHAT_MAX_ITERATIONS", "2048")) ENABLE_CHAT_RECONCILE_ON_STARTUP: bool = os.environ.get( "ENABLE_CHAT_RECONCILE_ON_STARTUP", "true" ).lower() in ("true", "1", "yes") + +# ── AI stream settings ────────────────────────────────────── +STREAM_CONNECT_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_CONNECT_TIMEOUT", "30")) +STREAM_READ_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_READ_TIMEOUT", "300")) +STREAM_WRITE_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_WRITE_TIMEOUT", "600")) diff --git a/cptr/utils/ai.py b/cptr/utils/ai.py index e4eabc4..a6a42c9 100644 --- a/cptr/utils/ai.py +++ b/cptr/utils/ai.py @@ -18,15 +18,27 @@ from typing import Dict, List import httpx from pydantic import BaseModel +from cptr.env import ( + STREAM_CONNECT_TIMEOUT_SECONDS, + STREAM_READ_TIMEOUT_SECONDS, + STREAM_WRITE_TIMEOUT_SECONDS, +) + logger = logging.getLogger(__name__) _STREAM_RETRY_ATTEMPTS = 3 +_STREAM_TIMEOUT = httpx.Timeout( + STREAM_CONNECT_TIMEOUT_SECONDS, + read=STREAM_READ_TIMEOUT_SECONDS, + write=STREAM_WRITE_TIMEOUT_SECONDS, +) _STREAM_RETRY_ERRORS = ( httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadError, httpx.ReadTimeout, httpx.RemoteProtocolError, + httpx.WriteTimeout, ) @@ -184,7 +196,7 @@ async def stream_anthropic( emitted = False for attempt in range(_STREAM_RETRY_ATTEMPTS): try: - async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client: + async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client: logger.info("[stream] anthropic POST %s/messages model=%s", url, form_data.model) async with client.stream( "POST", f"{url}/messages", json=body, headers=headers @@ -238,7 +250,7 @@ async def stream_anthropic( if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1: raise logger.warning( - "[stream] anthropic transient read failure before first event; retrying (%s/%s)", + "[stream] anthropic transient stream failure before first event; retrying (%s/%s)", attempt + 1, _STREAM_RETRY_ATTEMPTS, exc_info=True, @@ -289,7 +301,7 @@ async def stream_openai_completions( emitted = False for attempt in range(_STREAM_RETRY_ATTEMPTS): try: - async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client: + async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client: logger.info("[stream] openai completions POST %s/chat/completions model=%s", url, form_data.model) async with client.stream( "POST", f"{url}/chat/completions", json=body, headers=headers @@ -341,7 +353,7 @@ async def stream_openai_completions( if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1: raise logger.warning( - "[stream] openai completions transient read failure before first event; retrying (%s/%s)", + "[stream] openai completions transient stream failure before first event; retrying (%s/%s)", attempt + 1, _STREAM_RETRY_ATTEMPTS, exc_info=True, @@ -406,7 +418,7 @@ async def stream_openai_responses( emitted = False for attempt in range(_STREAM_RETRY_ATTEMPTS): try: - async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client: + async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client: logger.info("[stream] openai responses POST %s/responses model=%s", url, form_data.model) async with client.stream( "POST", f"{url}/responses", json=body, headers=headers @@ -446,7 +458,7 @@ async def stream_openai_responses( if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1: raise logger.warning( - "[stream] openai responses transient read failure before first event; retrying (%s/%s)", + "[stream] openai responses transient stream failure before first event; retrying (%s/%s)", attempt + 1, _STREAM_RETRY_ATTEMPTS, exc_info=True, diff --git a/cptr/utils/chat_task.py b/cptr/utils/chat_task.py index fd440ea..ed08201 100644 --- a/cptr/utils/chat_task.py +++ b/cptr/utils/chat_task.py @@ -474,21 +474,23 @@ async def run_chat_task( logger.info("[task %s] start: existing content=%d chars, output=%d items", message_id[:8], len(content), len(output_items)) - def _flush_text(): + def _flush_text() -> dict | None: """Flush accumulated text into a message output item.""" nonlocal text_buffer if not text_buffer: - return + return None logger.info("[task %s] flush_text: %d chars into message item", message_id[:8], len(text_buffer)) - output_items.append({ + item = { "type": "message", "id": str(uuid.uuid4()), "status": "completed", "role": "assistant", "content": [{"type": "output_text", "text": text_buffer}], - }) + } + output_items.append(item) text_buffer = "" + return item def _sync_state(): """Update in-memory state so API can serve it on refresh.""" @@ -546,7 +548,7 @@ async def run_chat_task( elif event["type"] == "tool_call": # Flush any text before the tool call - _flush_text() + flushed_item = _flush_text() name = event["name"] tool = TOOLS.get(name) @@ -574,6 +576,8 @@ async def run_chat_task( "output": result, } output_items.append(result_item) + if flushed_item: + await emit(output=flushed_item) await emit(output=item) await emit(output=result_item) _sync_state() @@ -614,6 +618,8 @@ async def run_chat_task( output=output_items, done=False, ) + if flushed_item: + await emit(output=flushed_item) await emit(output=item) await emit(done=True) return