This commit is contained in:
Timothy Jaeryang Baek
2026-06-06 11:01:52 +01:00
Unverified
parent b639b72364
commit 2950e4e2e8
3 changed files with 34 additions and 11 deletions
+5
View File
@@ -25,3 +25,8 @@ CHAT_MAX_ITERATIONS = int(os.environ.get("CHAT_MAX_ITERATIONS", "2048"))
ENABLE_CHAT_RECONCILE_ON_STARTUP: bool = os.environ.get(
"ENABLE_CHAT_RECONCILE_ON_STARTUP", "true"
).lower() in ("true", "1", "yes")
# ── AI stream settings ──────────────────────────────────────
STREAM_CONNECT_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_CONNECT_TIMEOUT", "30"))
STREAM_READ_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_READ_TIMEOUT", "300"))
STREAM_WRITE_TIMEOUT_SECONDS = float(os.environ.get("CPTR_STREAM_WRITE_TIMEOUT", "600"))
+18 -6
View File
@@ -18,15 +18,27 @@ from typing import Dict, List
import httpx
from pydantic import BaseModel
from cptr.env import (
STREAM_CONNECT_TIMEOUT_SECONDS,
STREAM_READ_TIMEOUT_SECONDS,
STREAM_WRITE_TIMEOUT_SECONDS,
)
logger = logging.getLogger(__name__)
_STREAM_RETRY_ATTEMPTS = 3
_STREAM_TIMEOUT = httpx.Timeout(
STREAM_CONNECT_TIMEOUT_SECONDS,
read=STREAM_READ_TIMEOUT_SECONDS,
write=STREAM_WRITE_TIMEOUT_SECONDS,
)
_STREAM_RETRY_ERRORS = (
httpx.ConnectError,
httpx.ConnectTimeout,
httpx.ReadError,
httpx.ReadTimeout,
httpx.RemoteProtocolError,
httpx.WriteTimeout,
)
@@ -184,7 +196,7 @@ async def stream_anthropic(
emitted = False
for attempt in range(_STREAM_RETRY_ATTEMPTS):
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client:
async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client:
logger.info("[stream] anthropic POST %s/messages model=%s", url, form_data.model)
async with client.stream(
"POST", f"{url}/messages", json=body, headers=headers
@@ -238,7 +250,7 @@ async def stream_anthropic(
if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1:
raise
logger.warning(
"[stream] anthropic transient read failure before first event; retrying (%s/%s)",
"[stream] anthropic transient stream failure before first event; retrying (%s/%s)",
attempt + 1,
_STREAM_RETRY_ATTEMPTS,
exc_info=True,
@@ -289,7 +301,7 @@ async def stream_openai_completions(
emitted = False
for attempt in range(_STREAM_RETRY_ATTEMPTS):
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client:
async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client:
logger.info("[stream] openai completions POST %s/chat/completions model=%s", url, form_data.model)
async with client.stream(
"POST", f"{url}/chat/completions", json=body, headers=headers
@@ -341,7 +353,7 @@ async def stream_openai_completions(
if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1:
raise
logger.warning(
"[stream] openai completions transient read failure before first event; retrying (%s/%s)",
"[stream] openai completions transient stream failure before first event; retrying (%s/%s)",
attempt + 1,
_STREAM_RETRY_ATTEMPTS,
exc_info=True,
@@ -406,7 +418,7 @@ async def stream_openai_responses(
emitted = False
for attempt in range(_STREAM_RETRY_ATTEMPTS):
try:
async with httpx.AsyncClient(timeout=httpx.Timeout(30, read=300)) as client:
async with httpx.AsyncClient(timeout=_STREAM_TIMEOUT) as client:
logger.info("[stream] openai responses POST %s/responses model=%s", url, form_data.model)
async with client.stream(
"POST", f"{url}/responses", json=body, headers=headers
@@ -446,7 +458,7 @@ async def stream_openai_responses(
if emitted or attempt == _STREAM_RETRY_ATTEMPTS - 1:
raise
logger.warning(
"[stream] openai responses transient read failure before first event; retrying (%s/%s)",
"[stream] openai responses transient stream failure before first event; retrying (%s/%s)",
attempt + 1,
_STREAM_RETRY_ATTEMPTS,
exc_info=True,
+11 -5
View File
@@ -474,21 +474,23 @@ async def run_chat_task(
logger.info("[task %s] start: existing content=%d chars, output=%d items",
message_id[:8], len(content), len(output_items))
def _flush_text():
def _flush_text() -> dict | None:
"""Flush accumulated text into a message output item."""
nonlocal text_buffer
if not text_buffer:
return
return None
logger.info("[task %s] flush_text: %d chars into message item",
message_id[:8], len(text_buffer))
output_items.append({
item = {
"type": "message",
"id": str(uuid.uuid4()),
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": text_buffer}],
})
}
output_items.append(item)
text_buffer = ""
return item
def _sync_state():
"""Update in-memory state so API can serve it on refresh."""
@@ -546,7 +548,7 @@ async def run_chat_task(
elif event["type"] == "tool_call":
# Flush any text before the tool call
_flush_text()
flushed_item = _flush_text()
name = event["name"]
tool = TOOLS.get(name)
@@ -574,6 +576,8 @@ async def run_chat_task(
"output": result,
}
output_items.append(result_item)
if flushed_item:
await emit(output=flushed_item)
await emit(output=item)
await emit(output=result_item)
_sync_state()
@@ -614,6 +618,8 @@ async def run_chat_task(
output=output_items,
done=False,
)
if flushed_item:
await emit(output=flushed_item)
await emit(output=item)
await emit(done=True)
return