mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-17 05:26:47 +03:00
fix(ai): support prompt caching for DigitalOcean inference (#51559)
Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
96339544bf
commit
7b04288099
@@ -49,6 +49,7 @@ const RESPECTS_INLINE_HINTS = new Set([
|
||||
"zai-coding-messages",
|
||||
"bedrock-converse",
|
||||
"openrouter",
|
||||
"digitalocean",
|
||||
])
|
||||
|
||||
// OpenRouter upstreams other than Anthropic and Alibaba Qwen cache without breakpoints. Gemini uses only the last
|
||||
|
||||
@@ -212,9 +212,11 @@ const OpenAIChatUsage = Schema.StructWithRest(
|
||||
prompt_tokens: optionalNull(Schema.Number),
|
||||
completion_tokens: optionalNull(Schema.Number),
|
||||
total_tokens: optionalNull(Schema.Number),
|
||||
// Zai reports cache hits as top-level `cached_tokens`; DeepSeek uses `prompt_cache_hit_tokens`.
|
||||
// Provider-specific cache accounting fields.
|
||||
cached_tokens: optionalNull(Schema.Number),
|
||||
prompt_cache_hit_tokens: optionalNull(Schema.Number),
|
||||
cache_read_input_tokens: optionalNull(Schema.Number),
|
||||
cache_created_input_tokens: optionalNull(Schema.Number),
|
||||
prompt_tokens_details: optionalNull(
|
||||
Schema.StructWithRest(
|
||||
Schema.Struct({
|
||||
@@ -903,16 +905,19 @@ const mapFinishReason = Effect.fn("OpenAIChat.mapFinishReason")(function* (event
|
||||
// satisfied on both sides.
|
||||
// Providers differ on cache-hit location: OpenAI uses
|
||||
// `prompt_tokens_details.cached_tokens`, DeepSeek uses
|
||||
// `prompt_cache_hit_tokens`, and Zai uses top-level `cached_tokens`.
|
||||
// `prompt_cache_hit_tokens`, Zai uses top-level `cached_tokens`, and
|
||||
// DigitalOcean uses top-level `cache_read_input_tokens` / `cache_created_input_tokens`.
|
||||
const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string): Usage | undefined => {
|
||||
if (!usage) return undefined
|
||||
const input = usage.prompt_tokens ?? undefined
|
||||
const output = usage.completion_tokens ?? undefined
|
||||
const cached = (usage.prompt_tokens_details?.cached_tokens ??
|
||||
(usage as { prompt_cache_hit_tokens?: number | null }).prompt_cache_hit_tokens ??
|
||||
(usage as { cached_tokens?: number | null }).cached_tokens ??
|
||||
undefined) as number | undefined
|
||||
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? undefined
|
||||
const cached =
|
||||
usage.prompt_tokens_details?.cached_tokens ??
|
||||
usage.prompt_cache_hit_tokens ??
|
||||
usage.cached_tokens ??
|
||||
usage.cache_read_input_tokens ??
|
||||
undefined
|
||||
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? usage.cache_created_input_tokens ?? undefined
|
||||
const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? undefined
|
||||
const nonCached = ProviderShared.subtractTokens(input, ProviderShared.sumTokens(cached, cacheWrite))
|
||||
return new Usage({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Shared counter and TTL mapping for provider cache-marker lowering.
|
||||
import type { CacheHint } from "../../schema/index.js"
|
||||
|
||||
export interface Breakpoints {
|
||||
remaining: number
|
||||
@@ -11,3 +12,12 @@ export const newBreakpoints = (cap: number): Breakpoints => ({ remaining: cap, d
|
||||
// requests omit the wire TTL and use the provider default.
|
||||
export const ttlBucket = (ttlSeconds: number | undefined): "1h" | undefined =>
|
||||
ttlSeconds !== undefined && ttlSeconds >= 3600 ? "1h" : undefined
|
||||
|
||||
export const cacheControl = () => {
|
||||
const breakpoints = newBreakpoints(4)
|
||||
return (cache: CacheHint | undefined) => {
|
||||
if (cache === undefined || breakpoints.remaining === 0) return undefined
|
||||
breakpoints.remaining -= 1
|
||||
return { type: "ephemeral" as const, ttl: ttlBucket(cache.ttlSeconds) }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { cacheControl } from "../protocols/utils/cache.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { ProviderID, type ModelID } from "../schema/index.js"
|
||||
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
|
||||
|
||||
export const id = ProviderID.make("digitalocean")
|
||||
const baseURL = "https://inference.do-ai.run/v1"
|
||||
|
||||
export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
|
||||
ProviderAuthOption<"optional"> & {
|
||||
readonly baseURL?: string
|
||||
readonly providerOptions?: OpenAIProviderOptionsInput
|
||||
}
|
||||
|
||||
export type Settings = ProviderPackage.Settings & OpenAIProviderOptionsInput & { readonly apiKey?: string }
|
||||
|
||||
export const protocol = Protocol.make({
|
||||
id: "digitalocean-chat",
|
||||
body: {
|
||||
schema: OpenAIChat.protocol.body.schema,
|
||||
from: (request) => OpenAIChat.fromRequest(request, { cacheControl: cacheControl() }),
|
||||
},
|
||||
stream: OpenAIChat.protocol.stream,
|
||||
})
|
||||
|
||||
export const route = Route.make({
|
||||
id: "digitalocean",
|
||||
provider: id,
|
||||
providerMetadataKey: "digitalocean",
|
||||
protocol,
|
||||
endpoint: Endpoint.path("/chat/completions", { baseURL }),
|
||||
framing: OpenAIChat.framing,
|
||||
})
|
||||
|
||||
export const routes = [route]
|
||||
|
||||
export const configure = (input: LanguageModelOptions = {}) => {
|
||||
const { apiKey: _apiKey, auth: _auth, baseURL: endpoint, ...defaults } = input
|
||||
const configured = route.with({
|
||||
...defaults,
|
||||
endpoint: { baseURL: endpoint ?? baseURL },
|
||||
auth: AuthOptions.bearer(input, ["DIGITALOCEAN_ACCESS_TOKEN", "DIGITALOCEAN_API_KEY", "DO_INFERENCE_API_KEY"]),
|
||||
})
|
||||
return {
|
||||
id,
|
||||
model: (modelID: string | ModelID) =>
|
||||
configured.model<OpenAIProviderOptionsInput>({
|
||||
id: modelID,
|
||||
compatibility: {
|
||||
supportsPromptCacheKey: true,
|
||||
},
|
||||
}),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput>["model"] = (
|
||||
modelID,
|
||||
{ apiKey, baseURL, body, headers, ...providerOptions },
|
||||
) =>
|
||||
configure({
|
||||
apiKey,
|
||||
baseURL,
|
||||
headers,
|
||||
http: { body },
|
||||
providerOptions,
|
||||
}).model(modelID)
|
||||
|
||||
export * as DigitalOcean from "./digitalocean.js"
|
||||
@@ -14,6 +14,7 @@ export * as CloudflareWorkersAI from "./cloudflare-workers-ai.js"
|
||||
export * as DeepInfra from "./deepinfra.js"
|
||||
export * as Deepgram from "./deepgram.js"
|
||||
export * as DeepSeek from "./deepseek.js"
|
||||
export * as DigitalOcean from "./digitalocean.js"
|
||||
export * as ElevenLabs from "./elevenlabs.js"
|
||||
export * as Fal from "./fal.js"
|
||||
export * as Fireworks from "./fireworks.js"
|
||||
|
||||
@@ -3,11 +3,11 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { HttpOptions, ProviderID, type CacheHint, type ModelID, type OpenString } from "../schema/index.js"
|
||||
import { HttpOptions, ProviderID, type ModelID, type OpenString } from "../schema/index.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { SystemOne } from "../experimental/system-one.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
|
||||
import { cacheControl } from "../protocols/utils/cache.js"
|
||||
import { isRecord, ProviderShared } from "../protocols/shared.js"
|
||||
|
||||
export const id = ProviderID.make("openrouter")
|
||||
@@ -131,18 +131,6 @@ export const protocol = Protocol.make({
|
||||
stream: OpenAIChat.protocol.stream,
|
||||
})
|
||||
|
||||
const cacheControl = () => {
|
||||
const breakpoints = newBreakpoints(4)
|
||||
return (cache: CacheHint | undefined) => {
|
||||
if (cache === undefined || breakpoints.remaining === 0) return undefined
|
||||
breakpoints.remaining -= 1
|
||||
return {
|
||||
type: "ephemeral" as const,
|
||||
...(ttlBucket(cache.ttlSeconds) === "1h" ? { ttl: "1h" } : {}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OpenRouter forwards `reasoning.max_tokens` as the upstream thinking budget. Upstreams such as Anthropic and Alibaba
|
||||
// reject one that is not below the output limit; 1,024 is Anthropic's minimum budget.
|
||||
const fitReasoning = (reasoning: Record<string, unknown>, maxTokens: number | undefined) =>
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
AmazonBedrock,
|
||||
AnthropicCompatible,
|
||||
CloudflareAIGateway,
|
||||
DigitalOcean,
|
||||
GoogleVertexMessages,
|
||||
Meta,
|
||||
MiniMax,
|
||||
@@ -117,6 +118,33 @@ describe("applyCachePolicy", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("'auto' emits cache_control markers on DigitalOcean", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: DigitalOcean.configure({ apiKey: "test" }).model("anthropic-claude-fable-5.1"),
|
||||
system: "You are concise.",
|
||||
tools: [{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } }],
|
||||
prompt: "hi",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
tools: [{ type: "function", function: { name: "lookup" }, cache_control: { type: "ephemeral" } }],
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: [{ text: "You are concise.", cache_control: { type: "ephemeral" } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [{ text: "hi", cache_control: { type: "ephemeral" } }],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("'auto' emits Anthropic cache markers on Anthropic-compatible routes", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "openai-gpt-5-nano",
|
||||
"tags": [
|
||||
"prefix:digitalocean-chat",
|
||||
"provider:digitalocean",
|
||||
"protocol:digitalocean-chat",
|
||||
"cache",
|
||||
"tool",
|
||||
"tool-loop"
|
||||
],
|
||||
"name": "digitalocean-chat/gpt-nano-continues-a-tool-call-with-cache-markers",
|
||||
"recordedAt": "2026-10-01T07:36:41.717Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://inference.do-ai.run/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"openai-gpt-5-nano\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"reasoning_effort\":\"minimal\",\"max_completion_tokens\":1024}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\",\"name\":\"get_weather\"},\"id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"city\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\\\":\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"Paris\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\\\"}\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\",\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":23,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":155,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":178}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://inference.do-ai.run/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"openai-gpt-5-nano\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"content\":[{\"type\":\"text\",\"text\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"reasoning_effort\":\"minimal\",\"max_completion_tokens\":1024}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"Paris\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" is\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" sunny\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\".\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":13,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":193,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":206}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+53
File diff suppressed because one or more lines are too long
+54
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"model": "anthropic-claude-haiku-4.5",
|
||||
"tags": [
|
||||
"prefix:digitalocean-chat",
|
||||
"provider:digitalocean",
|
||||
"protocol:digitalocean-chat",
|
||||
"cache",
|
||||
"tool",
|
||||
"tool-loop"
|
||||
],
|
||||
"name": "digitalocean-chat/haiku-continues-a-tool-call-with-cache-markers",
|
||||
"recordedAt": "2026-10-01T07:36:38.471Z"
|
||||
},
|
||||
"interactions": [
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://inference.do-ai.run/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic-claude-haiku-4.5\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_completion_tokens\":128}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\",\"name\":\"get_weather\"},\"id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"city\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\": \\\"Pari\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"s\\\"}\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\",\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":54,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":593,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":647}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
},
|
||||
{
|
||||
"transport": "http",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://inference.do-ai.run/v1/chat/completions",
|
||||
"headers": {
|
||||
"content-type": "application/json"
|
||||
},
|
||||
"body": "{\"model\":\"anthropic-claude-haiku-4.5\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"content\":[{\"type\":\"text\",\"text\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_completion_tokens\":128}"
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": {
|
||||
"content-type": "text/event-stream"
|
||||
},
|
||||
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" is sunny.\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":7,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":668,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":675}}\n\ndata: [DONE]\n\n"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+53
File diff suppressed because one or more lines are too long
@@ -0,0 +1,78 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { LLM, LLMRequest } from "../../src/index.js"
|
||||
import { DigitalOcean } from "../../src/providers/digitalocean.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
import {
|
||||
LARGE_CACHEABLE_SYSTEM,
|
||||
expectWeatherToolLoop,
|
||||
goldenWeatherToolLoopRequest,
|
||||
runWeatherToolLoop,
|
||||
} from "../recorded-scenarios.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "digitalocean-chat",
|
||||
provider: "digitalocean",
|
||||
protocol: "digitalocean-chat",
|
||||
requires: ["DIGITAL_OCEAN_OFFICIAL_API_KEY"],
|
||||
})
|
||||
|
||||
for (const item of [
|
||||
{ id: "anthropic-claude-haiku-4.5", name: "Haiku", maxTokens: 128, providerOptions: undefined },
|
||||
{ id: "openai-gpt-5-nano", name: "GPT Nano", maxTokens: 1024, providerOptions: { reasoningEffort: "minimal" } },
|
||||
] as const) {
|
||||
const model = DigitalOcean.configure({
|
||||
apiKey: process.env.DIGITAL_OCEAN_OFFICIAL_API_KEY ?? "fixture",
|
||||
providerOptions: item.providerOptions,
|
||||
}).model(item.id)
|
||||
|
||||
describe(`DigitalOcean ${item.name} recorded`, () => {
|
||||
recorded.effect.with(
|
||||
`${item.name} reuses a cached prompt`,
|
||||
{ tags: ["cache", "usage"], metadata: { model: item.id } },
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const request = LLM.request({
|
||||
model,
|
||||
system: LARGE_CACHEABLE_SYSTEM,
|
||||
prompt: "Reply exactly: OK",
|
||||
promptCacheKey: `digitalocean-recorded-${item.id}`,
|
||||
generation: { maxTokens: item.maxTokens },
|
||||
})
|
||||
const first = yield* LLMClient.generate(request)
|
||||
const second = yield* LLMClient.generate(request)
|
||||
|
||||
expect(first.text.trim()).toMatch(/^OK\.?$/)
|
||||
expect(second.text.trim()).toMatch(/^OK\.?$/)
|
||||
expect(second.usage.cacheReadInputTokens).toBeGreaterThan(0)
|
||||
for (const response of [first, second]) {
|
||||
expect(response.usage.inputTokens).toBeGreaterThan(4096)
|
||||
expect(response.usage.inputTokens).toBe(
|
||||
(response.usage.nonCachedInputTokens ?? 0) +
|
||||
(response.usage.cacheReadInputTokens ?? 0) +
|
||||
(response.usage.cacheWriteInputTokens ?? 0),
|
||||
)
|
||||
}
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
recorded.effect.with(
|
||||
`${item.name} continues a tool call with cache markers`,
|
||||
{ tags: ["cache", "tool", "tool-loop"], metadata: { model: item.id } },
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const request = goldenWeatherToolLoopRequest({
|
||||
id: `digitalocean-${item.id}-tool-loop`,
|
||||
model,
|
||||
maxTokens: item.maxTokens,
|
||||
temperature: false,
|
||||
})
|
||||
const events = yield* runWeatherToolLoop(LLMRequest.update(request, { cache: "auto" }))
|
||||
expectWeatherToolLoop(events)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { CacheHint, LLM } from "../../src/index.js"
|
||||
import { compileRequest } from "../../src/route/client.js"
|
||||
import { DigitalOcean } from "../../src/providers/digitalocean.js"
|
||||
import { it } from "../lib/effect.js"
|
||||
import { fixedResponse } from "../lib/http.js"
|
||||
import { sseEvents } from "../lib/sse.js"
|
||||
import { LLMClient } from "../../src/route.js"
|
||||
|
||||
describe("DigitalOcean", () => {
|
||||
test("preserves package entrypoint headers and body overrides", () => {
|
||||
const model = DigitalOcean.model("anthropic-claude-fable-5.1", {
|
||||
apiKey: "test-key",
|
||||
headers: { "X-Test": "fixture" },
|
||||
body: { temperature: 0 },
|
||||
})
|
||||
expect(model.route.defaults?.http).toMatchObject({
|
||||
headers: { "X-Test": "fixture" },
|
||||
body: { temperature: 0 },
|
||||
})
|
||||
})
|
||||
|
||||
it.effect("prepares DigitalOcean models with default endpoint and auth", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1")
|
||||
|
||||
expect(model).toMatchObject({
|
||||
id: "anthropic-claude-fable-5.1",
|
||||
provider: "digitalocean",
|
||||
route: { id: "digitalocean" },
|
||||
})
|
||||
expect(model.route.endpoint.baseURL).toBe("https://inference.do-ai.run/v1")
|
||||
|
||||
const prepared = yield* compileRequest(LLM.request({ model, prompt: "Say hello.", cache: "none" }))
|
||||
|
||||
expect(prepared.route).toBe("digitalocean")
|
||||
expect(prepared.body).toMatchObject({
|
||||
model: "anthropic-claude-fable-5.1",
|
||||
messages: [{ role: "user", content: "Say hello." }],
|
||||
stream: true,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("lowers the native cache policy to DigitalOcean cache_control markers", () =>
|
||||
Effect.gen(function* () {
|
||||
const prepared = yield* compileRequest(
|
||||
LLM.request({
|
||||
model: DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1"),
|
||||
system: [
|
||||
{ type: "text", text: "Base agent", cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3_600 }) },
|
||||
{ type: "text", text: "Project instructions" },
|
||||
],
|
||||
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }],
|
||||
prompt: "Hello",
|
||||
cache: { tools: true, system: true, messages: { tail: 1 } },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
tools: [{ cache_control: { type: "ephemeral" } }],
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
{ text: "Base agent", cache_control: { type: "ephemeral", ttl: "1h" } },
|
||||
{ text: "Project instructions", cache_control: { type: "ephemeral" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [{ text: "Hello", cache_control: { type: "ephemeral" } }],
|
||||
},
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("parses DigitalOcean cache usage fields into AI.Usage", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1")
|
||||
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Say OK" })).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
{
|
||||
id: "chatcmpl-1",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "anthropic-claude-fable-5.1",
|
||||
choices: [{ index: 0, delta: { content: "OK" }, finish_reason: null }],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl-1",
|
||||
object: "chat.completion.chunk",
|
||||
created: 1,
|
||||
model: "anthropic-claude-fable-5.1",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
usage: {
|
||||
prompt_tokens: 4491,
|
||||
completion_tokens: 8,
|
||||
total_tokens: 4499,
|
||||
cache_read_input_tokens: 4483,
|
||||
cache_created_input_tokens: 0,
|
||||
},
|
||||
},
|
||||
"[DONE]",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 4491,
|
||||
outputTokens: 8,
|
||||
nonCachedInputTokens: 8,
|
||||
cacheReadInputTokens: 4483,
|
||||
totalTokens: 4499,
|
||||
})
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -93,6 +93,7 @@ const HOSTS: Readonly<Record<string, Readonly<Record<string, string>>>> = {
|
||||
},
|
||||
"cloudflare-workers-ai": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/cloudflare-workers-ai" },
|
||||
deepseek: { "@ai-sdk/openai-compatible": "@opencode/ai/providers/deepseek" },
|
||||
digitalocean: { "@ai-sdk/openai-compatible": "@opencode/ai/providers/digitalocean" },
|
||||
"fireworks-ai": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/fireworks" },
|
||||
"google-vertex": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/google-vertex/chat" },
|
||||
"kimi-for-coding": protocols("moonshot"),
|
||||
|
||||
@@ -69,6 +69,7 @@ const builtins = new Map<string, () => Promise<unknown>>([
|
||||
["@opencode/ai/providers/cloudflare-workers-ai", () => import("@opencode/ai/providers/cloudflare-workers-ai")],
|
||||
["@opencode/ai/providers/deepinfra", () => import("@opencode/ai/providers/deepinfra")],
|
||||
["@opencode/ai/providers/deepseek", () => import("@opencode/ai/providers/deepseek")],
|
||||
["@opencode/ai/providers/digitalocean", () => import("@opencode/ai/providers/digitalocean")],
|
||||
["@opencode/ai/providers/fireworks", () => import("@opencode/ai/providers/fireworks")],
|
||||
["@opencode/ai/providers/google", () => import("@opencode/ai/providers/google")],
|
||||
["@opencode/ai/providers/google-vertex", () => import("@opencode/ai/providers/google-vertex")],
|
||||
|
||||
@@ -574,6 +574,7 @@ const PROTOCOLS: Readonly<Record<string, Protocol>> = {
|
||||
"@opencode/ai/providers/cloudflare-workers-ai": workersAIChat,
|
||||
"@opencode/ai/providers/deepinfra": deepinfraChat,
|
||||
"@opencode/ai/providers/deepseek": deepseekChat,
|
||||
"@opencode/ai/providers/digitalocean": openaiChat,
|
||||
"@opencode/ai/providers/fireworks": openaiChat,
|
||||
"@opencode/ai/providers/groq": openaiChat,
|
||||
"@opencode/ai/providers/meta/chat": openaiChat,
|
||||
|
||||
@@ -296,6 +296,17 @@ test("spells Workers AI thinking controls through the chat template", () => {
|
||||
})
|
||||
|
||||
test("spells Chat Completions variants for hosting providers", () => {
|
||||
expect(
|
||||
resolve(model("@opencode/ai/providers/digitalocean", "openai-gpt-5-nano", undefined, "digitalocean"), [
|
||||
{ type: "effort", values: ["minimal", "low", "medium", "high"] },
|
||||
]),
|
||||
).toEqual([
|
||||
{ id: "minimal", settings: { reasoningEffort: "minimal" } },
|
||||
{ id: "low", settings: { reasoningEffort: "low" } },
|
||||
{ id: "medium", settings: { reasoningEffort: "medium" } },
|
||||
{ id: "high", settings: { reasoningEffort: "high" } },
|
||||
])
|
||||
|
||||
expect(
|
||||
resolve(model("@opencode/ai/providers/openai-compatible", "deepseek-ai/deepseek-v4-pro", undefined, "nvidia"), [
|
||||
{ type: "effort", values: ["none", "high", "max"] },
|
||||
|
||||
Reference in New Issue
Block a user