fix(ai): support prompt caching for DigitalOcean inference (#51559)

Co-authored-by: Aiden Cline <aidenpcline@gmail.com>
This commit is contained in:
Rémy Sanchez
2026-10-01 18:56:04 +02:00
committed by GitHub
Unverified
parent 96339544bf
commit 7b04288099
17 changed files with 559 additions and 21 deletions
+1
View File
@@ -49,6 +49,7 @@ const RESPECTS_INLINE_HINTS = new Set([
"zai-coding-messages",
"bedrock-converse",
"openrouter",
"digitalocean",
])
// OpenRouter upstreams other than Anthropic and Alibaba Qwen cache without breakpoints. Gemini uses only the last
+12 -7
View File
@@ -212,9 +212,11 @@ const OpenAIChatUsage = Schema.StructWithRest(
prompt_tokens: optionalNull(Schema.Number),
completion_tokens: optionalNull(Schema.Number),
total_tokens: optionalNull(Schema.Number),
// Zai reports cache hits as top-level `cached_tokens`; DeepSeek uses `prompt_cache_hit_tokens`.
// Provider-specific cache accounting fields.
cached_tokens: optionalNull(Schema.Number),
prompt_cache_hit_tokens: optionalNull(Schema.Number),
cache_read_input_tokens: optionalNull(Schema.Number),
cache_created_input_tokens: optionalNull(Schema.Number),
prompt_tokens_details: optionalNull(
Schema.StructWithRest(
Schema.Struct({
@@ -903,16 +905,19 @@ const mapFinishReason = Effect.fn("OpenAIChat.mapFinishReason")(function* (event
// satisfied on both sides.
// Providers differ on cache-hit location: OpenAI uses
// `prompt_tokens_details.cached_tokens`, DeepSeek uses
// `prompt_cache_hit_tokens`, and Zai uses top-level `cached_tokens`.
// `prompt_cache_hit_tokens`, Zai uses top-level `cached_tokens`, and
// DigitalOcean uses top-level `cache_read_input_tokens` / `cache_created_input_tokens`.
const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string): Usage | undefined => {
if (!usage) return undefined
const input = usage.prompt_tokens ?? undefined
const output = usage.completion_tokens ?? undefined
const cached = (usage.prompt_tokens_details?.cached_tokens ??
(usage as { prompt_cache_hit_tokens?: number | null }).prompt_cache_hit_tokens ??
(usage as { cached_tokens?: number | null }).cached_tokens ??
undefined) as number | undefined
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? undefined
const cached =
usage.prompt_tokens_details?.cached_tokens ??
usage.prompt_cache_hit_tokens ??
usage.cached_tokens ??
usage.cache_read_input_tokens ??
undefined
const cacheWrite = usage.prompt_tokens_details?.cache_write_tokens ?? usage.cache_created_input_tokens ?? undefined
const reasoning = usage.completion_tokens_details?.reasoning_tokens ?? undefined
const nonCached = ProviderShared.subtractTokens(input, ProviderShared.sumTokens(cached, cacheWrite))
return new Usage({
+10
View File
@@ -1,4 +1,5 @@
// Shared counter and TTL mapping for provider cache-marker lowering.
import type { CacheHint } from "../../schema/index.js"
export interface Breakpoints {
remaining: number
@@ -11,3 +12,12 @@ export const newBreakpoints = (cap: number): Breakpoints => ({ remaining: cap, d
// requests omit the wire TTL and use the provider default.
export const ttlBucket = (ttlSeconds: number | undefined): "1h" | undefined =>
ttlSeconds !== undefined && ttlSeconds >= 3600 ? "1h" : undefined
export const cacheControl = () => {
const breakpoints = newBreakpoints(4)
return (cache: CacheHint | undefined) => {
if (cache === undefined || breakpoints.remaining === 0) return undefined
breakpoints.remaining -= 1
return { type: "ephemeral" as const, ttl: ttlBucket(cache.ttlSeconds) }
}
}
+76
View File
@@ -0,0 +1,76 @@
import type { ProviderPackage } from "../provider-package.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { cacheControl } from "../protocols/utils/cache.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { ProviderID, type ModelID } from "../schema/index.js"
import type { OpenAIProviderOptionsInput } from "./openai-options.js"
export const id = ProviderID.make("digitalocean")
const baseURL = "https://inference.do-ai.run/v1"
export type LanguageModelOptions = Omit<RouteDefaultsInput, "providerOptions"> &
ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly providerOptions?: OpenAIProviderOptionsInput
}
export type Settings = ProviderPackage.Settings & OpenAIProviderOptionsInput & { readonly apiKey?: string }
export const protocol = Protocol.make({
id: "digitalocean-chat",
body: {
schema: OpenAIChat.protocol.body.schema,
from: (request) => OpenAIChat.fromRequest(request, { cacheControl: cacheControl() }),
},
stream: OpenAIChat.protocol.stream,
})
export const route = Route.make({
id: "digitalocean",
provider: id,
providerMetadataKey: "digitalocean",
protocol,
endpoint: Endpoint.path("/chat/completions", { baseURL }),
framing: OpenAIChat.framing,
})
export const routes = [route]
export const configure = (input: LanguageModelOptions = {}) => {
const { apiKey: _apiKey, auth: _auth, baseURL: endpoint, ...defaults } = input
const configured = route.with({
...defaults,
endpoint: { baseURL: endpoint ?? baseURL },
auth: AuthOptions.bearer(input, ["DIGITALOCEAN_ACCESS_TOKEN", "DIGITALOCEAN_API_KEY", "DO_INFERENCE_API_KEY"]),
})
return {
id,
model: (modelID: string | ModelID) =>
configured.model<OpenAIProviderOptionsInput>({
id: modelID,
compatibility: {
supportsPromptCacheKey: true,
},
}),
configure,
}
}
export const provider = configure()
export const model: ProviderPackage.Definition<Settings, OpenAIProviderOptionsInput>["model"] = (
modelID,
{ apiKey, baseURL, body, headers, ...providerOptions },
) =>
configure({
apiKey,
baseURL,
headers,
http: { body },
providerOptions,
}).model(modelID)
export * as DigitalOcean from "./digitalocean.js"
+1
View File
@@ -14,6 +14,7 @@ export * as CloudflareWorkersAI from "./cloudflare-workers-ai.js"
export * as DeepInfra from "./deepinfra.js"
export * as Deepgram from "./deepgram.js"
export * as DeepSeek from "./deepseek.js"
export * as DigitalOcean from "./digitalocean.js"
export * as ElevenLabs from "./elevenlabs.js"
export * as Fal from "./fal.js"
export * as Fireworks from "./fireworks.js"
+2 -14
View File
@@ -3,11 +3,11 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { Protocol } from "../route/protocol.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { HttpOptions, ProviderID, type CacheHint, type ModelID, type OpenString } from "../schema/index.js"
import { HttpOptions, ProviderID, type ModelID, type OpenString } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import { SystemOne } from "../experimental/system-one.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { newBreakpoints, ttlBucket } from "../protocols/utils/cache.js"
import { cacheControl } from "../protocols/utils/cache.js"
import { isRecord, ProviderShared } from "../protocols/shared.js"
export const id = ProviderID.make("openrouter")
@@ -131,18 +131,6 @@ export const protocol = Protocol.make({
stream: OpenAIChat.protocol.stream,
})
const cacheControl = () => {
const breakpoints = newBreakpoints(4)
return (cache: CacheHint | undefined) => {
if (cache === undefined || breakpoints.remaining === 0) return undefined
breakpoints.remaining -= 1
return {
type: "ephemeral" as const,
...(ttlBucket(cache.ttlSeconds) === "1h" ? { ttl: "1h" } : {}),
}
}
}
// OpenRouter forwards `reasoning.max_tokens` as the upstream thinking budget. Upstreams such as Anthropic and Alibaba
// reject one that is not below the output limit; 1,024 is Anthropic's minimum budget.
const fitReasoning = (reasoning: Record<string, unknown>, maxTokens: number | undefined) =>
+28
View File
@@ -8,6 +8,7 @@ import {
AmazonBedrock,
AnthropicCompatible,
CloudflareAIGateway,
DigitalOcean,
GoogleVertexMessages,
Meta,
MiniMax,
@@ -117,6 +118,33 @@ describe("applyCachePolicy", () => {
}),
)
it.effect("'auto' emits cache_control markers on DigitalOcean", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: DigitalOcean.configure({ apiKey: "test" }).model("anthropic-claude-fable-5.1"),
system: "You are concise.",
tools: [{ name: "lookup", description: "Look up a value", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
}),
)
expect(prepared.body).toMatchObject({
tools: [{ type: "function", function: { name: "lookup" }, cache_control: { type: "ephemeral" } }],
messages: [
{
role: "system",
content: [{ text: "You are concise.", cache_control: { type: "ephemeral" } }],
},
{
role: "user",
content: [{ text: "hi", cache_control: { type: "ephemeral" } }],
},
],
})
}),
)
it.effect("'auto' emits Anthropic cache markers on Anthropic-compatible routes", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
@@ -0,0 +1,54 @@
{
"version": 1,
"metadata": {
"model": "openai-gpt-5-nano",
"tags": [
"prefix:digitalocean-chat",
"provider:digitalocean",
"protocol:digitalocean-chat",
"cache",
"tool",
"tool-loop"
],
"name": "digitalocean-chat/gpt-nano-continues-a-tool-call-with-cache-markers",
"recordedAt": "2026-10-01T07:36:41.717Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://inference.do-ai.run/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"openai-gpt-5-nano\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"reasoning_effort\":\"minimal\",\"max_completion_tokens\":1024}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\",\"name\":\"get_weather\"},\"id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"city\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\\\":\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"Paris\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\\\"}\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\",\"index\":0,\"logprobs\":null}],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840200,\"id\":\"chatcmpl-EU5dIREwFebLDdrnmX7ftaDIK4qrZ\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":23,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":155,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":178}}\n\ndata: [DONE]\n\n"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://inference.do-ai.run/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"openai-gpt-5-nano\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_zPwN7xhG4uXQINkufgtjVJ7L\",\"content\":[{\"type\":\"text\",\"text\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"reasoning_effort\":\"minimal\",\"max_completion_tokens\":1024}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"Paris\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" is\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" sunny\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\".\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null}],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840201,\"id\":\"chatcmpl-EU5dJprTkE7OHEKnmvot7rR0LSo5L\",\"model\":\"openai-gpt-5-nano\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":13,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":193,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":206}}\n\ndata: [DONE]\n\n"
}
}
]
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,54 @@
{
"version": 1,
"metadata": {
"model": "anthropic-claude-haiku-4.5",
"tags": [
"prefix:digitalocean-chat",
"provider:digitalocean",
"protocol:digitalocean-chat",
"cache",
"tool",
"tool-loop"
],
"name": "digitalocean-chat/haiku-continues-a-tool-call-with-cache-markers",
"recordedAt": "2026-10-01T07:36:38.471Z"
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://inference.do-ai.run/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"anthropic-claude-haiku-4.5\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_completion_tokens\":128}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\",\"name\":\"get_weather\"},\"id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"{\\\"city\\\"\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\": \\\"Pari\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"function\":{\"arguments\":\"s\\\"}\"},\"index\":0,\"type\":\"function\"}]},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\",\"index\":0,\"logprobs\":null}],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840197,\"id\":\"chatcmpl-ccddd771-e8a6-4b59-b819-614f79ac4624\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":54,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":593,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":647}}\n\ndata: [DONE]\n\n"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://inference.do-ai.run/v1/chat/completions",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"anthropic-claude-haiku-4.5\",\"messages\":[{\"role\":\"system\",\"content\":[{\"type\":\"text\",\"text\":\"Use the get_weather tool exactly once. After the tool result, reply exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"user\",\"content\":\"What is the weather in Paris?\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"toolu_014LEa2F66CGw2SoLmWBeH5C\",\"content\":[{\"type\":\"text\",\"text\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get current weather for a city.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"strict\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"stream_options\":{\"include_usage\":true},\"store\":false,\"max_completion_tokens\":128}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "data: {\"choices\":[{\"delta\":{\"content\":\"Paris\",\"role\":\"assistant\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\" is sunny.\"},\"finish_reason\":null,\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\",\"index\":0,\"logprobs\":null}],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\"}\n\ndata: {\"choices\":[],\"created\":1790840198,\"id\":\"chatcmpl-3914ee28-f256-4a49-994f-49365922a7a6\",\"model\":\"anthropic-claude-haiku-4.5\",\"object\":\"chat.completion.chunk\",\"usage\":{\"cache_created_input_tokens\":0,\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"cache_read_input_tokens\":0,\"completion_tokens\":7,\"completion_tokens_details\":{\"reasoning_tokens\":0},\"prompt_tokens\":668,\"prompt_tokens_details\":{\"cached_tokens\":0},\"speed\":null,\"total_tokens\":675}}\n\ndata: [DONE]\n\n"
}
}
]
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,78 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMRequest } from "../../src/index.js"
import { DigitalOcean } from "../../src/providers/digitalocean.js"
import { LLMClient } from "../../src/route.js"
import {
LARGE_CACHEABLE_SYSTEM,
expectWeatherToolLoop,
goldenWeatherToolLoopRequest,
runWeatherToolLoop,
} from "../recorded-scenarios.js"
import { recordedTests } from "../recorded-test.js"
const recorded = recordedTests({
prefix: "digitalocean-chat",
provider: "digitalocean",
protocol: "digitalocean-chat",
requires: ["DIGITAL_OCEAN_OFFICIAL_API_KEY"],
})
for (const item of [
{ id: "anthropic-claude-haiku-4.5", name: "Haiku", maxTokens: 128, providerOptions: undefined },
{ id: "openai-gpt-5-nano", name: "GPT Nano", maxTokens: 1024, providerOptions: { reasoningEffort: "minimal" } },
] as const) {
const model = DigitalOcean.configure({
apiKey: process.env.DIGITAL_OCEAN_OFFICIAL_API_KEY ?? "fixture",
providerOptions: item.providerOptions,
}).model(item.id)
describe(`DigitalOcean ${item.name} recorded`, () => {
recorded.effect.with(
`${item.name} reuses a cached prompt`,
{ tags: ["cache", "usage"], metadata: { model: item.id } },
() =>
Effect.gen(function* () {
const request = LLM.request({
model,
system: LARGE_CACHEABLE_SYSTEM,
prompt: "Reply exactly: OK",
promptCacheKey: `digitalocean-recorded-${item.id}`,
generation: { maxTokens: item.maxTokens },
})
const first = yield* LLMClient.generate(request)
const second = yield* LLMClient.generate(request)
expect(first.text.trim()).toMatch(/^OK\.?$/)
expect(second.text.trim()).toMatch(/^OK\.?$/)
expect(second.usage.cacheReadInputTokens).toBeGreaterThan(0)
for (const response of [first, second]) {
expect(response.usage.inputTokens).toBeGreaterThan(4096)
expect(response.usage.inputTokens).toBe(
(response.usage.nonCachedInputTokens ?? 0) +
(response.usage.cacheReadInputTokens ?? 0) +
(response.usage.cacheWriteInputTokens ?? 0),
)
}
}),
60_000,
)
recorded.effect.with(
`${item.name} continues a tool call with cache markers`,
{ tags: ["cache", "tool", "tool-loop"], metadata: { model: item.id } },
() =>
Effect.gen(function* () {
const request = goldenWeatherToolLoopRequest({
id: `digitalocean-${item.id}-tool-loop`,
model,
maxTokens: item.maxTokens,
temperature: false,
})
const events = yield* runWeatherToolLoop(LLMRequest.update(request, { cache: "auto" }))
expectWeatherToolLoop(events)
}),
60_000,
)
})
}
@@ -0,0 +1,123 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { CacheHint, LLM } from "../../src/index.js"
import { compileRequest } from "../../src/route/client.js"
import { DigitalOcean } from "../../src/providers/digitalocean.js"
import { it } from "../lib/effect.js"
import { fixedResponse } from "../lib/http.js"
import { sseEvents } from "../lib/sse.js"
import { LLMClient } from "../../src/route.js"
describe("DigitalOcean", () => {
test("preserves package entrypoint headers and body overrides", () => {
const model = DigitalOcean.model("anthropic-claude-fable-5.1", {
apiKey: "test-key",
headers: { "X-Test": "fixture" },
body: { temperature: 0 },
})
expect(model.route.defaults?.http).toMatchObject({
headers: { "X-Test": "fixture" },
body: { temperature: 0 },
})
})
it.effect("prepares DigitalOcean models with default endpoint and auth", () =>
Effect.gen(function* () {
const model = DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1")
expect(model).toMatchObject({
id: "anthropic-claude-fable-5.1",
provider: "digitalocean",
route: { id: "digitalocean" },
})
expect(model.route.endpoint.baseURL).toBe("https://inference.do-ai.run/v1")
const prepared = yield* compileRequest(LLM.request({ model, prompt: "Say hello.", cache: "none" }))
expect(prepared.route).toBe("digitalocean")
expect(prepared.body).toMatchObject({
model: "anthropic-claude-fable-5.1",
messages: [{ role: "user", content: "Say hello." }],
stream: true,
})
}),
)
it.effect("lowers the native cache policy to DigitalOcean cache_control markers", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1"),
system: [
{ type: "text", text: "Base agent", cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3_600 }) },
{ type: "text", text: "Project instructions" },
],
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }],
prompt: "Hello",
cache: { tools: true, system: true, messages: { tail: 1 } },
}),
)
expect(prepared.body).toMatchObject({
tools: [{ cache_control: { type: "ephemeral" } }],
messages: [
{
role: "system",
content: [
{ text: "Base agent", cache_control: { type: "ephemeral", ttl: "1h" } },
{ text: "Project instructions", cache_control: { type: "ephemeral" } },
],
},
{
role: "user",
content: [{ text: "Hello", cache_control: { type: "ephemeral" } }],
},
],
})
}),
)
it.effect("parses DigitalOcean cache usage fields into AI.Usage", () =>
Effect.gen(function* () {
const model = DigitalOcean.configure({ apiKey: "test-key" }).model("anthropic-claude-fable-5.1")
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Say OK" })).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{
id: "chatcmpl-1",
object: "chat.completion.chunk",
created: 1,
model: "anthropic-claude-fable-5.1",
choices: [{ index: 0, delta: { content: "OK" }, finish_reason: null }],
},
{
id: "chatcmpl-1",
object: "chat.completion.chunk",
created: 1,
model: "anthropic-claude-fable-5.1",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
usage: {
prompt_tokens: 4491,
completion_tokens: 8,
total_tokens: 4499,
cache_read_input_tokens: 4483,
cache_created_input_tokens: 0,
},
},
"[DONE]",
),
),
),
)
expect(response.usage).toMatchObject({
inputTokens: 4491,
outputTokens: 8,
nonCachedInputTokens: 8,
cacheReadInputTokens: 4483,
totalTokens: 4499,
})
}),
)
})
+1
View File
@@ -93,6 +93,7 @@ const HOSTS: Readonly<Record<string, Readonly<Record<string, string>>>> = {
},
"cloudflare-workers-ai": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/cloudflare-workers-ai" },
deepseek: { "@ai-sdk/openai-compatible": "@opencode/ai/providers/deepseek" },
digitalocean: { "@ai-sdk/openai-compatible": "@opencode/ai/providers/digitalocean" },
"fireworks-ai": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/fireworks" },
"google-vertex": { "@ai-sdk/openai-compatible": "@opencode/ai/providers/google-vertex/chat" },
"kimi-for-coding": protocols("moonshot"),
+1
View File
@@ -69,6 +69,7 @@ const builtins = new Map<string, () => Promise<unknown>>([
["@opencode/ai/providers/cloudflare-workers-ai", () => import("@opencode/ai/providers/cloudflare-workers-ai")],
["@opencode/ai/providers/deepinfra", () => import("@opencode/ai/providers/deepinfra")],
["@opencode/ai/providers/deepseek", () => import("@opencode/ai/providers/deepseek")],
["@opencode/ai/providers/digitalocean", () => import("@opencode/ai/providers/digitalocean")],
["@opencode/ai/providers/fireworks", () => import("@opencode/ai/providers/fireworks")],
["@opencode/ai/providers/google", () => import("@opencode/ai/providers/google")],
["@opencode/ai/providers/google-vertex", () => import("@opencode/ai/providers/google-vertex")],
+1
View File
@@ -574,6 +574,7 @@ const PROTOCOLS: Readonly<Record<string, Protocol>> = {
"@opencode/ai/providers/cloudflare-workers-ai": workersAIChat,
"@opencode/ai/providers/deepinfra": deepinfraChat,
"@opencode/ai/providers/deepseek": deepseekChat,
"@opencode/ai/providers/digitalocean": openaiChat,
"@opencode/ai/providers/fireworks": openaiChat,
"@opencode/ai/providers/groq": openaiChat,
"@opencode/ai/providers/meta/chat": openaiChat,
+11
View File
@@ -296,6 +296,17 @@ test("spells Workers AI thinking controls through the chat template", () => {
})
test("spells Chat Completions variants for hosting providers", () => {
expect(
resolve(model("@opencode/ai/providers/digitalocean", "openai-gpt-5-nano", undefined, "digitalocean"), [
{ type: "effort", values: ["minimal", "low", "medium", "high"] },
]),
).toEqual([
{ id: "minimal", settings: { reasoningEffort: "minimal" } },
{ id: "low", settings: { reasoningEffort: "low" } },
{ id: "medium", settings: { reasoningEffort: "medium" } },
{ id: "high", settings: { reasoningEffort: "high" } },
])
expect(
resolve(model("@opencode/ai/providers/openai-compatible", "deepseek-ai/deepseek-v4-pro", undefined, "nvidia"), [
{ type: "effort", values: ["none", "high", "max"] },