diff --git a/packages/ai/src/decision-client.ts b/packages/ai/src/decision-client.ts index 188038f5f83..f8f6edee28b 100644 --- a/packages/ai/src/decision-client.ts +++ b/packages/ai/src/decision-client.ts @@ -33,7 +33,10 @@ export const layer: Layer.Layer = Layer return Service.of({ generate: Effect.fnUntraced(function* (request) { const route = request.model.route - const response = yield* route.generate(request, executor.execute) + const response = yield* route.generate( + request, + RequestExecutor.withTls(executor.execute, request.model.http?.tls), + ) if (answersMatch(request.questions, response.answers)) return response as DecisionResponseFor return yield* new AIError({ diff --git a/packages/ai/src/media-client.ts b/packages/ai/src/media-client.ts index 622bfa7fdba..d5745413b0d 100644 --- a/packages/ai/src/media-client.ts +++ b/packages/ai/src/media-client.ts @@ -31,6 +31,7 @@ export const make = , Even service, Effect.gen(function* () { const executor = yield* RequestExecutor.Service + const execute = (model: Req["model"]) => RequestExecutor.withTls(executor.execute, model.http?.tls) const notQueued = (route: MediaRoute.AnyRoute, operation: string) => new AIError({ reason: new UnsupportedOperationError({ @@ -43,27 +44,29 @@ export const make = , Even const start = (request: Req) => { const route = request.model.route if (route.kind !== "queued") return Effect.fail(notQueued(route, "start")) - return route.start(request, executor.execute) + return route.start(request, execute(request.model)) } return service.of({ start, resume: (model, token) => { if (model.route.kind !== "queued") return Effect.fail(notQueued(model.route, "resume")) - return model.route.resume(model, token, executor.execute) + return model.route.resume(model, token, execute(model)) }, generate: (request, options) => { const route = request.model.route - if (route.kind !== "queued") return route.generate(request, executor.execute) + if (route.kind !== "queued") return route.generate(request, execute(request.model)) return start(request).pipe(Effect.flatMap((generation) => generation.await(options))) }, stream: (request, options) => { const route = request.model.route - if (route.kind === "stream") return route.stream(request, executor.execute) + if (route.kind === "stream") return route.stream(request, execute(request.model)) if (route.kind === "queued") return Stream.unwrap( start(request).pipe(Effect.map((generation) => resultEvents(generation, input.responseEvents, options))), ) - return Stream.fromIterableEffect(Effect.map(route.generate(request, executor.execute), input.responseEvents)) + return Stream.fromIterableEffect( + Effect.map(route.generate(request, execute(request.model)), input.responseEvents), + ) }, }) }), diff --git a/packages/ai/src/route/client.ts b/packages/ai/src/route/client.ts index 9781477d8f4..34c87af5528 100644 --- a/packages/ai/src/route/client.ts +++ b/packages/ai/src/route/client.ts @@ -608,7 +608,8 @@ const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest, o Stream.unwrap( Effect.gen(function* () { const compiled = yield* compile(request, options) - return compiled.route.streamPrepared(compiled.prepared, compiled.request, runtime, options) + const http = { execute: RequestExecutor.withTls(runtime.http.execute, compiled.request.http?.tls) } + return compiled.route.streamPrepared(compiled.prepared, compiled.request, { http }, options) }), ) @@ -670,10 +671,12 @@ export const layer: Layer.Layer = Layer ): Effect.Effect function compact(request: LLMRequest, options?: EndpointCompactOptions | TriggerCompactOptions) { return Effect.suspend((): Effect.Effect => { + const prepared = prepareRequest(request) + const http = { execute: RequestExecutor.withTls(executor.execute, prepared.http?.tls) } if (options?.mechanism === "trigger" && canCompact(request, options)) - return request.model.route.compact.trigger(prepareRequest(request), executor, options) + return request.model.route.compact.trigger(prepared, http, options) if ((options?.mechanism === undefined || options.mechanism === "endpoint") && canCompact(request)) - return request.model.route.compact.endpoint(prepareRequest(request), executor, options) + return request.model.route.compact.endpoint(prepared, http, options) return unsupportedCompaction(request.model, options?.mechanism) }) } diff --git a/packages/ai/src/route/executor.ts b/packages/ai/src/route/executor.ts index c32299a7eb6..cfd75cd49e9 100644 --- a/packages/ai/src/route/executor.ts +++ b/packages/ai/src/route/executor.ts @@ -9,7 +9,15 @@ import { } from "effect/http" import { HttpContext, HttpRateLimitDetails, AIError, TransportError } from "../schema/index.js" import { classifyProviderFailure, providerErrorMessage } from "../provider-error.js" -import { headerDetails, responseHttp, Service, type HttpMiddleware, type Interface } from "./executor-service.js" +import { + CurrentTls, + headerDetails, + responseHttp, + Service, + type HttpMiddleware, + type Interface, +} from "./executor-service.js" +import type { ClientTls } from "../schema/options.js" export { CurrentTls, responseHttp, Service } from "./executor-service.js" export type { HttpHandler, HttpMiddleware, Interface, RequestExecutorService } from "./executor-service.js" @@ -253,6 +261,10 @@ export const layer: Layer.Layer = Layer.e export const fetchLayer = layer.pipe(Layer.provide(FetchHttpClient.layer)) +/** Sends through `execute`, presenting `tls` on each request. The executor's HTTP client reads it from `CurrentTls`. */ +export const withTls = (execute: Interface["execute"], tls: ClientTls | undefined): Interface["execute"] => + tls ? (request, middleware) => execute(request, middleware).pipe(Effect.provideService(CurrentTls, tls)) : execute + /** * Per-call middleware that sends through `fetch` instead of the executor's HTTP client. Executor middleware and * `middleware` still wrap the call, and status classification still applies. diff --git a/packages/ai/src/route/transport/http.ts b/packages/ai/src/route/transport/http.ts index 03129617def..7924bbf02b3 100644 --- a/packages/ai/src/route/transport/http.ts +++ b/packages/ai/src/route/transport/http.ts @@ -110,7 +110,6 @@ export const httpJson = (input: HttpJsonInput): HttpJs // Unlike the header and chunk limits, the whole-request budget has no default. const total = limits.total ? Duration.millis(limits.total) : Duration.infinity const response = yield* runtime.http.execute(prepared.request, prepared.middleware).pipe( - Effect.provideService(RequestExecutor.CurrentTls, request.http?.tls), Effect.timeoutOrElse({ duration: Duration.min(timeoutDuration(limits.headers), total), orElse: () => timeout("request", "Timed out waiting for response headers"), diff --git a/packages/core/src/model-resolver.ts b/packages/core/src/model-resolver.ts index 861772959be..7eeba2e32cc 100644 --- a/packages/core/src/model-resolver.ts +++ b/packages/core/src/model-resolver.ts @@ -313,7 +313,8 @@ const fromCatalogMedia = Effect.fn("ModelResolver.fromCatalogMedia")(function* < credential?: Credential.Value, dependencies?: Dependencies, ) { - const { resolved, module, settings } = yield* loadCatalogPackage(model, credential, dependencies) + const { resolved, configured, module, settings } = yield* loadCatalogPackage(model, credential, dependencies) + const tls = Provider.tls(configured) const runtime: unknown = yield* Effect.try({ try: () => module.model(resolved.modelID ?? resolved.id, settings), catch: (cause) => constructionError(resolved, cause), @@ -328,7 +329,12 @@ const fromCatalogMedia = Effect.fn("ModelResolver.fromCatalogMedia")(function* < }) : runtime if (!is(adapted)) return yield* unsupported(resolved) - return adapted + if (!tls || !(adapted instanceof DecisionModel || adapted instanceof ImageModel)) return adapted + // Media and decision models are plain classes with readonly fields, so copy onto the same prototype. + const copy: Runtime = Object.assign(Object.create(Object.getPrototypeOf(adapted)), adapted, { + http: { ...adapted.http, tls }, + }) + return copy }) const constructionError = (model: RuntimeInfo, cause: unknown) => @@ -502,7 +508,8 @@ export const layer = Layer.effect( limit: selected.limit, compaction: runtimeInfo.settings?.compaction, cache: runtimeInfo.settings?.cache, - transport: provider?.settings?.transport, + // The WebSocket transport can't present a client certificate, so `tls` keeps the model on HTTP. + transport: Provider.tls(provider?.settings ?? {}) ? "http" : provider?.settings?.transport, chunkTimeout: Provider.timeout(provider?.settings?.chunkTimeout), } }) diff --git a/services/site/src/docs/content/network.mdx b/services/site/src/docs/content/network.mdx index 34e866e1a08..a1756a470c9 100644 --- a/services/site/src/docs/content/network.mdx +++ b/services/site/src/docs/content/network.mdx @@ -119,7 +119,9 @@ TLS handshake only sends it to servers that ask for one. - `passphrase` is only needed for an encrypted key. - Requests with a client certificate don't follow redirects, so the certificate isn't presented to another host. -Only model requests present the certificate. Tools such as webfetch, remote MCP servers and plugin installs don't. +Every model request of that provider presents the certificate: chat, compaction, image and decision models. A provider +with `tls` uses HTTP instead of WebSockets. Tools such as webfetch, remote MCP servers and plugin installs don't present +it. ## Foreground