feat(ai): scope HTTP middleware to any call, for every modality (#64152)

This commit is contained in:
Aiden Cline
2026-11-15 21:16:23 -06:00
committed by GitHub
Unverified
parent 0fad04b3d5
commit 367d24b70a
15 changed files with 462 additions and 62 deletions
+34
View File
@@ -23,6 +23,40 @@
]
},
"overrides": [
{
"files": ["packages/ai/src/**"],
"rules": {
"no-restricted-globals": [
"error",
{
"name": "Reflect",
"message": "Use typed property access or direct invocation. Suppress this rule only for genuine reflection."
},
{
"name": "fetch",
"message": "Send HTTP through RequestExecutor, which applies executor-wide and scoped middleware (HTTP hooks). A request sent any other way skips them."
}
],
"no-restricted-imports": [
"error",
{
"paths": [
{
"name": "effect/http",
"importNames": ["HttpClient", "FetchHttpClient"],
"message": "Send HTTP through RequestExecutor, which applies executor-wide and scoped middleware (HTTP hooks). Only route/executor.ts holds the HTTP client."
}
]
}
]
}
},
{
"files": ["packages/ai/src/route/executor.ts"],
"rules": {
"no-restricted-imports": "off"
}
},
{
"files": [
"packages/app/**",
+2 -1
View File
@@ -7,7 +7,8 @@
- **One canonical model.** The Schema classes in `src/schema/` are the data model. Do not add a parallel shape or a second construction path.
- **Independent of Session.** Auth, permissions, plugins, and orchestration belong in Core. `ai.stream`/`ai.generate` run exactly one model call.
- **Reuse protocols.** Providers compose existing protocols with `Route.make(...)` instead of cloning them.
- **One API, two runtimes.** `ai` from `@opencode/ai` and `ai` from `@opencode/ai/effect` have the same namespaces and operations, with the same names, overloads, input and options (plus `signal` on the Promise side); keep `test/promise-parity.types.ts` passing.
- **One API, two runtimes.** `ai` from `@opencode/ai` and `ai` from `@opencode/ai/effect` have the same namespaces and operations, with the same names, overloads, input and options (plus `RunOptions`, `signal` and `http`, on the Promise side); keep `test/promise-parity.types.ts` passing.
- **One way to hook HTTP.** Every request goes through `RequestExecutor`, which applies executor-wide middleware (`RequestExecutor.middleware`) and the caller's scoped middleware (`RequestExecutor.withMiddleware`, Promise `{ http }`). Don't add a per-method hook option or send HTTP any other way. A `url` asset downloads with its own `middleware`, which a route sets only for URLs on the provider's origin. Types already require every route to send through the executor it is given, and lint rejects `fetch` and the `HttpClient` import outside `route/executor.ts`. Keep `test/http-hooks.test.ts` passing; it lists every Promise method.
## Code style
+23
View File
@@ -249,6 +249,29 @@ const logged = RequestExecutor.middleware((request, next) =>
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
```
To hook only some calls, such as adding a per-user header, scope middleware to them. It runs on every provider request
they make, whatever the modality: submits, polls, results and cancels. Executor-wide middleware wraps it.
```ts
const signed: HttpMiddleware = (request, next) => next(HttpClientRequest.setHeader(request, "x-user", "user-1"))
// Effect: an Effect or a Stream
const video =
yield *
Effect.gen(function* () {
const job = yield* ai.video.start({ model, prompt: "A red balloon floats up" })
return yield* ai.video.result(model, job.id)
}).pipe(RequestExecutor.withMiddleware(signed))
// Promise: `http` on any call's options; a handle it returns keeps it
await AI.ai.video.start({ model, prompt: "A red balloon floats up" }, { http: signed })
```
Downloading a `url` asset never uses the caller's scoped middleware, so hooks that add credentials can't reach another
host such as a CDN or a presigned storage URL. A result URL on the provider's own origin carries the middleware of the
call that returned it (`asset.middleware`), so it downloads the same way later. Set `middleware` yourself with
`Media.url(url, { middleware })`.
## Venice AI
`Venice` provides native Chat Completions with streaming tools and reasoning. `model` and `chat`
+45 -26
View File
@@ -13,7 +13,7 @@ import {
type StreamOptions,
type TriggerCompactOptions,
} from "./route/client.js"
import { RequestExecutor } from "./route/executor.js"
import { RequestExecutor, type HttpMiddleware } from "./route/executor.js"
import {
AIError,
InvalidRequestError,
@@ -56,8 +56,16 @@ export interface Options {
export interface RunOptions {
readonly signal?: AbortSignal
/**
* Runs on every provider request the call makes, like `RequestExecutor.withMiddleware`: submits, polls, results and
* cancels included. A handle the call returns keeps it for its own calls.
*/
readonly http?: HttpMiddleware
}
/** Options for reading or writing an asset. Downloads use the asset's own `middleware`, never a caller's `http`. */
export type AssetRunOptions = Omit<RunOptions, "http">
export type Services = AIClient.Services
/**
@@ -97,7 +105,7 @@ export const make = (options: Options = {}) => {
/** Run any package Effect (for example `ai.compact(...)` from `@opencode/ai/effect`) inside this runtime. */
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
runtime.runPromise(Effect.raceFirst(effect, abortEffect(options?.signal)))
runtime.runPromise(Effect.raceFirst(scoped(effect, options?.http), abortEffect(options?.signal)))
const iterate = <A, E>(stream: Stream.Stream<A, E, Services>, options?: RunOptions): AsyncIterable<A> =>
Stream.toAsyncIterable(
@@ -105,7 +113,10 @@ export const make = (options: Options = {}) => {
runtime.contextEffect.pipe(
Effect.map(
(context): Stream.Stream<A, unknown> =>
stream.pipe(Stream.interruptWhen(abortEffect(options?.signal)), Stream.provideContext(context)),
scoped(stream, options?.http).pipe(
Stream.interruptWhen(abortEffect(options?.signal)),
Stream.provideContext(context),
),
),
),
),
@@ -119,8 +130,9 @@ export const make = (options: Options = {}) => {
input: LLMRequestInput<Model>,
options?: StreamOptions & RunOptions,
): Promise<LLMResponse>
// `run` and `iterate` apply `http` around the whole call, so the LLM call gets none of its own.
function generate(input: LLMRequestInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
return run(LLM.generate(input, options), options)
return run(LLM.generate(input, { ...options, http: undefined }), options)
}
function stream<const Model extends LanguageModel, S extends OutputSchema>(
@@ -132,7 +144,7 @@ export const make = (options: Options = {}) => {
options?: StreamOptions & RunOptions,
): AsyncIterable<LLMEvent>
function stream(input: LLMRequestInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
return iterate(LLM.stream(input, options), options)
return iterate(LLM.stream(input, { ...options, http: undefined }), options)
}
function loop<const Model extends LanguageModel, S extends OutputSchema>(
@@ -144,7 +156,7 @@ export const make = (options: Options = {}) => {
options?: StreamOptions & RunOptions,
): AsyncIterable<LLMEvent>
function loop(input: LLMLoopInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
return iterate(LLM.loop(input, options), options)
return iterate(LLM.loop(input, { ...options, http: undefined }), options)
}
/** Read `events` from `stream` or `loop` to the end and fold them into the final result. */
@@ -165,31 +177,33 @@ export const make = (options: Options = {}) => {
options?: (TriggerCompactOptions | EndpointCompactOptions) & RunOptions,
) {
return options?.mechanism === "trigger"
? run(LLM.compact(request as CheckpointRequest, options), options)
: run(LLM.compact(request as CompactionRequest, options), options)
? run(LLM.compact(request as CheckpointRequest, { ...options, http: undefined }), options)
: run(LLM.compact(request as CompactionRequest, { ...options, http: undefined }), options)
}
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
...generation.snapshot,
snapshot: generation.snapshot,
terminal: generation.terminal,
token: generation.token,
await: (options) => run(generation.await(options), options),
events: (options) => iterate(generation.events(options), options),
result: (options) => run(generation.result(), options),
refresh: (options) => run(generation.refresh(), options).then(handle),
cancel: (options) => run(generation.cancel(), options),
})
const handle =
(http: HttpMiddleware | undefined) =>
<Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
...generation.snapshot,
snapshot: generation.snapshot,
terminal: generation.terminal,
token: generation.token,
await: (options) => run(generation.await(options), { http, ...options }),
events: (options) => iterate(generation.events(options), { http, ...options }),
result: (options) => run(generation.result(), { http, ...options }),
refresh: (options) => run(generation.refresh(), { http, ...options }).then(handle(options?.http ?? http)),
cancel: (options) => run(generation.cancel(), { http, ...options }),
})
return {
run,
/** Decoded asset bytes, downloading `url` sources through the executor. */
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
bytes: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.bytes(), options),
base64: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.base64(), options),
/** Pull a `url` asset into owned bytes before the provider URL expires. */
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
materialize: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.materialize(), options),
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
file: async (path: string, options?: Media.AssetOptions & AssetRunOptions) => {
const { readFile } = await import("node:fs/promises")
return run(
Effect.tryPromise({
@@ -205,7 +219,7 @@ export const make = (options: Options = {}) => {
)
},
/** Write an asset's bytes like `Media.write`: streamed through a temporary file that replaces `path` on success. */
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
write: async (asset: Media.Asset, path: string, options?: AssetRunOptions) => {
const { open, rename, rm } = await import("node:fs/promises")
const temporary = `${path}.${crypto.randomUUID()}.tmp`
try {
@@ -299,9 +313,9 @@ export const make = (options: Options = {}) => {
start: <const Model extends TranscriptionModel>(
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
options?: RunOptions,
) => run(Transcription.start(input), options).then(handle),
) => run(Transcription.start(input), options).then(handle(options?.http)),
resume: (model: TranscriptionModel, token: unknown, options?: RunOptions) =>
run(Transcription.resume(model, token), options).then(handle),
run(Transcription.resume(model, token), options).then(handle(options?.http)),
},
},
dispose: () => runtime.dispose(),
@@ -310,6 +324,11 @@ export const make = (options: Options = {}) => {
export type Client = ReturnType<typeof make>
const scoped = <Self extends Effect.Effect<unknown, unknown, unknown> | Stream.Stream<unknown, unknown, unknown>>(
self: Self,
http: HttpMiddleware | undefined,
) => (http === undefined ? self : RequestExecutor.withMiddleware(http)(self))
const fileError = (message: string, cause: unknown) =>
new AIError({ reason: new InvalidRequestError({ message, cause }) })
+20 -9
View File
@@ -6,7 +6,7 @@ import { HttpClientRequest } from "effect/http"
import { ProviderID } from "./schema/ids.js"
import { AIError, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
import { ProviderMetadata } from "./schema/options.js"
import { responseHttp, Service } from "./route/executor-service.js"
import { CurrentMiddleware, type HttpMiddleware, responseHttp, Service } from "./route/executor-service.js"
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
export { detectMediaType } from "./utils/media-type.js"
@@ -109,6 +109,8 @@ export class Asset {
readonly providerMetadata?: ProviderMetadata
/** Transient download credentials for `url` sources; see `Asset.Input.headers`. */
readonly headers?: Record<string, string>
/** Middleware for downloading a `url` source; see `Asset.Input.middleware`. */
readonly middleware?: HttpMiddleware
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
@@ -127,6 +129,7 @@ export class Asset {
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
this.providerMetadata = input.providerMetadata
this.headers = input.source.type === "url" ? input.headers : undefined
this.middleware = input.source.type === "url" ? input.middleware : undefined
}
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
@@ -151,7 +154,7 @@ export class Asset {
? Effect.fromResult(Base64.decode(source.data)).pipe(
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
)
: download(source, this.headers)
: download(source, this.headers, this.middleware)
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
})
}
@@ -160,7 +163,7 @@ export class Asset {
stream(): Stream.Stream<Uint8Array, AIError, Service> {
const source = this.source
if (source.type !== "url" || this.#bytes !== undefined) return Stream.fromEffect(this.bytes())
return Stream.unwrap(downloadStream(source, this.headers))
return Stream.unwrap(downloadStream(source, this.headers, this.middleware))
}
base64(): Effect.Effect<string, AIError, Service> {
@@ -211,6 +214,12 @@ export namespace Asset {
* credentials and cannot be downloaded again after a round-trip. Call `materialize()` before persisting.
*/
readonly headers?: Record<string, string>
/**
* Middleware for downloading a `url` source, runtime-only like `headers`. A download never uses the caller's
* scoped middleware (`RequestExecutor.withMiddleware`), so hooks that add provider credentials can't reach another
* host. Routes set this to that middleware only for result URLs on the provider's origin.
*/
readonly middleware?: HttpMiddleware
}
}
@@ -256,11 +265,12 @@ export type ImageAsset = typeof ImageAssetSchema.Type
const download = Effect.fn("Media.download")(function* (
source: Extract<Source, { readonly type: "url" }>,
headers: Record<string, string> | undefined,
middleware: HttpMiddleware | undefined,
) {
const executor = yield* Service
const response = yield* executor.execute(
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
)
const response = yield* executor
.execute(HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})))
.pipe(Effect.provideService(CurrentMiddleware, middleware))
const buffer = yield* response.arrayBuffer.pipe(
Effect.mapError(
(cause) =>
@@ -279,11 +289,12 @@ const download = Effect.fn("Media.download")(function* (
const downloadStream = Effect.fn("Media.downloadStream")(function* (
source: Extract<Source, { readonly type: "url" }>,
headers: Record<string, string> | undefined,
middleware: HttpMiddleware | undefined,
) {
const executor = yield* Service
const response = yield* executor.execute(
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
)
const response = yield* executor
.execute(HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})))
.pipe(Effect.provideService(CurrentMiddleware, middleware))
return response.stream.pipe(
Stream.mapError(
(cause) =>
@@ -169,12 +169,12 @@ const decodeResult = Effect.fn("OpenRouterVideo.decodeResult")(function* (
if (urls.length === 0) return yield* output.invalid(`${route.name} completed without a video URL`)
// The content URLs are OpenRouter's own and only download with the API key.
const videos = yield* Effect.forEach(urls, (url) =>
context.headers(url).pipe(
Effect.map((headers) =>
context.credentials(url).pipe(
Effect.map((credentials) =>
Media.url(url, {
mediaType: "video/mp4",
expiresAt: job.expires_at === undefined || job.expires_at === null ? undefined : job.expires_at * 1000,
headers,
...credentials,
}),
),
),
+1
View File
@@ -199,3 +199,4 @@ export type {
} from "./tool.js"
export type { LLM } from "./llm.js"
export type { LLMLoopInput, LLMRequestInput } from "./llm.js"
export type { HttpHandler, HttpMiddleware } from "./route/executor-service.js"
+41 -1
View File
@@ -1,4 +1,4 @@
import { Context, type Effect } from "effect"
import { Context, Effect, Stream } from "effect"
import type { Headers, HttpClientRequest, HttpClientResponse } from "effect/http"
import { HttpContext, type AIError } from "../schema/errors.js"
import type { ClientTls } from "../schema/options.js"
@@ -34,6 +34,46 @@ export const CurrentTls = Context.Reference<ClientTls | undefined>("@opencode/AI
})
export type Service = RequestExecutorService
/**
* Middleware for every provider request made while it is set: model calls, media submits, polls, results and cancels,
* whatever the modality. Set it with {@link withMiddleware}. Downloads of a `url` asset don't use it; they use the
* asset's own `middleware`, which a route sets only for URLs on the provider's origin.
*/
export const CurrentMiddleware = Context.Reference<HttpMiddleware | undefined>(
"@opencode/AI/RequestExecutor/CurrentMiddleware",
{ defaultValue: () => undefined },
)
/** `outer` wraps `inner`: it sees the request first and the response last. */
export const composeMiddleware = (
outer: HttpMiddleware | undefined,
inner: HttpMiddleware | undefined,
): HttpMiddleware | undefined => {
if (outer === undefined) return inner
if (inner === undefined) return outer
return (request, handler) => outer(request, (forwarded) => inner(forwarded, handler))
}
/**
* Runs `fn` on every provider request `self` makes, an Effect or a Stream. Inside an outer `withMiddleware`, the outer
* one wraps this one. Executor-wide middleware (`RequestExecutor.middleware`) wraps both.
*/
export const withMiddleware =
(fn: HttpMiddleware) =>
<Self extends Effect.Effect<unknown, unknown, unknown> | Stream.Stream<unknown, unknown, unknown>>(
self: Self,
): Self => {
const merged = Effect.gen(function* () {
return composeMiddleware(yield* CurrentMiddleware, fn)
})
// One signature for both, so it infers inside `pipe`; each branch returns the same kind it was given.
return (
Stream.isStream(self)
? Stream.provideServiceEffect(self, CurrentMiddleware, merged)
: Effect.provideServiceEffect(self, CurrentMiddleware, merged)
) as Self
}
export const headerDetails = (headers: Headers.Headers) =>
Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, value]))
+29 -7
View File
@@ -10,6 +10,8 @@ import {
import { HttpContext, HttpRateLimitDetails, AIError, TransportError } from "../schema/index.js"
import { classifyProviderFailure, providerErrorMessage } from "../provider-error.js"
import {
composeMiddleware,
CurrentMiddleware,
CurrentTls,
headerDetails,
responseHttp,
@@ -19,7 +21,14 @@ import {
} from "./executor-service.js"
import type { ClientTls } from "../schema/options.js"
export { CurrentTls, responseHttp, Service } from "./executor-service.js"
export {
composeMiddleware,
CurrentMiddleware,
CurrentTls,
responseHttp,
Service,
withMiddleware,
} from "./executor-service.js"
export type { HttpHandler, HttpMiddleware, Interface, RequestExecutorService } from "./executor-service.js"
const normalizedHeaders = (headers: Headers.Headers) =>
@@ -254,7 +263,13 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
return yield* statusError(response)
})
return Service.of({
execute: executeOnce,
// Scoped middleware runs outside per-call middleware. It is cleared while the request runs, so a request a
// middleware makes itself isn't wrapped again.
execute: (request, next) =>
Effect.gen(function* () {
const middleware = composeMiddleware(yield* CurrentMiddleware, next)
return yield* executeOnce(request, middleware).pipe(Effect.provideService(CurrentMiddleware, undefined))
}),
})
}),
)
@@ -285,18 +300,25 @@ export const withFetch = (
return middleware === undefined ? send : (request) => middleware(request, send)
}
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
/**
* Run `fn` on every request: it sees the raw response before status classification, inside middleware already on
* `executor`, and outside per-call and scoped ({@link withMiddleware}) middleware.
*/
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
Layer.effect(
Service,
Effect.gen(function* () {
const inner = yield* Service
return Service.of({
// Scoped middleware sits where per-call middleware does, inside `fn`; it is read here and cleared so the
// executors below don't apply it again.
execute: (request, next) =>
inner.execute(
request,
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
),
Effect.gen(function* () {
const scoped = composeMiddleware(yield* CurrentMiddleware, next)
return yield* inner
.execute(request, composeMiddleware(fn, scoped))
.pipe(Effect.provideService(CurrentMiddleware, undefined))
}),
})
}),
).pipe(Layer.provide(executor))
+4 -3
View File
@@ -93,10 +93,11 @@ export interface Started<Token> {
export interface PollContext<Token> {
readonly token: Token
/**
* The headers an authenticated GET of `url` would send (deployment headers plus auth), for result URLs that need
* the API key to download. URLs on another origin than the route's base URL get no headers.
* What an authenticated download of `url` needs, for result URLs that only download with the API key: the headers a
* GET would send (deployment headers plus auth) and the caller's scoped middleware. Spread it into `Media.url`. URLs
* on another origin than the route's base URL get neither, so credentials and hooks never leave the provider's host.
*/
readonly headers: (url: string) => Effect.Effect<Record<string, string>, AIError>
readonly credentials: (url: string) => Effect.Effect<Pick<Media.AssetOptions, "headers" | "middleware">, AIError>
/**
* A further GET with the same auth and headers, retried like the poll itself, for providers whose result needs a
* second lookup (MiniMax v1 answers with a file id that resolves to a download URL).
+13 -4
View File
@@ -205,7 +205,16 @@ export const queued = <Request extends MediaRequest, Response, Token>(
}) =>
get(operation.path(token)).pipe(
Effect.flatMap((response) =>
operation.decode(response, { token, get, headers: (url) => transport.headers(url, exchange) }),
operation.decode(response, {
token,
get,
credentials: (url) =>
Effect.gen(function* () {
const headers = yield* transport.headers(url, exchange)
if (headers === undefined) return {}
return { headers, middleware: yield* RequestExecutor.CurrentMiddleware }
}),
}),
),
)
const status = poll(protocol.status)
@@ -399,7 +408,7 @@ const makeTransport = <Request extends MediaRequest>(
}),
/**
* The headers a follow-up GET of `url` sends, for result URLs that only download with the same credentials. A URL
* on another origin than the route's base URL gets none, so credentials never leave the provider's host.
* on another origin than the route's base URL gets undefined, so credentials never leave the provider's host.
*/
headers: (url: string, exchange: Exchange) =>
URL.canParse(url) && new URL(url).origin === baseURL("").origin
@@ -409,8 +418,8 @@ const makeTransport = <Request extends MediaRequest>(
url,
body: "",
headers: Headers.fromInput(exchange.headers),
}).pipe(Effect.map((headers): Record<string, string> => ({ ...headers })))
: Effect.succeed({}),
}).pipe(Effect.map((headers): Record<string, string> | undefined => ({ ...headers })))
: Effect.succeed(undefined),
/** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */
call: (method: AuthInput["method"], path: string, exchange: Exchange, execute: Execute) => {
// Provider-issued absolute URLs are used as-is; everything else resolves against the base.
+71
View File
@@ -305,6 +305,77 @@ describe("RequestExecutor", () => {
)
})
it.effect("runs scoped middleware between shared and per-call middleware, outer scopes first", () => {
const calls: Array<string> = []
const record =
(name: string): HttpMiddleware =>
(input, next) =>
Effect.sync(() => calls.push(name)).pipe(Effect.andThen(next(input)))
const base = RequestExecutor.layer.pipe(
Layer.provide(
handlerLayer((input) => Effect.sync(() => calls.push("handler")).pipe(Effect.as(input.respond("ok")))),
),
)
return Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
yield* executor
.execute(request, record("per-call"))
.pipe(RequestExecutor.withMiddleware(record("inner")), RequestExecutor.withMiddleware(record("outer")))
expect(calls).toEqual(["shared", "outer", "inner", "per-call", "handler"])
calls.length = 0
// Outside the scope, nothing scoped runs.
yield* executor.execute(request)
expect(calls).toEqual(["shared", "handler"])
}).pipe(Effect.provide(RequestExecutor.middleware(record("shared"), base)))
})
it.effect("applies scoped middleware to a stream and to requests sent through a per-request fetch", () => {
const seen: Array<string> = []
const scoped: HttpMiddleware = (input, next) =>
Effect.sync(() => seen.push(input.url)).pipe(Effect.andThen(next(input)))
const fetch: typeof globalThis.fetch = Object.assign(async () => new Response("from fetch"), {
preconnect: globalThis.fetch.preconnect,
})
return Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const body = yield* RequestExecutor.stream(executor, request).pipe(
Stream.decodeText(),
Stream.mkString,
RequestExecutor.withMiddleware(scoped),
)
expect(body).toBe("ok")
const fetched = yield* executor
.execute(sseRequest, RequestExecutor.withFetch(fetch))
.pipe(RequestExecutor.withMiddleware(scoped))
expect(yield* fetched.text).toBe("from fetch")
expect(seen).toEqual([request.url, sseRequest.url])
}).pipe(
Effect.provide(
RequestExecutor.layer.pipe(Layer.provide(handlerLayer((input) => Effect.succeed(input.respond("ok"))))),
),
)
})
it.effect("doesn't wrap a request that scoped middleware sends itself", () => {
let wrapped = 0
return Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
const scoped: HttpMiddleware = (input, next) =>
Effect.gen(function* () {
wrapped++
// Such as a token refresh made from inside an auth hook.
yield* executor.execute(sseRequest).pipe(Effect.orDie)
return yield* next(input)
})
yield* executor.execute(request).pipe(RequestExecutor.withMiddleware(scoped))
expect(wrapped).toBe(1)
}).pipe(
Effect.provide(
RequestExecutor.layer.pipe(Layer.provide(handlerLayer((input) => Effect.succeed(input.respond("ok"))))),
),
)
})
it.effect("classifies context overflow responses", () =>
Effect.gen(function* () {
const executor = yield* RequestExecutor.Service
+137
View File
@@ -0,0 +1,137 @@
import { describe, expect, test } from "bun:test"
import { Effect, Layer } from "effect"
import { HttpClientResponse } from "effect/http"
import { AI, AIError, Media } from "../src/index.js"
import { RequestExecutor, type HttpMiddleware } from "../src/route.js"
import { AssemblyAI, OpenAI, XAI } from "../src/providers.js"
import { handlerLayer } from "./lib/http.js"
// Every Promise method that talks to a provider must send its requests through the caller's `http` middleware. The
// Promise client runs the Effect API under `RequestExecutor.withMiddleware`, so this covers the Effect side too.
// Adding a method to the client fails "lists every method" until it is listed here: as a provider call, with a way to
// call it, or as a local helper.
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
const video = XAI.configure({ apiKey: "test", baseURL: "https://xai.test/v1" }).video("grok-imagine-video-1.5")
const transcription = AssemblyAI.configure({ apiKey: "test", baseURL: "https://assemblyai.test" }).transcription(
"universal-2",
)
const audio = Media.bytes(Uint8Array.from([0x49, 0x44, 0x33, 1, 2, 3]), "audio/mpeg")
const image = openai.image("gpt-image-2")
const chat = { model: openai.chat("gpt-4o-mini"), prompt: "Hello" }
type Client = ReturnType<typeof AI.make>
type Call = (ai: Client, http: HttpMiddleware) => Promise<unknown> | AsyncIterable<unknown>
const providerCalls: Record<string, Call> = {
generate: (ai, http) => ai.generate(chat, { http }),
stream: (ai, http) => ai.stream(chat, { http }),
loop: (ai, http) => ai.loop(chat, { http }),
compact: (ai, http) => ai.compact(ai.request({ model: openai.responses("gpt-5"), prompt: "Hello" }), { http }),
"image.generate": (ai, http) => ai.image.generate({ model: image, prompt: "A lighthouse" }, { http }),
"image.stream": (ai, http) => ai.image.stream({ model: image, prompt: "A lighthouse" }, { http }),
"image.start": (ai, http) => ai.image.start({ model: image, prompt: "A lighthouse" }, { http }),
"image.status": (ai, http) => ai.image.status(image, "job", { http }),
"image.result": (ai, http) => ai.image.result(image, "job", { http }),
"image.cancel": (ai, http) => ai.image.cancel(image, "job", { http }),
"decision.generate": (ai, http) =>
ai.decision.generate(
{
model: openai.decision("gpt-6-luna"),
input: "Is this a greeting? hello",
questions: [{ name: "greeting", type: "choice", instructions: "Greeting?", choices: [{ value: true }] }],
},
{ http },
),
"embedding.generate": (ai, http) =>
ai.embedding.generate({ model: openai.embedding("text-embedding-3-small"), values: ["hello"] }, { http }),
"video.start": (ai, http) => ai.video.start({ model: video, prompt: "A balloon" }, { http }),
"video.status": (ai, http) => ai.video.status(video, "job", { http }),
"video.result": (ai, http) => ai.video.result(video, "job", { http }),
"video.cancel": (ai, http) => ai.video.cancel(video, "job", { http }),
"experimental.speech.generate": (ai, http) =>
ai.experimental.speech.generate({ model: openai.speech("gpt-4o-mini-tts"), text: "Hello" }, { http }),
"experimental.speech.stream": (ai, http) =>
ai.experimental.speech.stream({ model: openai.speech("gpt-4o-mini-tts"), text: "Hello" }, { http }),
"experimental.transcription.generate": (ai, http) =>
ai.experimental.transcription.generate({ model: transcription, audio }, { http }),
"experimental.transcription.stream": (ai, http) =>
ai.experimental.transcription.stream({ model: transcription, audio }, { http }),
"experimental.transcription.start": (ai, http) =>
ai.experimental.transcription.start({ model: transcription, audio }, { http }),
"experimental.transcription.resume": (ai, http) =>
ai.experimental.transcription.resume(transcription, { transcriptID: "tr_1" }, { http }),
}
// No provider request of their own: request builders, local folds and asset helpers. Downloads use the asset's own
// `middleware` instead (see `Media.Asset`).
const local = [
"run",
"bytes",
"base64",
"materialize",
"file",
"write",
"request",
"collect",
"canCompact",
"image.request",
"decision.request",
"embedding.request",
"video.request",
"experimental.speech.request",
"experimental.transcription.request",
"dispose",
]
const methods = (value: object, prefix = ""): Array<string> =>
Object.entries(value).flatMap(([key, entry]) => {
if (typeof entry === "function") return [`${prefix}${key}`]
if (typeof entry === "object" && entry !== null) return methods(entry, `${prefix}${key}.`)
return []
})
const settle = async (result: Promise<unknown> | AsyncIterable<unknown>) => {
if (Symbol.asyncIterator in result) {
for await (const _ of result) break
return
}
await result
}
describe("http middleware", () => {
test("lists every method", () => {
const ai = AI.make()
expect(methods(ai).toSorted()).toEqual([...Object.keys(providerCalls), ...local].toSorted())
})
for (const [name, call] of Object.entries(providerCalls))
test(`${name} sends its requests through the caller's http`, async () => {
const hooked: Array<string> = []
const unhooked: Array<string> = []
const ai = AI.make({
layer: RequestExecutor.layer.pipe(
Layer.provide(
handlerLayer((input) =>
Effect.sync(() => unhooked.push(input.request.url)).pipe(Effect.andThen(Effect.die("unhooked request"))),
),
),
),
})
// Answers 400 so the call stops at once: a transport failure would be retried.
const http: HttpMiddleware = (request) =>
Effect.sync(() => {
hooked.push(request.url)
return HttpClientResponse.fromWeb(request, new Response("{}", { status: 400 }))
})
const error = await settle(call(ai, http)).then(
() => undefined,
(error: unknown) => error,
)
await ai.dispose()
expect(unhooked).toEqual([])
// A route that can't do the operation (no queued image route ships, xAI can't cancel) fails before any request.
if (error instanceof AIError && error.reason._tag === "UnsupportedOperation") return
expect(hooked.length).toBeGreaterThan(0)
})
})
+33 -2
View File
@@ -1,7 +1,8 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { Effect, Layer, Stream } from "effect"
import { Media, VideoClient, ai } from "../src/effect.js"
import { OpenRouter } from "../src/providers.js"
import { RequestExecutor, type HttpMiddleware } from "../src/route.js"
import { it } from "./lib/effect.js"
import { dynamicResponse, json, observe, type Call, type HandlerInput } from "./lib/http.js"
@@ -202,10 +203,40 @@ describe("ai.video.status / result / cancel / OpenRouter", () => {
}),
)
it.effect("keeps the caller's scoped middleware for a download on the provider's host, and drops it on another", () =>
Effect.gen(function* () {
const hooked: Array<string> = []
const hook: HttpMiddleware = (input, next) =>
Effect.sync(() => hooked.push(input.url)).pipe(Effect.andThen(next(input)))
const cdn = "https://cdn.example.test/video.mp4"
const respond = (input: HandlerInput) => {
if (input.request.url.endsWith("/content?index=0") || input.request.url === cdn)
return input.respond(Uint8Array.from([1]), { headers: { "content-type": "video/mp4" } })
return json(input, input.request.url.endsWith("other") ? { ...completed, unsigned_urls: [cdn] } : completed)
}
yield* Effect.gen(function* () {
// Read inside the scope; downloaded after it ends, as a caller storing the result later would.
const own = yield* ai.video.result(model, id).pipe(RequestExecutor.withMiddleware(hook))
yield* own.video.stream().pipe(Stream.runDrain)
// Downloaded inside the scope, but the URL isn't the provider's.
yield* Effect.gen(function* () {
const other = yield* ai.video.result(model, "other")
yield* other.video.stream().pipe(Stream.runDrain)
}).pipe(RequestExecutor.withMiddleware(hook))
}).pipe(Effect.provide(layer((input) => Effect.succeed(respond(input)))))
expect(hooked).toEqual([
"https://openrouter.test/api/v1/videos/gen-vid-1-abc",
"https://openrouter.test/api/v1/videos/gen-vid-1-abc/content?index=0",
"https://openrouter.test/api/v1/videos/other",
])
}),
)
it.effect("attaches no credentials to a result URL on another host", () =>
Effect.gen(function* () {
const response = yield* ai.video.result(model, id)
expect(response.video.headers).toEqual({})
expect(response.video.headers).toBeUndefined()
expect(response.video.middleware).toBeUndefined()
}).pipe(
Effect.provide(
layer((input) =>
+6 -6
View File
@@ -1,10 +1,10 @@
import type { Effect, Stream } from "effect"
import type { Client, GenerationHandle } from "../src/index.js"
import type { Client, GenerationHandle, RunOptions } from "../src/index.js"
import type { Generation, ai } from "../src/effect.js"
// The Promise `ai` mirrors the Effect `ai`: same namespaces, methods, arguments and results. The only allowed differences
// are `signal` on the options bag, Promise for Effect, AsyncIterable for Stream (in results and arguments) and
// GenerationHandle for Generation.
// are `RunOptions` (`signal`, `http`) on the options bag, Promise for Effect, AsyncIterable for Stream (in results and
// arguments) and GenerationHandle for Generation.
type Equal<A, B> = [A, B] extends [B, A] ? true : false
type Assert<T extends Record<string, true>> = T
@@ -27,17 +27,17 @@ type Signatures<F> = F extends {
: never
type Positional<P> = P extends readonly [infer A, ...infer Rest]
? [Iterated<WithoutSignal<A>>, ...Positional<Rest>]
? [Iterated<WithoutRunOptions<A>>, ...Positional<Rest>]
: []
type Iterated<T> = T extends Stream.Stream<infer A, infer _E, infer _R> ? AsyncIterable<A> : T
// Paired with its keys: an options bag of optional fields is otherwise assignable to `{}` and back.
type Options<P> = P extends readonly [unknown, ...infer Rest]
? Options<Rest>
: P extends readonly [(infer O)?]
? Keyed<WithoutSignal<NonNullable<O>>>
? Keyed<WithoutRunOptions<NonNullable<O>>>
: Keyed<{}>
type Keyed<T> = [keyof T, T]
type WithoutSignal<T> = T extends { readonly signal?: AbortSignal } ? Omit<T, "signal"> : T
type WithoutRunOptions<T> = T extends object ? Omit<T, keyof RunOptions> : T
type Promised<R> =
R extends Effect.Effect<Generation<infer A>, infer _E, infer _R>