mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 21:15:57 +03:00
feat(ai): scope HTTP middleware to any call, for every modality (#64152)
This commit is contained in:
committed by
GitHub
Unverified
parent
0fad04b3d5
commit
367d24b70a
@@ -23,6 +23,40 @@
|
||||
]
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"files": ["packages/ai/src/**"],
|
||||
"rules": {
|
||||
"no-restricted-globals": [
|
||||
"error",
|
||||
{
|
||||
"name": "Reflect",
|
||||
"message": "Use typed property access or direct invocation. Suppress this rule only for genuine reflection."
|
||||
},
|
||||
{
|
||||
"name": "fetch",
|
||||
"message": "Send HTTP through RequestExecutor, which applies executor-wide and scoped middleware (HTTP hooks). A request sent any other way skips them."
|
||||
}
|
||||
],
|
||||
"no-restricted-imports": [
|
||||
"error",
|
||||
{
|
||||
"paths": [
|
||||
{
|
||||
"name": "effect/http",
|
||||
"importNames": ["HttpClient", "FetchHttpClient"],
|
||||
"message": "Send HTTP through RequestExecutor, which applies executor-wide and scoped middleware (HTTP hooks). Only route/executor.ts holds the HTTP client."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["packages/ai/src/route/executor.ts"],
|
||||
"rules": {
|
||||
"no-restricted-imports": "off"
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": [
|
||||
"packages/app/**",
|
||||
|
||||
@@ -7,7 +7,8 @@
|
||||
- **One canonical model.** The Schema classes in `src/schema/` are the data model. Do not add a parallel shape or a second construction path.
|
||||
- **Independent of Session.** Auth, permissions, plugins, and orchestration belong in Core. `ai.stream`/`ai.generate` run exactly one model call.
|
||||
- **Reuse protocols.** Providers compose existing protocols with `Route.make(...)` instead of cloning them.
|
||||
- **One API, two runtimes.** `ai` from `@opencode/ai` and `ai` from `@opencode/ai/effect` have the same namespaces and operations, with the same names, overloads, input and options (plus `signal` on the Promise side); keep `test/promise-parity.types.ts` passing.
|
||||
- **One API, two runtimes.** `ai` from `@opencode/ai` and `ai` from `@opencode/ai/effect` have the same namespaces and operations, with the same names, overloads, input and options (plus `RunOptions`, `signal` and `http`, on the Promise side); keep `test/promise-parity.types.ts` passing.
|
||||
- **One way to hook HTTP.** Every request goes through `RequestExecutor`, which applies executor-wide middleware (`RequestExecutor.middleware`) and the caller's scoped middleware (`RequestExecutor.withMiddleware`, Promise `{ http }`). Don't add a per-method hook option or send HTTP any other way. A `url` asset downloads with its own `middleware`, which a route sets only for URLs on the provider's origin. Types already require every route to send through the executor it is given, and lint rejects `fetch` and the `HttpClient` import outside `route/executor.ts`. Keep `test/http-hooks.test.ts` passing; it lists every Promise method.
|
||||
|
||||
## Code style
|
||||
|
||||
|
||||
@@ -249,6 +249,29 @@ const logged = RequestExecutor.middleware((request, next) =>
|
||||
const everything = AIClient.layerWith(logged) // or AI.make({ layer: logged })
|
||||
```
|
||||
|
||||
To hook only some calls, such as adding a per-user header, scope middleware to them. It runs on every provider request
|
||||
they make, whatever the modality: submits, polls, results and cancels. Executor-wide middleware wraps it.
|
||||
|
||||
```ts
|
||||
const signed: HttpMiddleware = (request, next) => next(HttpClientRequest.setHeader(request, "x-user", "user-1"))
|
||||
|
||||
// Effect: an Effect or a Stream
|
||||
const video =
|
||||
yield *
|
||||
Effect.gen(function* () {
|
||||
const job = yield* ai.video.start({ model, prompt: "A red balloon floats up" })
|
||||
return yield* ai.video.result(model, job.id)
|
||||
}).pipe(RequestExecutor.withMiddleware(signed))
|
||||
|
||||
// Promise: `http` on any call's options; a handle it returns keeps it
|
||||
await AI.ai.video.start({ model, prompt: "A red balloon floats up" }, { http: signed })
|
||||
```
|
||||
|
||||
Downloading a `url` asset never uses the caller's scoped middleware, so hooks that add credentials can't reach another
|
||||
host such as a CDN or a presigned storage URL. A result URL on the provider's own origin carries the middleware of the
|
||||
call that returned it (`asset.middleware`), so it downloads the same way later. Set `middleware` yourself with
|
||||
`Media.url(url, { middleware })`.
|
||||
|
||||
## Venice AI
|
||||
|
||||
`Venice` provides native Chat Completions with streaming tools and reasoning. `model` and `chat`
|
||||
|
||||
+45
-26
@@ -13,7 +13,7 @@ import {
|
||||
type StreamOptions,
|
||||
type TriggerCompactOptions,
|
||||
} from "./route/client.js"
|
||||
import { RequestExecutor } from "./route/executor.js"
|
||||
import { RequestExecutor, type HttpMiddleware } from "./route/executor.js"
|
||||
import {
|
||||
AIError,
|
||||
InvalidRequestError,
|
||||
@@ -56,8 +56,16 @@ export interface Options {
|
||||
|
||||
export interface RunOptions {
|
||||
readonly signal?: AbortSignal
|
||||
/**
|
||||
* Runs on every provider request the call makes, like `RequestExecutor.withMiddleware`: submits, polls, results and
|
||||
* cancels included. A handle the call returns keeps it for its own calls.
|
||||
*/
|
||||
readonly http?: HttpMiddleware
|
||||
}
|
||||
|
||||
/** Options for reading or writing an asset. Downloads use the asset's own `middleware`, never a caller's `http`. */
|
||||
export type AssetRunOptions = Omit<RunOptions, "http">
|
||||
|
||||
export type Services = AIClient.Services
|
||||
|
||||
/**
|
||||
@@ -97,7 +105,7 @@ export const make = (options: Options = {}) => {
|
||||
|
||||
/** Run any package Effect (for example `ai.compact(...)` from `@opencode/ai/effect`) inside this runtime. */
|
||||
const run = <A, E>(effect: Effect.Effect<A, E, Services>, options?: RunOptions) =>
|
||||
runtime.runPromise(Effect.raceFirst(effect, abortEffect(options?.signal)))
|
||||
runtime.runPromise(Effect.raceFirst(scoped(effect, options?.http), abortEffect(options?.signal)))
|
||||
|
||||
const iterate = <A, E>(stream: Stream.Stream<A, E, Services>, options?: RunOptions): AsyncIterable<A> =>
|
||||
Stream.toAsyncIterable(
|
||||
@@ -105,7 +113,10 @@ export const make = (options: Options = {}) => {
|
||||
runtime.contextEffect.pipe(
|
||||
Effect.map(
|
||||
(context): Stream.Stream<A, unknown> =>
|
||||
stream.pipe(Stream.interruptWhen(abortEffect(options?.signal)), Stream.provideContext(context)),
|
||||
scoped(stream, options?.http).pipe(
|
||||
Stream.interruptWhen(abortEffect(options?.signal)),
|
||||
Stream.provideContext(context),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -119,8 +130,9 @@ export const make = (options: Options = {}) => {
|
||||
input: LLMRequestInput<Model>,
|
||||
options?: StreamOptions & RunOptions,
|
||||
): Promise<LLMResponse>
|
||||
// `run` and `iterate` apply `http` around the whole call, so the LLM call gets none of its own.
|
||||
function generate(input: LLMRequestInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
|
||||
return run(LLM.generate(input, options), options)
|
||||
return run(LLM.generate(input, { ...options, http: undefined }), options)
|
||||
}
|
||||
|
||||
function stream<const Model extends LanguageModel, S extends OutputSchema>(
|
||||
@@ -132,7 +144,7 @@ export const make = (options: Options = {}) => {
|
||||
options?: StreamOptions & RunOptions,
|
||||
): AsyncIterable<LLMEvent>
|
||||
function stream(input: LLMRequestInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
|
||||
return iterate(LLM.stream(input, options), options)
|
||||
return iterate(LLM.stream(input, { ...options, http: undefined }), options)
|
||||
}
|
||||
|
||||
function loop<const Model extends LanguageModel, S extends OutputSchema>(
|
||||
@@ -144,7 +156,7 @@ export const make = (options: Options = {}) => {
|
||||
options?: StreamOptions & RunOptions,
|
||||
): AsyncIterable<LLMEvent>
|
||||
function loop(input: LLMLoopInput & { readonly schema?: OutputSchema }, options?: StreamOptions & RunOptions) {
|
||||
return iterate(LLM.loop(input, options), options)
|
||||
return iterate(LLM.loop(input, { ...options, http: undefined }), options)
|
||||
}
|
||||
|
||||
/** Read `events` from `stream` or `loop` to the end and fold them into the final result. */
|
||||
@@ -165,31 +177,33 @@ export const make = (options: Options = {}) => {
|
||||
options?: (TriggerCompactOptions | EndpointCompactOptions) & RunOptions,
|
||||
) {
|
||||
return options?.mechanism === "trigger"
|
||||
? run(LLM.compact(request as CheckpointRequest, options), options)
|
||||
: run(LLM.compact(request as CompactionRequest, options), options)
|
||||
? run(LLM.compact(request as CheckpointRequest, { ...options, http: undefined }), options)
|
||||
: run(LLM.compact(request as CompactionRequest, { ...options, http: undefined }), options)
|
||||
}
|
||||
|
||||
const handle = <Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
|
||||
...generation.snapshot,
|
||||
snapshot: generation.snapshot,
|
||||
terminal: generation.terminal,
|
||||
token: generation.token,
|
||||
await: (options) => run(generation.await(options), options),
|
||||
events: (options) => iterate(generation.events(options), options),
|
||||
result: (options) => run(generation.result(), options),
|
||||
refresh: (options) => run(generation.refresh(), options).then(handle),
|
||||
cancel: (options) => run(generation.cancel(), options),
|
||||
})
|
||||
const handle =
|
||||
(http: HttpMiddleware | undefined) =>
|
||||
<Response>(generation: Generation<Response>): GenerationHandle<Response> => ({
|
||||
...generation.snapshot,
|
||||
snapshot: generation.snapshot,
|
||||
terminal: generation.terminal,
|
||||
token: generation.token,
|
||||
await: (options) => run(generation.await(options), { http, ...options }),
|
||||
events: (options) => iterate(generation.events(options), { http, ...options }),
|
||||
result: (options) => run(generation.result(), { http, ...options }),
|
||||
refresh: (options) => run(generation.refresh(), { http, ...options }).then(handle(options?.http ?? http)),
|
||||
cancel: (options) => run(generation.cancel(), { http, ...options }),
|
||||
})
|
||||
|
||||
return {
|
||||
run,
|
||||
/** Decoded asset bytes, downloading `url` sources through the executor. */
|
||||
bytes: (asset: Media.Asset, options?: RunOptions) => run(asset.bytes(), options),
|
||||
base64: (asset: Media.Asset, options?: RunOptions) => run(asset.base64(), options),
|
||||
bytes: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.bytes(), options),
|
||||
base64: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.base64(), options),
|
||||
/** Pull a `url` asset into owned bytes before the provider URL expires. */
|
||||
materialize: (asset: Media.Asset, options?: RunOptions) => run(asset.materialize(), options),
|
||||
materialize: (asset: Media.Asset, options?: AssetRunOptions) => run(asset.materialize(), options),
|
||||
/** Read a file into an asset like `Media.file`: sniffed media type, then the extension's. */
|
||||
file: async (path: string, options?: Media.AssetOptions & RunOptions) => {
|
||||
file: async (path: string, options?: Media.AssetOptions & AssetRunOptions) => {
|
||||
const { readFile } = await import("node:fs/promises")
|
||||
return run(
|
||||
Effect.tryPromise({
|
||||
@@ -205,7 +219,7 @@ export const make = (options: Options = {}) => {
|
||||
)
|
||||
},
|
||||
/** Write an asset's bytes like `Media.write`: streamed through a temporary file that replaces `path` on success. */
|
||||
write: async (asset: Media.Asset, path: string, options?: RunOptions) => {
|
||||
write: async (asset: Media.Asset, path: string, options?: AssetRunOptions) => {
|
||||
const { open, rename, rm } = await import("node:fs/promises")
|
||||
const temporary = `${path}.${crypto.randomUUID()}.tmp`
|
||||
try {
|
||||
@@ -299,9 +313,9 @@ export const make = (options: Options = {}) => {
|
||||
start: <const Model extends TranscriptionModel>(
|
||||
input: TranscriptionRequestInput<Model> | TranscriptionRequest,
|
||||
options?: RunOptions,
|
||||
) => run(Transcription.start(input), options).then(handle),
|
||||
) => run(Transcription.start(input), options).then(handle(options?.http)),
|
||||
resume: (model: TranscriptionModel, token: unknown, options?: RunOptions) =>
|
||||
run(Transcription.resume(model, token), options).then(handle),
|
||||
run(Transcription.resume(model, token), options).then(handle(options?.http)),
|
||||
},
|
||||
},
|
||||
dispose: () => runtime.dispose(),
|
||||
@@ -310,6 +324,11 @@ export const make = (options: Options = {}) => {
|
||||
|
||||
export type Client = ReturnType<typeof make>
|
||||
|
||||
const scoped = <Self extends Effect.Effect<unknown, unknown, unknown> | Stream.Stream<unknown, unknown, unknown>>(
|
||||
self: Self,
|
||||
http: HttpMiddleware | undefined,
|
||||
) => (http === undefined ? self : RequestExecutor.withMiddleware(http)(self))
|
||||
|
||||
const fileError = (message: string, cause: unknown) =>
|
||||
new AIError({ reason: new InvalidRequestError({ message, cause }) })
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import { HttpClientRequest } from "effect/http"
|
||||
import { ProviderID } from "./schema/ids.js"
|
||||
import { AIError, InvalidProviderOutputError, InvalidRequestError } from "./schema/errors.js"
|
||||
import { ProviderMetadata } from "./schema/options.js"
|
||||
import { responseHttp, Service } from "./route/executor-service.js"
|
||||
import { CurrentMiddleware, type HttpMiddleware, responseHttp, Service } from "./route/executor-service.js"
|
||||
import { detectMediaType, fileMediaType } from "./utils/media-type.js"
|
||||
|
||||
export { detectMediaType } from "./utils/media-type.js"
|
||||
@@ -109,6 +109,8 @@ export class Asset {
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
/** Transient download credentials for `url` sources; see `Asset.Input.headers`. */
|
||||
readonly headers?: Record<string, string>
|
||||
/** Middleware for downloading a `url` source; see `Asset.Input.middleware`. */
|
||||
readonly middleware?: HttpMiddleware
|
||||
|
||||
// Derived payload forms are cached on the instance because every protocol lowering re-reads the same payload. The
|
||||
// cache is check-then-set (concurrent first reads of a `url` source may both download) and is never observable
|
||||
@@ -127,6 +129,7 @@ export class Asset {
|
||||
this.expiresAt = input.source.type === "url" ? input.source.expiresAt : undefined
|
||||
this.providerMetadata = input.providerMetadata
|
||||
this.headers = input.source.type === "url" ? input.headers : undefined
|
||||
this.middleware = input.source.type === "url" ? input.middleware : undefined
|
||||
}
|
||||
|
||||
/** Inline payload without effects, for protocols that embed base64 or data URLs directly. */
|
||||
@@ -151,7 +154,7 @@ export class Asset {
|
||||
? Effect.fromResult(Base64.decode(source.data)).pipe(
|
||||
Effect.mapError((cause) => invalid(`Media asset contains invalid base64 data`, cause)),
|
||||
)
|
||||
: download(source, this.headers)
|
||||
: download(source, this.headers, this.middleware)
|
||||
return decoded.pipe(Effect.tap((data) => Effect.sync(() => (this.#bytes = data))))
|
||||
})
|
||||
}
|
||||
@@ -160,7 +163,7 @@ export class Asset {
|
||||
stream(): Stream.Stream<Uint8Array, AIError, Service> {
|
||||
const source = this.source
|
||||
if (source.type !== "url" || this.#bytes !== undefined) return Stream.fromEffect(this.bytes())
|
||||
return Stream.unwrap(downloadStream(source, this.headers))
|
||||
return Stream.unwrap(downloadStream(source, this.headers, this.middleware))
|
||||
}
|
||||
|
||||
base64(): Effect.Effect<string, AIError, Service> {
|
||||
@@ -211,6 +214,12 @@ export namespace Asset {
|
||||
* credentials and cannot be downloaded again after a round-trip. Call `materialize()` before persisting.
|
||||
*/
|
||||
readonly headers?: Record<string, string>
|
||||
/**
|
||||
* Middleware for downloading a `url` source, runtime-only like `headers`. A download never uses the caller's
|
||||
* scoped middleware (`RequestExecutor.withMiddleware`), so hooks that add provider credentials can't reach another
|
||||
* host. Routes set this to that middleware only for result URLs on the provider's origin.
|
||||
*/
|
||||
readonly middleware?: HttpMiddleware
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,11 +265,12 @@ export type ImageAsset = typeof ImageAssetSchema.Type
|
||||
const download = Effect.fn("Media.download")(function* (
|
||||
source: Extract<Source, { readonly type: "url" }>,
|
||||
headers: Record<string, string> | undefined,
|
||||
middleware: HttpMiddleware | undefined,
|
||||
) {
|
||||
const executor = yield* Service
|
||||
const response = yield* executor.execute(
|
||||
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
|
||||
)
|
||||
const response = yield* executor
|
||||
.execute(HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})))
|
||||
.pipe(Effect.provideService(CurrentMiddleware, middleware))
|
||||
const buffer = yield* response.arrayBuffer.pipe(
|
||||
Effect.mapError(
|
||||
(cause) =>
|
||||
@@ -279,11 +289,12 @@ const download = Effect.fn("Media.download")(function* (
|
||||
const downloadStream = Effect.fn("Media.downloadStream")(function* (
|
||||
source: Extract<Source, { readonly type: "url" }>,
|
||||
headers: Record<string, string> | undefined,
|
||||
middleware: HttpMiddleware | undefined,
|
||||
) {
|
||||
const executor = yield* Service
|
||||
const response = yield* executor.execute(
|
||||
HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})),
|
||||
)
|
||||
const response = yield* executor
|
||||
.execute(HttpClientRequest.get(source.url).pipe(HttpClientRequest.setHeaders(headers ?? {})))
|
||||
.pipe(Effect.provideService(CurrentMiddleware, middleware))
|
||||
return response.stream.pipe(
|
||||
Stream.mapError(
|
||||
(cause) =>
|
||||
|
||||
@@ -169,12 +169,12 @@ const decodeResult = Effect.fn("OpenRouterVideo.decodeResult")(function* (
|
||||
if (urls.length === 0) return yield* output.invalid(`${route.name} completed without a video URL`)
|
||||
// The content URLs are OpenRouter's own and only download with the API key.
|
||||
const videos = yield* Effect.forEach(urls, (url) =>
|
||||
context.headers(url).pipe(
|
||||
Effect.map((headers) =>
|
||||
context.credentials(url).pipe(
|
||||
Effect.map((credentials) =>
|
||||
Media.url(url, {
|
||||
mediaType: "video/mp4",
|
||||
expiresAt: job.expires_at === undefined || job.expires_at === null ? undefined : job.expires_at * 1000,
|
||||
headers,
|
||||
...credentials,
|
||||
}),
|
||||
),
|
||||
),
|
||||
|
||||
@@ -199,3 +199,4 @@ export type {
|
||||
} from "./tool.js"
|
||||
export type { LLM } from "./llm.js"
|
||||
export type { LLMLoopInput, LLMRequestInput } from "./llm.js"
|
||||
export type { HttpHandler, HttpMiddleware } from "./route/executor-service.js"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Context, type Effect } from "effect"
|
||||
import { Context, Effect, Stream } from "effect"
|
||||
import type { Headers, HttpClientRequest, HttpClientResponse } from "effect/http"
|
||||
import { HttpContext, type AIError } from "../schema/errors.js"
|
||||
import type { ClientTls } from "../schema/options.js"
|
||||
@@ -34,6 +34,46 @@ export const CurrentTls = Context.Reference<ClientTls | undefined>("@opencode/AI
|
||||
})
|
||||
export type Service = RequestExecutorService
|
||||
|
||||
/**
|
||||
* Middleware for every provider request made while it is set: model calls, media submits, polls, results and cancels,
|
||||
* whatever the modality. Set it with {@link withMiddleware}. Downloads of a `url` asset don't use it; they use the
|
||||
* asset's own `middleware`, which a route sets only for URLs on the provider's origin.
|
||||
*/
|
||||
export const CurrentMiddleware = Context.Reference<HttpMiddleware | undefined>(
|
||||
"@opencode/AI/RequestExecutor/CurrentMiddleware",
|
||||
{ defaultValue: () => undefined },
|
||||
)
|
||||
|
||||
/** `outer` wraps `inner`: it sees the request first and the response last. */
|
||||
export const composeMiddleware = (
|
||||
outer: HttpMiddleware | undefined,
|
||||
inner: HttpMiddleware | undefined,
|
||||
): HttpMiddleware | undefined => {
|
||||
if (outer === undefined) return inner
|
||||
if (inner === undefined) return outer
|
||||
return (request, handler) => outer(request, (forwarded) => inner(forwarded, handler))
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `fn` on every provider request `self` makes, an Effect or a Stream. Inside an outer `withMiddleware`, the outer
|
||||
* one wraps this one. Executor-wide middleware (`RequestExecutor.middleware`) wraps both.
|
||||
*/
|
||||
export const withMiddleware =
|
||||
(fn: HttpMiddleware) =>
|
||||
<Self extends Effect.Effect<unknown, unknown, unknown> | Stream.Stream<unknown, unknown, unknown>>(
|
||||
self: Self,
|
||||
): Self => {
|
||||
const merged = Effect.gen(function* () {
|
||||
return composeMiddleware(yield* CurrentMiddleware, fn)
|
||||
})
|
||||
// One signature for both, so it infers inside `pipe`; each branch returns the same kind it was given.
|
||||
return (
|
||||
Stream.isStream(self)
|
||||
? Stream.provideServiceEffect(self, CurrentMiddleware, merged)
|
||||
: Effect.provideServiceEffect(self, CurrentMiddleware, merged)
|
||||
) as Self
|
||||
}
|
||||
|
||||
export const headerDetails = (headers: Headers.Headers) =>
|
||||
Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, value]))
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ import {
|
||||
import { HttpContext, HttpRateLimitDetails, AIError, TransportError } from "../schema/index.js"
|
||||
import { classifyProviderFailure, providerErrorMessage } from "../provider-error.js"
|
||||
import {
|
||||
composeMiddleware,
|
||||
CurrentMiddleware,
|
||||
CurrentTls,
|
||||
headerDetails,
|
||||
responseHttp,
|
||||
@@ -19,7 +21,14 @@ import {
|
||||
} from "./executor-service.js"
|
||||
import type { ClientTls } from "../schema/options.js"
|
||||
|
||||
export { CurrentTls, responseHttp, Service } from "./executor-service.js"
|
||||
export {
|
||||
composeMiddleware,
|
||||
CurrentMiddleware,
|
||||
CurrentTls,
|
||||
responseHttp,
|
||||
Service,
|
||||
withMiddleware,
|
||||
} from "./executor-service.js"
|
||||
export type { HttpHandler, HttpMiddleware, Interface, RequestExecutorService } from "./executor-service.js"
|
||||
|
||||
const normalizedHeaders = (headers: Headers.Headers) =>
|
||||
@@ -254,7 +263,13 @@ export const layer: Layer.Layer<Service, never, HttpClient.HttpClient> = Layer.e
|
||||
return yield* statusError(response)
|
||||
})
|
||||
return Service.of({
|
||||
execute: executeOnce,
|
||||
// Scoped middleware runs outside per-call middleware. It is cleared while the request runs, so a request a
|
||||
// middleware makes itself isn't wrapped again.
|
||||
execute: (request, next) =>
|
||||
Effect.gen(function* () {
|
||||
const middleware = composeMiddleware(yield* CurrentMiddleware, next)
|
||||
return yield* executeOnce(request, middleware).pipe(Effect.provideService(CurrentMiddleware, undefined))
|
||||
}),
|
||||
})
|
||||
}),
|
||||
)
|
||||
@@ -285,18 +300,25 @@ export const withFetch = (
|
||||
return middleware === undefined ? send : (request) => middleware(request, send)
|
||||
}
|
||||
|
||||
/** Run `fn` on every request: it sees the raw response before status classification, inside middleware already on `executor`, and outside per-call middleware. */
|
||||
/**
|
||||
* Run `fn` on every request: it sees the raw response before status classification, inside middleware already on
|
||||
* `executor`, and outside per-call and scoped ({@link withMiddleware}) middleware.
|
||||
*/
|
||||
export const middleware = (fn: HttpMiddleware, executor: Layer.Layer<Service> = fetchLayer): Layer.Layer<Service> =>
|
||||
Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const inner = yield* Service
|
||||
return Service.of({
|
||||
// Scoped middleware sits where per-call middleware does, inside `fn`; it is read here and cleared so the
|
||||
// executors below don't apply it again.
|
||||
execute: (request, next) =>
|
||||
inner.execute(
|
||||
request,
|
||||
next === undefined ? fn : (input, handler) => fn(input, (forwarded) => next(forwarded, handler)),
|
||||
),
|
||||
Effect.gen(function* () {
|
||||
const scoped = composeMiddleware(yield* CurrentMiddleware, next)
|
||||
return yield* inner
|
||||
.execute(request, composeMiddleware(fn, scoped))
|
||||
.pipe(Effect.provideService(CurrentMiddleware, undefined))
|
||||
}),
|
||||
})
|
||||
}),
|
||||
).pipe(Layer.provide(executor))
|
||||
|
||||
@@ -93,10 +93,11 @@ export interface Started<Token> {
|
||||
export interface PollContext<Token> {
|
||||
readonly token: Token
|
||||
/**
|
||||
* The headers an authenticated GET of `url` would send (deployment headers plus auth), for result URLs that need
|
||||
* the API key to download. URLs on another origin than the route's base URL get no headers.
|
||||
* What an authenticated download of `url` needs, for result URLs that only download with the API key: the headers a
|
||||
* GET would send (deployment headers plus auth) and the caller's scoped middleware. Spread it into `Media.url`. URLs
|
||||
* on another origin than the route's base URL get neither, so credentials and hooks never leave the provider's host.
|
||||
*/
|
||||
readonly headers: (url: string) => Effect.Effect<Record<string, string>, AIError>
|
||||
readonly credentials: (url: string) => Effect.Effect<Pick<Media.AssetOptions, "headers" | "middleware">, AIError>
|
||||
/**
|
||||
* A further GET with the same auth and headers, retried like the poll itself, for providers whose result needs a
|
||||
* second lookup (MiniMax v1 answers with a file id that resolves to a download URL).
|
||||
|
||||
@@ -205,7 +205,16 @@ export const queued = <Request extends MediaRequest, Response, Token>(
|
||||
}) =>
|
||||
get(operation.path(token)).pipe(
|
||||
Effect.flatMap((response) =>
|
||||
operation.decode(response, { token, get, headers: (url) => transport.headers(url, exchange) }),
|
||||
operation.decode(response, {
|
||||
token,
|
||||
get,
|
||||
credentials: (url) =>
|
||||
Effect.gen(function* () {
|
||||
const headers = yield* transport.headers(url, exchange)
|
||||
if (headers === undefined) return {}
|
||||
return { headers, middleware: yield* RequestExecutor.CurrentMiddleware }
|
||||
}),
|
||||
}),
|
||||
),
|
||||
)
|
||||
const status = poll(protocol.status)
|
||||
@@ -399,7 +408,7 @@ const makeTransport = <Request extends MediaRequest>(
|
||||
}),
|
||||
/**
|
||||
* The headers a follow-up GET of `url` sends, for result URLs that only download with the same credentials. A URL
|
||||
* on another origin than the route's base URL gets none, so credentials never leave the provider's host.
|
||||
* on another origin than the route's base URL gets undefined, so credentials never leave the provider's host.
|
||||
*/
|
||||
headers: (url: string, exchange: Exchange) =>
|
||||
URL.canParse(url) && new URL(url).origin === baseURL("").origin
|
||||
@@ -409,8 +418,8 @@ const makeTransport = <Request extends MediaRequest>(
|
||||
url,
|
||||
body: "",
|
||||
headers: Headers.fromInput(exchange.headers),
|
||||
}).pipe(Effect.map((headers): Record<string, string> => ({ ...headers })))
|
||||
: Effect.succeed({}),
|
||||
}).pipe(Effect.map((headers): Record<string, string> | undefined => ({ ...headers })))
|
||||
: Effect.succeed(undefined),
|
||||
/** Bodiless follow-up call (status, result, cancel) with the same auth and headers as `submit`. */
|
||||
call: (method: AuthInput["method"], path: string, exchange: Exchange, execute: Execute) => {
|
||||
// Provider-issued absolute URLs are used as-is; everything else resolves against the base.
|
||||
|
||||
@@ -305,6 +305,77 @@ describe("RequestExecutor", () => {
|
||||
)
|
||||
})
|
||||
|
||||
it.effect("runs scoped middleware between shared and per-call middleware, outer scopes first", () => {
|
||||
const calls: Array<string> = []
|
||||
const record =
|
||||
(name: string): HttpMiddleware =>
|
||||
(input, next) =>
|
||||
Effect.sync(() => calls.push(name)).pipe(Effect.andThen(next(input)))
|
||||
const base = RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
handlerLayer((input) => Effect.sync(() => calls.push("handler")).pipe(Effect.as(input.respond("ok")))),
|
||||
),
|
||||
)
|
||||
return Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
yield* executor
|
||||
.execute(request, record("per-call"))
|
||||
.pipe(RequestExecutor.withMiddleware(record("inner")), RequestExecutor.withMiddleware(record("outer")))
|
||||
expect(calls).toEqual(["shared", "outer", "inner", "per-call", "handler"])
|
||||
calls.length = 0
|
||||
// Outside the scope, nothing scoped runs.
|
||||
yield* executor.execute(request)
|
||||
expect(calls).toEqual(["shared", "handler"])
|
||||
}).pipe(Effect.provide(RequestExecutor.middleware(record("shared"), base)))
|
||||
})
|
||||
|
||||
it.effect("applies scoped middleware to a stream and to requests sent through a per-request fetch", () => {
|
||||
const seen: Array<string> = []
|
||||
const scoped: HttpMiddleware = (input, next) =>
|
||||
Effect.sync(() => seen.push(input.url)).pipe(Effect.andThen(next(input)))
|
||||
const fetch: typeof globalThis.fetch = Object.assign(async () => new Response("from fetch"), {
|
||||
preconnect: globalThis.fetch.preconnect,
|
||||
})
|
||||
return Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const body = yield* RequestExecutor.stream(executor, request).pipe(
|
||||
Stream.decodeText(),
|
||||
Stream.mkString,
|
||||
RequestExecutor.withMiddleware(scoped),
|
||||
)
|
||||
expect(body).toBe("ok")
|
||||
const fetched = yield* executor
|
||||
.execute(sseRequest, RequestExecutor.withFetch(fetch))
|
||||
.pipe(RequestExecutor.withMiddleware(scoped))
|
||||
expect(yield* fetched.text).toBe("from fetch")
|
||||
expect(seen).toEqual([request.url, sseRequest.url])
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
RequestExecutor.layer.pipe(Layer.provide(handlerLayer((input) => Effect.succeed(input.respond("ok"))))),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
it.effect("doesn't wrap a request that scoped middleware sends itself", () => {
|
||||
let wrapped = 0
|
||||
return Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
const scoped: HttpMiddleware = (input, next) =>
|
||||
Effect.gen(function* () {
|
||||
wrapped++
|
||||
// Such as a token refresh made from inside an auth hook.
|
||||
yield* executor.execute(sseRequest).pipe(Effect.orDie)
|
||||
return yield* next(input)
|
||||
})
|
||||
yield* executor.execute(request).pipe(RequestExecutor.withMiddleware(scoped))
|
||||
expect(wrapped).toBe(1)
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
RequestExecutor.layer.pipe(Layer.provide(handlerLayer((input) => Effect.succeed(input.respond("ok"))))),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
it.effect("classifies context overflow responses", () =>
|
||||
Effect.gen(function* () {
|
||||
const executor = yield* RequestExecutor.Service
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { HttpClientResponse } from "effect/http"
|
||||
import { AI, AIError, Media } from "../src/index.js"
|
||||
import { RequestExecutor, type HttpMiddleware } from "../src/route.js"
|
||||
import { AssemblyAI, OpenAI, XAI } from "../src/providers.js"
|
||||
import { handlerLayer } from "./lib/http.js"
|
||||
|
||||
// Every Promise method that talks to a provider must send its requests through the caller's `http` middleware. The
|
||||
// Promise client runs the Effect API under `RequestExecutor.withMiddleware`, so this covers the Effect side too.
|
||||
// Adding a method to the client fails "lists every method" until it is listed here: as a provider call, with a way to
|
||||
// call it, or as a local helper.
|
||||
|
||||
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
|
||||
const video = XAI.configure({ apiKey: "test", baseURL: "https://xai.test/v1" }).video("grok-imagine-video-1.5")
|
||||
const transcription = AssemblyAI.configure({ apiKey: "test", baseURL: "https://assemblyai.test" }).transcription(
|
||||
"universal-2",
|
||||
)
|
||||
const audio = Media.bytes(Uint8Array.from([0x49, 0x44, 0x33, 1, 2, 3]), "audio/mpeg")
|
||||
const image = openai.image("gpt-image-2")
|
||||
const chat = { model: openai.chat("gpt-4o-mini"), prompt: "Hello" }
|
||||
|
||||
type Client = ReturnType<typeof AI.make>
|
||||
type Call = (ai: Client, http: HttpMiddleware) => Promise<unknown> | AsyncIterable<unknown>
|
||||
|
||||
const providerCalls: Record<string, Call> = {
|
||||
generate: (ai, http) => ai.generate(chat, { http }),
|
||||
stream: (ai, http) => ai.stream(chat, { http }),
|
||||
loop: (ai, http) => ai.loop(chat, { http }),
|
||||
compact: (ai, http) => ai.compact(ai.request({ model: openai.responses("gpt-5"), prompt: "Hello" }), { http }),
|
||||
"image.generate": (ai, http) => ai.image.generate({ model: image, prompt: "A lighthouse" }, { http }),
|
||||
"image.stream": (ai, http) => ai.image.stream({ model: image, prompt: "A lighthouse" }, { http }),
|
||||
"image.start": (ai, http) => ai.image.start({ model: image, prompt: "A lighthouse" }, { http }),
|
||||
"image.status": (ai, http) => ai.image.status(image, "job", { http }),
|
||||
"image.result": (ai, http) => ai.image.result(image, "job", { http }),
|
||||
"image.cancel": (ai, http) => ai.image.cancel(image, "job", { http }),
|
||||
"decision.generate": (ai, http) =>
|
||||
ai.decision.generate(
|
||||
{
|
||||
model: openai.decision("gpt-6-luna"),
|
||||
input: "Is this a greeting? hello",
|
||||
questions: [{ name: "greeting", type: "choice", instructions: "Greeting?", choices: [{ value: true }] }],
|
||||
},
|
||||
{ http },
|
||||
),
|
||||
"embedding.generate": (ai, http) =>
|
||||
ai.embedding.generate({ model: openai.embedding("text-embedding-3-small"), values: ["hello"] }, { http }),
|
||||
"video.start": (ai, http) => ai.video.start({ model: video, prompt: "A balloon" }, { http }),
|
||||
"video.status": (ai, http) => ai.video.status(video, "job", { http }),
|
||||
"video.result": (ai, http) => ai.video.result(video, "job", { http }),
|
||||
"video.cancel": (ai, http) => ai.video.cancel(video, "job", { http }),
|
||||
"experimental.speech.generate": (ai, http) =>
|
||||
ai.experimental.speech.generate({ model: openai.speech("gpt-4o-mini-tts"), text: "Hello" }, { http }),
|
||||
"experimental.speech.stream": (ai, http) =>
|
||||
ai.experimental.speech.stream({ model: openai.speech("gpt-4o-mini-tts"), text: "Hello" }, { http }),
|
||||
"experimental.transcription.generate": (ai, http) =>
|
||||
ai.experimental.transcription.generate({ model: transcription, audio }, { http }),
|
||||
"experimental.transcription.stream": (ai, http) =>
|
||||
ai.experimental.transcription.stream({ model: transcription, audio }, { http }),
|
||||
"experimental.transcription.start": (ai, http) =>
|
||||
ai.experimental.transcription.start({ model: transcription, audio }, { http }),
|
||||
"experimental.transcription.resume": (ai, http) =>
|
||||
ai.experimental.transcription.resume(transcription, { transcriptID: "tr_1" }, { http }),
|
||||
}
|
||||
|
||||
// No provider request of their own: request builders, local folds and asset helpers. Downloads use the asset's own
|
||||
// `middleware` instead (see `Media.Asset`).
|
||||
const local = [
|
||||
"run",
|
||||
"bytes",
|
||||
"base64",
|
||||
"materialize",
|
||||
"file",
|
||||
"write",
|
||||
"request",
|
||||
"collect",
|
||||
"canCompact",
|
||||
"image.request",
|
||||
"decision.request",
|
||||
"embedding.request",
|
||||
"video.request",
|
||||
"experimental.speech.request",
|
||||
"experimental.transcription.request",
|
||||
"dispose",
|
||||
]
|
||||
|
||||
const methods = (value: object, prefix = ""): Array<string> =>
|
||||
Object.entries(value).flatMap(([key, entry]) => {
|
||||
if (typeof entry === "function") return [`${prefix}${key}`]
|
||||
if (typeof entry === "object" && entry !== null) return methods(entry, `${prefix}${key}.`)
|
||||
return []
|
||||
})
|
||||
|
||||
const settle = async (result: Promise<unknown> | AsyncIterable<unknown>) => {
|
||||
if (Symbol.asyncIterator in result) {
|
||||
for await (const _ of result) break
|
||||
return
|
||||
}
|
||||
await result
|
||||
}
|
||||
|
||||
describe("http middleware", () => {
|
||||
test("lists every method", () => {
|
||||
const ai = AI.make()
|
||||
expect(methods(ai).toSorted()).toEqual([...Object.keys(providerCalls), ...local].toSorted())
|
||||
})
|
||||
|
||||
for (const [name, call] of Object.entries(providerCalls))
|
||||
test(`${name} sends its requests through the caller's http`, async () => {
|
||||
const hooked: Array<string> = []
|
||||
const unhooked: Array<string> = []
|
||||
const ai = AI.make({
|
||||
layer: RequestExecutor.layer.pipe(
|
||||
Layer.provide(
|
||||
handlerLayer((input) =>
|
||||
Effect.sync(() => unhooked.push(input.request.url)).pipe(Effect.andThen(Effect.die("unhooked request"))),
|
||||
),
|
||||
),
|
||||
),
|
||||
})
|
||||
// Answers 400 so the call stops at once: a transport failure would be retried.
|
||||
const http: HttpMiddleware = (request) =>
|
||||
Effect.sync(() => {
|
||||
hooked.push(request.url)
|
||||
return HttpClientResponse.fromWeb(request, new Response("{}", { status: 400 }))
|
||||
})
|
||||
const error = await settle(call(ai, http)).then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
)
|
||||
await ai.dispose()
|
||||
expect(unhooked).toEqual([])
|
||||
// A route that can't do the operation (no queued image route ships, xAI can't cancel) fails before any request.
|
||||
if (error instanceof AIError && error.reason._tag === "UnsupportedOperation") return
|
||||
expect(hooked.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
@@ -1,7 +1,8 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { Effect, Layer, Stream } from "effect"
|
||||
import { Media, VideoClient, ai } from "../src/effect.js"
|
||||
import { OpenRouter } from "../src/providers.js"
|
||||
import { RequestExecutor, type HttpMiddleware } from "../src/route.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse, json, observe, type Call, type HandlerInput } from "./lib/http.js"
|
||||
|
||||
@@ -202,10 +203,40 @@ describe("ai.video.status / result / cancel / OpenRouter", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps the caller's scoped middleware for a download on the provider's host, and drops it on another", () =>
|
||||
Effect.gen(function* () {
|
||||
const hooked: Array<string> = []
|
||||
const hook: HttpMiddleware = (input, next) =>
|
||||
Effect.sync(() => hooked.push(input.url)).pipe(Effect.andThen(next(input)))
|
||||
const cdn = "https://cdn.example.test/video.mp4"
|
||||
const respond = (input: HandlerInput) => {
|
||||
if (input.request.url.endsWith("/content?index=0") || input.request.url === cdn)
|
||||
return input.respond(Uint8Array.from([1]), { headers: { "content-type": "video/mp4" } })
|
||||
return json(input, input.request.url.endsWith("other") ? { ...completed, unsigned_urls: [cdn] } : completed)
|
||||
}
|
||||
yield* Effect.gen(function* () {
|
||||
// Read inside the scope; downloaded after it ends, as a caller storing the result later would.
|
||||
const own = yield* ai.video.result(model, id).pipe(RequestExecutor.withMiddleware(hook))
|
||||
yield* own.video.stream().pipe(Stream.runDrain)
|
||||
// Downloaded inside the scope, but the URL isn't the provider's.
|
||||
yield* Effect.gen(function* () {
|
||||
const other = yield* ai.video.result(model, "other")
|
||||
yield* other.video.stream().pipe(Stream.runDrain)
|
||||
}).pipe(RequestExecutor.withMiddleware(hook))
|
||||
}).pipe(Effect.provide(layer((input) => Effect.succeed(respond(input)))))
|
||||
expect(hooked).toEqual([
|
||||
"https://openrouter.test/api/v1/videos/gen-vid-1-abc",
|
||||
"https://openrouter.test/api/v1/videos/gen-vid-1-abc/content?index=0",
|
||||
"https://openrouter.test/api/v1/videos/other",
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("attaches no credentials to a result URL on another host", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* ai.video.result(model, id)
|
||||
expect(response.video.headers).toEqual({})
|
||||
expect(response.video.headers).toBeUndefined()
|
||||
expect(response.video.middleware).toBeUndefined()
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
layer((input) =>
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import type { Effect, Stream } from "effect"
|
||||
import type { Client, GenerationHandle } from "../src/index.js"
|
||||
import type { Client, GenerationHandle, RunOptions } from "../src/index.js"
|
||||
import type { Generation, ai } from "../src/effect.js"
|
||||
|
||||
// The Promise `ai` mirrors the Effect `ai`: same namespaces, methods, arguments and results. The only allowed differences
|
||||
// are `signal` on the options bag, Promise for Effect, AsyncIterable for Stream (in results and arguments) and
|
||||
// GenerationHandle for Generation.
|
||||
// are `RunOptions` (`signal`, `http`) on the options bag, Promise for Effect, AsyncIterable for Stream (in results and
|
||||
// arguments) and GenerationHandle for Generation.
|
||||
|
||||
type Equal<A, B> = [A, B] extends [B, A] ? true : false
|
||||
type Assert<T extends Record<string, true>> = T
|
||||
@@ -27,17 +27,17 @@ type Signatures<F> = F extends {
|
||||
: never
|
||||
|
||||
type Positional<P> = P extends readonly [infer A, ...infer Rest]
|
||||
? [Iterated<WithoutSignal<A>>, ...Positional<Rest>]
|
||||
? [Iterated<WithoutRunOptions<A>>, ...Positional<Rest>]
|
||||
: []
|
||||
type Iterated<T> = T extends Stream.Stream<infer A, infer _E, infer _R> ? AsyncIterable<A> : T
|
||||
// Paired with its keys: an options bag of optional fields is otherwise assignable to `{}` and back.
|
||||
type Options<P> = P extends readonly [unknown, ...infer Rest]
|
||||
? Options<Rest>
|
||||
: P extends readonly [(infer O)?]
|
||||
? Keyed<WithoutSignal<NonNullable<O>>>
|
||||
? Keyed<WithoutRunOptions<NonNullable<O>>>
|
||||
: Keyed<{}>
|
||||
type Keyed<T> = [keyof T, T]
|
||||
type WithoutSignal<T> = T extends { readonly signal?: AbortSignal } ? Omit<T, "signal"> : T
|
||||
type WithoutRunOptions<T> = T extends object ? Omit<T, keyof RunOptions> : T
|
||||
|
||||
type Promised<R> =
|
||||
R extends Effect.Effect<Generation<infer A>, infer _E, infer _R>
|
||||
|
||||
Reference in New Issue
Block a user