diff --git a/packages/ai/src/decision.ts b/packages/ai/src/decision.ts index 83009bd2c0b..ffd5f28feff 100644 --- a/packages/ai/src/decision.ts +++ b/packages/ai/src/decision.ts @@ -155,10 +155,11 @@ export interface DecisionModelRoute("Decision.ModelCompatibility")( { /** - * Where a System One host reads media, from models.dev `[api.systemone] compatibility.image_input`: - * `assets` (TypeSafe's assets map), `images` (top-level lists) or `state` (items inside a state list). + * Where a System One host reads media, from models.dev `[api.systemone] compatibility.media_input`, named after + * where the media goes in the body: `assets` (TypeSafe's assets map, referenced from the state), `state` (items + * inside a state list) or `root` (one top-level list per kind: `images`, `audio`, `videos`). */ - imageInput: Schema.optional(Schema.Literals(["assets", "images", "state"])), + mediaInput: Schema.optional(Schema.Literals(["assets", "state", "root"])), }, ) {} diff --git a/packages/ai/src/protocols/cloudflare-run-decisions.ts b/packages/ai/src/protocols/cloudflare-run-decisions.ts index 2d1a93cbf96..c806029ac4f 100644 --- a/packages/ai/src/protocols/cloudflare-run-decisions.ts +++ b/packages/ai/src/protocols/cloudflare-run-decisions.ts @@ -27,7 +27,7 @@ export const protocol = DecisionRoute.protocol({ inlineMedia: true, body: { from: (request) => - SystemOne.encodeBody(request, "images").pipe(Effect.map((input) => ({ model: request.model.id, input }))), + SystemOne.encodeBody(request, "root").pipe(Effect.map((input) => ({ model: request.model.id, input }))), }, response: { decode: (response, context) => diff --git a/packages/ai/src/protocols/system-one.ts b/packages/ai/src/protocols/system-one.ts index da9000be9cc..fd4b9382bbc 100644 --- a/packages/ai/src/protocols/system-one.ts +++ b/packages/ai/src/protocols/system-one.ts @@ -19,8 +19,8 @@ export const DEFAULT_PATH = "/systemone" export const DEFAULT_BASE_URL = "https://api.typesafe.ai/v1" -/** `messages` is Baseten's own placement (parts in a user message inside `state.messages`), set by its provider. */ -type ImageInput = NonNullable | "messages" +/** `state.messages` is Baseten's own placement (parts in a user message inside `state.messages`), set by its provider. */ +type MediaInput = NonNullable | "state.messages" const Probability = Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 1 })) const Answer = Schema.Union([ @@ -58,13 +58,13 @@ export type Response = Schema.Schema.Type const ID = "system-one" const NAME = "System One" -/** The System One body without `model`; media goes where the model's `imageInput` says, else `fallback`. */ +/** The System One body without `model`; media goes where the model's `mediaInput` says, else `fallback`. */ export const encodeBody = Effect.fn("SystemOne.encodeBody")(function* ( request: DecisionRequestFor, - fallback: ImageInput, + fallback: MediaInput, ) { const questions = yield* encodeQuestions(request.questions) - const placed = yield* placeMedia(request, request.model.compatibility?.imageInput ?? fallback) + const placed = yield* placeMedia(request, request.model.compatibility?.mediaInput ?? fallback) return { ...mergeJsonRecords(request.providerOptions, request.http?.body), ...placed, questions } }) @@ -103,7 +103,7 @@ const encodeQuestions = (questions: ReadonlyArray) => ] as const) }).pipe(Effect.map(Object.fromEntries)) -const placeMedia = Effect.fnUntraced(function* (request: DecisionRequestFor, imageInput: ImageInput) { +const placeMedia = Effect.fnUntraced(function* (request: DecisionRequestFor, mediaInput: MediaInput) { const parts = request.input const unsupported = (asset: Media.Asset, accepted: string) => new AIError({ @@ -116,9 +116,9 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequestFor, ima }) const media = parts.flatMap((part) => (part.type === "media" ? [part.media] : [])) if (media.length === 0) return { state: state(parts.map(plain)) } - if (imageInput === "state") return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported)) } + if (mediaInput === "state") return { state: yield* Effect.forEach(parts, (part) => stateItem(part, unsupported)) } // Chat-style content in one user message: text and JSON as text parts, media as in `state`. - if (imageInput === "messages") + if (mediaInput === "state.messages") return { state: { messages: [ @@ -133,7 +133,7 @@ const placeMedia = Effect.fnUntraced(function* (request: DecisionRequestFor, ima ], }, } - if (imageInput === "images") { + if (mediaInput === "root") { const lists = { image: "images", audio: "audio", video: "videos" } as const const unplaced = media.find((asset) => !Object.hasOwn(lists, asset.kind)) if (unplaced !== undefined) return yield* unsupported(unplaced, "images, audio and video") diff --git a/packages/ai/src/providers/baseten.ts b/packages/ai/src/providers/baseten.ts index e5a80243976..77d6f4fade9 100644 --- a/packages/ai/src/providers/baseten.ts +++ b/packages/ai/src/providers/baseten.ts @@ -47,7 +47,7 @@ const decisionProtocol = DecisionRoute.protocol({ id: "baseten-system-one", body: { from: (request) => - SystemOne.encodeBody(request, "messages").pipe(Effect.map((body) => ({ ...body, model: request.model.id }))), + SystemOne.encodeBody(request, "state.messages").pipe(Effect.map((body) => ({ ...body, model: request.model.id }))), }, }) diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index f0972ac7ec5..1ef52754f2f 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -203,7 +203,7 @@ export const configure = (input: LanguageModelOptions = {}) => { http: input.http, }) const decision = (modelID: string | ModelID) => - decisionRoute.model({ id: modelID, compatibility: { imageInput: "state" } }) + decisionRoute.model({ id: modelID, compatibility: { mediaInput: "state" } }) return { id, model: (modelID: string | ModelID) =>