From d142014e41eb7b117114b3c9372cf46279dec78b Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:03:35 -0500 Subject: [PATCH] feat(core): offer gpt-image-2 on the Codex login (#60085) --- packages/ai/src/protocols/openai-images.ts | 19 ++++++++---- packages/ai/src/providers/openai.ts | 4 ++- packages/ai/test/image.test.ts | 31 ++++++++++++++++++- packages/core/src/plugin/provider/openai.ts | 7 +++++ .../core/test/plugin/provider-openai.test.ts | 17 ++++++++++ 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts index ca56b9612fb..6ff18e77570 100644 --- a/packages/ai/src/protocols/openai-images.ts +++ b/packages/ai/src/protocols/openai-images.ts @@ -203,10 +203,12 @@ const reference = (asset: Media.Asset, provider: ProviderID) => ) /** - * How input images travel. `edits` is OpenAI's `/images/edits`; `references` keeps every request on - * `/images/generations` and lists them as `input_references`, for gateways without an edits endpoint (OpenRouter). + * How input images travel. `edits` is OpenAI's `/images/edits`, uploading owned bytes as multipart; `json-edits` sends + * every edit there as JSON with data URLs, for hosts that reject multipart (the ChatGPT backend); `references` keeps + * every request on `/images/generations` and lists them as `input_references`, for gateways without an edits endpoint + * (OpenRouter). */ -export type OpenAIImageInput = "edits" | "references" +export type OpenAIImageInput = "edits" | "json-edits" | "references" type Options = { readonly sizing: OpenAIImageSizing; readonly imageInput: OpenAIImageInput } @@ -253,8 +255,13 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* ( ) } - // Owned bytes go through multipart edits; remote URLs and file IDs use the JSON edits body instead. - if (images.length > 0 && images.every(isInline) && (mask === undefined || isInline(mask))) { + // Owned bytes go through multipart edits; remote URLs, file IDs and `json-edits` hosts use the JSON edits body. + if ( + options.imageInput === "edits" && + images.length > 0 && + images.every(isInline) && + (mask === undefined || isInline(mask)) + ) { const form = new FormData() MediaInput.appendFields( form, @@ -422,7 +429,7 @@ export const model = ( { protocol: input.provider === undefined ? base : { ...base, provider: ProviderID.make(input.provider) }, baseURL: DEFAULT_BASE_URL, - path: ({ request }) => (imageInput === "edits" && isEdit(request) ? EDIT_PATH : PATH), + path: ({ request }) => (imageInput !== "references" && isEdit(request) ? EDIT_PATH : PATH), }, input, ) diff --git a/packages/ai/src/providers/openai.ts b/packages/ai/src/providers/openai.ts index c19804854d0..b08747b3f8b 100644 --- a/packages/ai/src/providers/openai.ts +++ b/packages/ai/src/providers/openai.ts @@ -116,8 +116,10 @@ export const configure = (input: Config = {}) => { input.queryParams === undefined ? undefined : new HttpOptions({ query: input.queryParams }), ), } + // The ChatGPT backend (ChatGPT sign-in) rejects multipart image edits with "Unsupported content type". + const imageInput = media.baseURL?.startsWith("https://chatgpt.com/") ? "json-edits" : undefined const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) => - OpenAIImages.model({ ...media, id: modelID, providerOptions }) + OpenAIImages.model({ ...media, id: modelID, providerOptions, imageInput }) const speech = (modelID: string | ModelID) => OpenAISpeech.model({ ...media, id: modelID }) const transcription = (modelID: string | ModelID) => OpenAITranscription.model({ ...media, id: modelID }) const decisionRoute = OpenAIDecisions.route.with({ diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts index 3f14deb34f4..fe49cfb341c 100644 --- a/packages/ai/test/image.test.ts +++ b/packages/ai/test/image.test.ts @@ -164,7 +164,10 @@ describe("Image", () => { const bodies: Array = [] const compatible = OpenAICompatible.configure({ apiKey: "test", baseURL: "https://compatible.test/v1" }) const xai = XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }) - const generate = (model: Parameters[0]["model"], providerOptions?: Record) => + const generate = ( + model: Parameters[0]["model"], + providerOptions?: Record, + ) => Image.generate({ model, prompt: "A lighthouse", providerOptions }).pipe( Effect.provide( layer((input) => { @@ -580,6 +583,32 @@ describe("Image", () => { ) }) + it.effect("sends OpenAI edits as JSON to the ChatGPT backend, which rejects multipart", () => + Image.generate({ + model: OpenAI.configure({ apiKey: "test", baseURL: "https://chatgpt.com/backend-api/codex" }).image( + "gpt-image-2", + ), + prompt: "Add a red cap", + images: [Media.bytes(Uint8Array.from([1, 2, 3]), "image/png")], + }).pipe( + Effect.provide( + layer((input) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + expect(request.url).toBe("https://chatgpt.com/backend-api/codex/images/edits") + expect(request.headers.get("content-type")).toBe("application/json") + expect(JSON.parse(input.text)).toEqual({ + model: "gpt-image-2", + prompt: "Add a red cap", + images: [{ image_url: "data:image/png;base64,AQID" }], + }) + return json(input, { data: [{ b64_json: "AQID" }] }) + }), + ), + ), + ), + ) + it.effect("routes ordered xAI image inputs through JSON edits", () => Image.generate({ model: XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" }).image("future-model"), diff --git a/packages/core/src/plugin/provider/openai.ts b/packages/core/src/plugin/provider/openai.ts index fcf413c6e68..d005eb17ada 100644 --- a/packages/core/src/plugin/provider/openai.ts +++ b/packages/core/src/plugin/provider/openai.ts @@ -25,6 +25,8 @@ const browserMethodID = Integration.MethodID.make("chatgpt-browser") const headlessMethodID = Integration.MethodID.make("chatgpt-headless") // Codex rejects gpt-5.5 and older models, including gpt-5.3-codex-spark, for ChatGPT accounts. const codexDisallowed = new Set(["gpt-5.6"]) +// The image models the Codex client generates and edits with on the Codex login. +const codexImageModels = new Set(["gpt-image-2"]) type Pkce = { verifier: string @@ -285,6 +287,11 @@ export const OpenAIPlugin = define({ return } const apiID = draft.modelID ?? draft.id + if (draft.type === "image") { + if (codexImageModels.has(apiID)) draft.cost = [] + else draft.enabled = false + return + } const match = apiID.match(/^gpt-(\d+)(?:\.(\d+))?/) const major = Number(match?.[1]) const minor = Number(match?.[2] ?? 0) diff --git a/packages/core/test/plugin/provider-openai.test.ts b/packages/core/test/plugin/provider-openai.test.ts index dc416a4ce67..29957e56526 100644 --- a/packages/core/test/plugin/provider-openai.test.ts +++ b/packages/core/test/plugin/provider-openai.test.ts @@ -127,6 +127,19 @@ describe("OpenAIPlugin", () => { catalog.models.update(Provider.ID.openai, Model.ID.make("gpt-5"), () => {}) catalog.models.update(Provider.ID.openai, Model.ID.make("gpt-5.04-astra"), () => {}) catalog.models.update(Provider.ID.openai, Model.ID.make("gpt-4.99"), () => {}) + catalog.models.update(Provider.ID.openai, Model.ID.make("gpt-image-1.5"), (model) => { + model.type = "image" + }) + catalog.models.update(Provider.ID.openai, Model.ID.make("gpt-image-2"), (model) => { + model.type = "image" + model.cost = [ + { + input: Money.USDPerMillionTokens.make(8), + output: Money.USDPerMillionTokens.make(30), + cache: { read: Money.USDPerMillionTokens.zero, write: Money.USDPerMillionTokens.zero }, + }, + ] + }) }) yield* credentials.create({ integrationID: Integration.ID.make("openai"), @@ -184,6 +197,10 @@ describe("OpenAIPlugin", () => { expect(required(yield* models.get(Provider.ID.openai, Model.ID.make("gpt-5"))).enabled).toBe(false) expect(required(yield* models.get(Provider.ID.openai, Model.ID.make("gpt-5.04-astra"))).enabled).toBe(false) expect(required(yield* models.get(Provider.ID.openai, Model.ID.make("gpt-4.99"))).enabled).toBe(false) + // Only the image model the Codex client uses is offered, covered by the subscription. + const images = (yield* models.available(["image"])).filter((model) => model.providerID === Provider.ID.openai) + expect(images.map((model) => model.id)).toEqual([Model.ID.make("gpt-image-2")]) + expect(images[0]?.cost).toEqual([]) }), )