diff --git a/packages/core/src/plugin/provider/github-copilot.ts b/packages/core/src/plugin/provider/github-copilot.ts index b11f89f3b80..1eb7ad17321 100644 --- a/packages/core/src/plugin/provider/github-copilot.ts +++ b/packages/core/src/plugin/provider/github-copilot.ts @@ -465,11 +465,13 @@ function requestMetadata(url: string, body: unknown) { const last = body.input.at(-1) return { agent: !isRecord(last) || last.role !== "user", + // Tool results carry their images in `output`, user messages in `content`. vision: body.input.some( (item) => isRecord(item) && - Array.isArray(item.content) && - item.content.some((part) => isRecord(part) && part.type === "input_image"), + [item.content, item.output].some( + (parts) => Array.isArray(parts) && parts.some((part) => isRecord(part) && part.type === "input_image"), + ), ), } } diff --git a/packages/core/test/plugin/provider-github-copilot.test.ts b/packages/core/test/plugin/provider-github-copilot.test.ts index 5165f1c8c77..0f3fc1fb556 100644 --- a/packages/core/test/plugin/provider-github-copilot.test.ts +++ b/packages/core/test/plugin/provider-github-copilot.test.ts @@ -437,6 +437,31 @@ describe("GithubCopilotPlugin", () => { }), ) + it.effect("flags images returned by tools in Responses requests", () => + Effect.gen(function* () { + yield* addPlugin() + const event = yield* httpRequest( + new Request("https://api.githubcopilot.com/responses", { + method: "POST", + headers: { authorization: "Bearer token" }, + body: JSON.stringify({ + input: [ + { role: "user", content: [{ type: "input_text", text: "Take a screenshot." }] }, + { type: "function_call", call_id: "call_1", name: "screenshot", arguments: "{}" }, + { + type: "function_call_output", + call_id: "call_1", + output: [{ type: "input_image", image_url: "data:image/png;base64,AAAA" }], + }, + ], + }), + }), + ) + expect(event.request.headers.get("copilot-vision-request")).toBe("true") + expect(event.request.headers.get("x-initiator")).toBe("agent") + }), + ) + it.effect("keeps a declared agent initiator when the body looks user-initiated", () => Effect.gen(function* () { yield* addPlugin()