fix(ai): classify content policy errors by provider codes (#49099)

This commit is contained in:
Aiden Cline
2026-09-14 23:08:31 -05:00
committed by GitHub
Unverified
parent c31ac3c919
commit 6a30741c45
2 changed files with 161 additions and 8 deletions
+42 -8
View File
@@ -73,9 +73,27 @@ const SERVER_CODES = new Set([
"serviceunavailableexception",
])
const INVALID_REQUEST_CODES = new Set(["invalid_prompt", "invalid_request_error", "validationexception"])
// Azure OpenAI reports `content_filter` with `innererror.code` ResponsibleAIPolicyViolation.
// OpenRouter tags provider failures with a typed `error_type`; its Responses skin also
// emits `image_content_policy_violation` as the native code.
const CONTENT_POLICY_CODES = new Set([
"content_filter",
"responsibleaipolicyviolation",
"content_policy_violation",
"image_content_policy_violation",
"refusal",
])
// OpenCode Zen replaces upstream codes outside its allow-list but keeps the original
// as a `[code]` label at the start of the rewritten message.
const GATEWAY_CODE_LABEL = /^[^:\n]+: \[([A-Za-z0-9_.-]+)\]/
const RATE_LIMIT_TEXT = /rate increased too quickly|rate[-_\s]?limit|too[_\s]?many[_\s]?requests/i
const QUOTA_TEXT = /insufficient[-_\s]?quota|quota[-_\s]?exceeded/i
const CONTENT_POLICY_TEXT = /content[-_\s]?policy|content_filter|safety/i
// Policy rejections without a dedicated code, matched against the provider's own
// explanation only. OpenAI reuses `invalid_prompt` for usage-policy rejections while
// Bedrock Mantle reuses it for schema validation; Anthropic reports blocked output
// under `invalid_request_error`.
const CONTENT_POLICY_TEXT =
/violating our usage policy|blocked by content filtering policy|content[-_\s]?policy|rejected as a result of our safety system/i
const SERVER_ERROR_TEXT =
/\b(?:try again|(?:please |you can )?retry (?:the |this |your )?request|try (?:the |this |your )?request again|(?:currently |temporarily )?at capacity|overloaded|temporarily unavailable|service[-_\s]?unavailable|(?:server|internal)[-_\s]?error|server (?:is )?busy|provider returned (?:an )?error|resource[-_\s]?exhausted|upstream (?:connect|connection|request)|request buffer limit while retrying upstream)\b/i
@@ -102,9 +120,12 @@ export interface ProviderFailure {
export function classifyProviderFailure(input: ProviderFailure): AIError["reason"] {
const details = { message: input.message, body: input.rawBody, http: input.http, cause: input.cause }
const body = input.rawBody ?? ""
const codes = [...providerCodes(input.data), ...providerCodes(body), ...providerCodes(input.message)].map((code) =>
code.toLowerCase(),
)
const codes = [
...providerCodes(input.data),
...providerCodes(body),
...providerCodes(input.message),
...(GATEWAY_CODE_LABEL.exec(input.message)?.slice(1) ?? []),
].map((code) => code.toLowerCase())
// Scan the raw payload too so signals missing from the summary message
// (e.g. overflow phrases nested in a JSON error body) still classify.
const text = [input.message, body].filter((value) => value.length > 0).join("\n")
@@ -120,7 +141,8 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason
return new InvalidRequestError({ ...details, classification: "context-overflow" })
if (input.status === 413 || isPayloadTooLarge(text))
return new InvalidRequestError({ ...details, classification: "payload-too-large" })
if (CONTENT_POLICY_TEXT.test(text)) return new ContentPolicyError(details)
if (codes.some((code) => CONTENT_POLICY_CODES.has(code)) || (clientScoped && CONTENT_POLICY_TEXT.test(input.message)))
return new ContentPolicyError(details)
if (codes.some((code) => QUOTA_CODES.has(code)) || (input.status === 429 && QUOTA_TEXT.test(text)))
return new QuotaExceededError(details)
if (input.status === 401 || input.status === 403 || codes.some((code) => AUTH_CODES.has(code)))
@@ -160,12 +182,24 @@ function providerCodes(value: unknown) {
const decoded = typeof value === "string" ? Option.getOrUndefined(decodeJson(value)) : value
if (!isRecord(decoded)) return []
const error = isRecord(decoded.error) ? decoded.error : undefined
const inner = error && isRecord(error.innererror) ? error.innererror : undefined
const metadata = error && isRecord(error.metadata) ? error.metadata : undefined
const response = isRecord(decoded.response) ? decoded.response : undefined
const responseError = response && isRecord(response.error) ? response.error : undefined
const exception = isRecord(decoded.exception) ? decoded.exception : undefined
return [decoded.code, error?.code, error?.type, error?.status, responseError?.code, exception?.type].filter(
(value): value is string => typeof value === "string",
)
return [
decoded.code,
decoded.error_type,
error?.code,
error?.type,
error?.status,
error?.error_type,
inner?.code,
metadata?.error_type,
responseError?.code,
response?.error_type,
exception?.type,
].filter((value): value is string => typeof value === "string")
}
function isRecord(value: unknown): value is Record<string, unknown> {
+119
View File
@@ -121,6 +121,125 @@ describe("provider error classification", () => {
).toEqual(["Authentication", "QuotaExceeded", "RateLimit", "InvalidRequest", "ProviderInternal"])
})
test("classifies Azure content filter rejections by their structured codes", () => {
const message = "The response was filtered"
const azure = {
error: {
message,
type: null,
param: "prompt",
code: "content_filter",
status: 400,
innererror: { code: "ResponsibleAIPolicyViolation", content_filter_result: {} },
},
}
const innerOnly = { error: { message, code: null, innererror: { code: "ResponsibleAIPolicyViolation" } } }
expect(classifyProviderFailure({ message, status: 400, rawBody: JSON.stringify(azure) })._tag).toBe("ContentPolicy")
expect(classifyProviderFailure({ message, status: 400, rawBody: JSON.stringify(innerOnly) })._tag).toBe(
"ContentPolicy",
)
})
test("classifies policy rejections that reuse generic request codes by the provider explanation", () => {
const openai = {
error: {
message:
"Invalid prompt: your prompt was flagged as potentially violating our usage policy. Please try again with a different prompt.",
type: "invalid_request_error",
param: null,
code: "invalid_prompt",
},
}
const anthropic = {
type: "error",
error: { type: "invalid_request_error", message: "Output blocked by content filtering policy" },
}
expect(
[openai, anthropic].map(
(body) =>
classifyProviderFailure({ message: body.error.message, status: 400, rawBody: JSON.stringify(body) })._tag,
),
).toEqual(["ContentPolicy", "ContentPolicy"])
})
test("classifies OpenRouter typed policy errors across its API skins", () => {
const chat = {
id: "gen-abc123",
object: "chat.completion.chunk",
error: {
code: 403,
message: "Your input was flagged",
metadata: { error_type: "content_policy_violation", reasons: ["violence"], flagged_input: "..." },
},
choices: [{ index: 0, delta: { content: "" }, finish_reason: "error" }],
}
const responses = {
type: "response.failed",
response: {
id: "resp_abc123",
status: "failed",
error: { code: "image_content_policy_violation", message: "Your input was flagged" },
error_type: "content_policy_violation",
},
}
const messages = {
type: "error",
error: { type: "invalid_request_error", message: "Claude refused to respond", error_type: "refusal" },
}
expect(
[
classifyProviderFailure({ message: chat.error.message, status: 403, rawBody: JSON.stringify(chat) }),
classifyProviderFailure({ message: responses.response.error.message, rawBody: JSON.stringify(responses) }),
classifyProviderFailure({ message: messages.error.message, rawBody: JSON.stringify(messages) }),
].map((failure) => failure._tag),
).toEqual(["ContentPolicy", "ContentPolicy", "ContentPolicy"])
})
test("recovers policy codes that OpenCode Zen preserves only in its message label", () => {
// Zen drops upstream codes outside its allow-list and replaces the type, but
// `Protocol.errorMessage` keeps the original code as a `[code]` prefix.
const zen = {
error: {
type: "server_error",
message: "Upstream request failed: [content_filter] The response was filtered",
},
}
expect(
classifyProviderFailure({ message: zen.error.message, status: 400, rawBody: JSON.stringify(zen) })._tag,
).toBe("ContentPolicy")
})
test("keeps invalid_prompt schema validation failures as invalid requests", () => {
// Bedrock Mantle reports request validation failures under `invalid_prompt`
// with a pydantic dump that lists every input union member's fields.
const message = [
"SubmitRequestFailure: code=-32602, msg=219 validation errors for ResponsesRequest",
"input.list[union[EasyInputMessageParam,ResponseComputerToolCallParam]].2.ResponseComputerToolCallParam.pending_safety_checks",
" Field required [type=missing, input_value={'content': [...], 'role': 'assistant', 'type': 'message'}, input_type=dict]",
].join("\n")
const failure = classifyProviderFailure({
message,
status: 200,
rawBody: JSON.stringify({ response: { status: "failed", error: { code: "invalid_prompt", message } } }),
})
expect(failure._tag).toBe("InvalidRequest")
})
test("does not infer content policy from policy phrases outside the provider explanation", () => {
const failure = classifyProviderFailure({
message: "Provider request failed with HTTP 400",
status: 400,
rawBody: JSON.stringify({ error: { message: "Unknown parameter: content_policy_id" } }),
})
expect(failure._tag).toBe("InvalidRequest")
})
test("classifies transient client statuses as provider internal", () => {
expect([408, 409].map((status) => classifyProviderFailure({ message: `HTTP ${status}`, status })._tag)).toEqual([
"ProviderInternal",