feat(ai): add Seedance video generation for Volcengine and BytePlus (#63680)

This commit is contained in:
Aiden Cline
2026-11-13 00:20:46 -06:00
committed by GitHub
Unverified
parent b645291e9c
commit 989565390f
12 changed files with 905 additions and 1 deletions
+12 -1
View File
@@ -1036,7 +1036,8 @@ The hosted result is represented as a provider-executed tool call and a tool res
Video generation is a job: `start` submits it and returns at once, and the caller decides when to read its `status`,
fetch its `result`, or `cancel` it. Nothing polls for you. Every call after `start` takes the model and the job's
`token`, plain JSON you can store and read back from another process, so recovering after a restart is the same call
as a normal status read. Models come from `.video(...)` selectors on the `XAI` and `OpenRouter` facades.
as a normal status read. Models come from `.video(...)` selectors on the `XAI`, `OpenRouter`, `Volcengine` and
`BytePlus` facades.
```ts
import { Effect } from "effect"
@@ -1093,6 +1094,16 @@ Provider notes:
rejects reference videos or audio that aren't http(s) URLs (inline images work). Routing and upstream parameters go under `providerOptions.provider` (`only`, `order`,
`options: { <slug>: {...} }`), and `providerOptions.previous_job_id` edits or extends a finished job. Results are
OpenRouter content URLs that download with the API key, which the asset carries; there is no cancel endpoint.
- **Seedance** (`Volcengine.configure(...).video(...)` with `ARK_API_KEY`, `BytePlus.configure(...).video(...)` with
`BYTEPLUS_API_KEY`) builds Ark's `content[]`: the prompt as a `text` item, `start` and `end` as `image_url` items
with roles `first_frame` and `last_frame`, and references as `reference_image`, `reference_video` or
`reference_audio` by kind. Your own `Media.ref(...)` assets go as `asset://<id>`. `duration` goes to `duration`,
`aspect` to `ratio` (`auto` becomes `adaptive`), `resolution` to `resolution` and `sound` to `generate_audio`. Ark
doesn't allow frames and references in one task; that combination is sent and the provider's error comes back.
`providerOptions.draft_task: { id }` adds a `draft_task` item; `frames`, `seed`, `watermark`, `output_format`,
`omni_reference_task_type`, `return_last_frame` and the rest go into the body as-is. `cancel` sends `DELETE`, which
only cancels a queued task: Ark refuses it for a running one and deletes the record of a finished one. Video URLs
expire 24 hours after the task finished.
The promise client has the same calls, each accepting `{ signal }`:
+5
View File
@@ -44,6 +44,8 @@
"./providers/azure/responses": "./src/providers/azure/responses.ts",
"./providers/baseten": "./src/providers/baseten.ts",
"./providers/baseten/decisions": "./src/providers/baseten/decisions.ts",
"./providers/byteplus": "./src/providers/byteplus.ts",
"./providers/byteplus/videos": "./src/providers/byteplus/videos.ts",
"./providers/cartesia": "./src/providers/cartesia.ts",
"./providers/cerebras": "./src/providers/cerebras.ts",
"./providers/cloudflare-ai-gateway": "./src/providers/cloudflare-ai-gateway.ts",
@@ -121,6 +123,8 @@
"./providers/typesafe-ai/decisions": "./src/providers/typesafe-ai/decisions.ts",
"./providers/venice": "./src/providers/venice.ts",
"./providers/venice/responses": "./src/providers/venice/responses.ts",
"./providers/volcengine": "./src/providers/volcengine.ts",
"./providers/volcengine/videos": "./src/providers/volcengine/videos.ts",
"./providers/vercel-ai-gateway": "./src/providers/vercel-ai-gateway.ts",
"./providers/vercel-ai-gateway/chat": "./src/providers/vercel-ai-gateway/chat.ts",
"./providers/vercel-ai-gateway/decisions": "./src/providers/vercel-ai-gateway/decisions.ts",
@@ -154,6 +158,7 @@
"./protocols/openai-responses": "./src/protocols/openai-responses.ts",
"./protocols/openai-speech": "./src/protocols/openai-speech.ts",
"./protocols/openai-transcription": "./src/protocols/openai-transcription.ts",
"./protocols/seedance-video": "./src/protocols/seedance-video.ts",
"./protocols/system-one": "./src/protocols/system-one.ts",
"./protocols/xai-images": "./src/protocols/xai-images.ts",
"./protocols/xai-video": "./src/protocols/xai-video.ts",
+302
View File
@@ -0,0 +1,302 @@
import { Duration, Effect, Schema } from "effect"
import type { HttpClientResponse } from "effect/http"
import type { Status } from "../generation.js"
import { Media } from "../media.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords } from "../schema/index.js"
import { VideoModel, VideoResponse, seconds, type VideoRequest } from "../video.js"
import { ProviderShared, optionalNull } from "./shared.js"
// One Ark API (`/contents/generations/tasks`) served by Volcengine Ark in China and BytePlus ModelArk elsewhere.
export const VOLCENGINE_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"
export const BYTEPLUS_BASE_URL = "https://ark.ap-southeast.bytepluses.com/api/v3"
export const PATH = "/contents/generations/tasks"
// Video and last-frame URLs are documented as valid for 24 hours.
const OUTPUT_RETENTION = Duration.hours(24)
// ---------------------------------------------------------------------------
// 1. Public model input
// ---------------------------------------------------------------------------
/**
* Provider-native options, merged into the task body. Common fields (`start`, `end`, `references`, `duration`,
* `aspect`, `resolution`, `sound`) live on the request. `draft_task` adds a `draft_task` content item that turns a
* Draft video into the final one.
*/
export type SeedanceVideoOptions = {
readonly draft_task?: { readonly id: string }
readonly draft?: boolean
readonly frames?: number
readonly seed?: number
readonly camera_fixed?: boolean
readonly watermark?: boolean
readonly return_last_frame?: boolean
readonly omni_reference_task_type?: "auto" | "reference" | "edit" | "extend" | (string & {})
readonly output_format?: "mp4" | "mov" | (string & {})
readonly service_tier?: "default" | "flex" | (string & {})
readonly callback_url?: string
readonly execution_expires_after?: number
readonly priority?: number
readonly safety_identifier?: string
} & Record<string, unknown>
export type Request = VideoRequest<VideoModel<SeedanceVideoOptions>>
// ---------------------------------------------------------------------------
// 2. Token and response schemas
// ---------------------------------------------------------------------------
export const Token = Schema.Struct({ taskID: Schema.String })
export type Token = Schema.Schema.Type<typeof Token>
const StartResponse = Schema.Struct({ id: Schema.String })
const Task = Schema.Struct({
id: optionalNull(Schema.String),
model: optionalNull(Schema.String),
status: Schema.String,
content: optionalNull(
Schema.Struct({
video_url: optionalNull(Schema.String),
last_frame_url: optionalNull(Schema.String),
}),
),
error: optionalNull(
Schema.Struct({
code: optionalNull(Schema.String),
message: optionalNull(Schema.String),
}),
),
usage: optionalNull(
Schema.Struct({
completion_tokens: optionalNull(Schema.Number),
total_tokens: optionalNull(Schema.Number),
}),
),
updated_at: optionalNull(Schema.Number),
seed: optionalNull(Schema.Number),
resolution: optionalNull(Schema.String),
ratio: optionalNull(Schema.String),
duration: optionalNull(Schema.Number),
frames: optionalNull(Schema.Number),
framespersecond: optionalNull(Schema.Number),
generate_audio: optionalNull(Schema.Boolean),
output_format: optionalNull(Schema.String),
draft: optionalNull(Schema.Boolean),
service_tier: optionalNull(Schema.String),
})
const STATUS = {
queued: "queued",
running: "running",
succeeded: "completed",
failed: "failed",
cancelled: "cancelled",
expired: "expired",
} as const satisfies Record<string, Status>
// ---------------------------------------------------------------------------
// 3. Failures
// ---------------------------------------------------------------------------
// Ark error codes are dotted (`InvalidParameter.TaskTypeConstraint`); the leading segment is the category.
// `InternalServiceError`, `ServerOverloaded` and unknown codes are provider-side.
const FAILURE = {
InvalidParameter: "InvalidRequest",
MissingParameter: "InvalidRequest",
InvalidArgumentError: "InvalidRequest",
InvalidImageURL: "InvalidRequest",
AuthenticationError: "Authentication",
AccessDenied: "Authentication",
QuotaExceeded: "RateLimit",
RateLimitExceeded: "RateLimit",
} as const satisfies Record<string, MediaProtocol.Failure>
// Input and output moderation codes: `InputImageSensitiveContentDetected.PrivacyInformation`,
// `OutputVideoSensitiveContentDetected`, `SensitiveContentDetected`, ...
const moderated = (code: string | undefined) => code !== undefined && code.includes("SensitiveContentDetected")
// ---------------------------------------------------------------------------
// 4. Request body construction
// ---------------------------------------------------------------------------
const nativeOptions = (options: SeedanceVideoOptions | undefined) => {
if (!options) return undefined
const { draft_task: _draftTask, ...native } = options
return native
}
// ---------------------------------------------------------------------------
// 5. Response decoding
// ---------------------------------------------------------------------------
// Ark returns `frames` when the request set it, otherwise `duration` rounded down to whole seconds.
const durationSeconds = (task: Schema.Schema.Type<typeof Task>) => {
if (task.frames !== undefined && task.frames !== null && task.framespersecond)
return task.frames / task.framespersecond
return task.duration ?? undefined
}
// ---------------------------------------------------------------------------
// 6. Protocol and model, one per provider so assets and metadata carry the right provider ID
// ---------------------------------------------------------------------------
export type Provider = "volcengine" | "byteplus"
const make = (provider: Provider) => {
const route = MediaProtocol.identity({ id: "seedance-video", name: "Seedance Video", provider })
// `asset://<ID>` addresses a ModelArk digital character or authorized asset.
const item = (asset: Media.Asset, type: "image_url" | "video_url" | "audio_url", role: string) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
Effect.map(
(reference): Record<string, unknown> => ({
type,
[type]: { url: reference.type === "ref" ? `asset://${reference.value}` : reference.value },
role,
}),
),
)
const reference = (asset: Media.Asset) => {
if (asset.kind === "video") return item(asset, "video_url", "reference_video")
if (asset.kind === "audio") return item(asset, "audio_url", "reference_audio")
return item(asset, "image_url", "reference_image")
}
// Frames and references are separate Seedance modes; mixing them is sent as-is and rejected by the provider.
const fromRequest = Effect.fn("SeedanceVideo.fromRequest")(function* (request: Request) {
const content = [
// Every media mode accepts an optional prompt, so an empty one adds no text item.
...(request.prompt === "" ? [] : [{ type: "text", text: request.prompt }]),
...(request.start === undefined ? [] : [yield* item(request.start, "image_url", "first_frame")]),
...(request.end === undefined ? [] : [yield* item(request.end, "image_url", "last_frame")]),
...(yield* Effect.forEach(request.references ?? [], reference)),
...(request.providerOptions?.draft_task === undefined
? []
: [{ type: "draft_task", draft_task: request.providerOptions.draft_task }]),
]
return MediaProtocol.json(
mergeJsonRecords(
{
model: request.model.id,
content,
duration: request.duration === undefined ? undefined : seconds(request.duration),
ratio: request.aspect === "auto" ? "adaptive" : request.aspect,
resolution: request.resolution,
generate_audio: request.sound,
},
nativeOptions(request.providerOptions),
request.http?.body,
) ?? {},
)
})
const decodeStart = route.decodeStarted(StartResponse, (value) => ({
token: { taskID: value.id },
snapshot: { id: value.id, status: "queued" },
}))
const decodeTask = route.decodeJson(Task)
const decodeStatus = Effect.fn("SeedanceVideo.decodeStatus")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
return { id: context.token.taskID, status: MediaProtocol.status(STATUS, output.value.status) }
})
const decodeResult = Effect.fn("SeedanceVideo.decodeResult")(function* (
response: HttpClientResponse.HttpClientResponse,
context: MediaProtocol.PollContext<Token>,
) {
const output = yield* decodeTask(response)
const task = output.value
const status = MediaProtocol.status(STATUS, task.status)
if (status === "queued" || status === "running") return yield* output.pending(context.token.taskID)
if (status === "failed") {
const code = task.error?.code ?? undefined
const detail = task.error?.message ?? undefined
const message = `${route.name} generation failed${code === undefined ? "" : ` (${code})`}${detail === undefined ? "" : `: ${detail}`}`
if (moderated(code)) return yield* output.contentPolicy(message)
return yield* output.ended("failed", message, MediaProtocol.failure(FAILURE, code?.split(".")[0]))
}
if (status === "cancelled")
return yield* output.ended("cancelled", `${route.name} task ${context.token.taskID} was cancelled`)
if (status === "expired")
return yield* output.ended("expired", `${route.name} task ${context.token.taskID} expired`)
const videoURL = task.content?.video_url ?? undefined
if (videoURL === undefined) return yield* output.invalid(`${route.name} completed without a video URL`)
const duration = durationSeconds(task)
const options = {
mediaType: task.output_format === "mov" ? "video/quicktime" : "video/mp4",
info: duration === undefined ? undefined : { durationSeconds: duration },
}
// `updated_at` is when the task reached `succeeded`, which starts the URL's retention window.
const video =
task.updated_at === undefined || task.updated_at === null
? yield* MediaProtocol.expiringUrl(videoURL, OUTPUT_RETENTION, options)
: Media.url(videoURL, {
...options,
expiresAt: task.updated_at * 1000 + Duration.toMillis(OUTPUT_RETENTION),
})
return new VideoResponse({
videos: [video],
usage:
task.usage?.completion_tokens === undefined || task.usage.completion_tokens === null
? undefined
: {
type: "tokens",
output: task.usage.completion_tokens,
total: task.usage.total_tokens ?? undefined,
},
providerMetadata: {
[provider]: {
taskId: context.token.taskID,
model: task.model ?? undefined,
seed: task.seed ?? undefined,
ratio: task.ratio ?? undefined,
resolution: task.resolution ?? undefined,
duration: task.duration ?? undefined,
frames: task.frames ?? undefined,
framesPerSecond: task.framespersecond ?? undefined,
generateAudio: task.generate_audio ?? undefined,
outputFormat: task.output_format ?? undefined,
draft: task.draft ?? undefined,
serviceTier: task.service_tier ?? undefined,
lastFrameUrl: task.content?.last_frame_url ?? undefined,
},
},
})
})
const taskPath = (token: Token) => `${PATH}/${token.taskID}`
// DELETE cancels a queued task. Ark refuses it for a running task, and deletes the record of a finished one.
const protocol = MediaProtocol.queued<Request, VideoResponse, Token>(route, {
token: Token,
start: { body: { from: fromRequest }, decode: decodeStart },
status: { path: taskPath, decode: decodeStatus },
result: { path: taskPath, decode: decodeResult },
cancel: { method: "DELETE", path: taskPath },
})
const model = (input: MediaRoute.ModelInput) =>
VideoModel.fromRoute<SeedanceVideoOptions, Token>(
{
protocol,
baseURL: provider === "volcengine" ? VOLCENGINE_BASE_URL : BYTEPLUS_BASE_URL,
path: PATH,
},
input,
)
return { protocol, model }
}
export const SeedanceVideo = {
volcengine: make("volcengine"),
byteplus: make("byteplus"),
} as const
+40
View File
@@ -0,0 +1,40 @@
import type { ProviderPackage } from "../provider-package.js"
import { SeedanceVideo } from "../protocols/seedance-video.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
import { toConfig } from "./settings.js"
export const id = ProviderID.make("byteplus")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
export type Settings = ProviderPackage.Settings & {
readonly apiKey?: string | false
readonly baseURL?: string
}
export type { SeedanceVideoOptions } from "../protocols/seedance-video.js"
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "BYTEPLUS_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
return {
id,
video: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
SeedanceVideo.byteplus.model({ ...media, id: modelID, providerOptions }),
configure,
}
}
export const provider = configure()
export const videoModel = (modelID: string, settings?: Settings) => {
const input = toConfig(settings)
return configure(input).video(modelID, input.providerOptions)
}
export const video = provider.video
@@ -0,0 +1,2 @@
export { videoModel as model } from "../byteplus.js"
export type { Settings } from "../byteplus.js"
+2
View File
@@ -6,6 +6,7 @@ export * as AmazonBedrockMantle from "./amazon-bedrock-mantle.js"
export * as AssemblyAI from "./assemblyai.js"
export * as Azure from "./azure.js"
export * as Baseten from "./baseten.js"
export * as BytePlus from "./byteplus.js"
export * as Cartesia from "./cartesia.js"
export * as Cerebras from "./cerebras.js"
export * as Cohere from "./cohere.js"
@@ -38,6 +39,7 @@ export * as TogetherAI from "./togetherai.js"
export * as TypeSafeAI from "./typesafe-ai.js"
export * as VercelAIGateway from "./vercel-ai-gateway.js"
export * as Venice from "./venice.js"
export * as Volcengine from "./volcengine.js"
export * as XAI from "./xai.js"
export * as ZAI from "./zai.js"
export * as ZAICodingPlan from "./zai-coding-plan.js"
+40
View File
@@ -0,0 +1,40 @@
import type { ProviderPackage } from "../provider-package.js"
import { SeedanceVideo } from "../protocols/seedance-video.js"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { MediaRoute } from "../route/media.js"
import { type HttpOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
import { toConfig } from "./settings.js"
export const id = ProviderID.make("volcengine")
export type Config = ProviderAuthOption<"optional"> & {
readonly baseURL?: string
readonly headers?: Record<string, string>
readonly http?: HttpOptions
}
export type Settings = ProviderPackage.Settings & {
readonly apiKey?: string | false
readonly baseURL?: string
}
export type { SeedanceVideoOptions } from "../protocols/seedance-video.js"
const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "ARK_API_KEY")
export const configure = (input: Config = {}) => {
const media = MediaRoute.deployment(input, auth(input))
return {
id,
video: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
SeedanceVideo.volcengine.model({ ...media, id: modelID, providerOptions }),
configure,
}
}
export const provider = configure()
export const videoModel = (modelID: string, settings?: Settings) => {
const input = toConfig(settings)
return configure(input).video(modelID, input.providerOptions)
}
export const video = provider.video
@@ -0,0 +1,2 @@
export { videoModel as model } from "../volcengine.js"
export type { Settings } from "../volcengine.js"
@@ -513,6 +513,26 @@ const videos: ReadonlyArray<Case<VideoModel>> = [
auth: ["authorization", "Bearer openrouter-key"],
provider: "openrouter",
},
{
name: "byteplus",
load: async () => {
const { model } = await import("@opencode/ai/providers/byteplus/videos")
return model("dreamina-seedance-2-0-260128", { ...shared, apiKey: "byteplus-key" })
},
url: "https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks",
auth: ["authorization", "Bearer byteplus-key"],
provider: "byteplus",
},
{
name: "volcengine",
load: async () => {
const { model } = await import("@opencode/ai/providers/volcengine/videos")
return model("doubao-seedance-2-0-260128", { ...shared, apiKey: "ark-key" })
},
url: "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks",
auth: ["authorization", "Bearer ark-key"],
provider: "volcengine",
},
]
describe("video entrypoints", () => {
@@ -0,0 +1,52 @@
import { describe, expect } from "bun:test"
import { Effect, Schedule } from "effect"
import { ai, type VideoStatus } from "../../src/effect.js"
import { BytePlus } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
import { queuedPoll } from "./queued-recording.js"
// Seedance 2.0 mini is the cheapest 2.x model; 480p and 4 seconds are the lowest settings it accepts.
const model = BytePlus.configure({
apiKey: process.env.BYTEPLUS_API_KEY ?? "fixture",
}).video("dreamina-seedance-2-0-mini-260615")
const recorded = recordedTests({
prefix: "seedance-video",
provider: "byteplus",
protocol: "seedance-video",
requires: ["BYTEPLUS_API_KEY"],
})
describe("Seedance Video recorded", () => {
recorded.effect(
"generates a video",
() =>
Effect.gen(function* () {
const job = yield* ai.video.start({
model,
prompt: "A single red balloon drifting slowly upward against a clear blue sky.",
aspect: "16:9",
resolution: "480p",
duration: "4 seconds",
sound: false,
})
const observed: Array<VideoStatus> = []
yield* ai.video.status(model, job.token).pipe(
Effect.tap((status) => Effect.sync(() => observed.push(status))),
Effect.repeat({
schedule: Schedule.spaced(queuedPoll.interval ?? "10 millis"),
until: (status) => status.status !== "queued" && status.status !== "running",
}),
)
expect(observed.at(-1)?.status).toBe("completed")
const response = yield* ai.video.result(model, job.token)
expect(response.videos).toHaveLength(1)
expect(response.video.source.type).toBe("url")
expect(response.video.expiresAt).toBeGreaterThan(0)
expect(response.usage?.type).toBe("tokens")
expect((yield* response.video.bytes()).length).toBeGreaterThan(0)
}),
{ timeout: 15 * 60 * 1000 },
)
})
+426
View File
@@ -0,0 +1,426 @@
import { describe, expect } from "bun:test"
import { Effect, Layer } from "effect"
import { Media, VideoClient, ai } from "../src/effect.js"
import { BytePlus, Volcengine } from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse, json, observe, type Call } from "./lib/http.js"
const layer = (handler: Parameters<typeof dynamicResponse>[0]) =>
VideoClient.layer.pipe(Layer.provideMerge(dynamicResponse(handler)))
const model = Volcengine.configure({ apiKey: "test", baseURL: "https://ark.test/api/v3" }).video(
"doubao-seedance-2-0-260128",
)
const token = { taskID: "cgt-1" }
const succeeded = {
id: "cgt-1",
model: "doubao-seedance-2-0-260128",
status: "succeeded",
content: { video_url: "https://ark-content.test/out.mp4" },
usage: { completion_tokens: 108900, total_tokens: 108900 },
created_at: 1779348818,
updated_at: 1779348874,
seed: 78674,
resolution: "720p",
ratio: "16:9",
duration: 5,
framespersecond: 24,
service_tier: "default",
generate_audio: true,
draft: false,
}
const reply = (body: unknown) => layer((input) => Effect.succeed(json(input, body)))
// ---------------------------------------------------------------------------
// start
// ---------------------------------------------------------------------------
describe("ai.video.start / Seedance", () => {
it.effect("maps frames, shared fields and provider options onto Ark's content[] body", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
const job = yield* ai.video
.start({
model,
prompt: "The kitten yawns at the camera",
start: Media.fromDataUrl("data:image/png;base64,AQID"),
end: "https://example.test/last.png",
duration: "5 seconds",
aspect: "16:9",
resolution: "720p",
sound: false,
providerOptions: { watermark: true, return_last_frame: true, future_option: 1 },
headers: { "x-request": "yes" },
http: { query: { trace: "1" } },
})
.pipe(
Effect.provide(
layer((input) =>
Effect.gen(function* () {
const { call } = yield* observe(calls, input)
expect(call.headers.get("authorization")).toBe("Bearer test")
expect(call.headers.get("x-request")).toBe("yes")
return json(input, { id: "cgt-1" })
}),
),
),
)
expect(job).toEqual({ id: "cgt-1", token: { taskID: "cgt-1" } })
expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([
"POST https://ark.test/api/v3/contents/generations/tasks?trace=1",
])
expect(JSON.parse(calls[0].body)).toEqual({
model: "doubao-seedance-2-0-260128",
content: [
{ type: "text", text: "The kitten yawns at the camera" },
{ type: "image_url", image_url: { url: "data:image/png;base64,AQID" }, role: "first_frame" },
{ type: "image_url", image_url: { url: "https://example.test/last.png" }, role: "last_frame" },
],
duration: 5,
ratio: "16:9",
resolution: "720p",
generate_audio: false,
watermark: true,
return_last_frame: true,
future_option: 1,
})
}),
)
it.effect("maps image, video and audio references to their roles, with own assets as asset:// URIs", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
yield* ai.video
.start({
model,
prompt: "Match @Image1 to the motion of @Video1",
references: [
Media.ref("volcengine", "asset-123"),
Media.url("https://example.test/style.png", { mediaType: "image/png" }),
Media.url("https://example.test/motion.mp4", { mediaType: "video/mp4" }),
Media.url("https://example.test/voice.mp3", { mediaType: "audio/mpeg" }),
],
})
.pipe(
Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-2" }))))),
)
expect(JSON.parse(calls[0].body)).toEqual({
model: "doubao-seedance-2-0-260128",
content: [
{ type: "text", text: "Match @Image1 to the motion of @Video1" },
{ type: "image_url", image_url: { url: "asset://asset-123" }, role: "reference_image" },
{ type: "image_url", image_url: { url: "https://example.test/style.png" }, role: "reference_image" },
{ type: "video_url", video_url: { url: "https://example.test/motion.mp4" }, role: "reference_video" },
{ type: "audio_url", audio_url: { url: "https://example.test/voice.mp3" }, role: "reference_audio" },
],
})
}),
)
it.effect("sends aspect auto as adaptive and leaves unset fields out", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
yield* Effect.gen(function* () {
yield* ai.video.start({ model, prompt: "x", aspect: "auto" })
yield* ai.video.start({ model, prompt: "x" })
}).pipe(
Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-3" }))))),
)
expect(calls.map((call) => JSON.parse(call.body))).toEqual([
{ model: "doubao-seedance-2-0-260128", content: [{ type: "text", text: "x" }], ratio: "adaptive" },
{ model: "doubao-seedance-2-0-260128", content: [{ type: "text", text: "x" }] },
])
}),
)
it.effect("sends frames and references together for the provider to judge", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
yield* ai.video
.start({
model,
prompt: "x",
start: "https://example.test/first.png",
references: [Media.url("https://example.test/ref.png", { mediaType: "image/png" })],
})
.pipe(
Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-4" }))))),
)
expect(JSON.parse(calls[0].body).content.map((item: { role?: string }) => item.role)).toEqual([
undefined,
"first_frame",
"reference_image",
])
}),
)
it.effect("adds providerOptions.draft_task as a draft_task content item and omits an empty prompt", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
yield* ai.video
.start({ model, prompt: "", providerOptions: { draft_task: { id: "cgt-draft" }, resolution: "720p" } })
.pipe(
Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-5" }))))),
)
expect(JSON.parse(calls[0].body)).toEqual({
model: "doubao-seedance-2-0-260128",
content: [{ type: "draft_task", draft_task: { id: "cgt-draft" } }],
resolution: "720p",
})
}),
)
it.effect("rejects a reference owned by another provider before sending", () =>
Effect.gen(function* () {
const error = yield* ai.video
.start({ model, prompt: "x", references: [Media.ref("xai", "file_1")] })
.pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("volcengine references")
}).pipe(Effect.provide(layer(() => Effect.die("a foreign reference reached the network")))),
)
it.effect("uses each provider's Ark host and its own provider ID", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
const byteplus = BytePlus.configure({ apiKey: "bp" }).video("dreamina-seedance-2-0-260128")
const volcengine = Volcengine.configure({ apiKey: "ark" }).video("doubao-seedance-2-0-260128")
yield* Effect.gen(function* () {
yield* ai.video.start({ model: byteplus, prompt: "x", references: [Media.ref("byteplus", "a1")] })
yield* ai.video.start({ model: volcengine, prompt: "x" })
}).pipe(
Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-6" }))))),
)
expect(byteplus.provider).toBe("byteplus")
expect(volcengine.provider).toBe("volcengine")
expect(calls.map((call) => [call.url, call.headers.get("authorization")])).toEqual([
["https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks", "Bearer bp"],
["https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", "Bearer ark"],
])
expect(JSON.parse(calls[0].body).content[1].image_url.url).toBe("asset://a1")
}),
)
})
// ---------------------------------------------------------------------------
// status, result, cancel
// ---------------------------------------------------------------------------
describe("ai.video.status / result / cancel / Seedance", () => {
it.effect("reads one status per call from a JSON round-tripped token", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
const statuses = yield* Effect.gen(function* () {
const job = yield* ai.video.start({ model, prompt: "x" })
const restored: unknown = JSON.parse(JSON.stringify(job.token))
return yield* Effect.forEach([1, 2, 3], () => ai.video.status(model, restored))
}).pipe(
Effect.provide(
layer((input) =>
Effect.gen(function* () {
const { call, nth } = yield* observe(calls, input)
if (call.method === "POST") return json(input, { id: "cgt-1" })
if (nth === 1) return json(input, { id: "cgt-1", status: "queued" })
if (nth === 2) return json(input, { id: "cgt-1", status: "running" })
return json(input, succeeded)
}),
),
),
)
expect(statuses).toEqual([{ status: "queued" }, { status: "running" }, { status: "completed" }])
expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([
"POST https://ark.test/api/v3/contents/generations/tasks",
"GET https://ark.test/api/v3/contents/generations/tasks/cgt-1",
"GET https://ark.test/api/v3/contents/generations/tasks/cgt-1",
"GET https://ark.test/api/v3/contents/generations/tasks/cgt-1",
])
}),
)
it.effect("maps every documented status and keeps unknown ones running", () =>
Effect.gen(function* () {
const raw = ["queued", "running", "succeeded", "failed", "cancelled", "expired", "paused", "constructor"]
const statuses = yield* Effect.forEach(raw, (status) =>
ai.video.status(model, token).pipe(Effect.provide(reply({ id: "cgt-1", status }))),
)
expect(statuses.map((value) => value.status)).toEqual([
"queued",
"running",
"completed",
"failed",
"cancelled",
"expired",
"running",
"running",
])
}),
)
it.effect("returns the video as a URL that expires 24 hours after the task finished, with usage and metadata", () =>
Effect.gen(function* () {
const response = yield* ai.video.result(model, token).pipe(Effect.provide(reply(succeeded)))
expect(response.video.source).toEqual({
type: "url",
url: "https://ark-content.test/out.mp4",
mediaType: "video/mp4",
expiresAt: (1779348874 + 24 * 60 * 60) * 1000,
})
expect(response.video.info).toEqual({ durationSeconds: 5 })
expect(response.usage).toEqual({ type: "tokens", output: 108900, total: 108900 })
expect(response.providerMetadata).toEqual({
volcengine: {
taskId: "cgt-1",
model: "doubao-seedance-2-0-260128",
seed: 78674,
ratio: "16:9",
resolution: "720p",
duration: 5,
framesPerSecond: 24,
generateAudio: true,
draft: false,
serviceTier: "default",
},
})
}),
)
it.effect("derives duration from frames, types mov output and keeps the last frame URL", () =>
Effect.gen(function* () {
const response = yield* ai.video.result(model, token).pipe(
Effect.provide(
reply({
status: "succeeded",
content: {
video_url: "https://ark-content.test/out.mov",
last_frame_url: "https://ark-content.test/l.jpg",
},
frames: 57,
framespersecond: 24,
output_format: "mov",
}),
),
)
expect(response.video.mediaType).toBe("video/quicktime")
expect(response.video.info).toEqual({ durationSeconds: 57 / 24 })
expect(response.video.expiresAt).toBeGreaterThan(0)
expect(response.providerMetadata?.volcengine).toMatchObject({
frames: 57,
outputFormat: "mov",
lastFrameUrl: "https://ark-content.test/l.jpg",
})
}),
)
it.effect("fails a result read before the task finishes", () =>
Effect.gen(function* () {
const errors = yield* Effect.forEach(["queued", "running"], (status) =>
ai.video.result(model, token).pipe(Effect.flip, Effect.provide(reply({ id: "cgt-1", status }))),
)
expect(errors.map((error) => error.reason._tag)).toEqual(["InvalidRequest", "InvalidRequest"])
expect(errors[0].message).toBe(
"Seedance Video generation cgt-1 has not finished; await it before reading the result",
)
}),
)
for (const terminal of [
{
body: {
status: "failed",
error: { code: "InvalidParameter.TaskTypeConstraint", message: "ratio must be adaptive." },
},
tag: "InvalidRequest",
message: "Seedance Video generation failed (InvalidParameter.TaskTypeConstraint): ratio must be adaptive.",
},
{
body: { status: "failed", error: { code: "AuthenticationError", message: "Bad key." } },
tag: "Authentication",
message: "Seedance Video generation failed (AuthenticationError): Bad key.",
},
{
body: { status: "failed", error: { code: "QuotaExceeded", message: "Too many queued tasks." } },
tag: "RateLimit",
message: "Seedance Video generation failed (QuotaExceeded): Too many queued tasks.",
},
{
body: { status: "failed", error: { code: "InternalServiceError", message: "Retry later." } },
tag: "ProviderInternal",
message: "Seedance Video generation failed (InternalServiceError): Retry later.",
},
{
body: { status: "failed", error: { code: "constructor", message: "Future code." } },
tag: "ProviderInternal",
message: "Seedance Video generation failed (constructor): Future code.",
},
{
body: {
status: "failed",
error: { code: "OutputVideoSensitiveContentDetected", message: "The output video may be sensitive." },
},
tag: "ContentPolicy",
message:
"Seedance Video generation failed (OutputVideoSensitiveContentDetected): The output video may be sensitive.",
},
{
body: {
status: "failed",
error: { code: "InputImageSensitiveContentDetected.PrivacyInformation", message: "Real person." },
},
tag: "ContentPolicy",
message: "Seedance Video generation failed (InputImageSensitiveContentDetected.PrivacyInformation): Real person.",
},
{ body: { status: "cancelled" }, tag: "InvalidRequest", message: "Seedance Video task cgt-1 was cancelled" },
{ body: { status: "expired" }, tag: "InvalidRequest", message: "Seedance Video task cgt-1 expired" },
{
body: { status: "succeeded", content: {} },
tag: "InvalidProviderOutput",
message: "Seedance Video completed without a video URL",
},
]) {
it.effect(`fails the result of ${terminal.body.error?.code ?? terminal.body.status} tasks with the body`, () =>
Effect.gen(function* () {
const error = yield* ai.video.result(model, token).pipe(Effect.flip, Effect.provide(reply(terminal.body)))
expect(error.reason._tag).toBe(terminal.tag)
expect(error.message).toContain(terminal.message)
expect(error.reason.body).toBe(JSON.stringify(terminal.body))
}),
)
}
it.effect("cancels with DELETE on the task and does not retry a refusal", () =>
Effect.gen(function* () {
const calls: Array<Call> = []
yield* ai.video
.cancel(model, token)
.pipe(Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, {}))))))
const refused = yield* ai.video
.cancel(model, token)
.pipe(
Effect.flip,
Effect.provide(
layer((input) =>
observe(calls, input).pipe(
Effect.map(() =>
json(input, { error: { code: "InvalidParameter", message: "running" } }, { status: 400 }),
),
),
),
),
)
expect(refused.reason._tag).toBe("InvalidRequest")
expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([
"DELETE https://ark.test/api/v3/contents/generations/tasks/cgt-1",
"DELETE https://ark.test/api/v3/contents/generations/tasks/cgt-1",
])
}),
)
it.effect("rejects a token from another provider without a request", () =>
Effect.gen(function* () {
const error = yield* ai.video.status(model, { requestID: "req_1" }).pipe(Effect.flip)
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("cannot resume")
}).pipe(Effect.provide(layer(() => Effect.die("a foreign token reached the network")))),
)
})
+2
View File
@@ -59,6 +59,7 @@ const builtins = new Map<string, () => Promise<unknown>>([
["@opencode/ai/providers/azure/messages", () => import("@opencode/ai/providers/azure/messages")],
["@opencode/ai/providers/baseten", () => import("@opencode/ai/providers/baseten")],
["@opencode/ai/providers/baseten/decisions", () => import("@opencode/ai/providers/baseten/decisions")],
["@opencode/ai/providers/byteplus/videos", () => import("@opencode/ai/providers/byteplus/videos")],
["@opencode/ai/providers/cerebras", () => import("@opencode/ai/providers/cerebras")],
["@opencode/ai/providers/cloudflare-ai-gateway", () => import("@opencode/ai/providers/cloudflare-ai-gateway")],
[
@@ -155,6 +156,7 @@ const builtins = new Map<string, () => Promise<unknown>>([
["@opencode/ai/providers/typesafe-ai/decisions", () => import("@opencode/ai/providers/typesafe-ai/decisions")],
["@opencode/ai/providers/venice", () => import("@opencode/ai/providers/venice")],
["@opencode/ai/providers/venice/responses", () => import("@opencode/ai/providers/venice/responses")],
["@opencode/ai/providers/volcengine/videos", () => import("@opencode/ai/providers/volcengine/videos")],
["@opencode/ai/providers/vercel-ai-gateway", () => import("@opencode/ai/providers/vercel-ai-gateway")],
["@opencode/ai/providers/vercel-ai-gateway/chat", () => import("@opencode/ai/providers/vercel-ai-gateway/chat")],
[