diff --git a/packages/ai/README.md b/packages/ai/README.md index dcd7af49f44..0f6584873a0 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1036,7 +1036,8 @@ The hosted result is represented as a provider-executed tool call and a tool res Video generation is a job: `start` submits it and returns at once, and the caller decides when to read its `status`, fetch its `result`, or `cancel` it. Nothing polls for you. Every call after `start` takes the model and the job's `token`, plain JSON you can store and read back from another process, so recovering after a restart is the same call -as a normal status read. Models come from `.video(...)` selectors on the `XAI` and `OpenRouter` facades. +as a normal status read. Models come from `.video(...)` selectors on the `XAI`, `OpenRouter`, `Volcengine` and +`BytePlus` facades. ```ts import { Effect } from "effect" @@ -1093,6 +1094,16 @@ Provider notes: rejects reference videos or audio that aren't http(s) URLs (inline images work). Routing and upstream parameters go under `providerOptions.provider` (`only`, `order`, `options: { : {...} }`), and `providerOptions.previous_job_id` edits or extends a finished job. Results are OpenRouter content URLs that download with the API key, which the asset carries; there is no cancel endpoint. +- **Seedance** (`Volcengine.configure(...).video(...)` with `ARK_API_KEY`, `BytePlus.configure(...).video(...)` with + `BYTEPLUS_API_KEY`) builds Ark's `content[]`: the prompt as a `text` item, `start` and `end` as `image_url` items + with roles `first_frame` and `last_frame`, and references as `reference_image`, `reference_video` or + `reference_audio` by kind. Your own `Media.ref(...)` assets go as `asset://`. `duration` goes to `duration`, + `aspect` to `ratio` (`auto` becomes `adaptive`), `resolution` to `resolution` and `sound` to `generate_audio`. Ark + doesn't allow frames and references in one task; that combination is sent and the provider's error comes back. + `providerOptions.draft_task: { id }` adds a `draft_task` item; `frames`, `seed`, `watermark`, `output_format`, + `omni_reference_task_type`, `return_last_frame` and the rest go into the body as-is. `cancel` sends `DELETE`, which + only cancels a queued task: Ark refuses it for a running one and deletes the record of a finished one. Video URLs + expire 24 hours after the task finished. The promise client has the same calls, each accepting `{ signal }`: diff --git a/packages/ai/package.json b/packages/ai/package.json index 4cfbe96258e..765f0b1f1ff 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -44,6 +44,8 @@ "./providers/azure/responses": "./src/providers/azure/responses.ts", "./providers/baseten": "./src/providers/baseten.ts", "./providers/baseten/decisions": "./src/providers/baseten/decisions.ts", + "./providers/byteplus": "./src/providers/byteplus.ts", + "./providers/byteplus/videos": "./src/providers/byteplus/videos.ts", "./providers/cartesia": "./src/providers/cartesia.ts", "./providers/cerebras": "./src/providers/cerebras.ts", "./providers/cloudflare-ai-gateway": "./src/providers/cloudflare-ai-gateway.ts", @@ -121,6 +123,8 @@ "./providers/typesafe-ai/decisions": "./src/providers/typesafe-ai/decisions.ts", "./providers/venice": "./src/providers/venice.ts", "./providers/venice/responses": "./src/providers/venice/responses.ts", + "./providers/volcengine": "./src/providers/volcengine.ts", + "./providers/volcengine/videos": "./src/providers/volcengine/videos.ts", "./providers/vercel-ai-gateway": "./src/providers/vercel-ai-gateway.ts", "./providers/vercel-ai-gateway/chat": "./src/providers/vercel-ai-gateway/chat.ts", "./providers/vercel-ai-gateway/decisions": "./src/providers/vercel-ai-gateway/decisions.ts", @@ -154,6 +158,7 @@ "./protocols/openai-responses": "./src/protocols/openai-responses.ts", "./protocols/openai-speech": "./src/protocols/openai-speech.ts", "./protocols/openai-transcription": "./src/protocols/openai-transcription.ts", + "./protocols/seedance-video": "./src/protocols/seedance-video.ts", "./protocols/system-one": "./src/protocols/system-one.ts", "./protocols/xai-images": "./src/protocols/xai-images.ts", "./protocols/xai-video": "./src/protocols/xai-video.ts", diff --git a/packages/ai/src/protocols/seedance-video.ts b/packages/ai/src/protocols/seedance-video.ts new file mode 100644 index 00000000000..beafc561aba --- /dev/null +++ b/packages/ai/src/protocols/seedance-video.ts @@ -0,0 +1,302 @@ +import { Duration, Effect, Schema } from "effect" +import type { HttpClientResponse } from "effect/http" +import type { Status } from "../generation.js" +import { Media } from "../media.js" +import { MediaProtocol } from "../route/media-protocol.js" +import { MediaRoute } from "../route/media.js" +import { mergeJsonRecords } from "../schema/index.js" +import { VideoModel, VideoResponse, seconds, type VideoRequest } from "../video.js" +import { ProviderShared, optionalNull } from "./shared.js" + +// One Ark API (`/contents/generations/tasks`) served by Volcengine Ark in China and BytePlus ModelArk elsewhere. +export const VOLCENGINE_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3" +export const BYTEPLUS_BASE_URL = "https://ark.ap-southeast.bytepluses.com/api/v3" +export const PATH = "/contents/generations/tasks" +// Video and last-frame URLs are documented as valid for 24 hours. +const OUTPUT_RETENTION = Duration.hours(24) + +// --------------------------------------------------------------------------- +// 1. Public model input +// --------------------------------------------------------------------------- + +/** + * Provider-native options, merged into the task body. Common fields (`start`, `end`, `references`, `duration`, + * `aspect`, `resolution`, `sound`) live on the request. `draft_task` adds a `draft_task` content item that turns a + * Draft video into the final one. + */ +export type SeedanceVideoOptions = { + readonly draft_task?: { readonly id: string } + readonly draft?: boolean + readonly frames?: number + readonly seed?: number + readonly camera_fixed?: boolean + readonly watermark?: boolean + readonly return_last_frame?: boolean + readonly omni_reference_task_type?: "auto" | "reference" | "edit" | "extend" | (string & {}) + readonly output_format?: "mp4" | "mov" | (string & {}) + readonly service_tier?: "default" | "flex" | (string & {}) + readonly callback_url?: string + readonly execution_expires_after?: number + readonly priority?: number + readonly safety_identifier?: string +} & Record + +export type Request = VideoRequest> + +// --------------------------------------------------------------------------- +// 2. Token and response schemas +// --------------------------------------------------------------------------- + +export const Token = Schema.Struct({ taskID: Schema.String }) +export type Token = Schema.Schema.Type + +const StartResponse = Schema.Struct({ id: Schema.String }) + +const Task = Schema.Struct({ + id: optionalNull(Schema.String), + model: optionalNull(Schema.String), + status: Schema.String, + content: optionalNull( + Schema.Struct({ + video_url: optionalNull(Schema.String), + last_frame_url: optionalNull(Schema.String), + }), + ), + error: optionalNull( + Schema.Struct({ + code: optionalNull(Schema.String), + message: optionalNull(Schema.String), + }), + ), + usage: optionalNull( + Schema.Struct({ + completion_tokens: optionalNull(Schema.Number), + total_tokens: optionalNull(Schema.Number), + }), + ), + updated_at: optionalNull(Schema.Number), + seed: optionalNull(Schema.Number), + resolution: optionalNull(Schema.String), + ratio: optionalNull(Schema.String), + duration: optionalNull(Schema.Number), + frames: optionalNull(Schema.Number), + framespersecond: optionalNull(Schema.Number), + generate_audio: optionalNull(Schema.Boolean), + output_format: optionalNull(Schema.String), + draft: optionalNull(Schema.Boolean), + service_tier: optionalNull(Schema.String), +}) + +const STATUS = { + queued: "queued", + running: "running", + succeeded: "completed", + failed: "failed", + cancelled: "cancelled", + expired: "expired", +} as const satisfies Record + +// --------------------------------------------------------------------------- +// 3. Failures +// --------------------------------------------------------------------------- + +// Ark error codes are dotted (`InvalidParameter.TaskTypeConstraint`); the leading segment is the category. +// `InternalServiceError`, `ServerOverloaded` and unknown codes are provider-side. +const FAILURE = { + InvalidParameter: "InvalidRequest", + MissingParameter: "InvalidRequest", + InvalidArgumentError: "InvalidRequest", + InvalidImageURL: "InvalidRequest", + AuthenticationError: "Authentication", + AccessDenied: "Authentication", + QuotaExceeded: "RateLimit", + RateLimitExceeded: "RateLimit", +} as const satisfies Record + +// Input and output moderation codes: `InputImageSensitiveContentDetected.PrivacyInformation`, +// `OutputVideoSensitiveContentDetected`, `SensitiveContentDetected`, ... +const moderated = (code: string | undefined) => code !== undefined && code.includes("SensitiveContentDetected") + +// --------------------------------------------------------------------------- +// 4. Request body construction +// --------------------------------------------------------------------------- + +const nativeOptions = (options: SeedanceVideoOptions | undefined) => { + if (!options) return undefined + const { draft_task: _draftTask, ...native } = options + return native +} + +// --------------------------------------------------------------------------- +// 5. Response decoding +// --------------------------------------------------------------------------- + +// Ark returns `frames` when the request set it, otherwise `duration` rounded down to whole seconds. +const durationSeconds = (task: Schema.Schema.Type) => { + if (task.frames !== undefined && task.frames !== null && task.framespersecond) + return task.frames / task.framespersecond + return task.duration ?? undefined +} + +// --------------------------------------------------------------------------- +// 6. Protocol and model, one per provider so assets and metadata carry the right provider ID +// --------------------------------------------------------------------------- + +export type Provider = "volcengine" | "byteplus" + +const make = (provider: Provider) => { + const route = MediaProtocol.identity({ id: "seedance-video", name: "Seedance Video", provider }) + + // `asset://` addresses a ModelArk digital character or authorized asset. + const item = (asset: Media.Asset, type: "image_url" | "video_url" | "audio_url", role: string) => + ProviderShared.mediaReference(asset, route.provider, route.name).pipe( + Effect.map( + (reference): Record => ({ + type, + [type]: { url: reference.type === "ref" ? `asset://${reference.value}` : reference.value }, + role, + }), + ), + ) + + const reference = (asset: Media.Asset) => { + if (asset.kind === "video") return item(asset, "video_url", "reference_video") + if (asset.kind === "audio") return item(asset, "audio_url", "reference_audio") + return item(asset, "image_url", "reference_image") + } + + // Frames and references are separate Seedance modes; mixing them is sent as-is and rejected by the provider. + const fromRequest = Effect.fn("SeedanceVideo.fromRequest")(function* (request: Request) { + const content = [ + // Every media mode accepts an optional prompt, so an empty one adds no text item. + ...(request.prompt === "" ? [] : [{ type: "text", text: request.prompt }]), + ...(request.start === undefined ? [] : [yield* item(request.start, "image_url", "first_frame")]), + ...(request.end === undefined ? [] : [yield* item(request.end, "image_url", "last_frame")]), + ...(yield* Effect.forEach(request.references ?? [], reference)), + ...(request.providerOptions?.draft_task === undefined + ? [] + : [{ type: "draft_task", draft_task: request.providerOptions.draft_task }]), + ] + return MediaProtocol.json( + mergeJsonRecords( + { + model: request.model.id, + content, + duration: request.duration === undefined ? undefined : seconds(request.duration), + ratio: request.aspect === "auto" ? "adaptive" : request.aspect, + resolution: request.resolution, + generate_audio: request.sound, + }, + nativeOptions(request.providerOptions), + request.http?.body, + ) ?? {}, + ) + }) + + const decodeStart = route.decodeStarted(StartResponse, (value) => ({ + token: { taskID: value.id }, + snapshot: { id: value.id, status: "queued" }, + })) + + const decodeTask = route.decodeJson(Task) + + const decodeStatus = Effect.fn("SeedanceVideo.decodeStatus")(function* ( + response: HttpClientResponse.HttpClientResponse, + context: MediaProtocol.PollContext, + ) { + const output = yield* decodeTask(response) + return { id: context.token.taskID, status: MediaProtocol.status(STATUS, output.value.status) } + }) + + const decodeResult = Effect.fn("SeedanceVideo.decodeResult")(function* ( + response: HttpClientResponse.HttpClientResponse, + context: MediaProtocol.PollContext, + ) { + const output = yield* decodeTask(response) + const task = output.value + const status = MediaProtocol.status(STATUS, task.status) + if (status === "queued" || status === "running") return yield* output.pending(context.token.taskID) + if (status === "failed") { + const code = task.error?.code ?? undefined + const detail = task.error?.message ?? undefined + const message = `${route.name} generation failed${code === undefined ? "" : ` (${code})`}${detail === undefined ? "" : `: ${detail}`}` + if (moderated(code)) return yield* output.contentPolicy(message) + return yield* output.ended("failed", message, MediaProtocol.failure(FAILURE, code?.split(".")[0])) + } + if (status === "cancelled") + return yield* output.ended("cancelled", `${route.name} task ${context.token.taskID} was cancelled`) + if (status === "expired") + return yield* output.ended("expired", `${route.name} task ${context.token.taskID} expired`) + const videoURL = task.content?.video_url ?? undefined + if (videoURL === undefined) return yield* output.invalid(`${route.name} completed without a video URL`) + const duration = durationSeconds(task) + const options = { + mediaType: task.output_format === "mov" ? "video/quicktime" : "video/mp4", + info: duration === undefined ? undefined : { durationSeconds: duration }, + } + // `updated_at` is when the task reached `succeeded`, which starts the URL's retention window. + const video = + task.updated_at === undefined || task.updated_at === null + ? yield* MediaProtocol.expiringUrl(videoURL, OUTPUT_RETENTION, options) + : Media.url(videoURL, { + ...options, + expiresAt: task.updated_at * 1000 + Duration.toMillis(OUTPUT_RETENTION), + }) + return new VideoResponse({ + videos: [video], + usage: + task.usage?.completion_tokens === undefined || task.usage.completion_tokens === null + ? undefined + : { + type: "tokens", + output: task.usage.completion_tokens, + total: task.usage.total_tokens ?? undefined, + }, + providerMetadata: { + [provider]: { + taskId: context.token.taskID, + model: task.model ?? undefined, + seed: task.seed ?? undefined, + ratio: task.ratio ?? undefined, + resolution: task.resolution ?? undefined, + duration: task.duration ?? undefined, + frames: task.frames ?? undefined, + framesPerSecond: task.framespersecond ?? undefined, + generateAudio: task.generate_audio ?? undefined, + outputFormat: task.output_format ?? undefined, + draft: task.draft ?? undefined, + serviceTier: task.service_tier ?? undefined, + lastFrameUrl: task.content?.last_frame_url ?? undefined, + }, + }, + }) + }) + + const taskPath = (token: Token) => `${PATH}/${token.taskID}` + + // DELETE cancels a queued task. Ark refuses it for a running task, and deletes the record of a finished one. + const protocol = MediaProtocol.queued(route, { + token: Token, + start: { body: { from: fromRequest }, decode: decodeStart }, + status: { path: taskPath, decode: decodeStatus }, + result: { path: taskPath, decode: decodeResult }, + cancel: { method: "DELETE", path: taskPath }, + }) + + const model = (input: MediaRoute.ModelInput) => + VideoModel.fromRoute( + { + protocol, + baseURL: provider === "volcengine" ? VOLCENGINE_BASE_URL : BYTEPLUS_BASE_URL, + path: PATH, + }, + input, + ) + + return { protocol, model } +} + +export const SeedanceVideo = { + volcengine: make("volcengine"), + byteplus: make("byteplus"), +} as const diff --git a/packages/ai/src/providers/byteplus.ts b/packages/ai/src/providers/byteplus.ts new file mode 100644 index 00000000000..b710bfe1eff --- /dev/null +++ b/packages/ai/src/providers/byteplus.ts @@ -0,0 +1,40 @@ +import type { ProviderPackage } from "../provider-package.js" +import { SeedanceVideo } from "../protocols/seedance-video.js" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" +import { MediaRoute } from "../route/media.js" +import { type HttpOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js" +import { toConfig } from "./settings.js" + +export const id = ProviderID.make("byteplus") + +export type Config = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions +} + +export type Settings = ProviderPackage.Settings & { + readonly apiKey?: string | false + readonly baseURL?: string +} + +export type { SeedanceVideoOptions } from "../protocols/seedance-video.js" + +const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "BYTEPLUS_API_KEY") + +export const configure = (input: Config = {}) => { + const media = MediaRoute.deployment(input, auth(input)) + return { + id, + video: (modelID: string | ModelID, providerOptions?: ProviderOptions) => + SeedanceVideo.byteplus.model({ ...media, id: modelID, providerOptions }), + configure, + } +} + +export const provider = configure() +export const videoModel = (modelID: string, settings?: Settings) => { + const input = toConfig(settings) + return configure(input).video(modelID, input.providerOptions) +} +export const video = provider.video diff --git a/packages/ai/src/providers/byteplus/videos.ts b/packages/ai/src/providers/byteplus/videos.ts new file mode 100644 index 00000000000..f51bdec3cbe --- /dev/null +++ b/packages/ai/src/providers/byteplus/videos.ts @@ -0,0 +1,2 @@ +export { videoModel as model } from "../byteplus.js" +export type { Settings } from "../byteplus.js" diff --git a/packages/ai/src/providers/index.ts b/packages/ai/src/providers/index.ts index a9ac040a4e9..24d68030300 100644 --- a/packages/ai/src/providers/index.ts +++ b/packages/ai/src/providers/index.ts @@ -6,6 +6,7 @@ export * as AmazonBedrockMantle from "./amazon-bedrock-mantle.js" export * as AssemblyAI from "./assemblyai.js" export * as Azure from "./azure.js" export * as Baseten from "./baseten.js" +export * as BytePlus from "./byteplus.js" export * as Cartesia from "./cartesia.js" export * as Cerebras from "./cerebras.js" export * as Cohere from "./cohere.js" @@ -38,6 +39,7 @@ export * as TogetherAI from "./togetherai.js" export * as TypeSafeAI from "./typesafe-ai.js" export * as VercelAIGateway from "./vercel-ai-gateway.js" export * as Venice from "./venice.js" +export * as Volcengine from "./volcengine.js" export * as XAI from "./xai.js" export * as ZAI from "./zai.js" export * as ZAICodingPlan from "./zai-coding-plan.js" diff --git a/packages/ai/src/providers/volcengine.ts b/packages/ai/src/providers/volcengine.ts new file mode 100644 index 00000000000..935e8bfece5 --- /dev/null +++ b/packages/ai/src/providers/volcengine.ts @@ -0,0 +1,40 @@ +import type { ProviderPackage } from "../provider-package.js" +import { SeedanceVideo } from "../protocols/seedance-video.js" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" +import { MediaRoute } from "../route/media.js" +import { type HttpOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js" +import { toConfig } from "./settings.js" + +export const id = ProviderID.make("volcengine") + +export type Config = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions +} + +export type Settings = ProviderPackage.Settings & { + readonly apiKey?: string | false + readonly baseURL?: string +} + +export type { SeedanceVideoOptions } from "../protocols/seedance-video.js" + +const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "ARK_API_KEY") + +export const configure = (input: Config = {}) => { + const media = MediaRoute.deployment(input, auth(input)) + return { + id, + video: (modelID: string | ModelID, providerOptions?: ProviderOptions) => + SeedanceVideo.volcengine.model({ ...media, id: modelID, providerOptions }), + configure, + } +} + +export const provider = configure() +export const videoModel = (modelID: string, settings?: Settings) => { + const input = toConfig(settings) + return configure(input).video(modelID, input.providerOptions) +} +export const video = provider.video diff --git a/packages/ai/src/providers/volcengine/videos.ts b/packages/ai/src/providers/volcengine/videos.ts new file mode 100644 index 00000000000..fcfeec1c1dc --- /dev/null +++ b/packages/ai/src/providers/volcengine/videos.ts @@ -0,0 +1,2 @@ +export { videoModel as model } from "../volcengine.js" +export type { Settings } from "../volcengine.js" diff --git a/packages/ai/test/provider/media-entrypoints.test.ts b/packages/ai/test/provider/media-entrypoints.test.ts index 4a860a69801..fb00a8ac75c 100644 --- a/packages/ai/test/provider/media-entrypoints.test.ts +++ b/packages/ai/test/provider/media-entrypoints.test.ts @@ -513,6 +513,26 @@ const videos: ReadonlyArray> = [ auth: ["authorization", "Bearer openrouter-key"], provider: "openrouter", }, + { + name: "byteplus", + load: async () => { + const { model } = await import("@opencode/ai/providers/byteplus/videos") + return model("dreamina-seedance-2-0-260128", { ...shared, apiKey: "byteplus-key" }) + }, + url: "https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks", + auth: ["authorization", "Bearer byteplus-key"], + provider: "byteplus", + }, + { + name: "volcengine", + load: async () => { + const { model } = await import("@opencode/ai/providers/volcengine/videos") + return model("doubao-seedance-2-0-260128", { ...shared, apiKey: "ark-key" }) + }, + url: "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", + auth: ["authorization", "Bearer ark-key"], + provider: "volcengine", + }, ] describe("video entrypoints", () => { diff --git a/packages/ai/test/provider/seedance-video.recorded.test.ts b/packages/ai/test/provider/seedance-video.recorded.test.ts new file mode 100644 index 00000000000..669abe15051 --- /dev/null +++ b/packages/ai/test/provider/seedance-video.recorded.test.ts @@ -0,0 +1,52 @@ +import { describe, expect } from "bun:test" +import { Effect, Schedule } from "effect" +import { ai, type VideoStatus } from "../../src/effect.js" +import { BytePlus } from "../../src/providers.js" +import { recordedTests } from "../recorded-test.js" +import { queuedPoll } from "./queued-recording.js" + +// Seedance 2.0 mini is the cheapest 2.x model; 480p and 4 seconds are the lowest settings it accepts. +const model = BytePlus.configure({ + apiKey: process.env.BYTEPLUS_API_KEY ?? "fixture", +}).video("dreamina-seedance-2-0-mini-260615") + +const recorded = recordedTests({ + prefix: "seedance-video", + provider: "byteplus", + protocol: "seedance-video", + requires: ["BYTEPLUS_API_KEY"], +}) + +describe("Seedance Video recorded", () => { + recorded.effect( + "generates a video", + () => + Effect.gen(function* () { + const job = yield* ai.video.start({ + model, + prompt: "A single red balloon drifting slowly upward against a clear blue sky.", + aspect: "16:9", + resolution: "480p", + duration: "4 seconds", + sound: false, + }) + const observed: Array = [] + yield* ai.video.status(model, job.token).pipe( + Effect.tap((status) => Effect.sync(() => observed.push(status))), + Effect.repeat({ + schedule: Schedule.spaced(queuedPoll.interval ?? "10 millis"), + until: (status) => status.status !== "queued" && status.status !== "running", + }), + ) + expect(observed.at(-1)?.status).toBe("completed") + + const response = yield* ai.video.result(model, job.token) + expect(response.videos).toHaveLength(1) + expect(response.video.source.type).toBe("url") + expect(response.video.expiresAt).toBeGreaterThan(0) + expect(response.usage?.type).toBe("tokens") + expect((yield* response.video.bytes()).length).toBeGreaterThan(0) + }), + { timeout: 15 * 60 * 1000 }, + ) +}) diff --git a/packages/ai/test/seedance-video.test.ts b/packages/ai/test/seedance-video.test.ts new file mode 100644 index 00000000000..aa677c8739e --- /dev/null +++ b/packages/ai/test/seedance-video.test.ts @@ -0,0 +1,426 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import { Media, VideoClient, ai } from "../src/effect.js" +import { BytePlus, Volcengine } from "../src/providers.js" +import { it } from "./lib/effect.js" +import { dynamicResponse, json, observe, type Call } from "./lib/http.js" + +const layer = (handler: Parameters[0]) => + VideoClient.layer.pipe(Layer.provideMerge(dynamicResponse(handler))) + +const model = Volcengine.configure({ apiKey: "test", baseURL: "https://ark.test/api/v3" }).video( + "doubao-seedance-2-0-260128", +) +const token = { taskID: "cgt-1" } +const succeeded = { + id: "cgt-1", + model: "doubao-seedance-2-0-260128", + status: "succeeded", + content: { video_url: "https://ark-content.test/out.mp4" }, + usage: { completion_tokens: 108900, total_tokens: 108900 }, + created_at: 1779348818, + updated_at: 1779348874, + seed: 78674, + resolution: "720p", + ratio: "16:9", + duration: 5, + framespersecond: 24, + service_tier: "default", + generate_audio: true, + draft: false, +} + +const reply = (body: unknown) => layer((input) => Effect.succeed(json(input, body))) + +// --------------------------------------------------------------------------- +// start +// --------------------------------------------------------------------------- + +describe("ai.video.start / Seedance", () => { + it.effect("maps frames, shared fields and provider options onto Ark's content[] body", () => + Effect.gen(function* () { + const calls: Array = [] + const job = yield* ai.video + .start({ + model, + prompt: "The kitten yawns at the camera", + start: Media.fromDataUrl("data:image/png;base64,AQID"), + end: "https://example.test/last.png", + duration: "5 seconds", + aspect: "16:9", + resolution: "720p", + sound: false, + providerOptions: { watermark: true, return_last_frame: true, future_option: 1 }, + headers: { "x-request": "yes" }, + http: { query: { trace: "1" } }, + }) + .pipe( + Effect.provide( + layer((input) => + Effect.gen(function* () { + const { call } = yield* observe(calls, input) + expect(call.headers.get("authorization")).toBe("Bearer test") + expect(call.headers.get("x-request")).toBe("yes") + return json(input, { id: "cgt-1" }) + }), + ), + ), + ) + expect(job).toEqual({ id: "cgt-1", token: { taskID: "cgt-1" } }) + expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([ + "POST https://ark.test/api/v3/contents/generations/tasks?trace=1", + ]) + expect(JSON.parse(calls[0].body)).toEqual({ + model: "doubao-seedance-2-0-260128", + content: [ + { type: "text", text: "The kitten yawns at the camera" }, + { type: "image_url", image_url: { url: "data:image/png;base64,AQID" }, role: "first_frame" }, + { type: "image_url", image_url: { url: "https://example.test/last.png" }, role: "last_frame" }, + ], + duration: 5, + ratio: "16:9", + resolution: "720p", + generate_audio: false, + watermark: true, + return_last_frame: true, + future_option: 1, + }) + }), + ) + + it.effect("maps image, video and audio references to their roles, with own assets as asset:// URIs", () => + Effect.gen(function* () { + const calls: Array = [] + yield* ai.video + .start({ + model, + prompt: "Match @Image1 to the motion of @Video1", + references: [ + Media.ref("volcengine", "asset-123"), + Media.url("https://example.test/style.png", { mediaType: "image/png" }), + Media.url("https://example.test/motion.mp4", { mediaType: "video/mp4" }), + Media.url("https://example.test/voice.mp3", { mediaType: "audio/mpeg" }), + ], + }) + .pipe( + Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-2" }))))), + ) + expect(JSON.parse(calls[0].body)).toEqual({ + model: "doubao-seedance-2-0-260128", + content: [ + { type: "text", text: "Match @Image1 to the motion of @Video1" }, + { type: "image_url", image_url: { url: "asset://asset-123" }, role: "reference_image" }, + { type: "image_url", image_url: { url: "https://example.test/style.png" }, role: "reference_image" }, + { type: "video_url", video_url: { url: "https://example.test/motion.mp4" }, role: "reference_video" }, + { type: "audio_url", audio_url: { url: "https://example.test/voice.mp3" }, role: "reference_audio" }, + ], + }) + }), + ) + + it.effect("sends aspect auto as adaptive and leaves unset fields out", () => + Effect.gen(function* () { + const calls: Array = [] + yield* Effect.gen(function* () { + yield* ai.video.start({ model, prompt: "x", aspect: "auto" }) + yield* ai.video.start({ model, prompt: "x" }) + }).pipe( + Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-3" }))))), + ) + expect(calls.map((call) => JSON.parse(call.body))).toEqual([ + { model: "doubao-seedance-2-0-260128", content: [{ type: "text", text: "x" }], ratio: "adaptive" }, + { model: "doubao-seedance-2-0-260128", content: [{ type: "text", text: "x" }] }, + ]) + }), + ) + + it.effect("sends frames and references together for the provider to judge", () => + Effect.gen(function* () { + const calls: Array = [] + yield* ai.video + .start({ + model, + prompt: "x", + start: "https://example.test/first.png", + references: [Media.url("https://example.test/ref.png", { mediaType: "image/png" })], + }) + .pipe( + Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-4" }))))), + ) + expect(JSON.parse(calls[0].body).content.map((item: { role?: string }) => item.role)).toEqual([ + undefined, + "first_frame", + "reference_image", + ]) + }), + ) + + it.effect("adds providerOptions.draft_task as a draft_task content item and omits an empty prompt", () => + Effect.gen(function* () { + const calls: Array = [] + yield* ai.video + .start({ model, prompt: "", providerOptions: { draft_task: { id: "cgt-draft" }, resolution: "720p" } }) + .pipe( + Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-5" }))))), + ) + expect(JSON.parse(calls[0].body)).toEqual({ + model: "doubao-seedance-2-0-260128", + content: [{ type: "draft_task", draft_task: { id: "cgt-draft" } }], + resolution: "720p", + }) + }), + ) + + it.effect("rejects a reference owned by another provider before sending", () => + Effect.gen(function* () { + const error = yield* ai.video + .start({ model, prompt: "x", references: [Media.ref("xai", "file_1")] }) + .pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("volcengine references") + }).pipe(Effect.provide(layer(() => Effect.die("a foreign reference reached the network")))), + ) + + it.effect("uses each provider's Ark host and its own provider ID", () => + Effect.gen(function* () { + const calls: Array = [] + const byteplus = BytePlus.configure({ apiKey: "bp" }).video("dreamina-seedance-2-0-260128") + const volcengine = Volcengine.configure({ apiKey: "ark" }).video("doubao-seedance-2-0-260128") + yield* Effect.gen(function* () { + yield* ai.video.start({ model: byteplus, prompt: "x", references: [Media.ref("byteplus", "a1")] }) + yield* ai.video.start({ model: volcengine, prompt: "x" }) + }).pipe( + Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, { id: "cgt-6" }))))), + ) + expect(byteplus.provider).toBe("byteplus") + expect(volcengine.provider).toBe("volcengine") + expect(calls.map((call) => [call.url, call.headers.get("authorization")])).toEqual([ + ["https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks", "Bearer bp"], + ["https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", "Bearer ark"], + ]) + expect(JSON.parse(calls[0].body).content[1].image_url.url).toBe("asset://a1") + }), + ) +}) + +// --------------------------------------------------------------------------- +// status, result, cancel +// --------------------------------------------------------------------------- + +describe("ai.video.status / result / cancel / Seedance", () => { + it.effect("reads one status per call from a JSON round-tripped token", () => + Effect.gen(function* () { + const calls: Array = [] + const statuses = yield* Effect.gen(function* () { + const job = yield* ai.video.start({ model, prompt: "x" }) + const restored: unknown = JSON.parse(JSON.stringify(job.token)) + return yield* Effect.forEach([1, 2, 3], () => ai.video.status(model, restored)) + }).pipe( + Effect.provide( + layer((input) => + Effect.gen(function* () { + const { call, nth } = yield* observe(calls, input) + if (call.method === "POST") return json(input, { id: "cgt-1" }) + if (nth === 1) return json(input, { id: "cgt-1", status: "queued" }) + if (nth === 2) return json(input, { id: "cgt-1", status: "running" }) + return json(input, succeeded) + }), + ), + ), + ) + expect(statuses).toEqual([{ status: "queued" }, { status: "running" }, { status: "completed" }]) + expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([ + "POST https://ark.test/api/v3/contents/generations/tasks", + "GET https://ark.test/api/v3/contents/generations/tasks/cgt-1", + "GET https://ark.test/api/v3/contents/generations/tasks/cgt-1", + "GET https://ark.test/api/v3/contents/generations/tasks/cgt-1", + ]) + }), + ) + + it.effect("maps every documented status and keeps unknown ones running", () => + Effect.gen(function* () { + const raw = ["queued", "running", "succeeded", "failed", "cancelled", "expired", "paused", "constructor"] + const statuses = yield* Effect.forEach(raw, (status) => + ai.video.status(model, token).pipe(Effect.provide(reply({ id: "cgt-1", status }))), + ) + expect(statuses.map((value) => value.status)).toEqual([ + "queued", + "running", + "completed", + "failed", + "cancelled", + "expired", + "running", + "running", + ]) + }), + ) + + it.effect("returns the video as a URL that expires 24 hours after the task finished, with usage and metadata", () => + Effect.gen(function* () { + const response = yield* ai.video.result(model, token).pipe(Effect.provide(reply(succeeded))) + expect(response.video.source).toEqual({ + type: "url", + url: "https://ark-content.test/out.mp4", + mediaType: "video/mp4", + expiresAt: (1779348874 + 24 * 60 * 60) * 1000, + }) + expect(response.video.info).toEqual({ durationSeconds: 5 }) + expect(response.usage).toEqual({ type: "tokens", output: 108900, total: 108900 }) + expect(response.providerMetadata).toEqual({ + volcengine: { + taskId: "cgt-1", + model: "doubao-seedance-2-0-260128", + seed: 78674, + ratio: "16:9", + resolution: "720p", + duration: 5, + framesPerSecond: 24, + generateAudio: true, + draft: false, + serviceTier: "default", + }, + }) + }), + ) + + it.effect("derives duration from frames, types mov output and keeps the last frame URL", () => + Effect.gen(function* () { + const response = yield* ai.video.result(model, token).pipe( + Effect.provide( + reply({ + status: "succeeded", + content: { + video_url: "https://ark-content.test/out.mov", + last_frame_url: "https://ark-content.test/l.jpg", + }, + frames: 57, + framespersecond: 24, + output_format: "mov", + }), + ), + ) + expect(response.video.mediaType).toBe("video/quicktime") + expect(response.video.info).toEqual({ durationSeconds: 57 / 24 }) + expect(response.video.expiresAt).toBeGreaterThan(0) + expect(response.providerMetadata?.volcengine).toMatchObject({ + frames: 57, + outputFormat: "mov", + lastFrameUrl: "https://ark-content.test/l.jpg", + }) + }), + ) + + it.effect("fails a result read before the task finishes", () => + Effect.gen(function* () { + const errors = yield* Effect.forEach(["queued", "running"], (status) => + ai.video.result(model, token).pipe(Effect.flip, Effect.provide(reply({ id: "cgt-1", status }))), + ) + expect(errors.map((error) => error.reason._tag)).toEqual(["InvalidRequest", "InvalidRequest"]) + expect(errors[0].message).toBe( + "Seedance Video generation cgt-1 has not finished; await it before reading the result", + ) + }), + ) + + for (const terminal of [ + { + body: { + status: "failed", + error: { code: "InvalidParameter.TaskTypeConstraint", message: "ratio must be adaptive." }, + }, + tag: "InvalidRequest", + message: "Seedance Video generation failed (InvalidParameter.TaskTypeConstraint): ratio must be adaptive.", + }, + { + body: { status: "failed", error: { code: "AuthenticationError", message: "Bad key." } }, + tag: "Authentication", + message: "Seedance Video generation failed (AuthenticationError): Bad key.", + }, + { + body: { status: "failed", error: { code: "QuotaExceeded", message: "Too many queued tasks." } }, + tag: "RateLimit", + message: "Seedance Video generation failed (QuotaExceeded): Too many queued tasks.", + }, + { + body: { status: "failed", error: { code: "InternalServiceError", message: "Retry later." } }, + tag: "ProviderInternal", + message: "Seedance Video generation failed (InternalServiceError): Retry later.", + }, + { + body: { status: "failed", error: { code: "constructor", message: "Future code." } }, + tag: "ProviderInternal", + message: "Seedance Video generation failed (constructor): Future code.", + }, + { + body: { + status: "failed", + error: { code: "OutputVideoSensitiveContentDetected", message: "The output video may be sensitive." }, + }, + tag: "ContentPolicy", + message: + "Seedance Video generation failed (OutputVideoSensitiveContentDetected): The output video may be sensitive.", + }, + { + body: { + status: "failed", + error: { code: "InputImageSensitiveContentDetected.PrivacyInformation", message: "Real person." }, + }, + tag: "ContentPolicy", + message: "Seedance Video generation failed (InputImageSensitiveContentDetected.PrivacyInformation): Real person.", + }, + { body: { status: "cancelled" }, tag: "InvalidRequest", message: "Seedance Video task cgt-1 was cancelled" }, + { body: { status: "expired" }, tag: "InvalidRequest", message: "Seedance Video task cgt-1 expired" }, + { + body: { status: "succeeded", content: {} }, + tag: "InvalidProviderOutput", + message: "Seedance Video completed without a video URL", + }, + ]) { + it.effect(`fails the result of ${terminal.body.error?.code ?? terminal.body.status} tasks with the body`, () => + Effect.gen(function* () { + const error = yield* ai.video.result(model, token).pipe(Effect.flip, Effect.provide(reply(terminal.body))) + expect(error.reason._tag).toBe(terminal.tag) + expect(error.message).toContain(terminal.message) + expect(error.reason.body).toBe(JSON.stringify(terminal.body)) + }), + ) + } + + it.effect("cancels with DELETE on the task and does not retry a refusal", () => + Effect.gen(function* () { + const calls: Array = [] + yield* ai.video + .cancel(model, token) + .pipe(Effect.provide(layer((input) => observe(calls, input).pipe(Effect.map(() => json(input, {})))))) + const refused = yield* ai.video + .cancel(model, token) + .pipe( + Effect.flip, + Effect.provide( + layer((input) => + observe(calls, input).pipe( + Effect.map(() => + json(input, { error: { code: "InvalidParameter", message: "running" } }, { status: 400 }), + ), + ), + ), + ), + ) + expect(refused.reason._tag).toBe("InvalidRequest") + expect(calls.map((call) => `${call.method} ${call.url}`)).toEqual([ + "DELETE https://ark.test/api/v3/contents/generations/tasks/cgt-1", + "DELETE https://ark.test/api/v3/contents/generations/tasks/cgt-1", + ]) + }), + ) + + it.effect("rejects a token from another provider without a request", () => + Effect.gen(function* () { + const error = yield* ai.video.status(model, { requestID: "req_1" }).pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("cannot resume") + }).pipe(Effect.provide(layer(() => Effect.die("a foreign token reached the network")))), + ) +}) diff --git a/packages/core/src/provider.ts b/packages/core/src/provider.ts index f8c42351e28..98d1ab3d692 100644 --- a/packages/core/src/provider.ts +++ b/packages/core/src/provider.ts @@ -59,6 +59,7 @@ const builtins = new Map Promise>([ ["@opencode/ai/providers/azure/messages", () => import("@opencode/ai/providers/azure/messages")], ["@opencode/ai/providers/baseten", () => import("@opencode/ai/providers/baseten")], ["@opencode/ai/providers/baseten/decisions", () => import("@opencode/ai/providers/baseten/decisions")], + ["@opencode/ai/providers/byteplus/videos", () => import("@opencode/ai/providers/byteplus/videos")], ["@opencode/ai/providers/cerebras", () => import("@opencode/ai/providers/cerebras")], ["@opencode/ai/providers/cloudflare-ai-gateway", () => import("@opencode/ai/providers/cloudflare-ai-gateway")], [ @@ -155,6 +156,7 @@ const builtins = new Map Promise>([ ["@opencode/ai/providers/typesafe-ai/decisions", () => import("@opencode/ai/providers/typesafe-ai/decisions")], ["@opencode/ai/providers/venice", () => import("@opencode/ai/providers/venice")], ["@opencode/ai/providers/venice/responses", () => import("@opencode/ai/providers/venice/responses")], + ["@opencode/ai/providers/volcengine/videos", () => import("@opencode/ai/providers/volcengine/videos")], ["@opencode/ai/providers/vercel-ai-gateway", () => import("@opencode/ai/providers/vercel-ai-gateway")], ["@opencode/ai/providers/vercel-ai-gateway/chat", () => import("@opencode/ai/providers/vercel-ai-gateway/chat")], [