diff --git a/packages/core/src/config/plugin/skill.ts b/packages/core/src/config/plugin/skill.ts index 17837bcc129..c052172893f 100644 --- a/packages/core/src/config/plugin/skill.ts +++ b/packages/core/src/config/plugin/skill.ts @@ -28,10 +28,7 @@ export const Plugin = define({ const global = yield* Global.Service const location = yield* Location.Service const watcher = yield* Watcher.Service - const loaded: { entries: Entry[]; skills: Skill.Info[] } = { - entries: yield* config.entries(), - skills: [], - } + const loaded: { entries: Entry[]; skills: Skill.Info[] } = { entries: [], skills: [] } const watches = yield* FiberMap.make() const changes = yield* PubSub.sliding(1) const lock = Semaphore.makeUnsafe(1) @@ -191,10 +188,8 @@ export const Plugin = define({ Stream.runForEach((file) => refresh(file).pipe(Effect.andThen(ctx.skill.reload()))), Effect.forkScoped({ startImmediately: true }), ) - yield* refresh() - yield* ctx.skill.transform((editor) => { - for (const skill of loaded.skills) editor.add(skill) - }) + // Subscribe before the first config read so a config published while the first load runs, such as one an + // integration delivers, still triggers a reload. yield* ctx.event.subscribe({ types: ["config.updated"] }).pipe( Stream.runForEach(() => config.entries().pipe( @@ -205,5 +200,10 @@ export const Plugin = define({ ), Effect.forkScoped({ startImmediately: true }), ) + loaded.entries = yield* config.entries() + yield* refresh() + yield* ctx.skill.transform((editor) => { + for (const skill of loaded.skills) editor.add(skill) + }) }), }) diff --git a/packages/core/src/remote-http.ts b/packages/core/src/remote-http.ts index 80af015d60d..65facf0cb79 100644 --- a/packages/core/src/remote-http.ts +++ b/packages/core/src/remote-http.ts @@ -3,6 +3,7 @@ export * as RemoteHttp from "./remote-http.js" import { Effect, Scope, Stream } from "effect" import { FetchHttpClient, + Headers, HttpBody, HttpClient, HttpClientError, @@ -30,7 +31,12 @@ export function follow( request: HttpClientRequest.HttpClientRequest, sensitive: Iterable = [], ) { - return hop(http, request, new Set([...CREDENTIAL_HEADERS, ...Array.from(sensitive, (name) => name.toLowerCase())]), 0) + const strip = new Set([...CREDENTIAL_HEADERS, ...Array.from(sensitive, (name) => name.toLowerCase())]) + // The client records request headers on its trace spans, so every header that must not leave the origin is + // redacted there too. + return hop(http, request, strip, 0).pipe( + Effect.updateService(Headers.CurrentRedactedNames, (names) => [...names, ...strip]), + ) } /** GETs a configured remote source, sending its headers to the URL and to same-origin redirects only. */ diff --git a/packages/core/src/skill/discovery.ts b/packages/core/src/skill/discovery.ts index f18bd93970d..091cc2dea84 100644 --- a/packages/core/src/skill/discovery.ts +++ b/packages/core/src/skill/discovery.ts @@ -13,6 +13,8 @@ import { RemoteHttp } from "../remote-http.js" /** Records the version of a downloaded skill inside its cache directory. It is not part of the skill. */ export const versionFile = ".opencode-version" +// Config activation waits for skill sources, so an unresponsive catalog fails instead of holding it. +const requestTimeout = "30 seconds" const skillConcurrency = 4 const fileConcurrency = 8 @@ -91,6 +93,7 @@ const layer = Layer.effect( return yield* RemoteHttp.get(http, url, headers).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.scoped, + Effect.timeout(requestTimeout), Effect.flatMap((body) => fs.writeWithDirs(destination, new Uint8Array(body))), Effect.as(true), Effect.catch((error) => @@ -111,6 +114,7 @@ const layer = Layer.effect( ).pipe( Effect.flatMap(HttpClientResponse.schemaBodyJson(Index)), Effect.scoped, + Effect.timeout(requestTimeout), Effect.catch((error) => Effect.logError("failed to fetch skill index", { url: index, error }).pipe(Effect.as(undefined)), ), diff --git a/packages/core/test/config/skill.test.ts b/packages/core/test/config/skill.test.ts index 6805627d8e3..6684c73f3e4 100644 --- a/packages/core/test/config/skill.test.ts +++ b/packages/core/test/config/skill.test.ts @@ -1,7 +1,7 @@ import fs from "fs/promises" import path from "path" import { describe, expect, test } from "bun:test" -import { Deferred, Effect, Fiber, Layer, Schema, Stream } from "effect" +import { Deferred, Effect, Fiber, Layer, PubSub, Schema, Stream } from "effect" import { Config } from "@opencode/core/config" import { Directory, Document, type Entry, Info } from "@opencode/schema/config" import { ConfigIntegration } from "@opencode/core/config/integration" @@ -17,6 +17,7 @@ import { FSUtil } from "@opencode/util/fs-util" import { Global } from "@opencode/util/global" import { LayerNode } from "@opencode/util/effect/layer-node" import { Location } from "@opencode/core/location" +import { Event } from "@opencode/schema/event" import { IntegrationID } from "@opencode/schema/integration-id" import { AbsolutePath } from "@opencode/core/schema" import { Skill } from "@opencode/core/skill" @@ -24,6 +25,7 @@ import { SkillDiscovery } from "@opencode/core/skill/discovery" import { WellKnown } from "@opencode/core/wellknown" import { emptyCredentialNode, emptyWellknownNode } from "../fixture/config-nodes" import { advance, drain } from "../lib/clock" +import { waitUntil } from "../lib/eventually" import { tmpdir } from "../fixture/tmpdir" import { location } from "../fixture/location" import { testEffect } from "../lib/effect" @@ -35,6 +37,7 @@ const it = testEffect( Layer.merge(AppNodeBuilder.build(LayerNode.group([Skill.node, Bus.node, FSUtil.node])), watcherLayer), ) const decode = Schema.decodeUnknownSync(Info) +const configUpdated = () => ({ id: Event.ID.create(), created: Date.now(), type: "config.updated" as const, data: {} }) function write(directory: string, name: string, description: string) { return fs.writeFile( @@ -307,6 +310,64 @@ describe("ConfigSkillPlugin.Plugin", () => { ), ) + it.live("reloads for a config published while the first load is still running", () => + Effect.acquireDisposable(Effect.promise(() => tmpdir())).pipe( + Effect.flatMap((tmp) => { + const first = "https://first.example.test/skills/" + return Effect.gen(function* () { + const deploy = path.join(tmp.path, "cache", "deploy") + yield* Effect.promise(async () => { + await fs.mkdir(deploy, { recursive: true }) + await write(path.dirname(deploy), "deploy", "Deploy") + }) + const started = yield* Deferred.make() + const release = yield* Deferred.make() + const discovery = SkillDiscovery.Service.of({ + pull: (url) => + url === first + ? Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as([] as AbsolutePath[]), + ) + : Effect.succeed([AbsolutePath.make(deploy)]), + }) + const events = yield* PubSub.unbounded>() + const service = yield* Skill.Service + const configTest = yield* Config.Test + const activation = yield* ConfigSkillPlugin.Plugin.effect( + host({ + event: { subscribe: () => Stream.fromPubSub(events) }, + skill: { + list: () => Effect.die("unused skill.list"), + transform: service.transform, + reload: service.reload, + }, + }), + ).pipe( + Effect.provideService(SkillDiscovery.Service, discovery), + Effect.provideService(Global.Service, Global.Service.of({ ...Global.make(), home: tmp.path })), + Effect.provideService( + Location.Service, + Location.Service.of(location({ directory: AbsolutePath.make(tmp.path) })), + ), + Effect.forkScoped({ startImmediately: true }), + ) + + yield* Deferred.await(started) + yield* configTest.setEntries([ + new Document({ type: "document", info: decode({ skills: ["https://second.example.test/skills/"] }) }), + ]) + yield* PubSub.publish(events, configUpdated()) + yield* Deferred.succeed(release, undefined) + yield* Fiber.join(activation) + yield* waitUntil(service.list().pipe(Effect.map((list) => list.some((item) => item.id === "deploy")))) + }).pipe( + Effect.provide(Config.testLayer([new Document({ type: "document", info: decode({ skills: [first] }) })])), + ) + }), + ), + ) + it.live("prefers a worktree skill over the parent checkout copy", () => Effect.acquireDisposable(Effect.promise(() => tmpdir())).pipe( Effect.flatMap((tmp) => diff --git a/packages/core/test/remote-http.test.ts b/packages/core/test/remote-http.test.ts new file mode 100644 index 00000000000..84db1e2f610 --- /dev/null +++ b/packages/core/test/remote-http.test.ts @@ -0,0 +1,32 @@ +import { describe, expect } from "bun:test" +import { Effect, Tracer } from "effect" +import { HttpClient, HttpClientResponse } from "effect/http" +import { RemoteHttp } from "@opencode/core/remote-http" +import { it } from "./lib/effect" + +describe("RemoteHttp.get", () => { + it.effect("redacts a source's headers on the request's trace span", () => + Effect.gen(function* () { + const spans: Tracer.NativeSpan[] = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const http = HttpClient.make((request) => Effect.succeed(HttpClientResponse.fromWeb(request, new Response("ok")))) + + yield* RemoteHttp.get(http, "https://example.test/rules.md", { "X-Token": "secret", Accept: "text/plain" }).pipe( + Effect.scoped, + Effect.provideService(Tracer.Tracer, tracer), + // Effect records no headers by default; a tracing setup that opts in must still never see these values. + Effect.provideService(HttpClient.TracerHeaderFilter, () => true), + ) + + const attributes = Object.fromEntries(spans.flatMap((span) => Array.from(span.attributes))) + expect(attributes["http.request.header.x-token"]).toBe("") + expect(attributes["http.request.header.accept"]).toBe("") + }), + ) +}) diff --git a/packages/core/test/skill-discovery.test.ts b/packages/core/test/skill-discovery.test.ts index 8b66c0c6d2e..44ac131bf04 100644 --- a/packages/core/test/skill-discovery.test.ts +++ b/packages/core/test/skill-discovery.test.ts @@ -3,12 +3,14 @@ import path from "path" import { createServer } from "node:http" import { describe, expect } from "bun:test" import { NodeHttpServer } from "@effect/platform-node" -import { Effect } from "effect" -import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http" +import { Effect, Fiber, Layer } from "effect" +import { HttpClient, HttpClientResponse, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { Global } from "@opencode/util/global" +import { httpClient } from "@opencode/util/effect/app-node-platform" import { SkillDiscovery } from "@opencode/core/skill/discovery" import { tmpdir } from "./fixture/tmpdir" +import { advance } from "./lib/clock" import { it } from "./lib/effect" const fixture = Effect.gen(function* () { @@ -217,4 +219,38 @@ describe("SkillDiscovery.pull", () => { expect(yield* Effect.promise(() => Bun.file(path.join(root, "old.md")).exists())).toBe(false) }), ) + + it.effect( + "gives up on a catalog or a file download that never answers", + Effect.gen(function* () { + const tmp = yield* Effect.acquireDisposable(Effect.promise(() => tmpdir())) + const base = "https://skills.example.test/catalog/" + for (const hanging of [`${base}index.json`, `${base}deploy/SKILL.md`]) { + const http = HttpClient.make((request) => + request.url === hanging + ? Effect.never + : Effect.succeed( + HttpClientResponse.fromWeb( + request, + Response.json({ skills: [{ name: "deploy", version: hanging, files: ["SKILL.md"] }] }), + ), + ), + ) + const pull = yield* Effect.gen(function* () { + const discovery = yield* SkillDiscovery.Service + return yield* discovery.pull(base) + }).pipe( + Effect.provide( + AppNodeBuilder.build(SkillDiscovery.node, [ + Global.node.replace(Global.layerWith({ cache: tmp.path })), + httpClient.replace(Layer.succeed(HttpClient.HttpClient, http)), + ]), + ), + Effect.forkChild({ startImmediately: true }), + ) + yield* advance(() => pull.pollUnsafe() !== undefined) + expect(yield* Fiber.join(pull)).toEqual([]) + } + }), + ) })