mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
feat(core): run shell commands in the process daemon across handoffs (#62978)
This commit is contained in:
committed by
GitHub
Unverified
parent
c5a153288b
commit
d4bcde9121
@@ -4,14 +4,12 @@ import { Service } from "@opencode/client/effect/service"
|
||||
import { Commands } from "../../commands"
|
||||
import { Runtime } from "../../../framework/runtime"
|
||||
import { ServiceConfig } from "../../../services/service-config"
|
||||
import { ServerConnection } from "../../../services/server-connection"
|
||||
|
||||
export default Runtime.handler(
|
||||
Commands.commands.service.commands.restart,
|
||||
Effect.fn("cli.service.restart")(function* () {
|
||||
const options = yield* ServiceConfig.options()
|
||||
// Keep this explicit: automatic service replacement must preserve terminals.
|
||||
yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore)
|
||||
// No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes.
|
||||
yield* Service.stop(options)
|
||||
const transport = yield* Service.ensure(options)
|
||||
process.stdout.write(transport.url + EOL)
|
||||
|
||||
@@ -3,13 +3,12 @@ import { Service } from "@opencode/client/effect/service"
|
||||
import { Commands } from "../../commands"
|
||||
import { Runtime } from "../../../framework/runtime"
|
||||
import { ServiceConfig } from "../../../services/service-config"
|
||||
import { ServerConnection } from "../../../services/server-connection"
|
||||
|
||||
export default Runtime.handler(
|
||||
Commands.commands.service.commands.stop,
|
||||
Effect.fn("cli.service.stop")(function* () {
|
||||
const options = yield* ServiceConfig.options()
|
||||
yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore)
|
||||
// No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes.
|
||||
yield* Service.stop(options)
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -49,6 +49,7 @@ type Active = {
|
||||
isBackgrounded: boolean
|
||||
consumed: boolean
|
||||
recovery?: Recovery
|
||||
cancel?: Effect.Effect<void>
|
||||
}
|
||||
|
||||
type State = {
|
||||
@@ -59,6 +60,7 @@ type State = {
|
||||
type FinishResult = {
|
||||
info?: Info
|
||||
done?: Deferred.Deferred<Info>
|
||||
cancel?: Effect.Effect<void>
|
||||
scope?: Scope.Closeable
|
||||
generation?: Scope.Closeable
|
||||
}
|
||||
@@ -90,6 +92,9 @@ export type StartInput = {
|
||||
recovery?: Recovery
|
||||
notificationID?: SessionMessage.ID
|
||||
run: Effect.Effect<string, unknown>
|
||||
// Runs when the job is cancelled (`cancel`, `cancelSession`), before its work is interrupted. Teardown, such as
|
||||
// a server shutdown, interrupts the work without it.
|
||||
cancel?: Effect.Effect<void>
|
||||
}
|
||||
|
||||
export type WaitInput = {
|
||||
@@ -248,6 +253,11 @@ export const make = Effect.gen(function* () {
|
||||
if (!job) return [{}, jobs]
|
||||
if (job.scope !== scope) return [{}, jobs]
|
||||
if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs]
|
||||
// Teardown (this registry closing) is not a terminal outcome: leave the job and its durable row running for
|
||||
// the next server. The scope's own finalizer interrupts its observers. A cancel has already settled the job
|
||||
// as cancelled before interrupting its work.
|
||||
if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && scope.state._tag === "Closed")
|
||||
return [{}, jobs]
|
||||
const status: Exclude<Status, "running"> = Exit.isSuccess(exit)
|
||||
? "completed"
|
||||
: Cause.hasInterruptsOnly(exit.cause)
|
||||
@@ -295,6 +305,8 @@ export const make = Effect.gen(function* () {
|
||||
return [{ info: snapshot(existing) }, jobs]
|
||||
}
|
||||
const scope = yield* Scope.fork(state.scope, "parallel")
|
||||
// Closing the registry interrupts the work without settling it; interrupt its observers too.
|
||||
yield* Scope.addFinalizer(scope, Deferred.interrupt(done))
|
||||
const job = {
|
||||
info: {
|
||||
id: input.id,
|
||||
@@ -312,6 +324,7 @@ export const make = Effect.gen(function* () {
|
||||
isBackgrounded: false,
|
||||
consumed: false,
|
||||
recovery: input.recovery,
|
||||
cancel: input.cancel,
|
||||
}
|
||||
return [{ info: snapshot(job), scope }, new Map(jobs).set(input.id, job)]
|
||||
}),
|
||||
@@ -459,11 +472,12 @@ export const make = Effect.gen(function* () {
|
||||
}
|
||||
if (persist) yield* persistBackground(next)
|
||||
return [
|
||||
{ info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope },
|
||||
{ info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope, cancel: job.cancel },
|
||||
new Map(jobs).set(id, next),
|
||||
]
|
||||
}),
|
||||
)
|
||||
if (result.cancel) yield* result.cancel
|
||||
if (result.info && result.done) yield* Deferred.succeed(result.done, result.info)
|
||||
if (result.scope) yield* Scope.close(result.scope, Exit.void)
|
||||
if (result.generation) yield* consume(id, result.generation)
|
||||
|
||||
@@ -171,6 +171,11 @@ export interface Interface {
|
||||
readonly remove: (id: Pty.ID) => Effect.Effect<void, NotFoundError | UnavailableError>
|
||||
readonly shutdown: () => Effect.Effect<void, UnavailableError>
|
||||
readonly handoff: () => Effect.Effect<Handoff | null, UnavailableError>
|
||||
/**
|
||||
* Whether a handoff to the next server is prepared and its ticket still valid: this server is being replaced, so
|
||||
* the daemon's processes should be left running for the next one. A shutdown clears it.
|
||||
*/
|
||||
readonly handoffPrepared: () => boolean
|
||||
readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect<Attachment, NotFoundError | UnavailableError>
|
||||
/** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */
|
||||
readonly createPipe: (input: CreatePipeInput) => Effect.Effect<PipeInfo, UnavailableError | RequestError>
|
||||
@@ -361,14 +366,22 @@ const makeLayer = (options: Options = {}) =>
|
||||
return undefined
|
||||
})
|
||||
|
||||
// When the prepared handoff's ticket expires; 0 when none is prepared.
|
||||
let handoffExpiresAt = 0
|
||||
|
||||
const shutdown = Effect.fn("PersistentPty.shutdown")(function* () {
|
||||
handoffExpiresAt = 0
|
||||
const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable))
|
||||
current.clear()
|
||||
if (!response) return
|
||||
if (response.type !== "ok") return yield* unexpected(response)
|
||||
})
|
||||
|
||||
const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable)))
|
||||
const handoff = Effect.fn("PersistentPty.handoff")(function* () {
|
||||
const prepared = yield* daemon.handoff.pipe(Effect.mapError(unavailable))
|
||||
if (prepared) handoffExpiresAt = prepared.expiresAt
|
||||
return prepared
|
||||
})
|
||||
|
||||
const removeVisibleExit = (id: Pty.ID) => {
|
||||
if (removing.has(id)) return
|
||||
@@ -512,6 +525,7 @@ const makeLayer = (options: Options = {}) =>
|
||||
remove,
|
||||
shutdown,
|
||||
handoff,
|
||||
handoffPrepared: () => Date.now() < handoffExpiresAt,
|
||||
attach,
|
||||
createPipe,
|
||||
waitPipe,
|
||||
|
||||
@@ -208,6 +208,7 @@ export const requirements = LayerNode.group([
|
||||
InstructionDiscovery.node,
|
||||
Integration.node,
|
||||
Job.node,
|
||||
Shell.daemonNode,
|
||||
KV.node,
|
||||
llmClient,
|
||||
Location.node,
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
export * as SessionRestart from "./restart.js"
|
||||
|
||||
import { Context, Effect, Layer } from "effect"
|
||||
import { Context, Effect, Layer, Option, Schema } from "effect"
|
||||
import { isReadonlyArrayNonEmpty } from "effect/Array"
|
||||
import { makeGlobalNode } from "@opencode/util/effect/app-node"
|
||||
import { Shell as ShellSchema } from "@opencode/schema/shell"
|
||||
import { Bus } from "../../bus.js"
|
||||
import { Database } from "../../database/database.js"
|
||||
import { Instance } from "../../instance/service.js"
|
||||
import { Job } from "../../job.js"
|
||||
import { Session } from "../../session.js"
|
||||
import { Shell } from "../../shell.js"
|
||||
import { ShellTool } from "../../tool/plugin/shell.js"
|
||||
import { SessionEvent } from "../event.js"
|
||||
import { SessionExecution } from "../execution.js"
|
||||
import { SessionHistory } from "../history.js"
|
||||
import { SessionMessage } from "../message.js"
|
||||
import { SessionSchema } from "../schema.js"
|
||||
import { SessionStore } from "../store.js"
|
||||
import { ImageNotice } from "../../image/notice.js"
|
||||
@@ -72,6 +80,8 @@ export const layer = (options?: Options) =>
|
||||
const bus = yield* Bus.Service
|
||||
const jobs = yield* Job.Service
|
||||
const sessions = yield* Session.Service
|
||||
const instances = yield* Instance.Service
|
||||
const db = (yield* Database.Service).db
|
||||
const scope = yield* Effect.scope
|
||||
const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS
|
||||
|
||||
@@ -99,10 +109,7 @@ export const layer = (options?: Options) =>
|
||||
return true
|
||||
})
|
||||
|
||||
const restartMessage = (
|
||||
background: Job.Background,
|
||||
recovery: Exclude<Job.Recovery, { kind: "subagent" }>,
|
||||
) => {
|
||||
const restartMessage = (background: Job.Background, recovery: Exclude<Job.Recovery, { kind: "subagent" }>) => {
|
||||
const state = background.status === "running" ? "cancelled" : background.status
|
||||
if (recovery.kind === "shell") {
|
||||
const text =
|
||||
@@ -164,6 +171,100 @@ export const layer = (options?: Options) =>
|
||||
yield* jobs.completeBackground(background.notificationID)
|
||||
})
|
||||
|
||||
// The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it.
|
||||
const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
|
||||
const session = yield* store.get(sessionID)
|
||||
if (!session) return undefined
|
||||
return yield* Effect.serviceOption(Shell.Service).pipe(
|
||||
instances.provide(session),
|
||||
Effect.map(Option.getOrUndefined),
|
||||
// The session's directory is gone: its commands are reported like any lost background work.
|
||||
Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)),
|
||||
)
|
||||
})
|
||||
|
||||
const reattach = (
|
||||
shell: Shell.Interface,
|
||||
sessionID: SessionSchema.ID,
|
||||
info: ShellSchema.Info,
|
||||
suspended: ReadonlySet<SessionSchema.ID>,
|
||||
notificationID?: SessionMessage.ID,
|
||||
) =>
|
||||
ShellTool.reattach({
|
||||
shell,
|
||||
jobs,
|
||||
sessions,
|
||||
sessionID,
|
||||
info,
|
||||
notificationID,
|
||||
resume: () => (suspended.has(sessionID) ? false : undefined),
|
||||
})
|
||||
|
||||
// A shell job the previous server left running is watched again if its command is still known (running or
|
||||
// finished during the restart); otherwise it is reported as lost.
|
||||
const recoverShellJob = Effect.fnUntraced(function* (
|
||||
background: Job.Background,
|
||||
recovery: Extract<Job.Recovery, { kind: "shell" }>,
|
||||
suspended: ReadonlySet<SessionSchema.ID>,
|
||||
reattached: Set<string>,
|
||||
) {
|
||||
const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined
|
||||
const info =
|
||||
shell && Schema.is(ShellSchema.ID)(recovery.shellID)
|
||||
? yield* shell.get(recovery.shellID).pipe(Effect.option)
|
||||
: Option.none()
|
||||
if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery)
|
||||
reattached.add(info.value.id)
|
||||
yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID)
|
||||
})
|
||||
|
||||
// A server that stopped (shut down or killed) during a foreground shell call leaves the call unsettled, its
|
||||
// command still running in the daemon. Settle the call as moved to the background and watch the command.
|
||||
const settleBackgroundedCalls = Effect.fnUntraced(function* (
|
||||
sessionID: SessionSchema.ID,
|
||||
suspended: ReadonlySet<SessionSchema.ID>,
|
||||
reattached: ReadonlySet<string>,
|
||||
) {
|
||||
const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) =>
|
||||
message.type === "assistant"
|
||||
? message.content.flatMap((part) =>
|
||||
part.type === "tool" && part.name === ShellTool.name && part.state.status === "running"
|
||||
? [{ assistantMessageID: message.id, part }]
|
||||
: [],
|
||||
)
|
||||
: [],
|
||||
)
|
||||
if (calls.length === 0) return
|
||||
const shell = yield* shellOf(sessionID)
|
||||
if (!shell) return
|
||||
yield* Effect.forEach(
|
||||
calls,
|
||||
Effect.fnUntraced(function* (call) {
|
||||
const info = yield* shell.byToolCall({
|
||||
sessionID,
|
||||
assistantMessageID: call.assistantMessageID,
|
||||
callID: call.part.id,
|
||||
})
|
||||
if (!info) return
|
||||
const result = ShellTool.backgroundToolResult(info)
|
||||
if (!isReadonlyArrayNonEmpty(result.content)) return
|
||||
yield* bus.publish(SessionEvent.Tool.Success, {
|
||||
sessionID,
|
||||
assistantMessageID: call.assistantMessageID,
|
||||
id: call.part.id,
|
||||
content: result.content,
|
||||
metadata: result.metadata,
|
||||
executed: call.part.executed === true,
|
||||
})
|
||||
// Backgrounded but stopped before the call was settled: its job row was already reattached (and, for a
|
||||
// command that already finished, may even be done).
|
||||
if (reattached.has(info.id)) return
|
||||
yield* reattach(shell, sessionID, info, suspended)
|
||||
}),
|
||||
{ discard: true },
|
||||
)
|
||||
})
|
||||
|
||||
const recoverSubagent = Effect.fnUntraced(function* (
|
||||
background: Job.Background,
|
||||
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
|
||||
@@ -232,13 +333,37 @@ export const layer = (options?: Options) =>
|
||||
[...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)),
|
||||
)
|
||||
yield* store.releaseChildClaims(children)
|
||||
const recoverable = Effect.fnUntraced(function* (background: Job.Background) {
|
||||
return (yield* jobs.get(background.id))?.status !== "running"
|
||||
})
|
||||
// Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children,
|
||||
// whose first resume would otherwise fail their own unsettled shell calls.
|
||||
const reattached = new Set<string>()
|
||||
yield* Effect.forEach(
|
||||
// Admit shell outcomes before a recovered child can start its first model request.
|
||||
pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")),
|
||||
pending,
|
||||
Effect.fnUntraced(function* (background) {
|
||||
if ((yield* jobs.get(background.id))?.status === "running") return
|
||||
const recovery = background.recovery
|
||||
if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery)
|
||||
if (recovery.kind === "subagent" || !(yield* recoverable(background))) return
|
||||
if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached)
|
||||
yield* recoverToolJob(background, recovery)
|
||||
}),
|
||||
{ discard: true },
|
||||
)
|
||||
yield* Effect.forEach(
|
||||
suspended,
|
||||
(sessionID) =>
|
||||
settleBackgroundedCalls(sessionID, suspended, reattached).pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }),
|
||||
),
|
||||
),
|
||||
{ discard: true },
|
||||
)
|
||||
yield* Effect.forEach(
|
||||
pending,
|
||||
Effect.fnUntraced(function* (background) {
|
||||
const recovery = background.recovery
|
||||
if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return
|
||||
yield* recoverSubagent(background, recovery, suspended)
|
||||
}),
|
||||
{ discard: true },
|
||||
@@ -264,5 +389,14 @@ export const layer = (options?: Options) =>
|
||||
export const node = makeGlobalNode({
|
||||
service: Service,
|
||||
layer: layer(),
|
||||
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node],
|
||||
deps: [
|
||||
SessionStore.node,
|
||||
SessionExecution.node,
|
||||
Bus.node,
|
||||
Job.node,
|
||||
Session.node,
|
||||
Instance.node,
|
||||
Database.node,
|
||||
Shell.daemonNode,
|
||||
],
|
||||
})
|
||||
|
||||
@@ -29,6 +29,7 @@ import { llmClient } from "../../effect/app-node-platform.js"
|
||||
import { StepFailedError } from "../error.js"
|
||||
import { SessionRunnerRetry } from "./retry.js"
|
||||
import { SessionStep } from "./step.js"
|
||||
import { PersistentPty } from "../../persistent-pty/index.js"
|
||||
import { ToolOutput } from "../../tool-output.js"
|
||||
import { Plugin } from "../../plugin.js"
|
||||
import { PluginHooks } from "../../plugin/hooks.js"
|
||||
@@ -54,6 +55,7 @@ const layer = Layer.effect(
|
||||
const jobs = yield* Job.Service
|
||||
const title = yield* SessionTitle.Service
|
||||
const steps = yield* SessionStep.make
|
||||
const pty = yield* PersistentPty.Service
|
||||
// Title generation starts once input is visible and must not delay model execution.
|
||||
const titles = yield* FiberMap.make<SessionSchema.ID, void, never>()
|
||||
|
||||
@@ -293,6 +295,7 @@ const layer = Layer.effect(
|
||||
const outcome = yield* steps.attempt({
|
||||
isLocationClosed: lifecycle.isClosed,
|
||||
interruption,
|
||||
handoffPrepared: pty.handoffPrepared,
|
||||
sessionID,
|
||||
assistantMessageID,
|
||||
agent: loaded.agent.id,
|
||||
@@ -460,6 +463,7 @@ export const node = makeLocationNode({
|
||||
Job.node,
|
||||
SessionTitle.node,
|
||||
Snapshot.node,
|
||||
PersistentPty.node,
|
||||
ToolOutput.node,
|
||||
Database.node,
|
||||
],
|
||||
|
||||
@@ -44,6 +44,8 @@ interface Input {
|
||||
readonly isLocationClosed: () => boolean
|
||||
/** Why the execution is being interrupted, once it is: `"user"` when someone stopped it. */
|
||||
readonly interruption?: () => "user" | "shutdown" | undefined
|
||||
/** Whether this server is handing its process daemon over to the next one (see `PersistentPty.handoffPrepared`). */
|
||||
readonly handoffPrepared?: () => boolean
|
||||
readonly sessionID: SessionSchema.ID
|
||||
readonly assistantMessageID: SessionMessage.ID
|
||||
readonly agent: Agent.ID
|
||||
@@ -182,6 +184,16 @@ export const make = Effect.gen(function* () {
|
||||
// Read each tool's exit once all have ended, so what they raised while stopping (a decline, a defect) counts.
|
||||
const exits = Exit.isSuccess(joined) ? joined.value : yield* Fiber.awaitAll(fibers)
|
||||
const tools = classifyToolExits(exits, toolRuns, Exit.hasInterrupts(joined))
|
||||
// A server handing its process daemon over leaves the step as a crash would: unsettled calls and no step end.
|
||||
// The next server settles them when the Session resumes, and takes over the shell commands still running for
|
||||
// them. Any other interrupt settles the calls here.
|
||||
if (
|
||||
(streamInterrupted || tools.interrupted) &&
|
||||
tools.declines.length === 0 &&
|
||||
input.interruption?.() !== "user" &&
|
||||
input.handoffPrepared?.() === true
|
||||
)
|
||||
return yield* Effect.interrupt
|
||||
|
||||
if (
|
||||
!publisher.record().outputStarted &&
|
||||
@@ -363,7 +375,7 @@ type Cancel = ReturnType<typeof makeCancel>
|
||||
* A tool call's `cancel`. `trigger` completes it, and reports whether the tool was awaiting it, so would stop on its
|
||||
* own; a tool that was not is interrupted instead.
|
||||
*/
|
||||
const makeCancel = () => {
|
||||
export const makeCancel = () => {
|
||||
let listeners = 0
|
||||
let cancelled = false
|
||||
const gate = Deferred.makeUnsafe<void>()
|
||||
|
||||
+634
-34
@@ -1,7 +1,20 @@
|
||||
export * as Shell from "./shell.js"
|
||||
|
||||
import path from "path"
|
||||
import { Context, Deferred, Duration, Effect, Fiber, Latch, Layer, Option, Schema, Schedule, Stream } from "effect"
|
||||
import {
|
||||
Context,
|
||||
Deferred,
|
||||
Duration,
|
||||
Effect,
|
||||
Exit,
|
||||
Fiber,
|
||||
Latch,
|
||||
Layer,
|
||||
Option,
|
||||
Schema,
|
||||
Schedule,
|
||||
Stream,
|
||||
} from "effect"
|
||||
import { ChildProcess } from "effect/process"
|
||||
import { produce } from "immer"
|
||||
import { Shell } from "@opencode/schema/shell"
|
||||
@@ -14,6 +27,7 @@ import { Environment } from "./environment/index.js"
|
||||
import { FileRetention } from "./file-retention.js"
|
||||
import { Location } from "./location.js"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { PersistentPty } from "./persistent-pty.js"
|
||||
import { ShellSelect } from "./shell/select.js"
|
||||
import type { ShellCreateBefore } from "@opencode/plugin/effect/shell"
|
||||
import { PluginHooks } from "./plugin/hooks.js"
|
||||
@@ -33,17 +47,49 @@ const EXITED_LIMIT = 25
|
||||
export const RETENTION = Duration.days(7)
|
||||
export const DIRECTORY = "shell"
|
||||
const SPAN_COMMAND_LIMIT = 200
|
||||
const STOP_GRACE = Duration.seconds(3)
|
||||
const LOST = "Command was killed: the process daemon stopped"
|
||||
// Bounds the reads a result needs when the daemon did not report the line count, so stopping a call stays quick.
|
||||
const COUNT_NEWLINES_LIMIT = 16 * 1024 * 1024
|
||||
const LINE_START_SCAN_LIMIT = 1024 * 1024
|
||||
|
||||
type Info = Shell.Info
|
||||
export type StopReason = "timeout" | "killed"
|
||||
/** The tool call that started a command, so a server that took over can settle it. */
|
||||
export const ToolCall = Schema.Struct({
|
||||
sessionID: Schema.String,
|
||||
assistantMessageID: Schema.String,
|
||||
callID: Schema.String,
|
||||
})
|
||||
export type ToolCall = typeof ToolCall.Type
|
||||
|
||||
type CreateInput = Shell.CreateInput & {
|
||||
shell?: string
|
||||
toolCall?: ToolCall
|
||||
// A call waits on the command. A server that takes the command over moves it to the background, which clears
|
||||
// its timeout, so its deadline is not kept for a restart.
|
||||
waited?: boolean
|
||||
}
|
||||
|
||||
type Active = {
|
||||
// What a server needs to take a daemon-run command back after a restart, kept in the daemon's metadata.
|
||||
const Stored = Schema.Struct({
|
||||
id: Shell.ID,
|
||||
command: Schema.String,
|
||||
shell: Schema.String,
|
||||
location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }),
|
||||
metadata: Shell.Metadata,
|
||||
toolCall: Schema.optional(ToolCall),
|
||||
timeoutDeadline: Schema.optional(Schema.Number),
|
||||
stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])),
|
||||
})
|
||||
type Stored = typeof Stored.Type
|
||||
const decodeStored = Schema.decodeUnknownOption(Stored)
|
||||
|
||||
// A command this server spawned itself. It dies with its Location.
|
||||
type Spawned = {
|
||||
kind: "spawned"
|
||||
// Immutable snapshot; lifecycle updates replace it via immer `produce`.
|
||||
info: Info
|
||||
file: string
|
||||
size: number
|
||||
newlines: number
|
||||
// Byte offsets where the last two lines start, so a truncation notice can size a cut-off line.
|
||||
@@ -56,6 +102,29 @@ type Active = {
|
||||
stop?: (reason: StopReason) => Effect.Effect<void>
|
||||
}
|
||||
|
||||
// A command the process daemon runs. The daemon writes and counts its output file. It belongs to the server, not
|
||||
// to the Location that started it, so a Location closing or reloading never stops or forgets it.
|
||||
type Run = {
|
||||
kind: "daemon"
|
||||
info: Info
|
||||
process: number
|
||||
toolCall?: ToolCall
|
||||
// When a restart should stop it. Unset while a call waits on it: that call's timeout dies with this server.
|
||||
timeoutDeadline?: number
|
||||
stopReason?: StopReason
|
||||
// From the exit: newlines in the file, and why the daemon stopped writing it early.
|
||||
outputLines?: number
|
||||
outputError?: string
|
||||
// Why it was killed without a signal of its own.
|
||||
killed?: string
|
||||
done: Deferred.Deferred<Info, NotFoundError>
|
||||
timeoutFiber?: Fiber.Fiber<void>
|
||||
stopping?: Fiber.Fiber<void>
|
||||
wait?: Fiber.Fiber<void>
|
||||
}
|
||||
|
||||
type Command = Spawned | Run
|
||||
|
||||
type Ended = {
|
||||
status: "exited" | StopReason
|
||||
exit?: number
|
||||
@@ -66,10 +135,14 @@ type Ended = {
|
||||
/**
|
||||
* Location-owned non-interactive shell command process service.
|
||||
*
|
||||
* Each `create` spawns one shell command, captures combined stdout/stderr to a
|
||||
* Each `create` starts one shell command, captures combined stdout/stderr to a
|
||||
* file, and returns an ID. Clients poll `get` for status and `output` for
|
||||
* file-backed output by cursor. No session, message, or permission state lives
|
||||
* here; callers (e.g. `ShellTool`) own that association and store the shell ID.
|
||||
*
|
||||
* Local commands run in the process daemon when it is available (`Daemon`), so they outlive a planned server
|
||||
* restart: the next server takes them back. Remote workspaces, Windows and an unavailable daemon keep spawning
|
||||
* them from the server.
|
||||
*/
|
||||
export interface Interface {
|
||||
readonly create: <E = never, R = never>(
|
||||
@@ -95,8 +168,14 @@ export interface Interface {
|
||||
// with the given status and stays observable, file included, until retention evicts it. Returns at
|
||||
// once; `wait` observes the exit. Ignored once the command has ended or is already stopping.
|
||||
readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect<Shell.Info, NotFoundError>
|
||||
// The daemon command a tool call started, while it is known here (including one taken back after a restart).
|
||||
readonly byToolCall: (call: ToolCall) => Effect.Effect<Shell.Info | undefined>
|
||||
readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect<Shell.Output, NotFoundError>
|
||||
readonly remove: (id: Shell.ID) => Effect.Effect<void, NotFoundError>
|
||||
/** Runs `effect` until it ends or the server stops, outliving this Location (e.g. a job's completion notice). */
|
||||
readonly watch: DaemonInterface["watch"]
|
||||
/** Whether the server is handing its daemon commands over to the next one, which takes them over. */
|
||||
readonly handoffPrepared: () => boolean
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Shell") {}
|
||||
@@ -104,6 +183,7 @@ export class Service extends Context.Service<Service, Interface>()("@opencode/Sh
|
||||
export const cleanup = Effect.fn("Shell.cleanup")(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
const global = yield* Global.Service
|
||||
const pty = yield* PersistentPty.Service
|
||||
const directory = path.join(global.data, DIRECTORY)
|
||||
const projects = yield* fs.readDirectoryEntries(directory).pipe(
|
||||
Effect.map((entries) => entries.filter((entry) => entry.type === "Directory")),
|
||||
@@ -124,7 +204,29 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () {
|
||||
),
|
||||
{ concurrency: 8 },
|
||||
)
|
||||
yield* FileRetention.cleanup(fs, files.flat(), RETENTION)
|
||||
// Never starts the daemon. A command it still runs keeps its file; a finished one whose file expired is forgotten
|
||||
// below.
|
||||
const processes = new Map(
|
||||
(yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]),
|
||||
)
|
||||
yield* FileRetention.cleanup(
|
||||
fs,
|
||||
files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"),
|
||||
RETENTION,
|
||||
)
|
||||
// Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them.
|
||||
const cutoff = Date.now() - Duration.toMillis(RETENTION)
|
||||
yield* Effect.forEach(
|
||||
processes.values(),
|
||||
(process) =>
|
||||
Effect.gen(function* () {
|
||||
if (process.lifecycle.status === "running") return
|
||||
const stale = (process.exitedAt ?? process.startedAt) < cutoff
|
||||
if (!stale && (yield* fs.exists(process.outputPath).pipe(Effect.orElseSucceed(() => true)))) return
|
||||
yield* pty.terminatePipe(process.id).pipe(Effect.ignore)
|
||||
}),
|
||||
{ concurrency: 8, discard: true },
|
||||
)
|
||||
})
|
||||
|
||||
const cleanupLayer = Layer.effectDiscard(
|
||||
@@ -134,7 +236,365 @@ const cleanupLayer = Layer.effectDiscard(
|
||||
const cleanupNode = makeGlobalNode({
|
||||
name: "shell-output-cleanup",
|
||||
layer: cleanupLayer,
|
||||
deps: [FSUtil.node, Global.node],
|
||||
deps: [FSUtil.node, Global.node, PersistentPty.node],
|
||||
})
|
||||
|
||||
/** One Location's daemon commands, as `Daemon.at` hands them to its `Shell`. */
|
||||
interface Runs {
|
||||
readonly get: (id: Shell.ID) => Run | undefined
|
||||
readonly list: () => Run[]
|
||||
/** Fails with `UnavailableError` only when the daemon could not be reached or started: spawning instead is safe. */
|
||||
readonly create: (input: {
|
||||
readonly info: Info
|
||||
readonly args: readonly string[]
|
||||
readonly env: Readonly<Record<string, string>>
|
||||
readonly timeout: number
|
||||
readonly toolCall: ToolCall | undefined
|
||||
readonly waited: boolean
|
||||
}) => Effect.Effect<Info, PersistentPty.UnavailableError | PersistentPty.RequestError>
|
||||
readonly timeout: (run: Run, duration: number) => Effect.Effect<void>
|
||||
readonly stop: (run: Run, reason: StopReason) => Effect.Effect<void>
|
||||
readonly remove: (id: Shell.ID) => Effect.Effect<void>
|
||||
}
|
||||
|
||||
export interface DaemonInterface {
|
||||
/** A Location's daemon commands. The first call for a Location in this server takes its commands back from the daemon. */
|
||||
readonly at: (location: Location.Interface) => Effect.Effect<Runs>
|
||||
/** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */
|
||||
readonly watch: (effect: Effect.Effect<void, unknown>) => Effect.Effect<void>
|
||||
readonly handoffPrepared: () => boolean
|
||||
}
|
||||
|
||||
/** The process daemon's shell commands, owned by the server and grouped by Location. */
|
||||
export class Daemon extends Context.Service<Daemon, DaemonInterface>()("@opencode/Shell/Daemon") {}
|
||||
|
||||
const daemonLayer = Layer.effect(
|
||||
Daemon,
|
||||
Effect.gen(function* () {
|
||||
const bus = yield* Bus.Service
|
||||
const pty = yield* PersistentPty.Service
|
||||
const scope = yield* Effect.scope
|
||||
const context = yield* Effect.context()
|
||||
const global = yield* Global.Service
|
||||
const { unlink } = yield* Effect.promise(() => import("fs/promises"))
|
||||
// Keyed by canonical Location. `recovered` resolves once the Location's commands were taken back.
|
||||
const places = new Map<string, { runs: Runs; recovered: Deferred.Deferred<void> }>()
|
||||
// Fibers run with the server's context in its scope, so a Location closing never interrupts them. On shutdown
|
||||
// they end without settling anything: the commands keep running for the next server, and the jobs waiting on
|
||||
// them are left to the job registry, which keeps their rows for it.
|
||||
const fork = (effect: Effect.Effect<void>) =>
|
||||
effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context))
|
||||
|
||||
const place = (location: Location.Interface) => {
|
||||
const runs = new Map<Shell.ID, Run>()
|
||||
const exitOrder: Shell.ID[] = []
|
||||
const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID })
|
||||
const outputDir = path.join(global.data, DIRECTORY, location.project.id)
|
||||
|
||||
const persisted = (run: Run): Stored => ({
|
||||
id: run.info.id,
|
||||
command: run.info.command,
|
||||
shell: run.info.shell,
|
||||
location: {
|
||||
directory: location.directory,
|
||||
...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}),
|
||||
},
|
||||
metadata: run.info.metadata,
|
||||
toolCall: run.toolCall,
|
||||
timeoutDeadline: run.timeoutDeadline,
|
||||
stopReason: run.stopReason,
|
||||
})
|
||||
|
||||
// Records the command's latest timeout and stop reason in the daemon, so a restart sees them.
|
||||
const store = (run: Run) =>
|
||||
pty
|
||||
.setPipeMetadata(run.process, persisted(run))
|
||||
.pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("failed to update shell command metadata", { id: run.info.id, error: error.message }),
|
||||
),
|
||||
)
|
||||
|
||||
const remove = Effect.fnUntraced(function* (id: Shell.ID) {
|
||||
const run = runs.get(id)
|
||||
const index = exitOrder.indexOf(id)
|
||||
if (index !== -1) exitOrder.splice(index, 1)
|
||||
if (!run) return
|
||||
runs.delete(id)
|
||||
if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber)
|
||||
if (run.stopping) yield* Fiber.interrupt(run.stopping)
|
||||
// Stop waiting first, so forgetting the process is not mistaken for losing it.
|
||||
if (run.wait) yield* Fiber.interrupt(run.wait)
|
||||
// Forget the process before deleting its file. This also kills what is left of its group, even after a
|
||||
// successful exit, which a spawned command would leave running: the daemon cannot forget a process otherwise.
|
||||
yield* pty.terminatePipe(run.process).pipe(Effect.ignore)
|
||||
yield* Effect.promise(() => unlink(run.info.file).catch(() => {}))
|
||||
yield* Deferred.fail(run.done, new NotFoundError({ id }))
|
||||
// Every shell reports exactly one exit; removing a running one ends it as killed.
|
||||
if (run.info.status === "running")
|
||||
yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }, { location: ref })
|
||||
yield* bus.publish(Shell.Event.Deleted, { id }, { location: ref })
|
||||
})
|
||||
|
||||
const evict = Effect.fnUntraced(function* () {
|
||||
while (exitOrder.length > EXITED_LIMIT) {
|
||||
const oldest = exitOrder[0]
|
||||
if (!oldest) break
|
||||
yield* remove(oldest)
|
||||
}
|
||||
})
|
||||
|
||||
const finish = Effect.fnUntraced(function* (run: Run, outcome: Ended) {
|
||||
const id = run.info.id
|
||||
// A removed command already failed its waiters and published its exit and deletion.
|
||||
if (runs.get(id) !== run) return
|
||||
run.info = produce(run.info, (draft) => {
|
||||
draft.status = outcome.status
|
||||
if (outcome.exit !== undefined) draft.exit = outcome.exit
|
||||
if (outcome.signal !== undefined) draft.signal = outcome.signal
|
||||
draft.time.completed = outcome.completed
|
||||
})
|
||||
if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber)
|
||||
run.timeoutFiber = undefined
|
||||
// Resolve waiters before retention eviction, so an evicted command still reports its
|
||||
// terminal Info rather than the removal NotFoundError.
|
||||
yield* Deferred.succeed(run.done, run.info)
|
||||
yield* bus.publish(
|
||||
Shell.Event.Exited,
|
||||
{ id, ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), status: outcome.status },
|
||||
{ location: ref },
|
||||
)
|
||||
exitOrder.push(id)
|
||||
yield* evict()
|
||||
})
|
||||
|
||||
// Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. The stop is recorded
|
||||
// and the SIGTERM sent before this returns, uninterruptibly, so neither interrupting the caller (a timer being
|
||||
// retimed, removed or torn down) nor a server shutting down right after cancels it. The escalation runs on its
|
||||
// own.
|
||||
const stop = (run: Run, reason: StopReason) =>
|
||||
Effect.gen(function* () {
|
||||
if (run.stopReason !== undefined || run.info.status !== "running") return
|
||||
run.stopReason = reason
|
||||
run.timeoutDeadline = undefined
|
||||
yield* store(run)
|
||||
yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore)
|
||||
run.stopping = yield* fork(
|
||||
Effect.gen(function* () {
|
||||
const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit)
|
||||
if (Exit.isSuccess(ended) && Option.isNone(ended.value))
|
||||
yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore)
|
||||
}),
|
||||
)
|
||||
}).pipe(Effect.uninterruptible)
|
||||
|
||||
const arm = Effect.fnUntraced(function* (run: Run, duration: number) {
|
||||
run.timeoutFiber = yield* fork(
|
||||
Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop(run, "timeout"))),
|
||||
)
|
||||
})
|
||||
|
||||
const timeout = Effect.fnUntraced(function* (run: Run, duration: number) {
|
||||
if (run.stopReason !== undefined || run.info.status !== "running") return
|
||||
if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber)
|
||||
run.timeoutFiber = undefined
|
||||
run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration
|
||||
yield* store(run)
|
||||
if (duration !== 0) yield* arm(run, duration)
|
||||
})
|
||||
|
||||
// Watches the exit, and the timeout (in ms from now, none if undefined).
|
||||
const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) {
|
||||
run.wait = yield* fork(
|
||||
pty.waitPipe(run.process).pipe(
|
||||
Effect.map((exit): Ended => {
|
||||
run.outputLines = exit.outputLines
|
||||
run.outputError = exit.outputError
|
||||
// A stopped command reports its reason, and how the stopped process ended, as spawned ones do.
|
||||
return {
|
||||
status: run.stopReason ?? "exited",
|
||||
...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}),
|
||||
...(exit.signal !== undefined ? { signal: exit.signal } : {}),
|
||||
completed: exit.exitedAt,
|
||||
}
|
||||
}),
|
||||
// The daemon went away (and its processes with it), or forgot the process: it was killed.
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("lost a shell command's process", { id: run.info.id, error: error.message }).pipe(
|
||||
Effect.andThen(
|
||||
Effect.sync((): Ended => {
|
||||
run.killed = LOST
|
||||
return { status: "killed", completed: Date.now() }
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
// Like spawned commands: what a failed command left running in its group is stopped with it, while a
|
||||
// successful one may leave background processes (`cmd &`) behind.
|
||||
Effect.tap((outcome) =>
|
||||
outcome.status === "exited" && outcome.exit !== undefined && outcome.exit !== 0
|
||||
? pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore)
|
||||
: Effect.void,
|
||||
),
|
||||
Effect.flatMap((outcome) => finish(run, outcome)),
|
||||
),
|
||||
)
|
||||
// A stop interrupted by a restart lost its grace period: kill now.
|
||||
if (run.stopReason !== undefined) {
|
||||
run.stopping = yield* fork(pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore))
|
||||
return
|
||||
}
|
||||
if (duration === undefined) return
|
||||
if (duration > 0) return yield* arm(run, duration)
|
||||
yield* stop(run, "timeout")
|
||||
})
|
||||
|
||||
const create: Runs["create"] = Effect.fnUntraced(function* (input) {
|
||||
const run: Run = {
|
||||
kind: "daemon",
|
||||
info: input.info,
|
||||
process: 0,
|
||||
toolCall: input.toolCall,
|
||||
timeoutDeadline: !input.waited && input.timeout > 0 ? input.info.time.started + input.timeout : undefined,
|
||||
done: Deferred.makeUnsafe<Info, NotFoundError>(),
|
||||
}
|
||||
const created = yield* pty.createPipe({
|
||||
program: input.info.shell,
|
||||
args: input.args,
|
||||
cwd: input.info.cwd,
|
||||
env: input.env,
|
||||
groupID: location.project.id,
|
||||
outputPath: input.info.file,
|
||||
metadata: persisted(run),
|
||||
})
|
||||
run.process = created.id
|
||||
run.info = produce(input.info, (draft) => {
|
||||
if (created.pid !== undefined) draft.pid = created.pid
|
||||
// The daemon's clock, so the command reports the same start wherever it is read.
|
||||
draft.time.started = created.startedAt
|
||||
})
|
||||
runs.set(run.info.id, run)
|
||||
// Timed from now: starting the daemon may have taken a while.
|
||||
yield* attach(run, input.timeout > 0 ? input.timeout : undefined)
|
||||
yield* bus.publish(Shell.Event.Created, { info: run.info }, { location: ref })
|
||||
return run.info
|
||||
})
|
||||
|
||||
// Takes back this Location's commands, left by a previous server. Two Locations can share a project's output
|
||||
// directory, so match the Location itself.
|
||||
const recover = Effect.fnUntraced(function* () {
|
||||
const processes = yield* pty
|
||||
.listPipes()
|
||||
.pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe(
|
||||
Effect.as([]),
|
||||
),
|
||||
),
|
||||
)
|
||||
const finished: Run[] = []
|
||||
for (const process of processes) {
|
||||
const stored = Option.getOrUndefined(decodeStored(process.metadata))
|
||||
if (
|
||||
!stored ||
|
||||
stored.location.directory !== location.directory ||
|
||||
stored.location.workspaceID !== location.workspaceID ||
|
||||
path.dirname(process.outputPath) !== outputDir ||
|
||||
runs.has(stored.id)
|
||||
)
|
||||
continue
|
||||
const running = process.lifecycle.status === "running"
|
||||
const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined
|
||||
const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed"))
|
||||
const run: Run = {
|
||||
kind: "daemon",
|
||||
info: {
|
||||
id: stored.id,
|
||||
status,
|
||||
command: stored.command,
|
||||
cwd: process.cwd,
|
||||
shell: stored.shell,
|
||||
file: process.outputPath,
|
||||
...(process.pid !== undefined ? { pid: process.pid } : {}),
|
||||
...(exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}),
|
||||
...(exited?.signal !== undefined ? { signal: exited.signal } : {}),
|
||||
metadata: stored.metadata,
|
||||
time: {
|
||||
started: process.startedAt,
|
||||
...(running ? {} : { completed: process.exitedAt ?? process.startedAt }),
|
||||
},
|
||||
},
|
||||
process: process.id,
|
||||
toolCall: stored.toolCall,
|
||||
timeoutDeadline: stored.timeoutDeadline,
|
||||
stopReason: stored.stopReason,
|
||||
...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }),
|
||||
...(process.lifecycle.status === "failed" && !stored.stopReason
|
||||
? { killed: `Command was killed: ${process.lifecycle.message}` }
|
||||
: {}),
|
||||
done: Deferred.makeUnsafe<Info, NotFoundError>(),
|
||||
}
|
||||
runs.set(stored.id, run)
|
||||
if (running) {
|
||||
// A deadline that passed while no server ran fires at once.
|
||||
const deadline = stored.timeoutDeadline
|
||||
yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now()))
|
||||
continue
|
||||
}
|
||||
yield* Deferred.succeed(run.done, run.info)
|
||||
finished.push(run)
|
||||
}
|
||||
exitOrder.push(
|
||||
...finished
|
||||
.toSorted((a, b) => (a.info.time.completed ?? 0) - (b.info.time.completed ?? 0))
|
||||
.map((run) => run.info.id),
|
||||
)
|
||||
yield* evict()
|
||||
})
|
||||
|
||||
const handle: Runs = {
|
||||
get: (id) => runs.get(id),
|
||||
list: () => Array.from(runs.values()),
|
||||
create,
|
||||
timeout,
|
||||
stop,
|
||||
remove,
|
||||
}
|
||||
return { runs: handle, recover }
|
||||
}
|
||||
|
||||
const at = Effect.fnUntraced(function* (location: Location.Interface) {
|
||||
const key = `${location.directory}\0${location.workspaceID ?? ""}`
|
||||
const existing = places.get(key)
|
||||
if (existing) {
|
||||
yield* Deferred.await(existing.recovered)
|
||||
return existing.runs
|
||||
}
|
||||
const placed = place(location)
|
||||
const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe<void>() }
|
||||
places.set(key, entry)
|
||||
// Uninterruptible: the entry is shared, so a caller's interrupt must not leave it half recovered.
|
||||
yield* placed
|
||||
.recover()
|
||||
.pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined)), Effect.uninterruptible)
|
||||
return placed.runs
|
||||
})
|
||||
|
||||
return Daemon.of({
|
||||
at,
|
||||
handoffPrepared: pty.handoffPrepared,
|
||||
watch: (effect) =>
|
||||
fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe(
|
||||
Effect.asVoid,
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const daemonNode = makeGlobalNode({
|
||||
service: Daemon,
|
||||
layer: daemonLayer,
|
||||
deps: [Bus.node, Global.node, PersistentPty.node],
|
||||
})
|
||||
|
||||
const layer = () =>
|
||||
@@ -149,13 +609,21 @@ const layer = () =>
|
||||
const hooks = yield* PluginHooks.Service
|
||||
const environments = yield* SessionEnvironment.Service
|
||||
const config = yield* Config.Service
|
||||
const daemon = yield* Daemon
|
||||
const context = yield* Effect.context()
|
||||
const runFork = Effect.runForkWith(context)
|
||||
const commands = new Map<Shell.ID, Active>()
|
||||
// Commands this server spawned for the Location; daemon commands live in `runs`, owned by the server.
|
||||
const commands = new Map<Shell.ID, Spawned>()
|
||||
const exitOrder: Shell.ID[] = []
|
||||
// Remote workspaces and Windows keep spawning commands from the server. Taken back before the service is
|
||||
// returned, so session restart recovery sees them.
|
||||
const runs =
|
||||
location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available
|
||||
? yield* daemon.at(location)
|
||||
: undefined
|
||||
|
||||
const outputDir = path.join(global.data, DIRECTORY, location.project.id)
|
||||
const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises"))
|
||||
const { mkdir, unlink, stat } = yield* Effect.promise(() => import("fs/promises"))
|
||||
const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs"))
|
||||
yield* Effect.promise(() => mkdir(outputDir, { recursive: true }))
|
||||
|
||||
@@ -163,7 +631,8 @@ const layer = () =>
|
||||
Effect.gen(function* () {
|
||||
for (const command of commands.values()) {
|
||||
if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber)
|
||||
// Teardown interrupts pending commands; it is not a terminal command failure.
|
||||
// Teardown interrupts pending commands; it is not a terminal command failure. Daemon commands are left
|
||||
// to the server.
|
||||
yield* Deferred.interrupt(command.done)
|
||||
}
|
||||
commands.clear()
|
||||
@@ -172,7 +641,7 @@ const layer = () =>
|
||||
)
|
||||
|
||||
const require = Effect.fnUntraced(function* (id: Shell.ID) {
|
||||
const command = commands.get(id)
|
||||
const command: Command | undefined = commands.get(id) ?? runs?.get(id)
|
||||
if (!command) return yield* new NotFoundError({ id })
|
||||
return command
|
||||
})
|
||||
@@ -186,19 +655,21 @@ const layer = () =>
|
||||
if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber)
|
||||
// Unblock any wait still pending when the command is removed before it terminated.
|
||||
yield* Deferred.fail(command.done, new NotFoundError({ id }))
|
||||
yield* Effect.promise(() => unlink(command.file).catch(() => {}))
|
||||
yield* Effect.promise(() => unlink(command.info.file).catch(() => {}))
|
||||
// Every shell reports exactly one exit; removing a running one ends it as killed.
|
||||
if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" })
|
||||
yield* bus.publish(Shell.Event.Deleted, { id })
|
||||
})
|
||||
|
||||
const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) {
|
||||
yield* annotate((yield* require(id)).info)
|
||||
const command = yield* require(id)
|
||||
yield* annotate(command.info)
|
||||
if (command.kind === "daemon") return yield* runs?.remove(id) ?? Effect.void
|
||||
yield* removeCommand(id)
|
||||
})
|
||||
|
||||
const list = Effect.fn("Shell.list")(function* () {
|
||||
return Array.from(commands.values())
|
||||
return [...commands.values(), ...(runs?.list() ?? [])]
|
||||
.filter((command) => command.info.status === "running")
|
||||
.map((command) => command.info)
|
||||
})
|
||||
@@ -220,19 +691,31 @@ const layer = () =>
|
||||
const timeout = Effect.fn("Shell.timeout")(function* (id: Shell.ID, duration: number) {
|
||||
const command = yield* require(id)
|
||||
yield* annotate(command.info)
|
||||
if (command.info.status !== "running" || !command.timeout) return command.info
|
||||
yield* command.timeout(duration)
|
||||
if (command.kind === "daemon") yield* runs?.timeout(command, duration) ?? Effect.void
|
||||
if (command.kind === "spawned" && command.info.status === "running" && command.timeout)
|
||||
yield* command.timeout(duration)
|
||||
return command.info
|
||||
})
|
||||
|
||||
const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) {
|
||||
const command = yield* require(id)
|
||||
yield* annotate(command.info)
|
||||
if (command.info.status !== "running" || !command.stop) return command.info
|
||||
yield* command.stop(reason)
|
||||
if (command.kind === "daemon") yield* runs?.stop(command, reason) ?? Effect.void
|
||||
if (command.kind === "spawned" && command.info.status === "running" && command.stop) yield* command.stop(reason)
|
||||
return command.info
|
||||
})
|
||||
|
||||
const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) {
|
||||
return runs
|
||||
?.list()
|
||||
.find(
|
||||
(run) =>
|
||||
run.toolCall?.sessionID === call.sessionID &&
|
||||
run.toolCall.assistantMessageID === call.assistantMessageID &&
|
||||
run.toolCall.callID === call.callID,
|
||||
)?.info
|
||||
})
|
||||
|
||||
const readRange = (file: string, start: number, length: number) =>
|
||||
length <= 0
|
||||
? Effect.succeed(Buffer.alloc(0))
|
||||
@@ -252,21 +735,70 @@ const layer = () =>
|
||||
}),
|
||||
)
|
||||
|
||||
// A daemon command's file is the only record of its output, so its size is the file's; a missing file is empty.
|
||||
const sizeOf = (command: Command) =>
|
||||
command.kind === "spawned"
|
||||
? Effect.succeed(command.size)
|
||||
: Effect.promise(() =>
|
||||
stat(command.info.file).then(
|
||||
(info) => info.size,
|
||||
() => 0,
|
||||
),
|
||||
)
|
||||
|
||||
const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) {
|
||||
const command = yield* require(id)
|
||||
const cursor = input?.cursor ?? 0
|
||||
const limit = input?.limit ?? 65536
|
||||
if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false }
|
||||
const size = yield* sizeOf(command)
|
||||
if (cursor >= size) return { output: "", cursor: size, size, truncated: false }
|
||||
const start = Math.max(0, cursor)
|
||||
const bytes = yield* readRange(command.file, start, Math.min(limit, command.size - start))
|
||||
const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start))
|
||||
return {
|
||||
output: bytes.toString("utf8"),
|
||||
cursor: start + bytes.length,
|
||||
size: command.size,
|
||||
size,
|
||||
truncated: false,
|
||||
}
|
||||
})
|
||||
|
||||
// Only for a command whose exit report is missing (still running, or lost with the daemon); the daemon counts
|
||||
// newlines otherwise. Gives up on large files, whose line numbers then go unreported, to keep a result quick.
|
||||
const countNewlines = (file: string, size: number) =>
|
||||
Effect.promise(
|
||||
() =>
|
||||
new Promise<number | undefined>((resolve) => {
|
||||
if (size === 0) return resolve(0)
|
||||
if (size > COUNT_NEWLINES_LIMIT) return resolve(undefined)
|
||||
let count = 0
|
||||
const stream = createReadStream(file, { end: size - 1 })
|
||||
stream.on("data", (chunk: string | Buffer) => {
|
||||
const bytes = Buffer.from(chunk)
|
||||
for (let index = bytes.indexOf(10); index !== -1; index = bytes.indexOf(10, index + 1)) count++
|
||||
})
|
||||
stream.on("end", () => resolve(count))
|
||||
stream.on("error", () => resolve(count))
|
||||
}),
|
||||
)
|
||||
|
||||
// Where the last two lines of a daemon command's output start, found by reading back from the end. Only needed
|
||||
// to size a single cut-off line, which the spawner tracks as it writes. A start further back than the scan
|
||||
// limit is reported as unknown.
|
||||
const lineStartsOf = Effect.fnUntraced(function* (file: string, size: number) {
|
||||
const starts: number[] = []
|
||||
let end = size
|
||||
const floor = Math.max(0, size - LINE_START_SCAN_LIMIT)
|
||||
while (end > floor && starts.length < 2) {
|
||||
const from = Math.max(floor, end - 65536)
|
||||
const chunk = yield* readRange(file, from, end - from)
|
||||
for (let index = chunk.length - 1; index >= 0 && starts.length < 2; index--)
|
||||
if (chunk[index] === 10) starts.push(from + index + 1)
|
||||
end = from
|
||||
}
|
||||
const unknown = floor > 0 ? undefined : 0
|
||||
return [starts[1] ?? unknown, starts[0] ?? unknown] as [number | undefined, number | undefined]
|
||||
})
|
||||
|
||||
const result = Effect.fn("Shell.result")(function* (
|
||||
started: Shell.Info,
|
||||
options?: { readonly wait?: Duration.Input },
|
||||
@@ -291,35 +823,59 @@ const layer = () =>
|
||||
const maxLines = limits?.max_lines ?? ToolOutput.MAX_LINES
|
||||
const maxBytes = limits?.max_bytes ?? ToolOutput.MAX_BYTES
|
||||
const command = yield* require(info.id)
|
||||
const start = Math.max(0, command.size - maxBytes)
|
||||
const size = yield* sizeOf(command)
|
||||
const start = Math.max(0, size - maxBytes)
|
||||
// Read the byte before the window too, to tell whether the window starts on a line boundary.
|
||||
const before = Math.max(0, start - 1)
|
||||
const window = yield* readRange(command.file, before, command.size - before)
|
||||
const window = yield* readRange(info.file, before, size - before)
|
||||
const page = window.subarray(charBoundary(window, start - before)).toString("utf8")
|
||||
const partial = start > 0 && window[0] !== 10
|
||||
const endsWithNewline = page.endsWith("\n")
|
||||
const lines = page.split("\n")
|
||||
if (endsWithNewline) lines.pop()
|
||||
const total = command.newlines + (endsWithNewline ? 0 : 1)
|
||||
const newlines =
|
||||
command.kind === "spawned"
|
||||
? command.newlines
|
||||
: (command.outputLines ?? (yield* countNewlines(info.file, size)))
|
||||
const total = newlines === undefined ? undefined : newlines + (endsWithNewline ? 0 : 1)
|
||||
const suffix = `full output saved to ${info.file}]`
|
||||
// After a write error the daemon kept running the command, but its file stopped early.
|
||||
const stoppedEarly =
|
||||
command.kind === "daemon" && command.outputError
|
||||
? `\n\n[output file ${info.file} stopped early: ${command.outputError}]`
|
||||
: ""
|
||||
const killed = command.kind === "daemon" && command.killed ? { killed: command.killed } : {}
|
||||
if (partial && lines.length === 1) {
|
||||
const lineStart = endsWithNewline ? command.lineStarts[0] : command.lineStarts[1]
|
||||
const lineSize = command.size - (endsWithNewline ? 1 : 0) - lineStart
|
||||
const lineStarts = command.kind === "spawned" ? command.lineStarts : yield* lineStartsOf(info.file, size)
|
||||
const lineStart = endsWithNewline ? lineStarts[0] : lineStarts[1]
|
||||
const end = size - (endsWithNewline ? 1 : 0)
|
||||
const lineSize =
|
||||
lineStart === undefined
|
||||
? `over ${formatSize(Math.min(end, LINE_START_SCAN_LIMIT))}`
|
||||
: formatSize(end - lineStart)
|
||||
const shownSize = Buffer.byteLength(lines[0])
|
||||
const line = total === undefined ? "the last line" : `line ${total}`
|
||||
return {
|
||||
output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of line ${total} (${formatSize(lineSize)}); ${suffix}`,
|
||||
output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of ${line} (${lineSize}); ${suffix}${stoppedEarly}`,
|
||||
truncated: true,
|
||||
...killed,
|
||||
}
|
||||
}
|
||||
if (start > 0 || lines.length > maxLines) {
|
||||
// A window that starts mid-line drops that partial line so the line numbers stay exact.
|
||||
const kept = lines.slice(-Math.min(maxLines, lines.length - (partial ? 1 : 0)))
|
||||
return {
|
||||
output: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}`,
|
||||
output:
|
||||
total === undefined
|
||||
? `${kept.join("\n")}\n\n[showing the last ${kept.length} lines; ${suffix}${stoppedEarly}`
|
||||
: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}${stoppedEarly}`,
|
||||
truncated: true,
|
||||
...killed,
|
||||
}
|
||||
}
|
||||
return { output: page || "(no output)", truncated: false }
|
||||
// One blank line before the notice, whether or not the output ended with a newline.
|
||||
const shown = stoppedEarly && page.endsWith("\n") ? page.slice(0, -1) : page
|
||||
return { output: `${shown || "(no output)"}${stoppedEarly}`, truncated: false, ...killed }
|
||||
}).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined)))
|
||||
return { info, capture }
|
||||
})
|
||||
@@ -346,6 +902,8 @@ const layer = () =>
|
||||
}
|
||||
yield* hooks.trigger("shell", "create.before", invocation)
|
||||
if (before) yield* before(invocation)
|
||||
// Absolute for local commands, so the command reports the same cwd wherever it runs.
|
||||
if (location.workspaceID === undefined) invocation.cwd = path.resolve(location.directory, invocation.cwd)
|
||||
|
||||
const id = Shell.ID.ascending()
|
||||
const args = ShellSelect.args(invocation.shell, invocation.command)
|
||||
@@ -363,10 +921,38 @@ const layer = () =>
|
||||
}
|
||||
yield* annotate(info)
|
||||
|
||||
if (runs) {
|
||||
const created = yield* runs
|
||||
.create({
|
||||
info,
|
||||
args,
|
||||
// The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner.
|
||||
env: Object.fromEntries(
|
||||
Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined),
|
||||
),
|
||||
timeout: invocation.timeout,
|
||||
toolCall: input.toolCall,
|
||||
waited: input.waited === true,
|
||||
})
|
||||
.pipe(
|
||||
Effect.map(Option.some),
|
||||
Effect.catchTags({
|
||||
"PersistentPty.UnavailableError": (error) =>
|
||||
Effect.logWarning("process daemon unavailable; spawning shell command instead", {
|
||||
error: error.message,
|
||||
}).pipe(Effect.as(Option.none<Info>())),
|
||||
// The daemon may have run it: never run it again here.
|
||||
"PersistentPty.RequestError": (cause) =>
|
||||
Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })),
|
||||
}),
|
||||
)
|
||||
if (Option.isSome(created)) return created.value
|
||||
}
|
||||
|
||||
// Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so
|
||||
// the managing fiber keeps its scope open until the command terminates (it awaits `done` at the
|
||||
// end). `create` returns once `ready` resolves with the registered command.
|
||||
const ready = Deferred.makeUnsafe<Active, AppProcess.AppProcessError>()
|
||||
const ready = Deferred.makeUnsafe<Spawned, AppProcess.AppProcessError>()
|
||||
runFork(
|
||||
Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
@@ -377,17 +963,17 @@ const layer = () =>
|
||||
env: invocation.env,
|
||||
stdin: "ignore",
|
||||
detached: process.platform !== "win32",
|
||||
forceKillAfter: Duration.seconds(3),
|
||||
forceKillAfter: STOP_GRACE,
|
||||
}),
|
||||
)
|
||||
.pipe(
|
||||
Effect.mapError((cause) => new AppProcess.AppProcessError({ command: invocation.command, cause })),
|
||||
)
|
||||
const command: Active = {
|
||||
const command: Spawned = {
|
||||
kind: "spawned",
|
||||
info: produce(info, (draft) => {
|
||||
draft.pid = handle.pid
|
||||
}),
|
||||
file,
|
||||
size: 0,
|
||||
newlines: 0,
|
||||
lineStarts: [0, 0],
|
||||
@@ -441,7 +1027,7 @@ const layer = () =>
|
||||
command.stop = (reason) =>
|
||||
Effect.gen(function* () {
|
||||
if (!(yield* end({ status: reason }))) return
|
||||
runFork(handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void)))
|
||||
runFork(handle.kill({ forceKillAfter: STOP_GRACE }).pipe(Effect.catch(() => Effect.void)))
|
||||
}).pipe(Effect.uninterruptible)
|
||||
|
||||
command.timeout = (duration) =>
|
||||
@@ -522,7 +1108,20 @@ const layer = () =>
|
||||
return command.info
|
||||
})
|
||||
|
||||
return Service.of({ create, list, get, wait, result, timeout, stop, output, remove })
|
||||
return Service.of({
|
||||
create,
|
||||
list,
|
||||
get,
|
||||
wait,
|
||||
result,
|
||||
timeout,
|
||||
stop,
|
||||
byToolCall,
|
||||
output,
|
||||
remove,
|
||||
watch: daemon.watch,
|
||||
handoffPrepared: daemon.handoffPrepared,
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -570,6 +1169,7 @@ export const node = makeLocationNode({
|
||||
PluginHooks.node,
|
||||
SessionEnvironment.node,
|
||||
Config.node,
|
||||
daemonNode,
|
||||
cleanupNode,
|
||||
],
|
||||
})
|
||||
|
||||
@@ -5,10 +5,11 @@ import type { Shell } from "@opencode/schema/shell"
|
||||
|
||||
export type Result = {
|
||||
info: Shell.Info
|
||||
capture: { output: string; truncated: boolean } | undefined
|
||||
// `killed` says why a command was killed when no exit code or signal of its own tells, e.g. the process daemon stopped.
|
||||
capture: { output: string; truncated: boolean; killed?: string } | undefined
|
||||
}
|
||||
|
||||
type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean }
|
||||
type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean; killed?: string }
|
||||
|
||||
const missing = "Shell command output is no longer available."
|
||||
export const unavailable: Shell.Output = {
|
||||
@@ -25,6 +26,7 @@ export function output(result: Result): Output {
|
||||
...(result.info.exit !== undefined ? { exit: result.info.exit } : {}),
|
||||
...(result.info.signal !== undefined ? { signal: result.info.signal } : {}),
|
||||
...(result.info.status === "timeout" ? { timeout: true } : {}),
|
||||
...(result.capture?.killed !== undefined ? { killed: result.capture.killed } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,7 +47,8 @@ export function interrupted(result: Result) {
|
||||
return { output, notice, message: output ? `${output}\n\n${notice}` : notice }
|
||||
}
|
||||
|
||||
export function notice(output: Pick<Output, "exit" | "signal" | "timeout">, timeout?: number) {
|
||||
export function notice(output: Pick<Output, "exit" | "signal" | "timeout" | "killed">, timeout?: number) {
|
||||
if (output.killed !== undefined) return output.killed
|
||||
if (output.timeout)
|
||||
return `Command timed out${timeout ? ` after ${timeout / 1_000} seconds` : ""}. If it needs longer and isn't waiting for input, rerun with a larger timeout.\nExit code: ${exitCode(output)}`
|
||||
if (output.signal !== undefined || (output.exit !== undefined && output.exit !== 0))
|
||||
@@ -65,6 +68,7 @@ export function metadata(output: Output) {
|
||||
...(output.exit !== undefined ? { exit: output.exit } : {}),
|
||||
...(output.signal !== undefined ? { signal: output.signal } : {}),
|
||||
...(output.timeout !== undefined ? { timeout: output.timeout } : {}),
|
||||
...(output.killed !== undefined ? { killed: true } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session"
|
||||
import type { ShellCreateBefore } from "@opencode/plugin/effect/shell"
|
||||
import type { Shell as ShellSchema } from "@opencode/schema/shell"
|
||||
import type { Tool } from "@opencode/schema/tool"
|
||||
import { Deferred, Effect, Fiber, Option, Schema, Scope } from "effect"
|
||||
import { Deferred, Effect, Fiber, Option, Schema } from "effect"
|
||||
import { Environment } from "../../environment/index.js"
|
||||
import { Job } from "../../job.js"
|
||||
import { FileAccess } from "../../file-access.js"
|
||||
@@ -14,6 +14,7 @@ import { Flag } from "../../flag.js"
|
||||
import { Permission } from "../../permission.js"
|
||||
import { NonNegativeInt } from "../../schema.js"
|
||||
import { Session } from "../../session.js"
|
||||
import { SessionMessage } from "../../session/message.js"
|
||||
import { SessionSchema } from "../../session/schema.js"
|
||||
import { Shell } from "../../shell.js"
|
||||
import { ShellParse } from "../../shell/parse.js"
|
||||
@@ -72,6 +73,7 @@ const StructuredOutput = Schema.Struct({
|
||||
pid: Schema.optionalKey(NonNegativeInt),
|
||||
truncated: Schema.Boolean,
|
||||
timeout: Schema.optionalKey(Schema.Boolean),
|
||||
killed: Schema.optionalKey(Schema.String),
|
||||
})
|
||||
|
||||
const Output = Schema.Struct({
|
||||
@@ -118,12 +120,143 @@ const backgroundResult = (info: ShellResult.Result["info"], timeout?: number) =>
|
||||
status: "running" as const,
|
||||
})
|
||||
|
||||
// Tells the session how a background command ended, once its job settles.
|
||||
const notify = Effect.fnUntraced(function* (
|
||||
sessions: Session.Interface,
|
||||
jobs: Job.Interface,
|
||||
input: {
|
||||
readonly sessionID: SessionSchema.ID
|
||||
readonly id: string
|
||||
readonly shellID: string
|
||||
readonly command: string
|
||||
readonly settled: Deferred.Deferred<Output>
|
||||
// Read at delivery; `false` admits the notice without waking the session.
|
||||
readonly resume?: () => false | undefined
|
||||
},
|
||||
) {
|
||||
const info = (yield* jobs.wait({ id: input.id })).info
|
||||
if (!info || info.status === "running") return
|
||||
const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined
|
||||
const text = output ? (info.output ?? "") : info.status === "error" ? (info.error ?? "Command failed") : "Cancelled"
|
||||
const resume = input.resume?.()
|
||||
// A session deleted while its command ran has no one left to tell.
|
||||
yield* sessions
|
||||
.synthetic({
|
||||
...(info.notificationID ? { id: info.notificationID } : {}),
|
||||
sessionID: input.sessionID,
|
||||
description: input.command,
|
||||
...ShellResult.notification({
|
||||
jobID: input.id,
|
||||
shellID: input.shellID,
|
||||
command: input.command,
|
||||
state: info.status,
|
||||
text,
|
||||
output,
|
||||
}),
|
||||
...(resume === false ? { resume } : {}),
|
||||
})
|
||||
.pipe(Effect.catchTag("Session.NotFoundError", () => Effect.void))
|
||||
if (info.notificationID) yield* jobs.completeBackground(info.notificationID)
|
||||
})
|
||||
|
||||
// The job that waits on a command for its call (or its session, once backgrounded).
|
||||
const startJob = Effect.fnUntraced(function* (
|
||||
jobs: Job.Interface,
|
||||
shell: Shell.Interface,
|
||||
input: {
|
||||
readonly sessionID: SessionSchema.ID
|
||||
readonly info: ShellSchema.Info
|
||||
readonly notificationID?: SessionMessage.ID
|
||||
readonly timeout?: () => number
|
||||
},
|
||||
) {
|
||||
const info = input.info
|
||||
const settled = yield* Deferred.make<Output>()
|
||||
const job = yield* jobs.start({
|
||||
// CodeMode children share a tool-call ID, but each shell must own its job.
|
||||
id: info.id,
|
||||
type: name,
|
||||
title: info.command,
|
||||
metadata: { sessionID: input.sessionID, shellID: info.id },
|
||||
recovery: {
|
||||
kind: "shell",
|
||||
sessionID: input.sessionID,
|
||||
shellID: info.id,
|
||||
command: info.command,
|
||||
},
|
||||
...(input.notificationID ? { notificationID: input.notificationID } : {}),
|
||||
run: Effect.gen(function* () {
|
||||
const result = yield* shell.result(info)
|
||||
if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id })
|
||||
return { ...ShellResult.output(result), status: "completed" as const }
|
||||
}).pipe(
|
||||
Effect.tap((output) => Deferred.succeed(settled, output)),
|
||||
Effect.map((output) => resultMessages(output, input.timeout?.()).join("\n\n")),
|
||||
// Interrupting the job (its registry closing) stops the command too, unless the server is handing its daemon
|
||||
// over: then the command keeps running for the next server.
|
||||
Effect.onInterrupt(() =>
|
||||
shell.handoffPrepared() ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid),
|
||||
),
|
||||
),
|
||||
// Cancelling the job stops the command, keeping it and its output file.
|
||||
cancel: shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid),
|
||||
})
|
||||
return { job, settled }
|
||||
})
|
||||
|
||||
/** The tool result of a call whose command moved to the background. */
|
||||
export const backgroundToolResult = (info: ShellSchema.Info) => {
|
||||
const result = toolResult(backgroundResult(info))
|
||||
// Narrowed to what a backgrounded command reports, so it can be published as is.
|
||||
const metadata = {
|
||||
shellID: info.id,
|
||||
status: "running" as const,
|
||||
truncated: result.metadata.truncated,
|
||||
...(info.pid === undefined ? {} : { pid: info.pid }),
|
||||
}
|
||||
return { content: result.content, metadata }
|
||||
}
|
||||
|
||||
/**
|
||||
* Watches a command taken over from a previous server as a background job of its session, and notifies the
|
||||
* session when it finishes, as for a command the model ran in the background. A finished command's notice is
|
||||
* admitted before this returns.
|
||||
*/
|
||||
export const reattach = Effect.fnUntraced(function* (input: {
|
||||
readonly shell: Shell.Interface
|
||||
readonly jobs: Job.Interface
|
||||
readonly sessions: Session.Interface
|
||||
readonly sessionID: SessionSchema.ID
|
||||
readonly info: ShellSchema.Info
|
||||
// The job's existing durable row, if the previous server had backgrounded it.
|
||||
readonly notificationID?: SessionMessage.ID
|
||||
readonly resume?: () => false | undefined
|
||||
}) {
|
||||
const { job, settled } = yield* startJob(input.jobs, input.shell, input)
|
||||
yield* input.jobs.background(job.id)
|
||||
const notifying = notify(input.sessions, input.jobs, {
|
||||
sessionID: input.sessionID,
|
||||
id: job.id,
|
||||
shellID: input.info.id,
|
||||
command: input.info.command,
|
||||
settled,
|
||||
resume: input.resume,
|
||||
})
|
||||
const current = yield* input.shell.get(input.info.id).pipe(Effect.option)
|
||||
if (Option.isSome(current) && current.value.status !== "running")
|
||||
return yield* notifying.pipe(
|
||||
Effect.catch((error) =>
|
||||
Effect.logWarning("failed to report a recovered shell command", { id: input.info.id, error }),
|
||||
),
|
||||
)
|
||||
yield* input.shell.watch(notifying)
|
||||
})
|
||||
|
||||
export const Plugin = {
|
||||
id: "opencode.tool.shell",
|
||||
effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) {
|
||||
const sessions = yield* Session.Service
|
||||
const jobs = yield* Job.Service
|
||||
const scope = yield* Scope.Scope
|
||||
const environment = yield* Environment.Service
|
||||
const access = yield* FileAccess.Service
|
||||
const shell = yield* Shell.Service
|
||||
@@ -162,39 +295,15 @@ export const Plugin = {
|
||||
return timeout
|
||||
})
|
||||
|
||||
const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")(
|
||||
function* (
|
||||
sessionID: SessionSchema.ID,
|
||||
id: string,
|
||||
shellID: string,
|
||||
command: string,
|
||||
settled: Deferred.Deferred<Output>,
|
||||
) {
|
||||
const info = (yield* jobs.wait({ id })).info
|
||||
if (!info || info.status === "running") return
|
||||
const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined
|
||||
const text = output
|
||||
? (info.output ?? "")
|
||||
: info.status === "error"
|
||||
? (info.error ?? "Command failed")
|
||||
: "Cancelled"
|
||||
yield* sessions.synthetic({
|
||||
...(info.notificationID ? { id: info.notificationID } : {}),
|
||||
sessionID,
|
||||
description: command,
|
||||
...ShellResult.notification({
|
||||
jobID: id,
|
||||
shellID,
|
||||
command,
|
||||
state: info.status,
|
||||
text,
|
||||
output,
|
||||
}),
|
||||
})
|
||||
if (info.notificationID) yield* jobs.completeBackground(info.notificationID)
|
||||
},
|
||||
Effect.forkIn(scope, { startImmediately: true }),
|
||||
)
|
||||
// Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and
|
||||
// its session must still hear how it ended.
|
||||
const notifyWhenDone = (
|
||||
sessionID: SessionSchema.ID,
|
||||
id: string,
|
||||
shellID: string,
|
||||
command: string,
|
||||
settled: Deferred.Deferred<Output>,
|
||||
) => shell.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled }))
|
||||
|
||||
// Stops the command of a call the user asked to stop, and fails the call with the output so far. The command
|
||||
// stays observable, output file included.
|
||||
@@ -249,6 +358,12 @@ export const Plugin = {
|
||||
timeout,
|
||||
shell: yield* compatibleShell,
|
||||
metadata: { sessionID: context.sessionID },
|
||||
waited: input.background !== true,
|
||||
toolCall: {
|
||||
sessionID: context.sessionID,
|
||||
assistantMessageID: context.messageID,
|
||||
callID: context.id,
|
||||
},
|
||||
},
|
||||
(invocation) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -274,30 +389,10 @@ export const Plugin = {
|
||||
)
|
||||
yield* context.progress({ shellID: info.id })
|
||||
|
||||
const settled = yield* Deferred.make<Output>()
|
||||
const run = Effect.gen(function* () {
|
||||
const result = yield* shell.result(info)
|
||||
if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id })
|
||||
return { ...ShellResult.output(result), status: "completed" as const }
|
||||
}).pipe(
|
||||
Effect.tap((output) => Deferred.succeed(settled, output)),
|
||||
Effect.map((output) => resultMessages(output, finalTimeout).join("\n\n")),
|
||||
// Cancelling the job stops the command but keeps it, output file included.
|
||||
Effect.onInterrupt(() => shell.stop(info.id, "killed").pipe(Effect.ignore)),
|
||||
)
|
||||
const job = yield* jobs.start({
|
||||
// CodeMode children share a tool-call ID, but each shell must own its job.
|
||||
id: info.id,
|
||||
type: name,
|
||||
title: info.command,
|
||||
metadata: { sessionID: context.sessionID, shellID: info.id },
|
||||
recovery: {
|
||||
kind: "shell",
|
||||
sessionID: context.sessionID,
|
||||
shellID: info.id,
|
||||
command: info.command,
|
||||
},
|
||||
run,
|
||||
const { job, settled } = yield* startJob(jobs, shell, {
|
||||
sessionID: context.sessionID,
|
||||
info,
|
||||
timeout: () => finalTimeout,
|
||||
})
|
||||
// Stopped while the command was starting: stop it now.
|
||||
if (yield* Deferred.isDone(stopped)) return yield* cancelled(info, job.id)
|
||||
@@ -326,13 +421,19 @@ export const Plugin = {
|
||||
),
|
||||
)
|
||||
: waiting
|
||||
// A user's stop arrives through `cancel`; an interrupt (a server shutdown) cancels the job as before.
|
||||
// A user's stop arrives through `cancel`. Any interrupt stops the command too, unless the server is
|
||||
// handing its daemon over: then the command and its job keep running, and the next server takes the
|
||||
// call over and moves the command to the background.
|
||||
const outcome = yield* restore(
|
||||
Effect.raceFirst(
|
||||
blocked.pipe(Effect.map((result) => ({ type: "settled" as const, result }))),
|
||||
Deferred.await(stopped).pipe(Effect.as({ type: "cancelled" as const })),
|
||||
),
|
||||
).pipe(Effect.onInterrupt(() => jobs.cancel(job.id).pipe(Effect.ignore)))
|
||||
).pipe(
|
||||
Effect.onInterrupt(() =>
|
||||
shell.handoffPrepared() ? Effect.void : jobs.cancel(job.id).pipe(Effect.ignore),
|
||||
),
|
||||
)
|
||||
if (outcome.type === "cancelled") return yield* cancelled(info, job.id)
|
||||
const result = outcome.result
|
||||
if (result?.type === "backgrounded") {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { appendFileSync, writeFileSync } from "node:fs"
|
||||
import { appendFileSync, rmSync, writeFileSync } from "node:fs"
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
|
||||
import net from "node:net"
|
||||
import os from "node:os"
|
||||
@@ -16,7 +16,11 @@ export function temporaryDirectory() {
|
||||
|
||||
export function listen(
|
||||
socketPath: string,
|
||||
handle: (socket: net.Socket, request: Record<string, unknown>, token: string) => object | undefined,
|
||||
handle: (
|
||||
socket: net.Socket,
|
||||
request: Record<string, unknown>,
|
||||
token: string,
|
||||
) => object | undefined | Promise<object | undefined>,
|
||||
) {
|
||||
return Effect.acquireRelease(
|
||||
Effect.promise(async () => {
|
||||
@@ -33,8 +37,10 @@ export function listen(
|
||||
const owning = envelope.request.op === "own"
|
||||
if (owning) owner = true
|
||||
const response = owning ? { type: "owned" } : handle(socket, envelope.request, envelope.token)
|
||||
if (response) socket.write(frame(response))
|
||||
if (owner) return serve()
|
||||
return Promise.resolve(response).then((response) => {
|
||||
if (response) socket.write(frame(response))
|
||||
if (owner) return serve()
|
||||
})
|
||||
})
|
||||
void serve().catch(() => socket.destroy())
|
||||
})
|
||||
@@ -132,6 +138,12 @@ export const fakeDaemon = Effect.fn(function* () {
|
||||
const failTerminate = new Set<string>()
|
||||
const trap = new Set<number>()
|
||||
const failCreate = { message: undefined as string | undefined }
|
||||
// How long a prepared handoff's ticket stays valid.
|
||||
const handoffTTL = { ms: 60_000 }
|
||||
// While set, `create_pipe` answers only once it resolves, like a daemon slow to start a command.
|
||||
const holdCreate: { until?: Promise<void> } = {}
|
||||
// While set, `list` answers only once it resolves, like a busy daemon.
|
||||
const holdList: { until?: Promise<void> } = {}
|
||||
const ops: string[] = []
|
||||
const requests: Record<string, unknown>[] = []
|
||||
let nextID = 1
|
||||
@@ -171,8 +183,10 @@ export const fakeDaemon = Effect.fn(function* () {
|
||||
ops.push(String(request.op))
|
||||
requests.push(request)
|
||||
if (request.op === "ping") return pong
|
||||
if (request.op === "list")
|
||||
return { type: "processes", processes: processes.filter((item) => item.mode === request.mode) }
|
||||
if (request.op === "list") {
|
||||
const list = () => ({ type: "processes", processes: processes.filter((item) => item.mode === request.mode) })
|
||||
return holdList.until ? holdList.until.then(list) : list()
|
||||
}
|
||||
if (request.op === "get") {
|
||||
const found = find(request.id)
|
||||
if (!found) return { type: "error", message: `terminal ${String(request.id)} not found` }
|
||||
@@ -187,13 +201,22 @@ export const fakeDaemon = Effect.fn(function* () {
|
||||
if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd))
|
||||
return { type: "error", message: "cwd must be absolute" }
|
||||
if (failCreate.message !== undefined) return { type: "error", message: failCreate.message }
|
||||
const created = pipe(nextID++, request)
|
||||
writeFileSync(String(request.output_path), "")
|
||||
processes.push(created)
|
||||
return { type: "created", process: created }
|
||||
const create = () => {
|
||||
const created = pipe(nextID++, request)
|
||||
writeFileSync(String(request.output_path), "")
|
||||
processes.push(created)
|
||||
return { type: "created", process: created }
|
||||
}
|
||||
return holdCreate.until ? holdCreate.until.then(create) : create()
|
||||
}
|
||||
if (request.op === "control") return { type: "ok" }
|
||||
if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 }
|
||||
// Unregisters, so the server sees it stopped; it keeps serving connections already open.
|
||||
if (request.op === "shutdown") {
|
||||
rmSync(path.join(directory, "service.json"), { force: true })
|
||||
return { type: "ok" }
|
||||
}
|
||||
if (request.op === "prepare_handoff")
|
||||
return { type: "handoff", ticket: "next", expires_at: Date.now() + handoffTTL.ms }
|
||||
if (request.op === "rename") {
|
||||
const found = find(request.id)
|
||||
if (!found?.pty) return { type: "error", message: "terminal not found" }
|
||||
@@ -245,6 +268,9 @@ export const fakeDaemon = Effect.fn(function* () {
|
||||
return {
|
||||
failTerminate,
|
||||
failCreate,
|
||||
holdCreate,
|
||||
holdList,
|
||||
handoffTTL,
|
||||
trap,
|
||||
ops,
|
||||
requests,
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Effect, Layer } from "effect"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
|
||||
// No process daemon: shell commands fall back to the Environment's spawner, and nothing is ever running in it.
|
||||
export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, {
|
||||
createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })),
|
||||
listPipes: () => Effect.succeed([]),
|
||||
handoffPrepared: () => false,
|
||||
})
|
||||
@@ -0,0 +1,122 @@
|
||||
import path from "node:path"
|
||||
import { Context, Effect, Exit, Layer, Scope } from "effect"
|
||||
import { ChildProcessSpawner } from "effect/process"
|
||||
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
|
||||
import { Config } from "@opencode/core/config"
|
||||
import { Environment } from "@opencode/core/environment/index"
|
||||
import { Location } from "@opencode/core/location"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Bus } from "@opencode/core/bus"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { LocationLifecycle } from "@opencode/core/location-lifecycle"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { hostEnvironmentLayer } from "./environment"
|
||||
import { location } from "./location"
|
||||
import { fakeDaemon } from "./persistent-pty-daemon"
|
||||
import { tmpdirScoped } from "./tmpdir"
|
||||
|
||||
// One fake daemon and data directory shared by every Location (and server) a test builds.
|
||||
export const daemonShells = Effect.fn(function* () {
|
||||
const daemon = yield* fakeDaemon()
|
||||
const data = (yield* tmpdirScoped()).path
|
||||
const global = Global.layerWith({ data, state: path.join(data, "state") })
|
||||
const context = yield* Layer.build(
|
||||
AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]),
|
||||
)
|
||||
const pty = Context.get(context, PersistentPty.Service)
|
||||
let spawned = 0
|
||||
const environment = Layer.effect(
|
||||
Environment.Service,
|
||||
Effect.gen(function* () {
|
||||
const host = yield* Environment.Service
|
||||
return Environment.Service.of({
|
||||
...host,
|
||||
spawner: ChildProcessSpawner.make(() =>
|
||||
Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))),
|
||||
),
|
||||
})
|
||||
}),
|
||||
).pipe(Layer.provide(hostEnvironmentLayer))
|
||||
// `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own.
|
||||
const shellLayer = (directory: string, server?: Shell.DaemonInterface) =>
|
||||
AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [
|
||||
...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []),
|
||||
Location.node.replace(
|
||||
Layer.succeed(
|
||||
Location.Service,
|
||||
Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))),
|
||||
),
|
||||
),
|
||||
Global.node.replace(global),
|
||||
Config.node.replace(Config.testLayer()),
|
||||
Environment.node.replace(environment),
|
||||
PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)),
|
||||
])
|
||||
// The server that `openLocation` Locations share, as long as the test runs.
|
||||
const serverContext = yield* Layer.build(
|
||||
AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [
|
||||
Global.node.replace(global),
|
||||
PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)),
|
||||
]),
|
||||
)
|
||||
const server = Context.get(serverContext, Shell.Daemon)
|
||||
// A Location's Shell, on a server of its own, that stays open for the rest of the test.
|
||||
const openShell = (directory: string) =>
|
||||
Effect.gen(function* () {
|
||||
const context = yield* Layer.build(shellLayer(directory))
|
||||
return Context.get(context, Shell.Service)
|
||||
})
|
||||
// A Location's Shell and its services, closed by `close` as a reload or an eviction does while the server runs.
|
||||
const openLocation = (directory: string) =>
|
||||
Effect.gen(function* () {
|
||||
const scope = yield* Scope.make()
|
||||
const context = yield* Layer.buildWithScope(shellLayer(directory, server), scope)
|
||||
return {
|
||||
shell: Context.get(context, Shell.Service),
|
||||
bus: Context.get(context, Bus.Service),
|
||||
close: Context.get(context, LocationLifecycle.Service).shutdown.pipe(
|
||||
Effect.andThen(Scope.close(scope, Exit.void)),
|
||||
),
|
||||
}
|
||||
})
|
||||
// Runs `body` against the Location's Shell on a server of its own, then stops that server, as a restart does.
|
||||
const withShell = <A, E>(
|
||||
directory: string,
|
||||
body: (shell: Shell.Interface) => Effect.Effect<A, E, LocationLifecycle.Service | Bus.Service>,
|
||||
) =>
|
||||
Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const shell = yield* Shell.Service
|
||||
return yield* body(shell)
|
||||
}).pipe(Effect.provide(shellLayer(directory))),
|
||||
)
|
||||
const directory = (yield* tmpdirScoped()).path
|
||||
return {
|
||||
daemon,
|
||||
data,
|
||||
directory,
|
||||
global,
|
||||
pty,
|
||||
withShell,
|
||||
openShell,
|
||||
openLocation,
|
||||
// Where the server's daemon commands report.
|
||||
serverBus: Context.get(serverContext, Bus.Service),
|
||||
spawned: () => spawned,
|
||||
created: () => daemon.requests.filter((request) => request.op === "create_pipe"),
|
||||
signals: (id: number) =>
|
||||
daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])),
|
||||
}
|
||||
})
|
||||
|
||||
// Polls in real time until `check` holds: the fake daemon answers over a socket, which the test clock does not drive.
|
||||
export const until = (check: (() => boolean) | Effect.Effect<boolean>) =>
|
||||
Effect.gen(function* () {
|
||||
for (let attempt = 0; attempt < 2_000; attempt++) {
|
||||
if (typeof check === "function" ? check() : yield* check) return
|
||||
yield* Effect.promise(() => Bun.sleep(2))
|
||||
}
|
||||
return yield* Effect.die(new Error("condition never held"))
|
||||
})
|
||||
@@ -457,6 +457,59 @@ describe("Job", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("interrupts observers without settling a background job, or running its cancel hook, on teardown", () =>
|
||||
Effect.gen(function* () {
|
||||
const scope = yield* Scope.make()
|
||||
const jobs = yield* Job.make.pipe(Scope.provide(scope))
|
||||
let cancels = 0
|
||||
const job = yield* jobs.start({
|
||||
id: "shell_teardown",
|
||||
type: "shell",
|
||||
recovery: {
|
||||
kind: "shell",
|
||||
sessionID: SessionSchema.ID.make("ses_teardown"),
|
||||
shellID: "shell_teardown",
|
||||
command: "sleep 60",
|
||||
},
|
||||
run: Effect.never,
|
||||
cancel: Effect.sync(() => void cancels++),
|
||||
})
|
||||
const background = yield* jobs.background(job.id)
|
||||
const observer = yield* jobs.wait({ id: job.id }).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
|
||||
yield* Scope.close(scope, Exit.void)
|
||||
|
||||
const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second"))
|
||||
expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true)
|
||||
expect(cancels).toBe(0)
|
||||
const current = yield* Job.make
|
||||
expect(yield* current.pendingBackground).toMatchObject([
|
||||
{ id: job.id, notificationID: background?.notificationID, status: "running" },
|
||||
])
|
||||
if (background?.notificationID) yield* current.completeBackground(background.notificationID)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("runs a job's cancel hook when it is cancelled, alone or with its session", () =>
|
||||
Effect.gen(function* () {
|
||||
const jobs = yield* Job.Service
|
||||
const cancelled: string[] = []
|
||||
const start = (id: string) =>
|
||||
jobs.start({
|
||||
id,
|
||||
type: "shell",
|
||||
recovery: { kind: "shell", sessionID: SessionSchema.ID.make("ses_cancel_hook"), shellID: id, command: "x" },
|
||||
run: Effect.never,
|
||||
cancel: Effect.sync(() => void cancelled.push(id)),
|
||||
})
|
||||
yield* start("shell_cancel_one")
|
||||
yield* start("shell_cancel_session")
|
||||
yield* jobs.cancel("shell_cancel_one")
|
||||
yield* jobs.cancelSession(SessionSchema.ID.make("ses_cancel_hook"))
|
||||
expect(cancelled).toEqual(["shell_cancel_one", "shell_cancel_session"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () =>
|
||||
Effect.gen(function* () {
|
||||
const jobs = yield* Job.Service
|
||||
|
||||
@@ -122,6 +122,34 @@ describe.skipIf(process.platform === "win32")("PersistentPty pipe processes", ()
|
||||
metadata: { id: "sh_test", command: "echo hi" },
|
||||
}
|
||||
|
||||
it.live("knows a handoff is prepared until a shutdown or until its ticket expires", () =>
|
||||
Effect.gen(function* () {
|
||||
const daemon = yield* fakeDaemon()
|
||||
yield* Effect.gen(function* () {
|
||||
const pty = yield* PersistentPty.Service
|
||||
expect(pty.handoffPrepared()).toBe(false)
|
||||
yield* pty.createPipe(input)
|
||||
expect(yield* pty.handoff()).not.toBeNull()
|
||||
expect(pty.handoffPrepared()).toBe(true)
|
||||
// Abandoned by a shutdown.
|
||||
yield* pty.shutdown()
|
||||
expect(pty.handoffPrepared()).toBe(false)
|
||||
}).pipe(Effect.provide(layer(daemon)))
|
||||
|
||||
// Abandoned by letting the ticket expire.
|
||||
const expiring = yield* fakeDaemon()
|
||||
expiring.handoffTTL.ms = 50
|
||||
yield* Effect.gen(function* () {
|
||||
const pty = yield* PersistentPty.Service
|
||||
yield* pty.createPipe(input)
|
||||
expect(yield* pty.handoff()).not.toBeNull()
|
||||
expect(pty.handoffPrepared()).toBe(true)
|
||||
yield* Effect.sleep(80)
|
||||
expect(pty.handoffPrepared()).toBe(false)
|
||||
}).pipe(Effect.provide(layer(expiring)))
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("creates, inspects, signals and waits for a pipe process", () =>
|
||||
Effect.gen(function* () {
|
||||
const daemon = yield* fakeDaemon()
|
||||
|
||||
@@ -22,6 +22,11 @@ import { SessionMessage } from "@opencode/core/session/message"
|
||||
import { SessionRunner } from "@opencode/core/session/runner/index"
|
||||
import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql"
|
||||
import { SessionStore } from "@opencode/core/session/store"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Agent } from "@opencode/core/agent"
|
||||
import { ID } from "@opencode/core/model"
|
||||
import { Provider } from "@opencode/core/provider"
|
||||
import { daemonShells } from "./fixture/shell-daemon"
|
||||
import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect"
|
||||
import { eq } from "drizzle-orm"
|
||||
import { testEffect } from "./lib/effect"
|
||||
@@ -397,6 +402,9 @@ describe("SessionExecution lifecycle", () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so the daemon shell cases skip there.
|
||||
const daemonLive = process.platform === "win32" ? it.live.skip : it.live
|
||||
|
||||
describe("SessionRestart background recovery", () => {
|
||||
it.effect("keeps shell owners idle until a user prompt delivers recovered notices exactly once", () =>
|
||||
Effect.gen(function* () {
|
||||
@@ -486,6 +494,276 @@ describe("SessionRestart background recovery", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
daemonLive("watches shell jobs whose commands survived the restart", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const jobs = yield* Job.Service
|
||||
const bus = yield* Bus.Service
|
||||
const parent = Session.ID.make("ses_background_shell_survived")
|
||||
yield* seedSessions(database, [parent])
|
||||
const daemon = yield* daemonShells()
|
||||
const before = yield* daemon.withShell("/project", (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const running = yield* shell.create({ shell: "/bin/sh", command: "running", cwd: "/tmp" })
|
||||
const finished = yield* shell.create({ shell: "/bin/sh", command: "finished", cwd: "/tmp" })
|
||||
return { running, finished }
|
||||
}),
|
||||
)
|
||||
yield* seedBackground(jobs, parent, [
|
||||
{ id: before.running.id, shellID: before.running.id, command: "running" },
|
||||
{ id: before.finished.id, shellID: before.finished.id, command: "finished" },
|
||||
])
|
||||
// The finished command ended while no server watched it.
|
||||
daemon.daemon.output(2, "done\n")
|
||||
daemon.daemon.exit(2, 0)
|
||||
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid),
|
||||
undefined,
|
||||
restarted,
|
||||
shell,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
|
||||
const notices = Effect.gen(function* () {
|
||||
return (yield* store.context(parent)).filter(
|
||||
(message) => message.type === "synthetic" && message.metadata?.source === "shell",
|
||||
)
|
||||
})
|
||||
while ((yield* notices).length < 1) yield* Effect.sleep(5)
|
||||
expect(yield* notices).toMatchObject([
|
||||
{
|
||||
text: expect.stringContaining("done"),
|
||||
metadata: { source: "shell", shellID: before.finished.id, state: "completed", exit: 0 },
|
||||
},
|
||||
])
|
||||
expect((yield* restarted.get(before.running.id))?.status).toBe("running")
|
||||
expect((yield* restarted.pendingBackground).map((item) => item.id)).toEqual([before.running.id])
|
||||
|
||||
daemon.daemon.exit(1, 2)
|
||||
while ((yield* notices).length < 2) yield* Effect.sleep(5)
|
||||
expect((yield* notices)[1]).toMatchObject({
|
||||
text: expect.stringContaining("Exit code: 2"),
|
||||
metadata: { source: "shell", shellID: before.running.id, state: "completed", exit: 2 },
|
||||
})
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
daemonLive("settles a shell call a killed server left running as moved to the background", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const bus = yield* Bus.Service
|
||||
const parent = Session.ID.make("ses_background_shell_killed")
|
||||
yield* seedSessions(database, [parent], { time_suspended: Date.now() })
|
||||
const assistantMessageID = SessionMessage.ID.create()
|
||||
yield* bus.publish(SessionEvent.Step.Started, {
|
||||
sessionID: parent,
|
||||
assistantMessageID,
|
||||
agent: Agent.ID.make("build"),
|
||||
model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
|
||||
started: 0,
|
||||
})
|
||||
yield* bus.publish(SessionEvent.Tool.Input.Started, {
|
||||
sessionID: parent,
|
||||
assistantMessageID,
|
||||
id: "call-shell",
|
||||
name: "shell",
|
||||
})
|
||||
yield* bus.publish(SessionEvent.Tool.Called, {
|
||||
sessionID: parent,
|
||||
assistantMessageID,
|
||||
id: "call-shell",
|
||||
input: { command: "sleep 60" },
|
||||
executed: false,
|
||||
})
|
||||
const daemon = yield* daemonShells()
|
||||
const info = yield* daemon.withShell("/project", (shell) =>
|
||||
shell.create({
|
||||
shell: "/bin/sh",
|
||||
command: "sleep 60",
|
||||
cwd: "/tmp",
|
||||
timeout: 120_000,
|
||||
waited: true,
|
||||
toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" },
|
||||
}),
|
||||
)
|
||||
// A call waits on it, so its deadline is not kept: taking it over never times it out.
|
||||
expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline")
|
||||
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid),
|
||||
undefined,
|
||||
restarted,
|
||||
shell,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
|
||||
const [assistant] = (yield* store.context(parent)).filter((message) => message.type === "assistant")
|
||||
expect(assistant?.type === "assistant" ? assistant.content : []).toMatchObject([
|
||||
{
|
||||
type: "tool",
|
||||
id: "call-shell",
|
||||
state: {
|
||||
status: "completed",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `Command moved to the background (shell ID: ${info.id}).\nPID: ${info.pid}\nOutput is streaming to: ${info.file}`,
|
||||
},
|
||||
{ type: "text", text: expect.stringContaining("You will be notified automatically") },
|
||||
],
|
||||
},
|
||||
},
|
||||
])
|
||||
expect((yield* restarted.get(info.id))?.status).toBe("running")
|
||||
expect(daemon.signals(1)).toEqual([])
|
||||
|
||||
daemon.daemon.exit(1, 0)
|
||||
const notices = Effect.gen(function* () {
|
||||
return (yield* store.context(parent)).filter(
|
||||
(message) => message.type === "synthetic" && message.metadata?.source === "shell",
|
||||
)
|
||||
})
|
||||
while ((yield* notices).length < 1) yield* Effect.sleep(5)
|
||||
expect((yield* notices)[0]).toMatchObject({ metadata: { source: "shell", shellID: info.id, state: "completed" } })
|
||||
}),
|
||||
)
|
||||
|
||||
for (const exited of [false, true])
|
||||
daemonLive(
|
||||
`reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const bus = yield* Bus.Service
|
||||
const jobs = yield* Job.Service
|
||||
const parent = Session.ID.make(`ses_background_shell_unsettled_${exited}`)
|
||||
yield* seedSessions(database, [parent], { time_suspended: Date.now() })
|
||||
const assistantMessageID = yield* seedShellCall(bus, parent, "call-shell")
|
||||
const daemon = yield* daemonShells()
|
||||
const info = yield* daemon.withShell("/project", (shell) =>
|
||||
shell.create({
|
||||
shell: "/bin/sh",
|
||||
command: "work",
|
||||
cwd: "/tmp",
|
||||
waited: true,
|
||||
toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" },
|
||||
}),
|
||||
)
|
||||
// The old server persisted the background job, then died before the step recorded the call's result.
|
||||
yield* seedBackground(jobs, parent, [{ id: info.id, shellID: info.id, command: "work" }])
|
||||
const [row] = yield* jobs.pendingBackground
|
||||
if (exited) {
|
||||
daemon.daemon.output(1, "finished output\n")
|
||||
daemon.daemon.exit(1, 0)
|
||||
}
|
||||
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid),
|
||||
undefined,
|
||||
restarted,
|
||||
shell,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
|
||||
expect(yield* shellCallState(store, parent)).toMatchObject({ status: "completed" })
|
||||
const notices = shellNotices(store, parent)
|
||||
// A finished command's result is admitted during recovery, before any session resumes.
|
||||
if (exited)
|
||||
expect((yield* SessionInbox.list(database.db, parent)).length + (yield* notices).length).toBeGreaterThan(0)
|
||||
if (!exited) {
|
||||
expect((yield* restarted.pendingBackground).map((item) => item.notificationID)).toEqual([
|
||||
row?.notificationID,
|
||||
])
|
||||
daemon.daemon.exit(1, 0)
|
||||
}
|
||||
while ((yield* notices).length < 1) yield* Effect.sleep(5)
|
||||
yield* Effect.sleep(50)
|
||||
expect(yield* notices).toMatchObject([{ id: row?.notificationID, metadata: { shellID: info.id } }])
|
||||
if (exited) expect(yield* notices).toMatchObject([{ text: expect.stringContaining("finished output") }])
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
daemonLive("settles a recovered child's unsettled shell call before the child resumes", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const bus = yield* Bus.Service
|
||||
const jobs = yield* Job.Service
|
||||
const parent = Session.ID.make("ses_background_shell_child_parent")
|
||||
const child = Session.ID.make("ses_background_shell_child")
|
||||
yield* seedSessions(database, [parent])
|
||||
yield* seedSessions(database, [child], { parent_id: parent, time_suspended: Date.now() })
|
||||
yield* jobs.start({
|
||||
id: child,
|
||||
type: "subagent",
|
||||
recovery: {
|
||||
kind: "subagent",
|
||||
parentSessionID: parent,
|
||||
childSessionID: child,
|
||||
agent: "explore",
|
||||
description: "Inspect recovery",
|
||||
},
|
||||
run: Effect.never,
|
||||
})
|
||||
yield* jobs.background(child)
|
||||
const assistantMessageID = yield* seedShellCall(bus, child, "call-child-shell")
|
||||
const daemon = yield* daemonShells()
|
||||
const info = yield* daemon.withShell("/project", (shell) =>
|
||||
shell.create({
|
||||
shell: "/bin/sh",
|
||||
command: "child work",
|
||||
cwd: "/tmp",
|
||||
waited: true,
|
||||
toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" },
|
||||
}),
|
||||
)
|
||||
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope))
|
||||
const seen = yield* Deferred.make<unknown>()
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) =>
|
||||
sessionID === child
|
||||
? shellCallState(store, child).pipe(Effect.flatMap((state) => Deferred.succeed(seen, state)))
|
||||
: Effect.void,
|
||||
undefined,
|
||||
restarted,
|
||||
shell,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
expect(yield* Deferred.await(seen).pipe(Effect.timeout("2 seconds"))).toMatchObject({ status: "completed" })
|
||||
expect((yield* restarted.get(info.id))?.status).toBe("running")
|
||||
// Cancelling a reattached job stops its command.
|
||||
yield* restarted.cancel(info.id)
|
||||
while (daemon.signals(1).length === 0) yield* Effect.sleep(5)
|
||||
expect(daemon.signals(1)).toEqual(["SIGTERM"])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("preserves locally running background work", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
@@ -1400,6 +1678,7 @@ function buildExecution(
|
||||
drain: (input: Parameters<SessionRunner.Interface["drain"]>[0]) => Effect.Effect<void, SessionRunner.RunError>,
|
||||
options?: SessionRestart.Options,
|
||||
overrideJobs?: Job.Interface,
|
||||
shell?: Shell.Interface,
|
||||
) {
|
||||
return Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
@@ -1430,9 +1709,12 @@ function buildExecution(
|
||||
LocationServiceMap.Service,
|
||||
LayerMap.make(
|
||||
() =>
|
||||
// The local execution test only needs the Session runner from the Location graph.
|
||||
// The local execution test only needs the Session runner (and the Shell, if given) from the Location graph.
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
|
||||
runner as unknown as Layer.Layer<LocationServices, FileSystem.DirectoryNotFoundError>,
|
||||
(shell ? Layer.merge(runner, Layer.succeed(Shell.Service, shell)) : runner) as unknown as Layer.Layer<
|
||||
LocationServices,
|
||||
FileSystem.DirectoryNotFoundError
|
||||
>,
|
||||
),
|
||||
)
|
||||
return yield* Layer.buildWithScope(
|
||||
@@ -1443,6 +1725,16 @@ function buildExecution(
|
||||
Layer.provide(Layer.succeed(Bus.Service, bus)),
|
||||
Layer.provide(Layer.succeed(SessionStore.Service, store)),
|
||||
Layer.provide(Layer.succeed(Job.Service, jobs)),
|
||||
Layer.provide(
|
||||
Layer.succeed(
|
||||
Shell.Daemon,
|
||||
Shell.Daemon.of({
|
||||
at: () => Effect.die("the harness has no daemon registry"),
|
||||
watch: (effect) => effect.pipe(Effect.ignore, Effect.forkIn(scope), Effect.asVoid),
|
||||
handoffPrepared: () => false,
|
||||
}),
|
||||
),
|
||||
),
|
||||
// Do not reuse the outer harness's selector with its already-captured Location map.
|
||||
Layer.provide(
|
||||
AppNodeBuilder.build(Instance.node, [LocationServiceMap.node.replace(locations)]).pipe(Layer.fresh),
|
||||
@@ -1452,3 +1744,49 @@ function buildExecution(
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** Records a shell call the model made that is still running, as a killed server leaves it. */
|
||||
function seedShellCall(bus: Bus.Interface, sessionID: Session.ID, callID: string) {
|
||||
return Effect.gen(function* () {
|
||||
const assistantMessageID = SessionMessage.ID.create()
|
||||
yield* bus.publish(SessionEvent.Step.Started, {
|
||||
sessionID,
|
||||
assistantMessageID,
|
||||
agent: Agent.ID.make("build"),
|
||||
model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") },
|
||||
started: 0,
|
||||
})
|
||||
yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: callID, name: "shell" })
|
||||
yield* bus.publish(SessionEvent.Tool.Called, {
|
||||
sessionID,
|
||||
assistantMessageID,
|
||||
id: callID,
|
||||
input: { command: "work" },
|
||||
executed: false,
|
||||
})
|
||||
return assistantMessageID
|
||||
})
|
||||
}
|
||||
|
||||
function shellCallState(store: SessionStore.Interface, sessionID: Session.ID) {
|
||||
return store.context(sessionID).pipe(
|
||||
Effect.map((messages) =>
|
||||
messages.flatMap((message) =>
|
||||
message.type === "assistant"
|
||||
? message.content.flatMap((part) => (part.type === "tool" ? [part.state] : []))
|
||||
: [],
|
||||
),
|
||||
),
|
||||
Effect.map((states) => states[0]),
|
||||
)
|
||||
}
|
||||
|
||||
function shellNotices(store: SessionStore.Interface, sessionID: Session.ID) {
|
||||
return store
|
||||
.context(sessionID)
|
||||
.pipe(
|
||||
Effect.map((messages) =>
|
||||
messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -125,7 +125,7 @@ const setup = Effect.fnUntraced(function* (options?: {
|
||||
const services = Layer.mergeAll(
|
||||
Layer.succeed(PluginHooks.Service, hooks),
|
||||
Layer.mock(Image.Service, {}),
|
||||
options?.shell ?? Layer.mock(Shell.Service, {}),
|
||||
options?.shell ?? Layer.mock(Shell.Service, { handoffPrepared: () => false }),
|
||||
)
|
||||
const servicesFor = (ref: Location.Ref) => {
|
||||
locations.push(ref)
|
||||
@@ -508,6 +508,7 @@ describe("Session-owned handles", () => {
|
||||
})
|
||||
const fixture = yield* setup({
|
||||
shell: Layer.mock(Shell.Service, {
|
||||
handoffPrepared: () => false,
|
||||
create: (input) =>
|
||||
Effect.sync(() => {
|
||||
expect(input).toEqual({
|
||||
|
||||
@@ -233,8 +233,9 @@ it.effect("records the cost the endpoint reports instead of the catalog price",
|
||||
// Interrupts an attempt running a tool that stops on `cancel` and one that never does, and returns how the step and
|
||||
// its calls were recorded.
|
||||
const stopTwoTools = (how: {
|
||||
reason: "user" | "shutdown"
|
||||
reason: "user" | "shutdown" | undefined
|
||||
locationClosed: boolean
|
||||
handoffPrepared?: boolean
|
||||
// What the cooperative tool does once asked to stop; by default it fails with its partial output.
|
||||
stopped?: Effect.Effect<never, Permission.DeclinedError>
|
||||
}) =>
|
||||
@@ -284,6 +285,7 @@ const stopTwoTools = (how: {
|
||||
.attempt({
|
||||
isLocationClosed: () => how.locationClosed,
|
||||
interruption: () => how.reason,
|
||||
handoffPrepared: () => how.handoffPrepared === true,
|
||||
sessionID,
|
||||
assistantMessageID,
|
||||
agent: Agent.defaultID,
|
||||
@@ -392,17 +394,42 @@ it.live("classifies what a tool raises while the user's stop settles it: a defec
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("interrupts every tool on a server shutdown", () =>
|
||||
// Without a handoff nothing will take the calls over, so they settle here however the step was interrupted.
|
||||
for (const reason of ["shutdown", undefined] as const)
|
||||
it.live(
|
||||
`settles every call as interrupted on ${reason === "shutdown" ? "a server shutdown" : "an unexplained interrupt"} without a handoff`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const { exit, data, requests, interrupted } = yield* stopTwoTools({ reason, locationClosed: false })
|
||||
expect(Exit.hasInterrupts(exit)).toBe(true)
|
||||
expect(requests).toBe(1)
|
||||
expect(interrupted).toBe(true)
|
||||
const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }
|
||||
expect(data).toMatchObject({
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
content: [
|
||||
{ type: "tool", id: "call-cooperative", state: generic },
|
||||
{ type: "tool", id: "call-stuck", state: generic },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("leaves an interrupted step as a crash would while the server hands its daemon over", () =>
|
||||
Effect.gen(function* () {
|
||||
const { exit, data, requests } = yield* stopTwoTools({ reason: "shutdown", locationClosed: false })
|
||||
const { exit, data, requests } = yield* stopTwoTools({
|
||||
reason: "shutdown",
|
||||
locationClosed: false,
|
||||
handoffPrepared: true,
|
||||
})
|
||||
expect(Exit.hasInterrupts(exit)).toBe(true)
|
||||
expect(requests).toBe(1)
|
||||
const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }
|
||||
// No step end and no settled call: the next server finds the calls unsettled.
|
||||
expect(data).not.toHaveProperty("error")
|
||||
expect(data).toMatchObject({
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
content: [
|
||||
{ type: "tool", id: "call-cooperative", state: generic },
|
||||
{ type: "tool", id: "call-stuck", state: generic },
|
||||
{ type: "tool", id: "call-cooperative", state: { status: "running" } },
|
||||
{ type: "tool", id: "call-stuck", state: { status: "running" } },
|
||||
],
|
||||
})
|
||||
}),
|
||||
|
||||
@@ -6,13 +6,16 @@ import { Shell } from "@opencode/core/shell"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { FSUtil } from "@opencode/util/fs-util"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { disabledPersistentPtyLayer } from "./fixture/persistent-pty"
|
||||
import { withTempDir } from "./fixture/tmpdir"
|
||||
import { it } from "./lib/effect"
|
||||
|
||||
const withStore = <A, E, R>(body: (fs: FSUtil.Interface, root: string) => Effect.Effect<A, E, R>) =>
|
||||
withTempDir((tmp) => {
|
||||
const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [
|
||||
const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [
|
||||
Global.node.replace(Global.layerWith({ data: tmp.path })),
|
||||
PersistentPty.node.replace(disabledPersistentPtyLayer),
|
||||
])
|
||||
return Effect.gen(function* () {
|
||||
const fs = yield* FSUtil.Service
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { existsSync, readdirSync, realpathSync } from "node:fs"
|
||||
import { mkdir, rm, utimes, writeFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { Context, Effect, Fiber, Layer } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { Config } from "@opencode/core/config"
|
||||
import { Environment } from "@opencode/core/environment/index"
|
||||
import { Location } from "@opencode/core/location"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Bus } from "@opencode/core/bus"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { FSUtil } from "@opencode/util/fs-util"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { AppProcess } from "@opencode/util/process"
|
||||
import { hostEnvironmentLayer } from "./fixture/environment"
|
||||
import { location } from "./fixture/location"
|
||||
import { daemonShells, until } from "./fixture/shell-daemon"
|
||||
import { tmpdirScoped } from "./fixture/tmpdir"
|
||||
import { binaryPath } from "@opencode/pty"
|
||||
import { it } from "./lib/effect"
|
||||
|
||||
const unix = process.platform !== "win32"
|
||||
const test = unix ? it.live : it.live.skip
|
||||
// On the test clock, for timers.
|
||||
const timed = unix ? it.effect : it.effect.skip
|
||||
|
||||
const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("")
|
||||
|
||||
describe("Shell on the process daemon", () => {
|
||||
test("runs a command with the server's environment, an absolute cwd and its metadata", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* Effect.promise(() => mkdir(path.join(env.directory, "sub")))
|
||||
process.env.OPENCODE_TEST_REMOVED = "1"
|
||||
yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED))
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create(
|
||||
{
|
||||
shell: "/bin/sh",
|
||||
command: "echo hi",
|
||||
cwd: "sub",
|
||||
timeout: 60_000,
|
||||
metadata: { sessionID: "ses_test" },
|
||||
},
|
||||
(invocation) =>
|
||||
Effect.sync(() => {
|
||||
delete invocation.env.OPENCODE_TEST_REMOVED
|
||||
invocation.env.OPENCODE_TEST_UNSET = undefined
|
||||
}),
|
||||
)
|
||||
const [request] = env.created()
|
||||
expect(request).toMatchObject({
|
||||
program: "/bin/sh",
|
||||
args: ["-c", "echo hi"],
|
||||
cwd: path.join(env.directory, "sub"),
|
||||
output_path: info.file,
|
||||
metadata: {
|
||||
id: info.id,
|
||||
command: "echo hi",
|
||||
shell: "/bin/sh",
|
||||
location: { directory: env.directory },
|
||||
metadata: { sessionID: "ses_test" },
|
||||
timeoutDeadline: expect.any(Number),
|
||||
},
|
||||
})
|
||||
expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" })
|
||||
expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED")
|
||||
expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET")
|
||||
expect(info).toMatchObject({ status: "running", pid: 1001, cwd: path.join(env.directory, "sub") })
|
||||
expect(env.spawned()).toBe(0)
|
||||
|
||||
env.daemon.output(1, "first\nsecond")
|
||||
expect(yield* shell.output(info.id)).toEqual({
|
||||
output: "first\nsecond",
|
||||
cursor: 12,
|
||||
size: 12,
|
||||
truncated: false,
|
||||
})
|
||||
expect(yield* shell.output(info.id, { cursor: 6, limit: 3 })).toEqual({
|
||||
output: "sec",
|
||||
cursor: 9,
|
||||
size: 12,
|
||||
truncated: false,
|
||||
})
|
||||
expect(yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).toEqual({
|
||||
output: "",
|
||||
cursor: 12,
|
||||
size: 12,
|
||||
truncated: false,
|
||||
})
|
||||
expect((yield* shell.list()).map((item) => item.id)).toEqual([info.id])
|
||||
|
||||
env.daemon.exit(1, 0)
|
||||
expect(yield* shell.result(info)).toMatchObject({
|
||||
info: { status: "exited", exit: 0, time: { completed: expect.any(Number) } },
|
||||
capture: { output: "first\nsecond", truncated: false },
|
||||
})
|
||||
expect(yield* shell.list()).toEqual([])
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("counts lines like the spawner and reads only what reached the file after a write error", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const long = yield* shell.create({ shell: "/bin/sh", command: "long" })
|
||||
env.daemon.output(1, lines(3000))
|
||||
env.daemon.exit(1, 0)
|
||||
const result = yield* shell.result(long)
|
||||
expect(result.capture?.output).toEndWith(
|
||||
`line 3000\n\n[showing lines 1001-3000 of 3000; full output saved to ${long.file}]`,
|
||||
)
|
||||
|
||||
const failed = yield* shell.create({ shell: "/bin/sh", command: "failed" })
|
||||
env.daemon.output(2, "kept\n")
|
||||
env.daemon.fail(2, "No space left on device")
|
||||
env.daemon.output(2, "lost\n")
|
||||
env.daemon.exit(2, 1)
|
||||
expect(yield* shell.result(failed)).toMatchObject({
|
||||
info: { status: "exited", exit: 1 },
|
||||
capture: {
|
||||
output: `kept\n\n[output file ${failed.file} stopped early: No space left on device]`,
|
||||
},
|
||||
})
|
||||
|
||||
// A single line longer than the window is sized from the file, which the daemon does not track.
|
||||
const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" })
|
||||
env.daemon.output(3, `short\n${"x".repeat(60 * 1024)}\n`)
|
||||
env.daemon.exit(3, 0)
|
||||
expect((yield* shell.result(wide)).capture?.output).toEndWith(
|
||||
`\n\n[showing the last 50 KB of line 2 (60 KB); full output saved to ${wide.file}]`,
|
||||
)
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("bounds what it reads to number lines the daemon has not counted yet", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
// Still running, so the daemon has not reported a line count, and the file is too big to count quickly.
|
||||
const big = yield* shell.create({ shell: "/bin/sh", command: "big" })
|
||||
env.daemon.output(1, `${"y".repeat(17 * 1024 * 1024)}\n${lines(10)}`)
|
||||
expect((yield* shell.result(big, { wait: 0 })).capture?.output).toEndWith(
|
||||
`line 10\n\n[showing the last 10 lines; full output saved to ${big.file}]`,
|
||||
)
|
||||
|
||||
// A cut-off line that started too far back to look for.
|
||||
const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" })
|
||||
env.daemon.output(2, `short\n${"x".repeat(2 * 1024 * 1024)}`)
|
||||
expect((yield* shell.result(wide, { wait: 0 })).capture?.output).toEndWith(
|
||||
`\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${wide.file}]`,
|
||||
)
|
||||
|
||||
// The same, ending with a newline: the line's start is still too far back to know.
|
||||
const ended = yield* shell.create({ shell: "/bin/sh", command: "ended" })
|
||||
env.daemon.output(3, `short\n${"x".repeat(2 * 1024 * 1024)}\n`)
|
||||
expect((yield* shell.result(ended, { wait: 0 })).capture?.output).toEndWith(
|
||||
`\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${ended.file}]`,
|
||||
)
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("reports a command lost with the daemon as killed, with what it wrote", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create({ shell: "/bin/sh", command: "work" })
|
||||
env.daemon.output(1, "partial\n")
|
||||
yield* until(() => env.daemon.waiters(1) > 0)
|
||||
env.daemon.drop(1)
|
||||
expect(yield* shell.result(info)).toMatchObject({
|
||||
info: { status: "killed" },
|
||||
capture: { output: "partial\n", killed: "Command was killed: the process daemon stopped" },
|
||||
})
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("stops what a failed command left in its group, and leaves a successful one's", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const succeeded = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 0" })
|
||||
const failed = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 3" })
|
||||
yield* until(() => env.daemon.waiters(1) > 0 && env.daemon.waiters(2) > 0)
|
||||
env.daemon.exit(1, 0)
|
||||
env.daemon.exit(2, 3)
|
||||
expect(yield* shell.wait(failed.id)).toMatchObject({ status: "exited", exit: 3 })
|
||||
expect(yield* shell.wait(succeeded.id)).toMatchObject({ status: "exited", exit: 0 })
|
||||
expect(env.signals(2)).toEqual(["SIGTERM"])
|
||||
expect(env.signals(1)).toEqual([])
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("fails a command the daemon rejected instead of spawning it", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
env.daemon.failCreate.message = "output path is a symlink"
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* shell.create({ shell: "/bin/sh", command: "echo hi" }).pipe(Effect.flip)
|
||||
expect(error).toBeInstanceOf(AppProcess.AppProcessError)
|
||||
expect(env.spawned()).toBe(0)
|
||||
expect(yield* shell.list()).toEqual([])
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
// On the test clock: the timeout and the grace period pass without waiting for them.
|
||||
timed("stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
env.daemon.trap.add(1)
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 })
|
||||
yield* TestClock.adjust("10 millis")
|
||||
yield* until(() => env.signals(1).length > 0)
|
||||
expect(env.signals(1)).toEqual(["SIGTERM"])
|
||||
expect((yield* shell.get(info.id)).status).toBe("running")
|
||||
// The grace period starts once the daemon acknowledged SIGTERM, so keep the clock moving until it ends.
|
||||
yield* until(TestClock.adjust("1 second").pipe(Effect.map(() => env.signals(1).length > 1)))
|
||||
expect((yield* shell.wait(info.id)).status).toBe("timeout")
|
||||
expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"])
|
||||
expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" })
|
||||
expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline")
|
||||
expect(yield* shell.result(info)).toMatchObject({
|
||||
info: { status: "timeout" },
|
||||
capture: { output: "(no output)" },
|
||||
})
|
||||
// Like a spawned command, it records how the stopped process ended.
|
||||
expect((yield* shell.result(info)).info).toMatchObject({ signal: "SIGKILL" })
|
||||
expect(env.daemon.ops).not.toContain("terminate")
|
||||
expect(existsSync(info.file)).toBe(true)
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
test("forgets a command in the daemon before deleting its file, on remove and on eviction", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" })
|
||||
yield* shell.remove(running.id)
|
||||
expect(env.daemon.requests.filter((request) => request.op === "terminate")).toEqual([
|
||||
expect.objectContaining({ id: 1 }),
|
||||
])
|
||||
expect(existsSync(running.file)).toBe(false)
|
||||
expect(env.daemon.processes).toEqual([])
|
||||
expect(yield* shell.result(running)).toMatchObject({ info: { status: "killed" }, capture: undefined })
|
||||
|
||||
const finished = yield* Effect.forEach(Array.from({ length: 26 }), (_, index) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* shell.create({ shell: "/bin/sh", command: `echo ${index}` })
|
||||
env.daemon.exit(index + 2, 0)
|
||||
yield* shell.wait(info.id)
|
||||
return info
|
||||
}),
|
||||
)
|
||||
// Waiters resolve before retention runs, which forgets the process and then deletes its file.
|
||||
yield* until(() => !existsSync(finished[0]!.file))
|
||||
expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError)
|
||||
expect(existsSync(finished[0]!.file)).toBe(false)
|
||||
expect(env.daemon.processes.map((process) => process.id)).not.toContain(2)
|
||||
expect(existsSync(finished[1]!.file)).toBe(true)
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
test("takes its commands back after a restart and leaves other Locations' alone", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
env.daemon.trap.add(4)
|
||||
const before = yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const running = yield* shell.create({ shell: "/bin/sh", command: "running", metadata: { tag: "a" } })
|
||||
const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" })
|
||||
env.daemon.output(2, "done\n")
|
||||
env.daemon.exit(2, 3)
|
||||
const exited = yield* shell.wait(finished.id)
|
||||
const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 60_000 })
|
||||
const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" })
|
||||
yield* shell.stop(stopping.id, "killed")
|
||||
yield* until(() => env.signals(4).length > 0)
|
||||
// A call waits on this one: a restart moves it to the background, so its timeout is not kept.
|
||||
const waited = yield* shell.create({ shell: "/bin/sh", command: "waited", timeout: 60_000, waited: true })
|
||||
return { running, exited, expiring, stopping, waited }
|
||||
}),
|
||||
)
|
||||
// Closing the Location only detached from its commands.
|
||||
expect(env.daemon.ops).not.toContain("terminate")
|
||||
expect(env.signals(1)).toEqual([])
|
||||
expect(env.signals(4)).toEqual(["SIGTERM"])
|
||||
// The expiring command's deadline passes while no server runs.
|
||||
const expiring = env.daemon.processes.find((process) => process.id === 3)?.pipe
|
||||
if (expiring) expiring.metadata = { ...expiring.metadata, timeoutDeadline: Date.now() - 1 }
|
||||
|
||||
const other = (yield* tmpdirScoped()).path
|
||||
yield* env.withShell(other, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
expect(yield* shell.get(before.running.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError)
|
||||
expect(yield* shell.list()).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
expect(yield* shell.get(before.running.id)).toMatchObject({
|
||||
status: "running",
|
||||
command: "running",
|
||||
shell: "/bin/sh",
|
||||
file: before.running.file,
|
||||
metadata: { tag: "a" },
|
||||
pid: before.running.pid,
|
||||
})
|
||||
expect(yield* shell.get(before.exited.id)).toEqual({ ...before.exited, cwd: env.directory })
|
||||
expect((yield* shell.result(before.exited)).capture).toMatchObject({ output: "done\n" })
|
||||
// The deadline passed during the restart, and the interrupted stop goes straight to SIGKILL.
|
||||
expect((yield* shell.wait(before.expiring.id)).status).toBe("timeout")
|
||||
expect(env.signals(3)).toEqual(["SIGTERM"])
|
||||
expect((yield* shell.wait(before.stopping.id)).status).toBe("killed")
|
||||
expect(env.signals(4)).toEqual(["SIGTERM", "SIGKILL"])
|
||||
|
||||
expect(env.daemon.processes.find((process) => process.id === 5)?.pipe?.metadata).not.toHaveProperty(
|
||||
"timeoutDeadline",
|
||||
)
|
||||
expect((yield* shell.get(before.waited.id)).status).toBe("running")
|
||||
expect(env.signals(5)).toEqual([])
|
||||
|
||||
env.daemon.exit(1, 0)
|
||||
expect(yield* shell.wait(before.running.id)).toMatchObject({ status: "exited", exit: 0 })
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
// A reload or an eviction closes a Location while the server keeps running.
|
||||
timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
const closing = yield* env.openLocation(env.directory)
|
||||
const info = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 })
|
||||
yield* closing.close
|
||||
expect(env.signals(1)).toEqual([])
|
||||
yield* TestClock.adjust("50 millis")
|
||||
yield* until(() => env.signals(1).length > 0)
|
||||
expect(env.signals(1)).toEqual(["SIGTERM"])
|
||||
expect(yield* closing.shell.wait(info.id)).toMatchObject({ status: "timeout" })
|
||||
}),
|
||||
)
|
||||
|
||||
timed("a closed Location and its replacement see the same commands, with one timer and one report each", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
const closing = yield* env.openLocation(env.directory)
|
||||
const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 })
|
||||
const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" })
|
||||
const removed = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" })
|
||||
yield* closing.close
|
||||
let exits = 0
|
||||
yield* env.serverBus.listen((event) =>
|
||||
Effect.sync(() => {
|
||||
if (event.type === "shell.exited") exits++
|
||||
}),
|
||||
)
|
||||
const replacement = yield* env.openLocation(env.directory)
|
||||
expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([
|
||||
backgrounded.id,
|
||||
stopped.id,
|
||||
removed.id,
|
||||
])
|
||||
|
||||
// The call that started it, still in the closed Location, moves it to the background and clears its timeout.
|
||||
yield* closing.shell.timeout(backgrounded.id, 0)
|
||||
yield* TestClock.adjust("100 millis")
|
||||
expect(env.signals(1)).toEqual([])
|
||||
|
||||
// Cancelling its job in the closed Location stops it; both see why it ended, and it is reported once.
|
||||
yield* closing.shell.stop(stopped.id, "killed")
|
||||
expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" })
|
||||
expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" })
|
||||
yield* until(() => exits === 1)
|
||||
|
||||
// Removed through one, it is gone from both.
|
||||
yield* replacement.shell.remove(removed.id)
|
||||
expect(yield* closing.shell.get(removed.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError)
|
||||
// The server watched them all along: the replacement took nothing back and waited on nothing more.
|
||||
expect(env.daemon.ops.filter((op) => op === "wait")).toHaveLength(3)
|
||||
yield* replacement.close
|
||||
}),
|
||||
)
|
||||
|
||||
test("finishes taking a Location's commands back even if the call that started it is interrupted", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
const created = yield* env.withShell(env.directory, (shell) =>
|
||||
shell.create({ shell: "/bin/sh", command: "running" }),
|
||||
)
|
||||
// A new server, whose first request for the Location is interrupted while the daemon is slow to list.
|
||||
const server = Context.get(
|
||||
yield* Layer.build(
|
||||
AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [
|
||||
Global.node.replace(env.global),
|
||||
PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)),
|
||||
]),
|
||||
),
|
||||
Shell.Daemon,
|
||||
)
|
||||
const place = location(Location.Ref.make({ directory: AbsolutePath.make(env.directory) }))
|
||||
let release = () => {}
|
||||
env.daemon.holdList.until = new Promise<void>((resolve) => (release = resolve))
|
||||
const lists = () => env.daemon.ops.filter((op) => op === "list").length
|
||||
const before = lists()
|
||||
const first = yield* server.at(place).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
yield* until(() => lists() > before)
|
||||
const interrupting = yield* Fiber.interrupt(first).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
env.daemon.holdList.until = undefined
|
||||
release()
|
||||
yield* Fiber.join(interrupting)
|
||||
// Recovery ran to the end, and is not run again.
|
||||
const runs = yield* server.at(place)
|
||||
expect(runs.list().map((run) => run.info.id)).toEqual([created.id])
|
||||
expect(lists()).toBe(before + 1)
|
||||
}))
|
||||
|
||||
test("the sweep keeps a running command's file, and deletes a finished one's file and forgets it", () =>
|
||||
Effect.gen(function* () {
|
||||
const env = yield* daemonShells()
|
||||
const ids = yield* env.withShell(env.directory, (shell) =>
|
||||
Effect.gen(function* () {
|
||||
const running = yield* shell.create({ shell: "/bin/sh", command: "running" })
|
||||
const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" })
|
||||
env.daemon.exit(2, 0)
|
||||
yield* shell.wait(finished.id)
|
||||
const lostFile = yield* shell.create({ shell: "/bin/sh", command: "lost file" })
|
||||
env.daemon.exit(3, 0)
|
||||
yield* shell.wait(lostFile.id)
|
||||
const old = yield* shell.create({ shell: "/bin/sh", command: "old" })
|
||||
env.daemon.exit(4, 0)
|
||||
yield* shell.wait(old.id)
|
||||
return { running, finished, lostFile, old }
|
||||
}),
|
||||
)
|
||||
// A finished process whose file was deleted elsewhere, and one that ended more than 7 days ago.
|
||||
yield* Effect.promise(() => rm(ids.lostFile.file))
|
||||
const ended = env.daemon.processes.find((process) => process.id === 4)
|
||||
if (ended) ended.exited_at = Date.now() - 8 * 24 * 60 * 60 * 1_000
|
||||
const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000)
|
||||
yield* Effect.promise(() => utimes(ids.running.file, expired, expired))
|
||||
yield* Effect.promise(() => utimes(ids.finished.file, expired, expired))
|
||||
const orphan = path.join(path.dirname(ids.running.file), "sh_0123456789abORPHANORPHANOR.out")
|
||||
yield* Effect.promise(() => writeFile(orphan, "orphan"))
|
||||
yield* Effect.promise(() => utimes(orphan, expired, expired))
|
||||
|
||||
yield* Shell.cleanup().pipe(
|
||||
Effect.provide(
|
||||
AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [
|
||||
Global.node.replace(env.global),
|
||||
PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)),
|
||||
]),
|
||||
),
|
||||
)
|
||||
|
||||
expect(existsSync(ids.running.file)).toBe(true)
|
||||
expect(existsSync(ids.finished.file)).toBe(false)
|
||||
expect(existsSync(orphan)).toBe(false)
|
||||
expect(env.daemon.processes.map((process) => process.id)).toEqual([1])
|
||||
// The old process's file is recent, so it stays.
|
||||
expect(existsSync(ids.old.file)).toBe(true)
|
||||
}))
|
||||
})
|
||||
|
||||
// Against the real opencode-pty, like server/test/persistent-pty.test.ts.
|
||||
const binary = unix ? (process.env.OPENCODE_PTY_BIN ?? binaryPath) : undefined
|
||||
const real = binary !== undefined && existsSync(binary) ? it.live : it.live.skip
|
||||
|
||||
describe("Shell on a real process daemon", () => {
|
||||
real(
|
||||
"starts the daemon on the first command, never on boot or sweep, and runs it with the server's environment",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const data = (yield* tmpdirScoped()).path
|
||||
const directory = (yield* tmpdirScoped()).path
|
||||
const state = path.join(data, "state")
|
||||
const global = Global.layerWith({ data, state })
|
||||
const started = () => existsSync(path.join(state, "pty")) && readdirSync(path.join(state, "pty")).length > 0
|
||||
process.env.OPENCODE_TEST_REMOVED = "1"
|
||||
yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED))
|
||||
yield* Effect.gen(function* () {
|
||||
const shell = yield* Shell.Service
|
||||
yield* Shell.cleanup()
|
||||
expect(started()).toBe(false)
|
||||
const info = yield* shell.create(
|
||||
{
|
||||
shell: "/bin/sh",
|
||||
command: 'printf "%s|%s|%s\\n" "${OPENCODE_TEST_REMOVED:-unset}" "$TERM" "$(pwd)"; printf tail',
|
||||
},
|
||||
(invocation) =>
|
||||
Effect.sync(() => {
|
||||
delete invocation.env.OPENCODE_TEST_REMOVED
|
||||
}),
|
||||
)
|
||||
expect(started()).toBe(true)
|
||||
expect(yield* shell.result(info)).toMatchObject({
|
||||
info: { status: "exited", exit: 0 },
|
||||
capture: {
|
||||
output: `unset|xterm-256color|${realpathSync(directory)}\ntail`,
|
||||
},
|
||||
})
|
||||
}).pipe(
|
||||
Effect.scoped,
|
||||
Effect.provide(
|
||||
AppNodeBuilder.build(LayerNode.group([Shell.node, FSUtil.node, Global.node, PersistentPty.node]), [
|
||||
Location.node.replace(
|
||||
Layer.succeed(
|
||||
Location.Service,
|
||||
Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))),
|
||||
),
|
||||
),
|
||||
Global.node.replace(global),
|
||||
Config.node.replace(Config.testLayer()),
|
||||
Environment.node.replace(hostEnvironmentLayer),
|
||||
]),
|
||||
),
|
||||
)
|
||||
}),
|
||||
{ timeout: 20_000 },
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,267 @@
|
||||
// A foreground shell call across a graceful restart, through the whole app: server 1 runs the turn, prepares a
|
||||
// handoff and is torn down mid-call, server 2 recovers the command from the same process daemon and resumes the session.
|
||||
import { describe, expect } from "bun:test"
|
||||
import path from "path"
|
||||
import { Context, Effect, Exit, Fiber, Layer, Scope } from "effect"
|
||||
import type { ChildProcess } from "effect/process"
|
||||
import { LanguageModel, LLMClient } from "@opencode/ai/effect"
|
||||
import { OpenAIChat } from "@opencode/ai/protocols/openai-chat"
|
||||
import { TestLLM } from "@opencode/ai/testing"
|
||||
import { Agent } from "@opencode/core/agent"
|
||||
import { Database } from "@opencode/core/database/database"
|
||||
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
|
||||
import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform"
|
||||
import { Watcher } from "@opencode/core/filesystem/watcher"
|
||||
import { LocationServiceMap } from "@opencode/core/location-service-map"
|
||||
import { Model } from "@opencode/core/model"
|
||||
import { Permission } from "@opencode/core/permission"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { Plugin } from "@opencode/core/plugin"
|
||||
import { Provider } from "@opencode/core/provider"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Session } from "@opencode/core/session"
|
||||
import { SessionRestart } from "@opencode/core/session/execution/restart"
|
||||
import { SessionRunnerModel } from "@opencode/core/session/runner/model"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Environment } from "@opencode/core/environment/index"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { fakeDaemon } from "./fixture/persistent-pty-daemon"
|
||||
import { recordingEnvironmentLayer } from "./fixture/environment"
|
||||
import { tempGlobalLayer } from "./fixture/global"
|
||||
import { offlineModels } from "./fixture/models"
|
||||
import { tmpdirScoped } from "./fixture/tmpdir"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { permissionLayer } from "./lib/permission"
|
||||
|
||||
const it = testEffect(tempGlobalLayer)
|
||||
|
||||
const model = SessionRunnerModel.node.replace(
|
||||
Layer.succeed(SessionRunnerModel.Service, {
|
||||
resolve: () =>
|
||||
Effect.succeed(
|
||||
SessionRunnerModel.resolved(LanguageModel.make({ id: "restart", provider: "test", route: OpenAIChat.route }), {
|
||||
capabilities: { tools: true, input: ["text"], output: ["text"] },
|
||||
cost: [],
|
||||
limit: { context: 200_000, output: 32_000 },
|
||||
}),
|
||||
),
|
||||
}),
|
||||
)
|
||||
|
||||
// Everything both servers share: the process daemon, the data directory, the database and the model.
|
||||
const world = Effect.gen(function* () {
|
||||
const daemon = yield* fakeDaemon()
|
||||
const root = (yield* tmpdirScoped()).path
|
||||
const data = path.join(root, "data")
|
||||
const global = Global.layerWith({
|
||||
data,
|
||||
cache: path.join(root, "cache"),
|
||||
config: path.join(root, "config"),
|
||||
state: path.join(root, "state"),
|
||||
tmp: path.join(root, "tmp"),
|
||||
bin: path.join(root, "cache", "bin"),
|
||||
log: path.join(data, "log"),
|
||||
repos: path.join(data, "repos"),
|
||||
})
|
||||
const database = yield* Layer.build(Database.layer())
|
||||
const llm = yield* Layer.build(TestLLM.testLayer({ fallback: TestLLM.text("Done", "done") }))
|
||||
const project = (yield* tmpdirScoped()).path
|
||||
// Every command the servers spawn instead of handing to the daemon.
|
||||
const spawns: ChildProcess.Command[] = []
|
||||
// A server: the whole app on the shared pieces, torn down when its scope closes. The daemon comes first, as in the
|
||||
// server's routes, so it is released after the sessions and jobs.
|
||||
const server = (scope: Scope.Scope) =>
|
||||
Layer.buildWithScope(
|
||||
AppNodeBuilder.build(
|
||||
LayerNode.group([
|
||||
PersistentPty.node,
|
||||
Shell.daemonNode,
|
||||
Session.node,
|
||||
LocationServiceMap.node,
|
||||
SessionRestart.node,
|
||||
]),
|
||||
[
|
||||
Global.node.replace(global),
|
||||
Database.node.replace(Layer.succeedContext(database)),
|
||||
LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))),
|
||||
PersistentPty.node.replace(PersistentPty.configured(daemon.options)),
|
||||
Environment.node.replace(recordingEnvironmentLayer(spawns)),
|
||||
Permission.node.replace(permissionLayer({ assert: () => Effect.void })),
|
||||
Watcher.node.replace(Watcher.configured({ enabled: false })),
|
||||
offlineModels,
|
||||
model,
|
||||
],
|
||||
),
|
||||
scope,
|
||||
)
|
||||
const shellSpawns = () => spawns.filter((command) => JSON.stringify(command).includes("sleep 60"))
|
||||
return { daemon, llm: Context.get(llm, TestLLM.Test), project, server, shellSpawns }
|
||||
})
|
||||
|
||||
const shellCall = (sessions: Session.Interface, sessionID: Session.ID) =>
|
||||
sessions
|
||||
.context(sessionID)
|
||||
.pipe(
|
||||
Effect.map((messages) =>
|
||||
messages.flatMap((message) =>
|
||||
message.type === "assistant" ? message.content.filter((part) => part.type === "tool") : [],
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
const until = (check: () => boolean) =>
|
||||
Effect.gen(function* () {
|
||||
while (!check()) yield* Effect.sleep(5)
|
||||
})
|
||||
|
||||
// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only).
|
||||
describe.skipIf(process.platform === "win32")("shell commands across a graceful restart", () => {
|
||||
for (const finished of [false, true])
|
||||
it.live(
|
||||
`settles a foreground call ${finished ? "whose command finished during the restart" : "whose command still runs"} and resumes its session`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const { daemon, llm, project, server } = yield* world
|
||||
// The turn's first request runs the command; anything else (a title, the resumed turn) just answers.
|
||||
let turns = 0
|
||||
yield* llm.serve((request) =>
|
||||
request.tools.some((tool) => tool.name === "shell") && turns++ === 0
|
||||
? TestLLM.toolCall("call-restart", "shell", { command: "sleep 60" })
|
||||
: TestLLM.text("Done", "done"),
|
||||
)
|
||||
|
||||
// Server 1 starts the turn, which waits on the command, then shuts down.
|
||||
const first = yield* Scope.make()
|
||||
const one = yield* server(first)
|
||||
const sessions = Context.get(one, Session.Service)
|
||||
const session = yield* sessions.create({
|
||||
location: { directory: AbsolutePath.make(project) },
|
||||
agent: Agent.ID.make("build"),
|
||||
model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }),
|
||||
})
|
||||
yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe(
|
||||
Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)),
|
||||
)
|
||||
yield* sessions.prompt({ sessionID: session.id, text: "run it" })
|
||||
yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0)
|
||||
const command = daemon.processes[0]!
|
||||
// A graceful restart: the server hands its daemon over to the next one, then shuts down.
|
||||
expect(yield* Context.get(one, PersistentPty.Service).handoff()).not.toBeNull()
|
||||
yield* Scope.close(first, Exit.void)
|
||||
|
||||
// The shutdown stopped nothing and settled nothing.
|
||||
expect(command.lifecycle.status).toBe("running")
|
||||
expect(daemon.ops).not.toContain("signal")
|
||||
expect(turns).toBe(1)
|
||||
if (finished) {
|
||||
daemon.output(command.id, "finished output\n")
|
||||
daemon.exit(command.id, 0)
|
||||
}
|
||||
|
||||
// Server 2 takes the command back, settles the call and resumes the turn.
|
||||
const second = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(second, Exit.void))
|
||||
const two = yield* server(second)
|
||||
const resumed = Context.get(two, Session.Service)
|
||||
yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions
|
||||
yield* until(() => turns >= 2)
|
||||
const shellID = String(command.pipe?.metadata.id)
|
||||
expect(yield* shellCall(resumed, session.id)).toMatchObject([
|
||||
{
|
||||
id: "call-restart",
|
||||
state: {
|
||||
status: "completed",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: expect.stringContaining(`Command moved to the background (shell ID: ${shellID})`),
|
||||
},
|
||||
{ type: "text", text: expect.stringContaining("You will be notified automatically") },
|
||||
],
|
||||
},
|
||||
},
|
||||
])
|
||||
if (!finished) {
|
||||
// The resumed turn ends while the command still runs; its completion wakes the session again.
|
||||
yield* resumed.wait(session.id)
|
||||
expect(turns).toBe(2)
|
||||
daemon.exit(command.id, 0)
|
||||
yield* until(() => turns >= 3)
|
||||
}
|
||||
// Either way the session hears how it ended.
|
||||
const notices = Effect.map(resumed.context(session.id), (messages) =>
|
||||
messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"),
|
||||
)
|
||||
while ((yield* notices).length === 0) yield* Effect.sleep(5)
|
||||
expect((yield* notices)[0]).toMatchObject({ metadata: { shellID, state: "completed" } })
|
||||
if (finished) expect((yield* notices)[0]).toMatchObject({ text: expect.stringContaining("finished output") })
|
||||
yield* resumed.wait(session.id)
|
||||
}),
|
||||
)
|
||||
|
||||
// Without a handoff nothing takes the commands over: the shutdown stops them, while the daemon is still reachable.
|
||||
for (const starting of [false, true])
|
||||
it.live(
|
||||
`stops a command ${starting ? "still being created" : "a foreground call waits on"} when the server shuts down without a handoff`,
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const { daemon, llm, project, server, shellSpawns } = yield* world
|
||||
let turns = 0
|
||||
yield* llm.serve((request) =>
|
||||
request.tools.some((tool) => tool.name === "shell") && turns++ === 0
|
||||
? TestLLM.toolCall("call-shutdown", "shell", { command: "sleep 60" })
|
||||
: TestLLM.text("Done", "done"),
|
||||
)
|
||||
let release = () => {}
|
||||
if (starting) daemon.holdCreate.until = new Promise<void>((resolve) => (release = resolve))
|
||||
|
||||
const first = yield* Scope.make()
|
||||
const one = yield* server(first)
|
||||
const sessions = Context.get(one, Session.Service)
|
||||
const session = yield* sessions.create({
|
||||
location: { directory: AbsolutePath.make(project) },
|
||||
agent: Agent.ID.make("build"),
|
||||
model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }),
|
||||
})
|
||||
yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe(
|
||||
Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)),
|
||||
)
|
||||
yield* sessions.prompt({ sessionID: session.id, text: "run it" })
|
||||
if (starting) yield* until(() => daemon.ops.includes("create_pipe"))
|
||||
else yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0)
|
||||
|
||||
const closing = yield* Scope.close(first, Exit.void).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
if (starting) {
|
||||
yield* Effect.sleep(20)
|
||||
daemon.holdCreate.until = undefined
|
||||
release()
|
||||
}
|
||||
yield* Fiber.join(closing)
|
||||
|
||||
// The command reached the daemon, and the shutdown stopped it there; nothing was spawned instead.
|
||||
const command = daemon.processes[0]
|
||||
expect(command).toBeDefined()
|
||||
expect(daemon.requests).toContainEqual(
|
||||
expect.objectContaining({ op: "signal", id: command?.id, signal: "SIGTERM" }),
|
||||
)
|
||||
expect(command?.lifecycle.status).toBe("exited")
|
||||
expect(shellSpawns()).toEqual([])
|
||||
|
||||
// The call was settled on the way down, and the next server resumes the session.
|
||||
const second = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(second, Exit.void))
|
||||
const two = yield* server(second)
|
||||
const resumed = Context.get(two, Session.Service)
|
||||
expect(yield* shellCall(resumed, session.id)).toMatchObject([
|
||||
{
|
||||
id: "call-shutdown",
|
||||
state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } },
|
||||
},
|
||||
])
|
||||
yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions
|
||||
yield* until(() => turns >= 2)
|
||||
yield* resumed.wait(session.id)
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -46,3 +46,20 @@ describe("ShellResult.interrupted", () => {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("ShellResult.output", () => {
|
||||
test("reports a command killed without an exit code or signal of its own", () => {
|
||||
const lost = ShellResult.output({
|
||||
info: info("killed"),
|
||||
capture: { output: "partial\n", truncated: false, killed: "Command was killed: the process daemon stopped" },
|
||||
})
|
||||
expect(ShellResult.notice(lost)).toBe("Command was killed: the process daemon stopped")
|
||||
expect(ShellResult.metadata(lost)).toMatchObject({ killed: true })
|
||||
})
|
||||
|
||||
test("keeps a removed command's result unchanged", () => {
|
||||
const removed = ShellResult.output({ info: info("killed"), capture: undefined })
|
||||
expect(removed).toEqual({ output: ShellResult.unavailable.output, truncated: false })
|
||||
expect(ShellResult.notice(removed)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -10,6 +10,8 @@ import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Workspace } from "@opencode/core/workspace"
|
||||
import { Global } from "@opencode/util/global"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { disabledPersistentPtyLayer } from "./fixture/persistent-pty"
|
||||
import { hostEnvironmentLayer } from "./fixture/environment"
|
||||
import { tempGlobalLayer } from "./fixture/global"
|
||||
import { location, tempLocationLayer } from "./fixture/location"
|
||||
@@ -95,6 +97,7 @@ it.live("eviction makes progress past an already-removed shell", () =>
|
||||
Global.node.replace(tempGlobalLayer),
|
||||
Config.node.replace(Config.testLayer()),
|
||||
Environment.node.replace(environment),
|
||||
PersistentPty.node.replace(disabledPersistentPtyLayer),
|
||||
]),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -42,6 +42,10 @@ import { ShellTool } from "@opencode/core/tool/plugin/shell"
|
||||
import { ToolOutput } from "@opencode/core/tool-output"
|
||||
import { Tool } from "@opencode/core/tool"
|
||||
import { PluginHooks } from "@opencode/core/plugin/hooks"
|
||||
import { SessionStep } from "@opencode/core/session/runner/step"
|
||||
import { PersistentPty } from "@opencode/core/persistent-pty"
|
||||
import { disabledPersistentPtyLayer } from "./fixture/persistent-pty"
|
||||
import { fakeDaemon } from "./fixture/persistent-pty-daemon"
|
||||
import { tmpdir, tmpdirScoped, withTempDir } from "./fixture/tmpdir"
|
||||
import { tempGlobalLayer } from "./fixture/global"
|
||||
import { offlineModels } from "./fixture/models"
|
||||
@@ -142,12 +146,15 @@ const shellPluginSupervisor = makeLocationNode({
|
||||
Session.node,
|
||||
Job.node,
|
||||
Shell.node,
|
||||
Shell.daemonNode,
|
||||
ShellSelect.node,
|
||||
Tool.node,
|
||||
],
|
||||
})
|
||||
|
||||
const nodes = LayerNode.group([
|
||||
PersistentPty.node,
|
||||
Shell.daemonNode,
|
||||
Database.node,
|
||||
Bus.node,
|
||||
Job.node,
|
||||
@@ -158,21 +165,61 @@ const nodes = LayerNode.group([
|
||||
FSUtil.node,
|
||||
Global.node,
|
||||
])
|
||||
const replacements = [
|
||||
const daemonReplacements = [
|
||||
SessionExecution.node.replace(executionNode),
|
||||
Permission.node.replace(permission),
|
||||
Global.node.replace(tempGlobalLayer),
|
||||
offlineModels,
|
||||
] satisfies LayerNode.Replacements
|
||||
// Most suites spawn commands from the server, so CI starts no daemon for them; `parity` runs a subset on both.
|
||||
const replacements = [
|
||||
...daemonReplacements,
|
||||
PersistentPty.node.replace(disabledPersistentPtyLayer),
|
||||
] satisfies LayerNode.Replacements
|
||||
const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements))
|
||||
const it = testEffect(
|
||||
AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]),
|
||||
)
|
||||
// On the real process daemon (when its binary is compatible; otherwise this falls back to spawning).
|
||||
const daemonIt = testEffect(
|
||||
AppNodeBuilder.build(nodes, [...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor)]),
|
||||
)
|
||||
const parity = {
|
||||
live: (name: string, body: Parameters<typeof it.live>[1], options?: Parameters<typeof it.live>[2]) => {
|
||||
it.live(`${name} (spawned)`, body, options)
|
||||
daemonIt.live(`${name} (daemon)`, body, options)
|
||||
},
|
||||
skip: (name: string, body: Parameters<typeof it.live>[1], options?: Parameters<typeof it.live>[2]) =>
|
||||
it.live.skip(name, body, options),
|
||||
}
|
||||
// On an in-process fake daemon, so daemon-only behavior is covered without the real binary.
|
||||
let fake: Effect.Success<ReturnType<typeof fakeDaemon>> | undefined
|
||||
const fakeDaemonIt = testEffect(
|
||||
AppNodeBuilder.build(nodes, [
|
||||
...daemonReplacements,
|
||||
PluginSupervisor.node.replace(shellPluginSupervisor),
|
||||
PersistentPty.node.replace(
|
||||
makeGlobalNode({
|
||||
service: PersistentPty.Service,
|
||||
layer: Layer.unwrap(
|
||||
fakeDaemon().pipe(
|
||||
Effect.map((daemon) => {
|
||||
fake = daemon
|
||||
return PersistentPty.layerWith(daemon.options)
|
||||
}),
|
||||
),
|
||||
),
|
||||
deps: [Bus.node, Global.node],
|
||||
}),
|
||||
),
|
||||
]),
|
||||
)
|
||||
const permissionIt = testEffect(
|
||||
AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [
|
||||
SessionExecution.node.replace(executionNode),
|
||||
Global.node.replace(tempGlobalLayer),
|
||||
PluginSupervisor.node.replace(shellPluginSupervisor),
|
||||
PersistentPty.node.replace(disabledPersistentPtyLayer),
|
||||
offlineModels,
|
||||
]),
|
||||
)
|
||||
@@ -184,6 +231,21 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({
|
||||
})
|
||||
|
||||
const isWindows = process.platform === "win32"
|
||||
// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so these cases have no daemon there.
|
||||
const daemonLive = isWindows ? fakeDaemonIt.live.skip : fakeDaemonIt.live
|
||||
|
||||
// The process daemon keeps a finished command's zombie until the command is forgotten.
|
||||
const running = (pid: number) => {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
return !Bun.spawnSync(["ps", "-o", "stat=", "-p", String(pid)])
|
||||
.stdout.toString()
|
||||
.trim()
|
||||
.startsWith("Z")
|
||||
}
|
||||
const cwdCommand = isWindows ? "(Get-Location).Path; Start-Sleep -Milliseconds 100" : "pwd"
|
||||
const helloCommand = isWindows ? "[Console]::Out.Write('hello'); Start-Sleep -Milliseconds 100" : "printf hello"
|
||||
const stderrCommand = isWindows
|
||||
@@ -1199,6 +1261,7 @@ describe("ShellTool", () => {
|
||||
Session.node,
|
||||
Job.node,
|
||||
Shell.node,
|
||||
Shell.daemonNode,
|
||||
ShellSelect.node,
|
||||
Tool.node,
|
||||
],
|
||||
@@ -1268,7 +1331,7 @@ describe("ShellTool", () => {
|
||||
)
|
||||
}
|
||||
|
||||
it.live("keeps non-zero exits useful", () =>
|
||||
parity.live("keeps non-zero exits useful", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
@@ -1558,7 +1621,7 @@ describe("ShellTool", () => {
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
it.live("uses configured line limits", () =>
|
||||
parity.live("uses configured line limits", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return Effect.gen(function* () {
|
||||
@@ -1742,7 +1805,79 @@ describe("ShellTool", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("returns the shell id and PID for a background command", () =>
|
||||
daemonLive("keeps a background daemon command's job across a Location reload and reports how it ended", () =>
|
||||
withTempDir((tmp) =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
const sessions = yield* Session.Service
|
||||
const jobs = yield* Job.Service
|
||||
const bus = yield* Bus.Service
|
||||
const locations = yield* LocationServiceMap.Service
|
||||
const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) })
|
||||
yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel })
|
||||
const admitted = yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe(
|
||||
Stream.filter((event) => event.data.sessionID === sessionID && event.data.item.type === "synthetic"),
|
||||
Stream.take(2),
|
||||
Stream.runCollect,
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
const started = yield* Effect.gen(function* () {
|
||||
const plugins = yield* Plugin.Service
|
||||
yield* plugins.awaitActivation
|
||||
const registry = yield* Tool.Service
|
||||
const shell = yield* Shell.Service
|
||||
const run = (id: string) =>
|
||||
Effect.gen(function* () {
|
||||
const settled = yield* executeTool(registry, call({ command: "work", background: true }, id))
|
||||
return yield* shell.get(ID.make(String(settled.metadata?.shellID)))
|
||||
})
|
||||
return { finishing: yield* run("call-finishing"), cancelled: yield* run("call-cancelled") }
|
||||
}).pipe(Effect.provide(locations.get(location)))
|
||||
const process = (info: { file: string }) => {
|
||||
const found = fake?.processes.find((item) => item.pipe?.output_path === info.file)
|
||||
if (!found) throw new Error("no daemon process for the command")
|
||||
return found.id
|
||||
}
|
||||
|
||||
// A reload closes the Location and boots its replacement, which takes the commands back.
|
||||
yield* locations.invalidate(location)
|
||||
const replacement = yield* Effect.gen(function* () {
|
||||
return yield* Shell.Service
|
||||
}).pipe(Effect.provide(locations.get(location)))
|
||||
expect((yield* replacement.list()).map((info) => info.id).toSorted()).toEqual(
|
||||
[started.finishing.id, started.cancelled.id].toSorted(),
|
||||
)
|
||||
expect((yield* jobs.get(started.finishing.id))?.status).toBe("running")
|
||||
|
||||
// Cancelling the session's job still stops its command.
|
||||
yield* jobs.cancel(started.cancelled.id)
|
||||
while (!fake?.ops.includes("signal")) yield* Effect.promise(() => Bun.sleep(2))
|
||||
expect(fake?.requests).toContainEqual(
|
||||
expect.objectContaining({ op: "signal", id: process(started.cancelled), signal: "SIGTERM" }),
|
||||
)
|
||||
|
||||
fake?.output(process(started.finishing), "done\n")
|
||||
fake?.exit(process(started.finishing), 0)
|
||||
const notices = Array.from(yield* Fiber.join(admitted)).map((event) => event.data.item.payload)
|
||||
expect(notices).toContainEqual(
|
||||
expect.objectContaining({
|
||||
text: expect.stringContaining("done"),
|
||||
metadata: expect.objectContaining({ shellID: started.finishing.id, state: "completed", exit: 0 }),
|
||||
}),
|
||||
)
|
||||
expect(notices).toContainEqual(
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({ shellID: started.cancelled.id, state: "cancelled" }),
|
||||
}),
|
||||
)
|
||||
expect((yield* jobs.get(started.finishing.id))?.status).toBe("completed")
|
||||
expect((yield* replacement.get(started.finishing.id)).status).toBe("exited")
|
||||
yield* locations.invalidate(location)
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
parity.live("returns the shell id and PID for a background command", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
@@ -1910,8 +2045,8 @@ describe("ShellTool", () => {
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
const signalTest = isWindows ? it.live.skip : it.live
|
||||
signalTest(
|
||||
const signalParity = isWindows ? parity.skip : parity.live
|
||||
signalParity(
|
||||
"escalates a shell timeout when the ready process ignores SIGTERM",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
@@ -1946,7 +2081,7 @@ describe("ShellTool", () => {
|
||||
})
|
||||
const pid = info.pid
|
||||
if (pid === undefined) throw new Error("Expected shell PID")
|
||||
expect(() => process.kill(pid, 0)).toThrow()
|
||||
expect(running(pid)).toBe(false)
|
||||
}),
|
||||
(info) =>
|
||||
Effect.try(() => {
|
||||
@@ -2009,7 +2144,7 @@ describe("ShellTool", () => {
|
||||
)
|
||||
}
|
||||
|
||||
it.live("backgrounds a foreground command when the session is signaled", () =>
|
||||
parity.live("backgrounds a foreground command when the session is signaled", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
@@ -2089,7 +2224,7 @@ describe("ShellTool stopped", () => {
|
||||
return { shellID, exit, result: Exit.isSuccess(exit) ? exit.value : undefined }
|
||||
})
|
||||
|
||||
const test = isWindows ? it.live.skip : it.live
|
||||
const test = isWindows ? parity.skip : parity.live
|
||||
|
||||
test("stops a command the user stopped, keeps its file and fails the call with the tail, file and shell ID", () =>
|
||||
withTempDir((tmp) => {
|
||||
@@ -2181,6 +2316,155 @@ describe("ShellTool stopped", () => {
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
|
||||
// Starts a call whose daemon command is slow to be created, and returns once the daemon was asked for it.
|
||||
const startSlowCall = (registry: Tool.Interface, id: string) =>
|
||||
Effect.gen(function* () {
|
||||
let release = () => {}
|
||||
if (fake) fake.holdCreate.until = new Promise<void>((resolve) => (release = resolve))
|
||||
// The step's own gate: a stop goes through `cancel` only if the tool is awaiting it.
|
||||
const cancel = SessionStep.makeCancel()
|
||||
const fiber = yield* executeTool(registry, {
|
||||
...call({ command: "sleep 60" }, id),
|
||||
cancel: cancel.effect,
|
||||
}).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
while (!fake?.ops.includes("create_pipe")) yield* Effect.promise(() => Bun.sleep(2))
|
||||
return {
|
||||
fiber,
|
||||
stop: Effect.sync(() => cancel.trigger()),
|
||||
release: Effect.sync(() => {
|
||||
if (fake) fake.holdCreate.until = undefined
|
||||
release()
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
daemonLive("stops a command the user stopped while it was being created", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
Effect.gen(function* () {
|
||||
const jobs = yield* Job.Service
|
||||
const shell = yield* Shell.Service
|
||||
const call = yield* startSlowCall(registry, "call-slow-create")
|
||||
// The tool already listens while its command is being created, so the step would not interrupt it.
|
||||
expect(yield* call.stop).toBe(true)
|
||||
yield* call.release
|
||||
const result = yield* Fiber.join(call.fiber)
|
||||
expect(result.error?.message).toContain("Command interrupted by the user")
|
||||
const process = fake?.processes[0]
|
||||
expect(fake?.requests).toContainEqual(
|
||||
expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }),
|
||||
)
|
||||
expect(process?.lifecycle.status).toBe("exited")
|
||||
expect(yield* shell.list()).toEqual([])
|
||||
expect((yield* jobs.get(String(process?.pipe?.metadata.id)))?.status).not.toBe("running")
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
daemonLive("stops a command the user stopped after its Location reloaded", () =>
|
||||
withTempDir((tmp) =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
const sessions = yield* Session.Service
|
||||
const locations = yield* LocationServiceMap.Service
|
||||
const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) })
|
||||
yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel })
|
||||
yield* Effect.gen(function* () {
|
||||
const plugins = yield* Plugin.Service
|
||||
yield* plugins.awaitActivation
|
||||
const registry = yield* Tool.Service
|
||||
const jobs = yield* Job.Service
|
||||
const cancel = SessionStep.makeCancel()
|
||||
const started = yield* Deferred.make<string>()
|
||||
const fiber = yield* executeTool(registry, {
|
||||
...call({ command: "sleep 60" }, "call-reloaded"),
|
||||
progress: (update) =>
|
||||
typeof update.shellID === "string"
|
||||
? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid)
|
||||
: Effect.void,
|
||||
cancel: cancel.effect,
|
||||
}).pipe(Effect.forkChild({ startImmediately: true }))
|
||||
const shellID = yield* Deferred.await(started)
|
||||
// The call still holds the old Location when a reload closes it.
|
||||
yield* locations.invalidate(location)
|
||||
expect(cancel.trigger()).toBe(true)
|
||||
const result = yield* Fiber.join(fiber)
|
||||
expect(result.error?.message).toContain(`Command interrupted by the user (shell ID: ${shellID})`)
|
||||
expect(fake?.processes[0]?.lifecycle.status).toBe("exited")
|
||||
expect((yield* jobs.get(shellID))?.status).not.toBe("running")
|
||||
}).pipe(Effect.provide(locations.get(location)))
|
||||
yield* locations.invalidate(location)
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
for (const handoff of [false, true])
|
||||
for (const starting of [true, false])
|
||||
daemonLive(
|
||||
`${handoff ? "leaves a command running" : "stops a command"} when an interrupt ${handoff ? "during a handoff " : ""}cuts its call off while it ${starting ? "starts" : "runs"}`,
|
||||
() =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
Effect.gen(function* () {
|
||||
const jobs = yield* Job.Service
|
||||
const pty = yield* PersistentPty.Service
|
||||
const call = yield* startSlowCall(registry, "call-interrupted")
|
||||
if (handoff) expect(yield* pty.handoff()).not.toBeNull()
|
||||
if (!starting) {
|
||||
yield* call.release
|
||||
while (fake!.waiters(fake!.processes[0]?.id ?? -1) === 0) yield* Effect.sleep(2)
|
||||
}
|
||||
const interrupting = yield* Fiber.interrupt(call.fiber).pipe(
|
||||
Effect.forkChild({ startImmediately: true }),
|
||||
)
|
||||
if (starting) yield* call.release
|
||||
yield* Fiber.join(interrupting)
|
||||
expect(Exit.hasInterrupts(yield* Fiber.await(call.fiber))).toBe(true)
|
||||
const process = fake?.processes[0]
|
||||
const shellID = String(process?.pipe?.metadata.id)
|
||||
if (handoff) {
|
||||
// Nothing stopped it: the next server takes it over through its recorded tool call.
|
||||
expect(process?.lifecycle.status).toBe("running")
|
||||
expect(process?.pipe?.metadata).toMatchObject({ toolCall: { callID: "call-interrupted" } })
|
||||
expect(fake?.ops).not.toContain("signal")
|
||||
expect((yield* jobs.get(shellID))?.status).toBe("running")
|
||||
yield* jobs.cancel(shellID)
|
||||
return
|
||||
}
|
||||
// Nothing will take it over, so it is stopped and its job ends.
|
||||
const shell = yield* Shell.Service
|
||||
expect((yield* shell.wait(ID.make(shellID))).status).toBe("killed")
|
||||
expect(fake?.requests).toContainEqual(
|
||||
expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }),
|
||||
)
|
||||
expect(process?.lifecycle.status).toBe("exited")
|
||||
expect((yield* jobs.get(shellID))?.status).toBe("cancelled")
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
daemonLive("records which call waits on a daemon command, and keeps no timeout for a restart", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
Effect.gen(function* () {
|
||||
const { shellID } = yield* stopCall(registry, "sleep 60", "call-recorded", "interrupt", false)
|
||||
const request = fake?.requests.find((item) => item.op === "create_pipe")
|
||||
expect(request?.metadata).toMatchObject({
|
||||
id: shellID,
|
||||
toolCall: { sessionID, callID: "call-recorded" },
|
||||
})
|
||||
// The call's timeout applies to this server only: a restart moves the command to the background.
|
||||
expect(request?.metadata).not.toHaveProperty("timeoutDeadline")
|
||||
}),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
test("lets an execute.after hook replace a stopped call's failure", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
@@ -2209,18 +2493,17 @@ describe("ShellTool stopped", () => {
|
||||
)
|
||||
}))
|
||||
|
||||
test("leaves a call a shutdown interrupted to the step", () =>
|
||||
test("stops a command whose call an interrupt cut off, with no handoff to take it over", () =>
|
||||
withTempDir((tmp) => {
|
||||
reset()
|
||||
return withSession(tmp.path, (registry) =>
|
||||
Effect.gen(function* () {
|
||||
const shell = yield* Shell.Service
|
||||
const jobs = yield* Job.Service
|
||||
const { shellID, exit } = yield* stopCall(registry, "printf ready; sleep 60", "call-teardown", "interrupt")
|
||||
expect(Exit.hasInterrupts(exit)).toBe(true)
|
||||
// The job is cancelled as before, which stops the command; its file stays.
|
||||
const info = yield* shell.wait(shellID)
|
||||
expect(info.status).toBe("killed")
|
||||
expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("ready")
|
||||
expect((yield* shell.wait(shellID)).status).toBe("killed")
|
||||
expect((yield* jobs.get(shellID))?.status).toBe("cancelled")
|
||||
}),
|
||||
)
|
||||
}))
|
||||
|
||||
@@ -20,6 +20,7 @@ import { Instance } from "@opencode/core/instance/service"
|
||||
import { SessionTransfer } from "@opencode/core/session/transfer"
|
||||
import { InstructionEntry } from "@opencode/core/session/instruction-entry"
|
||||
import { ShellSelect } from "@opencode/core/shell/select"
|
||||
import { Shell } from "@opencode/core/shell"
|
||||
import { Job } from "@opencode/core/job"
|
||||
import { KV } from "@opencode/core/kv"
|
||||
import { Mcp } from "@opencode/core/mcp/index"
|
||||
@@ -60,6 +61,11 @@ const applicationServiceNodes = [
|
||||
Database.node,
|
||||
KV.node,
|
||||
Bus.node,
|
||||
// Before the jobs and sessions, so they are released after them: a shutdown's interrupted shell calls and jobs
|
||||
// still reach the daemon to stop their commands, and a command being created never falls back to spawning.
|
||||
PersistentPty.node,
|
||||
// Daemon-run shell commands belong to the server, so a Location closing never takes them down.
|
||||
Shell.daemonNode,
|
||||
EventLogger.node,
|
||||
httpClient,
|
||||
Job.node,
|
||||
@@ -74,7 +80,6 @@ const applicationServiceNodes = [
|
||||
PluginUpdate.node,
|
||||
PermissionSaved.node,
|
||||
PtyTicket.node,
|
||||
PersistentPty.node,
|
||||
Credential.node,
|
||||
WellKnown.node,
|
||||
ServerPairing.node,
|
||||
|
||||
Reference in New Issue
Block a user