From eee9e31a7905e5f902a56244eec77c1f4c141a4f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 16 Nov 2026 19:02:12 -0500 Subject: [PATCH] fix: Deliver Lane Cleanup and Quarantine Receipts When a Worker Run Stops A concurrent worker run that stopped after a lane's settlement committed dropped the cleanup acknowledgement and the quarantine receipt, because both were bound to the run's abort signal. Code API offers only unsettled work on reconnect, so nothing re-sent them and the lane's fence stayed until reset, also blocking its parent checkout. One failed registration heartbeat was enough to stop every slot this way. The cleanup acknowledgement for a committed settlement and the quarantine receipt are now delivered with their bounded retries regardless of the run stopping. A quarantine receipt whose delivery stays uncertain is kept and delivered after the next registration, before any slot leases. A concurrent run absorbs one transient heartbeat failure before stopping its slots. Uncertain outcomes still quarantine; the receipt now also reaches Code API. --- packages/code/src/worker-receipts.test.ts | 268 ++++++++++++++++++++++ packages/code/src/worker.ts | 64 ++++-- 2 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 packages/code/src/worker-receipts.test.ts diff --git a/packages/code/src/worker-receipts.test.ts b/packages/code/src/worker-receipts.test.ts new file mode 100644 index 00000000..425f564d --- /dev/null +++ b/packages/code/src/worker-receipts.test.ts @@ -0,0 +1,268 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { BridgeWorker } from './worker.js'; +import type { WorkspaceMutationQuarantine } from './worker.js'; +import type { + BridgeAssignment, + BridgeWorkspaceToolCapabilities, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; + +const incarnationId = 'incarnation-receipts'; +const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: 1, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'a' }], +}; + +interface Exchange { + action: string; + body: Record; +} + +interface Fixture { + operation: 'read_file' | 'execute_command'; + execute(signal: AbortSignal | undefined, controller: AbortController): Promise; + guard?: Partial; + /** A response that replaces a registration, by 1-based registration attempt. */ + registration?: (attempt: number) => Response | undefined; + /** Throw to model a lost control receipt, by 1-based delivery attempt. */ + receipt?: (action: string, attempt: number) => void; + leaseTtlMs?: number; +} + +function result(request: WorkspaceToolRequest): WorkspaceToolResult { + return (request.operation === 'execute_command' + ? { protocolVersion: 1, operation: 'execute_command', workspaceId: 'a', stdout: 'done', stderr: '', exitCode: 0, truncated: false, timedOut: false } + : { protocolVersion: 1, operation: 'read_file', workspaceId: 'a', path: 'file.txt', content: 'done', startLine: 1, endLine: 1, truncated: false }) as WorkspaceToolResult; +} + +/** A worker on one lane-scoped slot against a fixture bridge that records every control exchange. */ +function laneWorker(fixture: Fixture): { + worker: BridgeWorker; + exchanges: Exchange[]; + errors: unknown[]; + run(): { controller: AbortController; done: Promise }; +} { + const exchanges: Exchange[] = []; + const errors: unknown[] = []; + const attempts = new Map(); + let registrations = 0; + let leased = false; + let controller = new AbortController(); + const request = { + protocolVersion: 1, + operation: fixture.operation, + workspaceId: 'a', + ...(fixture.operation === 'execute_command' ? { command: 'true' } : { path: 'file.txt' }), + } as WorkspaceToolRequest; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId, + sandboxEndpoint: 'http://localhost:2', + leaseWaitMs: 10, + reconnectDelayMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + mutationFailuresAreAtomic: true, + execute: async (_request, signal) => await fixture.execute(signal, controller), + }, + workspaceQuarantines: new Map([ + ['a', { + async assertAvailable() {}, + async arm() {}, + async clear() {}, + async quarantine() {}, + ...fixture.guard, + }], + ]), + onError: (error) => errors.push(error), + fetchImpl: async (url, init) => { + const path = new URL(String(url)).pathname; + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + const action = path.split('/').at(-1)!; + if (action === 'register') { + registrations += 1; + return fixture.registration?.(registrations) ?? Response.json({ + protocolVersion: 1, workerId: 'worker', incarnationId, + registrationGeneration: registrations, registeredAt: new Date().toISOString(), + leaseTtlMs: fixture.leaseTtlMs ?? 60_000, workspaceLeaseSlots: 2, + supportedWorkspaceToolOperations: capabilities.operations, + }); + } + if (action === 'ready') return Response.json({ protocolVersion: 1, ready: true }); + if (action === 'lease') { + exchanges.push({ action, body }); + if (!leased && body.workspaceLeaseSlot === 0) { + leased = true; + return Response.json({ + protocolVersion: 1, serverElapsedMs: 0, + assignment: { + protocolVersion: 1, assignmentId: 'assignment-a', workerId: 'worker', incarnationId, + generation: 1, leaseToken: 'l'.repeat(43), workspaceLeaseSlot: 0, + expiresAt: new Date(Date.now() + 30_000).toISOString(), remainingMs: 30_000, + executionKind: 'workspace_tool', request, + } satisfies BridgeAssignment, + }); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + } + if (action === 'ack') return Response.json({ protocolVersion: 1, accepted: true }); + if (action === 'cancellation') return Response.json({ protocolVersion: 1, cancelled: false }); + const attempt = (attempts.get(action) ?? 0) + 1; + attempts.set(action, attempt); + fixture.receipt?.(action, attempt); + exchanges.push({ action, body }); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + return { + worker, + exchanges, + errors, + run() { + controller = new AbortController(); + return { controller, done: worker.run(controller.signal) }; + }, + }; +} + +function actions(exchanges: Exchange[]): string[] { + return exchanges.map((exchange) => exchange.action).filter((action) => action !== 'lease'); +} + +function receipt(exchanges: Exchange[], action: string): Record | undefined { + return exchanges.find((exchange) => exchange.action === action)?.body; +} + +async function until(condition: () => boolean, message: string): Promise { + for (let attempt = 0; attempt < 500 && !condition(); attempt++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + assert.ok(condition(), message); +} + +for (const operation of ['read_file', 'execute_command'] as const) { + test(`a committed ${operation} result still releases its lane when the run stops before cleanup is acknowledged`, async () => { + let stop!: () => void; + const lane = laneWorker({ + operation, + execute: async (_signal, controller) => { + stop = () => controller.abort(); + return result({ operation } as WorkspaceToolRequest); + }, + // The run stops in the gap between the committed settlement and the cleanup acknowledgement. + receipt: (action) => { + if (action === 'settle') stop(); + }, + }); + await lane.run().done; + assert.deepEqual(actions(lane.exchanges), ['settle', 'workspace-cleanup']); + assert.equal(receipt(lane.exchanges, 'settle')?.status, 'fulfilled'); + }); +} + +test('an interrupted command still quarantines its lane, and the receipt is delivered after the run stops', async () => { + let quarantined = 0; + const lane = laneWorker({ + operation: 'execute_command', + guard: { async quarantine() { quarantined += 1; } }, + execute: async (signal, controller) => { + controller.abort(); + assert.equal(signal?.aborted, true); + throw new DOMException('aborted', 'AbortError'); + }, + }); + await lane.run().done; + assert.equal(quarantined, 1); + assert.deepEqual(actions(lane.exchanges), ['quarantine']); +}); + +test('a quarantine receipt that cannot be delivered while the run stops is delivered before the next run leases', async () => { + let outage = true; + const lane = laneWorker({ + operation: 'execute_command', + execute: async (_signal, controller) => { + controller.abort(); + throw new DOMException('aborted', 'AbortError'); + }, + receipt: () => { + if (outage) throw new TypeError('fetch failed'); + }, + }); + await lane.run().done; + assert.deepEqual(actions(lane.exchanges), []); + + outage = false; + lane.exchanges.length = 0; + const next = lane.run(); + try { + await until(() => lane.exchanges.some((exchange) => exchange.action === 'lease'), 'the next run never leased'); + } finally { + next.controller.abort(); + await next.done; + } + assert.equal(lane.exchanges[0].action, 'quarantine'); + assert.equal(receipt(lane.exchanges, 'quarantine')?.leaseToken, 'l'.repeat(43)); + + lane.exchanges.length = 0; + const later = lane.run(); + try { + await until(() => lane.exchanges.some((exchange) => exchange.action === 'lease'), 'the later run never leased'); + } finally { + later.controller.abort(); + await later.done; + } + assert.deepEqual(actions(lane.exchanges), []); +}); + +for (const failures of [1, 2]) { + test(`${failures} consecutive failed heartbeat${failures === 1 ? ' is retried' : 's stop every slot'}`, async () => { + let started = false; + let failed = 0; + let executionAborted: boolean | undefined; + const lane = laneWorker({ + operation: 'read_file', + leaseTtlMs: 100, + registration: () => { + if (!started || failed >= failures) return undefined; + failed += 1; + return Response.json({ error: 'Bad gateway' }, { status: 502 }); + }, + execute: async (signal) => { + started = true; + await until(() => failed === failures || signal?.aborted === true, 'the heartbeat never failed'); + await new Promise((resolve) => setTimeout(resolve, 150)); + executionAborted = signal?.aborted === true; + return result({ operation: 'read_file' } as WorkspaceToolRequest); + }, + }); + const { controller, done } = lane.run(); + try { + await until(() => executionAborted !== undefined, 'the execution never finished'); + if (failures === 1) { + await until(() => actions(lane.exchanges).includes('workspace-cleanup'), 'the lane was never released'); + } + } finally { + controller.abort(); + await done; + } + assert.equal(executionAborted, failures > 1); + if (failures > 1) return; + assert.deepEqual(actions(lane.exchanges), ['settle', 'workspace-cleanup']); + assert.equal(receipt(lane.exchanges, 'settle')?.status, 'fulfilled'); + assert.deepEqual(lane.errors, []); + }); +} diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index c2ae9924..fb7bb1ed 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -140,6 +140,8 @@ const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; const CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS = 1_000; const MIN_REGISTRATION_HEARTBEAT_MS = 25; const REGISTRATION_RETRY_DELAY_MS = 100; +/** Consecutive transient heartbeat failures a concurrent run absorbs before stopping every slot. */ +const CONCURRENT_HEARTBEAT_RETRIES = 1; const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; const SETTLEMENT_RETRY_DELAY_MS = 100; const REJECTION_ACK_GRACE_MS = 30_000; @@ -500,6 +502,8 @@ export class BridgeWorker { private maintenanceOnly = false; private mutationGuardArmed = false; private readonly quarantinedWorkspaces = new Set(); + /** Lane quarantine receipts a stopped run could not deliver, by assignment ID. */ + private readonly undeliveredQuarantineReceipts = new Map(); private readonly activeWorkspaceAssignments = new Map< string, { id: string; done: Promise } @@ -1167,7 +1171,17 @@ export class BridgeWorker { controller.abort(error); }; this.concurrentRunning = true; - const heartbeat = this.maintainRegistration(controller.signal).catch(fail); + const heartbeat = this.maintainRegistration( + controller.signal, + CONCURRENT_HEARTBEAT_RETRIES, + ).catch(fail); + // Code API offers only unsettled work again, so nothing else re-sends these receipts. + for (const pending of [...this.undeliveredQuarantineReceipts.values()]) { + if (controller.signal.aborted) break; + await this.deliverQuarantineReceipt(pending, controller.signal).catch( + (error) => this.options.onError?.(error), + ); + } const lane = async (slot?: number): Promise => { let retries = 0; while (!controller.signal.aborted) { @@ -1184,12 +1198,9 @@ export class BridgeWorker { ) { // The durable local guard is retained. A distinct receipt tells // Code API to release this slot without declaring the root clean. + // A stopping run still delivers it: the lane's fence outlives this run. try { - await this.reportWorkspaceOwnership( - assignment, - 'quarantine', - controller.signal, - ); + await this.deliverQuarantineReceipt(assignment); this.options.onError?.(error); continue; } catch (quarantineError) { @@ -2260,7 +2271,7 @@ export class BridgeWorker { const recoveryHeartbeatController = new AbortController(); const recoveryHeartbeat = this.maintainRegistration( recoveryHeartbeatController.signal, - true, + Number.POSITIVE_INFINITY, ).catch(() => undefined); const rejectionAckGraceMs = Math.max( 0, @@ -2333,11 +2344,8 @@ export class BridgeWorker { } if (assignment.workspaceLeaseSlot !== undefined) { try { - await this.reportWorkspaceOwnership( - assignment, - 'workspace-cleanup', - signal, - ); + // The settlement has committed, so a stopping run must still release the lane's fence. + await this.reportWorkspaceOwnership(assignment, 'workspace-cleanup'); } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace cleanup acknowledgement could not be confirmed', @@ -2356,6 +2364,28 @@ export class BridgeWorker { } } + /** Deliver a lane quarantine receipt, keeping it for the next run while delivery is uncertain. */ + private async deliverQuarantineReceipt( + assignment: BridgeAssignment, + signal?: AbortSignal, + ): Promise { + try { + await this.reportWorkspaceOwnership(assignment, 'quarantine', signal); + this.undeliveredQuarantineReceipts.delete(assignment.assignmentId); + } catch (error) { + const refused = + error instanceof BridgeProtocolError && + error.status != null && + error.status >= 400 && + error.status < 500 && + error.status !== 408 && + error.status !== 429; + if (refused) this.undeliveredQuarantineReceipts.delete(assignment.assignmentId); + else this.undeliveredQuarantineReceipts.set(assignment.assignmentId, assignment); + throw error; + } + } + private async reportWorkspaceOwnership( assignment: BridgeAssignment, operation: 'quarantine' | 'workspace-cleanup', @@ -2497,8 +2527,9 @@ export class BridgeWorker { private async maintainRegistration( signal: AbortSignal, - retryTransient = false, + transientRetries = 0, ): Promise { + let failures = 0; while (!signal.aborted) { const heartbeatIntervalMs = Math.max( MIN_REGISTRATION_HEARTBEAT_MS, @@ -2515,6 +2546,7 @@ export class BridgeWorker { signal, this.mutationGuardArmed || this.armedWorkspaces.size > 0, ); + failures = 0; } catch (error) { const terminal = error instanceof BridgeProtocolError && @@ -2522,7 +2554,9 @@ export class BridgeWorker { error.status === 403 || error.code === 'WORKER_FENCED' || error.code === 'WORKER_QUARANTINED'); - if (!retryTransient || terminal || signal.aborted) throw error; + if (failures >= transientRetries || terminal || signal.aborted) + throw error; + failures += 1; await this.delay(REGISTRATION_RETRY_DELAY_MS, signal); } } @@ -2535,7 +2569,7 @@ export class BridgeWorker { const heartbeatController = new AbortController(); const heartbeat = this.maintainRegistration( heartbeatController.signal, - true, + Number.POSITIVE_INFINITY, ).catch(() => undefined); try { await this.settleWithRetry(