Repository navigation
feat(api): abort signal support for unbound, vercel-ai-gateway, zoo-gateway #1653
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
a3fb762
8d8ee07
044e22b
8fd0325
07aeb6d
8043f3e
1b9814a
cd8edc3
470e9cf
5c4c598
fe3dfe5
76f0dc8
ed73cd8
17975dd
09aa5a0
7f78d17
6b796fe
aad2004
644eb4c
b1bd578
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| import { Anthropic } from "@anthropic-ai/sdk" | ||
| import OpenAI from "openai" | ||
| import OpenAI, { APIConnectionTimeoutError } from "openai" | ||
|
|
||
| import { | ||
| type ModelInfo, | ||
|
|
@@ -29,6 +29,14 @@ import type { | |
| } from "../index" | ||
| import { handleOpenAIError } from "./utils/error-handler" | ||
| import { applyRouterToolPreferences } from "./utils/router-tool-preferences" | ||
| import { | ||
| bridgeExternalAbortSignal, | ||
| createAbortError, | ||
| hasUsableTimeout, | ||
| isRequestAborted, | ||
| resolveModelWithAbort, | ||
| toSdkTimeoutMs, | ||
| } from "./utils/abort-signal" | ||
| import { extractReasoningFromDelta } from "./utils/extract-reasoning" | ||
|
|
||
| // Unbound usage includes extra fields for Anthropic cache tokens. | ||
|
|
@@ -152,14 +160,21 @@ export class UnboundHandler extends BaseProvider implements SingleCompletionHand | |
| messages: Anthropic.Messages.MessageParam[], | ||
| metadata?: ApiHandlerCreateMessageMetadata, | ||
| ): ApiStream { | ||
| // Establish the cancellation scope around model resolution: a | ||
| // pre-aborted signal rejects before the lookup starts, and a signal | ||
| // that fires while model metadata is loading settles on the | ||
| // standardized AbortError; any other resolution failure propagates | ||
| // unchanged. | ||
| const externalAbortSignal = metadata?.abortSignal | ||
| const resolved = await resolveModelWithAbort(() => this.fetchModel(), externalAbortSignal, "Unbound") | ||
| const { | ||
| id: model, | ||
| info, | ||
| maxTokens: max_tokens, | ||
| temperature, | ||
| reasoningEffort: reasoning_effort, | ||
| reasoning: thinking, | ||
| } = await this.fetchModel() | ||
| } = resolved | ||
|
|
||
| const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [ | ||
| { role: "system", content: systemPrompt }, | ||
|
|
@@ -185,51 +200,86 @@ export class UnboundHandler extends BaseProvider implements SingleCompletionHand | |
| tool_choice: metadata?.tool_choice, | ||
| } | ||
|
|
||
| let stream | ||
| // Per-request signal bridged from the caller's abort signal via the shared helper | ||
| // (pre-aborted guard, once-only listener, retained cleanup). The never-aborting | ||
| // fallback preserves the previous always-signal behaviour when the caller has no | ||
| // external signal, and dispose() detaches the listener when the request ends. | ||
| const { signal: bridgedSignal, dispose: detachExternalAbortSignal } = | ||
| bridgeExternalAbortSignal(externalAbortSignal) | ||
| const requestSignal = bridgedSignal ?? new AbortController().signal | ||
|
|
||
| try { | ||
| stream = await this.client.chat.completions.create(completionParams) | ||
| } catch (error) { | ||
| throw handleOpenAIError(error, this.providerName) | ||
| } | ||
| let lastUsage: any = undefined | ||
| let stream | ||
| try { | ||
| stream = await this.client.chat.completions.create(completionParams, { signal: requestSignal }) | ||
| } catch (error) { | ||
| // Preserve abort identity (series standard): a cancelled request | ||
| // must surface as a DOM-standard AbortError, not a wrapped | ||
| // completion error. | ||
| if (isRequestAborted(error, externalAbortSignal)) { | ||
| throw createAbortError("Unbound") | ||
| } | ||
| throw handleOpenAIError(error, this.providerName) | ||
| } | ||
| let lastUsage: any = undefined | ||
|
|
||
| for await (const chunk of stream) { | ||
| const delta = chunk.choices[0]?.delta | ||
| try { | ||
| for await (const chunk of stream) { | ||
| const delta = chunk.choices[0]?.delta | ||
|
|
||
| const reasoningText = extractReasoningFromDelta(delta) | ||
| if (reasoningText) { | ||
| yield { type: "reasoning", text: reasoningText } | ||
| } | ||
| const reasoningText = extractReasoningFromDelta(delta) | ||
| if (reasoningText) { | ||
| yield { type: "reasoning", text: reasoningText } | ||
| } | ||
|
|
||
| if (delta?.content) { | ||
| yield { type: "text", text: delta.content } | ||
| } | ||
| if (delta?.content) { | ||
| yield { type: "text", text: delta.content } | ||
| } | ||
|
|
||
| // Handle native tool calls | ||
| if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { | ||
| for (const toolCall of delta.tool_calls) { | ||
| yield { | ||
| type: "tool_call_partial", | ||
| index: toolCall.index, | ||
| id: toolCall.id, | ||
| name: toolCall.function?.name, | ||
| arguments: toolCall.function?.arguments, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Handle native tool calls | ||
| if (delta && "tool_calls" in delta && Array.isArray(delta.tool_calls)) { | ||
| for (const toolCall of delta.tool_calls) { | ||
| yield { | ||
| type: "tool_call_partial", | ||
| index: toolCall.index, | ||
| id: toolCall.id, | ||
| name: toolCall.function?.name, | ||
| arguments: toolCall.function?.arguments, | ||
| if (chunk.usage) { | ||
| lastUsage = chunk.usage | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (chunk.usage) { | ||
| lastUsage = chunk.usage | ||
| if (lastUsage) { | ||
| yield this.processUsageMetrics(lastUsage, info) | ||
| } | ||
| } catch (error) { | ||
| // Preserve abort identity (series standard): a cancellation that | ||
| // surfaces after the stream has started must also normalize to | ||
| // the standardized AbortError, not the raw SDK rejection. | ||
| if (isRequestAborted(error, externalAbortSignal)) { | ||
| throw createAbortError("Unbound") | ||
| } | ||
| throw error | ||
| } | ||
| } | ||
|
|
||
| if (lastUsage) { | ||
| yield this.processUsageMetrics(lastUsage, info) | ||
| } finally { | ||
| detachExternalAbortSignal() | ||
| } | ||
| } | ||
|
|
||
| async completePrompt(prompt: string, _options?: CompletePromptOptions): Promise<string> { | ||
| const { id: model, maxTokens: max_tokens, temperature } = await this.fetchModel() | ||
| async completePrompt(prompt: string, options?: CompletePromptOptions): Promise<string> { | ||
| // Model resolution must honour the caller's cancellation scope the same | ||
| // way createMessage does: a signal that fires during the lookup settles on | ||
| // the standardized AbortError instead of waiting for the catalog to finish. | ||
| const { | ||
| id: model, | ||
| maxTokens: max_tokens, | ||
| temperature, | ||
| } = await resolveModelWithAbort(() => this.fetchModel(), options?.abortSignal, "Unbound") | ||
|
|
||
| const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [{ role: "system", content: prompt }] | ||
|
|
||
|
|
@@ -239,11 +289,34 @@ export class UnboundHandler extends BaseProvider implements SingleCompletionHand | |
| messages: openAiMessages, | ||
| temperature: temperature, | ||
| } | ||
| // Build request options with abortSignal and/or timeout. | ||
| // timeoutMs <= 0 means "no explicit timeout": omit the SDK timeout | ||
| // option entirely — the OpenAI SDK treats timeout: 0 as an immediate | ||
| // abort, which would cancel the request right away. | ||
| const createOptions: OpenAI.RequestOptions = {} | ||
| if (options?.abortSignal) { | ||
| createOptions.signal = options.abortSignal | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keep SDK abort listeners off the caller's signal. If a task reuses one signal across completions, OpenAI 5.23.2 adds an anonymous abort listener to it on every fetch attempt and does not remove the listener when the request ends. Completed request controllers therefore accumulate on the live task signal. Use
|
||
| } | ||
| const timeoutMs = options?.timeoutMs | ||
| if (hasUsableTimeout(timeoutMs)) { | ||
| createOptions.timeout = toSdkTimeoutMs(timeoutMs) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Enforce the timeout through response-body consumption. With
|
||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| let response: OpenAI.Chat.ChatCompletion | ||
| try { | ||
| response = await this.client.chat.completions.create(completionParams) | ||
| response = await this.client.chat.completions.create(completionParams, createOptions) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Settle caller cancellation during SDK retry backoff. If the SDK is waiting on a 429
|
||
| } catch (error) { | ||
| // Preserve abort identity (series standard): caller-initiated | ||
| // cancellations and request timeouts must surface as a | ||
| // DOM-standard AbortError, not a wrapped completion error. The | ||
| // OpenAI SDK reports both with messages ending in a period | ||
| // ("Request was aborted.", "Request timed out."), which would not | ||
| // match task-level abort detection (message ending in "aborted"). | ||
| // SDK request timeouts are not aborts, but the series standard maps | ||
| // them to the same AbortError identity as caller cancellations. | ||
| if (isRequestAborted(error, options?.abortSignal) || error instanceof APIConnectionTimeoutError) { | ||
| throw createAbortError("Unbound") | ||
| } | ||
| throw handleOpenAIError(error, this.providerName) | ||
| } | ||
| return response.choices[0]?.message.content || "" | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Reject cancellation when the SDK ends the stream without throwing.
After a partial chunk, aborting the bridged signal can make OpenAI 5.23.2 end its SSE iterator normally. The SDK catches the body-read
AbortErrorand returns. The provider then skips this catch and can emit retained usage before ending successfully. Check the caller signal after the loop, before yielding final usage. Preserve normal EOF when the caller signal is live. (raw.githubusercontent.com)