From 5e11c44d1d9ea33ebfd7b1bd33e17c8f67e3f49e Mon Sep 17 00:00:00 2001 From: Narendranath Gogineni Date: Fri, 4 Apr 2025 19:29:35 +0530 Subject: [PATCH] sync changes --- eslint.config.js | 1 + ...sesHandler.ts => modelResponsesHandler.ts} | 4 +- src/handlers/responseHandlers.ts | 18 +- src/handlers/streamHandler.ts | 34 +- src/index.ts | 10 +- src/providers/open-ai-base/constants.ts | 375 +++++++++++++ .../open-ai-base/createModelResponse.ts | 162 ++++++ src/providers/open-ai-base/helpers.ts | 508 ++++++++++++++++++ src/providers/open-ai-base/index.ts | 2 +- src/types/inputList.ts | 2 +- src/types/{responses.ts => modelResponses.ts} | 177 +++--- 11 files changed, 1167 insertions(+), 126 deletions(-) rename src/handlers/{responsesHandler.ts => modelResponsesHandler.ts} (94%) create mode 100644 src/providers/open-ai-base/constants.ts create mode 100644 src/providers/open-ai-base/helpers.ts rename src/types/{responses.ts => modelResponses.ts} (95%) diff --git a/eslint.config.js b/eslint.config.js index 228289c3..726b6251 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -12,5 +12,6 @@ export default tseslint.config({ rules: { // '@typescript-eslint/no-unused-vars': 'error', 'no-duplicate-imports': 'error', + '@typescript-eslint/no-namespace': 'off', }, }); diff --git a/src/handlers/responsesHandler.ts b/src/handlers/modelResponsesHandler.ts similarity index 94% rename from src/handlers/responsesHandler.ts rename to src/handlers/modelResponsesHandler.ts index 720feb93..0d65eb08 100644 --- a/src/handlers/responsesHandler.ts +++ b/src/handlers/modelResponsesHandler.ts @@ -5,7 +5,7 @@ import { } from './handlerUtils'; import { endpointStrings } from '../providers/types'; -function responsesHandler( +function modelResponsesHandler( endpoint: endpointStrings, method: 'POST' | 'GET' | 'DELETE' ) { @@ -44,4 +44,4 @@ function responsesHandler( return handler; } -export default responsesHandler; +export default modelResponsesHandler; diff --git a/src/handlers/responseHandlers.ts b/src/handlers/responseHandlers.ts index 11cd04a5..bcc1cb79 100644 --- a/src/handlers/responseHandlers.ts +++ b/src/handlers/responseHandlers.ts @@ -16,6 +16,7 @@ import { } from './streamHandler'; import { HookSpan } from '../middlewares/hooks'; import { env } from 'hono/adapter'; +import { OpenAIModelResponseJSONToStreamGenerator } from '../providers/open-ai-base/createModelResponse'; /** * Handles various types of responses based on the specified parameters @@ -74,10 +75,19 @@ export async function responseHandler( // JSON to text/event-stream conversion is only allowed for unified routes: chat completions and completions. // Set the transformer to OpenAI json to stream convertor function in that case. if (responseTransformer && streamingMode && isCacheHit) { - responseTransformerFunction = - responseTransformer === 'chatComplete' - ? OpenAIChatCompleteJSONToStreamResponseTransform - : OpenAICompleteJSONToStreamResponseTransform; + switch (responseTransformer) { + case 'chatComplete': + responseTransformerFunction = + OpenAIChatCompleteJSONToStreamResponseTransform; + break; + case 'createModelResponse': + responseTransformerFunction = OpenAIModelResponseJSONToStreamGenerator; + break; + default: + responseTransformerFunction = + OpenAICompleteJSONToStreamResponseTransform; + break; + } } else if (responseTransformer && !streamingMode && isCacheHit) { responseTransformerFunction = undefined; } diff --git a/src/handlers/streamHandler.ts b/src/handlers/streamHandler.ts index 70dbbe39..403a68a8 100644 --- a/src/handlers/streamHandler.ts +++ b/src/handlers/streamHandler.ts @@ -345,14 +345,34 @@ export async function handleJSONToStreamResponse( const encoder = new TextEncoder(); const responseJSON: OpenAIChatCompleteResponse | OpenAICompleteResponse = await response.clone().json(); - const streamChunkArray = responseTransformerFunction(responseJSON, provider); - (async () => { - for (const chunk of streamChunkArray) { - await writer.write(encoder.encode(chunk)); - } - writer.close(); - })(); + if ( + Object.prototype.toString.call(responseTransformerFunction) === + '[object GeneratorFunction]' + ) { + const generator = responseTransformerFunction(responseJSON, provider); + (async () => { + while (true) { + const chunk = generator.next(); + if (chunk.done) { + break; + } + await writer.write(encoder.encode(chunk.value)); + } + writer.close(); + })(); + } else { + const streamChunkArray = responseTransformerFunction( + responseJSON, + provider + ); + (async () => { + for (const chunk of streamChunkArray) { + await writer.write(encoder.encode(chunk)); + } + writer.close(); + })(); + } return new Response(readable, { headers: new Headers({ diff --git a/src/index.ts b/src/index.ts index e1829739..ad01093e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,7 +34,7 @@ import finetuneHandler from './handlers/finetuneHandler'; // Config import conf from '../conf.json'; -import responsesHandler from './handlers/responsesHandler'; +import modelResponsesHandler from './handlers/modelResponsesHandler'; // Create a new Hono server instance const app = new Hono(); @@ -204,22 +204,22 @@ app.get('/v1/batches', requestValidator, batchesHandler('listBatches', 'GET')); app.post( '/v1/responses', requestValidator, - responsesHandler('createModelResponse', 'POST') + modelResponsesHandler('createModelResponse', 'POST') ); app.get( '/v1/responses/:id', requestValidator, - responsesHandler('getModelResponse', 'GET') + modelResponsesHandler('getModelResponse', 'GET') ); app.delete( '/v1/responses/:id', requestValidator, - responsesHandler('deleteModelResponse', 'DELETE') + modelResponsesHandler('deleteModelResponse', 'DELETE') ); app.get( '/v1/responses/:id/input_items', requestValidator, - responsesHandler('listResponseInputItems', 'GET') + modelResponsesHandler('listResponseInputItems', 'GET') ); app.all( diff --git a/src/providers/open-ai-base/constants.ts b/src/providers/open-ai-base/constants.ts new file mode 100644 index 00000000..c706aac4 --- /dev/null +++ b/src/providers/open-ai-base/constants.ts @@ -0,0 +1,375 @@ +import { + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseCreatedEvent, + ResponseErrorEvent, + ResponseFileSearchCallCompletedEvent, + ResponseFileSearchCallInProgressEvent, + ResponseFileSearchCallSearchingEvent, + ResponseFunctionCallArgumentsDeltaEvent, + ResponseFunctionCallArgumentsDoneEvent, + ResponseInProgressEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseRefusalDeltaEvent, + ResponseRefusalDoneEvent, + ResponseTextAnnotationDeltaEvent, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, + ResponseWebSearchCallCompletedEvent, + ResponseWebSearchCallInProgressEvent, + ResponseWebSearchCallSearchingEvent, +} from '../../types/modelResponses'; + +export const RESPONSE_ERROR_EVENT: ResponseErrorEvent = { + type: 'error', + code: 'ERR_SOMETHING', + message: 'Something went wrong', + param: null, +}; + +//response.created +export const RESPONSE_CREATED_EVENT: ResponseCreatedEvent = { + type: 'response.created', + response: { + id: '', + object: 'response', + created_at: 0, + status: 'in_progress', + error: null, + incomplete_details: null, + instructions: null, + max_output_tokens: null, + model: 'gpt-4o-2024-08-06', + output: [], + parallel_tool_calls: true, + previous_response_id: null, + reasoning: { + effort: null, + generate_summary: null, + }, + store: true, + temperature: 1.0, + text: { + format: { + type: 'text', + }, + }, + tool_choice: 'auto', + tools: [], + top_p: 1.0, + truncation: 'disabled', + usage: null, + user: null, + metadata: {}, + }, +}; + +//response.in_progress +export const RESPONSE_IN_PROGRESS_EVENT: ResponseInProgressEvent = { + type: 'response.in_progress', + response: { + id: '', + object: 'response', + created_at: 1743576815, + status: 'in_progress', + error: null, + incomplete_details: null, + instructions: null, + max_output_tokens: null, + model: '', + output: [], + parallel_tool_calls: true, + previous_response_id: null, + reasoning: { + effort: null, + generate_summary: null, + }, + store: true, + temperature: 1.0, + text: { + format: { + type: 'text', + }, + }, + tool_choice: 'auto', + tools: [], + top_p: 1.0, + truncation: 'disabled', + usage: null, + user: null, + metadata: {}, + }, +}; + +//response.output_item.added message +export const RESPONSE_OUTPUT_MESSAGE_ITEM_ADDED_EVENT: ResponseOutputItemAddedEvent = + { + type: 'response.output_item.added', + output_index: 0, + item: { + type: 'message', + id: '', + status: 'in_progress', + role: 'assistant', + content: [], + }, + }; + +export const RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_ADDED_EVENT: ResponseContentPartAddedEvent = + { + type: 'response.content_part.added', + item_id: '', + output_index: 0, + content_index: 0, + part: { + type: 'output_text', + text: '', + annotations: [], + }, + }; + +export const RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_ADDED_EVENT: ResponseContentPartAddedEvent = + { + type: 'response.content_part.added', + item_id: '', + output_index: 0, + content_index: 0, + part: { + type: 'refusal', + refusal: '', + }, + }; + +export const RESPONSE_OUTPUT_TEXT_DELTA_EVENT: ResponseTextDeltaEvent = { + type: 'response.output_text.delta', + item_id: '', + output_index: 0, + content_index: 0, + delta: '', +}; + +export const RESPONSE_OUTPUT_TEXT_ANNOTATION_ADDED_EVENT: ResponseTextAnnotationDeltaEvent = + { + type: 'response.output_text.annotation.added', + item_id: '', + output_index: 1, + content_index: 0, + annotation_index: 0, + annotation: { + type: 'file_citation', + index: 390, + file_id: '', + }, + }; + +export const RESPONSE_OUTPUT_TEXT_DONE_EVENT: ResponseTextDoneEvent = { + type: 'response.output_text.done', + item_id: '', + output_index: 0, + content_index: 0, + text: '', +}; + +export const RESPONSE_OUTPUT_REFUSAL_DELTA_EVENT: ResponseRefusalDeltaEvent = { + type: 'response.refusal.delta', + item_id: '', + output_index: 0, + content_index: 0, + delta: '', +}; + +export const RESPONSE_OUTPUT_REFUSAL_DONE_EVENT: ResponseRefusalDoneEvent = { + type: 'response.refusal.done', + item_id: '', + output_index: 1, + content_index: 2, + refusal: '', +}; + +export const RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_DONE_EVENT: ResponseContentPartDoneEvent = + { + type: 'response.content_part.done', + item_id: '', + output_index: 0, + content_index: 0, + part: { + type: 'refusal', + refusal: '', + }, + }; + +export const RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_DONE_EVENT: ResponseContentPartDoneEvent = + { + type: 'response.content_part.done', + item_id: '', + output_index: 0, + content_index: 0, + part: { + type: 'output_text', + text: '', + annotations: [], + }, + }; + +//response.output_item.done message +export const RESPONSE_OUTPUT_MESSAGE_ITEM_DONE_EVENT: ResponseOutputItemDoneEvent = + { + type: 'response.output_item.done', + output_index: 0, + item: { + type: 'message', + id: '', + status: 'completed', + role: 'assistant', + content: [ + { + type: 'output_text', + text: '', + annotations: [], + }, + ], + }, + }; + +//response.output_item.added function_call +export const RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_ADDED_EVENT: ResponseOutputItemAddedEvent = + { + type: 'response.output_item.added', + output_index: 0, + item: { + type: 'function_call', + id: '', + call_id: '', + name: '', + arguments: '', + status: 'in_progress', + }, + }; + +//response.function_call_arguments.delta +export const RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DELTA_EVENT: ResponseFunctionCallArgumentsDeltaEvent = + { + type: 'response.function_call_arguments.delta', + item_id: '', + output_index: 0, + delta: '', + }; + +//response.function_call_arguments.done +export const RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DONE_EVENT: ResponseFunctionCallArgumentsDoneEvent = + { + type: 'response.function_call_arguments.done', + item_id: '', + output_index: 0, + arguments: '', + }; + +//response.output_item.done function_call +export const RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_DONE_EVENT: ResponseOutputItemDoneEvent = + { + type: 'response.output_item.done', + output_index: 0, + item: { + type: 'function_call', + id: '', + call_id: '', + name: '', + arguments: '', + status: 'completed', + }, + }; + +//response.output_item.added web_search +export const RESPONSE_OUTPUT_WEB_SEARCH_ITEM_ADDED_EVENT: ResponseOutputItemAddedEvent = + { + type: 'response.output_item.added', + output_index: 0, + item: { + type: 'web_search_call', + id: '', + status: 'in_progress', + }, + }; + +//response.web_search_call.in_progress +export const RESPONSE_OUTPUT_WEB_SEARCH_CALL_IN_PROGRESS_EVENT: ResponseWebSearchCallInProgressEvent = + { + type: 'response.web_search_call.in_progress', + output_index: 0, + item_id: '', + }; + +//response.web_search_call.searching +export const RESPONSE_OUTPUT_WEB_SEARCH_CALL_SEARCHING_EVENT: ResponseWebSearchCallSearchingEvent = + { + type: 'response.web_search_call.searching', + output_index: 0, + item_id: '', + }; + +//response.web_search_call.completed +export const RESPONSE_OUTPUT_WEB_SEARCH_CALL_COMPLETED_EVENT: ResponseWebSearchCallCompletedEvent = + { + type: 'response.web_search_call.completed', + output_index: 0, + item_id: '', + }; + +export const RESPONSE_OUTPUT_WEB_SEARCH_ITEM_DONE_EVENT: ResponseOutputItemDoneEvent = + { + type: 'response.output_item.done', + output_index: 0, + item: { + type: 'web_search_call', + id: '', + status: 'completed', + }, + }; + +export const RESPONSE_OUTPUT_FILE_SEARCH_ITEM_ADDED_EVENT: ResponseOutputItemAddedEvent = + { + type: 'response.output_item.added', + output_index: 0, + item: { + type: 'file_search_call', + id: '', + queries: [], + results: [], + status: 'in_progress', + }, + }; + +export const RESPONSE_OUTPUT_FILE_SEARCH_CALL_IN_PROGRESS_EVENT: ResponseFileSearchCallInProgressEvent = + { + type: 'response.file_search_call.in_progress', + output_index: 0, + item_id: '', + }; + +export const RESPONSE_OUTPUT_FILE_SEARCH_CALL_SEARCHING_EVENT: ResponseFileSearchCallSearchingEvent = + { + type: 'response.file_search_call.searching', + output_index: 0, + item_id: '', + }; + +export const RESPONSE_OUTPUT_FILE_SEARCH_CALL_COMPLETED_EVENT: ResponseFileSearchCallCompletedEvent = + { + type: 'response.file_search_call.completed', + output_index: 0, + item_id: '', + }; + +export const RESPONSE_OUTPUT_FILE_SEARCH_ITEM_DONE_EVENT: ResponseOutputItemDoneEvent = + { + type: 'response.output_item.done', + output_index: 0, + item: { + type: 'file_search_call', + id: '', + queries: [], + results: [], + status: 'completed', + }, + }; diff --git a/src/providers/open-ai-base/createModelResponse.ts b/src/providers/open-ai-base/createModelResponse.ts index c940e340..297caba8 100644 --- a/src/providers/open-ai-base/createModelResponse.ts +++ b/src/providers/open-ai-base/createModelResponse.ts @@ -1,4 +1,37 @@ import { ProviderConfig } from '../types'; +import { OpenAIResponse } from '../../types/modelResponses'; +import { + getRandomId, + getResponseCompletedEvent, + getResponseOutputFileSearchCallCompletedEvent, + getResponseOutputFileSearchCallInProgressEvent, + getResponseOutputFileSearchCallSearchingEvent, + getResponseOutputFileSearchItemAddedEvent, + getResponseOutputFileSearchItemDoneEvent, + getResponseOutputMessageItemAddedEvent, + getResponseOutputMessageItemDoneEvent, + getResponseOutputMessageOutputTextContentPartAddedEvent, + getResponseOutputMessageOutputTextContentPartDeltaEvent, + getResponseOutputMessageOutputTextContentPartDoneEvent, + getResponseOutputMessageRefusalContentPartAddedEvent, + getResponseOutputMessageRefusalContentPartDeltaEvent, + getResponseOutputTextDoneEvent, + getResponseOutputWebSearchCallCompletedEvent, + getResponseOutputWebSearchCallInProgressEvent, + getResponseOutputWebSearchCallSearchingEvent, + getResponseOutputWebSearchItemAddedEvent, + getResponseOutputWebSearchItemDoneEvent, + getResponseOutputMessageRefusalDoneEvent, + getResponseOutputMessageRefusalContentPartDoneEvent, + getResponseErrorEvent, + getResponseFailedEvent, + getResponseIncompleteEvent, + getResponseCreatedEvent, + getResponseFunctionCallArgumentsDeltaEvents, + getResponseInProgressEvent, + getResponseOutputFunctionCallItemAddedEvent, + getResponseOutputFunctionCallItemDoneEvent, +} from './helpers'; export const OpenAICreateModelResponseConfig: ProviderConfig = { input: { @@ -70,3 +103,132 @@ export const OpenAICreateModelResponseConfig: ProviderConfig = { required: false, }, }; + +export function* OpenAIModelResponseJSONToStreamGenerator( + response: OpenAIResponse +): Generator { + if (response.error?.code) { + yield getResponseErrorEvent(response.error); + return; + } + + if (response.status === 'failed') { + yield getResponseFailedEvent(response); + return; + } + + if (response.status === 'incomplete') { + yield getResponseIncompleteEvent(response); + return; + } + + const responseId = getRandomId(); + yield getResponseCreatedEvent(response, responseId); + yield getResponseInProgressEvent(response, responseId); + for (const [index, outputItem] of response.output.entries()) { + const outputItemId = getRandomId(); + if (outputItem.type === 'function_call') { + const functionCallId = getRandomId(); + yield getResponseOutputFunctionCallItemAddedEvent( + index, + outputItemId, + functionCallId, + outputItem + ); + yield getResponseFunctionCallArgumentsDeltaEvents( + index, + outputItemId, + outputItem + ); + yield getResponseOutputFunctionCallItemDoneEvent( + index, + outputItemId, + functionCallId, + outputItem + ); + } else if (outputItem.type === 'web_search_call') { + yield getResponseOutputWebSearchItemAddedEvent(index, outputItemId); + yield getResponseOutputWebSearchCallInProgressEvent(index, outputItemId); + yield getResponseOutputWebSearchCallSearchingEvent(index, outputItemId); + yield getResponseOutputWebSearchCallCompletedEvent(index, outputItemId); + yield getResponseOutputWebSearchItemDoneEvent(index, outputItemId); + } else if (outputItem.type === 'file_search_call') { + // TODO: validate this + yield getResponseOutputFileSearchItemAddedEvent(index, outputItemId); + yield getResponseOutputFileSearchCallInProgressEvent(index, outputItemId); + yield getResponseOutputFileSearchCallSearchingEvent(index, outputItemId); + yield getResponseOutputFileSearchCallCompletedEvent(index, outputItemId); + yield getResponseOutputFileSearchItemDoneEvent( + index, + outputItemId, + outputItem + ); + } else if (outputItem.type === 'message') { + yield getResponseOutputMessageItemAddedEvent(index, outputItemId); + for (const [ + contentPartIndex, + contentPart, + ] of outputItem.content.entries()) { + if (contentPart.type === 'output_text') { + yield getResponseOutputMessageOutputTextContentPartAddedEvent( + index, + outputItemId, + contentPartIndex + ); + for (let i = 0; i < contentPart.text.length; i += 500) { + yield getResponseOutputMessageOutputTextContentPartDeltaEvent( + index, + outputItemId, + contentPartIndex, + contentPart.text.slice(i, i + 500) + ); + } + yield getResponseOutputTextDoneEvent( + index, + outputItemId, + contentPartIndex, + contentPart + ); + yield getResponseOutputMessageOutputTextContentPartDoneEvent( + index, + outputItemId, + contentPartIndex, + contentPart + ); + } else if (contentPart.type === 'refusal') { + yield getResponseOutputMessageRefusalContentPartAddedEvent( + index, + outputItemId, + contentPartIndex + ); + for (let i = 0; i < contentPart.refusal.length; i += 500) { + yield getResponseOutputMessageRefusalContentPartDeltaEvent( + index, + outputItemId, + contentPartIndex, + contentPart.refusal.slice(i, i + 500) + ); + } + yield getResponseOutputMessageRefusalDoneEvent( + index, + outputItemId, + contentPartIndex, + contentPart + ); + yield getResponseOutputMessageRefusalContentPartDoneEvent( + index, + outputItemId, + contentPartIndex, + contentPart + ); + } + } + yield getResponseOutputMessageItemDoneEvent( + index, + outputItemId, + outputItem + ); + } + } + yield getResponseCompletedEvent(response, responseId); +} diff --git a/src/providers/open-ai-base/helpers.ts b/src/providers/open-ai-base/helpers.ts new file mode 100644 index 00000000..68abec34 --- /dev/null +++ b/src/providers/open-ai-base/helpers.ts @@ -0,0 +1,508 @@ +import { randomUUID } from 'crypto'; +import { + OpenAIResponse, + ResponseCreatedEvent, + ResponseFunctionCallArgumentsDeltaEvent, + ResponseFunctionCallArgumentsDoneEvent, + ResponseInProgressEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseOutputItem, + ResponseFunctionToolCall, + ResponseCompletedEvent, + ResponseFunctionWebSearch, + ResponseWebSearchCallInProgressEvent, + ResponseWebSearchCallSearchingEvent, + ResponseWebSearchCallCompletedEvent, + ResponseOutputMessage, + ResponseFileSearchToolCall, + ResponseFileSearchCallInProgressEvent, + ResponseFileSearchCallSearchingEvent, + ResponseFileSearchCallCompletedEvent, + ResponseOutputText, + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, + ResponseOutputRefusal, + ResponseRefusalDeltaEvent, + ResponseRefusalDoneEvent, + ResponseError, + ResponseErrorEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, +} from '../../types/modelResponses'; +import { + RESPONSE_CREATED_EVENT, + RESPONSE_IN_PROGRESS_EVENT, + RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DELTA_EVENT, + RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DONE_EVENT, + RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_ADDED_EVENT, + RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_DONE_EVENT, + RESPONSE_OUTPUT_MESSAGE_ITEM_DONE_EVENT, + RESPONSE_OUTPUT_MESSAGE_ITEM_ADDED_EVENT, + RESPONSE_OUTPUT_WEB_SEARCH_CALL_COMPLETED_EVENT, + RESPONSE_OUTPUT_WEB_SEARCH_CALL_IN_PROGRESS_EVENT, + RESPONSE_OUTPUT_WEB_SEARCH_CALL_SEARCHING_EVENT, + RESPONSE_OUTPUT_WEB_SEARCH_ITEM_ADDED_EVENT, + RESPONSE_OUTPUT_WEB_SEARCH_ITEM_DONE_EVENT, + RESPONSE_OUTPUT_FILE_SEARCH_ITEM_ADDED_EVENT, + RESPONSE_OUTPUT_FILE_SEARCH_CALL_IN_PROGRESS_EVENT, + RESPONSE_OUTPUT_FILE_SEARCH_CALL_SEARCHING_EVENT, + RESPONSE_OUTPUT_FILE_SEARCH_CALL_COMPLETED_EVENT, + RESPONSE_OUTPUT_FILE_SEARCH_ITEM_DONE_EVENT, + RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_ADDED_EVENT, + RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_DONE_EVENT, + RESPONSE_OUTPUT_TEXT_DELTA_EVENT, + RESPONSE_OUTPUT_TEXT_DONE_EVENT, + RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_ADDED_EVENT, + RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_DONE_EVENT, + RESPONSE_OUTPUT_REFUSAL_DELTA_EVENT, + RESPONSE_OUTPUT_REFUSAL_DONE_EVENT, + RESPONSE_ERROR_EVENT, +} from './constants'; + +export const getRandomId = () => { + return 'portkey_cache_' + randomUUID(); +}; + +export const getResponseErrorEvent = (error: ResponseError): string => { + const template: ResponseErrorEvent = { ...RESPONSE_ERROR_EVENT }; + template.code = error.code; + template.message = error.message; + return `event: error\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseFailedEvent = (response: OpenAIResponse): string => { + const chunk: ResponseFailedEvent = { + type: 'response.failed', + response: response, + }; + chunk.response.created_at = Math.floor(Date.now() / 1000); + chunk.response.id = getRandomId(); + return `event: response.failed\ndata: ${JSON.stringify(chunk)}\n\n`; +}; + +export const getResponseIncompleteEvent = ( + response: OpenAIResponse +): string => { + const chunk: ResponseIncompleteEvent = { + type: 'response.incomplete', + response: response, + }; + chunk.response.created_at = Math.floor(Date.now() / 1000); + chunk.response.id = getRandomId(); + return `event: response.incomplete\ndata: ${JSON.stringify(chunk)}\n\n`; +}; + +// response.created +export const getResponseCreatedEvent = ( + finalResponse: OpenAIResponse, + responseId: string +): string => { + const template: ResponseCreatedEvent = { + ...RESPONSE_CREATED_EVENT, + ...finalResponse, + }; + template.response.output = []; + template.response.created_at = Math.floor(Date.now() / 1000); + template.response.id = responseId; + return `event: response.created\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.in_progress +export const getResponseInProgressEvent = ( + finalResponse: OpenAIResponse, + responseId: string +): string => { + const template: ResponseInProgressEvent = { + ...RESPONSE_IN_PROGRESS_EVENT, + ...finalResponse, + }; + template.response.output = []; + template.response.created_at = Math.floor(Date.now() / 1000); + template.response.id = responseId; + return `event: response.in_progress\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.completed +export const getResponseCompletedEvent = ( + finalResponse: OpenAIResponse, + responseId: string +): string => { + const template: ResponseCompletedEvent = { + type: 'response.completed', + response: finalResponse, + }; + template.response.id = responseId; + template.response.created_at = Math.floor(Date.now() / 1000); + return `event: response.completed\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.added message +export const getResponseOutputMessageItemAddedEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseOutputItemAddedEvent = { + ...RESPONSE_OUTPUT_MESSAGE_ITEM_ADDED_EVENT, + }; + template.output_index = index; + const item: ResponseOutputItem = template.item as ResponseOutputItem; + item.id = outputItemId; + return `event: response.output_item.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageOutputTextContentPartAddedEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number +): string => { + const template: ResponseContentPartAddedEvent = { + ...RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_ADDED_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + return `event: response.content_part.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageOutputTextContentPartDeltaEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + delta: string +): string => { + const template: ResponseTextDeltaEvent = { + ...RESPONSE_OUTPUT_TEXT_DELTA_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.delta = delta; + return `event: response.output_text.delta\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputTextDoneEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + contentPart: ResponseOutputText +): string => { + const template: ResponseTextDoneEvent = { + ...RESPONSE_OUTPUT_TEXT_DONE_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.text = contentPart.text; + return `event: response.output_text.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageOutputTextContentPartDoneEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + contentPart: ResponseOutputText +): string => { + const template: ResponseContentPartDoneEvent = { + ...RESPONSE_OUTPUT_MESSAGE_OUTPUT_TEXT_CONTENT_PART_DONE_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.part = contentPart; + return `event: response.content_part.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageRefusalContentPartAddedEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number +): string => { + const template: ResponseContentPartAddedEvent = { + ...RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_ADDED_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + return `event: response.content_part.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageRefusalContentPartDeltaEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + delta: string +): string => { + const template: ResponseRefusalDeltaEvent = { + ...RESPONSE_OUTPUT_REFUSAL_DELTA_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.delta = delta; + return `event: response.refusal.delta\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageRefusalDoneEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + contentPart: ResponseOutputRefusal +): string => { + const template: ResponseRefusalDoneEvent = { + ...RESPONSE_OUTPUT_REFUSAL_DONE_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.refusal = contentPart.refusal; + return `event: response.refusal.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +export const getResponseOutputMessageRefusalContentPartDoneEvent = ( + index: number, + outputItemId: string, + contentPartIndex: number, + contentPart: ResponseOutputRefusal +): string => { + const template: ResponseContentPartDoneEvent = { + ...RESPONSE_OUTPUT_MESSAGE_REFUSAL_CONTENT_PART_DONE_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + template.content_index = contentPartIndex; + template.part = contentPart; + return `event: response.content_part.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.done message +export const getResponseOutputMessageItemDoneEvent = ( + index: number, + outputItemId: string, + contentItem: ResponseOutputMessage +): string => { + const template: ResponseOutputItemDoneEvent = { + ...RESPONSE_OUTPUT_MESSAGE_ITEM_DONE_EVENT, + }; + template.output_index = index; + const item: ResponseOutputMessage = template.item as ResponseOutputMessage; + item.id = outputItemId; + item.content = contentItem.content; + return `event: response.output_item.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.added function_call +export const getResponseOutputFunctionCallItemAddedEvent = ( + index: number, + outputItemId: string, + functionCallId: string, + outputItem: ResponseOutputItem +): string => { + if (outputItem.type !== 'function_call') return '\n\n'; // TOOD: this is for type safety, make it prettier + const template: ResponseOutputItemAddedEvent = { + ...RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_ADDED_EVENT, + }; + template.output_index = index; + const item: ResponseFunctionToolCall = + template.item as ResponseFunctionToolCall; + item.id = outputItemId; + item.call_id = functionCallId; + item.name = outputItem.name; + return `event: response.output_item.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.function_call_arguments.delta +// we could return this as multiple events, but one delta event is enough for now +export const getResponseFunctionCallArgumentsDeltaEvents = ( + index: number, + itemId: string, + contentItem: ResponseFunctionToolCall +): string => { + const template: ResponseFunctionCallArgumentsDeltaEvent = { + ...RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DELTA_EVENT, + }; + template.output_index = index; + template.item_id = itemId; + template.delta = contentItem.arguments; + return `event: response.function_call_arguments.delta\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.function_call_arguments.done +export const getResponseFunctionCallArgumentsDoneEvent = ( + index: number, + itemId: string, + contentItem: ResponseFunctionToolCall +): string => { + const template: ResponseFunctionCallArgumentsDoneEvent = { + ...RESPONSE_OUTPUT_FUNCTION_CALL_ARGUMENTS_DONE_EVENT, + }; + template.output_index = index; + template.item_id = itemId; + template.arguments = contentItem.arguments; + return `event: response.function_call_arguments.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response function call item done +export const getResponseOutputFunctionCallItemDoneEvent = ( + index: number, + outputItemId: string, + functionCallId: string, + contentItem: ResponseOutputItem +): string => { + if (contentItem.type !== 'function_call') return '\n\n'; // TOOD: this is for type safety, make it prettier + const template: ResponseOutputItemDoneEvent = { + ...RESPONSE_OUTPUT_FUNCTION_CALL_ITEM_DONE_EVENT, + }; + template.output_index = index; + const item: ResponseFunctionToolCall = + template.item as ResponseFunctionToolCall; + item.id = outputItemId; + item.call_id = functionCallId; + item.name = contentItem.name; + item.arguments = contentItem.arguments; + return `event: response.output_item.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.added web_search +export const getResponseOutputWebSearchItemAddedEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseOutputItemAddedEvent = { + ...RESPONSE_OUTPUT_WEB_SEARCH_ITEM_ADDED_EVENT, + }; + template.output_index = index; + const item: ResponseFunctionWebSearch = + template.item as ResponseFunctionWebSearch; + item.id = outputItemId; + return `event: response.output_item.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.web_search_call.in_progress +export const getResponseOutputWebSearchCallInProgressEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseWebSearchCallInProgressEvent = { + ...RESPONSE_OUTPUT_WEB_SEARCH_CALL_IN_PROGRESS_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.web_search_call.in_progress\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.web_search_call.searching +export const getResponseOutputWebSearchCallSearchingEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseWebSearchCallSearchingEvent = { + ...RESPONSE_OUTPUT_WEB_SEARCH_CALL_SEARCHING_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.web_search_call.searching\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.web_search_call.completed +export const getResponseOutputWebSearchCallCompletedEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseWebSearchCallCompletedEvent = { + ...RESPONSE_OUTPUT_WEB_SEARCH_CALL_COMPLETED_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.web_search_call.completed\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.done web_search +export const getResponseOutputWebSearchItemDoneEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseOutputItemDoneEvent = { + ...RESPONSE_OUTPUT_WEB_SEARCH_ITEM_DONE_EVENT, + }; + template.output_index = index; + const item: ResponseFunctionWebSearch = + template.item as ResponseFunctionWebSearch; + item.id = outputItemId; + return `event: response.output_item.done\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.added file_search +export const getResponseOutputFileSearchItemAddedEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseOutputItemAddedEvent = { + ...RESPONSE_OUTPUT_FILE_SEARCH_ITEM_ADDED_EVENT, + }; + template.output_index = index; + const item: ResponseFileSearchToolCall = + template.item as ResponseFileSearchToolCall; + item.id = outputItemId; + return `event: response.output_item.added\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.file_search_call.in_progress +export const getResponseOutputFileSearchCallInProgressEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseFileSearchCallInProgressEvent = { + ...RESPONSE_OUTPUT_FILE_SEARCH_CALL_IN_PROGRESS_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.file_search_call.in_progress\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.file_search_call.searching +export const getResponseOutputFileSearchCallSearchingEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseFileSearchCallSearchingEvent = { + ...RESPONSE_OUTPUT_FILE_SEARCH_CALL_SEARCHING_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.file_search_call.searching\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.file_search_call.completed +export const getResponseOutputFileSearchCallCompletedEvent = ( + index: number, + outputItemId: string +): string => { + const template: ResponseFileSearchCallCompletedEvent = { + ...RESPONSE_OUTPUT_FILE_SEARCH_CALL_COMPLETED_EVENT, + }; + template.output_index = index; + template.item_id = outputItemId; + return `event: response.file_search_call.completed\ndata: ${JSON.stringify(template)}\n\n`; +}; + +// response.output_item.done file_search +export const getResponseOutputFileSearchItemDoneEvent = ( + index: number, + outputItemId: string, + contentItem: ResponseOutputItem +): string => { + if (contentItem.type !== 'file_search_call') return '\n\n'; // TOOD: this is for type safety, make it prettier + const template: ResponseOutputItemDoneEvent = { + ...RESPONSE_OUTPUT_FILE_SEARCH_ITEM_DONE_EVENT, + }; + template.output_index = index; + const item: ResponseFileSearchToolCall = + template.item as ResponseFileSearchToolCall; + item.id = outputItemId; + item.queries = contentItem.queries; + item.results = contentItem.results; + return `event: response.output_item.done\ndata: ${JSON.stringify(template)}\n\n`; +}; diff --git a/src/providers/open-ai-base/index.ts b/src/providers/open-ai-base/index.ts index 0f89932f..ac6f98a8 100644 --- a/src/providers/open-ai-base/index.ts +++ b/src/providers/open-ai-base/index.ts @@ -5,7 +5,7 @@ import { Params, Message } from '../../types/requestBody'; import { OpenAIResponse, ModelResponseDeleteResponse, -} from '../../types/responses'; +} from '../../types/modelResponses'; import { OpenAIChatCompleteResponse } from '../openai/chatComplete'; import { OpenAICompleteResponse } from '../openai/complete'; import { OpenAIErrorResponseTransform } from '../openai/utils'; diff --git a/src/types/inputList.ts b/src/types/inputList.ts index d260b384..4b1da097 100644 --- a/src/types/inputList.ts +++ b/src/types/inputList.ts @@ -1,6 +1,6 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -import * as ResponsesAPI from './responses'; +import * as ResponsesAPI from './modelResponses'; /** * A list of Response items. diff --git a/src/types/responses.ts b/src/types/modelResponses.ts similarity index 95% rename from src/types/responses.ts rename to src/types/modelResponses.ts index ba3dbd20..058ee8ec 100644 --- a/src/types/responses.ts +++ b/src/types/modelResponses.ts @@ -26,7 +26,7 @@ export type ParsedResponseOutputItem = | ResponseFileSearchToolCall | ResponseFunctionWebSearch | ResponseComputerToolCall - | ResponseOutputItem.Reasoning; + | ResponseOutputItemReasoning; export interface ParsedResponse extends OpenAIResponse { output: Array>; @@ -118,26 +118,21 @@ export interface FileSearchTool { /** * Ranking options for search. */ - ranking_options?: FileSearchTool.RankingOptions; + ranking_options?: FileSearchToolRankingOptions; } -export namespace FileSearchTool { +export interface FileSearchToolRankingOptions { /** - * Ranking options for search. + * The ranker to use for the file search. */ - export interface RankingOptions { - /** - * The ranker to use for the file search. - */ - ranker?: 'auto' | 'default-2024-11-15'; + ranker?: 'auto' | 'default-2024-11-15'; - /** - * The score threshold for the file search, a number between 0 and 1. Numbers - * closer to 1 will attempt to return only the most relevant results, but may - * return fewer results. - */ - score_threshold?: number; - } + /** + * The score threshold for the file search, a number between 0 and 1. Numbers + * closer to 1 will attempt to return only the most relevant results, but may + * return fewer results. + */ + score_threshold?: number; } /** @@ -184,12 +179,15 @@ export interface OpenAIResponse { */ created_at: number; - output_text: string; + /** + * An error object returned when the model fails to generate a Response. + */ + error: ResponseError | null; /** * Details about why the response is incomplete. */ - incomplete_details: Response.IncompleteDetails | null; + incomplete_details: IncompleteDetails | null; /** * Inserts a system (or developer) message as the first item in the model's @@ -335,26 +333,23 @@ export interface OpenAIResponse { * Represents token usage details including input tokens, output tokens, a * breakdown of output tokens, and the total tokens used. */ - usage?: ResponseUsage; + usage?: ResponseUsage | null; /** * A unique identifier representing your end-user, which can help OpenAI to monitor * and detect abuse. * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices#end-user-ids). */ - user?: string; + user?: string | null; + + store?: boolean; } -export namespace Response { +export interface IncompleteDetails { /** - * Details about why the response is incomplete. + * The reason why the response is incomplete. */ - export interface IncompleteDetails { - /** - * The reason why the response is incomplete. - */ - reason?: 'max_output_tokens' | 'content_filter'; - } + reason?: 'max_output_tokens' | 'content_filter'; } /** @@ -954,25 +949,7 @@ export interface ResponseError { /** * The error code for the response. */ - code: - | 'server_error' - | 'rate_limit_exceeded' - | 'invalid_prompt' - | 'vector_store_timeout' - | 'invalid_image' - | 'invalid_image_format' - | 'invalid_base64_image' - | 'invalid_image_url' - | 'image_too_large' - | 'image_too_small' - | 'image_parse_error' - | 'image_content_policy_violation' - | 'invalid_image_mode' - | 'image_file_too_large' - | 'unsupported_image_media_type' - | 'empty_image_file' - | 'failed_to_download_image' - | 'image_file_not_found'; + code: string; /** * A human-readable description of the error. @@ -1713,49 +1690,39 @@ export type ResponseOutputItem = | ResponseFunctionToolCall | ResponseFunctionWebSearch | ResponseComputerToolCall - | ResponseOutputItem.Reasoning; + | ResponseOutputItemReasoning; -export namespace ResponseOutputItem { +export interface ResponseOutputItemReasoning { /** - * A description of the chain of thought used by a reasoning model while generating - * a response. + * The unique identifier of the reasoning content. */ - export interface Reasoning { + id: string; + + /** + * Reasoning text contents. + */ + content: { /** - * The unique identifier of the reasoning content. + * A short summary of the reasoning used by the model when generating the response. */ - id: string; + text: string; /** - * Reasoning text contents. + * The type of the object. Always `text`. */ - content: Array; + type: 'reasoning_summary'; + }; - /** - * The type of the object. Always `reasoning`. - */ - type: 'reasoning'; + /** + * The type of the object. Always `reasoning`. + */ + type: 'reasoning'; - /** - * The status of the item. One of `in_progress`, `completed`, or `incomplete`. - * Populated when items are returned via API. - */ - status?: 'in_progress' | 'completed' | 'incomplete'; - } - - export namespace Reasoning { - export interface Content { - /** - * A short summary of the reasoning used by the model when generating the response. - */ - text: string; - - /** - * The type of the object. Always `text`. - */ - type: 'reasoning_summary'; - } - } + /** + * The status of the item. One of `in_progress`, `completed`, or `incomplete`. + * Populated when items are returned via API. + */ + status?: 'in_progress' | 'completed' | 'incomplete'; } /** @@ -2417,38 +2384,36 @@ export interface WebSearchTool { */ search_context_size?: 'low' | 'medium' | 'high'; - user_location?: WebSearchTool.UserLocation | null; + user_location?: WebSearchToolUserLocation | null; } -export namespace WebSearchTool { - export interface UserLocation { - /** - * The type of location approximation. Always `approximate`. - */ - type: 'approximate'; +interface WebSearchToolUserLocation { + /** + * The type of location approximation. Always `approximate`. + */ + type: 'approximate'; - /** - * Free text input for the city of the user, e.g. `San Francisco`. - */ - city?: string; + /** + * Free text input for the city of the user, e.g. `San Francisco`. + */ + city?: string; - /** - * The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of - * the user, e.g. `US`. - */ - country?: string; + /** + * The two-letter [ISO country code](https://en.wikipedia.org/wiki/ISO_3166-1) of + * the user, e.g. `US`. + */ + country?: string; - /** - * Free text input for the region of the user, e.g. `California`. - */ - region?: string; + /** + * Free text input for the region of the user, e.g. `California`. + */ + region?: string; - /** - * The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the - * user, e.g. `America/Los_Angeles`. - */ - timezone?: string; - } + /** + * The [IANA timezone](https://timeapi.io/documentation/iana-timezones) of the + * user, e.g. `America/Los_Angeles`. + */ + timezone?: string; } export type ResponseCreateParams =