From aad975f778916bf3f202a9ba7a9cf2d2d5586cee Mon Sep 17 00:00:00 2001 From: visargD Date: Sun, 21 Jan 2024 16:00:15 +0530 Subject: [PATCH 01/10] fix: change mistral-ai safe_mode param to safe_prompt --- src/providers/mistral-ai/chatComplete.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/providers/mistral-ai/chatComplete.ts b/src/providers/mistral-ai/chatComplete.ts index bbea47c6..e9b6c3f4 100644 --- a/src/providers/mistral-ai/chatComplete.ts +++ b/src/providers/mistral-ai/chatComplete.ts @@ -40,8 +40,9 @@ export const MistralAIChatCompleteConfig: ProviderConfig = { param: "random_seed", default: null, }, - safe_mode: { - param: "safe_mode", + safe_prompt: { + param: "safe_prompt", + default: false }, }; From 208e0045e2bffbdfd77f7a73aaef510e2876b152 Mon Sep 17 00:00:00 2001 From: visargD Date: Sun, 21 Jan 2024 16:27:22 +0530 Subject: [PATCH 02/10] fix: add fallback mapping for mistral-ai safe_mode param --- src/providers/mistral-ai/chatComplete.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/providers/mistral-ai/chatComplete.ts b/src/providers/mistral-ai/chatComplete.ts index e9b6c3f4..a64f0c4b 100644 --- a/src/providers/mistral-ai/chatComplete.ts +++ b/src/providers/mistral-ai/chatComplete.ts @@ -44,6 +44,11 @@ export const MistralAIChatCompleteConfig: ProviderConfig = { param: "safe_prompt", default: false }, + // TODO: deprecate this and move to safe_prompt in next release + safe_mode: { + param: "safe_prompt", + default: false + }, }; interface MistralAIChatCompleteResponse extends ChatCompletionResponse { From 0cf477934211e6f7f743e26fb8aa30eb5a9a4534 Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 15:07:00 +0530 Subject: [PATCH 03/10] feat: add new content types constants --- src/globals.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/globals.ts b/src/globals.ts index 036ec9ff..a867ef9e 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -39,5 +39,7 @@ export const PROXY_REQUEST_PATH_PREFIX:string = "/v1/proxy"; export const CONTENT_TYPES = { APPLICATION_JSON: "application/json", MULTIPART_FORM_DATA: "multipart/form-data", - EVENT_STREAM: "text/event-stream" + EVENT_STREAM: "text/event-stream", + AUDIO_MPEG: "audio/mpeg", + APPLICATION_OCTET_STREAM: "application/octet-stream" } \ No newline at end of file From 39e147797440b36b86fe88db4d4e03e9e68b0a7b Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 16:27:29 +0530 Subject: [PATCH 04/10] feat: add openai chat completions json to stream transformer --- src/providers/openai/chatComplete.ts | 103 ++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/src/providers/openai/chatComplete.ts b/src/providers/openai/chatComplete.ts index 927e9b8b..d5ce73a0 100644 --- a/src/providers/openai/chatComplete.ts +++ b/src/providers/openai/chatComplete.ts @@ -83,8 +83,109 @@ export const OpenAIChatCompleteConfig: ProviderConfig = { } }; -interface OpenAIChatCompleteResponse extends ChatCompletionResponse { +export interface OpenAIChatCompleteResponse extends ChatCompletionResponse { system_fingerprint: string; } export const OpenAIChatCompleteResponseTransform: (response: OpenAIChatCompleteResponse) => ChatCompletionResponse = (response) => response; + +/** + * Transforms an OpenAI-format chat completions JSON response into an array of formatted OpenAI compatible text/event-stream chunks. + * + * @param {Object} response - The OpenAIChatCompleteResponse object. + * @param {string} provider - The provider string. + * @returns {Array} - An array of formatted stream chunks. + */ +export const OpenAIChatCompleteJSONToStreamResponseTransform: (response: OpenAIChatCompleteResponse, provider: string) => Array = (response, provider) => { + const streamChunkArray: Array = []; + const { id, model, system_fingerprint, choices } = response; + const streamChunkTemplate = { + id, + object: "chat.completion.chunk", + created: Date.now(), + model: model || "", + system_fingerprint: system_fingerprint || null, + provider + } + + for (const [index, choice] of choices.entries()) { + if (choice.message && choice.message.tool_calls) { + const toolCallNameChunk = { + index: 0, + function: { + name: choice.message.tool_calls[0].name, + arguments: "" + } + } + + const toolCallArgumentChunk = { + index: 0, + function: { + arguments: choice.message.tool_calls[0].arguments + } + } + + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + delta: { + role: "assistant", + content: null, + tool_calls: [toolCallNameChunk] + } + } + ] + })}\n\n`) + + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + delta: { + role: "assistant", + tool_calls: [toolCallArgumentChunk] + } + } + ] + })}\n\n`) + } + + if (choice.message && choice.message.content && typeof choice.message.content === "string") { + const inidividualWords: Array = []; + for (let i = 0; i < choice.message.content.length; i += 4) { + inidividualWords.push(choice.message.content.slice(i, i+4)); + } + inidividualWords.forEach((word: string) => { + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + delta: { + role: "assistant", + content: word + } + } + ] + }) + }\n\n`)}) + } + + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + delta: {}, + finish_reason: choice.finish_reason + } + ] + })}\n\n`) + } + + streamChunkArray.push(`data: [DONE]\n\n`) + return streamChunkArray; +} From 62588b168ea72d5f5c0308c7fcebf845480e5bd2 Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 16:27:53 +0530 Subject: [PATCH 05/10] feat: add openai completions json to stream transformer --- src/providers/openai/complete.ts | 55 +++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/providers/openai/complete.ts b/src/providers/openai/complete.ts index 26c7ecdf..4159fb66 100644 --- a/src/providers/openai/complete.ts +++ b/src/providers/openai/complete.ts @@ -75,8 +75,61 @@ export const OpenAICompleteConfig: ProviderConfig = { } }; -interface OpenAICompleteResponse extends CompletionResponse { +export interface OpenAICompleteResponse extends CompletionResponse { system_fingerprint: string; } export const OpenAICompleteResponseTransform: (response: OpenAICompleteResponse) => CompletionResponse = (response) => response; + +/** + * Transforms an OpenAI-format completions JSON into an array of formatted OpenAI compatible text/event-stream chunks. + * + * @param {Object} response - The OpenAICompleteResponse object. + * @param {string} provider - The provider string. + * @returns {Array} - An array of formatted stream chunks. + */ +export const OpenAICompleteJSONToStreamResponseTransform: (response: OpenAICompleteResponse, provider: string) => Array = (response, provider) => { + const streamChunkArray: Array = []; + const { id, model, choices } = response; + const streamChunkTemplate = { + id: id, + object: "text_completion", + created: Date.now(), + model: model ?? "", + provider + } + + for (const [index, choice] of choices.entries()) { + if (choice.text) { + const inidividualWords = []; + for (let i = 0; i < choice.text.length; i += 4) { + inidividualWords.push(choice.text.slice(i, i+4)); + } + inidividualWords.forEach((word: string) => { + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + text: word + } + ] + }) + }\n\n`)}) + } + + streamChunkArray.push(`data: ${JSON.stringify({ + ...streamChunkTemplate, + choices: [ + { + index: index, + text: "", + finish_reason: choice.finish_reason + } + ] + })}\n\n`) + } + + streamChunkArray.push(`data: [DONE]\n\n`) + return streamChunkArray; +} \ No newline at end of file From 9660c44f1cc60259215237c86c30e35b557ea722 Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 16:29:30 +0530 Subject: [PATCH 06/10] feat: add json to stream response handler for unified routes --- src/handlers/streamHandler.ts | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/handlers/streamHandler.ts b/src/handlers/streamHandler.ts index d4ae6efc..0cb7e30f 100644 --- a/src/handlers/streamHandler.ts +++ b/src/handlers/streamHandler.ts @@ -1,4 +1,6 @@ -import { AZURE_OPEN_AI, GOOGLE } from "../globals"; +import { AZURE_OPEN_AI, CONTENT_TYPES, GOOGLE } from "../globals"; +import { OpenAIChatCompleteResponse } from "../providers/openai/chatComplete"; +import { OpenAICompleteResponse } from "../providers/openai/complete"; import { getStreamModeSplitPattern } from "../utils"; export async function* readStream(reader: ReadableStreamDefaultReader, splitPattern: string, transformFunction: Function | undefined, isSleepTimeRequired: boolean, fallbackChunkId: string) { @@ -104,3 +106,25 @@ export async function handleStreamingMode(response: Response, proxyProvider: str return new Response(readable, response); } +export async function handleJSONToStreamResponse(response: Response, provider: string, responseTransformerFunction: Function): Promise { + const { readable, writable } = new TransformStream(); + const writer = writable.getWriter(); + const encoder = new TextEncoder(); + const responseJSON: OpenAIChatCompleteResponse | OpenAICompleteResponse = await response.clone().json(); + const streamChunkArray = responseTransformerFunction(responseJSON, provider); + + (async () => { + for (const chunk of streamChunkArray) { + await writer.write(encoder.encode(chunk)); + } + writer.close(); + })(); + + return new Response(readable, { + headers: new Headers({ + ...Object.fromEntries(response.headers), + 'content-type': CONTENT_TYPES.EVENT_STREAM + }) + }); +} + From 51b23429feecac996f4aed420816c6cdb9356bf7 Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 16:30:15 +0530 Subject: [PATCH 07/10] feat: add support for streaming from json cache response --- src/handlers/handlerUtils.ts | 55 +++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index 38baa547..a5b200b3 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -1,13 +1,15 @@ import { Context } from "hono"; -import { AZURE_OPEN_AI, GOOGLE, HEADER_KEYS, PALM, POWERED_BY, RESPONSE_HEADER_KEYS, RETRY_STATUS_CODES } from "../globals"; +import { AZURE_OPEN_AI, CONTENT_TYPES, GOOGLE, HEADER_KEYS, PALM, POWERED_BY, RESPONSE_HEADER_KEYS, RETRY_STATUS_CODES } from "../globals"; import Providers from "../providers"; import { ProviderAPIConfig, endpointStrings } from "../providers/types"; import transformToProviderRequest from "../services/transformToProviderRequest"; import { Config, Options, Params, RequestBody, ShortConfig, Targets } from "../types/requestBody"; import { convertKeysToCamelCase } from "../utils"; import { retryRequest } from "./retryHandler"; -import { handleAudioResponse, handleNonStreamingMode, handleOctetStreamResponse, handleStreamingMode } from "./streamHandler"; +import { handleAudioResponse, handleJSONToStreamResponse, handleNonStreamingMode, handleOctetStreamResponse, handleStreamingMode } from "./streamHandler"; import { env } from "hono/adapter"; +import { OpenAIChatCompleteJSONToStreamResponseTransform } from "../providers/openai/chatComplete"; +import { OpenAICompleteJSONToStreamResponseTransform } from "../providers/openai/complete"; /** * Constructs the request options for the API call. @@ -374,15 +376,12 @@ export async function tryPost(c: Context, providerOption:Options, inputParams: P ); if (cacheResponse) { response = await responseHandler( - new Response(cacheResponse, { - headers: { - "content-type": "application/json", - }, - }), - false, + new Response(cacheResponse), + isStreamingMode, provider, - undefined, - url + fn, + url, + true ); c.set("requestOptions", [ ...requestOptions, @@ -471,21 +470,43 @@ export async function tryProvidersInSequence(c: Context, providers:Options[], pa throw new Error(JSON.stringify(errors)); } -// Response Handlers for streaming & non-streaming -export function responseHandler(response: Response, streamingMode: boolean, proxyProvider: string, responseTransformer: string | undefined, requestURL: string): Promise { - // Checking status 200 so that errors are not considered as stream mode. +/** + * Handles various types of responses based on the specified parameters + * and returns a mapped response + * @param {Response} response - The HTTP response recieved from LLM. + * @param {boolean} streamingMode - Indicates whether streaming mode is enabled. + * @param {string} proxyProvider - The provider string. + * @param {string | undefined} responseTransformer - The response transformer to determine type of call. + * @param {string} requestURL - The URL of the original LLM request. + * @param {boolean} [isCacheHit=false] - Indicates whether the response is a cache hit. + * @returns {Promise} - A promise that resolves to the processed response. + */ +export function responseHandler(response: Response, streamingMode: boolean, proxyProvider: string, responseTransformer: string | undefined, requestURL: string, isCacheHit: boolean = false): Promise { let responseTransformerFunction: Function | undefined; + const responseContentType = response.headers?.get("content-type"); + + // Checking status 200 so that errors are not considered as stream mode. if (responseTransformer && streamingMode && response.status === 200) { responseTransformerFunction = Providers[proxyProvider]?.responseTransforms?.[`stream-${responseTransformer}`]; } else if (responseTransformer) { responseTransformerFunction = Providers[proxyProvider]?.responseTransforms?.[responseTransformer]; } - if (streamingMode && response.status === 200) { - return handleStreamingMode(response, proxyProvider, responseTransformerFunction, requestURL) - } else if (response.headers?.get("content-type") === "audio/mpeg") { + // JSON to text/event-stream conversion is only allowed for unified routes: chat completions and completions. + // Set the transformer to OpenAI json to stream convertor function in that case. + if (responseTransformer && streamingMode && isCacheHit) { + responseTransformerFunction = responseTransformer === "chatComplete" ? OpenAIChatCompleteJSONToStreamResponseTransform : OpenAICompleteJSONToStreamResponseTransform; + } else if (responseTransformer && !streamingMode && isCacheHit) { + responseTransformerFunction = undefined; + } + + if (streamingMode && response.status === 200 && isCacheHit && responseTransformer && responseTransformerFunction) { + return handleJSONToStreamResponse(response, proxyProvider, responseTransformerFunction) + } else if (streamingMode && response.status === 200) { + return handleStreamingMode(response, proxyProvider, responseTransformerFunction, requestURL) + } else if (responseContentType === CONTENT_TYPES.AUDIO_MPEG) { return handleAudioResponse(response) - } else if (response.headers?.get("content-type") === "application/octet-stream") { + } else if (responseContentType === CONTENT_TYPES.APPLICATION_OCTET_STREAM) { return handleOctetStreamResponse(response) } else { return handleNonStreamingMode(response, responseTransformerFunction) From 2deb968263ff4ae08a144eece460a59803367de2 Mon Sep 17 00:00:00 2001 From: visargD Date: Mon, 22 Jan 2024 19:43:48 +0530 Subject: [PATCH 08/10] add usage to json to stream transformed chunks --- src/providers/openai/chatComplete.ts | 9 ++++++++- src/providers/openai/complete.ts | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/providers/openai/chatComplete.ts b/src/providers/openai/chatComplete.ts index d5ce73a0..5d05e0cf 100644 --- a/src/providers/openai/chatComplete.ts +++ b/src/providers/openai/chatComplete.ts @@ -105,7 +105,14 @@ export const OpenAIChatCompleteJSONToStreamResponseTransform: (response: OpenAIC created: Date.now(), model: model || "", system_fingerprint: system_fingerprint || null, - provider + provider, + usage: {} + } + + if (response.usage?.completion_tokens) { + streamChunkTemplate.usage = { + completion_tokens: response.usage?.completion_tokens + } } for (const [index, choice] of choices.entries()) { diff --git a/src/providers/openai/complete.ts b/src/providers/openai/complete.ts index 4159fb66..dc73bb6b 100644 --- a/src/providers/openai/complete.ts +++ b/src/providers/openai/complete.ts @@ -96,7 +96,14 @@ export const OpenAICompleteJSONToStreamResponseTransform: (response: OpenAICompl object: "text_completion", created: Date.now(), model: model ?? "", - provider + provider, + usage: {} + } + + if (response.usage?.completion_tokens) { + streamChunkTemplate.usage = { + completion_tokens: response.usage?.completion_tokens + } } for (const [index, choice] of choices.entries()) { From de11fc69e19c5bf5e3d27af167567c14c58932ee Mon Sep 17 00:00:00 2001 From: visargD Date: Tue, 23 Jan 2024 19:45:37 +0530 Subject: [PATCH 09/10] chore: minor refactoring and enhancements --- src/handlers/handlerUtils.ts | 2 +- src/providers/openai/chatComplete.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index a5b200b3..c0857bd2 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -376,7 +376,7 @@ export async function tryPost(c: Context, providerOption:Options, inputParams: P ); if (cacheResponse) { response = await responseHandler( - new Response(cacheResponse), + new Response(cacheResponse, { headers: {"content-type": "application/json"}}), isStreamingMode, provider, fn, diff --git a/src/providers/openai/chatComplete.ts b/src/providers/openai/chatComplete.ts index 5d05e0cf..fdc236fa 100644 --- a/src/providers/openai/chatComplete.ts +++ b/src/providers/openai/chatComplete.ts @@ -117,10 +117,13 @@ export const OpenAIChatCompleteJSONToStreamResponseTransform: (response: OpenAIC for (const [index, choice] of choices.entries()) { if (choice.message && choice.message.tool_calls) { + const currentToolCall = choice.message.tool_calls[0]; const toolCallNameChunk = { index: 0, + id: currentToolCall.id, + type: "function", function: { - name: choice.message.tool_calls[0].name, + name: currentToolCall.function.name, arguments: "" } } @@ -128,7 +131,7 @@ export const OpenAIChatCompleteJSONToStreamResponseTransform: (response: OpenAIC const toolCallArgumentChunk = { index: 0, function: { - arguments: choice.message.tool_calls[0].arguments + arguments: currentToolCall.function.arguments } } From ae9786bea74cb8c4b39ead3095ec1ee2b5d0c46d Mon Sep 17 00:00:00 2001 From: visargD Date: Tue, 23 Jan 2024 20:30:33 +0530 Subject: [PATCH 10/10] minor cleaning --- src/handlers/handlerUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/handlers/handlerUtils.ts b/src/handlers/handlerUtils.ts index c0857bd2..d36da115 100644 --- a/src/handlers/handlerUtils.ts +++ b/src/handlers/handlerUtils.ts @@ -500,7 +500,7 @@ export function responseHandler(response: Response, streamingMode: boolean, prox responseTransformerFunction = undefined; } - if (streamingMode && response.status === 200 && isCacheHit && responseTransformer && responseTransformerFunction) { + if (streamingMode && response.status === 200 && isCacheHit && responseTransformerFunction) { return handleJSONToStreamResponse(response, proxyProvider, responseTransformerFunction) } else if (streamingMode && response.status === 200) { return handleStreamingMode(response, proxyProvider, responseTransformerFunction, requestURL)