From 613255e0d05f4176aadce636689ad14b91c39e13 Mon Sep 17 00:00:00 2001 From: Dmitry Rubtsov Date: Wed, 15 Jul 2026 03:30:23 +0600 Subject: [PATCH 1/2] perf(translator): build input and messages arrays in a single pass - Replaced per-item `sjson.SetRawBytes(out, "input.-1"/"messages.-1", ...)` appends, which reparsed and rewrote the growing output document on every conversation item, causing O(n^2) work on long histories. - Applied the same approach to both interactions translation directions and the OpenAI Responses to Chat Completions path. --- internal/translator/common/bytes.go | 19 ++++ internal/translator/common/bytes_test.go | 23 +++++ .../interactions_openai_responses_request.go | 95 +++++++++++-------- .../openai_openai-responses_request.go | 24 +++-- 4 files changed, 112 insertions(+), 49 deletions(-) create mode 100644 internal/translator/common/bytes_test.go diff --git a/internal/translator/common/bytes.go b/internal/translator/common/bytes.go index 96bec594..3b5ad814 100644 --- a/internal/translator/common/bytes.go +++ b/internal/translator/common/bytes.go @@ -22,6 +22,25 @@ func ClaudeInputTokensJSON(count int64) []byte { return out } +func JoinRawArray(items [][]byte) []byte { + if len(items) == 0 { + return []byte("[]") + } + size := len(items) + 1 + for _, item := range items { + size += len(item) + } + out := make([]byte, 0, size) + out = append(out, '[') + for i, item := range items { + if i > 0 { + out = append(out, ',') + } + out = append(out, item...) + } + return append(out, ']') +} + func SSEEventData(event string, payload []byte) []byte { out := make([]byte, 0, len(event)+len(payload)+14) out = append(out, "event: "...) diff --git a/internal/translator/common/bytes_test.go b/internal/translator/common/bytes_test.go new file mode 100644 index 00000000..2ce96705 --- /dev/null +++ b/internal/translator/common/bytes_test.go @@ -0,0 +1,23 @@ +package common + +import "testing" + +func TestJoinRawArray(t *testing.T) { + tests := []struct { + name string + items [][]byte + want string + }{ + {name: "empty", want: "[]"}, + {name: "single", items: [][]byte{[]byte(`{"id":1}`)}, want: `[{"id":1}]`}, + {name: "multiple", items: [][]byte{[]byte(`{"id":1}`), []byte(`{"id":2}`)}, want: `[{"id":1},{"id":2}]`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := string(JoinRawArray(test.items)); got != test.want { + t.Fatalf("JoinRawArray() = %s, want %s", got, test.want) + } + }) + } +} diff --git a/internal/translator/openai/interactions/responses/interactions_openai_responses_request.go b/internal/translator/openai/interactions/responses/interactions_openai_responses_request.go index d6e45bad..22c3e118 100644 --- a/internal/translator/openai/interactions/responses/interactions_openai_responses_request.go +++ b/internal/translator/openai/interactions/responses/interactions_openai_responses_request.go @@ -3,6 +3,7 @@ package responses import ( "strings" + translatorcommon "github.com/router-for-me/CLIProxyAPI/v7/internal/translator/common" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -21,7 +22,7 @@ func ConvertOpenAIResponsesRequestToInteractions(modelName string, inputRawJSON out, _ = sjson.SetBytes(out, "previous_interaction_id", previousResponseID.String()) } if input := root.Get("input"); input.Exists() { - out = appendResponsesInputToInteractions(out, input) + out = setResponsesInputOnInteractions(out, input) } out = appendResponsesToolsToInteractions(out, root.Get("tools")) if toolChoice := root.Get("tool_choice"); toolChoice.Exists() { @@ -55,7 +56,7 @@ func ConvertInteractionsRequestToOpenAIResponses(modelName string, inputRawJSON out, _ = sjson.SetBytes(out, "previous_response_id", previousInteractionID.String()) } if input := root.Get("input"); input.Exists() { - out = appendInteractionsInputToResponses(out, input) + out = setInteractionsInputOnResponses(out, input) } out = appendInteractionsToolsToResponses(out, root.Get("tools")) if toolChoice := root.Get("generation_config.tool_choice"); toolChoice.Exists() { @@ -156,25 +157,30 @@ func interactionsThinkingEffort(root gjson.Result) string { return "" } -func appendResponsesInputToInteractions(out []byte, input gjson.Result) []byte { +func setResponsesInputOnInteractions(out []byte, input gjson.Result) []byte { functionNamesByCallID := make(map[string]string) + items := make([][]byte, 0) if input.Type == gjson.String { - return appendInteractionsTextStep(out, "user_input", input.String()) - } - if input.IsArray() { + items = append(items, interactionsTextStep("user_input", input.String())) + } else if input.IsArray() { input.ForEach(func(_, item gjson.Result) bool { - out = appendResponsesInputItemToInteractions(out, item, functionNamesByCallID) + if converted := responsesInputItemToInteractions(item, functionNamesByCallID); converted != nil { + items = append(items, converted) + } return true }) - return out + } else if input.IsObject() { + if converted := responsesInputItemToInteractions(input, functionNamesByCallID); converted != nil { + items = append(items, converted) + } } - if input.IsObject() { - return appendResponsesInputItemToInteractions(out, input, functionNamesByCallID) + if len(items) > 0 { + out, _ = sjson.SetRawBytes(out, "input", translatorcommon.JoinRawArray(items)) } return out } -func appendResponsesInputItemToInteractions(out []byte, item gjson.Result, functionNamesByCallID map[string]string) []byte { +func responsesInputItemToInteractions(item gjson.Result, functionNamesByCallID map[string]string) []byte { switch item.Get("type").String() { case "message": stepType := "user_input" @@ -183,8 +189,7 @@ func appendResponsesInputItemToInteractions(out []byte, item gjson.Result, funct } step := []byte(`{"type":"","content":[]}`) step, _ = sjson.SetBytes(step, "type", stepType) - step = appendResponsesContentToInteractions(step, item.Get("content"), stepType) - out, _ = sjson.SetRawBytes(out, "input.-1", step) + return appendResponsesContentToInteractions(step, item.Get("content"), stepType) case "function_call": callID := firstNonEmpty(item.Get("call_id").String(), item.Get("id").String()) if callID != "" { @@ -192,15 +197,15 @@ func appendResponsesInputItemToInteractions(out []byte, item gjson.Result, funct functionNamesByCallID[callID] = name } } - out, _ = sjson.SetRawBytes(out, "input.-1", responsesFunctionCallToInteractions(item)) + return responsesFunctionCallToInteractions(item) case "function_call_output": - out, _ = sjson.SetRawBytes(out, "input.-1", responsesFunctionOutputToInteractions(item, functionNamesByCallID)) + return responsesFunctionOutputToInteractions(item, functionNamesByCallID) case "input_text", "output_text", "text": stepType := "user_input" if item.Get("type").String() == "output_text" { stepType = "model_output" } - out = appendInteractionsTextStep(out, stepType, item.Get("text").String()) + return interactionsTextStep(stepType, item.Get("text").String()) case "input_image", "output_image": stepType := "user_input" if item.Get("type").String() == "output_image" { @@ -211,15 +216,14 @@ func appendResponsesInputItemToInteractions(out []byte, item gjson.Result, funct if part, ok := responsesContentPartToInteractions(item); ok { step, _ = sjson.SetRawBytes(step, "content.-1", part) } - out, _ = sjson.SetRawBytes(out, "input.-1", step) + return step default: if content := item.Get("content"); content.Exists() { step := []byte(`{"type":"user_input","content":[]}`) - step = appendResponsesContentToInteractions(step, content, "user_input") - out, _ = sjson.SetRawBytes(out, "input.-1", step) + return appendResponsesContentToInteractions(step, content, "user_input") } } - return out + return nil } func appendResponsesContentToInteractions(step []byte, content gjson.Result, stepType string) []byte { @@ -317,12 +321,11 @@ func responsesFunctionOutputToInteractions(item gjson.Result, functionNamesByCal return out } -func appendInteractionsTextStep(out []byte, stepType, text string) []byte { +func interactionsTextStep(stepType, text string) []byte { step := []byte(`{"type":"","content":[{"type":"text","text":""}]}`) step, _ = sjson.SetBytes(step, "type", stepType) step, _ = sjson.SetBytes(step, "content.0.text", text) - out, _ = sjson.SetRawBytes(out, "input.-1", step) - return out + return step } func appendResponsesToolsToInteractions(out []byte, tools gjson.Result) []byte { @@ -380,44 +383,52 @@ func functionDeclarationFromTool(tool gjson.Result) ([]byte, bool) { return out, true } -func appendInteractionsInputToResponses(out []byte, input gjson.Result) []byte { +func setInteractionsInputOnResponses(out []byte, input gjson.Result) []byte { + items := make([][]byte, 0) if input.Type == gjson.String { - item := []byte(`{"type":"message","role":"user","content":[{"type":"input_text","text":""}]}`) - item, _ = sjson.SetBytes(item, "content.0.text", input.String()) - out, _ = sjson.SetRawBytes(out, "input.-1", item) - return out - } - if input.IsArray() { + items = append(items, interactionsTextMessage(input.String())) + } else if input.IsArray() { input.ForEach(func(_, item gjson.Result) bool { - out = appendInteractionsInputItemToResponses(out, item) + if converted := interactionsInputItemToResponses(item); converted != nil { + items = append(items, converted) + } return true }) - return out + } else if input.IsObject() { + if converted := interactionsInputItemToResponses(input); converted != nil { + items = append(items, converted) + } } - if input.IsObject() { - return appendInteractionsInputItemToResponses(out, input) + if len(items) > 0 { + out, _ = sjson.SetRawBytes(out, "input", translatorcommon.JoinRawArray(items)) } return out } -func appendInteractionsInputItemToResponses(out []byte, item gjson.Result) []byte { +func interactionsTextMessage(text string) []byte { + item := []byte(`{"type":"message","role":"user","content":[{"type":"input_text","text":""}]}`) + item, _ = sjson.SetBytes(item, "content.0.text", text) + return item +} + +func interactionsInputItemToResponses(item gjson.Result) []byte { switch item.Get("type").String() { case "user_input": - out, _ = sjson.SetRawBytes(out, "input.-1", interactionsMessageToResponses(item, "user")) + return interactionsMessageToResponses(item, "user") case "model_output": - out, _ = sjson.SetRawBytes(out, "input.-1", interactionsMessageToResponses(item, "assistant")) + return interactionsMessageToResponses(item, "assistant") case "thought": - out, _ = sjson.SetRawBytes(out, "input.-1", interactionsThoughtToResponses(item)) + return interactionsThoughtToResponses(item) case "function_call": - out, _ = sjson.SetRawBytes(out, "input.-1", interactionsFunctionCallToResponses(item)) + return interactionsFunctionCallToResponses(item) case "function_result": - out, _ = sjson.SetRawBytes(out, "input.-1", interactionsFunctionResultToResponses(item)) + return interactionsFunctionResultToResponses(item) default: if item.Type == gjson.String { - return appendInteractionsInputToResponses(out, item) + return interactionsTextMessage(item.String()) } } - return out + return nil } func interactionsMessageToResponses(item gjson.Result, role string) []byte { diff --git a/internal/translator/openai/openai/responses/openai_openai-responses_request.go b/internal/translator/openai/openai/responses/openai_openai-responses_request.go index c5e76dc4..b03e9009 100644 --- a/internal/translator/openai/openai/responses/openai_openai-responses_request.go +++ b/internal/translator/openai/openai/responses/openai_openai-responses_request.go @@ -3,6 +3,7 @@ package responses import ( "strings" + translatorcommon "github.com/router-for-me/CLIProxyAPI/v7/internal/translator/common" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -33,6 +34,11 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu root := gjson.ParseBytes(rawJSON) + messages := make([][]byte, 0) + appendMessage := func(message []byte) { + messages = append(messages, message) + } + // Set model name out, _ = sjson.SetBytes(out, "model", modelName) @@ -52,7 +58,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu if instructions := root.Get("instructions"); instructions.Exists() { systemMessage := []byte(`{"role":"system","content":""}`) systemMessage, _ = sjson.SetBytes(systemMessage, "content", instructions.String()) - out, _ = sjson.SetRawBytes(out, "messages.-1", systemMessage) + appendMessage(systemMessage) } // Convert input array to messages @@ -91,7 +97,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu if reasoningContent := takePendingReasoningContent(); reasoningContent != "" { assistantMessage, _ = sjson.SetBytes(assistantMessage, "reasoning_content", reasoningContent) } - out, _ = sjson.SetRawBytes(out, "messages.-1", assistantMessage) + appendMessage(assistantMessage) for _, id := range pendingToolCallIDs { if strings.TrimSpace(id) == "" { continue @@ -103,7 +109,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu } flushDeferredMessages := func() { for _, message := range deferredMessages { - out, _ = sjson.SetRawBytes(out, "messages.-1", message) + appendMessage(message) } deferredMessages = deferredMessages[:0] } @@ -122,7 +128,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu deferredMessages = append(deferredMessages, message) return } - out, _ = sjson.SetRawBytes(out, "messages.-1", message) + appendMessage(message) } appendPendingReasoningMessage := func() { reasoningContent := takePendingReasoningContent() @@ -251,7 +257,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu toolMessage, _ = sjson.SetBytes(toolMessage, "content", output.String()) } - out, _ = sjson.SetRawBytes(out, "messages.-1", toolMessage) + appendMessage(toolMessage) if callID != "" { delete(awaitingToolOutputs, callID) } @@ -278,7 +284,7 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu callID := strings.TrimSpace(item.Get("call_id").String()) toolMessage, _ = sjson.SetBytes(toolMessage, "tool_call_id", callID) toolMessage, _ = sjson.SetBytes(toolMessage, "content", responsesToolOutputText(item.Get("output"))) - out, _ = sjson.SetRawBytes(out, "messages.-1", toolMessage) + appendMessage(toolMessage) if callID != "" { delete(awaitingToolOutputs, callID) } @@ -295,7 +301,11 @@ func ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName string, inpu msg := []byte(`{}`) msg, _ = sjson.SetBytes(msg, "role", "user") msg, _ = sjson.SetBytes(msg, "content", input.String()) - out, _ = sjson.SetRawBytes(out, "messages.-1", msg) + appendMessage(msg) + } + + if len(messages) > 0 { + out, _ = sjson.SetRawBytes(out, "messages", translatorcommon.JoinRawArray(messages)) } // Convert tools from responses format to chat completions format. From 3c6c7b736e9f419fcbd02e0ea03f5da3213526be Mon Sep 17 00:00:00 2001 From: Dmitry Rubtsov Date: Wed, 15 Jul 2026 18:21:00 +0600 Subject: [PATCH 2/2] perf(translator): build claude system and messages arrays in a single pass - Replaced per-item `sjson.SetRawBytes(out, "system.-1"/"messages.-1", ...)` appends, which reparsed and rewrote the growing output document on every conversation item, causing O(n^2) work on long histories. - Collected system and message blocks into slices and wrote each array once via `common.JoinRawArray`, moving the cache_control post-processing and system-only fallback turn onto the collected slices. --- .../chat-completions/claude_openai_request.go | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/internal/translator/claude/openai/chat-completions/claude_openai_request.go b/internal/translator/claude/openai/chat-completions/claude_openai_request.go index fb7fb2b8..611df009 100644 --- a/internal/translator/claude/openai/chat-completions/claude_openai_request.go +++ b/internal/translator/claude/openai/chat-completions/claude_openai_request.go @@ -163,39 +163,36 @@ func ConvertOpenAIRequestToClaude(modelName string, inputRawJSON []byte, stream // Process messages and transform them to Claude Code format if messages := root.Get("messages"); messages.Exists() && messages.IsArray() { - messageIndex := 0 + systemBlocks := make([][]byte, 0) + messageBlocks := make([][]byte, 0) messages.ForEach(func(_, message gjson.Result) bool { role := message.Get("role").String() contentResult := message.Get("content") switch role { case "system": - systemStart := len(gjson.GetBytes(out, "system").Array()) + systemStart := len(systemBlocks) if contentResult.Exists() && contentResult.Type == gjson.String && contentResult.String() != "" { textPart := []byte(`{"type":"text","text":""}`) textPart, _ = sjson.SetBytes(textPart, "text", contentResult.String()) textPart = common.AttachCacheControl(textPart, message) - out, _ = sjson.SetRawBytes(out, "system.-1", textPart) + systemBlocks = append(systemBlocks, textPart) } else if contentResult.Exists() && contentResult.IsArray() { contentResult.ForEach(func(_, part gjson.Result) bool { if part.Get("type").String() == "text" { textPart := []byte(`{"type":"text","text":""}`) textPart, _ = sjson.SetBytes(textPart, "text", part.Get("text").String()) textPart = common.AttachCacheControl(textPart, part) - out, _ = sjson.SetRawBytes(out, "system.-1", textPart) + systemBlocks = append(systemBlocks, textPart) } return true }) // Message-level cache_control applies to the last system block from this message. if message.Get("cache_control").Exists() { - systemArr := gjson.GetBytes(out, "system").Array() - if len(systemArr) > systemStart { - lastIdx := len(systemArr) - 1 - if !systemArr[lastIdx].Get("cache_control").Exists() { - path := fmt.Sprintf("system.%d", lastIdx) - block := []byte(systemArr[lastIdx].Raw) - block = common.AttachCacheControl(block, message) - out, _ = sjson.SetRawBytes(out, path, block) + if len(systemBlocks) > systemStart { + lastIdx := len(systemBlocks) - 1 + if !gjson.GetBytes(systemBlocks[lastIdx], "cache_control").Exists() { + systemBlocks[lastIdx] = common.AttachCacheControl(systemBlocks[lastIdx], message) } } } @@ -258,8 +255,7 @@ func ConvertOpenAIRequestToClaude(modelName string, inputRawJSON []byte, stream } msg = common.AttachMessageCacheControl(msg, message) - out, _ = sjson.SetRawBytes(out, "messages.-1", msg) - messageIndex++ + messageBlocks = append(messageBlocks, msg) case "tool": // Handle tool result messages conversion @@ -276,20 +272,22 @@ func ConvertOpenAIRequestToClaude(modelName string, inputRawJSON []byte, stream msg, _ = sjson.SetBytes(msg, "content.0.content", toolResultContent) } msg = common.AttachMessageCacheControl(msg, message) - out, _ = sjson.SetRawBytes(out, "messages.-1", msg) - messageIndex++ + messageBlocks = append(messageBlocks, msg) } return true }) // Preserve a minimal conversational turn for system-only inputs. // Claude payloads with top-level system instructions but no messages are risky for downstream validation. - if messageIndex == 0 { - system := gjson.GetBytes(out, "system") - if system.Exists() && system.IsArray() && len(system.Array()) > 0 { - fallbackMsg := []byte(`{"role":"user","content":[{"type":"text","text":""}]}`) - out, _ = sjson.SetRawBytes(out, "messages.-1", fallbackMsg) - } + if len(messageBlocks) == 0 && len(systemBlocks) > 0 { + messageBlocks = append(messageBlocks, []byte(`{"role":"user","content":[{"type":"text","text":""}]}`)) + } + + if len(systemBlocks) > 0 { + out, _ = sjson.SetRawBytes(out, "system", common.JoinRawArray(systemBlocks)) + } + if len(messageBlocks) > 0 { + out, _ = sjson.SetRawBytes(out, "messages", common.JoinRawArray(messageBlocks)) } }