mirror of
https://gh-proxy.org/https://github.com/klinsmaya/CLIProxyAPI.git
synced 2026-11-10 02:16:49 +08:00
feat(translator): preserve input_audio fields in OpenAI request conversions
- Updated `ConvertOpenAIRequestToGemini` and `ConvertOpenAIRequestToCodex` to handle `input_audio`, retaining `data` and `format` fields. - Added helper `openAIInputAudioMimeType` for determining MIME types from audio formats. - Introduced unit tests to validate correct preservation of `input_audio` data and format. Closes: #3447
This commit is contained in:
@@ -193,6 +193,20 @@ func ConvertOpenAIRequestToCodex(modelName string, inputRawJSON []byte, stream b
|
||||
msg, _ = sjson.SetRawBytes(msg, "content.-1", part)
|
||||
}
|
||||
}
|
||||
case "input_audio":
|
||||
if role == "user" {
|
||||
audioData := it.Get("input_audio.data").String()
|
||||
audioFormat := it.Get("input_audio.format").String()
|
||||
if audioData != "" {
|
||||
part := []byte(`{}`)
|
||||
part, _ = sjson.SetBytes(part, "type", "input_audio")
|
||||
part, _ = sjson.SetBytes(part, "data", audioData)
|
||||
if audioFormat != "" {
|
||||
part, _ = sjson.SetBytes(part, "format", audioFormat)
|
||||
}
|
||||
msg, _ = sjson.SetRawBytes(msg, "content.-1", part)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,6 +352,39 @@ func TestToolCallOutputWithNonStringJSONContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertOpenAIRequestToCodexPreservesInputAudio(t *testing.T) {
|
||||
input := []byte(`{
|
||||
"model": "gpt-5.5",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Transcribe this audio verbatim."},
|
||||
{"type": "input_audio", "input_audio": {"data": "SUQzBA==", "format": "mp3"}}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`)
|
||||
|
||||
out := ConvertOpenAIRequestToCodex("gpt-5.5", input, true)
|
||||
parts := gjson.GetBytes(out, "input.0.content").Array()
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("expected 2 content parts, got %d: %s", len(parts), gjson.GetBytes(out, "input.0.content").Raw)
|
||||
}
|
||||
if parts[0].Get("type").String() != "input_text" || parts[0].Get("text").String() != "Transcribe this audio verbatim." {
|
||||
t.Fatalf("part 0: expected input_text with prompt text, got %s", parts[0].Raw)
|
||||
}
|
||||
if parts[1].Get("type").String() != "input_audio" {
|
||||
t.Fatalf("part 1: expected input_audio, got %s", parts[1].Raw)
|
||||
}
|
||||
if parts[1].Get("data").String() != "SUQzBA==" {
|
||||
t.Fatalf("part 1: expected audio data to be preserved, got %s", parts[1].Get("data").String())
|
||||
}
|
||||
if parts[1].Get("format").String() != "mp3" {
|
||||
t.Fatalf("part 1: expected audio format mp3, got %s", parts[1].Get("format").String())
|
||||
}
|
||||
}
|
||||
|
||||
// Parallel tool calls: assistant invokes 3 tools at once, all call_ids
|
||||
// and outputs must be translated and paired correctly.
|
||||
func TestMultipleToolCalls(t *testing.T) {
|
||||
|
||||
@@ -210,6 +210,14 @@ func ConvertOpenAIRequestToGemini(modelName string, inputRawJSON []byte, _ bool)
|
||||
} else {
|
||||
log.Warnf("Unknown file name extension '%s' in user message, skip", ext)
|
||||
}
|
||||
case "input_audio":
|
||||
audioData := item.Get("input_audio.data").String()
|
||||
if audioData != "" {
|
||||
mimeType := openAIInputAudioMimeType(item.Get("input_audio.format").String())
|
||||
node, _ = sjson.SetBytes(node, "parts."+itoa(p)+".inlineData.mime_type", mimeType)
|
||||
node, _ = sjson.SetBytes(node, "parts."+itoa(p)+".inlineData.data", audioData)
|
||||
p++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -438,3 +446,26 @@ func openAIToolCallGeminiThoughtSignature(toolCall gjson.Result) string {
|
||||
|
||||
// itoa converts int to string without strconv import for few usages.
|
||||
func itoa(i int) string { return fmt.Sprintf("%d", i) }
|
||||
|
||||
func openAIInputAudioMimeType(audioFormat string) string {
|
||||
switch audioFormat {
|
||||
case "", "wav":
|
||||
return "audio/wav"
|
||||
case "mp3":
|
||||
return "audio/mpeg"
|
||||
case "ogg":
|
||||
return "audio/ogg"
|
||||
case "flac":
|
||||
return "audio/flac"
|
||||
case "aac":
|
||||
return "audio/aac"
|
||||
case "webm":
|
||||
return "audio/webm"
|
||||
case "pcm16":
|
||||
return "audio/pcm"
|
||||
case "g711_ulaw", "g711_alaw":
|
||||
return "audio/basic"
|
||||
default:
|
||||
return "audio/" + audioFormat
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,3 +26,35 @@ func TestConvertOpenAIRequestToGemini_StripsTrailingAssistantPrefill(t *testing.
|
||||
t.Fatalf("final remaining role = %q, want %q", got, "user")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertOpenAIRequestToGeminiPreservesInputAudio(t *testing.T) {
|
||||
inputJSON := `{
|
||||
"model": "gpt-5.5",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Transcribe this audio verbatim."},
|
||||
{"type": "input_audio", "input_audio": {"data": "SUQzBA==", "format": "mp3"}}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
result := ConvertOpenAIRequestToGemini("gemini-3.1-pro-high", []byte(inputJSON), false)
|
||||
resultJSON := gjson.ParseBytes(result)
|
||||
parts := resultJSON.Get("contents.0.parts").Array()
|
||||
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("parts length = %d, want 2. parts=%s", len(parts), resultJSON.Get("contents.0.parts").Raw)
|
||||
}
|
||||
if got := parts[0].Get("text").String(); got != "Transcribe this audio verbatim." {
|
||||
t.Fatalf("text part = %q, want prompt text", got)
|
||||
}
|
||||
if got := parts[1].Get("inlineData.mime_type").String(); got != "audio/mpeg" {
|
||||
t.Fatalf("audio mime_type = %q, want %q", got, "audio/mpeg")
|
||||
}
|
||||
if got := parts[1].Get("inlineData.data").String(); got != "SUQzBA==" {
|
||||
t.Fatalf("audio data = %q, want %q", got, "SUQzBA==")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user