feat(translator): preserve input_audio fields in OpenAI request conversions

- Updated `ConvertOpenAIRequestToGemini` and `ConvertOpenAIRequestToCodex` to handle `input_audio`, retaining `data` and `format` fields.
- Added helper `openAIInputAudioMimeType` for determining MIME types from audio formats.
- Introduced unit tests to validate correct preservation of `input_audio` data and format.

Closes: #3447
This commit is contained in:
Luis Pater
2026-06-20 00:03:54 +08:00
parent 1581700646
commit 51aa5ba925
4 changed files with 110 additions and 0 deletions
@@ -193,6 +193,20 @@ func ConvertOpenAIRequestToCodex(modelName string, inputRawJSON []byte, stream b
msg, _ = sjson.SetRawBytes(msg, "content.-1", part)
}
}
case "input_audio":
if role == "user" {
audioData := it.Get("input_audio.data").String()
audioFormat := it.Get("input_audio.format").String()
if audioData != "" {
part := []byte(`{}`)
part, _ = sjson.SetBytes(part, "type", "input_audio")
part, _ = sjson.SetBytes(part, "data", audioData)
if audioFormat != "" {
part, _ = sjson.SetBytes(part, "format", audioFormat)
}
msg, _ = sjson.SetRawBytes(msg, "content.-1", part)
}
}
}
}
}
@@ -352,6 +352,39 @@ func TestToolCallOutputWithNonStringJSONContent(t *testing.T) {
}
}
func TestConvertOpenAIRequestToCodexPreservesInputAudio(t *testing.T) {
input := []byte(`{
"model": "gpt-5.5",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe this audio verbatim."},
{"type": "input_audio", "input_audio": {"data": "SUQzBA==", "format": "mp3"}}
]
}
]
}`)
out := ConvertOpenAIRequestToCodex("gpt-5.5", input, true)
parts := gjson.GetBytes(out, "input.0.content").Array()
if len(parts) != 2 {
t.Fatalf("expected 2 content parts, got %d: %s", len(parts), gjson.GetBytes(out, "input.0.content").Raw)
}
if parts[0].Get("type").String() != "input_text" || parts[0].Get("text").String() != "Transcribe this audio verbatim." {
t.Fatalf("part 0: expected input_text with prompt text, got %s", parts[0].Raw)
}
if parts[1].Get("type").String() != "input_audio" {
t.Fatalf("part 1: expected input_audio, got %s", parts[1].Raw)
}
if parts[1].Get("data").String() != "SUQzBA==" {
t.Fatalf("part 1: expected audio data to be preserved, got %s", parts[1].Get("data").String())
}
if parts[1].Get("format").String() != "mp3" {
t.Fatalf("part 1: expected audio format mp3, got %s", parts[1].Get("format").String())
}
}
// Parallel tool calls: assistant invokes 3 tools at once, all call_ids
// and outputs must be translated and paired correctly.
func TestMultipleToolCalls(t *testing.T) {
@@ -210,6 +210,14 @@ func ConvertOpenAIRequestToGemini(modelName string, inputRawJSON []byte, _ bool)
} else {
log.Warnf("Unknown file name extension '%s' in user message, skip", ext)
}
case "input_audio":
audioData := item.Get("input_audio.data").String()
if audioData != "" {
mimeType := openAIInputAudioMimeType(item.Get("input_audio.format").String())
node, _ = sjson.SetBytes(node, "parts."+itoa(p)+".inlineData.mime_type", mimeType)
node, _ = sjson.SetBytes(node, "parts."+itoa(p)+".inlineData.data", audioData)
p++
}
}
}
}
@@ -438,3 +446,26 @@ func openAIToolCallGeminiThoughtSignature(toolCall gjson.Result) string {
// itoa converts int to string without strconv import for few usages.
func itoa(i int) string { return fmt.Sprintf("%d", i) }
func openAIInputAudioMimeType(audioFormat string) string {
switch audioFormat {
case "", "wav":
return "audio/wav"
case "mp3":
return "audio/mpeg"
case "ogg":
return "audio/ogg"
case "flac":
return "audio/flac"
case "aac":
return "audio/aac"
case "webm":
return "audio/webm"
case "pcm16":
return "audio/pcm"
case "g711_ulaw", "g711_alaw":
return "audio/basic"
default:
return "audio/" + audioFormat
}
}
@@ -26,3 +26,35 @@ func TestConvertOpenAIRequestToGemini_StripsTrailingAssistantPrefill(t *testing.
t.Fatalf("final remaining role = %q, want %q", got, "user")
}
}
func TestConvertOpenAIRequestToGeminiPreservesInputAudio(t *testing.T) {
inputJSON := `{
"model": "gpt-5.5",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Transcribe this audio verbatim."},
{"type": "input_audio", "input_audio": {"data": "SUQzBA==", "format": "mp3"}}
]
}
]
}`
result := ConvertOpenAIRequestToGemini("gemini-3.1-pro-high", []byte(inputJSON), false)
resultJSON := gjson.ParseBytes(result)
parts := resultJSON.Get("contents.0.parts").Array()
if len(parts) != 2 {
t.Fatalf("parts length = %d, want 2. parts=%s", len(parts), resultJSON.Get("contents.0.parts").Raw)
}
if got := parts[0].Get("text").String(); got != "Transcribe this audio verbatim." {
t.Fatalf("text part = %q, want prompt text", got)
}
if got := parts[1].Get("inlineData.mime_type").String(); got != "audio/mpeg" {
t.Fatalf("audio mime_type = %q, want %q", got, "audio/mpeg")
}
if got := parts[1].Get("inlineData.data").String(); got != "SUQzBA==" {
t.Fatalf("audio data = %q, want %q", got, "SUQzBA==")
}
}