diff --git a/config.example.yaml b/config.example.yaml index d593c2f7..e435742a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -341,7 +341,9 @@ nonstream-keepalive-interval: 0 # models: # The models supported by the provider. # - name: "moonshotai/kimi-k2:free" # The actual model name. # alias: "kimi-k2" # The alias used in the API. -# image: false # optional: set true to allow this model on /v1/images/generations and /v1/images/edits +# image: false # optional: set true to allow this model on /v1/images/generations and /v1/images/edits (not chat/responses image input) +# input-modalities: [text, image] # optional: declare /v1/chat/completions and /v1/responses multimodal input for Codex clients +# output-modalities: [text] # optional: declare output modalities when known # thinking: # optional: omit to default to levels ["low","medium","high"] # levels: ["low", "medium", "high"] # # You may repeat the same alias to build an internal model pool. diff --git a/internal/config/config.go b/internal/config/config.go index b8e603e3..fcee1bf7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -660,6 +660,13 @@ type OpenAICompatibilityModel struct { // Image marks this model as callable through /v1/images/generations and /v1/images/edits. Image bool `yaml:"image,omitempty" json:"image,omitempty"` + // InputModalities declares chat/responses input capabilities (e.g. text, image) for Codex and other clients. + // This is separate from Image, which only enables /v1/images/* endpoints. + InputModalities []string `yaml:"input-modalities,omitempty" json:"input-modalities,omitempty"` + + // OutputModalities declares supported output modalities when known (e.g. text, image). + OutputModalities []string `yaml:"output-modalities,omitempty" json:"output-modalities,omitempty"` + // Thinking configures the thinking/reasoning capability for this model. // If nil, the model defaults to level-based reasoning with levels ["low", "medium", "high"]. Thinking *registry.ThinkingSupport `yaml:"thinking,omitempty" json:"thinking,omitempty"` diff --git a/sdk/api/handlers/openai/codex_client_models.go b/sdk/api/handlers/openai/codex_client_models.go index 41d8e120..4851376b 100644 --- a/sdk/api/handlers/openai/codex_client_models.go +++ b/sdk/api/handlers/openai/codex_client_models.go @@ -173,6 +173,10 @@ func applyCodexClientModelMetadata(entry map[string]any, id string, model map[st } if info.Type == registry.OpenAIImageModelType { entry["visibility"] = "hide" + delete(entry, "input_modalities") + delete(entry, "supports_image_detail_original") + } else { + applyCodexClientInputModalitiesMetadata(entry, info.SupportedInputModalities) } applyCodexClientThinkingMetadata(entry, info.Thinking) } @@ -213,6 +217,33 @@ func applyCodexClientVisibilityOverride(entry map[string]any, id string) { } } +func applyCodexClientInputModalitiesMetadata(entry map[string]any, modalities []string) { + if len(modalities) == 0 { + return + } + codexModalities := make([]any, 0, len(modalities)) + supportsImage := false + for _, raw := range modalities { + modality := strings.ToLower(strings.TrimSpace(raw)) + if modality == "" { + continue + } + codexModalities = append(codexModalities, modality) + if modality == "image" { + supportsImage = true + } + } + if len(codexModalities) == 0 { + return + } + entry["input_modalities"] = codexModalities + if supportsImage { + entry["supports_image_detail_original"] = true + } else { + delete(entry, "supports_image_detail_original") + } +} + func applyCodexClientThinkingMetadata(entry map[string]any, thinking *registry.ThinkingSupport) { if thinking == nil || len(thinking.Levels) == 0 { return diff --git a/sdk/api/handlers/openai/codex_client_models_test.go b/sdk/api/handlers/openai/codex_client_models_test.go new file mode 100644 index 00000000..4f8fa052 --- /dev/null +++ b/sdk/api/handlers/openai/codex_client_models_test.go @@ -0,0 +1,102 @@ +package openai + +import ( + "testing" + + "github.com/router-for-me/CLIProxyAPI/v7/internal/registry" +) + +func TestCodexClientModelsResponse_InputModalitiesFromRegistry(t *testing.T) { + modelID := "mimo-v2.5-pro-codex-test" + textOnlyModelID := "mimo-text-only-codex-test" + modelRegistry := registry.GetGlobalRegistry() + modelRegistry.RegisterClient("codex-input-modalities-test", "openai-compatibility", []*registry.ModelInfo{ + { + ID: modelID, + Object: "model", + OwnedBy: "mimo", + Type: "openai-compatibility", + DisplayName: modelID, + SupportedInputModalities: []string{"text", "image"}, + }, + { + ID: textOnlyModelID, + Object: "model", + OwnedBy: "mimo", + Type: "openai-compatibility", + DisplayName: textOnlyModelID, + SupportedInputModalities: []string{"text"}, + }, + { + ID: "compat-image-only-codex-test", + Object: "model", + OwnedBy: "mimo", + Type: registry.OpenAIImageModelType, + }, + }) + t.Cleanup(func() { + modelRegistry.UnregisterClient("codex-input-modalities-test") + }) + + openaiModels := modelRegistry.GetAvailableModels("openai") + resp := CodexClientModelsResponse(openaiModels) + models, ok := resp["models"].([]map[string]any) + if !ok { + t.Fatalf("models type = %T, want []map[string]any", resp["models"]) + } + + var visionEntry map[string]any + var textOnlyEntry map[string]any + var imageEntry map[string]any + for _, entry := range models { + slug := stringModelValue(entry, "slug") + switch slug { + case modelID: + visionEntry = entry + case textOnlyModelID: + textOnlyEntry = entry + case "compat-image-only-codex-test": + imageEntry = entry + } + } + if visionEntry == nil { + t.Fatalf("expected codex entry for %q", modelID) + } + modalities, ok := visionEntry["input_modalities"].([]any) + if !ok || len(modalities) != 2 { + t.Fatalf("input_modalities = %#v, want [text image]", visionEntry["input_modalities"]) + } + if got, _ := modalities[0].(string); got != "text" { + t.Fatalf("input_modalities[0] = %q, want text", got) + } + if got, _ := modalities[1].(string); got != "image" { + t.Fatalf("input_modalities[1] = %q, want image", got) + } + if got, ok := visionEntry["supports_image_detail_original"].(bool); !ok || !got { + t.Fatalf("supports_image_detail_original = %#v, want true", visionEntry["supports_image_detail_original"]) + } + + if textOnlyEntry == nil { + t.Fatalf("expected codex entry for %q", textOnlyModelID) + } + textOnlyModalities, ok := textOnlyEntry["input_modalities"].([]any) + if !ok || len(textOnlyModalities) != 1 { + t.Fatalf("text-only input_modalities = %#v, want [text]", textOnlyEntry["input_modalities"]) + } + if got, _ := textOnlyModalities[0].(string); got != "text" { + t.Fatalf("text-only input_modalities[0] = %q, want text", got) + } + if _, exists := textOnlyEntry["supports_image_detail_original"]; exists { + t.Fatalf("text-only model should not expose supports_image_detail_original: %#v", textOnlyEntry["supports_image_detail_original"]) + } + + if imageEntry == nil { + t.Fatal("expected codex entry for image-only compat model") + } + if got, _ := imageEntry["visibility"].(string); got != "hide" { + t.Fatalf("image model visibility = %q, want hide", got) + } + if _, exists := imageEntry["input_modalities"]; exists { + t.Fatalf("image endpoint model should not expose input_modalities from registry: %#v", imageEntry["input_modalities"]) + } +} diff --git a/sdk/cliproxy/openai_compat_config_models_test.go b/sdk/cliproxy/openai_compat_config_models_test.go new file mode 100644 index 00000000..608d8986 --- /dev/null +++ b/sdk/cliproxy/openai_compat_config_models_test.go @@ -0,0 +1,71 @@ +package cliproxy + +import ( + "testing" + + "github.com/router-for-me/CLIProxyAPI/v7/internal/config" + "github.com/router-for-me/CLIProxyAPI/v7/internal/registry" +) + +func TestBuildOpenAICompatibilityConfigModels_InputModalities(t *testing.T) { + compat := &config.OpenAICompatibility{ + Name: "mimo", + Models: []config.OpenAICompatibilityModel{ + { + Name: "upstream-vision", + Alias: "mimo-v2.5-pro", + InputModalities: []string{"TEXT", "image", "image"}, + }, + { + Name: "upstream-image", + Alias: "compat-image", + Image: true, + }, + }, + } + + models := buildOpenAICompatibilityConfigModels(compat) + if len(models) != 2 { + t.Fatalf("model count = %d, want 2", len(models)) + } + + var vision *ModelInfo + var imageModel *ModelInfo + for _, model := range models { + if model == nil { + continue + } + switch model.ID { + case "mimo-v2.5-pro": + vision = model + case "compat-image": + imageModel = model + } + } + if vision == nil { + t.Fatal("expected vision model") + } + if got := joinModalities(vision.SupportedInputModalities); got != "text,image" { + t.Fatalf("SupportedInputModalities = %q, want text,image", got) + } + if imageModel == nil { + t.Fatal("expected image model") + } + if imageModel.Type != registry.OpenAIImageModelType { + t.Fatalf("image model type = %q, want %q", imageModel.Type, registry.OpenAIImageModelType) + } + if len(imageModel.SupportedInputModalities) != 0 { + t.Fatalf("image model input modalities = %+v, want none", imageModel.SupportedInputModalities) + } +} + +func joinModalities(modalities []string) string { + if len(modalities) == 0 { + return "" + } + out := modalities[0] + for i := 1; i < len(modalities); i++ { + out += "," + modalities[i] + } + return out +} diff --git a/sdk/cliproxy/service.go b/sdk/cliproxy/service.go index d1040c96..9fe0afb7 100644 --- a/sdk/cliproxy/service.go +++ b/sdk/cliproxy/service.go @@ -2497,20 +2497,47 @@ func buildOpenAICompatibilityConfigModels(compat *config.OpenAICompatibility) [] if thinking == nil && !model.Image { thinking = ®istry.ThinkingSupport{Levels: []string{"low", "medium", "high"}} } + inputModalities := normalizeCompatConfigModalities(model.InputModalities) + outputModalities := normalizeCompatConfigModalities(model.OutputModalities) models = append(models, &ModelInfo{ - ID: modelID, - Object: "model", - Created: now, - OwnedBy: compat.Name, - Type: modelType, - DisplayName: modelID, - UserDefined: false, - Thinking: thinking, + ID: modelID, + Object: "model", + Created: now, + OwnedBy: compat.Name, + Type: modelType, + DisplayName: modelID, + UserDefined: false, + Thinking: thinking, + SupportedInputModalities: inputModalities, + SupportedOutputModalities: outputModalities, }) } return models } +func normalizeCompatConfigModalities(raw []string) []string { + if len(raw) == 0 { + return nil + } + out := make([]string, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, item := range raw { + modality := strings.ToLower(strings.TrimSpace(item)) + if modality == "" { + continue + } + if _, exists := seen[modality]; exists { + continue + } + seen[modality] = struct{}{} + out = append(out, modality) + } + if len(out) == 0 { + return nil + } + return out +} + func buildConfigModels[T modelEntry](models []T, ownedBy, modelType string) []*ModelInfo { if len(models) == 0 { return nil diff --git a/sdk/cliproxy/service_excluded_models_test.go b/sdk/cliproxy/service_excluded_models_test.go index 96490743..c176d9da 100644 --- a/sdk/cliproxy/service_excluded_models_test.go +++ b/sdk/cliproxy/service_excluded_models_test.go @@ -136,6 +136,82 @@ func TestRegisterModelsForAuth_OpenAICompatibilityImageModelType(t *testing.T) { } } +func TestRegisterModelsForAuth_OpenAICompatibilityInputModalities(t *testing.T) { + service := &Service{ + cfg: &config.Config{ + OpenAICompatibility: []config.OpenAICompatibility{ + { + Name: "mimo", + BaseURL: "https://example.com/v1", + Models: []config.OpenAICompatibilityModel{ + { + Name: "mimo-v2.5-pro", + Alias: "mimo-v2.5-pro", + InputModalities: []string{"text", "image"}, + OutputModalities: []string{"text"}, + }, + {Name: "upstream-image", Alias: "compat-image", Image: true}, + }, + }, + }, + }, + } + auth := &coreauth.Auth{ + ID: "auth-openai-compat-modalities", + Provider: "openai-compatibility", + Status: coreauth.StatusActive, + Attributes: map[string]string{ + "auth_kind": "api_key", + "compat_name": "mimo", + "provider_key": "mimo", + }, + } + + modelRegistry := internalregistry.GetGlobalRegistry() + modelRegistry.UnregisterClient(auth.ID) + t.Cleanup(func() { + modelRegistry.UnregisterClient(auth.ID) + }) + + service.registerModelsForAuth(context.Background(), auth) + + models := modelRegistry.GetModelsForClient(auth.ID) + var visionModel *internalregistry.ModelInfo + var imageEndpointModel *internalregistry.ModelInfo + for _, model := range models { + if model == nil { + continue + } + switch strings.TrimSpace(model.ID) { + case "mimo-v2.5-pro": + visionModel = model + case "compat-image": + imageEndpointModel = model + } + } + if visionModel == nil { + t.Fatal("expected mimo-v2.5-pro to be registered") + } + if visionModel.Type != "openai-compatibility" { + t.Fatalf("vision model type = %q, want openai-compatibility", visionModel.Type) + } + if got := strings.Join(visionModel.SupportedInputModalities, ","); got != "text,image" { + t.Fatalf("SupportedInputModalities = %q, want text,image", got) + } + if got := strings.Join(visionModel.SupportedOutputModalities, ","); got != "text" { + t.Fatalf("SupportedOutputModalities = %q, want text", got) + } + if imageEndpointModel == nil { + t.Fatal("expected compat-image to be registered") + } + if imageEndpointModel.Type != internalregistry.OpenAIImageModelType { + t.Fatalf("image endpoint model type = %q, want %q", imageEndpointModel.Type, internalregistry.OpenAIImageModelType) + } + if len(imageEndpointModel.SupportedInputModalities) != 0 { + t.Fatalf("image endpoint model should not inherit chat input modalities: %+v", imageEndpointModel.SupportedInputModalities) + } +} + func TestRegisterModelsForAuth_AntigravityFetchesWebSearchCapability(t *testing.T) { var sawFetch bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {