feat(openai): add support for input/output modalities in Codex client models

- Added handling of input/output modalities metadata for Codex and compatibility clients.
- Updated model registry and configurations to support "text" and "image" modalities.
- Introduced new tests to validate input/output modalities registration and processing.

Closes: #3976

config.example.yaml

@@ -341,7 +341,9 @@ nonstream-keepalive-interval: 0
 #     models: # The models supported by the provider.
 #       - name: "moonshotai/kimi-k2:free" # The actual model name.
 #         alias: "kimi-k2"               # The alias used in the API.
-#         image: false                   # optional: set true to allow this model on /v1/images/generations and /v1/images/edits
+#         image: false                   # optional: set true to allow this model on /v1/images/generations and /v1/images/edits (not chat/responses image input)
+#         input-modalities: [text, image] # optional: declare /v1/chat/completions and /v1/responses multimodal input for Codex clients
+#         output-modalities: [text]       # optional: declare output modalities when known
 #         thinking:                      # optional: omit to default to levels ["low","medium","high"]
 #           levels: ["low", "medium", "high"]
 #       # You may repeat the same alias to build an internal model pool.

internal/config/config.go

@@ -660,6 +660,13 @@ type OpenAICompatibilityModel struct {
 	// Image marks this model as callable through /v1/images/generations and /v1/images/edits.
 	Image bool `yaml:"image,omitempty" json:"image,omitempty"`
 
+	// InputModalities declares chat/responses input capabilities (e.g. text, image) for Codex and other clients.
+	// This is separate from Image, which only enables /v1/images/* endpoints.
+	InputModalities []string `yaml:"input-modalities,omitempty" json:"input-modalities,omitempty"`
+
+	// OutputModalities declares supported output modalities when known (e.g. text, image).
+	OutputModalities []string `yaml:"output-modalities,omitempty" json:"output-modalities,omitempty"`
+
 	// Thinking configures the thinking/reasoning capability for this model.
 	// If nil, the model defaults to level-based reasoning with levels ["low", "medium", "high"].
 	Thinking *registry.ThinkingSupport `yaml:"thinking,omitempty" json:"thinking,omitempty"`

sdk/api/handlers/openai/codex_client_models.go

@@ -173,6 +173,10 @@ func applyCodexClientModelMetadata(entry map[string]any, id string, model map[st
 		}
 		if info.Type == registry.OpenAIImageModelType {
 			entry["visibility"] = "hide"
+			delete(entry, "input_modalities")
+			delete(entry, "supports_image_detail_original")
+		} else {
+			applyCodexClientInputModalitiesMetadata(entry, info.SupportedInputModalities)
 		}
 		applyCodexClientThinkingMetadata(entry, info.Thinking)
 	}
@@ -213,6 +217,33 @@ func applyCodexClientVisibilityOverride(entry map[string]any, id string) {
 	}
 }
 
+func applyCodexClientInputModalitiesMetadata(entry map[string]any, modalities []string) {
+	if len(modalities) == 0 {
+		return
+	}
+	codexModalities := make([]any, 0, len(modalities))
+	supportsImage := false
+	for _, raw := range modalities {
+		modality := strings.ToLower(strings.TrimSpace(raw))
+		if modality == "" {
+			continue
+		}
+		codexModalities = append(codexModalities, modality)
+		if modality == "image" {
+			supportsImage = true
+		}
+	}
+	if len(codexModalities) == 0 {
+		return
+	}
+	entry["input_modalities"] = codexModalities
+	if supportsImage {
+		entry["supports_image_detail_original"] = true
+	} else {
+		delete(entry, "supports_image_detail_original")
+	}
+}
+
 func applyCodexClientThinkingMetadata(entry map[string]any, thinking *registry.ThinkingSupport) {
 	if thinking == nil || len(thinking.Levels) == 0 {
 		return

sdk/api/handlers/openai/codex_client_models_test.go

@@ -0,0 +1,102 @@
+package openai
+
+import (
+	"testing"
+
+	"github.com/router-for-me/CLIProxyAPI/v7/internal/registry"
+)
+
+func TestCodexClientModelsResponse_InputModalitiesFromRegistry(t *testing.T) {
+	modelID := "mimo-v2.5-pro-codex-test"
+	textOnlyModelID := "mimo-text-only-codex-test"
+	modelRegistry := registry.GetGlobalRegistry()
+	modelRegistry.RegisterClient("codex-input-modalities-test", "openai-compatibility", []*registry.ModelInfo{
+		{
+			ID:                       modelID,
+			Object:                   "model",
+			OwnedBy:                  "mimo",
+			Type:                     "openai-compatibility",
+			DisplayName:              modelID,
+			SupportedInputModalities: []string{"text", "image"},
+		},
+		{
+			ID:                       textOnlyModelID,
+			Object:                   "model",
+			OwnedBy:                  "mimo",
+			Type:                     "openai-compatibility",
+			DisplayName:              textOnlyModelID,
+			SupportedInputModalities: []string{"text"},
+		},
+		{
+			ID:      "compat-image-only-codex-test",
+			Object:  "model",
+			OwnedBy: "mimo",
+			Type:    registry.OpenAIImageModelType,
+		},
+	})
+	t.Cleanup(func() {
+		modelRegistry.UnregisterClient("codex-input-modalities-test")
+	})
+
+	openaiModels := modelRegistry.GetAvailableModels("openai")
+	resp := CodexClientModelsResponse(openaiModels)
+	models, ok := resp["models"].([]map[string]any)
+	if !ok {
+		t.Fatalf("models type = %T, want []map[string]any", resp["models"])
+	}
+
+	var visionEntry map[string]any
+	var textOnlyEntry map[string]any
+	var imageEntry map[string]any
+	for _, entry := range models {
+		slug := stringModelValue(entry, "slug")
+		switch slug {
+		case modelID:
+			visionEntry = entry
+		case textOnlyModelID:
+			textOnlyEntry = entry
+		case "compat-image-only-codex-test":
+			imageEntry = entry
+		}
+	}
+	if visionEntry == nil {
+		t.Fatalf("expected codex entry for %q", modelID)
+	}
+	modalities, ok := visionEntry["input_modalities"].([]any)
+	if !ok || len(modalities) != 2 {
+		t.Fatalf("input_modalities = %#v, want [text image]", visionEntry["input_modalities"])
+	}
+	if got, _ := modalities[0].(string); got != "text" {
+		t.Fatalf("input_modalities[0] = %q, want text", got)
+	}
+	if got, _ := modalities[1].(string); got != "image" {
+		t.Fatalf("input_modalities[1] = %q, want image", got)
+	}
+	if got, ok := visionEntry["supports_image_detail_original"].(bool); !ok || !got {
+		t.Fatalf("supports_image_detail_original = %#v, want true", visionEntry["supports_image_detail_original"])
+	}
+
+	if textOnlyEntry == nil {
+		t.Fatalf("expected codex entry for %q", textOnlyModelID)
+	}
+	textOnlyModalities, ok := textOnlyEntry["input_modalities"].([]any)
+	if !ok || len(textOnlyModalities) != 1 {
+		t.Fatalf("text-only input_modalities = %#v, want [text]", textOnlyEntry["input_modalities"])
+	}
+	if got, _ := textOnlyModalities[0].(string); got != "text" {
+		t.Fatalf("text-only input_modalities[0] = %q, want text", got)
+	}
+	if _, exists := textOnlyEntry["supports_image_detail_original"]; exists {
+		t.Fatalf("text-only model should not expose supports_image_detail_original: %#v", textOnlyEntry["supports_image_detail_original"])
+	}
+
+	if imageEntry == nil {
+		t.Fatal("expected codex entry for image-only compat model")
+	}
+	if got, _ := imageEntry["visibility"].(string); got != "hide" {
+		t.Fatalf("image model visibility = %q, want hide", got)
+	}
+	if _, exists := imageEntry["input_modalities"]; exists {
+		t.Fatalf("image endpoint model should not expose input_modalities from registry: %#v", imageEntry["input_modalities"])
+	}
+}

sdk/cliproxy/openai_compat_config_models_test.go

@@ -0,0 +1,71 @@
+package cliproxy
+
+import (
+	"testing"
+
+	"github.com/router-for-me/CLIProxyAPI/v7/internal/config"
+	"github.com/router-for-me/CLIProxyAPI/v7/internal/registry"
+)
+
+func TestBuildOpenAICompatibilityConfigModels_InputModalities(t *testing.T) {
+	compat := &config.OpenAICompatibility{
+		Name: "mimo",
+		Models: []config.OpenAICompatibilityModel{
+			{
+				Name:            "upstream-vision",
+				Alias:           "mimo-v2.5-pro",
+				InputModalities: []string{"TEXT", "image", "image"},
+			},
+			{
+				Name:  "upstream-image",
+				Alias: "compat-image",
+				Image: true,
+			},
+		},
+	}
+
+	models := buildOpenAICompatibilityConfigModels(compat)
+	if len(models) != 2 {
+		t.Fatalf("model count = %d, want 2", len(models))
+	}
+
+	var vision *ModelInfo
+	var imageModel *ModelInfo
+	for _, model := range models {
+		if model == nil {
+			continue
+		}
+		switch model.ID {
+		case "mimo-v2.5-pro":
+			vision = model
+		case "compat-image":
+			imageModel = model
+		}
+	}
+	if vision == nil {
+		t.Fatal("expected vision model")
+	}
+	if got := joinModalities(vision.SupportedInputModalities); got != "text,image" {
+		t.Fatalf("SupportedInputModalities = %q, want text,image", got)
+	}
+	if imageModel == nil {
+		t.Fatal("expected image model")
+	}
+	if imageModel.Type != registry.OpenAIImageModelType {
+		t.Fatalf("image model type = %q, want %q", imageModel.Type, registry.OpenAIImageModelType)
+	}
+	if len(imageModel.SupportedInputModalities) != 0 {
+		t.Fatalf("image model input modalities = %+v, want none", imageModel.SupportedInputModalities)
+	}
+}
+
+func joinModalities(modalities []string) string {
+	if len(modalities) == 0 {
+		return ""
+	}
+	out := modalities[0]
+	for i := 1; i < len(modalities); i++ {
+		out += "," + modalities[i]
+	}
+	return out
+}

sdk/cliproxy/service.go

@@ -2497,20 +2497,47 @@ func buildOpenAICompatibilityConfigModels(compat *config.OpenAICompatibility) []
 		if thinking == nil && !model.Image {
 			thinking = &registry.ThinkingSupport{Levels: []string{"low", "medium", "high"}}
 		}
+		inputModalities := normalizeCompatConfigModalities(model.InputModalities)
+		outputModalities := normalizeCompatConfigModalities(model.OutputModalities)
 		models = append(models, &ModelInfo{
-			ID:          modelID,
-			Object:      "model",
-			Created:     now,
-			OwnedBy:     compat.Name,
-			Type:        modelType,
-			DisplayName: modelID,
-			UserDefined: false,
-			Thinking:    thinking,
+			ID:                        modelID,
+			Object:                    "model",
+			Created:                   now,
+			OwnedBy:                   compat.Name,
+			Type:                      modelType,
+			DisplayName:               modelID,
+			UserDefined:               false,
+			Thinking:                  thinking,
+			SupportedInputModalities:  inputModalities,
+			SupportedOutputModalities: outputModalities,
 		})
 	}
 	return models
 }
 
+func normalizeCompatConfigModalities(raw []string) []string {
+	if len(raw) == 0 {
+		return nil
+	}
+	out := make([]string, 0, len(raw))
+	seen := make(map[string]struct{}, len(raw))
+	for _, item := range raw {
+		modality := strings.ToLower(strings.TrimSpace(item))
+		if modality == "" {
+			continue
+		}
+		if _, exists := seen[modality]; exists {
+			continue
+		}
+		seen[modality] = struct{}{}
+		out = append(out, modality)
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
 func buildConfigModels[T modelEntry](models []T, ownedBy, modelType string) []*ModelInfo {
 	if len(models) == 0 {
 		return nil

sdk/cliproxy/service_excluded_models_test.go

@@ -136,6 +136,82 @@ func TestRegisterModelsForAuth_OpenAICompatibilityImageModelType(t *testing.T) {
 	}
 }
 
+func TestRegisterModelsForAuth_OpenAICompatibilityInputModalities(t *testing.T) {
+	service := &Service{
+		cfg: &config.Config{
+			OpenAICompatibility: []config.OpenAICompatibility{
+				{
+					Name:    "mimo",
+					BaseURL: "https://example.com/v1",
+					Models: []config.OpenAICompatibilityModel{
+						{
+							Name:             "mimo-v2.5-pro",
+							Alias:            "mimo-v2.5-pro",
+							InputModalities:  []string{"text", "image"},
+							OutputModalities: []string{"text"},
+						},
+						{Name: "upstream-image", Alias: "compat-image", Image: true},
+					},
+				},
+			},
+		},
+	}
+	auth := &coreauth.Auth{
+		ID:       "auth-openai-compat-modalities",
+		Provider: "openai-compatibility",
+		Status:   coreauth.StatusActive,
+		Attributes: map[string]string{
+			"auth_kind":    "api_key",
+			"compat_name":  "mimo",
+			"provider_key": "mimo",
+		},
+	}
+
+	modelRegistry := internalregistry.GetGlobalRegistry()
+	modelRegistry.UnregisterClient(auth.ID)
+	t.Cleanup(func() {
+		modelRegistry.UnregisterClient(auth.ID)
+	})
+
+	service.registerModelsForAuth(context.Background(), auth)
+
+	models := modelRegistry.GetModelsForClient(auth.ID)
+	var visionModel *internalregistry.ModelInfo
+	var imageEndpointModel *internalregistry.ModelInfo
+	for _, model := range models {
+		if model == nil {
+			continue
+		}
+		switch strings.TrimSpace(model.ID) {
+		case "mimo-v2.5-pro":
+			visionModel = model
+		case "compat-image":
+			imageEndpointModel = model
+		}
+	}
+	if visionModel == nil {
+		t.Fatal("expected mimo-v2.5-pro to be registered")
+	}
+	if visionModel.Type != "openai-compatibility" {
+		t.Fatalf("vision model type = %q, want openai-compatibility", visionModel.Type)
+	}
+	if got := strings.Join(visionModel.SupportedInputModalities, ","); got != "text,image" {
+		t.Fatalf("SupportedInputModalities = %q, want text,image", got)
+	}
+	if got := strings.Join(visionModel.SupportedOutputModalities, ","); got != "text" {
+		t.Fatalf("SupportedOutputModalities = %q, want text", got)
+	}
+	if imageEndpointModel == nil {
+		t.Fatal("expected compat-image to be registered")
+	}
+	if imageEndpointModel.Type != internalregistry.OpenAIImageModelType {
+		t.Fatalf("image endpoint model type = %q, want %q", imageEndpointModel.Type, internalregistry.OpenAIImageModelType)
+	}
+	if len(imageEndpointModel.SupportedInputModalities) != 0 {
+		t.Fatalf("image endpoint model should not inherit chat input modalities: %+v", imageEndpointModel.SupportedInputModalities)
+	}
+}
+
 func TestRegisterModelsForAuth_AntigravityFetchesWebSearchCapability(t *testing.T) {
 	var sawFetch bool
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {