mirror of
https://gh-proxy.org/https://github.com/klinsmaya/new-api.git
synced 2026-11-10 02:17:24 +08:00
feitianbubuandCaIon
afe73b08ff
* feat: add ali MultimodalEmbedding qwen3-vl-embedding * fix: report image/video tokens in qwen3-vl-embedding usage for billing * fix(ali): serve every Bailian multimodal embedding model natively Route qwen2.5-vl-embedding, the tongyi-embedding-vision series and multimodal-embedding-v1, dated snapshots included, to the native multimodal-embedding endpoint like qwen3-vl-embedding. Text embedding models stay in compatible mode. Take tongyi-embedding-vision's text and image tokens from input_tokens_details so the image ratio applies to its image tokens, and set total_tokens to the billed input tokens: tongyi counts its unpriced output_tokens in total_tokens. A response without usage bills the request estimate, as compatible mode does. Return an error code carried in a 200 body as a 500, close the response body when reading it fails, and fold the input conversion into one function. Count the text, image and video parts of embedding input in the request estimate the way chat content is counted, so image calls are pre-charged and a response without usage is not free. With the default settings no image is downloaded for the estimate. * fix(gemini): reject embedding input that is not plain text Gemini embeds text only, so the converter kept the strings and dropped image, video and text parts. The embedding request estimate now counts those parts, and Gemini embeddings bill that estimate, so a dropped part would be charged without ever reaching Gemini. A part now fails the request with a 400 that skips retries, before anything is sent or charged. --------- Co-authored-by: CaIon <[email protected]>
110 lines
3.1 KiB
Go
110 lines
3.1 KiB
Go
package dto
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
|
|
kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
|
|
"github.com/QuantumNous/new-api/relaykit/types"
|
|
)
|
|
|
|
type EmbeddingOptions struct {
|
|
Seed int `json:"seed,omitempty"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopK int `json:"top_k,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"`
|
|
PresencePenalty *float64 `json:"presence_penalty,omitempty"`
|
|
NumPredict int `json:"num_predict,omitempty"`
|
|
NumCtx int `json:"num_ctx,omitempty"`
|
|
}
|
|
|
|
type EmbeddingRequest struct {
|
|
Model string `json:"model"`
|
|
Input any `json:"input"`
|
|
EncodingFormat string `json:"encoding_format,omitempty"`
|
|
Dimensions *int `json:"dimensions,omitempty"`
|
|
User string `json:"user,omitempty"`
|
|
Seed *float64 `json:"seed,omitempty"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"`
|
|
PresencePenalty *float64 `json:"presence_penalty,omitempty"`
|
|
}
|
|
|
|
func (r *EmbeddingRequest) GetTokenCountMeta() *types.TokenCountMeta {
|
|
texts := r.ParseInput()
|
|
var files []*types.FileMeta
|
|
// Chat-style parts count like chat message content: text parts as text,
|
|
// image and video parts as files.
|
|
items, _ := r.Input.([]any)
|
|
for _, item := range items {
|
|
if _, ok := item.(map[string]any); !ok {
|
|
continue
|
|
}
|
|
part, err := kitutil.Any2Type[MediaContent](item)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch part.Type {
|
|
case ContentTypeText:
|
|
texts = append(texts, part.Text)
|
|
case ContentTypeImageURL:
|
|
if source := part.ToFileSource(); source != nil {
|
|
files = append(files, &types.FileMeta{FileType: types.FileTypeImage, Source: source, Detail: part.GetImageMedia().Detail})
|
|
}
|
|
case ContentTypeVideoUrl:
|
|
if source := part.ToFileSource(); source != nil {
|
|
files = append(files, &types.FileMeta{FileType: types.FileTypeVideo, Source: source})
|
|
}
|
|
}
|
|
}
|
|
|
|
return &types.TokenCountMeta{
|
|
CombineText: strings.Join(texts, "\n"),
|
|
Files: files,
|
|
}
|
|
}
|
|
|
|
func (r *EmbeddingRequest) IsStream(c *http.Request) bool {
|
|
return false
|
|
}
|
|
|
|
func (r *EmbeddingRequest) SetModelName(modelName string) {
|
|
if modelName != "" {
|
|
r.Model = modelName
|
|
}
|
|
}
|
|
|
|
func (r *EmbeddingRequest) ParseInput() []string {
|
|
if r.Input == nil {
|
|
return make([]string, 0)
|
|
}
|
|
var input []string
|
|
switch r.Input.(type) {
|
|
case string:
|
|
input = []string{r.Input.(string)}
|
|
case []any:
|
|
input = make([]string, 0, len(r.Input.([]any)))
|
|
for _, item := range r.Input.([]any) {
|
|
if str, ok := item.(string); ok {
|
|
input = append(input, str)
|
|
}
|
|
}
|
|
}
|
|
return input
|
|
}
|
|
|
|
type EmbeddingResponseItem struct {
|
|
Object string `json:"object"`
|
|
Index int `json:"index"`
|
|
Embedding []float64 `json:"embedding"`
|
|
}
|
|
|
|
type EmbeddingResponse struct {
|
|
Object string `json:"object"`
|
|
Data []EmbeddingResponseItem `json:"data"`
|
|
Model string `json:"model"`
|
|
Usage `json:"usage"`
|
|
}
|