diff --git a/docs/plugin-api/v1.d.ts b/docs/plugin-api/v1.d.ts index 3910785b3..7a4a27fde 100644 --- a/docs/plugin-api/v1.d.ts +++ b/docs/plugin-api/v1.d.ts @@ -86,6 +86,7 @@ export interface SubmitEventDeltaResult {changes: readonly JSONChange[]; state: export declare function parseSubmitEventDelta(ctx: DriverContext, event: SubmitEvent, previousState: JSONValue | null): SubmitEventDeltaResult; export declare function parseSubmitEvent(ctx: DriverContext, event: SubmitEvent, previousState: JSONValue | null): {state: JSONValue; done: boolean}; export declare function parseSubmitResponse(ctx: DriverContext, response: UpstreamResponse): {taskId: string; taskData?: unknown; immediate?: NormalizedTaskResult; state?: unknown}; +/** Query redirects retain credentials only within the channel host and meta.allowedHosts; SSRF checks and HTTPS downgrade protection still apply. */ export declare function buildQueryRequest(ctx: TaskQueryContext): RequestDescriptor; export declare function buildBatchQueryRequest(ctx: BatchQueryContext, tasks: readonly TaskQueryContext[]): RequestDescriptor; export declare function parseTaskResult(ctx: TaskQueryContext, body: unknown, response: HookHTTPResponse): NormalizedTaskResult; diff --git a/docs/plugin-api/v1.md b/docs/plugin-api/v1.md index 60c8886c2..41d489189 100644 --- a/docs/plugin-api/v1.md +++ b/docs/plugin-api/v1.md @@ -42,7 +42,7 @@ A plugin whose vendor has no LobeHub icon ships its logo as a sidecar file, `ico `baseUrl` is an optional default upstream address for channels of type 61 ("Task Plugin"). When an administrator binds the plugin and leaves the channel Base URL empty, the host copies this value onto the channel before validation and records `base_url_source: plugin_default` in the channel audit event; the persisted channel value is what every later request uses, so a later plugin update that changes `baseUrl` does not move existing channels. The value must be an absolute `http` or `https` URL without credentials, query, or fragment; the host is lowercased and must be ASCII (use punycode), trailing slashes are stripped, and the normalized value must not exceed 191 characters (the width of the channel `base_url` column on MySQL). Private, loopback, and plain `http` addresses are allowed for self-hosted upstreams; the admin UI flags them before binding. `baseUrl` is ignored on legacy `channelTypes` channels, which keep the built-in per-type default. The default host is not implicitly added to `allowedHosts`: if the administrator points the channel at a different host, plugin requests to the author's default host are rejected. -`allowedHosts` lists extra hosts that plugin requests may target besides the channel base URL host. Each entry is `host` or `host:port` (IPv6 literals bracketed, for example `[::1]:8080`); schemes, paths, credentials, and queries are rejected. Entries are lowercased and must be unique after normalization. An entry with a port matches only requests to that port (default ports `80`/`443` match requests that omit them). +`allowedHosts` lists extra hosts that plugin requests may target besides the channel base URL host. These are trusted credential recipients for both direct requests and task query redirects; add only hosts permitted to receive the channel credentials. Each entry is `host` or `host:port` (IPv6 literals bracketed, for example `[::1]:8080`); schemes, paths, credentials, and queries are rejected. Entries are lowercased and must be unique after normalization. An entry with a port matches only requests to that port (default ports `80`/`443` match requests that omit them). `upstreams` lists the kinds of upstream the driver's request builders can address. `vendor` is the provider API itself and is implied for every plugin; `new_api` is another New API gateway that has the same plugin installed. Only a plugin that declares `new_api` can be bound to a type-60 New API channel: the host rejects other bindings when the channel is saved, and the channel form lists only declaring plugins. A gateway does not serve vendor paths at their vendor location. It serves a plugin's wire format only on that plugin's own native routes, which carry the plugin's prefix because `/api` is a reserved host namespace and vendor paths would collide across plugins (`/doubao/api/v3/...`, `/ali/api/v1/...`, `/kling/v1/videos/...`), and it authenticates native routes and host protocols with `Authorization: Bearer `. The host therefore tells the driver which kind of upstream the executing channel is through `ctx.upstream` (see Persisted data and driver hooks) instead of guessing a prefix from `meta.routes`; a plugin whose vendor paths already equal its native routes (sunoapi) or that only uses host protocols (sora) declares `new_api` without changing any URL. Values must be unique and drawn from the two kinds above; listing `vendor` is optional and descriptive. This is an additive apiVersion 1 extension: older hosts reject the unknown field, and a declaring plugin keeps building vendor URLs on hosts that never set `ctx.upstream`. @@ -491,6 +491,24 @@ Query and parse hooks use `TaskQueryContext`, not `DriverContext`. The host rebu `buildBatchQueryRequest(ctx, tasks)` and `parseBatchResult` receive `tasks: TaskQueryContext[]`. `parseTaskResult` / `parseBatchResult` also receive `{status, headers}` for the upstream HTTP response. +Single and batch task query redirects use the same credential boundary as direct +plugin requests: the channel base URL host and `meta.allowedHosts`, with the +existing exact host and port matching. An omitted `allowedHosts` grants no extra +hosts. Plugins that need another credential recipient add it to this existing +list; no redirect option or query descriptor field is needed. + +Redirects outside that boundary continue without the original headers except +`Accept` and `Accept-Language`; `Referer` and URL userinfo are removed as well. +After any untrusted hop, credentials stay removed even if a later hop returns +to a trusted host. Arbitrary plugin and channel credential headers are covered. +307/308 may replay a query body only along a trusted chain; other body replays +fail that poll and use the existing poll failure handling. + +Every redirect still passes the existing SSRF checks and the limit of 10 +requests in a redirect chain. HTTPS-to-HTTP redirects are rejected even to an +allowed host. HTTPS upgrades within the trusted hosts are allowed. +`utils.fetch` keeps returning redirects to the plugin. + `status: "UNKNOWN"` means the plugin does not recognize the response. Do not write `|| "IN_PROGRESS"` (or equivalent) for a missing table entry. The host treats `UNKNOWN`, hook errors, empty status, and unrecognized status strings as consecutive poll failures. The host classifies the HTTP status before trusting a non-terminal parse: diff --git a/relay/channel/task/jsplugin/adaptor.go b/relay/channel/task/jsplugin/adaptor.go index 6e66cd9b7..c298a6490 100644 --- a/relay/channel/task/jsplugin/adaptor.go +++ b/relay/channel/task/jsplugin/adaptor.go @@ -753,8 +753,69 @@ func (a *TaskAdaptor) doFetchDescriptor(baseURL, key, proxy string, descriptor r if err != nil { return nil, err } + // Clients are cached across channels. Keep the redirect policy and its + // credential state local to this query, while reusing the transport. + queryClient := *client + credentialsStripped := false + var redirectPolicyErr error + queryClient.CheckRedirect = func(next *http.Request, via []*http.Request) error { + if strings.EqualFold(via[len(via)-1].URL.Scheme, "https") && !strings.EqualFold(next.URL.Scheme, "https") { + redirectPolicyErr = fmt.Errorf("task query redirect cannot downgrade HTTPS") + return redirectPolicyErr + } + // Redirects use the same credential recipients as direct plugin + // requests: the channel base host and the plugin's allowedHosts. + if pluginruntime.ValidateRequestURL(next.URL.String(), baseURL, a.plugin.Meta.AllowedHosts) != nil { + credentialsStripped = true + } + if credentialsStripped { + // A 307/308 can replay credentials or private task data in the + // body even after the credential headers have been removed. + if next.Body != nil && next.Body != http.NoBody { + redirectPolicyErr = fmt.Errorf("task query redirect cannot replay a body to an origin without credential authorization") + return redirectPolicyErr + } + // Plugin and channel headers can use arbitrary credential names. + // Keep only representation preferences, and do this on every + // later hop: net/http copies headers from the initial request. + for name := range next.Header { + switch http.CanonicalHeaderKey(name) { + case "Accept", "Accept-Language": + default: + delete(next.Header, name) + } + } + next.Host = "" + next.URL.User = nil + } else { + // net/http may have already removed standard credentials for a + // different domain. Restore them only along an authorized chain; + // crossing an untrusted origin can never restore credentials later. + for _, name := range []string{"Authorization", "Www-Authenticate", "Cookie", "Cookie2", "Proxy-Authorization", "Proxy-Authenticate"} { + if values, ok := req.Header[name]; ok { + next.Header[name] = slices.Clone(values) + } + } + } + if client.CheckRedirect != nil { + if err := client.CheckRedirect(next, via); err != nil { + if err == http.ErrUseLastResponse { + return err + } + // The shared SSRF policy includes the target URL in errors. + // Keep its diagnostic reason without exposing signed URLs. + redirectPolicyErr = errors.New(strings.ReplaceAll(err.Error(), next.URL.String(), next.URL.Scheme+"://"+next.URL.Host)) + return redirectPolicyErr + } + return nil + } + if len(via) >= 10 { + return fmt.Errorf("stopped after 10 redirects") + } + return service.ValidateSSRFProtectedFetchURL(next.URL.String()) + } started := time.Now() - resp, err := client.Do(req) + resp, err := queryClient.Do(req) if err != nil { logger.LogDebug( context.Background(), @@ -763,6 +824,11 @@ func (a *TaskAdaptor) doFetchDescriptor(baseURL, key, proxy string, descriptor r method, time.Since(started).Milliseconds(), ) + if redirectPolicyErr != nil { + // net/http wraps redirect errors with the Location URL, which + // may contain credentials. Polling logs only the policy reason. + return nil, redirectPolicyErr + } return nil, err } logger.LogDebug( diff --git a/relay/channel/task/jsplugin/adaptor_test.go b/relay/channel/task/jsplugin/adaptor_test.go index 1580d7ce3..86cc63ac8 100644 --- a/relay/channel/task/jsplugin/adaptor_test.go +++ b/relay/channel/task/jsplugin/adaptor_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "io" "math" "mime" @@ -27,6 +28,7 @@ import ( "github.com/QuantumNous/new-api/relay/helper" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/service" + "github.com/QuantumNous/new-api/setting/system_setting" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -651,6 +653,238 @@ export function buildContentRequest(ctx) { assert.ErrorContains(t, err, "utils.fetch is not available in protocols.openai_video.render") } +type taskQueryRoundTripper func(*http.Request) (*http.Response, error) + +func (f taskQueryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestTaskQueryRedirects(t *testing.T) { + service.InitHttpClient() + client := service.GetHttpClient() + originalClient := *client + fetchSetting := system_setting.GetFetchSetting() + originalSetting := *fetchSetting + t.Cleanup(func() { + *client = originalClient + *fetchSetting = originalSetting + }) + *fetchSetting = system_setting.FetchSetting{ + EnableSSRFProtection: true, + DomainList: []string{"blocked.example"}, + AllowedPorts: []string{"1-65535"}, + } + const source = ` +export const meta = {apiVersion:1,key:"redirect",name:"Redirect",version:"1.0.0",author:{name:"Test"},models:["m"],fetchMode:"batch"}; +export function buildSubmitRequest(ctx) {return {url:ctx.baseUrl+"/submit"};} +export function parseSubmitResponse() {return {taskId:"task"};} +export function buildQueryRequest(ctx) {return ctx.state;} +export function parseTaskResult() {return {status:"SUCCESS"};} +export function buildBatchQueryRequest(ctx,tasks) {return tasks[0].state;} +export function parseBatchResult() {return [];} +` + defaultPlugin, err := pluginruntime.NewRegistry().Register(source, pluginruntime.Options{}) + require.NoError(t, err) + type redirectCase struct { + name string + start string + locations []string + allowedHosts []string + status int + body string + batch bool + ordered bool + loop bool + wantRequests int + credentialHops int + wantError string + } + cases := []redirectCase{ + {name: "host case and explicit default port", locations: []string{"https://API.example:443/final"}, wantRequests: 2, credentialHops: 2}, + {name: "port change is cross origin", locations: []string{"https://api.example:8443/final"}, wantRequests: 2, credentialHops: 1}, + {name: "subdomain is cross origin", locations: []string{"https://sub.api.example/final"}, wantRequests: 2, credentialHops: 1}, + {name: "HTTPS upgrade keeps the configured host", start: "http://api.example/query", locations: []string{"https://api.example/final"}, wantRequests: 2, credentialHops: 2}, + {name: "authorized cross origin", locations: []string{"https://cdn.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 2, credentialHops: 2}, + {name: "allowedHosts does not authorize unlisted hosts", locations: []string{"https://other.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 2, credentialHops: 1}, + {name: "explicit allowed port", locations: []string{"https://api.example:8443/final"}, allowedHosts: []string{"api.example:8443"}, wantRequests: 2, credentialHops: 2}, + {name: "authorized origin does not authorize its subdomain", locations: []string{"https://sub.cdn.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 2, credentialHops: 1}, + {name: "authorized origin does not authorize other ports", locations: []string{"https://cdn.example:8443/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 2, credentialHops: 1}, + {name: "untrusted chain never restores credentials", locations: []string{"https://cdn.example/next", "/more", "https://api.example/final"}, wantRequests: 4, credentialHops: 1}, + {name: "trust after an untrusted hop cannot restore credentials", locations: []string{"https://other.example/next", "https://cdn.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 3, credentialHops: 1}, + {name: "authorized chain can return to origin", locations: []string{"https://cdn.example/next", "https://api.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 3, credentialHops: 3}, + {name: "redirect URL credentials are removed", locations: []string{"https://username:location-secret@cdn.example/final"}, wantRequests: 2, credentialHops: 1}, + {name: "SSRF still rejects an authorized origin", locations: []string{"https://blocked.example/final?key=location-secret"}, allowedHosts: []string{"blocked.example"}, wantRequests: 1, credentialHops: 1, wantError: "blocked"}, + {name: "SSRF checks later hops", locations: []string{"https://cdn.example/next", "https://blocked.example/final"}, wantRequests: 2, credentialHops: 1, wantError: "blocked"}, + {name: "HTTPS downgrade is rejected even for the channel host", locations: []string{"http://api.example/final?key=location-secret"}, wantRequests: 1, credentialHops: 1, wantError: "cannot downgrade HTTPS"}, + {name: "redirect limit remains enforced", loop: true, wantRequests: 10, credentialHops: 10, wantError: "stopped after 10 redirects"}, + } + for _, status := range []int{301, 302, 303, 307, 308} { + cases = append(cases, + redirectCase{name: fmt.Sprintf("same origin %d", status), status: status, locations: []string{"/final"}, wantRequests: 2, credentialHops: 2}, + redirectCase{name: fmt.Sprintf("cross origin %d", status), status: status, locations: []string{"https://cdn.example/final?signature=download-token"}, wantRequests: 2, credentialHops: 1}, + ) + } + for _, status := range []int{307, 308} { + cases = append(cases, + redirectCase{name: fmt.Sprintf("batch same origin body %d", status), status: status, body: `{"ids":["task"]}`, batch: true, locations: []string{"/final"}, wantRequests: 2, credentialHops: 2}, + redirectCase{name: fmt.Sprintf("batch untrusted body %d", status), status: status, body: `{"ids":["task"]}`, batch: true, locations: []string{"https://cdn.example/final?key=location-secret"}, wantRequests: 1, credentialHops: 1, wantError: "cannot replay a body"}, + redirectCase{name: fmt.Sprintf("batch authorized body %d", status), status: status, body: `{"ids":["task"]}`, batch: true, ordered: true, locations: []string{"https://cdn.example/final"}, allowedHosts: []string{"cdn.example"}, wantRequests: 2, credentialHops: 2}, + ) + } + cases = append(cases, redirectCase{name: "POST to GET across origins", status: 303, body: `{"ids":["task"]}`, batch: true, locations: []string{"https://cdn.example/final"}, wantRequests: 2, credentialHops: 1}) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.start == "" { + tc.start = "https://api.example/query?key=query-secret" + } + if tc.status == 0 { + tc.status = http.StatusFound + } + descriptor := requestDescriptor{URL: tc.start, Method: http.MethodGet, Headers: map[string]string{ + "Authorization": "Bearer channel-secret", "Cookie": "session=channel-secret", + "X-API-Key": "channel-secret", "x-goog-api-key": "channel-secret", "X-Vendor-Credential": "channel-secret", + "Accept": "application/json", "Accept-Language": "en", "Content-Type": "application/json", + }} + if tc.body != "" { + descriptor.Method, descriptor.Body = http.MethodPost, tc.body + } + state, err := common.Marshal(descriptor) + require.NoError(t, err) + task := &model.Task{TaskID: "task", PrivateData: model.TaskPrivateData{PluginState: state}} + type sentRequest struct { + request *http.Request + body string + } + var sent []sentRequest + // Exercise net/http's real redirect machinery without external DNS + // or TLS servers; the transport records exactly what it would send. + client.Transport = taskQueryRoundTripper(func(req *http.Request) (*http.Response, error) { + var body []byte + if req.Body != nil { + var readErr error + body, readErr = io.ReadAll(req.Body) + require.NoError(t, readErr) + require.NoError(t, req.Body.Close()) + } + sent = append(sent, sentRequest{request: req.Clone(req.Context()), body: string(body)}) + resp := &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"status":"SUCCESS"}`)), Request: req} + if tc.loop { + resp.StatusCode = tc.status + resp.Header.Set("Location", "/query") + } else if len(sent) <= len(tc.locations) { + resp.StatusCode = tc.status + resp.Header.Set("Location", tc.locations[len(sent)-1]) + } + return resp, nil + }) + pluginSource := source + if len(tc.allowedHosts) > 0 { + hosts, marshalErr := common.Marshal(tc.allowedHosts) + require.NoError(t, marshalErr) + pluginSource = strings.Replace(pluginSource, `fetchMode:"batch"`, `fetchMode:"batch",allowedHosts:`+string(hosts), 1) + } + if tc.ordered { + pluginSource = strings.Replace(pluginSource, `fetchMode:"batch"`, `fetchMode:"batch",requiredCapabilities:["json-order@1"]`, 1) + } + plugin, registerErr := pluginruntime.NewRegistry().Register(pluginSource, pluginruntime.Options{}) + require.NoError(t, registerErr) + adaptor := New(plugin) + var resp *http.Response + if tc.batch { + resp, err = adaptor.FetchBatchTasks(tc.start, "channel-secret", []*model.Task{task}, "") + } else { + resp, err = adaptor.FetchTask(tc.start, "channel-secret", task, "") + } + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + assert.NotContains(t, err.Error(), "location-secret") + } else { + require.NoError(t, err) + require.NotNil(t, resp) + assert.Equal(t, http.StatusOK, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + } + require.Len(t, sent, tc.wantRequests) + for i, hop := range sent { + for _, name := range []string{"Authorization", "Cookie", "X-API-Key", "X-Goog-Api-Key", "X-Vendor-Credential"} { + if i < tc.credentialHops { + assert.Contains(t, hop.request.Header.Get(name), "channel-secret", "hop %d, header %s", i, name) + } else { + assert.Empty(t, hop.request.Header.Get(name), "hop %d, header %s", i, name) + } + } + if i >= tc.credentialHops { + assert.Empty(t, hop.request.Header.Get("Referer"), "hop %d", i) + assert.Nil(t, hop.request.URL.User, "hop %d", i) + } + assert.Equal(t, "application/json", hop.request.Header.Get("Accept")) + assert.Equal(t, "en", hop.request.Header.Get("Accept-Language")) + if i == 0 || tc.status == 307 || tc.status == 308 { + assert.Equal(t, tc.body, hop.body) + assert.Equal(t, descriptor.Method, hop.request.Method) + } else { + assert.Empty(t, hop.body) + assert.Equal(t, http.MethodGet, hop.request.Method) + } + } + if tc.wantError == "" && strings.Contains(tc.locations[len(tc.locations)-1], "signature=") { + assert.Equal(t, "download-token", sent[len(sent)-1].request.URL.Query().Get("signature")) + } + }) + } + + t.Run("query policy does not change the shared client", func(t *testing.T) { + var headers []http.Header + client.Transport = taskQueryRoundTripper(func(req *http.Request) (*http.Response, error) { + headers = append(headers, req.Header.Clone()) + resp := &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody, Request: req} + if len(headers) == 1 { + resp.StatusCode = http.StatusFound + resp.Header.Set("Location", "https://cdn.example/final") + } + return resp, nil + }) + req, err := http.NewRequest(http.MethodGet, "https://api.example/query", nil) + require.NoError(t, err) + req.Header.Set("X-API-Key", "shared-client-value") + resp, err := client.Do(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Len(t, headers, 2) + assert.Equal(t, "shared-client-value", headers[1].Get("X-API-Key")) + }) + + t.Run("real HTTP receivers do not receive cross origin credentials", func(t *testing.T) { + client.Transport = originalClient.Transport + fetchSetting.AllowPrivateIp = true + received := make(chan http.Header, 2) + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + received <- r.Header.Clone() + _, _ = w.Write([]byte(`{"status":"SUCCESS"}`)) + })) + defer target.Close() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + received <- r.Header.Clone() + http.Redirect(w, r, target.URL+"/final", http.StatusFound) + })) + defer origin.Close() + resp, err := New(defaultPlugin).doFetchDescriptor(origin.URL, "", "", requestDescriptor{ + URL: origin.URL + "/query?key=query-secret", + Headers: map[string]string{"X-API-Key": "channel-secret", "x-goog-api-key": "channel-secret", "X-Vendor-Credential": "channel-secret"}, + }) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + assert.JSONEq(t, `{"status":"SUCCESS"}`, string(body)) + initialHeaders, redirectedHeaders := <-received, <-received + assert.Equal(t, "channel-secret", initialHeaders.Get("X-API-Key")) + for _, name := range []string{"X-API-Key", "X-Goog-Api-Key", "X-Vendor-Credential", "Referer"} { + assert.Empty(t, redirectedHeaders.Get(name), name) + } + }) +} + func TestTaskAdaptorMapsJSContract(t *testing.T) { gin.SetMode(gin.TestMode) service.InitHttpClient()