mirror of
https://gh-proxy.org/https://github.com/klinsmaya/multica.git
synced 2026-11-08 01:16:45 +08:00
Bohan Jiang
c923ace620
Agents on the built-in omp (Oh-My-Pi) runtime could not set a reasoning effort at all. omp dispatches to the pi backend but was missing from providerThinkingEnums, so the server rejected every value, and parseOmpModels discarded omp's per-model effort catalog, leaving the picker hidden and the daemon's guard nothing to validate against. `--thinking` is blocked in custom_args by design, so the only workaround was folding the level into the model string (`devin/swe-2:max`). What a catalog entry means is taken from can1357/oh-my-pi v18.2.0 rather than inferred from pi, because two of pi's rules are inverted for omp: - `models --json` emits `thinking` as an effort array or `null`, never an object. - omp's Effort vocabulary excludes `off`, yet resolveThinkingLevelForModel returns `off` before clamping runs, so it is honoured for every reasoning model. - `reasoning: true` with `thinking: null` means "reasons, no controllable dial"; inferring levels there would offer efforts omp then clamps away. So a reasoning model advertises `off` plus exactly the efforts the catalog listed, a non-reasoning model gets no picker, and nothing is inferred. `auto` is deliberately not exposed: omp keeps it as a session-level sentinel that is never an Effort or ThinkingLevel. An effort with no pinned model is refused rather than stored, because omp resolves its own default role model at task time and clamps the level to what that model supports — so a stored level would save cleanly, read back as set, and then run at a different level. The refusal holds in all three layers: the inspector hides the picker, CreateAgent/UpdateAgent return 400 (Update against the state the request lands on, so clearing the model while an effort is carried over is caught too), and ValidateThinkingLevelWith rejects before any catalog load, so a discovery error cannot reach the daemon's fail-open path. codex shares that execution constraint but stays out of the API-level refusal via a separate narrower predicate: agents already hold an effort with an empty model, and 400ing that pair would block unrelated edits to them. omp has no such history. Closes #8458
1143 lines
44 KiB
Go
1143 lines
44 KiB
Go
package handler
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// TestCreateAgent_ThinkingLevel_ValidationConsistency exercises the
|
|
// MUL-2339 invariant: when an HTTP caller sends a literal-invalid
|
|
// thinking_level the API MUST return 400, regardless of which other
|
|
// field combination the same request mutates. The constraint comes
|
|
// from Trump's PR1 review: "invalid value 的 API 行为请保持一致，
|
|
// 不要同一类变更有时 400、有时静默清空".
|
|
func TestCreateAgent_ThinkingLevel_ValidationConsistency(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx,
|
|
`DELETE FROM agent WHERE workspace_id = $1 AND name LIKE 'thinking-test-%'`,
|
|
testWorkspaceID,
|
|
)
|
|
})
|
|
|
|
t.Run("empty value succeeds", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "thinking-test-empty",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("empty thinking_level: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("known claude value succeeds", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "thinking-test-known",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "high",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("thinking_level=high: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "high" {
|
|
t.Errorf("expected thinking_level=high in response, got %v", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
t.Run("codex-only token rejected for claude runtime", func(t *testing.T) {
|
|
// `none` is a valid Codex token but NOT a Claude token. The
|
|
// gate must always 400 regardless of which other fields the
|
|
// request also tried to change.
|
|
body := map[string]any{
|
|
"name": "thinking-test-codex-only",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "none",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("codex-only thinking_level on claude runtime: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("garbage value rejected", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "thinking-test-garbage",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "supersonic",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("garbage thinking_level: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestCreateAgent_PiThinkingLevelValidation(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
piRuntimeID := createPiProviderRuntime(t)
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx,
|
|
`DELETE FROM agent WHERE workspace_id = $1 AND name LIKE 'pi-thinking-%'`,
|
|
testWorkspaceID,
|
|
)
|
|
})
|
|
|
|
t.Run("native max value succeeds", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "pi-thinking-max",
|
|
"runtime_id": piRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "max",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("Pi thinking_level=max: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "max" {
|
|
t.Fatalf("Pi thinking_level response = %v, want max", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
t.Run("non-Pi token is rejected", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "pi-thinking-ultra",
|
|
"runtime_id": piRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "ultra",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("Pi thinking_level=ultra: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAgentServiceTierValidationAndTriState(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
codexRuntimeID := createCodexProviderRuntime(t)
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
|
|
t.Run("create persists Codex catalog id", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "service-tier-create",
|
|
"runtime_id": codexRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"model": "gpt-5.6-sol",
|
|
"service_tier": "priority",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("create service_tier: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["service_tier"] != "priority" {
|
|
t.Fatalf("service_tier response = %v, want priority", resp["service_tier"])
|
|
}
|
|
agentID, _ := resp["id"].(string)
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx, `DELETE FROM agent WHERE id = $1`, agentID)
|
|
})
|
|
|
|
clear := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, map[string]any{
|
|
"service_tier": "",
|
|
}), "id", agentID)
|
|
testHandler.UpdateAgent(clear, req)
|
|
if clear.Code != http.StatusOK {
|
|
t.Fatalf("clear service_tier: expected 200, got %d: %s", clear.Code, clear.Body.String())
|
|
}
|
|
var cleared map[string]any
|
|
_ = json.NewDecoder(clear.Body).Decode(&cleared)
|
|
if cleared["service_tier"] != "" {
|
|
t.Fatalf("cleared service_tier = %v, want empty", cleared["service_tier"])
|
|
}
|
|
})
|
|
|
|
t.Run("non-Codex runtime rejects tier", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "service-tier-rejected",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"service_tier": "priority",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("Claude service_tier: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestUpdateAgent_ThinkingLevel_TriState covers the three modes of
|
|
// the field on PATCH:
|
|
// - field omitted → leave the existing value alone (the silent-clear
|
|
// anti-pattern flagged by Trump's review must NOT happen here)
|
|
// - explicit "" → clear back to NULL
|
|
// - non-empty → validate against the CURRENT runtime's provider enum
|
|
//
|
|
// All three branches share the same 400 / 200 outcome rule: validation
|
|
// failures are always 400, never auto-clear.
|
|
func TestUpdateAgent_ThinkingLevel_TriState(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
agentID := createAgentOnRuntime(t, "thinking-update-test", claudeRuntimeID, "high")
|
|
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx, `DELETE FROM agent WHERE id = $1`, agentID)
|
|
})
|
|
|
|
// 1. Omitted field — name-only update must NOT touch thinking_level.
|
|
t.Run("omitted field leaves value alone", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "thinking-update-test-renamed",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("name-only update: expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "high" {
|
|
t.Errorf("name-only update silently changed thinking_level: got %v, want high", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
// 2. Explicit "" — must clear.
|
|
t.Run("empty string clears", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"thinking_level": "",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("clear update: expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "" {
|
|
t.Errorf("empty thinking_level should clear: got %v", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
// 3. Garbage value — always 400, never silently clear.
|
|
t.Run("garbage value is always 400", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"thinking_level": "warp-speed",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("garbage thinking_level: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
// 4. Codex-only token while bound to a Claude runtime → 400. This
|
|
// is the "consistency" case from Trump's review: the API does
|
|
// NOT auto-clear or coerce; the same token that's valid for a
|
|
// Codex runtime is rejected here.
|
|
t.Run("codex token on claude runtime is 400, not silent clear", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"thinking_level": "minimal",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("codex token on claude runtime: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestUpdateAgent_RuntimeSwitch_PreservesValidValueRejectsInvalid covers
|
|
// the gap Elon flagged in PR1 review: a PATCH that switches `runtime_id`
|
|
// without explicitly touching `thinking_level` used to silently keep
|
|
// the existing value, so a Claude agent storing `max` could land on a
|
|
// Codex runtime where `max` is not a recognised token at all, and the
|
|
// daemon would receive a literal-invalid level.
|
|
//
|
|
// The contract the test pins, matching the existing "always 400 on
|
|
// literal-invalid" rule:
|
|
//
|
|
// - existing value still valid for the new runtime → 200, value kept
|
|
// - existing value invalid for the new runtime → 400, never silent
|
|
// clear or coerce
|
|
// - caller can recover by re-sending with `thinking_level: ""` to clear
|
|
// in the same PATCH
|
|
func TestUpdateAgent_RuntimeSwitch_PreservesValidValueRejectsInvalid(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
codexRuntimeID := createCodexProviderRuntime(t)
|
|
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx, `DELETE FROM agent WHERE workspace_id = $1 AND name LIKE 'runtime-switch-%'`, testWorkspaceID)
|
|
})
|
|
|
|
// Direction is Codex → Claude: Codex's enum is a superset of Claude's
|
|
// (it adds none/minimal plus the gpt-5.6 max/ultra levels), so a
|
|
// Codex-only token is the value that becomes invalid on switch.
|
|
t.Run("existing value still valid for new runtime is kept", func(t *testing.T) {
|
|
// `high` is valid for both Codex and Claude enums — switching
|
|
// runtime without touching thinking_level should keep it.
|
|
agentID := createAgentOnRuntime(t, "runtime-switch-keep", codexRuntimeID, "high")
|
|
body := map[string]any{
|
|
"runtime_id": claudeRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 when existing value is still valid, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "high" {
|
|
t.Errorf("expected thinking_level=high preserved across runtime switch, got %v", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
t.Run("existing value invalid for new runtime is 400, not silent", func(t *testing.T) {
|
|
// `ultra` is Codex-only (added for the gpt-5.6 series); switching to
|
|
// Claude must NOT silently keep it. Behaviour stays consistent with
|
|
// the explicit-set path: always 400 on literal-invalid.
|
|
agentID := createAgentOnRuntime(t, "runtime-switch-reject", codexRuntimeID, "ultra")
|
|
body := map[string]any{
|
|
"runtime_id": claudeRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 when existing value is invalid for new runtime, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("simultaneous explicit clear lets the switch through", func(t *testing.T) {
|
|
// The 400 above is recoverable: pass `thinking_level: ""` in
|
|
// the same PATCH and the switch goes through with a cleared
|
|
// value. This is the documented escape hatch in the error
|
|
// message; the test pins it so the contract holds.
|
|
agentID := createAgentOnRuntime(t, "runtime-switch-clear", codexRuntimeID, "ultra")
|
|
body := map[string]any{
|
|
"runtime_id": claudeRuntimeID,
|
|
"thinking_level": "",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 with simultaneous clear, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "" {
|
|
t.Errorf("expected thinking_level cleared, got %v", resp["thinking_level"])
|
|
}
|
|
})
|
|
|
|
t.Run("simultaneous explicit set to valid value lets the switch through", func(t *testing.T) {
|
|
// The other recovery: caller picks a value valid for the new
|
|
// runtime. Same PATCH, no need for a separate roundtrip.
|
|
agentID := createAgentOnRuntime(t, "runtime-switch-replace", codexRuntimeID, "ultra")
|
|
body := map[string]any{
|
|
"runtime_id": claudeRuntimeID,
|
|
"thinking_level": "high",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 with simultaneous set, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["thinking_level"] != "high" {
|
|
t.Errorf("expected thinking_level=high, got %v", resp["thinking_level"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestUpdateAgent_RuntimeSwitch_ClearsKnownIncompatibleModel covers the
|
|
// runtime/model persistence bug from MUL-3341: a runtime_id-only PATCH used
|
|
// to preserve a provider-native model string, so switching a Claude Code
|
|
// agent to Codex could leave agent.model = "claude-..." and fail at task
|
|
// execution. Unknown custom models are intentionally preserved because the
|
|
// API supports manual entries and cannot prove they are invalid.
|
|
func TestUpdateAgent_RuntimeSwitch_ClearsKnownIncompatibleModel(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
|
|
ctx := context.Background()
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
secondClaudeRuntimeID := createClaudeProviderRuntime(t)
|
|
codexRuntimeID := createCodexProviderRuntime(t)
|
|
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx, `DELETE FROM agent WHERE workspace_id = $1 AND name LIKE 'runtime-model-switch-%'`, testWorkspaceID)
|
|
})
|
|
|
|
t.Run("runtime-only switch clears known foreign model", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-clear", claudeRuntimeID, "claude-sonnet-4-6")
|
|
body := map[string]any{
|
|
"runtime_id": codexRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 switching runtime with incompatible model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "" {
|
|
t.Errorf("expected model cleared across Claude->Codex runtime switch, got %v", resp["model"])
|
|
}
|
|
})
|
|
|
|
t.Run("runtime-only switch clears provider-prefixed model not accepted by target", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-prefixed", claudeRuntimeID, "openai/gpt-4o")
|
|
body := map[string]any{
|
|
"runtime_id": codexRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 switching runtime with provider-prefixed model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "" {
|
|
t.Errorf("expected provider-prefixed model cleared across runtime switch, got %v", resp["model"])
|
|
}
|
|
})
|
|
|
|
t.Run("runtime-only switch keeps exact target accepted model", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-accepted", claudeRuntimeID, "gpt-5.5")
|
|
body := map[string]any{
|
|
"runtime_id": codexRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 preserving exact target accepted model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "gpt-5.5" {
|
|
t.Errorf("expected exact target model preserved, got %v", resp["model"])
|
|
}
|
|
})
|
|
|
|
t.Run("runtime-only switch keeps context-tagged target model", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-context-tag", claudeRuntimeID, "claude-opus-5[1m]")
|
|
body := map[string]any{
|
|
"runtime_id": secondClaudeRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 preserving context-tagged Claude model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "claude-opus-5[1m]" {
|
|
t.Errorf("expected context-tagged Claude model preserved, got %v", resp["model"])
|
|
}
|
|
})
|
|
|
|
t.Run("explicit replacement model wins during switch", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-replace", claudeRuntimeID, "claude-sonnet-4-6")
|
|
body := map[string]any{
|
|
"runtime_id": codexRuntimeID,
|
|
"model": "gpt-5.5",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 with explicit replacement model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "gpt-5.5" {
|
|
t.Errorf("expected explicit model to be persisted, got %v", resp["model"])
|
|
}
|
|
})
|
|
|
|
t.Run("unknown custom model is preserved", func(t *testing.T) {
|
|
agentID := createAgentOnRuntimeWithModel(t, "runtime-model-switch-custom", claudeRuntimeID, "private-lab-model")
|
|
body := map[string]any{
|
|
"runtime_id": codexRuntimeID,
|
|
}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 preserving unknown custom model, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
var resp map[string]any
|
|
_ = json.NewDecoder(w.Body).Decode(&resp)
|
|
if resp["model"] != "private-lab-model" {
|
|
t.Errorf("expected unknown custom model preserved, got %v", resp["model"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestThinkingLevelRejectionCopy pins WHY a thinking_level was refused, not
|
|
// just that it was. A runtime with no reasoning control must not be described
|
|
// as receiving an unrecognised value: "high" is a fine effort token, and the
|
|
// old shared sentence sent users looking for a spelling that cannot exist
|
|
// (MUL-5770).
|
|
//
|
|
// Copilot is the standing example: it discovers over ACP but executes through
|
|
// its own CLI, so no live ACP session exists to carry an effort. Hermes used
|
|
// to play this role and no longer can — jcode runs under that provider and
|
|
// applies an advertised effort, so the provider-level gate is open for it.
|
|
func TestThinkingLevelRejectionCopy(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("runtime without reasoning control names the capability gap", func(t *testing.T) {
|
|
msg := thinkingLevelRejection("copilot", "high")
|
|
if !strings.Contains(msg, "does not support a per-agent reasoning effort") {
|
|
t.Errorf("expected a capability explanation, got %q", msg)
|
|
}
|
|
if strings.Contains(msg, "not a recognised value") {
|
|
t.Errorf("capability gap must not read as a bad token, got %q", msg)
|
|
}
|
|
if !strings.Contains(msg, `"copilot"`) {
|
|
t.Errorf("expected the runtime named in %q", msg)
|
|
}
|
|
})
|
|
|
|
t.Run("runtime with reasoning control names the value", func(t *testing.T) {
|
|
msg := thinkingLevelRejection("claude", "supersonic")
|
|
if !strings.Contains(msg, "not a recognised value") {
|
|
t.Errorf("expected a value-level explanation, got %q", msg)
|
|
}
|
|
if !strings.Contains(msg, `"supersonic"`) {
|
|
t.Errorf("expected the rejected token echoed in %q", msg)
|
|
}
|
|
})
|
|
|
|
t.Run("carry-over path always offers the clear escape hatch", func(t *testing.T) {
|
|
for _, provider := range []string{"copilot", "claude"} {
|
|
msg := existingThinkingLevelRejection(provider, "xhigh")
|
|
if !strings.Contains(msg, `thinking_level=""`) {
|
|
t.Errorf("provider %q: expected the clear instruction, got %q", provider, msg)
|
|
}
|
|
}
|
|
if msg := existingThinkingLevelRejection("copilot", "xhigh"); !strings.Contains(msg, "does not support a per-agent reasoning effort") {
|
|
t.Errorf("expected the capability explanation on the carry-over path, got %q", msg)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestCreateAgent_NoReasoningControlRejectsThinkingLevel covers the reported
|
|
// flow end-to-end: creating an agent on a runtime with no reasoning control
|
|
// 400s, and the body explains the capability gap rather than blaming the
|
|
// value. Empty stays valid — it means "runtime default".
|
|
//
|
|
// It also pins the other side of the MUL-5991 change: hermes, which used to be
|
|
// this test's subject, now accepts a level because jcode applies one.
|
|
func TestCreateAgent_NoReasoningControlRejectsThinkingLevel(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
gapRuntimeID := createProviderRuntime(t, "copilot")
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(),
|
|
`DELETE FROM agent WHERE workspace_id = $1 AND name LIKE '%-thinking-%'`,
|
|
testWorkspaceID,
|
|
)
|
|
})
|
|
|
|
t.Run("effort value rejected with a capability message", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "copilot-thinking-high",
|
|
"runtime_id": gapRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "high",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("copilot thinking_level=high: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "does not support a per-agent reasoning effort") {
|
|
t.Errorf("expected a capability explanation in the 400 body, got %s", w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("empty value still creates", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "copilot-thinking-empty",
|
|
"runtime_id": gapRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("copilot empty thinking_level: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
// hermes with no discovered catalog must REFUSE, not accept. The catalog is
|
|
// written only when a client asks for a model list, so a CLI-only caller can
|
|
// sit in the undiscovered state forever — treating it as "supported" is the
|
|
// permanent hole that let a Hermes Agent user persist a level the daemon
|
|
// then drops. The message names the missing evidence rather than claiming a
|
|
// limitation that would be false for jcode.
|
|
t.Run("hermes with no discovered catalog is refused", func(t *testing.T) {
|
|
hermesRuntimeID := createProviderRuntime(t, "hermes")
|
|
body := map[string]any{
|
|
"name": "hermes-thinking-high",
|
|
"runtime_id": hermesRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "high",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("hermes thinking_level=high with no catalog: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
if !strings.Contains(w.Body.String(), "has not reported a model catalog yet") {
|
|
t.Errorf("expected the not-yet-discovered explanation, got %s", w.Body.String())
|
|
}
|
|
if strings.Contains(w.Body.String(), "does not support") {
|
|
t.Errorf("must not claim a limitation we have not established, got %s", w.Body.String())
|
|
}
|
|
})
|
|
|
|
// Empty stays valid on the same runtime — "runtime default" needs no
|
|
// capability evidence.
|
|
t.Run("hermes with no catalog still accepts an empty value", func(t *testing.T) {
|
|
hermesRuntimeID := createProviderRuntime(t, "hermes")
|
|
body := map[string]any{
|
|
"name": "hermes-thinking-empty",
|
|
"runtime_id": hermesRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("hermes empty thinking_level: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestUpdateAgent_ThinkingLevelCarriesOverToUndiscoveredHermes covers the
|
|
// branch where the caller never mentions thinking_level at all: an agent that
|
|
// already carries one is switched onto a hermes runtime whose capability we
|
|
// cannot confirm. Preserving the value there would smuggle a possibly-inert
|
|
// setting onto the new runtime, so the switch is refused with the escape hatch
|
|
// spelled out.
|
|
func TestUpdateAgent_ThinkingLevelCarriesOverToUndiscoveredHermes(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
claudeRuntimeID := createClaudeProviderRuntime(t)
|
|
hermesRuntimeID := createProviderRuntime(t, "hermes")
|
|
|
|
created := httptest.NewRecorder()
|
|
testHandler.CreateAgent(created, newRequest(http.MethodPost, "/api/agents", map[string]any{
|
|
"name": "carryover-thinking-agent",
|
|
"runtime_id": claudeRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "high",
|
|
}))
|
|
if created.Code != http.StatusCreated {
|
|
t.Fatalf("seed agent: expected 201, got %d: %s", created.Code, created.Body.String())
|
|
}
|
|
var seeded map[string]any
|
|
_ = json.NewDecoder(created.Body).Decode(&seeded)
|
|
agentID, _ := seeded["id"].(string)
|
|
t.Cleanup(func() { testPool.Exec(ctx, `DELETE FROM agent WHERE id = $1`, agentID) })
|
|
|
|
w := httptest.NewRecorder()
|
|
testHandler.UpdateAgent(w, withURLParam(
|
|
newRequest(http.MethodPatch, "/api/agents/"+agentID, map[string]any{
|
|
"runtime_id": hermesRuntimeID,
|
|
}), "id", agentID))
|
|
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("carry-over onto undiscovered hermes: expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
// Decode rather than substring-match the raw body: the escape hatch contains
|
|
// quotes, which JSON-escapes to `thinking_level=\"\"` on the wire.
|
|
var errBody struct {
|
|
Error string `json:"error"`
|
|
}
|
|
if err := json.NewDecoder(w.Body).Decode(&errBody); err != nil {
|
|
t.Fatalf("decode error body: %v", err)
|
|
}
|
|
if !strings.Contains(errBody.Error, "has not reported a model catalog yet") {
|
|
t.Errorf("expected the not-yet-discovered explanation, got %q", errBody.Error)
|
|
}
|
|
if !strings.Contains(errBody.Error, `thinking_level=""`) {
|
|
t.Errorf("expected the clear-it escape hatch, got %q", errBody.Error)
|
|
}
|
|
}
|
|
|
|
// TestAcpThinkingDecision is the fix for the regression the hermes
|
|
// change introduced: `hermes` covers jcode (advertises an effort) and Hermes
|
|
// Agent (advertises none), and the provider name cannot tell them apart. The
|
|
// discovered catalog can, so the capability 400 survives for the binary that
|
|
// genuinely has no reasoning dial.
|
|
func TestAcpThinkingDecision(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := context.Background()
|
|
runtimeID := parseUUID("11111111-1111-1111-1111-111111111111")
|
|
|
|
withCatalog := func(t *testing.T, models []ModelEntry) *Handler {
|
|
t.Helper()
|
|
cache := NewInMemoryModelCatalogCache()
|
|
if err := cache.Put(ctx, uuidToString(runtimeID), models, nil, true); err != nil {
|
|
t.Fatalf("seed catalog: %v", err)
|
|
}
|
|
return &Handler{ModelCatalogCache: cache}
|
|
}
|
|
|
|
withEffort := []ModelEntry{{
|
|
ID: "gpt-5.6-sol",
|
|
Thinking: &ModelThinking{SupportedLevels: []ThinkingLevel{{Value: "high", Label: "High"}}},
|
|
}}
|
|
withoutEffort := []ModelEntry{{ID: "hermes-4"}}
|
|
|
|
t.Run("jcode-shaped catalog keeps the level accepted", func(t *testing.T) {
|
|
if got := withCatalog(t, withEffort).acpThinkingDecision(ctx, "hermes", runtimeID); got != acpEffortPresent {
|
|
t.Errorf("decision = %v, want acpEffortPresent", got)
|
|
}
|
|
})
|
|
|
|
t.Run("Hermes-Agent-shaped catalog restores the capability answer", func(t *testing.T) {
|
|
if got := withCatalog(t, withoutEffort).acpThinkingDecision(ctx, "hermes", runtimeID); got != acpEffortAbsent {
|
|
t.Errorf("decision = %v, want acpEffortAbsent", got)
|
|
}
|
|
})
|
|
|
|
// Undiscovered is its own answer for an ambiguous provider — not "supported".
|
|
// The catalog is only written once a client asks for a model list, so this
|
|
// state persists indefinitely for CLI-only callers.
|
|
t.Run("no cache is unknown for hermes", func(t *testing.T) {
|
|
if got := (&Handler{}).acpThinkingDecision(ctx, "hermes", runtimeID); got != acpEffortUnknown {
|
|
t.Errorf("decision = %v, want acpEffortUnknown", got)
|
|
}
|
|
})
|
|
|
|
t.Run("empty catalog is unknown for hermes", func(t *testing.T) {
|
|
if got := withCatalog(t, nil).acpThinkingDecision(ctx, "hermes", runtimeID); got != acpEffortUnknown {
|
|
t.Errorf("decision = %v, want acpEffortUnknown", got)
|
|
}
|
|
})
|
|
|
|
// reasonix names exactly one binary, and that binary supports an effort, so
|
|
// an undiscovered reasonix runtime is allowed rather than blocked before its
|
|
// first discovery. Tightening hermes must not tighten this.
|
|
t.Run("no cache still allows reasonix", func(t *testing.T) {
|
|
if got := (&Handler{}).acpThinkingDecision(ctx, "reasonix", runtimeID); got != acpEffortPresent {
|
|
t.Errorf("decision = %v, want acpEffortPresent — reasonix is unambiguous", got)
|
|
}
|
|
})
|
|
|
|
// Providers outside the ACP-catalog set are answered by name alone; this
|
|
// check must not start second-guessing them from a catalog.
|
|
t.Run("non-ACP provider is not consulted", func(t *testing.T) {
|
|
if got := withCatalog(t, withoutEffort).acpThinkingDecision(ctx, "claude", runtimeID); got != acpEffortPresent {
|
|
t.Errorf("decision = %v, want acpEffortPresent — claude is decided by provider", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// createProviderRuntime stands up a runtime row for the given provider so a
|
|
// test can exercise provider-level gates end-to-end.
|
|
func createProviderRuntime(t *testing.T, provider string) string {
|
|
t.Helper()
|
|
var runtimeID string
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent_runtime (
|
|
workspace_id, daemon_id, name, runtime_mode, provider, status,
|
|
device_info, metadata, last_seen_at, owner_id
|
|
)
|
|
VALUES ($1, NULL, $2, 'cloud', $5, 'online', $3, '{}'::jsonb, now(), $4)
|
|
RETURNING id
|
|
`, testWorkspaceID, provider+" Thinking Runtime", provider+" thinking-level test runtime", testUserID, provider).Scan(&runtimeID)
|
|
if err != nil {
|
|
t.Fatalf("create %s runtime: %v", provider, err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID)
|
|
})
|
|
return runtimeID
|
|
}
|
|
|
|
func createPiProviderRuntime(t *testing.T) string {
|
|
t.Helper()
|
|
var runtimeID string
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent_runtime (
|
|
workspace_id, daemon_id, name, runtime_mode, provider, status,
|
|
device_info, metadata, last_seen_at, owner_id
|
|
)
|
|
VALUES ($1, NULL, $2, 'cloud', 'pi', 'online', $3, '{}'::jsonb, now(), $4)
|
|
RETURNING id
|
|
`, testWorkspaceID, "Pi Thinking Runtime", "Pi thinking-level test runtime", testUserID).Scan(&runtimeID)
|
|
if err != nil {
|
|
t.Fatalf("create pi runtime: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID)
|
|
})
|
|
return runtimeID
|
|
}
|
|
|
|
// createCodexProviderRuntime mirrors createClaudeProviderRuntime but for
|
|
// the codex provider, so runtime-switch tests can exercise a real
|
|
// cross-provider transition.
|
|
func createCodexProviderRuntime(t *testing.T) string {
|
|
t.Helper()
|
|
var runtimeID string
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent_runtime (
|
|
workspace_id, daemon_id, name, runtime_mode, provider, status,
|
|
device_info, metadata, last_seen_at, owner_id
|
|
)
|
|
VALUES ($1, NULL, $2, 'cloud', 'codex', 'online', $3, '{}'::jsonb, now(), $4)
|
|
RETURNING id
|
|
`, testWorkspaceID, "Codex Thinking Runtime", "Codex thinking-level test runtime", testUserID).Scan(&runtimeID)
|
|
if err != nil {
|
|
t.Fatalf("create codex runtime: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID)
|
|
})
|
|
return runtimeID
|
|
}
|
|
|
|
// createClaudeProviderRuntime stands up a runtime row with provider
|
|
// "claude" so the thinking_level gate runs against the real Claude
|
|
// enum (the default test runtime uses a fake provider). The runtime
|
|
// is workspace-private but visible to the test owner.
|
|
func createClaudeProviderRuntime(t *testing.T) string {
|
|
t.Helper()
|
|
var runtimeID string
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent_runtime (
|
|
workspace_id, daemon_id, name, runtime_mode, provider, status,
|
|
device_info, metadata, last_seen_at, owner_id
|
|
)
|
|
VALUES ($1, NULL, $2, 'cloud', 'claude', 'online', $3, '{}'::jsonb, now(), $4)
|
|
RETURNING id
|
|
`, testWorkspaceID, "Claude Thinking Runtime", "Claude thinking-level test runtime", testUserID).Scan(&runtimeID)
|
|
if err != nil {
|
|
t.Fatalf("create claude runtime: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent_runtime WHERE id = $1`, runtimeID)
|
|
})
|
|
return runtimeID
|
|
}
|
|
|
|
// createAgentOnRuntime seeds an agent row bound to the given runtime
|
|
// with the given initial thinking_level (empty for NULL).
|
|
func createAgentOnRuntime(t *testing.T, name, runtimeID, level string) string {
|
|
t.Helper()
|
|
var agentID string
|
|
var levelArg any
|
|
if level == "" {
|
|
levelArg = nil
|
|
} else {
|
|
levelArg = level
|
|
}
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent (
|
|
workspace_id, name, description, runtime_mode, runtime_config,
|
|
runtime_id, visibility, max_concurrent_tasks, owner_id,
|
|
instructions, custom_env, custom_args, thinking_level
|
|
)
|
|
VALUES ($1, $2, '', 'cloud', '{}'::jsonb, $3, 'private', 1, $4, '', '{}'::jsonb, '[]'::jsonb, $5)
|
|
RETURNING id
|
|
`, testWorkspaceID, name, runtimeID, testUserID, levelArg).Scan(&agentID)
|
|
if err != nil {
|
|
t.Fatalf("create agent on runtime %s: %v", runtimeID, err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent WHERE id = $1`, agentID)
|
|
})
|
|
return agentID
|
|
}
|
|
|
|
func createAgentOnRuntimeWithModel(t *testing.T, name, runtimeID, model string) string {
|
|
t.Helper()
|
|
var agentID string
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent (
|
|
workspace_id, name, description, runtime_mode, runtime_config,
|
|
runtime_id, visibility, max_concurrent_tasks, owner_id,
|
|
instructions, custom_env, custom_args, model
|
|
)
|
|
VALUES ($1, $2, '', 'cloud', '{}'::jsonb, $3, 'private', 1, $4, '', '{}'::jsonb, '[]'::jsonb, $5)
|
|
RETURNING id
|
|
`, testWorkspaceID, name, runtimeID, testUserID, model).Scan(&agentID)
|
|
if err != nil {
|
|
t.Fatalf("create agent on runtime %s with model %s: %v", runtimeID, model, err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent WHERE id = $1`, agentID)
|
|
})
|
|
return agentID
|
|
}
|
|
|
|
// createOmpAgent inserts an omp-bound agent with an explicit model and level.
|
|
// Neither shared helper sets both, and the combination is exactly what the
|
|
// clear-the-model case needs.
|
|
func createOmpAgent(t *testing.T, name, runtimeID, model, level string) string {
|
|
t.Helper()
|
|
var agentID string
|
|
var levelArg any
|
|
if level != "" {
|
|
levelArg = level
|
|
}
|
|
err := testPool.QueryRow(context.Background(), `
|
|
INSERT INTO agent (
|
|
workspace_id, name, description, runtime_mode, runtime_config,
|
|
runtime_id, visibility, max_concurrent_tasks, owner_id,
|
|
instructions, custom_env, custom_args, model, thinking_level
|
|
)
|
|
VALUES ($1, $2, '', 'cloud', '{}'::jsonb, $3, 'private', 1, $4, '', '{}'::jsonb, '[]'::jsonb, $5, $6)
|
|
RETURNING id
|
|
`, testWorkspaceID, name, runtimeID, testUserID, model, levelArg).Scan(&agentID)
|
|
if err != nil {
|
|
t.Fatalf("create omp agent: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
testPool.Exec(context.Background(), `DELETE FROM agent WHERE id = $1`, agentID)
|
|
})
|
|
return agentID
|
|
}
|
|
|
|
// TestAgent_OmpThinkingLevelRequiresExplicitModel is the API half of MUL-7412.
|
|
// An omp effort with no pinned model must never reach storage: omp resolves its
|
|
// own default role model at task time and clamps the level to what THAT model
|
|
// supports, so a stored level would save cleanly, read back as set, and then run
|
|
// at a different level. Hiding the picker in the inspector does not cover
|
|
// `multica agent create/update` or a direct API call, so the refusal has to live
|
|
// here.
|
|
func TestAgent_OmpThinkingLevelRequiresExplicitModel(t *testing.T) {
|
|
if testHandler == nil {
|
|
t.Skip("database not available")
|
|
}
|
|
ctx := context.Background()
|
|
ompRuntimeID := createProviderRuntime(t, "omp")
|
|
|
|
t.Cleanup(func() {
|
|
testPool.Exec(ctx,
|
|
`DELETE FROM agent WHERE workspace_id = $1 AND name LIKE 'omp-model-gate-%'`,
|
|
testWorkspaceID,
|
|
)
|
|
})
|
|
|
|
t.Run("create with a level and no model is 400", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "omp-model-gate-nomodel",
|
|
"runtime_id": ompRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "max",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
// Nothing may persist from a rejected create.
|
|
var count int
|
|
if err := testPool.QueryRow(ctx,
|
|
`SELECT count(*) FROM agent WHERE workspace_id = $1 AND name = 'omp-model-gate-nomodel'`,
|
|
testWorkspaceID).Scan(&count); err != nil {
|
|
t.Fatalf("count agents: %v", err)
|
|
}
|
|
if count != 0 {
|
|
t.Errorf("rejected create persisted %d agent(s)", count)
|
|
}
|
|
})
|
|
|
|
t.Run("create with a level and a pinned model succeeds", func(t *testing.T) {
|
|
body := map[string]any{
|
|
"name": "omp-model-gate-pinned",
|
|
"runtime_id": ompRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"model": "devin/swe-2",
|
|
"thinking_level": "max",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("create with a whitespace-only model is 400", func(t *testing.T) {
|
|
// A blank model is stored as non-empty but resolves to nothing, so it
|
|
// must not be a way around the gate.
|
|
body := map[string]any{
|
|
"name": "omp-model-gate-blank",
|
|
"runtime_id": ompRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"model": " ",
|
|
"thinking_level": "max",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("setting a level on a model-less agent is 400", func(t *testing.T) {
|
|
agentID := createOmpAgent(t, "omp-model-gate-set", ompRuntimeID, "", "")
|
|
body := map[string]any{"thinking_level": "high"}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("clearing the model while a level persists is 400", func(t *testing.T) {
|
|
// The level is untouched by this request — the model change alone creates
|
|
// the invalid pair, so validating only the submitted field would miss it.
|
|
agentID := createOmpAgent(t, "omp-model-gate-clearmodel", ompRuntimeID, "devin/swe-2", "max")
|
|
body := map[string]any{"model": ""}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
// The stored pair must be untouched by the rejection.
|
|
var model, level string
|
|
if err := testPool.QueryRow(ctx,
|
|
`SELECT coalesce(model, ''), coalesce(thinking_level, '') FROM agent WHERE id = $1`,
|
|
agentID).Scan(&model, &level); err != nil {
|
|
t.Fatalf("read agent: %v", err)
|
|
}
|
|
if model != "devin/swe-2" || level != "max" {
|
|
t.Errorf("rejected update mutated state: model=%q level=%q", model, level)
|
|
}
|
|
})
|
|
|
|
t.Run("clearing model and level together succeeds", func(t *testing.T) {
|
|
agentID := createOmpAgent(t, "omp-model-gate-clearboth", ompRuntimeID, "devin/swe-2", "max")
|
|
body := map[string]any{"model": "", "thinking_level": ""}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("an unrelated edit on a valid agent still succeeds", func(t *testing.T) {
|
|
// The gate must not lock an agent out of ordinary edits.
|
|
agentID := createOmpAgent(t, "omp-model-gate-rename", ompRuntimeID, "devin/swe-2", "max")
|
|
body := map[string]any{"name": "omp-model-gate-rename-2"}
|
|
w := httptest.NewRecorder()
|
|
req := withURLParam(newRequest(http.MethodPatch, "/api/agents/"+agentID, body), "id", agentID)
|
|
testHandler.UpdateAgent(w, req)
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("a level with no model is still storable on codex", func(t *testing.T) {
|
|
// codex shares the execution constraint but is deliberately grandfathered:
|
|
// agents already hold this pair and the daemon drops the level at launch.
|
|
// Rejecting it here would block unrelated edits to them. Pinned so the
|
|
// carve-out is a decision on record rather than an oversight.
|
|
codexRuntimeID := createCodexProviderRuntime(t)
|
|
body := map[string]any{
|
|
"name": "omp-model-gate-codex",
|
|
"runtime_id": codexRuntimeID,
|
|
"visibility": "private",
|
|
"max_concurrent_tasks": 1,
|
|
"thinking_level": "high",
|
|
}
|
|
w := httptest.NewRecorder()
|
|
testHandler.CreateAgent(w, newRequest(http.MethodPost, "/api/agents", body))
|
|
if w.Code != http.StatusCreated {
|
|
t.Fatalf("codex empty-model create: expected 201, got %d: %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|