multica/server/internal/daemon/runtime_not_executable_test.go

package daemon

import (
	"context"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/multica-ai/multica/server/pkg/agent"
)

// stubNotExecutableProbe makes every version probe fail the way the OS fails a
// file it will not execute — an npm placeholder stub whose postinstall was
// blocked (MUL-6164) — wrapped exactly as detectCLIVersion wraps it, so the
// daemon sees the same error shape it sees in the field.
func stubNotExecutableProbe(t *testing.T, path string) {
	t.Helper()
	orig := detectAgentVersion
	t.Cleanup(func() { detectAgentVersion = orig })
	detectAgentVersion = func(_ context.Context, runtimeCmd agent.Command) (string, error) {
		p := runtimeCmd.Path
		enoexec := &fs.PathError{Op: "fork/exec", Path: p, Err: syscall.ENOEXEC}
		return "", fmt.Errorf("detect version for %s: %w", p, agent.ExplainExecError(enoexec))
	}
}

// stubConfirmWindow collapses the confirmation window so a second probe round
// counts as confirmation immediately. Two rounds are still required.
func stubConfirmWindow(t *testing.T, window time.Duration) {
	t.Helper()
	orig := condemnedConfirmWindow
	t.Cleanup(func() { condemnedConfirmWindow = orig })
	condemnedConfirmWindow = window
}

// npmPackagedCLI stages the layout an npm-installed agent CLI has — a bin entry
// under a package whose manifest declares the postinstall that was supposed to
// replace it — so the daemon can derive a real repair command from it.
func npmPackagedCLI(t *testing.T) string {
	t.Helper()
	root := filepath.Join(t.TempDir(), "claude-code")
	if err := os.MkdirAll(filepath.Join(root, "bin"), 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	manifest := `{"name":"@anthropic-ai/claude-code","scripts":{"postinstall":"node install.cjs"}}`
	if err := os.WriteFile(filepath.Join(root, "package.json"), []byte(manifest), 0o644); err != nil {
		t.Fatalf("write manifest: %v", err)
	}
	return filepath.Join(root, "bin", "claude.exe")
}

func notExecutableDaemon(t *testing.T) *Daemon {
	t.Helper()
	stubProbeRetry(t, time.Millisecond, time.Millisecond)
	execPath := npmPackagedCLI(t)
	stubNotExecutableProbe(t, execPath)
	d := freshDaemon("")
	d.cfg.Agents = map[string]AgentEntry{"claude": {Path: execPath, Command: "claude"}}
	return d
}

// A CLI that cannot exec is a fact about the file on disk, so it must end up on
// the demotable side of the probe rather than the transient one — that is what
// takes the runtime offline instead of leaving it online to claim tasks it
// cannot run. The first sighting still waits: an installer overwriting the bin
// entry in place is briefly indistinguishable from a broken install.
func TestDetectBuiltinRuntimes_NotExecutableDemotesOnlyAfterConfirmation(t *testing.T) {
	stubConfirmWindow(t, 0)
	d := notExecutableDaemon(t)

	runtimes, demotable, unavailable := d.detectBuiltinRuntimes(context.Background())
	if len(runtimes) != 0 {
		t.Fatalf("registered %v, want none: the CLI cannot run", runtimes)
	}
	if _, ok := demotable["claude"]; ok {
		t.Errorf("first sighting demoted %v, want it held for confirmation", demotable)
	}
	if _, ok := unavailable["claude"]; !ok {
		t.Errorf("first sighting = %v, want claude treated as transient for one round", unavailable)
	}

	_, demotable, unavailable = d.detectBuiltinRuntimes(context.Background())
	if _, ok := unavailable["claude"]; ok {
		t.Errorf("second sighting = %v, want claude condemned, not held again", unavailable)
	}
	verdict, ok := demotable["claude"]
	if !ok {
		t.Fatalf("second sighting did not condemn claude: %v", demotable)
	}
	// The reason is what /health shows and what the demotion logs, so it has to
	// carry the repair instructions, not just the errno.
	for _, want := range []string{"not executable", "postinstall"} {
		if !strings.Contains(verdict.reason, want) {
			t.Errorf("reason is missing %q, so the user still cannot act on it:\n%s", want, verdict.reason)
		}
	}
	// And the structured half the server stores: without a code the admission
	// paths cannot tell this apart from a sleeping laptop.
	if verdict.offline == nil || verdict.offline.Code != RuntimeOfflineCodeNotExecutable {
		t.Fatalf("verdict carries no offline reason for the server: %+v", verdict.offline)
	}
	if verdict.offline.Repair == nil || !strings.Contains(verdict.offline.Repair.Command, "install.cjs") {
		t.Errorf("offline reason lost the repair command: %+v", verdict.offline.Repair)
	}
}

// Two probes inside the window are two samples of the same instant as far as
// this verdict is concerned — an in-place install can span them.
func TestDetectBuiltinRuntimes_NotExecutableWaitsOutTheWindow(t *testing.T) {
	stubConfirmWindow(t, time.Hour)
	d := notExecutableDaemon(t)

	for round := 1; round <= 3; round++ {
		_, demotable, _ := d.detectBuiltinRuntimes(context.Background())
		if _, ok := demotable["claude"]; ok {
			t.Fatalf("round %d demoted inside the confirmation window", round)
		}
	}
}

// The window is measured from the FIRST sighting, and a healthy probe resets it
// — otherwise an unrelated failure months later would inherit a stale clock and
// demote on its first sighting.
func TestConfirmNotExecutable_WindowStartsAtFirstSighting(t *testing.T) {
	stubConfirmWindow(t, time.Minute)
	d := freshDaemon("")
	start := time.Now()

	if d.confirmCondemned(builtinProbeNotExecutable, "claude", start) {
		t.Fatal("first sighting confirmed the verdict on its own")
	}
	if d.confirmCondemned(builtinProbeNotExecutable, "claude", start.Add(59*time.Second)) {
		t.Error("confirmed before the window elapsed")
	}
	if !d.confirmCondemned(builtinProbeNotExecutable, "claude", start.Add(time.Minute)) {
		t.Error("did not confirm after the window elapsed")
	}

	d.clearCondemned("claude")
	if d.confirmCondemned(builtinProbeNotExecutable, "claude", start.Add(time.Hour)) {
		t.Error("a recovered provider must start a fresh window, not inherit the old one")
	}
}

// The user-visible half of the feature: once the verdict is confirmed, the
// provider's runtimes actually go offline instead of staying online to claim
// tasks that die at fork/exec — and they come back on their own after the
// repair, without a daemon restart.
func TestRefreshAgentVersions_TakesNotExecutableRuntimeOfflineAndBack(t *testing.T) {
	stubConfirmWindow(t, 0)
	stubProbeRetry(t, time.Millisecond, time.Second)
	fx := newVersionRefreshFixture(t)
	d := fx.daemon

	// npm blocks the postinstall on the next `npm i -g`: same path, same
	// executable bit, a placeholder stub where the binary used to be.
	fx.setProbeErr(func(path string, _ int) error {
		enoexec := &fs.PathError{Op: "fork/exec", Path: path, Err: syscall.ENOEXEC}
		return fmt.Errorf("detect version for %s: %w", path, agent.ExplainExecError(enoexec))
	})

	d.refreshAgentVersions(context.Background()) // first sighting: held
	for _, workspaceID := range []string{"ws-1", "ws-2"} {
		if got := registeredProviders(t, d, workspaceID); len(got) != 1 {
			t.Fatalf("%s providers = %v, want codex still online during confirmation", workspaceID, got)
		}
	}

	d.refreshAgentVersions(context.Background()) // confirmed: demote
	for _, workspaceID := range []string{"ws-1", "ws-2"} {
		if got := registeredProviders(t, d, workspaceID); len(got) != 0 {
			t.Errorf("%s providers = %v, want none: the CLI cannot run, so the runtime must not keep claiming tasks", workspaceID, got)
		}
	}
	if reason := d.skippedAgentsSnapshot()["codex"]; !strings.Contains(reason, "not executable") {
		t.Errorf("/health reason = %q, want it to say why the runtime went offline", reason)
	}

	// The user runs the repair. Nothing restarts; the converge path re-registers.
	fx.setProbeErr(nil)
	d.convergeRuntimeRegistrations(context.Background())
	for _, workspaceID := range []string{"ws-1", "ws-2"} {
		if got := registeredProviders(t, d, workspaceID); len(got) != 1 || got[0] != "codex" {
			t.Errorf("%s providers after repair = %v, want [codex] back without a restart", workspaceID, got)
		}
	}
}

// A provider that probes OK again clears its pending verdict, which is what
// makes recovery automatic: no runtime + no hold puts it back in
// providersMissingRuntimes for the converge path to register.
func TestDetectBuiltinRuntimes_HealthyProbeClearsPendingVerdict(t *testing.T) {
	stubConfirmWindow(t, 0)
	stubProbeRetry(t, time.Millisecond, time.Millisecond)
	d := freshDaemon("")
	d.cfg.Agents = map[string]AgentEntry{"claude": {Path: "/fake/claude", Command: "claude"}}

	broken := true
	orig := detectAgentVersion
	t.Cleanup(func() { detectAgentVersion = orig })
	detectAgentVersion = func(_ context.Context, runtimeCmd agent.Command) (string, error) {
		p := runtimeCmd.Path
		if broken {
			enoexec := &fs.PathError{Op: "fork/exec", Path: p, Err: syscall.ENOEXEC}
			return "", fmt.Errorf("detect version for %s: %w", p, agent.ExplainExecError(enoexec))
		}
		return "2.1.231", nil
	}

	d.detectBuiltinRuntimes(context.Background()) // first sighting: clock starts
	broken = false
	if runtimes, _, _ := d.detectBuiltinRuntimes(context.Background()); len(runtimes) != 1 {
		t.Fatalf("repaired CLI registered %v, want one runtime", runtimes)
	}
	broken = true
	if _, demotable, _ := d.detectBuiltinRuntimes(context.Background()); len(demotable) != 0 {
		t.Errorf("a fresh failure demoted immediately (%v): the healthy probe should have reset the clock", demotable)
	}
}

// The verdict has to survive the server losing it.
//
// A register whose REQUEST is still in flight across the verdict is upserted
// after it, and that upsert overwrites the runtime row's metadata wholesale —
// including the reason the demotion just stored. The local hold correctly
// refuses the response, but if the cleanup that takes the revived row offline
// again sends no reason, the server is left with a bare "offline". Every
// admission path then reads that as "the machine will come back" and goes right
// back to queueing, which is the behaviour this change exists to remove
// (MUL-6164).
func TestDeregisterRevivedRuntimes_ReattachesTheUnusableReason(t *testing.T) {
	stubConfirmWindow(t, 0)
	stubProbeRetry(t, time.Millisecond, time.Second)
	// Stable ids before the first registration: the interleave only exists when
	// the stale response names the same row the demotion took offline.
	fx := newVersionRefreshFixtureWith(t, func(fx *batchFixture) { fx.enableStableRuntimeIDs() })
	d := fx.daemon
	runtimeID := fx.runtimeIDFor("ws-1", "codex")
	if runtimeID == "" {
		t.Fatal("fixture did not expose a stable runtime id")
	}

	// Hold ONE register inside the server handler, before it upserts. That is
	// the request the verdict has to survive; every later register passes
	// straight through.
	var once sync.Once
	reached, release := make(chan struct{}), make(chan struct{})
	fx.setRegisterGate(func(string) {
		once.Do(func() {
			close(reached)
			<-release
		})
	})

	inFlight := []map[string]string{{"name": "Codex", "type": "codex", "version": "9.9.9", "status": "online"}}
	respCh := make(chan *RegisterResponse, 1)
	go func() {
		resp, err := d.registerBuiltinRuntimesForWorkspaceLocked(context.Background(), "ws-1", inFlight)
		if err != nil {
			respCh <- nil
			return
		}
		respCh <- resp
	}()
	<-reached

	// While it is held: the CLI breaks and the verdict lands, reason and all.
	fx.setProbeErr(func(path string, _ int) error {
		enoexec := &fs.PathError{Op: "fork/exec", Path: path, Err: syscall.ENOEXEC}
		return fmt.Errorf("detect version for %s: %w", path, agent.ExplainExecError(enoexec))
	})
	d.refreshAgentVersions(context.Background()) // first sighting
	d.refreshAgentVersions(context.Background()) // confirmed: demote
	if _, ok := fx.runtimeOfflineReason(runtimeID); !ok {
		t.Fatal("demotion did not record a reason on the runtime row (precondition)")
	}

	// The held register now upserts, wiping the reason and putting the row back
	// online — the loss this test exists for.
	close(release)
	staleResp := <-respCh
	if staleResp == nil {
		t.Fatal("in-flight register failed")
	}
	if _, ok := fx.runtimeOfflineReason(runtimeID); ok {
		t.Fatal("precondition: the late upsert should have cleared the stored reason")
	}

	_, revived, ok := d.mergeBuiltinRegisterResponse("ws-1", staleResp)
	if !ok {
		t.Fatal("merge of the late register response failed")
	}
	if len(revived.ids) == 0 {
		t.Fatal("late response was rejected locally but named no row to take offline again")
	}

	d.deregisterRevivedRuntimes(context.Background(), "ws-1", revived)

	reason, ok := fx.runtimeOfflineReason(runtimeID)
	if !ok {
		t.Fatal("the revived row went offline with no reason: admission would treat an unusable CLI as a " +
			"sleeping machine and queue for it again")
	}
	if reason.Code != RuntimeOfflineCodeNotExecutable {
		t.Errorf("re-attached reason = %q, want %q", reason.Code, RuntimeOfflineCodeNotExecutable)
	}
}