feat: support selective provider bundles in headless Bun builds (#565)

Signed-off-by: Kevin Cui <[email protected]>

.github/workflows/publish-npm.yml

@@ -84,6 +84,11 @@ jobs:
             await connector.close();
           }
           JS
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+        with:
+          bun-version-file: .bun-version
+      - name: Verify selective Bun builds
+        run: bun scripts/smoke-headless-build.ts "$RUNNER_TEMP/consumer"
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: headless-package

docs/headless.md

@@ -130,6 +130,33 @@ For Bun executables, include `getConnectorAssetDirectory()` in
 the asset directory named `open-connector`; package assets resolve
 independently of the working directory.
 
+To include only selected providers, use the build helper:
+
+```ts
+import { getConnectorBuildOptions } from "@oomol-lab/open-connector/build";
+
+const prepared = await getConnectorBuildOptions({ providers: ["github", "slack"] });
+try {
+  await Bun.build({
+    entrypoints: ["./server.ts"],
+    target: "bun",
+    format: "esm",
+    splitting: true,
+    plugins: prepared.plugins,
+    external: prepared.external,
+    compile: { outfile: "./app", assets: prepared.assets },
+  });
+} finally {
+  await prepared.dispose();
+}
+```
+
+Omitting options or `providers` includes all providers; `providers: []` includes
+none. Unknown IDs fail preparation. Selection includes each provider's complete
+executor module and catalog entry, including authentication and action schemas.
+Shared runtime code and migrations remain included. The runtime API is unchanged,
+and the installed npm package still contains all providers.
+
 ## Documentation
 
 - [Runtime API and MCP](https://github.com/oomol-lab/open-connector/blob/main/docs/runtime-api.md)

scripts/build-runtime-package.ts

@@ -34,7 +34,10 @@ await writeFile(
       description: "Embeddable Open Connector runtime for Node.js and Bun.",
       license: "Apache-2.0",
       type: "module",
-      exports: { ".": { types: "./src/server/connector-runtime.d.ts", import: "./src/server/connector-runtime.js" } },
+      exports: {
+        ".": { types: "./src/server/connector-runtime.d.ts", import: "./src/server/connector-runtime.js" },
+        "./build": { types: "./src/server/connector-build.d.ts", import: "./src/server/connector-build.js" },
+      },
       dependencies: source.dependencies,
       engines: { node: ">=22.18.0" },
       keywords: ["connector", "oauth", "mcp", "agents", "integrations"],

scripts/fixtures/headless-host.mjs

@@ -0,0 +1,18 @@
+import { createConnectorRuntime, getConnectorAssetDirectory } from "@oomol-lab/open-connector";
+import assert from "node:assert/strict";
+import { readdir } from "node:fs/promises";
+import { join } from "node:path";
+
+const expected = JSON.parse(process.argv[2]);
+const files = await readdir(join(getConnectorAssetDirectory(), "catalog/apps"));
+assert.deepEqual(files.filter((name) => name.endsWith(".json")).sort(), expected.map((id) => `${id}.json`).sort());
+
+const runtime = await createConnectorRuntime({ dataDir: "./data", publicOrigin: "https://host.example" });
+try {
+  const response = await runtime.fetch(new Request("https://host.example/v1/providers"));
+  assert.equal(response.status, 200);
+  const catalog = await response.json();
+  assert.deepEqual(catalog.data.map((provider) => provider.service).sort(), expected);
+} finally {
+  await runtime.close();
+}

scripts/smoke-headless-build.ts

@@ -0,0 +1,70 @@
+// Run with Bun after installing the packed headless package into the supplied consumer directory.
+import assert from "node:assert/strict";
+import { cp, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+const consumer = resolve(process.argv[2]);
+const { getConnectorBuildOptions } = (await import(
+  import.meta.resolve("@oomol-lab/open-connector/build", join(consumer, "package.json"))
+)) as typeof import("../src/server/connector-build.ts");
+const execution = await mkdtemp(join(tmpdir(), "connector-build-smoke-"));
+const entrypoint = join(consumer, "headless-host.mjs");
+
+try {
+  await cp(new URL("./fixtures/headless-host.mjs", import.meta.url), entrypoint);
+  await assert.rejects(getConnectorBuildOptions({ providers: ["missing-provider"] }), /Unknown provider/);
+  await assert.rejects(getConnectorBuildOptions({ providers: ["../github"] }), /provider id/);
+  const defaults = await getConnectorBuildOptions();
+  const omitted = await getConnectorBuildOptions({});
+  assert.deepEqual(omitted.assets, defaults.assets);
+  await omitted.dispose();
+  const index = JSON.parse(await readFile(join(defaults.assets[0]!, "catalog/apps-index.json"), "utf8"));
+  const all = index.providers.map((entry: { provider: { service: string } }) => entry.provider.service).sort();
+  await defaults.dispose();
+  for (const providers of [undefined, ["github", "github"], []]) {
+    const expected = providers === undefined ? all : [...new Set(providers)].sort();
+    const prepared = await getConnectorBuildOptions({ providers });
+    const outfile = join(execution, "host");
+    try {
+      const result = await Bun.build({
+        entrypoints: [entrypoint],
+        target: "bun",
+        format: "esm",
+        splitting: true,
+        plugins: prepared.plugins,
+        external: prepared.external,
+        compile: { outfile, assets: prepared.assets },
+        metafile: true,
+      });
+      assert.ok(result.success, result.logs.join("\n"));
+      assert.ok(result.metafile);
+      const bundledProviders = new Set<string>();
+      for (const path of Object.keys(result.metafile.inputs)) {
+        const match = /\/src\/providers\/([^/]+)\//.exec(path.replaceAll("\\", "/"));
+        if (match) bundledProviders.add(match[1]!);
+      }
+      assert.deepEqual([...bundledProviders].sort(), expected);
+    } finally {
+      await prepared.dispose();
+    }
+    if (providers !== undefined) await assert.rejects(readdir(prepared.assets[0]!));
+    // Bun 1.4.0 writes an ad-hoc signature that macOS 27 refuses to start (oven-sh/bun#39837); re-sign it in place
+    // like scripts/build-binary.ts does so the smoke also runs on a macOS checkout.
+    if (process.platform === "darwin") {
+      const signed = Bun.spawnSync(["codesign", "--force", "--sign", "-", outfile], { stderr: "inherit" });
+      assert.ok(signed.success, `codesign failed with exit code ${signed.exitCode}`);
+    }
+    const child = Bun.spawn([outfile, JSON.stringify(expected)], {
+      cwd: execution,
+      stdout: "inherit",
+      stderr: "inherit",
+    });
+    assert.equal(await child.exited, 0);
+    await rm(join(execution, "data"), { recursive: true, force: true });
+    console.log(`Compiled host passed with ${expected.length} providers.`);
+  }
+} finally {
+  await rm(entrypoint, { force: true });
+  await rm(execution, { recursive: true, force: true });
+}

scripts/tsconfig.package.json

@@ -6,5 +6,5 @@
     "outDir": "../dist/package/src",
     "rewriteRelativeImportExtensions": true
   },
-  "include": ["../src/server/connector-runtime.ts"]
+  "include": ["../src/server/connector-runtime.ts", "../src/server/connector-build.ts"]
 }

src/server/connector-assets.ts

@@ -0,0 +1,14 @@
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/** Directory to include in a host's Bun compile.assets. Its basename keeps connector assets namespaced. */
+export function getConnectorAssetDirectory(): string {
+  return isStandaloneExecutable()
+    ? join(import.meta.dirname, "open-connector")
+    : fileURLToPath(new URL("../../assets/open-connector/", import.meta.url));
+}
+
+/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */
+export function isStandaloneExecutable(): boolean {
+  return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true;
+}

src/server/connector-build.ts

@@ -0,0 +1,106 @@
+import type { CatalogIndexSource } from "../catalog-index.ts";
+import type { ProviderDefinition } from "../core/types.ts";
+
+import { access, cp, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { extname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { catalogIndexFileName, createCatalogIndex } from "../catalog-index.ts";
+import { assertProviderId } from "../core/provider-id.ts";
+import { getConnectorAssetDirectory } from "./connector-assets.ts";
+
+export interface ConnectorBuildOptions {
+  /** Omit to include all providers. An empty array includes none. */
+  providers?: readonly string[];
+}
+
+export interface ConnectorBuild {
+  /** Pass to Bun.build's compile.assets. */
+  assets: string[];
+  /** Pass to Bun.build alongside plugins. */
+  external: string[];
+  plugins: ConnectorBuildPlugin[];
+  /** Remove staged assets after the build, including when it fails. */
+  dispose(): Promise<void>;
+}
+
+// Keep the Bun plugin boundary structural so importing this helper does not change Node's global types.
+interface ConnectorBuildPlugin {
+  name: string;
+  setup(build: ConnectorBuildHooks): void;
+}
+
+interface ConnectorBuildHooks {
+  onLoad(
+    options: { filter: RegExp },
+    callback: (args: { path: string }) => { contents: string; loader: "js" } | undefined,
+  ): unknown;
+  onEnd(callback: (result: { success: boolean }) => void): unknown;
+}
+
+/** Prepare a headless Bun executable with only the requested provider code and catalog entries. */
+export async function getConnectorBuildOptions(options: ConnectorBuildOptions = {}): Promise<ConnectorBuild> {
+  const sourceAssets = getConnectorAssetDirectory();
+  const external = ["proxy-agent"];
+  if (options.providers === undefined) {
+    return { assets: [sourceAssets], external, plugins: [], dispose: () => Promise.resolve() };
+  }
+
+  const services = [...new Set(options.providers)].sort();
+  const catalogDir = join(sourceAssets, "catalog/apps");
+  const files = new Set(await readdir(catalogDir));
+  for (const service of services) {
+    assertProviderId(service);
+    if (!files.has(`${service}.json`)) throw new Error(`Unknown provider: ${service}`);
+  }
+
+  const directory = await mkdtemp(join(tmpdir(), "open-connector-build-"));
+  const dispose = (): Promise<void> => rm(directory, { recursive: true, force: true });
+  try {
+    const assets = join(directory, "open-connector");
+    const apps = join(assets, "catalog/apps");
+    await mkdir(apps, { recursive: true });
+    const providerDir = fileURLToPath(new URL("../providers/", import.meta.url));
+    const extension = extname(import.meta.filename);
+    const registry = await realpath(join(providerDir, `registry.generated${extension}`));
+    const sources: CatalogIndexSource[] = [];
+    for (const service of services) {
+      await access(join(providerDir, service, `executors${extension}`));
+      const file = `${service}.json`;
+      const content = await readFile(join(catalogDir, file));
+      const provider = JSON.parse(content.toString()) as ProviderDefinition;
+      if (provider.service !== service) throw new Error(`Catalog service does not match ${file}`);
+      await writeFile(join(apps, file), content);
+      sources.push({ file, bytes: content.byteLength, provider });
+    }
+    // Bun embeds files, so preserve the directory that the runtime reads even with no providers.
+    if (!services.length) await writeFile(join(apps, "empty"), "");
+    await writeFile(join(assets, "catalog", catalogIndexFileName), JSON.stringify(createCatalogIndex(sources)));
+    await cp(join(sourceAssets, "migrations"), join(assets, "migrations"), { recursive: true });
+    const contents = `export const executorModules = {\n${services
+      .map(
+        (service) =>
+          `${JSON.stringify(service)}: () => import(${JSON.stringify(`./${service}/executors${extension}`)}),`,
+      )
+      .join("\n")}\n};`;
+    const plugin: ConnectorBuildPlugin = {
+      name: "open-connector-providers",
+      setup(build) {
+        let replaced = false;
+        build.onLoad({ filter: /[/\\]providers[/\\]registry\.generated\.[jt]s$/ }, ({ path }) => {
+          if (path !== registry) return;
+          replaced = true;
+          return { contents, loader: "js" };
+        });
+        build.onEnd((result) => {
+          if (result.success && !replaced)
+            throw new Error("The build did not load Open Connector's executor registry.");
+        });
+      },
+    };
+    return { assets: [assets], external, plugins: [plugin], dispose };
+  } catch (error) {
+    await dispose();
+    throw error;
+  }
+}

src/server/connector-runtime.ts

@@ -6,7 +6,6 @@ import type { IStagedTransitFileService } from "./files/transit-file-store.ts";
 
 import { mkdir } from "node:fs/promises";
 import { join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
 import { loadCatalog } from "../catalog-store.ts";
 import { ActionPolicyService } from "../core/action-policy.ts";
 import { setEgressTrustedHosts, setPrivateNetworkAccessAllowed } from "../core/request.ts";
@@ -14,10 +13,10 @@ import { ProviderLoader } from "../providers/provider-loader.ts";
 import { executorModules } from "../providers/registry.generated.ts";
 import { createRuntimeJwtVerifier } from "./api/runtime-jwt.ts";
 import { createConnectApp } from "./connect-app.ts";
+import { getConnectorAssetDirectory } from "./connector-assets.ts";
 import { cleanupStagedTransitFiles, createNodeTransitFileUpload } from "./files/node-transit-file-upload.ts";
 import { TransitFileService } from "./files/transit-files.ts";
 import { createSecretCodec } from "./secrets/secret-codec.ts";
-import { isStandaloneExecutable } from "./server-assets.ts";
 import { createDirectoryMigrationSource } from "./storage/migration-source.ts";
 import { createNodeRuntimeDatabase } from "./storage/node-runtime-database.ts";
 import { DEFAULT_RUN_LIMIT } from "./storage/runtime-store.ts";
@@ -108,12 +107,7 @@ export interface ConnectorRuntime {
 
 let runtimeActive = false;
 
-/** Directory to include in a host's Bun compile.assets. Its basename keeps connector assets namespaced. */
-export function getConnectorAssetDirectory(): string {
-  return isStandaloneExecutable()
-    ? join(import.meta.dirname, "open-connector")
-    : fileURLToPath(new URL("../../assets/open-connector/", import.meta.url));
-}
+export { getConnectorAssetDirectory } from "./connector-assets.ts";
 
 /** Create a headless runtime without opening a listener or installing process signal handlers. */
 export async function createConnectorRuntime(options: ConnectorRuntimeOptions): Promise<ConnectorRuntime> {

src/server/server-assets.ts

@@ -3,6 +3,7 @@ import type { ConnectorAssets } from "./connector-runtime.ts";
 import { access } from "node:fs/promises";
 import { join } from "node:path";
 import { catalogIndexFileName } from "../catalog-index.ts";
+import { isStandaloneExecutable } from "./connector-assets.ts";
 
 /**
  * Locations of the assets that are generated or built outside `src` and read
@@ -51,11 +52,6 @@ export async function resolveServerAssets(): Promise<ServerAssets> {
   };
 }
 
-/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */
-export function isStandaloneExecutable(): boolean {
-  return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true;
-}
-
 async function resolveStaticRoot(root: string): Promise<string | undefined> {
   const indexHtml = await resolveExistingFile(join(root, "index.html"));
   return indexHtml === undefined ? undefined : root;