diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 528ed06f..03b4687d 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -84,6 +84,11 @@ jobs: await connector.close(); } JS + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + - name: Verify selective Bun builds + run: bun scripts/smoke-headless-build.ts "$RUNNER_TEMP/consumer" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: headless-package diff --git a/docs/headless.md b/docs/headless.md index 5cc375a4..232df3b6 100644 --- a/docs/headless.md +++ b/docs/headless.md @@ -130,6 +130,33 @@ For Bun executables, include `getConnectorAssetDirectory()` in the asset directory named `open-connector`; package assets resolve independently of the working directory. +To include only selected providers, use the build helper: + +```ts +import { getConnectorBuildOptions } from "@oomol-lab/open-connector/build"; + +const prepared = await getConnectorBuildOptions({ providers: ["github", "slack"] }); +try { + await Bun.build({ + entrypoints: ["./server.ts"], + target: "bun", + format: "esm", + splitting: true, + plugins: prepared.plugins, + external: prepared.external, + compile: { outfile: "./app", assets: prepared.assets }, + }); +} finally { + await prepared.dispose(); +} +``` + +Omitting options or `providers` includes all providers; `providers: []` includes +none. Unknown IDs fail preparation. Selection includes each provider's complete +executor module and catalog entry, including authentication and action schemas. +Shared runtime code and migrations remain included. The runtime API is unchanged, +and the installed npm package still contains all providers. + ## Documentation - [Runtime API and MCP](https://github.com/oomol-lab/open-connector/blob/main/docs/runtime-api.md) diff --git a/scripts/build-runtime-package.ts b/scripts/build-runtime-package.ts index 1de6ea75..ff4d5ffe 100644 --- a/scripts/build-runtime-package.ts +++ b/scripts/build-runtime-package.ts @@ -34,7 +34,10 @@ await writeFile( description: "Embeddable Open Connector runtime for Node.js and Bun.", license: "Apache-2.0", type: "module", - exports: { ".": { types: "./src/server/connector-runtime.d.ts", import: "./src/server/connector-runtime.js" } }, + exports: { + ".": { types: "./src/server/connector-runtime.d.ts", import: "./src/server/connector-runtime.js" }, + "./build": { types: "./src/server/connector-build.d.ts", import: "./src/server/connector-build.js" }, + }, dependencies: source.dependencies, engines: { node: ">=22.18.0" }, keywords: ["connector", "oauth", "mcp", "agents", "integrations"], diff --git a/scripts/fixtures/headless-host.mjs b/scripts/fixtures/headless-host.mjs new file mode 100644 index 00000000..996d20e8 --- /dev/null +++ b/scripts/fixtures/headless-host.mjs @@ -0,0 +1,18 @@ +import { createConnectorRuntime, getConnectorAssetDirectory } from "@oomol-lab/open-connector"; +import assert from "node:assert/strict"; +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; + +const expected = JSON.parse(process.argv[2]); +const files = await readdir(join(getConnectorAssetDirectory(), "catalog/apps")); +assert.deepEqual(files.filter((name) => name.endsWith(".json")).sort(), expected.map((id) => `${id}.json`).sort()); + +const runtime = await createConnectorRuntime({ dataDir: "./data", publicOrigin: "https://host.example" }); +try { + const response = await runtime.fetch(new Request("https://host.example/v1/providers")); + assert.equal(response.status, 200); + const catalog = await response.json(); + assert.deepEqual(catalog.data.map((provider) => provider.service).sort(), expected); +} finally { + await runtime.close(); +} diff --git a/scripts/smoke-headless-build.ts b/scripts/smoke-headless-build.ts new file mode 100644 index 00000000..c4a8b4ea --- /dev/null +++ b/scripts/smoke-headless-build.ts @@ -0,0 +1,70 @@ +// Run with Bun after installing the packed headless package into the supplied consumer directory. +import assert from "node:assert/strict"; +import { cp, mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const consumer = resolve(process.argv[2]); +const { getConnectorBuildOptions } = (await import( + import.meta.resolve("@oomol-lab/open-connector/build", join(consumer, "package.json")) +)) as typeof import("../src/server/connector-build.ts"); +const execution = await mkdtemp(join(tmpdir(), "connector-build-smoke-")); +const entrypoint = join(consumer, "headless-host.mjs"); + +try { + await cp(new URL("./fixtures/headless-host.mjs", import.meta.url), entrypoint); + await assert.rejects(getConnectorBuildOptions({ providers: ["missing-provider"] }), /Unknown provider/); + await assert.rejects(getConnectorBuildOptions({ providers: ["../github"] }), /provider id/); + const defaults = await getConnectorBuildOptions(); + const omitted = await getConnectorBuildOptions({}); + assert.deepEqual(omitted.assets, defaults.assets); + await omitted.dispose(); + const index = JSON.parse(await readFile(join(defaults.assets[0]!, "catalog/apps-index.json"), "utf8")); + const all = index.providers.map((entry: { provider: { service: string } }) => entry.provider.service).sort(); + await defaults.dispose(); + for (const providers of [undefined, ["github", "github"], []]) { + const expected = providers === undefined ? all : [...new Set(providers)].sort(); + const prepared = await getConnectorBuildOptions({ providers }); + const outfile = join(execution, "host"); + try { + const result = await Bun.build({ + entrypoints: [entrypoint], + target: "bun", + format: "esm", + splitting: true, + plugins: prepared.plugins, + external: prepared.external, + compile: { outfile, assets: prepared.assets }, + metafile: true, + }); + assert.ok(result.success, result.logs.join("\n")); + assert.ok(result.metafile); + const bundledProviders = new Set(); + for (const path of Object.keys(result.metafile.inputs)) { + const match = /\/src\/providers\/([^/]+)\//.exec(path.replaceAll("\\", "/")); + if (match) bundledProviders.add(match[1]!); + } + assert.deepEqual([...bundledProviders].sort(), expected); + } finally { + await prepared.dispose(); + } + if (providers !== undefined) await assert.rejects(readdir(prepared.assets[0]!)); + // Bun 1.4.0 writes an ad-hoc signature that macOS 27 refuses to start (oven-sh/bun#39837); re-sign it in place + // like scripts/build-binary.ts does so the smoke also runs on a macOS checkout. + if (process.platform === "darwin") { + const signed = Bun.spawnSync(["codesign", "--force", "--sign", "-", outfile], { stderr: "inherit" }); + assert.ok(signed.success, `codesign failed with exit code ${signed.exitCode}`); + } + const child = Bun.spawn([outfile, JSON.stringify(expected)], { + cwd: execution, + stdout: "inherit", + stderr: "inherit", + }); + assert.equal(await child.exited, 0); + await rm(join(execution, "data"), { recursive: true, force: true }); + console.log(`Compiled host passed with ${expected.length} providers.`); + } +} finally { + await rm(entrypoint, { force: true }); + await rm(execution, { recursive: true, force: true }); +} diff --git a/scripts/tsconfig.package.json b/scripts/tsconfig.package.json index 6ba21ac9..278c0ce9 100644 --- a/scripts/tsconfig.package.json +++ b/scripts/tsconfig.package.json @@ -6,5 +6,5 @@ "outDir": "../dist/package/src", "rewriteRelativeImportExtensions": true }, - "include": ["../src/server/connector-runtime.ts"] + "include": ["../src/server/connector-runtime.ts", "../src/server/connector-build.ts"] } diff --git a/src/server/connector-assets.ts b/src/server/connector-assets.ts new file mode 100644 index 00000000..c18a006d --- /dev/null +++ b/src/server/connector-assets.ts @@ -0,0 +1,14 @@ +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** Directory to include in a host's Bun compile.assets. Its basename keeps connector assets namespaced. */ +export function getConnectorAssetDirectory(): string { + return isStandaloneExecutable() + ? join(import.meta.dirname, "open-connector") + : fileURLToPath(new URL("../../assets/open-connector/", import.meta.url)); +} + +/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */ +export function isStandaloneExecutable(): boolean { + return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true; +} diff --git a/src/server/connector-build.ts b/src/server/connector-build.ts new file mode 100644 index 00000000..3fb8aba4 --- /dev/null +++ b/src/server/connector-build.ts @@ -0,0 +1,106 @@ +import type { CatalogIndexSource } from "../catalog-index.ts"; +import type { ProviderDefinition } from "../core/types.ts"; + +import { access, cp, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { extname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { catalogIndexFileName, createCatalogIndex } from "../catalog-index.ts"; +import { assertProviderId } from "../core/provider-id.ts"; +import { getConnectorAssetDirectory } from "./connector-assets.ts"; + +export interface ConnectorBuildOptions { + /** Omit to include all providers. An empty array includes none. */ + providers?: readonly string[]; +} + +export interface ConnectorBuild { + /** Pass to Bun.build's compile.assets. */ + assets: string[]; + /** Pass to Bun.build alongside plugins. */ + external: string[]; + plugins: ConnectorBuildPlugin[]; + /** Remove staged assets after the build, including when it fails. */ + dispose(): Promise; +} + +// Keep the Bun plugin boundary structural so importing this helper does not change Node's global types. +interface ConnectorBuildPlugin { + name: string; + setup(build: ConnectorBuildHooks): void; +} + +interface ConnectorBuildHooks { + onLoad( + options: { filter: RegExp }, + callback: (args: { path: string }) => { contents: string; loader: "js" } | undefined, + ): unknown; + onEnd(callback: (result: { success: boolean }) => void): unknown; +} + +/** Prepare a headless Bun executable with only the requested provider code and catalog entries. */ +export async function getConnectorBuildOptions(options: ConnectorBuildOptions = {}): Promise { + const sourceAssets = getConnectorAssetDirectory(); + const external = ["proxy-agent"]; + if (options.providers === undefined) { + return { assets: [sourceAssets], external, plugins: [], dispose: () => Promise.resolve() }; + } + + const services = [...new Set(options.providers)].sort(); + const catalogDir = join(sourceAssets, "catalog/apps"); + const files = new Set(await readdir(catalogDir)); + for (const service of services) { + assertProviderId(service); + if (!files.has(`${service}.json`)) throw new Error(`Unknown provider: ${service}`); + } + + const directory = await mkdtemp(join(tmpdir(), "open-connector-build-")); + const dispose = (): Promise => rm(directory, { recursive: true, force: true }); + try { + const assets = join(directory, "open-connector"); + const apps = join(assets, "catalog/apps"); + await mkdir(apps, { recursive: true }); + const providerDir = fileURLToPath(new URL("../providers/", import.meta.url)); + const extension = extname(import.meta.filename); + const registry = await realpath(join(providerDir, `registry.generated${extension}`)); + const sources: CatalogIndexSource[] = []; + for (const service of services) { + await access(join(providerDir, service, `executors${extension}`)); + const file = `${service}.json`; + const content = await readFile(join(catalogDir, file)); + const provider = JSON.parse(content.toString()) as ProviderDefinition; + if (provider.service !== service) throw new Error(`Catalog service does not match ${file}`); + await writeFile(join(apps, file), content); + sources.push({ file, bytes: content.byteLength, provider }); + } + // Bun embeds files, so preserve the directory that the runtime reads even with no providers. + if (!services.length) await writeFile(join(apps, "empty"), ""); + await writeFile(join(assets, "catalog", catalogIndexFileName), JSON.stringify(createCatalogIndex(sources))); + await cp(join(sourceAssets, "migrations"), join(assets, "migrations"), { recursive: true }); + const contents = `export const executorModules = {\n${services + .map( + (service) => + `${JSON.stringify(service)}: () => import(${JSON.stringify(`./${service}/executors${extension}`)}),`, + ) + .join("\n")}\n};`; + const plugin: ConnectorBuildPlugin = { + name: "open-connector-providers", + setup(build) { + let replaced = false; + build.onLoad({ filter: /[/\\]providers[/\\]registry\.generated\.[jt]s$/ }, ({ path }) => { + if (path !== registry) return; + replaced = true; + return { contents, loader: "js" }; + }); + build.onEnd((result) => { + if (result.success && !replaced) + throw new Error("The build did not load Open Connector's executor registry."); + }); + }, + }; + return { assets: [assets], external, plugins: [plugin], dispose }; + } catch (error) { + await dispose(); + throw error; + } +} diff --git a/src/server/connector-runtime.ts b/src/server/connector-runtime.ts index 5be3c17a..a4a7de1d 100644 --- a/src/server/connector-runtime.ts +++ b/src/server/connector-runtime.ts @@ -6,7 +6,6 @@ import type { IStagedTransitFileService } from "./files/transit-file-store.ts"; import { mkdir } from "node:fs/promises"; import { join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; import { loadCatalog } from "../catalog-store.ts"; import { ActionPolicyService } from "../core/action-policy.ts"; import { setEgressTrustedHosts, setPrivateNetworkAccessAllowed } from "../core/request.ts"; @@ -14,10 +13,10 @@ import { ProviderLoader } from "../providers/provider-loader.ts"; import { executorModules } from "../providers/registry.generated.ts"; import { createRuntimeJwtVerifier } from "./api/runtime-jwt.ts"; import { createConnectApp } from "./connect-app.ts"; +import { getConnectorAssetDirectory } from "./connector-assets.ts"; import { cleanupStagedTransitFiles, createNodeTransitFileUpload } from "./files/node-transit-file-upload.ts"; import { TransitFileService } from "./files/transit-files.ts"; import { createSecretCodec } from "./secrets/secret-codec.ts"; -import { isStandaloneExecutable } from "./server-assets.ts"; import { createDirectoryMigrationSource } from "./storage/migration-source.ts"; import { createNodeRuntimeDatabase } from "./storage/node-runtime-database.ts"; import { DEFAULT_RUN_LIMIT } from "./storage/runtime-store.ts"; @@ -108,12 +107,7 @@ export interface ConnectorRuntime { let runtimeActive = false; -/** Directory to include in a host's Bun compile.assets. Its basename keeps connector assets namespaced. */ -export function getConnectorAssetDirectory(): string { - return isStandaloneExecutable() - ? join(import.meta.dirname, "open-connector") - : fileURLToPath(new URL("../../assets/open-connector/", import.meta.url)); -} +export { getConnectorAssetDirectory } from "./connector-assets.ts"; /** Create a headless runtime without opening a listener or installing process signal handlers. */ export async function createConnectorRuntime(options: ConnectorRuntimeOptions): Promise { diff --git a/src/server/server-assets.ts b/src/server/server-assets.ts index 10178c9a..87676ad5 100644 --- a/src/server/server-assets.ts +++ b/src/server/server-assets.ts @@ -3,6 +3,7 @@ import type { ConnectorAssets } from "./connector-runtime.ts"; import { access } from "node:fs/promises"; import { join } from "node:path"; import { catalogIndexFileName } from "../catalog-index.ts"; +import { isStandaloneExecutable } from "./connector-assets.ts"; /** * Locations of the assets that are generated or built outside `src` and read @@ -51,11 +52,6 @@ export async function resolveServerAssets(): Promise { }; } -/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */ -export function isStandaloneExecutable(): boolean { - return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true; -} - async function resolveStaticRoot(root: string): Promise { const indexHtml = await resolveExistingFile(join(root, "index.html")); return indexHtml === undefined ? undefined : root;