mirror of
https://gh-proxy.org/https://github.com/klinsmaya/open-connector.git
synced 2026-11-08 01:16:52 +08:00
feat: compile the server into Bun single-file executables (#485)
Signed-off-by: Kevin Cui <[email protected]>
This commit is contained in:
Kevin Cui
@@ -0,0 +1 @@
|
||||
1.4.0
|
||||
@@ -0,0 +1,247 @@
|
||||
name: Build Binaries
|
||||
|
||||
# Builds the Bun single-file executables (`npm run build:binary`, six targets)
|
||||
# and smoke-tests every one of them on real hardware:
|
||||
#
|
||||
# build (Linux x64) -> smoke linux-arm64 (Blacksmith Ubuntu ARM)
|
||||
# -> smoke windows-x64 (Blacksmith Windows Server 2025)
|
||||
# -> smoke windows-arm64 (GitHub-hosted windows-11-arm)
|
||||
# -> smoke darwin-arm64 (Blacksmith macOS 15, Apple Silicon)
|
||||
# -> smoke darwin-x64 (GitHub-hosted macos-15-intel)
|
||||
#
|
||||
# All six targets are cross-compiled on ONE Linux runner: Bun downloads a
|
||||
# ~80 MB runtime per target and compiling is a matter of seconds, whereas
|
||||
# cross-compiling ON a Windows runner is broken upstream (oven-sh/bun#11198,
|
||||
# the workspace and the Bun cache sit on different drives). The Linux job also
|
||||
# smoke-tests its own linux-x64 output twice, once against SQLite and once
|
||||
# against the PostgreSQL service container (service containers are Linux-only,
|
||||
# which is one reason the smoke matrix is a separate job).
|
||||
#
|
||||
# The smoke matrix mixes runner vendors because Blacksmith offers no Windows
|
||||
# ARM64 and no macOS Intel runners; those two targets run on GitHub-hosted
|
||||
# runners, which are free for public repositories. Every smoke job only
|
||||
# downloads its artifact and runs `node scripts/smoke-binary.ts`, so it needs
|
||||
# neither `npm ci` nor Bun.
|
||||
#
|
||||
# The build job's `npm run build:binary` runs `npm run build:web`, which fetches
|
||||
# https://oomol.com/en/apps/catalog.json (the provider icon map) at build time,
|
||||
# so an outage there fails the build job. This is the first PR-gated path
|
||||
# through the web build.
|
||||
#
|
||||
# darwin binaries built on Linux carry an invalid ad-hoc signature (Bun 1.4.0
|
||||
# writes a wrong page hash, which macOS 27 rejects), so the macOS smoke jobs
|
||||
# re-sign the binary with `codesign` and verify the signature before running it.
|
||||
#
|
||||
# A `runs-on` label that no runner serves queues forever instead of failing, so
|
||||
# `workflow_dispatch` is here to verify the labels before relying on them.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
# Cancel superseded runs for the same ref (e.g. new push to an open PR).
|
||||
concurrency:
|
||||
group: build-binary-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
# Least privilege: these jobs only read the repository; the binaries are
|
||||
# workflow artifacts, nothing is published.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
# This project runs TypeScript directly with `node` (native type stripping) and
|
||||
# forbids `tsx` / `--experimental-strip-types` (see AGENTS.md). Type stripping is
|
||||
# on by default only on Node >= 22.18 / >= 23.6, so pin an LTS that supports it.
|
||||
# Do NOT downgrade below 22.18 — `npm ci` (postinstall codegen) and typecheck fail.
|
||||
NODE_VERSION: "24"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build all targets
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout-minutes: 30
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_DB: open_connector_test
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_USER: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# No step needs the persisted GITHUB_TOKEN in git config; dropping it
|
||||
# keeps `npm ci` postinstall scripts from reading it (zizmor hardening).
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: npm
|
||||
|
||||
# Pinned to an immutable SHA rather than the mutable v2 tag: this
|
||||
# third-party action installs the toolchain that produces the shipped
|
||||
# binaries, so its supply chain must not be silently updatable.
|
||||
# `.bun-version` is the single owner of the pinned Bun version; the build
|
||||
# script refuses to run under any other version.
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version-file: .bun-version
|
||||
|
||||
# Bun downloads one runtime per cross-compile target (~80 MB each) from
|
||||
# registry.npmjs.org into ~/.bun/install/cache; setup-bun caches only the
|
||||
# host executable. The key changes with the pinned Bun version.
|
||||
- name: Cache Bun cross-compile runtimes
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.bun/install/cache/bun-*-v*
|
||||
key: bun-runtimes-${{ runner.os }}-${{ hashFiles('.bun-version') }}
|
||||
|
||||
# Runs postinstall codegen (provider registry + catalog).
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
# Regenerates the catalog, builds the web console, then compiles all six targets.
|
||||
- name: Build binaries
|
||||
run: npm run build:binary
|
||||
|
||||
# SQLite mode: the embedded catalog, migrations and console all come from the binary.
|
||||
- name: Smoke test linux-x64 (SQLite)
|
||||
run: node scripts/smoke-binary.ts dist/open-connector-linux-x64
|
||||
|
||||
# PostgreSQL mode: the binary's own `migrate` subcommand applies the embedded
|
||||
# postgresql/ migrations, then the server must pass its schema check.
|
||||
- name: Migrate PostgreSQL with linux-x64
|
||||
env:
|
||||
OOMOL_CONNECT_DATABASE_URL: postgresql://postgres:[email protected]:5432/open_connector_test
|
||||
run: dist/open-connector-linux-x64 migrate
|
||||
|
||||
- name: Smoke test linux-x64 (PostgreSQL)
|
||||
env:
|
||||
OOMOL_CONNECT_DATABASE_URL: postgresql://postgres:[email protected]:5432/open_connector_test
|
||||
run: node scripts/smoke-binary.ts dist/open-connector-linux-x64
|
||||
|
||||
# One artifact per target so each smoke job downloads only its own ~35 MB
|
||||
# zip. upload-artifact drops the exec bit; the smoke jobs chmod after download.
|
||||
- name: Upload linux-x64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-linux-x64
|
||||
path: dist/open-connector-linux-x64
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload linux-arm64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-linux-arm64
|
||||
path: dist/open-connector-linux-arm64
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload darwin-x64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-darwin-x64
|
||||
path: dist/open-connector-darwin-x64
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload darwin-arm64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-darwin-arm64
|
||||
path: dist/open-connector-darwin-arm64
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload windows-x64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-windows-x64
|
||||
path: dist/open-connector-windows-x64.exe
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload windows-arm64
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: binary-windows-arm64
|
||||
path: dist/open-connector-windows-arm64.exe
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Run each remaining target on its native OS and architecture. linux-x64 was
|
||||
# already exercised by the build job.
|
||||
smoke:
|
||||
name: Smoke ${{ matrix.target }}
|
||||
needs: build
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- target: linux-arm64
|
||||
runner: blacksmith-2vcpu-ubuntu-2404-arm
|
||||
file: open-connector-linux-arm64
|
||||
- target: windows-x64
|
||||
runner: blacksmith-2vcpu-windows-2025
|
||||
file: open-connector-windows-x64.exe
|
||||
# Blacksmith has no Windows ARM64 runners.
|
||||
- target: windows-arm64
|
||||
runner: windows-11-arm
|
||||
file: open-connector-windows-arm64.exe
|
||||
- target: darwin-arm64
|
||||
runner: blacksmith-6vcpu-macos-15
|
||||
file: open-connector-darwin-arm64
|
||||
# Blacksmith has no macOS Intel runners.
|
||||
- target: darwin-x64
|
||||
runner: macos-15-intel
|
||||
file: open-connector-darwin-x64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# The smoke script uses only node: builtins, so no `npm ci` is needed.
|
||||
- name: Setup Node.js ${{ env.NODE_VERSION }}
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Download ${{ matrix.target }} binary
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: binary-${{ matrix.target }}
|
||||
path: dist
|
||||
|
||||
# Artifact zips do not preserve file modes.
|
||||
- name: Restore executable bit
|
||||
if: runner.os != 'Windows'
|
||||
run: chmod +x dist/${{ matrix.file }}
|
||||
|
||||
# Linux-built darwin binaries carry an invalid ad-hoc signature (Bun 1.4.0
|
||||
# writes a wrong page hash); re-sign them and assert the signature is valid,
|
||||
# since macOS 15 would tolerate an invalid one silently.
|
||||
- name: Ad-hoc sign and verify
|
||||
if: runner.os == 'macOS'
|
||||
run: |
|
||||
codesign --force --sign - dist/${{ matrix.file }}
|
||||
codesign --verify --verbose=2 dist/${{ matrix.file }}
|
||||
|
||||
- name: Smoke test ${{ matrix.target }}
|
||||
run: node scripts/smoke-binary.ts dist/${{ matrix.file }}
|
||||
@@ -275,6 +275,7 @@ Issues and pull requests are welcome.
|
||||
- [Fly.io deployment](docs/fly-io.md)
|
||||
- [Cloudflare deployment](docs/cloudflare.md)
|
||||
- [Docker image (GHCR)](docs/docker-ghcr.md)
|
||||
- [Single binary](docs/single-binary.md)
|
||||
- [Configuration](docs/configuration.md)
|
||||
- [Credentials and OAuth](docs/credentials.md)
|
||||
- [Catalog format](docs/catalog-format.md)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# Single Binary
|
||||
|
||||
OpenConnector can be compiled into one self-contained executable per platform with
|
||||
[Bun](https://bun.com/docs/bundler/executables). The binary embeds the runtime, the generated
|
||||
provider catalog, the database migrations, and the built web console, so it runs from any directory
|
||||
without a Node.js installation, a checkout, or `node_modules`. Nothing is extracted to disk at
|
||||
runtime.
|
||||
|
||||
## Build
|
||||
|
||||
Building requires Node.js for the npm scripts and the Bun version pinned in `.bun-version`; the
|
||||
build script exits with an error under any other Bun version. Compile on Linux or macOS.
|
||||
Cross-compiling on a Windows host fails upstream
|
||||
([oven-sh/bun#11198](https://github.com/oven-sh/bun/issues/11198)).
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build:binary
|
||||
```
|
||||
|
||||
`npm run build:binary` regenerates the catalog, builds the web console, and then writes six files
|
||||
under `dist/`:
|
||||
|
||||
```text
|
||||
dist/open-connector-linux-x64
|
||||
dist/open-connector-linux-arm64
|
||||
dist/open-connector-darwin-x64
|
||||
dist/open-connector-darwin-arm64
|
||||
dist/open-connector-windows-x64.exe
|
||||
dist/open-connector-windows-arm64.exe
|
||||
```
|
||||
|
||||
Each file is roughly 145 to 170 MiB. To build a subset, pass one or more target names after `--`:
|
||||
|
||||
```bash
|
||||
npm run build:binary -- linux-x64 darwin-arm64
|
||||
```
|
||||
|
||||
Bun downloads a runtime for every target that differs from the host (about 80 MB each, from
|
||||
`registry.npmjs.org`) into `~/.bun/install/cache` on first use. These downloads are not integrity
|
||||
checked by Bun; TLS is the only protection.
|
||||
|
||||
`npm run build:web` also fetches the provider icon map from https://oomol.com/en/apps/catalog.json
|
||||
and fails when it is unreachable.
|
||||
|
||||
`.bun-version` and the `@types/bun` devDependency pin in `package.json` are bumped together.
|
||||
|
||||
## Run
|
||||
|
||||
The binary takes the same environment variables as `npm start`; see
|
||||
[configuration.md](configuration.md) for the full reference. The ones you will usually set:
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
| ---------------------------- | ----------- | ----------------------------------------------------------------- |
|
||||
| `OOMOL_CONNECT_DATA_DIR` | `./data` | SQLite database, transit files, and upload staging. |
|
||||
| `PORT` | `3000` | HTTP port. |
|
||||
| `HOST` | `127.0.0.1` | Bind address. |
|
||||
| `OOMOL_CONNECT_DATABASE_URL` | unset | PostgreSQL connection URL. When unset, SQLite under the data dir. |
|
||||
|
||||
```bash
|
||||
OOMOL_CONNECT_DATA_DIR="$HOME/open-connector-data" \
|
||||
PORT=3000 \
|
||||
./dist/open-connector-linux-x64
|
||||
```
|
||||
|
||||
With SQLite, migrations are applied automatically when the database opens, exactly as with
|
||||
`npm start`.
|
||||
|
||||
### PostgreSQL Migrations
|
||||
|
||||
PostgreSQL migrations are explicit. The binary has a `migrate` subcommand that applies the embedded
|
||||
migrations and exits without starting the server. Run it before the first start and before starting
|
||||
a newer binary that contains pending migrations:
|
||||
|
||||
```bash
|
||||
OOMOL_CONNECT_DATABASE_URL="postgresql://open_connector:[email protected]:5432/open_connector?sslmode=verify-full" \
|
||||
./dist/open-connector-linux-x64 migrate
|
||||
|
||||
OOMOL_CONNECT_DATABASE_URL="postgresql://open_connector:[email protected]:5432/open_connector?sslmode=verify-full" \
|
||||
./dist/open-connector-linux-x64
|
||||
```
|
||||
|
||||
The server only checks schema readiness at startup and refuses to start when migrations are
|
||||
missing; it never applies PostgreSQL DDL itself. Without `OOMOL_CONNECT_DATABASE_URL`, `migrate`
|
||||
prints a notice that SQLite migrations are applied automatically and exits.
|
||||
|
||||
## Differences From `npm start`
|
||||
|
||||
- `NODE_ENV` is fixed to `production` inside the binary, so logs are always JSON (no pretty
|
||||
printing). `OOMOL_CONNECT_LOG_LEVEL` and every other environment variable are read at runtime as
|
||||
usual.
|
||||
- macOS: binaries built on macOS are ad-hoc signed by the build script. Binaries built on another
|
||||
operating system carry an invalid ad-hoc signature, and macOS 27 and newer refuses to run them
|
||||
until you re-sign them:
|
||||
|
||||
```bash
|
||||
codesign --force --sign - dist/open-connector-darwin-arm64
|
||||
```
|
||||
|
||||
`codesign --verify --verbose=2 <file>` prints "valid on disk" for a usable binary and "invalid
|
||||
signature" for one that still needs re-signing.
|
||||
|
||||
## Notes
|
||||
|
||||
- Like `npm start`, the binary does not load `.env` files (Bun's automatic loading is disabled at
|
||||
build time).
|
||||
- On Windows, stopping the process from a process manager or `taskkill` terminates it immediately;
|
||||
the graceful shutdown hook that closes the HTTP server and the database on Linux and macOS does
|
||||
not run. This matches `node src/server/index.ts` on Windows.
|
||||
Generated
+21
@@ -45,6 +45,7 @@
|
||||
"@electric-sql/pglite-socket": "^0.2.8",
|
||||
"@hyrious/configs": "^0.1.6",
|
||||
"@types/ali-oss": "^6.23.3",
|
||||
"@types/bun": "1.4.0",
|
||||
"@types/busboy": "^1.5.4",
|
||||
"@types/mailparser": "^3.4.6",
|
||||
"@types/mdast": "^4.0.4",
|
||||
@@ -4125,6 +4126,16 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/bun": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.4.0.tgz",
|
||||
"integrity": "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bun-types": "1.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/busboy": {
|
||||
"version": "1.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz",
|
||||
@@ -5115,6 +5126,16 @@
|
||||
"integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bun-types": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.4.0.tgz",
|
||||
"integrity": "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"dev": "node scripts/dev-local.ts",
|
||||
"build": "npm run typecheck",
|
||||
"build:web": "npm run build --workspace web",
|
||||
"build:binary": "npm run generate:catalog && npm run build:web && bun scripts/build-binary.ts",
|
||||
"dev:cloudflare": "npm run generate:catalog && npm run build:web && node scripts/copy-catalog-assets.ts && wrangler dev --config wrangler.local.jsonc",
|
||||
"deploy:cloudflare": "npm run generate:catalog && npm run build:web && node scripts/copy-catalog-assets.ts && wrangler deploy --config wrangler.local.jsonc --minify",
|
||||
"start": "node scripts/ensure-generated.ts && node src/server/index.ts",
|
||||
@@ -61,6 +62,7 @@
|
||||
"@electric-sql/pglite-socket": "^0.2.8",
|
||||
"@hyrious/configs": "^0.1.6",
|
||||
"@types/ali-oss": "^6.23.3",
|
||||
"@types/bun": "1.4.0",
|
||||
"@types/busboy": "^1.5.4",
|
||||
"@types/mailparser": "^3.4.6",
|
||||
"@types/mdast": "^4.0.4",
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
|
||||
// Build the single-file server executables with Bun.
|
||||
//
|
||||
// Usage: `bun scripts/build-binary.ts [target-name ...]` (normally through `npm run build:binary`, which
|
||||
// regenerates the catalog and builds the web console first). No arguments builds every target.
|
||||
|
||||
interface BinaryTarget {
|
||||
/** Suffix of the output file: `dist/open-connector-<name>` (Bun appends `.exe` for Windows targets). */
|
||||
name: string;
|
||||
target: Bun.Build.CompileTarget;
|
||||
}
|
||||
|
||||
const rootDir = resolve(import.meta.dirname, "..");
|
||||
const bunVersionFile = join(rootDir, ".bun-version");
|
||||
const targets: BinaryTarget[] = [
|
||||
{ name: "linux-x64", target: "bun-linux-x64" },
|
||||
{ name: "linux-arm64", target: "bun-linux-arm64" },
|
||||
{ name: "darwin-x64", target: "bun-darwin-x64" },
|
||||
{ name: "darwin-arm64", target: "bun-darwin-arm64" },
|
||||
{ name: "windows-x64", target: "bun-windows-x64" },
|
||||
{ name: "windows-arm64", target: "bun-windows-arm64" },
|
||||
];
|
||||
|
||||
assertPinnedBunVersion();
|
||||
assertBuildInputs();
|
||||
for (const target of selectTargets(process.argv.slice(2))) {
|
||||
await buildTarget(target);
|
||||
}
|
||||
|
||||
/** `.bun-version` is the single owner of the pinned Bun version; CI installs it and local builds must match it. */
|
||||
function assertPinnedBunVersion(): void {
|
||||
const pinnedVersion = readFileSync(bunVersionFile, "utf8").trim();
|
||||
if (Bun.version !== pinnedVersion) {
|
||||
fail(
|
||||
`Bun ${Bun.version} is running but .bun-version pins ${pinnedVersion}. Install it with: curl -fsSL https://bun.sh/install | bash -s "bun-v${pinnedVersion}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** The embedded directories are generated; refuse to build a binary that would ship an empty catalog or console. */
|
||||
function assertBuildInputs(): void {
|
||||
const problems: string[] = [];
|
||||
if (listFiles(join(rootDir, "catalog/apps"), ".json").length === 0) {
|
||||
problems.push("catalog/apps contains no .json files");
|
||||
}
|
||||
if (listFiles(join(rootDir, "migrations"), ".sql").length === 0) {
|
||||
problems.push("migrations contains no .sql files");
|
||||
}
|
||||
if (listFiles(join(rootDir, "migrations/postgresql"), ".sql").length === 0) {
|
||||
problems.push("migrations/postgresql contains no .sql files");
|
||||
}
|
||||
if (!existsSync(join(rootDir, "dist/web/index.html"))) {
|
||||
problems.push("dist/web/index.html is missing");
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
fail(
|
||||
`Build inputs are missing:\n - ${problems.join("\n - ")}\nRun "npm run build:binary" so the catalog and the web console are generated before the binary is built.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function listFiles(directory: string, extension: string): string[] {
|
||||
try {
|
||||
return readdirSync(directory).filter((name) => name.endsWith(extension));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function selectTargets(names: string[]): BinaryTarget[] {
|
||||
if (names.length === 0) {
|
||||
return targets;
|
||||
}
|
||||
|
||||
return names.map((name) => {
|
||||
const target = targets.find((candidate) => candidate.name === name);
|
||||
if (!target) {
|
||||
fail(`Unknown target "${name}". Valid targets: ${targets.map((candidate) => candidate.name).join(", ")}.`);
|
||||
}
|
||||
|
||||
return target;
|
||||
});
|
||||
}
|
||||
|
||||
async function buildTarget(binaryTarget: BinaryTarget): Promise<void> {
|
||||
const outfile = join(rootDir, "dist", `open-connector-${binaryTarget.name}`);
|
||||
// Bun appends .exe to Windows outputs; clear both spellings so an earlier build can never be mistaken for this one.
|
||||
rmSync(outfile, { force: true });
|
||||
rmSync(`${outfile}.exe`, { force: true });
|
||||
|
||||
const result = await runBunBuild(binaryTarget.target, outfile);
|
||||
const [artifact] = result.outputs;
|
||||
if (!artifact) {
|
||||
fail(`Bun.build produced no output for ${binaryTarget.name}.`);
|
||||
}
|
||||
|
||||
if (binaryTarget.target.startsWith("bun-darwin-")) {
|
||||
signDarwinBinary(artifact.path);
|
||||
}
|
||||
|
||||
const sizeMiB = statSync(artifact.path).size / (1024 * 1024);
|
||||
console.log(`built ${relative(rootDir, artifact.path)} (${sizeMiB.toFixed(1)} MiB)`);
|
||||
}
|
||||
|
||||
async function runBunBuild(target: Bun.Build.CompileTarget, outfile: string): Promise<Bun.BuildOutput> {
|
||||
let result: Bun.BuildOutput;
|
||||
try {
|
||||
result = await Bun.build({
|
||||
entrypoints: [join(rootDir, "src/server/index.ts")],
|
||||
target: "bun",
|
||||
format: "esm",
|
||||
// ali-oss depends on urllib, which lazily `require("proxy-agent")`, an optional peer dependency that is not
|
||||
// installed here. The bundler cannot resolve it, so it stays a runtime require that is never reached.
|
||||
external: ["proxy-agent"],
|
||||
// Bun inlines `process.env.NODE_ENV` at compile time ("development" unless defined). src/server/logger.ts then
|
||||
// loads the pino-pretty worker transport, which cannot run inside a standalone executable. Every other
|
||||
// environment variable is still read at runtime.
|
||||
define: { "process.env.NODE_ENV": JSON.stringify("production") },
|
||||
compile: {
|
||||
target,
|
||||
outfile,
|
||||
// Each directory is embedded under its basename next to the bundle: migrations/, apps/ and web/.
|
||||
// catalog/apps rather than catalog/: an interrupted `npm run generate:catalog` leaves catalog/.apps-<pid>-<ts>
|
||||
// temp directories behind, and they must never end up inside a release.
|
||||
assets: [join(rootDir, "migrations"), join(rootDir, "catalog/apps"), join(rootDir, "dist/web")],
|
||||
// `node src/server/index.ts` reads neither .env nor bunfig.toml; keep the binary's configuration surface the same.
|
||||
autoloadDotenv: false,
|
||||
autoloadBunfig: false,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
// Bun.build rejects with an AggregateError whose `errors` hold the BuildMessage / ResolveMessage entries.
|
||||
if (error instanceof AggregateError) {
|
||||
for (const message of error.errors) {
|
||||
console.error(describeBuildMessage(message));
|
||||
}
|
||||
fail(`Bundling ${target} failed.`);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
for (const message of result.logs) {
|
||||
console.error(describeBuildMessage(message));
|
||||
}
|
||||
if (!result.success) {
|
||||
fail(`Bundling ${target} failed.`);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function describeBuildMessage(message: unknown): string {
|
||||
if (message instanceof BuildMessage || message instanceof ResolveMessage) {
|
||||
const position = message.position;
|
||||
const location = position ? ` (${position.file}:${position.line}:${position.column})` : "";
|
||||
return `${message.level}: ${message.message}${location}`;
|
||||
}
|
||||
|
||||
return String(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bun 1.4.0 writes an ad-hoc signature whose last page hash is wrong (oven-sh/bun#39837, fixed upstream but
|
||||
* unreleased). macOS 27 refuses to start such a binary (SIGKILL at exec), so re-sign it in place. codesign only
|
||||
* exists on macOS; darwin outputs built elsewhere are re-signed on the macOS smoke runner instead.
|
||||
*/
|
||||
function signDarwinBinary(output: string): void {
|
||||
const displayPath = relative(rootDir, output);
|
||||
if (process.platform !== "darwin") {
|
||||
console.warn(
|
||||
`warning: ${displayPath} still carries Bun's invalid ad-hoc signature because codesign is only available on macOS; run "codesign --force --sign - ${displayPath}" on macOS 27 or later before executing it.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const result = spawnSync("codesign", ["--force", "--sign", "-", output], { stdio: "inherit" });
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
fail(
|
||||
`codesign failed for ${displayPath} (${result.signal ? `signal ${result.signal}` : `exit code ${result.status}`}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function fail(message: string): never {
|
||||
console.error(message);
|
||||
process.exit(1);
|
||||
}
|
||||
+9
-14
@@ -1,11 +1,13 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { Pool } from "pg";
|
||||
import { logger } from "../src/server/logger.ts";
|
||||
import { createSecretCodec } from "../src/server/secrets/secret-codec.ts";
|
||||
import { assertPostgresDatabaseUrl, createNodeRuntimeDatabase } from "../src/server/storage/node-runtime-database.ts";
|
||||
import { migratePostgresDatabase } from "../src/server/storage/postgres-migrations.ts";
|
||||
import {
|
||||
createNodeRuntimeDatabase,
|
||||
migratePostgresRuntimeDatabase,
|
||||
sqliteMigrationsNotice,
|
||||
} from "../src/server/storage/node-runtime-database.ts";
|
||||
|
||||
const { positionals, values: options } = parseArgs({
|
||||
args: process.argv.slice(2),
|
||||
@@ -49,20 +51,13 @@ const dataDir = resolve(options["data-dir"] ?? process.env.OOMOL_CONNECT_DATA_DI
|
||||
const databasePath = join(dataDir, "connect.sqlite");
|
||||
if (command === "migrate") {
|
||||
if (!databaseUrl) {
|
||||
console.log("SQLite migrations are applied automatically when the local runtime database opens.");
|
||||
console.log(sqliteMigrationsNotice);
|
||||
} else {
|
||||
assertPostgresDatabaseUrl(databaseUrl);
|
||||
const pool = new Pool({
|
||||
application_name: "open-connector-migrate",
|
||||
await migratePostgresRuntimeDatabase({
|
||||
connectionString: databaseUrl,
|
||||
connectionTimeoutMillis: readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000),
|
||||
max: 1,
|
||||
connectionTimeoutMs: readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000),
|
||||
logger,
|
||||
});
|
||||
try {
|
||||
await migratePostgresDatabase({ pool, logger });
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const secretCodec = createSecretCodec(process.env.OOMOL_CONNECT_ENCRYPTION_KEY);
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
|
||||
import { strict as assert } from "node:assert";
|
||||
import { spawn } from "node:child_process";
|
||||
import { access, mkdtemp, rm } from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
|
||||
// Start a built single-file executable against a fresh data directory and check that its embedded catalog,
|
||||
// web console, migrations and shutdown path work.
|
||||
//
|
||||
// Usage: `node scripts/smoke-binary.ts <path-to-binary>`. Set OOMOL_CONNECT_DATABASE_URL to run the PostgreSQL
|
||||
// mode; every other OOMOL_CONNECT_* variable is removed from the server's environment. Uses only Node built-ins so
|
||||
// the smoke runners need no `npm ci`.
|
||||
|
||||
interface ProcessExit {
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
}
|
||||
|
||||
interface ServerProcessOptions {
|
||||
binaryPath: string;
|
||||
dataDir: string;
|
||||
port: number;
|
||||
databaseUrl: string | undefined;
|
||||
}
|
||||
|
||||
const healthTimeoutMs = 60_000;
|
||||
const healthPollIntervalMs = 500;
|
||||
const requestTimeoutMs = 10_000;
|
||||
const shutdownTimeoutMs = 10_000;
|
||||
|
||||
/** The server binary with its captured output and exit state. */
|
||||
class ServerProcess {
|
||||
private readonly child: ChildProcess;
|
||||
private readonly stdoutChunks: Buffer[] = [];
|
||||
private readonly stderrChunks: Buffer[] = [];
|
||||
private readonly closed: Promise<void>;
|
||||
private exitState: ProcessExit | undefined;
|
||||
private spawnError: Error | undefined;
|
||||
|
||||
constructor(options: ServerProcessOptions) {
|
||||
this.child = spawn(options.binaryPath, [], {
|
||||
cwd: options.dataDir,
|
||||
env: buildServerEnvironment(options),
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
this.child.stdout?.on("data", (chunk: Buffer) => this.stdoutChunks.push(chunk));
|
||||
this.child.stderr?.on("data", (chunk: Buffer) => this.stderrChunks.push(chunk));
|
||||
this.closed = new Promise((resolveClosed) => {
|
||||
this.child.once("error", (error) => {
|
||||
this.spawnError = error;
|
||||
this.exitState ??= { code: null, signal: null };
|
||||
resolveClosed();
|
||||
});
|
||||
this.child.once("close", (code, signal) => {
|
||||
this.exitState = { code, signal };
|
||||
resolveClosed();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
get exit(): ProcessExit | undefined {
|
||||
return this.exitState;
|
||||
}
|
||||
|
||||
get stdout(): string {
|
||||
return Buffer.concat(this.stdoutChunks).toString("utf8");
|
||||
}
|
||||
|
||||
get stderr(): string {
|
||||
return Buffer.concat(this.stderrChunks).toString("utf8");
|
||||
}
|
||||
|
||||
describeExit(): string {
|
||||
if (this.spawnError) {
|
||||
return `spawn failed: ${this.spawnError.message}`;
|
||||
}
|
||||
if (!this.exitState) {
|
||||
return "still running";
|
||||
}
|
||||
|
||||
return this.exitState.signal ? `signal ${this.exitState.signal}` : `exit code ${this.exitState.code}`;
|
||||
}
|
||||
|
||||
kill(signal: NodeJS.Signals): void {
|
||||
this.child.kill(signal);
|
||||
}
|
||||
|
||||
/** Resolve with the exit state, or undefined when the process is still running after the timeout. */
|
||||
async waitForExit(timeoutMs: number): Promise<ProcessExit | undefined> {
|
||||
await Promise.race([this.closed, sleep(timeoutMs, undefined, { ref: false })]);
|
||||
return this.exitState;
|
||||
}
|
||||
|
||||
/** Terminate a process that is still running; no-op after it exited. */
|
||||
async forceStop(): Promise<void> {
|
||||
if (this.exitState) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.kill("SIGKILL");
|
||||
await this.waitForExit(shutdownTimeoutMs);
|
||||
}
|
||||
}
|
||||
|
||||
const binaryPath = await resolveBinaryPath(process.argv[2]);
|
||||
const databaseUrl = process.env.OOMOL_CONNECT_DATABASE_URL?.trim() || undefined;
|
||||
const mode = databaseUrl ? "postgresql" : "sqlite";
|
||||
const startedAt = performance.now();
|
||||
// Probe the port before creating the data directory so a probe failure cannot leak the temp directory.
|
||||
const port = await findFreePort();
|
||||
const dataDir = await mkdtemp(join(tmpdir(), "open-connector-smoke-"));
|
||||
const baseUrl = `http://127.0.0.1:${port}`;
|
||||
const server = new ServerProcess({ binaryPath, dataDir, port, databaseUrl });
|
||||
|
||||
try {
|
||||
await waitForHealth(server, baseUrl);
|
||||
const readyAt = performance.now();
|
||||
const indexHtml = await checkConsoleIndex(baseUrl);
|
||||
await checkConsoleAssets(baseUrl, indexHtml);
|
||||
await checkProviders(baseUrl);
|
||||
await checkApps(baseUrl);
|
||||
await checkDatabaseBackend(server, dataDir, databaseUrl);
|
||||
await checkGracefulShutdown(server);
|
||||
await removeDataDir(dataDir);
|
||||
console.log(
|
||||
`PASS ${binaryPath} mode=${mode} startup=${formatMs(readyAt - startedAt)} total=${formatMs(performance.now() - startedAt)}`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`FAIL ${binaryPath}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
printServerLogs(server);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await server.forceStop();
|
||||
await removeDataDir(dataDir).catch(() => undefined);
|
||||
}
|
||||
|
||||
/** Resolve the binary before spawning: spawn resolves relative paths against the child's cwd, which is the data dir. */
|
||||
async function resolveBinaryPath(argument: string | undefined): Promise<string> {
|
||||
if (!argument) {
|
||||
console.error("Usage: node scripts/smoke-binary.ts <path-to-binary>");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const resolved = resolve(argument);
|
||||
try {
|
||||
await access(resolved);
|
||||
} catch {
|
||||
console.error(`Binary not found: ${resolved}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** process.env without OOMOL_CONNECT_* so the caller's shell cannot leak configuration into the server under test. */
|
||||
function buildServerEnvironment(options: ServerProcessOptions): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = {};
|
||||
for (const [key, value] of Object.entries(process.env)) {
|
||||
if (!key.toUpperCase().startsWith("OOMOL_CONNECT_")) {
|
||||
env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
env.PORT = String(options.port);
|
||||
env.HOST = "127.0.0.1";
|
||||
env.OOMOL_CONNECT_DATA_DIR = options.dataDir;
|
||||
if (options.databaseUrl) {
|
||||
env.OOMOL_CONNECT_DATABASE_URL = options.databaseUrl;
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
function findFreePort(): Promise<number> {
|
||||
return new Promise((resolvePort, reject) => {
|
||||
const probe = createServer();
|
||||
probe.once("error", reject);
|
||||
probe.listen(0, "127.0.0.1", () => {
|
||||
const address = probe.address();
|
||||
if (!address || typeof address === "string") {
|
||||
probe.close();
|
||||
reject(new Error("Could not determine a free port."));
|
||||
return;
|
||||
}
|
||||
|
||||
const { port: freePort } = address;
|
||||
probe.close((error) => (error ? reject(error) : resolvePort(freePort)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForHealth(server: ServerProcess, baseUrl: string): Promise<void> {
|
||||
const deadline = Date.now() + healthTimeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (server.exit) {
|
||||
throw new Error(`server exited before /health responded (${server.describeExit()})`);
|
||||
}
|
||||
if (await isHealthy(baseUrl)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await sleep(healthPollIntervalMs);
|
||||
}
|
||||
|
||||
throw new Error(`/health did not respond within ${healthTimeoutMs} ms`);
|
||||
}
|
||||
|
||||
async function isHealthy(baseUrl: string): Promise<boolean> {
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(healthPollIntervalMs * 4) });
|
||||
await response.text();
|
||||
return response.status === 200;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkConsoleIndex(baseUrl: string): Promise<string> {
|
||||
const response = await fetch(`${baseUrl}/`, { signal: AbortSignal.timeout(requestTimeoutMs) });
|
||||
const body = await response.text();
|
||||
assert(response.status === 200, `GET / returned ${response.status}`);
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
assert(contentType.startsWith("text/html"), `GET / content-type is ${contentType || "missing"}`);
|
||||
assert(body.includes("<script"), "GET / body contains no <script tag");
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SPA fallback answers any unknown console path with index.html, so GET / proves nothing about asset serving.
|
||||
* Fetching the hashed files index.html references proves the embedded static map, including its path separators.
|
||||
*/
|
||||
async function checkConsoleAssets(baseUrl: string, indexHtml: string): Promise<void> {
|
||||
const assetPaths = collectAssetPaths(indexHtml);
|
||||
assert(assetPaths.length > 0, 'GET / references no src="/..." or href="/..." assets');
|
||||
for (const assetPath of assetPaths) {
|
||||
const response = await fetch(`${baseUrl}${assetPath}`, { signal: AbortSignal.timeout(requestTimeoutMs) });
|
||||
const body = await response.arrayBuffer();
|
||||
assert(response.status === 200, `GET ${assetPath} returned ${response.status}`);
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
assert(
|
||||
!contentType.startsWith("text/html"),
|
||||
`GET ${assetPath} was answered by the SPA fallback (content-type ${contentType || "missing"})`,
|
||||
);
|
||||
const contentLength = response.headers.get("content-length");
|
||||
assert(
|
||||
Number(contentLength) > 0 && Number(contentLength) === body.byteLength,
|
||||
`GET ${assetPath} Content-Length ${contentLength ?? "missing"} does not match the ${body.byteLength}-byte body`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Root-relative `src` and `href` values; protocol-relative `//host/...` URLs are not served by the binary. */
|
||||
function collectAssetPaths(html: string): string[] {
|
||||
const paths = new Set<string>();
|
||||
for (const match of html.matchAll(/\b(?:src|href)="(\/[^"]*)"/g)) {
|
||||
if (!match[1].startsWith("//")) {
|
||||
paths.add(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
async function checkProviders(baseUrl: string): Promise<void> {
|
||||
const data = await fetchEnvelopeData(baseUrl, "/v1/providers");
|
||||
assert(Array.isArray(data), "/v1/providers data is not an array");
|
||||
assert(data.length > 1000, `/v1/providers returned ${data.length} providers; expected more than 1000`);
|
||||
assert(
|
||||
data.some((entry) => isRecord(entry) && entry.service === "slack"),
|
||||
"/v1/providers does not include slack",
|
||||
);
|
||||
}
|
||||
|
||||
async function checkApps(baseUrl: string): Promise<void> {
|
||||
const data = await fetchEnvelopeData(baseUrl, "/v1/apps");
|
||||
assert(Array.isArray(data), "/v1/apps data is not an array");
|
||||
}
|
||||
|
||||
/** Return `data` of a `{ success: true, data }` envelope. */
|
||||
async function fetchEnvelopeData(baseUrl: string, path: string): Promise<unknown> {
|
||||
const response = await fetch(`${baseUrl}${path}`, { signal: AbortSignal.timeout(requestTimeoutMs) });
|
||||
const body = await response.text();
|
||||
assert(response.status === 200, `GET ${path} returned ${response.status}`);
|
||||
const payload: unknown = JSON.parse(body);
|
||||
assert(isRecord(payload), `GET ${path} did not return a JSON object`);
|
||||
assert(payload.success === true, `GET ${path} envelope success is ${JSON.stringify(payload.success)}`);
|
||||
return payload.data;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
async function checkDatabaseBackend(
|
||||
server: ServerProcess,
|
||||
dataDir: string,
|
||||
databaseUrl: string | undefined,
|
||||
): Promise<void> {
|
||||
if (databaseUrl) {
|
||||
assert(server.stdout.includes('"backend":"postgresql"'), 'server log has no "backend":"postgresql" line');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await access(join(dataDir, "connect.sqlite"));
|
||||
} catch {
|
||||
throw new Error(`connect.sqlite was not created in ${dataDir}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function checkGracefulShutdown(server: ServerProcess): Promise<void> {
|
||||
server.kill("SIGTERM");
|
||||
const exit = await server.waitForExit(shutdownTimeoutMs);
|
||||
if (!exit) {
|
||||
await server.forceStop();
|
||||
throw new Error(`server did not exit within ${shutdownTimeoutMs} ms after SIGTERM`);
|
||||
}
|
||||
if (exit.code === 0) {
|
||||
return;
|
||||
}
|
||||
// Node's kill() is TerminateProcess on Windows: the SIGTERM handler never runs, so graceful shutdown is not
|
||||
// exercised there and the forced termination is the expected outcome.
|
||||
if (process.platform === "win32" && exit.code === null && exit.signal === "SIGTERM") {
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Error(`server exited with ${server.describeExit()} after SIGTERM; expected exit code 0`);
|
||||
}
|
||||
|
||||
/** Windows can report EBUSY for a short while after the process exits; retry instead of failing. */
|
||||
function removeDataDir(directory: string): Promise<void> {
|
||||
return rm(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
|
||||
}
|
||||
|
||||
function printServerLogs(server: ServerProcess): void {
|
||||
console.error(`server ${server.describeExit()}`);
|
||||
console.error("--- server stdout ---");
|
||||
console.error(server.stdout);
|
||||
console.error("--- server stderr ---");
|
||||
console.error(server.stderr);
|
||||
}
|
||||
|
||||
function formatMs(value: number): string {
|
||||
return `${Math.round(value)}ms`;
|
||||
}
|
||||
@@ -143,10 +143,7 @@ function toProviderSummary(provider: RuntimeProviderDefinition): ProviderSummary
|
||||
/**
|
||||
* Load generated provider catalog files from disk.
|
||||
*/
|
||||
export async function loadCatalog(
|
||||
catalogDir: string = join(process.cwd(), "catalog/apps"),
|
||||
options: LoadCatalogOptions = {},
|
||||
): Promise<CatalogStore> {
|
||||
export async function loadCatalog(catalogDir: string, options: LoadCatalogOptions = {}): Promise<CatalogStore> {
|
||||
const entries = await readdir(catalogDir, { withFileTypes: true });
|
||||
const providers = await Promise.all(
|
||||
entries
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { Hono } from "hono";
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { registerStaticRoutes } from "./static-routes.ts";
|
||||
|
||||
const indexHtml = '<!doctype html><div id="root"></div>';
|
||||
const consoleScript = "console.log('ok');";
|
||||
const headersFile = "/*\n X-Frame-Options: DENY\n";
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("registerStaticRoutes in embedded mode", () => {
|
||||
it("serves index.html for the root path", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toContain("text/html");
|
||||
expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(indexHtml)));
|
||||
await expect(response.text()).resolves.toBe(indexHtml);
|
||||
});
|
||||
|
||||
it("serves nested assets with their content type", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/assets/x.js");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toMatch(/^text\/javascript/);
|
||||
expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(consoleScript)));
|
||||
await expect(response.text()).resolves.toBe(consoleScript);
|
||||
});
|
||||
|
||||
it("falls back to application/octet-stream for files without an extension", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/_headers");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toBe("application/octet-stream");
|
||||
await expect(response.text()).resolves.toBe(headersFile);
|
||||
});
|
||||
|
||||
it("answers HEAD with headers only", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/assets/x.js", { method: "HEAD" });
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toMatch(/^text\/javascript/);
|
||||
expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(consoleScript)));
|
||||
await expect(response.text()).resolves.toBe("");
|
||||
});
|
||||
|
||||
it("serves the console shell for unknown console paths", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/providers/github");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toContain("text/html");
|
||||
await expect(response.text()).resolves.toBe(indexHtml);
|
||||
});
|
||||
|
||||
it("keeps JSON 404 responses for API paths", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/v1/nope");
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
expect(response.headers.get("content-type")).toContain("application/json");
|
||||
await expect(response.json()).resolves.toEqual({
|
||||
error: { code: "not_found", message: "Not found." },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not serve assets for POST requests", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const response = await app.request("/assets/x.js", { method: "POST" });
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
await expect(response.json()).resolves.toEqual({
|
||||
error: { code: "not_found", message: "Not found." },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks content-hashed assets immutable and leaves index.html uncached", async () => {
|
||||
const app = await createEmbeddedApp();
|
||||
|
||||
const asset = await app.request("/assets/x.js");
|
||||
const index = await app.request("/index.html");
|
||||
|
||||
expect(asset.status).toBe(200);
|
||||
expect(asset.headers.get("cache-control")).toBe("public, max-age=31536000, immutable");
|
||||
expect(index.status).toBe(200);
|
||||
expect(index.headers.get("cache-control")).toBeNull();
|
||||
});
|
||||
|
||||
it("loads the tree into memory at registration", async () => {
|
||||
const root = await createConsoleRoot();
|
||||
const app = new Hono();
|
||||
registerStaticRoutes(app, { root, embedded: true });
|
||||
await rm(root, { recursive: true, force: true });
|
||||
|
||||
const response = await app.request("/assets/x.js");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
await expect(response.text()).resolves.toBe(consoleScript);
|
||||
});
|
||||
});
|
||||
|
||||
describe("registerStaticRoutes in Node mode", () => {
|
||||
it("streams assets from disk", async () => {
|
||||
const root = await createConsoleRoot();
|
||||
const app = new Hono();
|
||||
registerStaticRoutes(app, { root });
|
||||
|
||||
const response = await app.request("/assets/x.js");
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toMatch(/^text\/javascript/);
|
||||
await expect(response.text()).resolves.toBe(consoleScript);
|
||||
});
|
||||
});
|
||||
|
||||
async function createEmbeddedApp(): Promise<Hono> {
|
||||
const app = new Hono();
|
||||
registerStaticRoutes(app, { root: await createConsoleRoot(), embedded: true });
|
||||
return app;
|
||||
}
|
||||
|
||||
async function createConsoleRoot(): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), "oomol-connect-static-"));
|
||||
tempDirs.push(root);
|
||||
await mkdir(join(root, "assets"), { recursive: true });
|
||||
await writeFile(join(root, "index.html"), indexHtml);
|
||||
await writeFile(join(root, "assets", "x.js"), consoleScript);
|
||||
await writeFile(join(root, "_headers"), headersFile);
|
||||
return root;
|
||||
}
|
||||
@@ -1,27 +1,57 @@
|
||||
import type { Hono } from "hono";
|
||||
|
||||
import { serveStatic } from "@hono/node-server/serve-static";
|
||||
import { getMimeType } from "hono/utils/mime";
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { join, relative, sep } from "node:path";
|
||||
import { isConsoleShellRequest } from "./console-paths.ts";
|
||||
import { notFound } from "./http-utils.ts";
|
||||
|
||||
export interface StaticRoutesOptions {
|
||||
/** Built console directory; undefined when the console is not built. */
|
||||
root?: string;
|
||||
/** Serve from an embedded (non-streamable, epoch-mtime) tree: files are loaded into memory once at registration. */
|
||||
embedded?: boolean;
|
||||
}
|
||||
|
||||
interface EmbeddedFile {
|
||||
/** Backed by a plain ArrayBuffer, which is what Hono's response body type requires. */
|
||||
body: Uint8Array<ArrayBuffer>;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
/** The console entry file is owned here: `/` resolves to `/index.html` in both serving modes. */
|
||||
function rewriteRequestPath(path: string): string {
|
||||
return path === "/" ? "/index.html" : path;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register static web-console routes for the local server.
|
||||
*
|
||||
* The web console is intentionally outside `src` and may be absent during
|
||||
* backend development. Hono's static middleware handles real files; this
|
||||
* wrapper owns only the fallback behavior for API and browser requests.
|
||||
*
|
||||
* Inside a Bun standalone executable the console lives in the embedded tree,
|
||||
* where `createReadStream` does not work and every mtime is the Unix epoch, so
|
||||
* the streaming middleware cannot be used. Embedded mode instead reads the
|
||||
* whole tree into memory once and answers from that map.
|
||||
*/
|
||||
export function registerStaticRoutes(app: Hono, root?: string): void {
|
||||
export function registerStaticRoutes(app: Hono, options: StaticRoutesOptions = {}): void {
|
||||
const root = options.root;
|
||||
if (root) {
|
||||
app.use(
|
||||
"*",
|
||||
serveStatic({
|
||||
root,
|
||||
rewriteRequestPath: (path) => (path === "/" ? "/index.html" : path),
|
||||
}),
|
||||
);
|
||||
if (options.embedded) {
|
||||
registerEmbeddedFiles(app, readEmbeddedFiles(root));
|
||||
} else {
|
||||
app.use(
|
||||
"*",
|
||||
serveStatic({
|
||||
root,
|
||||
rewriteRequestPath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
app.notFound(async (context) => {
|
||||
@@ -48,3 +78,60 @@ export function registerStaticRoutes(app: Hono, root?: string): void {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the console tree synchronously: `createApp()` registers routes
|
||||
* synchronously, so an asynchronous walk would race the first request.
|
||||
*/
|
||||
function readEmbeddedFiles(root: string): Map<string, EmbeddedFile> {
|
||||
const files = new Map<string, EmbeddedFile>();
|
||||
for (const entry of readdirSync(root, { withFileTypes: true, recursive: true })) {
|
||||
if (!entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
const absolute = join(entry.parentPath, entry.name);
|
||||
// The recursive walk yields backslash-separated names on Windows; request
|
||||
// paths always use forward slashes, so normalise the key or nothing matches.
|
||||
const key = `/${relative(root, absolute).split(sep).join("/")}`;
|
||||
files.set(key, {
|
||||
body: readFileSync(absolute),
|
||||
contentType: getMimeType(entry.name) ?? "application/octet-stream",
|
||||
});
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve the in-memory console files. No Last-Modified or ETag headers: embedded
|
||||
* mtimes are the epoch, so unlike the streaming middleware there is nothing
|
||||
* truthful to validate against. Content-hashed assets are marked immutable
|
||||
* instead; everything else is served uncached.
|
||||
*/
|
||||
function registerEmbeddedFiles(app: Hono, files: Map<string, EmbeddedFile>): void {
|
||||
app.use("*", async (context, next) => {
|
||||
const method = context.req.method;
|
||||
if (method !== "GET" && method !== "HEAD") {
|
||||
return next();
|
||||
}
|
||||
|
||||
const path = rewriteRequestPath(context.req.path);
|
||||
const file = files.get(path);
|
||||
if (!file) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": file.contentType,
|
||||
"Content-Length": String(file.body.byteLength),
|
||||
};
|
||||
// Vite names everything under assets/ by content hash, so a URL never changes
|
||||
// meaning and a year-long immutable cache is safe; index.html and the other
|
||||
// root files keep their names between releases and stay uncached. Embedded
|
||||
// mode cannot offer Last-Modified as the alternative: every mtime in the
|
||||
// embedded tree is the Unix epoch, so a validator would never see a change.
|
||||
if (path.startsWith("/assets/")) {
|
||||
headers["Cache-Control"] = "public, max-age=31536000, immutable";
|
||||
}
|
||||
return method === "HEAD" ? context.body(null, 200, headers) : context.body(file.body, 200, headers);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3728,7 +3728,7 @@ function createTestServer(providers: ProviderDefinition[], options: CreateTestSe
|
||||
uploadTransitFile: options.uploadTransitFile,
|
||||
runtimeTokens,
|
||||
runtimePolicyStore: options.runtimePolicyStore ?? new MemoryRuntimePolicyStore(),
|
||||
registerStaticRoutes: staticRoot ? (app) => registerStaticRoutes(app, staticRoot) : undefined,
|
||||
registerStaticRoutes: staticRoot ? (app) => registerStaticRoutes(app, { root: staticRoot }) : undefined,
|
||||
auth: {
|
||||
...options.auth,
|
||||
hasRuntimeTokens: async () => (await runtimeTokens.listTokens()).length > 0,
|
||||
|
||||
+42
-17
@@ -3,7 +3,7 @@ import type { ServerType } from "@hono/node-server";
|
||||
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { serve } from "@hono/node-server";
|
||||
import { access, mkdir } from "node:fs/promises";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { loadCatalog } from "../catalog-store.ts";
|
||||
import { ActionPolicyService, parseActionPolicyList } from "../core/action-policy.ts";
|
||||
@@ -23,7 +23,12 @@ import { S3TransitFileService } from "./files/s3-transit-files.ts";
|
||||
import { TransitFileService } from "./files/transit-files.ts";
|
||||
import { logger } from "./logger.ts";
|
||||
import { createSecretCodec } from "./secrets/secret-codec.ts";
|
||||
import { createNodeRuntimeDatabase } from "./storage/node-runtime-database.ts";
|
||||
import { resolveServerAssets } from "./server-assets.ts";
|
||||
import {
|
||||
createNodeRuntimeDatabase,
|
||||
migratePostgresRuntimeDatabase,
|
||||
sqliteMigrationsNotice,
|
||||
} from "./storage/node-runtime-database.ts";
|
||||
import { DEFAULT_RUN_LIMIT } from "./storage/runtime-store.ts";
|
||||
|
||||
const port = Number(process.env.PORT ?? 3000);
|
||||
@@ -37,10 +42,22 @@ const databaseUrl = optionalEnv("OOMOL_CONNECT_DATABASE_URL");
|
||||
const databasePoolMax = readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_POOL_MAX", 10);
|
||||
const databaseConnectTimeoutMs = readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000);
|
||||
|
||||
// The standalone binary embeds migrations/postgresql, but the PostgreSQL startup validator refuses to serve until
|
||||
// they are applied and its error text points at `npm run runtime:migrate`, which a binary user does not have.
|
||||
// `migrate` applies them from the same source the validator reads, so validation and execution cannot diverge.
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
|
||||
try {
|
||||
await main();
|
||||
if (command === undefined) {
|
||||
await main();
|
||||
} else if (command === "migrate" && rest.length === 0) {
|
||||
await runMigrateCommand();
|
||||
} else {
|
||||
console.error("Usage: open-connector [migrate]");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ err: error }, "connect server failed");
|
||||
logger.error({ err: error }, command === "migrate" ? "migrate failed" : "connect server failed");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
@@ -65,8 +82,8 @@ async function main(): Promise<void> {
|
||||
const allowedCustomOAuth = parseActionPolicyList(process.env.OOMOL_CONNECT_ALLOWED_CUSTOM_OAUTH);
|
||||
|
||||
await mkdir(dataDir, { recursive: true });
|
||||
const staticRoot = await resolveStaticRoot(join(process.cwd(), "dist/web"));
|
||||
const catalog = await loadCatalog(undefined, {
|
||||
const assets = await resolveServerAssets();
|
||||
const catalog = await loadCatalog(assets.catalogDir, {
|
||||
executableServices: Object.keys(executorModules),
|
||||
});
|
||||
const runtimeDatabase = databaseUrl
|
||||
@@ -78,6 +95,7 @@ async function main(): Promise<void> {
|
||||
runLimit,
|
||||
poolMax: databasePoolMax,
|
||||
connectionTimeoutMs: databaseConnectTimeoutMs,
|
||||
migrations: assets.migrations,
|
||||
})
|
||||
: await createNodeRuntimeDatabase({
|
||||
backend: "sqlite",
|
||||
@@ -85,6 +103,7 @@ async function main(): Promise<void> {
|
||||
logger,
|
||||
secretCodec,
|
||||
runLimit,
|
||||
migrations: assets.migrations,
|
||||
});
|
||||
|
||||
try {
|
||||
@@ -106,7 +125,7 @@ async function main(): Promise<void> {
|
||||
verifyRuntimeJwt,
|
||||
actionPolicy,
|
||||
allowedCustomOAuth,
|
||||
registerStaticRoutes: (app) => registerStaticRoutes(app, staticRoot),
|
||||
registerStaticRoutes: (app) => registerStaticRoutes(app, { root: assets.staticRoot, embedded: assets.embedded }),
|
||||
logger,
|
||||
});
|
||||
|
||||
@@ -133,7 +152,7 @@ async function main(): Promise<void> {
|
||||
"runtime data encryption is disabled; set OOMOL_CONNECT_ENCRYPTION_KEY to encrypt stored credentials, Marketplace API keys, OAuth client configuration, pending OAuth state, and completed idempotent action responses",
|
||||
);
|
||||
}
|
||||
if (!staticRoot) {
|
||||
if (!assets.staticRoot) {
|
||||
logger.warn("web console assets are not built; use http://localhost:5173 for local console development");
|
||||
}
|
||||
},
|
||||
@@ -145,6 +164,21 @@ async function main(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function runMigrateCommand(): Promise<void> {
|
||||
if (!databaseUrl) {
|
||||
logger.info(sqliteMigrationsNotice);
|
||||
return;
|
||||
}
|
||||
|
||||
const assets = await resolveServerAssets();
|
||||
await migratePostgresRuntimeDatabase({
|
||||
connectionString: databaseUrl,
|
||||
connectionTimeoutMs: databaseConnectTimeoutMs,
|
||||
logger,
|
||||
migrations: assets.migrations,
|
||||
});
|
||||
}
|
||||
|
||||
function waitForShutdown(server: ServerType): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let closing = false;
|
||||
@@ -169,15 +203,6 @@ function waitForShutdown(server: ServerType): Promise<void> {
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveStaticRoot(root: string): Promise<string | undefined> {
|
||||
try {
|
||||
await access(join(root, "index.html"));
|
||||
return root;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function readPositiveIntegerEnv(name: string, fallback: number): number {
|
||||
const value = process.env[name];
|
||||
if (value === undefined) {
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveServerAssets } from "./server-assets.ts";
|
||||
import { defaultMigrationSource } from "./storage/migration-source.ts";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("resolveServerAssets", () => {
|
||||
it("uses the working directory layout of npm start under Node", async () => {
|
||||
const cwd = await createTempCwd();
|
||||
await mkdir(join(cwd, "dist", "web"), { recursive: true });
|
||||
await writeFile(join(cwd, "dist", "web", "index.html"), "<!doctype html>");
|
||||
|
||||
const assets = await resolveServerAssets();
|
||||
|
||||
expect(assets.catalogDir).toBe(join(cwd, "catalog/apps"));
|
||||
expect(assets.migrations).toBe(defaultMigrationSource);
|
||||
expect(assets.staticRoot).toBe(join(cwd, "dist/web"));
|
||||
expect(assets.embedded).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the console as not built when dist/web/index.html is missing", async () => {
|
||||
const cwd = await createTempCwd();
|
||||
await mkdir(join(cwd, "dist", "web"), { recursive: true });
|
||||
|
||||
const assets = await resolveServerAssets();
|
||||
|
||||
expect(assets.catalogDir).toBe(join(cwd, "catalog/apps"));
|
||||
expect(assets.migrations).toBe(defaultMigrationSource);
|
||||
expect(assets.staticRoot).toBeUndefined();
|
||||
expect(assets.embedded).toBe(false);
|
||||
});
|
||||
|
||||
it("reads the tree embedded next to the module inside a Bun standalone executable", async () => {
|
||||
vi.stubGlobal("Bun", { isStandaloneExecutable: true });
|
||||
const root = import.meta.dirname;
|
||||
|
||||
const assets = await resolveServerAssets();
|
||||
|
||||
expect(assets.embedded).toBe(true);
|
||||
expect(assets.catalogDir).toBe(join(root, "apps"));
|
||||
// The directory source touches the filesystem lazily; the ENOENT it raises names the embedded directory.
|
||||
expect(assets.migrations).not.toBe(defaultMigrationSource);
|
||||
expect(() => assets.migrations.readMigrations("sqlite")).toThrow(join(root, "migrations"));
|
||||
// No web/index.html lives beside this module, so the console is reported as not built.
|
||||
expect(assets.staticRoot).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
async function createTempCwd(): Promise<string> {
|
||||
const cwd = await mkdtemp(join(tmpdir(), "oomol-connect-assets-"));
|
||||
tempDirs.push(cwd);
|
||||
vi.spyOn(process, "cwd").mockReturnValue(cwd);
|
||||
return cwd;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { MigrationSource } from "./storage/migration-source.ts";
|
||||
|
||||
import { access } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { createDirectoryMigrationSource, defaultMigrationSource } from "./storage/migration-source.ts";
|
||||
|
||||
/**
|
||||
* Locations of the assets that are generated or built outside `src` and read
|
||||
* by the server at startup.
|
||||
*/
|
||||
export interface ServerAssets {
|
||||
/** Directory of generated provider catalog JSON files. */
|
||||
catalogDir: string;
|
||||
migrations: MigrationSource;
|
||||
/** Built web console directory, or undefined when the console is not built (index.html missing). */
|
||||
staticRoot: string | undefined;
|
||||
/**
|
||||
* True when the directories above live in the read-only tree embedded in a
|
||||
* Bun standalone executable. Embedded files cannot be streamed and report
|
||||
* epoch mtimes.
|
||||
*/
|
||||
embedded: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve where catalog, migrations and web console live for this process: the
|
||||
* embedded tree in a standalone executable, otherwise the working directory and
|
||||
* repository layout used by `npm start`.
|
||||
*
|
||||
* The build script embeds `migrations/`, `catalog/apps/` and `dist/web/` next
|
||||
* to the bundled entry point, so inside the executable they appear under
|
||||
* `import.meta.dirname` as `migrations/`, `apps/` and `web/`.
|
||||
*/
|
||||
export async function resolveServerAssets(): Promise<ServerAssets> {
|
||||
if (isStandaloneExecutable()) {
|
||||
const root = import.meta.dirname;
|
||||
return {
|
||||
catalogDir: join(root, "apps"),
|
||||
migrations: createDirectoryMigrationSource(join(root, "migrations")),
|
||||
staticRoot: await resolveStaticRoot(join(root, "web")),
|
||||
embedded: true,
|
||||
};
|
||||
}
|
||||
|
||||
const cwd = process.cwd();
|
||||
return {
|
||||
catalogDir: join(cwd, "catalog/apps"),
|
||||
migrations: defaultMigrationSource,
|
||||
staticRoot: await resolveStaticRoot(join(cwd, "dist/web")),
|
||||
embedded: false,
|
||||
};
|
||||
}
|
||||
|
||||
/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */
|
||||
function isStandaloneExecutable(): boolean {
|
||||
return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true;
|
||||
}
|
||||
|
||||
async function resolveStaticRoot(root: string): Promise<string | undefined> {
|
||||
try {
|
||||
await access(join(root, "index.html"));
|
||||
return root;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,13 @@
|
||||
import type { RuntimeActionHttpResult } from "../api/runtime-api.ts";
|
||||
import type { D1DatabaseBinding, D1PreparedStatementBinding } from "../cloudflare/cloudflare-bindings.ts";
|
||||
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AesGcmSecretCodec } from "../secrets/secret-codec.ts";
|
||||
import { D1RuntimeDatabase } from "./d1-runtime-store.ts";
|
||||
import { defaultMigrationSource } from "./migration-source.ts";
|
||||
import { RuntimeTokenService } from "./runtime-token-service.ts";
|
||||
|
||||
const migrationDirectory = new URL("../../../migrations/", import.meta.url);
|
||||
|
||||
const githubProfile = {
|
||||
accountId: "github:octocat",
|
||||
displayName: "octocat",
|
||||
@@ -607,14 +605,8 @@ class SqliteD1Database implements D1DatabaseBinding {
|
||||
private readonly database = new DatabaseSync(":memory:");
|
||||
|
||||
constructor() {
|
||||
// Derive the migration list the same way runSqliteMigrations does, so a new migration reaches the
|
||||
// D1 harness on its own. The three lines stay duplicated here because sharing an owner would mean
|
||||
// exporting from production code, which this test-only fix deliberately leaves alone.
|
||||
const migrationFiles = readdirSync(migrationDirectory)
|
||||
.filter((name) => /^\d+_.*\.sql$/.test(name))
|
||||
.sort();
|
||||
for (const file of migrationFiles) {
|
||||
this.database.exec(readFileSync(new URL(file, migrationDirectory), "utf8"));
|
||||
for (const migration of defaultMigrationSource.readMigrations("sqlite")) {
|
||||
this.database.exec(migration.sql);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createDirectoryMigrationSource, defaultMigrationSource } from "./migration-source.ts";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
describe("createDirectoryMigrationSource", () => {
|
||||
it("reads sqlite migrations from the directory and postgresql migrations from its subdirectory", async () => {
|
||||
const directory = await createMigrationDirectory();
|
||||
await writeFile(join(directory, "0001_sqlite.sql"), "create table sqlite_only (id integer);");
|
||||
await writeFile(join(directory, "postgresql", "0001_postgres.sql"), "create table postgres_only (id integer);");
|
||||
|
||||
const source = createDirectoryMigrationSource(directory);
|
||||
expect(source.readMigrations("sqlite")).toEqual([
|
||||
{ name: "0001_sqlite.sql", sql: "create table sqlite_only (id integer);" },
|
||||
]);
|
||||
expect(source.readMigrations("postgresql")).toEqual([
|
||||
{ name: "0001_postgres.sql", sql: "create table postgres_only (id integer);" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps only numbered .sql files and ignores other entries and directories", async () => {
|
||||
const directory = await createMigrationDirectory();
|
||||
await writeFile(join(directory, "0001_runtime.sql"), "select 1;");
|
||||
await writeFile(join(directory, "README.md"), "# migrations");
|
||||
await writeFile(join(directory, "notes.sql"), "select 2;");
|
||||
await writeFile(join(directory, "0002_backup.sql.bak"), "select 3;");
|
||||
await mkdir(join(directory, "archive"));
|
||||
await mkdir(join(directory, "0002_archive.sql"));
|
||||
|
||||
expect(createDirectoryMigrationSource(directory).readMigrations("sqlite")).toEqual([
|
||||
{ name: "0001_runtime.sql", sql: "select 1;" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("sorts migrations by file name using string order", async () => {
|
||||
const directory = await createMigrationDirectory();
|
||||
await writeFile(join(directory, "0010_second.sql"), "select 10;");
|
||||
await writeFile(join(directory, "0002_first.sql"), "select 2;");
|
||||
await writeFile(join(directory, "10_last.sql"), "select 3;");
|
||||
|
||||
expect(
|
||||
createDirectoryMigrationSource(directory)
|
||||
.readMigrations("sqlite")
|
||||
.map((migration) => migration.name),
|
||||
).toEqual(["0002_first.sql", "0010_second.sql", "10_last.sql"]);
|
||||
});
|
||||
|
||||
it("reads migration bodies as utf8", async () => {
|
||||
const directory = await createMigrationDirectory();
|
||||
const sql = "-- 运行时表\ncreate table runtime (name text);\n";
|
||||
await writeFile(join(directory, "0001_runtime.sql"), sql, "utf8");
|
||||
|
||||
expect(createDirectoryMigrationSource(directory).readMigrations("sqlite")).toEqual([
|
||||
{ name: "0001_runtime.sql", sql },
|
||||
]);
|
||||
});
|
||||
|
||||
it("defers filesystem errors to readMigrations", async () => {
|
||||
const directory = await createMigrationDirectory();
|
||||
await writeFile(join(directory, "0001_runtime.sql"), "select 1;");
|
||||
await rm(join(directory, "postgresql"), { recursive: true });
|
||||
|
||||
const missing = createDirectoryMigrationSource(join(directory, "missing"));
|
||||
expect(() => missing.readMigrations("sqlite")).toThrow(/ENOENT/);
|
||||
expect(() => missing.readMigrations("postgresql")).toThrow(/ENOENT/);
|
||||
|
||||
const withoutPostgres = createDirectoryMigrationSource(directory);
|
||||
expect(withoutPostgres.readMigrations("sqlite")).toEqual([{ name: "0001_runtime.sql", sql: "select 1;" }]);
|
||||
expect(() => withoutPostgres.readMigrations("postgresql")).toThrow(/ENOENT/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultMigrationSource", () => {
|
||||
it("resolves the repository migrations directory", () => {
|
||||
const sqlite = defaultMigrationSource.readMigrations("sqlite");
|
||||
expect(sqlite[0]).toMatchObject({ name: "0001_runtime.sql" });
|
||||
expect(sqlite[0]?.sql).toContain("create table");
|
||||
expect(sqlite.every((migration) => migration.name.endsWith(".sql"))).toBe(true);
|
||||
|
||||
const postgresql = defaultMigrationSource.readMigrations("postgresql");
|
||||
expect(postgresql[0]).toMatchObject({ name: "0010_runtime.sql" });
|
||||
expect(postgresql.map((migration) => migration.name)).not.toContain("0001_runtime.sql");
|
||||
});
|
||||
});
|
||||
|
||||
async function createMigrationDirectory(): Promise<string> {
|
||||
const directory = await mkdtemp(join(tmpdir(), "oomol-connect-migrations-"));
|
||||
tempDirs.push(directory);
|
||||
await mkdir(join(directory, "postgresql"));
|
||||
return directory;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
/**
|
||||
* Where runtime migrations come from. Validation and execution must read the same source so that
|
||||
* a migration cannot be required by one and invisible to the other.
|
||||
*/
|
||||
export interface MigrationSource {
|
||||
/** Migrations for one dialect, sorted by file name. Synchronous because SqliteRuntimeDatabase migrates inside its constructor. */
|
||||
readMigrations(dialect: "sqlite" | "postgresql"): { name: string; sql: string }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Directory layout: <directory>/*.sql = sqlite, <directory>/postgresql/*.sql = postgresql.
|
||||
* Construction performs no filesystem access; errors (ENOENT etc.) surface from readMigrations().
|
||||
* Accepts a directory path string only (no URL): a file: URL base without a trailing slash silently resolves against the
|
||||
* parent directory; not accepting URLs removes the trap by construction.
|
||||
*/
|
||||
export function createDirectoryMigrationSource(directory: string): MigrationSource {
|
||||
return {
|
||||
readMigrations(dialect) {
|
||||
const dir = dialect === "sqlite" ? directory : join(directory, "postgresql");
|
||||
return readdirSync(dir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && /^\d+_.*\.sql$/.test(entry.name))
|
||||
.map((entry) => entry.name)
|
||||
.sort()
|
||||
.map((name) => ({ name, sql: readFileSync(join(dir, name), "utf8") }));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** The repository's migrations/ directory resolved relative to this module. Used by every Node (non-binary) entry point. */
|
||||
export const defaultMigrationSource: MigrationSource = createDirectoryMigrationSource(
|
||||
fileURLToPath(new URL("../../../migrations/", import.meta.url)),
|
||||
);
|
||||
@@ -1,7 +1,10 @@
|
||||
import type { RuntimeLogger } from "../../core/types.ts";
|
||||
import type { ISecretCodec } from "../secrets/secret-codec-core.ts";
|
||||
import type { MigrationSource } from "./migration-source.ts";
|
||||
import type { RuntimeDatabase } from "./runtime-database.ts";
|
||||
|
||||
import { Pool } from "pg";
|
||||
import { migratePostgresDatabase } from "./postgres-migrations.ts";
|
||||
import { PostgresRuntimeDatabase } from "./postgres-runtime-store.ts";
|
||||
import { SqliteRuntimeDatabase } from "./sqlite-runtime-store.ts";
|
||||
|
||||
@@ -15,6 +18,7 @@ interface CommonOptions {
|
||||
logger?: RuntimeLogger;
|
||||
runLimit?: number;
|
||||
secretCodec?: ISecretCodec;
|
||||
migrations?: MigrationSource;
|
||||
}
|
||||
|
||||
interface SqliteOptions extends CommonOptions {
|
||||
@@ -31,6 +35,13 @@ interface PostgresOptions extends CommonOptions {
|
||||
|
||||
export type NodeRuntimeDatabaseOptions = SqliteOptions | PostgresOptions;
|
||||
|
||||
export interface MigratePostgresRuntimeDatabaseOptions {
|
||||
connectionString: string;
|
||||
connectionTimeoutMs: number;
|
||||
logger?: RuntimeLogger;
|
||||
migrations?: MigrationSource;
|
||||
}
|
||||
|
||||
export async function createNodeRuntimeDatabase(options: NodeRuntimeDatabaseOptions): Promise<NodeRuntimeDatabase> {
|
||||
if (options.backend === "sqlite") {
|
||||
return new SqliteRuntimeDatabase(options.path, options);
|
||||
@@ -41,7 +52,27 @@ export async function createNodeRuntimeDatabase(options: NodeRuntimeDatabaseOpti
|
||||
return await PostgresRuntimeDatabase.open(connectionString, options);
|
||||
}
|
||||
|
||||
export function assertPostgresDatabaseUrl(value: string): void {
|
||||
/** What `migrate` entry points print without OOMOL_CONNECT_DATABASE_URL: SQLite has no explicit migrate step. */
|
||||
export const sqliteMigrationsNotice =
|
||||
"SQLite migrations are applied automatically when the local runtime database opens.";
|
||||
|
||||
/** Validate the URL, open a single-connection pool named open-connector-migrate, apply pending migrations, and close the pool. */
|
||||
export async function migratePostgresRuntimeDatabase(options: MigratePostgresRuntimeDatabaseOptions): Promise<void> {
|
||||
assertPostgresDatabaseUrl(options.connectionString);
|
||||
const pool = new Pool({
|
||||
application_name: "open-connector-migrate",
|
||||
connectionString: options.connectionString,
|
||||
connectionTimeoutMillis: options.connectionTimeoutMs,
|
||||
max: 1,
|
||||
});
|
||||
try {
|
||||
await migratePostgresDatabase({ pool, logger: options.logger, migrations: options.migrations });
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
function assertPostgresDatabaseUrl(value: string): void {
|
||||
let protocol: string;
|
||||
try {
|
||||
protocol = new URL(value).protocol;
|
||||
|
||||
@@ -1,20 +1,16 @@
|
||||
import type { RuntimeLogger } from "../../core/types.ts";
|
||||
import type { MigrationSource } from "./migration-source.ts";
|
||||
import type { Pool, PoolClient } from "pg";
|
||||
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { defaultMigrationSource } from "./migration-source.ts";
|
||||
|
||||
const migrationDirectory = new URL("../../../migrations/postgresql/", import.meta.url);
|
||||
const migrationLockNamespace = 1_326_382_671;
|
||||
const migrationLockId = 1;
|
||||
|
||||
interface PostgresMigration {
|
||||
name: string;
|
||||
sql: string;
|
||||
}
|
||||
|
||||
export interface PostgresMigrationOptions {
|
||||
pool: Pool;
|
||||
logger?: RuntimeLogger;
|
||||
migrations?: MigrationSource;
|
||||
}
|
||||
|
||||
export async function migratePostgresDatabase(options: PostgresMigrationOptions): Promise<void> {
|
||||
@@ -31,7 +27,7 @@ export async function migratePostgresDatabase(options: PostgresMigrationOptions)
|
||||
`);
|
||||
|
||||
const startedAt = Date.now();
|
||||
const migrations = readPostgresMigrations();
|
||||
const migrations = (options.migrations ?? defaultMigrationSource).readMigrations("postgresql");
|
||||
const applied = await readAppliedMigrations(client);
|
||||
let newlyAppliedCount = 0;
|
||||
|
||||
@@ -86,8 +82,11 @@ export async function migratePostgresDatabase(options: PostgresMigrationOptions)
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertPostgresSchemaReady(pool: Pool): Promise<void> {
|
||||
const migrations = readPostgresMigrations();
|
||||
export async function assertPostgresSchemaReady(
|
||||
pool: Pool,
|
||||
migrations: MigrationSource = defaultMigrationSource,
|
||||
): Promise<void> {
|
||||
const required = migrations.readMigrations("postgresql");
|
||||
const relation = await pool.query<{ name: string | null }>("select to_regclass($1) as name", ["runtime_migrations"]);
|
||||
if (!relation.rows[0]?.name) {
|
||||
throw new Error(
|
||||
@@ -96,7 +95,7 @@ export async function assertPostgresSchemaReady(pool: Pool): Promise<void> {
|
||||
}
|
||||
|
||||
const applied = await readAppliedMigrations(pool);
|
||||
const missing = migrations.filter((migration) => !applied.has(migration.name)).map((migration) => migration.name);
|
||||
const missing = required.filter((migration) => !applied.has(migration.name)).map((migration) => migration.name);
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`PostgreSQL runtime schema is not ready. Missing migrations: ${missing.join(", ")}. Run \`npm run runtime:migrate\` before starting the server.`,
|
||||
@@ -104,16 +103,6 @@ export async function assertPostgresSchemaReady(pool: Pool): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
function readPostgresMigrations(): PostgresMigration[] {
|
||||
return readdirSync(migrationDirectory)
|
||||
.filter((name) => /^\d+_.*\.sql$/.test(name))
|
||||
.sort()
|
||||
.map((name) => ({
|
||||
name,
|
||||
sql: readFileSync(new URL(name, migrationDirectory), "utf8"),
|
||||
}));
|
||||
}
|
||||
|
||||
async function readAppliedMigrations(queryable: Pool | PoolClient): Promise<Set<string>> {
|
||||
const result = await queryable.query<{ name: string }>("select name from runtime_migrations");
|
||||
return new Set(result.rows.map((row) => row.name));
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import type { RuntimeActionHttpResult } from "../api/runtime-api.ts";
|
||||
import type { MigrationSource } from "./migration-source.ts";
|
||||
|
||||
import { PGlite } from "@electric-sql/pglite";
|
||||
import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
|
||||
import { Pool } from "pg";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { AesGcmSecretCodec } from "../secrets/secret-codec.ts";
|
||||
import { defaultMigrationSource } from "./migration-source.ts";
|
||||
import { assertPostgresSchemaReady, migratePostgresDatabase } from "./postgres-migrations.ts";
|
||||
import { PostgresRuntimeDatabase } from "./postgres-runtime-store.ts";
|
||||
import { RuntimeTokenService } from "./runtime-token-service.ts";
|
||||
@@ -68,6 +70,62 @@ describe("PostgreSQL migrations with PGlite", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("PostgreSQL migrations with a custom migration source", () => {
|
||||
let testServer: PGliteTestServer;
|
||||
|
||||
beforeAll(async () => {
|
||||
testServer = await startPGliteTestServer();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await testServer.server.stop();
|
||||
await testServer.database.close();
|
||||
});
|
||||
|
||||
it("validates and executes migrations through the same source", async () => {
|
||||
const migrations: MigrationSource = {
|
||||
readMigrations(dialect) {
|
||||
return [
|
||||
...defaultMigrationSource.readMigrations(dialect),
|
||||
{ name: "9998_custom.sql", sql: "create table custom_records (id integer primary key)" },
|
||||
];
|
||||
},
|
||||
};
|
||||
const unapplied: MigrationSource = {
|
||||
readMigrations: () => [{ name: "9999_unapplied.sql", sql: "select 1" }],
|
||||
};
|
||||
|
||||
const pool = new Pool({ connectionString: testServer.url, max: 1 });
|
||||
try {
|
||||
await migratePostgresDatabase({ pool, migrations });
|
||||
await expect(assertPostgresSchemaReady(pool, migrations)).resolves.toBeUndefined();
|
||||
await expect(assertPostgresSchemaReady(pool)).resolves.toBeUndefined();
|
||||
await expect(assertPostgresSchemaReady(pool, unapplied)).rejects.toThrow(
|
||||
"Missing migrations: 9999_unapplied.sql",
|
||||
);
|
||||
await expect(pool.query("select name from runtime_migrations order by name")).resolves.toMatchObject({
|
||||
rows: [
|
||||
{ name: "0010_runtime.sql" },
|
||||
{ name: "0011_runtime_token_connection_scope.sql" },
|
||||
{ name: "0012_marketplace.sql" },
|
||||
{ name: "9998_custom.sql" },
|
||||
],
|
||||
});
|
||||
await expect(pool.query("select to_regclass($1) as name", ["custom_records"])).resolves.toMatchObject({
|
||||
rows: [{ name: "custom_records" }],
|
||||
});
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
await expect(PostgresRuntimeDatabase.open(testServer.url, { migrations: unapplied })).rejects.toThrow(
|
||||
"Missing migrations: 9999_unapplied.sql",
|
||||
);
|
||||
const database = await PostgresRuntimeDatabase.open(testServer.url, { migrations });
|
||||
await database.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("PostgresRuntimeDatabase with PGlite", () => {
|
||||
let testServer: PGliteTestServer;
|
||||
let database: PostgresRuntimeDatabase;
|
||||
|
||||
@@ -15,6 +15,7 @@ import type {
|
||||
IdempotencyClaimResult,
|
||||
IIdempotencyStore,
|
||||
} from "./idempotency-store.ts";
|
||||
import type { MigrationSource } from "./migration-source.ts";
|
||||
import type { RuntimeDatabase } from "./runtime-database.ts";
|
||||
import type { IRuntimePolicyStore, RuntimePolicyRecord } from "./runtime-policy-store.ts";
|
||||
import type { RuntimeRow } from "./runtime-sql.ts";
|
||||
@@ -43,6 +44,7 @@ export interface PostgresRuntimeDatabaseOptions {
|
||||
secretCodec?: ISecretCodec;
|
||||
poolMax?: number;
|
||||
connectionTimeoutMs?: number;
|
||||
migrations?: MigrationSource;
|
||||
}
|
||||
|
||||
export class PostgresRuntimeDatabase implements RuntimeDatabase {
|
||||
@@ -86,7 +88,7 @@ export class PostgresRuntimeDatabase implements RuntimeDatabase {
|
||||
});
|
||||
|
||||
try {
|
||||
await assertPostgresSchemaReady(pool);
|
||||
await assertPostgresSchemaReady(pool, options.migrations);
|
||||
return new PostgresRuntimeDatabase(pool, options);
|
||||
} catch (error) {
|
||||
await pool.end();
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import type { RuntimeActionHttpResult } from "../api/runtime-api.ts";
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { AesGcmSecretCodec } from "../secrets/secret-codec.ts";
|
||||
import { createDirectoryMigrationSource } from "./migration-source.ts";
|
||||
import { RuntimeTokenService } from "./runtime-token-service.ts";
|
||||
import { SqliteRunLogStore, SqliteRuntimeDatabase } from "./sqlite-runtime-store.ts";
|
||||
|
||||
@@ -459,6 +460,31 @@ describe("SqliteRuntimeDatabase", () => {
|
||||
database.close();
|
||||
});
|
||||
|
||||
it("applies migrations from a custom migration source", async () => {
|
||||
const databasePath = await createDatabasePath();
|
||||
const migrationDirectory = join(dirname(databasePath), "migrations");
|
||||
await mkdir(migrationDirectory);
|
||||
await writeFile(
|
||||
join(migrationDirectory, "0001_custom.sql"),
|
||||
"create table custom_records (id integer primary key);",
|
||||
);
|
||||
|
||||
const database = new SqliteRuntimeDatabase(databasePath, {
|
||||
migrations: createDirectoryMigrationSource(migrationDirectory),
|
||||
});
|
||||
database.close();
|
||||
|
||||
const inspected = new DatabaseSync(databasePath);
|
||||
expect(inspected.prepare("select name from runtime_migrations order by name").all()).toEqual([
|
||||
{ name: "0001_custom.sql" },
|
||||
]);
|
||||
expect(
|
||||
inspected.prepare("select name from sqlite_master where type = 'table' and name = 'custom_records'").get(),
|
||||
).toEqual({ name: "custom_records" });
|
||||
expect(inspected.prepare("select name from sqlite_master where name = 'connections'").get()).toBeUndefined();
|
||||
inspected.close();
|
||||
});
|
||||
|
||||
it("keeps an inserted run when retention cleanup fails", async () => {
|
||||
const raw = new DatabaseSync(":memory:");
|
||||
for (const migration of [
|
||||
|
||||
@@ -15,15 +15,16 @@ import type {
|
||||
IdempotencyClaimResult,
|
||||
IIdempotencyStore,
|
||||
} from "./idempotency-store.ts";
|
||||
import type { MigrationSource } from "./migration-source.ts";
|
||||
import type { RuntimeDatabase } from "./runtime-database.ts";
|
||||
import type { IRuntimePolicyStore, RuntimePolicyRecord } from "./runtime-policy-store.ts";
|
||||
import type { IRunLogStore, RunLog, RunLogListInput, RunLogPage, RunLogWriteResult } from "./runtime-store.ts";
|
||||
import type { IRuntimeTokenStore, RuntimeTokenRecord } from "./runtime-token-service.ts";
|
||||
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { parseRuntimeActionHttpResult } from "../api/runtime-api.ts";
|
||||
import { PlainTextSecretCodec } from "../secrets/secret-codec-core.ts";
|
||||
import { defaultMigrationSource } from "./migration-source.ts";
|
||||
import {
|
||||
listRunLogs,
|
||||
parseJson,
|
||||
@@ -36,12 +37,12 @@ import {
|
||||
import { DEFAULT_RUN_LIMIT } from "./runtime-store.ts";
|
||||
|
||||
type SecretJsonTable = "oauth_client_configs";
|
||||
const migrationDirectory = new URL("../../../migrations/", import.meta.url);
|
||||
|
||||
export interface SqliteRuntimeDatabaseOptions {
|
||||
logger?: RuntimeLogger;
|
||||
runLimit?: number;
|
||||
secretCodec?: ISecretCodec;
|
||||
migrations?: MigrationSource;
|
||||
}
|
||||
|
||||
interface SecretJsonInput {
|
||||
@@ -95,7 +96,7 @@ export class SqliteRuntimeDatabase implements RuntimeDatabase {
|
||||
constructor(filename: string, options: SqliteRuntimeDatabaseOptions = {}) {
|
||||
this.database = new DatabaseSync(filename);
|
||||
this.secretCodec = options.secretCodec ?? new PlainTextSecretCodec();
|
||||
this.initialize(options.logger);
|
||||
this.initialize(options.migrations ?? defaultMigrationSource, options.logger);
|
||||
this.connectionStore = new SqliteConnectionStore(this.database, this.secretCodec);
|
||||
this.oauthClientConfigStore = new SqliteOAuthClientConfigStore(this.database, this.secretCodec);
|
||||
this.oauthStateStore = new SqliteOAuthStateStore(this.database, this.secretCodec);
|
||||
@@ -156,9 +157,9 @@ export class SqliteRuntimeDatabase implements RuntimeDatabase {
|
||||
`);
|
||||
}
|
||||
|
||||
private initialize(logger?: RuntimeLogger): void {
|
||||
private initialize(migrations: MigrationSource, logger?: RuntimeLogger): void {
|
||||
this.database.exec("pragma journal_mode = wal;");
|
||||
runSqliteMigrations(this.database, logger);
|
||||
runSqliteMigrations(this.database, migrations, logger);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -648,7 +649,7 @@ function insertRun(database: DatabaseSync, run: RunLog): void {
|
||||
);
|
||||
}
|
||||
|
||||
function runSqliteMigrations(database: DatabaseSync, logger?: RuntimeLogger): void {
|
||||
function runSqliteMigrations(database: DatabaseSync, migrations: MigrationSource, logger?: RuntimeLogger): void {
|
||||
const startedAt = Date.now();
|
||||
database.exec(`
|
||||
create table if not exists runtime_migrations (
|
||||
@@ -662,42 +663,42 @@ function runSqliteMigrations(database: DatabaseSync, logger?: RuntimeLogger): vo
|
||||
.all()
|
||||
.map((row) => readString(row, "name")),
|
||||
);
|
||||
const migrationFiles = readdirSync(migrationDirectory)
|
||||
.filter((name) => /^\d+_.*\.sql$/.test(name))
|
||||
.sort();
|
||||
const migrationFiles = migrations.readMigrations("sqlite");
|
||||
let newlyAppliedCount = 0;
|
||||
|
||||
for (const file of migrationFiles) {
|
||||
if (applied.has(file)) {
|
||||
for (const migration of migrationFiles) {
|
||||
if (applied.has(migration.name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const migrationStartedAt = Date.now();
|
||||
logger?.info({ migration: file }, "sqlite migration started");
|
||||
logger?.info({ migration: migration.name }, "sqlite migration started");
|
||||
try {
|
||||
const sql = readFileSync(new URL(file, migrationDirectory), "utf8");
|
||||
runInTransaction(database, () => {
|
||||
database.exec(sql);
|
||||
database.exec(migration.sql);
|
||||
database
|
||||
.prepare("insert into runtime_migrations (name, applied_at) values (?, ?)")
|
||||
.run(file, new Date().toISOString());
|
||||
.run(migration.name, new Date().toISOString());
|
||||
});
|
||||
} catch (error) {
|
||||
logger?.error(
|
||||
{ migration: file, durationMs: Date.now() - migrationStartedAt, err: error },
|
||||
{ migration: migration.name, durationMs: Date.now() - migrationStartedAt, err: error },
|
||||
"sqlite migration failed",
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
applied.add(file);
|
||||
applied.add(migration.name);
|
||||
newlyAppliedCount += 1;
|
||||
logger?.info({ migration: file, durationMs: Date.now() - migrationStartedAt }, "sqlite migration completed");
|
||||
logger?.info(
|
||||
{ migration: migration.name, durationMs: Date.now() - migrationStartedAt },
|
||||
"sqlite migration completed",
|
||||
);
|
||||
}
|
||||
|
||||
logger?.info(
|
||||
{
|
||||
migrationCount: migrationFiles.length,
|
||||
appliedCount: migrationFiles.filter((file) => applied.has(file)).length,
|
||||
appliedCount: migrationFiles.filter((migration) => applied.has(migration.name)).length,
|
||||
newlyAppliedCount,
|
||||
durationMs: Date.now() - startedAt,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user