MUL-6502: fix startup recovery during PostgreSQL outages (#7372)

* fix(startup): retry transient database outages Co-authored-by: multica-agent <[email protected]> * fix(startup): address database retry review Co-authored-by: multica-agent <[email protected]> * test(startup): address retry review nits Co-authored-by: multica-agent <[email protected]> * fix(startup): preserve native pgx connect timeout Co-authored-by: multica-agent <[email protected]> * fix(startup): apply timeout fallback to services Co-authored-by: multica-agent <[email protected]> --------- Co-authored-by: Sol-Boy <[email protected]> Co-authored-by: multica-agent <[email protected]>
2026-11-08 01:16:45 +08:00 · 2026-08-21 14:33:04 +08:00
parent 5e2d349ec4
commit 53e15b35c6
19 changed files with 1070 additions and 27 deletions
@@ -9,6 +9,14 @@ DATABASE_URL=postgres://multica:multica@localhost:5432/multica?sslmode=disable
 # DATABASE_URL; env vars below take precedence over URL params.
 # DATABASE_MAX_CONNS=25
 # DATABASE_MIN_CONNS=5
+# Retry transient database failures for up to this duration across the image
+# entrypoint's migration and API startup phases. Set to 0 for a single
+# fail-fast attempt per phase. Each connection attempt is separately bounded
+# so an unreachable host cannot consume the whole budget. A native pgx
+# connect_timeout in DATABASE_URL, PGCONNECT_TIMEOUT, or a service file takes
+# precedence over MULTICA_DATABASE_CONNECT_TIMEOUT for compatibility.
+# MULTICA_DATABASE_STARTUP_TIMEOUT=3m
+# MULTICA_DATABASE_CONNECT_TIMEOUT=5s

 # Server
 # APP_ENV gates production safety checks. Docker self-host pins APP_ENV to
@@ -100,6 +100,8 @@ jobs:
              - 'scripts/agent-cli-command-names.txt'
              - 'scripts/go-test-with-agent-cli-guard.sh'
              - 'scripts/helm-config.test.sh'
+              - 'scripts/entrypoint.test.sh'
+              - 'docker/entrypoint.sh'
              - 'scripts/test-go.sh'
              - 'scripts/test-go.test.sh'
              # makefile-build.test.sh asserts how `make build` names its
@@ -502,6 +504,10 @@ jobs:
        if: ${{ needs.changes.outputs.backend == 'true' }}
        run: bash scripts/helm-config.test.sh

+      - name: Test backend entrypoint signals
+        if: ${{ needs.changes.outputs.backend == 'true' }}
+        run: bash scripts/entrypoint.test.sh
+
      - name: Test build output naming
        if: ${{ needs.changes.outputs.backend == 'true' }}
        run: bash scripts/makefile-build.test.sh
@@ -33,6 +33,8 @@ MULTICA_PUBLIC_URL=https://api.multica.example.com
 | `DATABASE_URL` | ローカルの `multica` データベース | PostgreSQL 接続文字列 |
 | `DATABASE_MAX_CONNS` | `25` | API プロセスあたりの最大データベース接続数 |
 | `DATABASE_MIN_CONNS` | `5` | API プロセスが保持する最小接続数 |
+| `MULTICA_DATABASE_STARTUP_TIMEOUT` | `3m` | コンテナ起動時に migration と API が共有する、一時的なデータベース障害への再試行時間。`0` にすると各フェーズで 1 回だけ試行します |
+| `MULTICA_DATABASE_CONNECT_TIMEOUT` | `5s` | 起動時の接続試行ごとのフォールバックタイムアウト。pgx ネイティブの `connect_timeout`、`PGCONNECT_TIMEOUT`、service 設定が優先されます |
 | `PORT` | `8080` | API のリッスンポート |
 | `JWT_SECRET` | 本番環境で必須 | ログイン JWT と一部の署名フローに使うシークレット。本番では空または既知のプレースホルダーだと起動を拒否 |
 | `APP_ENV` | 空 | プロダクションでは `production` を設定 |
@@ -33,6 +33,8 @@ MULTICA_PUBLIC_URL=https://api.multica.example.com
 | `DATABASE_URL` | 로컬 `multica` 데이터베이스 | PostgreSQL 연결 주소 |
 | `DATABASE_MAX_CONNS` | `25` | API 프로세스 하나의 최대 데이터베이스 연결 수 |
 | `DATABASE_MIN_CONNS` | `5` | API 프로세스 하나가 유지하는 최소 연결 수 |
+| `MULTICA_DATABASE_STARTUP_TIMEOUT` | `3m` | 컨테이너 시작 중 migration과 API가 공유하는 일시적인 데이터베이스 장애 재시도 시간. `0`으로 설정하면 각 단계에서 한 번만 시도 |
+| `MULTICA_DATABASE_CONNECT_TIMEOUT` | `5s` | 시작 중 각 연결 시도의 폴백 제한 시간. pgx 기본 `connect_timeout`, `PGCONNECT_TIMEOUT` 또는 service 설정이 우선 적용됨 |
 | `PORT` | `8080` | API listen port |
 | `JWT_SECRET` | 프로덕션에서 필수 | 로그인 JWT와 일부 서명 절차에서 사용하는 키. 프로덕션에서는 비어 있거나 알려진 자리표시자 값으로 부팅 거부 |
 | `APP_ENV` | 비어 있음 | 프로덕션 환경에서는 `production`으로 설정 |
@@ -33,6 +33,8 @@ You also need to pick an email service, otherwise verification codes and invitat
 | `DATABASE_URL` | local `multica` database | PostgreSQL connection string |
 | `DATABASE_MAX_CONNS` | `25` | Maximum database connections per API process |
 | `DATABASE_MIN_CONNS` | `5` | Minimum connections each API process keeps |
+| `MULTICA_DATABASE_STARTUP_TIMEOUT` | `3m` | Container-level retry budget shared by migration and API startup; set to `0` for one fail-fast attempt per phase |
+| `MULTICA_DATABASE_CONNECT_TIMEOUT` | `5s` | Fallback timeout for each startup connection attempt; native pgx `connect_timeout`, `PGCONNECT_TIMEOUT`, or service settings take precedence |
 | `PORT` | `8080` | API listen port |
 | `JWT_SECRET` | required in production | Secret for sign-in JWTs and some signing flows; production refuses to boot on empty or known placeholders |
 | `APP_ENV` | empty | Set to `production` in production |
@@ -33,6 +33,8 @@ MULTICA_PUBLIC_URL=https://api.multica.example.com
 | `DATABASE_URL` | 本地 `multica` 数据库 | PostgreSQL 连接地址 |
 | `DATABASE_MAX_CONNS` | `25` | 单个 API 进程的最大数据库连接数 |
 | `DATABASE_MIN_CONNS` | `5` | 单个 API 进程保留的最小连接数 |
+| `MULTICA_DATABASE_STARTUP_TIMEOUT` | `3m` | 容器启动过程中，migration 与 API 共享的暂时性数据库故障重试时长；设为 `0` 时每个阶段只尝试一次 |
+| `MULTICA_DATABASE_CONNECT_TIMEOUT` | `5s` | 启动期间单次数据库连接的兜底超时；pgx 原生 `connect_timeout`、`PGCONNECT_TIMEOUT` 或 service 配置优先 |
 | `PORT` | `8080` | API 监听端口 |
 | `JWT_SECRET` | 生产环境必填 | 登录 JWT、部分签名流程使用的密钥；生产环境对空值或已知占位值拒绝启动 |
 | `APP_ENV` | 空 | 生产环境设为 `production` |
@@ -80,15 +80,15 @@ spec:
          {{- end }}
          # The entrypoint runs `./migrate up` before serving traffic. On a
          # cold cluster (Postgres still coming up) this can take minutes. The
-          # startupProbe holds off liveness/readiness until /healthz is OK,
-          # giving migrations a ~5 min budget (30 * 10s) so the pod is never
-          # killed mid-migration. Once it passes, the aggressive livenessProbe
-          # below takes over.
+          # startupProbe holds off liveness/readiness until /healthz is OK.
+          # Its 10 minute window exceeds the shared 3 minute database retry
+          # budget and leaves time for migrations before kubelet may restart
+          # the pod.
          startupProbe:
            httpGet:
              path: /healthz
              port: 8080
-            failureThreshold: 30
+            failureThreshold: 60
            periodSeconds: 10
          readinessProbe:
            httpGet:
@@ -98,7 +98,9 @@ spec:
            periodSeconds: 10
          livenessProbe:
            httpGet:
-              path: /healthz
+              # Liveness must not depend on PostgreSQL. A database outage makes
+              # the pod NotReady, but should not restart an otherwise live API.
+              path: /health
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 30
@@ -7,6 +7,8 @@ metadata:
 data:
  # --- Backend ---
  PORT: "8080"
+  MULTICA_DATABASE_STARTUP_TIMEOUT: {{ .Values.backend.config.databaseStartupTimeout | quote }}
+  MULTICA_DATABASE_CONNECT_TIMEOUT: {{ .Values.backend.config.databaseConnectTimeout | quote }}
  APP_ENV: {{ .Values.backend.config.appEnv | quote }}
  MULTICA_APP_URL: {{ .Values.backend.config.appUrl | quote }}
  FRONTEND_ORIGIN: {{ .Values.backend.config.frontendOrigin | quote }}
@@ -98,8 +98,7 @@ backend:
      # ReadWriteOnce, which prevents `backend.replicas > 1` from scheduling a
      # second pod (Multi-Attach error); disabling persistence — or switching
      # accessModes to a ReadWriteMany class — removes that constraint on the
-      # storage side. Note: backend startup migrations are not yet serialized
-      # across pods, so multi-replica still requires care beyond the chart.
+      # storage side.
      enabled: true
      size: 5Gi
      # Leave empty to use the cluster's default StorageClass.
@@ -113,6 +112,13 @@ backend:
        - ReadWriteOnce
  # All non-secret backend env. Secret values come from `existingSecret`.
  config:
+    # The migrator and API server retry transient PostgreSQL startup failures
+    # within this budget. Set to 0 to restore fail-fast startup behavior.
+    databaseStartupTimeout: 3m
+    # Fallback bound for each PostgreSQL connection attempt during startup.
+    # Native connect_timeout / PGCONNECT_TIMEOUT / service settings take
+    # precedence so existing deployments keep their pgx connection behavior.
+    databaseConnectTimeout: 5s
    appEnv: production
    appUrl: http://multica.dev.lan
    frontendOrigin: http://multica.dev.lan
@@ -96,6 +96,8 @@ services:
      APP_ENV: ${APP_ENV:-production}
      MULTICA_DEV_VERIFICATION_CODE: ${MULTICA_DEV_VERIFICATION_CODE:-}
      MULTICA_APP_URL: ${MULTICA_APP_URL:-http://localhost:3000}
+      MULTICA_DATABASE_STARTUP_TIMEOUT: ${MULTICA_DATABASE_STARTUP_TIMEOUT:-3m}
+      MULTICA_DATABASE_CONNECT_TIMEOUT: ${MULTICA_DATABASE_CONNECT_TIMEOUT:-5s}
      MULTICA_SHUTDOWN_HOLD_DURATION: ${MULTICA_SHUTDOWN_HOLD_DURATION:-}
      MULTICA_RUNTIME_RECONNECT_GRACE: ${MULTICA_RUNTIME_RECONNECT_GRACE:-}
      ALLOW_SIGNUP: ${ALLOW_SIGNUP:-true}
@@ -1,8 +1,41 @@
 #!/bin/sh
-set -e
+set -eu
+
+# The migrator and API server are separate processes, but startup retry time is
+# one container-level budget. Both binaries derive their remaining time from
+# this entrypoint timestamp. Always overwrite inherited values to avoid reusing
+# a stale timestamp after a container restart.
+export MULTICA_INTERNAL_DATABASE_STARTUP_STARTED_AT_UNIX="$(date +%s)"
+
+migrate_pid=""
+
+stop_migration() {
+  signal="$1"
+  exit_code="$2"
+  trap - TERM INT
+  if [ -n "$migrate_pid" ]; then
+    kill "-$signal" "$migrate_pid" 2>/dev/null || true
+    wait "$migrate_pid" 2>/dev/null || true
+  fi
+  exit "$exit_code"
+}
+
+trap 'stop_migration TERM 143' TERM
+trap 'stop_migration INT 130' INT

 echo "Running database migrations..."
-./migrate up
+./migrate up &
+migrate_pid=$!
+if wait "$migrate_pid"; then
+  migrate_status=0
+else
+  migrate_status=$?
+fi
+migrate_pid=""
+trap - TERM INT
+if [ "$migrate_status" -ne 0 ]; then
+  exit "$migrate_status"
+fi

 echo "Starting server..."
 exec ./server
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TEST_DIR="$(mktemp -d)"
+ENTRYPOINT_PID=""
+
+cleanup() {
+  if [ -n "$ENTRYPOINT_PID" ]; then
+    kill -KILL "$ENTRYPOINT_PID" 2>/dev/null || true
+  fi
+  if [ -s "$TEST_DIR/migrate.pid" ]; then
+    kill -KILL "$(cat "$TEST_DIR/migrate.pid")" 2>/dev/null || true
+  fi
+  rm -rf "$TEST_DIR"
+}
+trap cleanup EXIT
+
+cp "$ROOT_DIR/docker/entrypoint.sh" "$TEST_DIR/entrypoint.sh"
+
+cat >"$TEST_DIR/migrate" <<'SCRIPT'
+#!/bin/sh
+trap 'printf "TERM\n" >"$MIGRATE_SIGNAL_FILE"; exit 143' TERM
+printf '%s\n' "$$" >"$MIGRATE_PID_FILE"
+printf '%s\n' "$MULTICA_INTERNAL_DATABASE_STARTUP_STARTED_AT_UNIX" >"$MIGRATE_STARTED_AT_FILE"
+while :; do
+  sleep 1
+done
+SCRIPT
+
+cat >"$TEST_DIR/server" <<'SCRIPT'
+#!/bin/sh
+printf 'started\n' >"$SERVER_STARTED_FILE"
+printf '%s\n' "$MULTICA_INTERNAL_DATABASE_STARTUP_STARTED_AT_UNIX" >"$SERVER_STARTED_AT_FILE"
+SCRIPT
+
+chmod +x "$TEST_DIR/entrypoint.sh" "$TEST_DIR/migrate" "$TEST_DIR/server"
+
+export MIGRATE_PID_FILE="$TEST_DIR/migrate.pid"
+export MIGRATE_SIGNAL_FILE="$TEST_DIR/migrate.signal"
+export MIGRATE_STARTED_AT_FILE="$TEST_DIR/migrate.started-at"
+export SERVER_STARTED_FILE="$TEST_DIR/server.started"
+export SERVER_STARTED_AT_FILE="$TEST_DIR/server.started-at"
+
+(
+  cd "$TEST_DIR"
+  exec ./entrypoint.sh
+) &
+ENTRYPOINT_PID=$!
+
+for _ in $(seq 1 50); do
+  if [ -s "$MIGRATE_PID_FILE" ]; then
+    break
+  fi
+  sleep 0.1
+done
+if [ ! -s "$MIGRATE_PID_FILE" ]; then
+  echo "migrator did not start"
+  exit 1
+fi
+
+kill -TERM "$ENTRYPOINT_PID"
+set +e
+wait "$ENTRYPOINT_PID"
+status=$?
+set -e
+ENTRYPOINT_PID=""
+
+if [ "$status" -ne 143 ]; then
+  echo "entrypoint exited with $status, want 143 after SIGTERM"
+  exit 1
+fi
+if [ "$(cat "$MIGRATE_SIGNAL_FILE" 2>/dev/null || true)" != "TERM" ]; then
+  echo "entrypoint did not forward SIGTERM to the migrator"
+  exit 1
+fi
+if [ -e "$SERVER_STARTED_FILE" ]; then
+  echo "entrypoint started the server after interrupted migrations"
+  exit 1
+fi
+
+cat >"$TEST_DIR/migrate" <<'SCRIPT'
+#!/bin/sh
+printf '%s\n' "$MULTICA_INTERNAL_DATABASE_STARTUP_STARTED_AT_UNIX" >"$MIGRATE_STARTED_AT_FILE"
+exit 0
+SCRIPT
+chmod +x "$TEST_DIR/migrate"
+
+(
+  cd "$TEST_DIR"
+  ./entrypoint.sh
+)
+if [ "$(cat "$SERVER_STARTED_FILE" 2>/dev/null || true)" != "started" ]; then
+  echo "entrypoint did not start the server after successful migrations"
+  exit 1
+fi
+started_at="$(cat "$MIGRATE_STARTED_AT_FILE" 2>/dev/null || true)"
+if [ -z "$started_at" ] || [ "$started_at" != "$(cat "$SERVER_STARTED_AT_FILE" 2>/dev/null || true)" ]; then
+  echo "entrypoint did not share one database startup timestamp"
+  exit 1
+fi
+case "$started_at" in
+  *[!0-9]*)
+    echo "entrypoint database startup timestamp is not a Unix timestamp"
+    exit 1
+    ;;
+esac
+
+echo "entrypoint startup and signal forwarding ok"
@@ -35,8 +35,19 @@ default_config="$(
 require_rendered_value "$default_config" 'MULTICA_VCS_INTEGRATION_ENABLED: "true"'
 require_rendered_value "$default_config" 'MULTICA_ENTITLEMENT_POLICY_ENABLED: "false"'
 require_rendered_value "$default_config" 'MULTICA_ENTITLEMENT_POLICY_URL: ""'
+require_rendered_value "$default_config" 'MULTICA_DATABASE_STARTUP_TIMEOUT: "3m"'
+require_rendered_value "$default_config" 'MULTICA_DATABASE_CONNECT_TIMEOUT: "5s"'
 reject_rendered_value "$default_config" 'MULTICA_ENTITLEMENT_SERVICE_TOKEN'

+default_backend="$(
+  helm template multica "$CHART_DIR" \
+    --show-only templates/backend.yaml
+)"
+require_rendered_value "$default_backend" 'failureThreshold: 60'
+liveness_block="$(sed -n '/livenessProbe:/,/resources:/p' <<<"$default_backend")"
+require_rendered_value "$liveness_block" 'path: /health'
+reject_rendered_value "$liveness_block" 'path: /healthz'
+
 disabled_config="$(
  helm template multica "$CHART_DIR" \
    --show-only templates/configmap.yaml \
@@ -56,6 +56,8 @@ require_config "$config" 'FRONTEND_ORIGIN: http://localhost:3100'
 require_config "$config" 'GOOGLE_REDIRECT_URI: http://localhost:3100/auth/callback'
 require_config "$config" 'MULTICA_APP_URL: http://localhost:3100'
 require_config "$config" 'SMTP_FROM_EMAIL: [email protected]'
+require_config "$config" 'MULTICA_DATABASE_STARTUP_TIMEOUT: 3m'
+require_config "$config" 'MULTICA_DATABASE_CONNECT_TIMEOUT: 5s'

 # The backend environment is an explicit allowlist, so a variable documented in
 # .env.example but missing here silently never reaches the container: the
@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/signal"
 	"strings"
+	"syscall"

 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"github.com/multica-ai/multica/server/internal/attributionbackfill"
+	"github.com/multica-ai/multica/server/internal/dbstartup"
 	"github.com/multica-ai/multica/server/internal/logger"
 	"github.com/multica-ai/multica/server/internal/migrations"
 	"github.com/multica-ai/multica/server/internal/taskusagebackfill"
@@ -536,32 +539,64 @@ func main() {
 		dbURL = "postgres://multica:multica@localhost:5432/multica?sslmode=disable"
 	}

-	ctx := context.Background()
-	pool, err := pgxpool.New(ctx, dbURL)
+	startupSettings := dbstartup.SettingsFromEnv()
+	pool, err := dbstartup.NewPool(context.Background(), dbURL, startupSettings.ConnectTimeout)
 	if err != nil {
 		slog.Error("unable to connect to database", "error", err)
 		os.Exit(1)
 	}
 	defer pool.Close()

-	if err := pool.Ping(ctx); err != nil {
-		slog.Error("unable to ping database", "error", err)
-		os.Exit(1)
-	}
-
 	files, err := migrations.Files(direction)
 	if err != nil {
 		slog.Error("failed to find migration files", "error", err)
 		os.Exit(1)
 	}

-	if err := runMigrations(ctx, pool, runOptions{
+	options := runOptions{
 		Direction:  direction,
 		Files:      files,
 		Hooks:      hooksForDirection(direction),
 		Conditions: conditionsForDirection(direction),
-	}); err != nil {
-		slog.Error("migration run failed", "error", err)
+	}
+	startupCtx, stopStartup := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stopStartup()
+	retryOptions := startupSettings.RetryOptions()
+	retryOptions.ShouldRetry = dbstartup.IsTransientDatabaseError
+	retryOptions.OnRetry = func(event dbstartup.RetryEvent) {
+		slog.Warn("database unavailable before migrations; retrying",
+			"attempt", event.Attempt,
+			"retry_in", event.Delay,
+			"error", event.Err,
+		)
+	}
+	if err := dbstartup.Retry(startupCtx, retryOptions, pool.Ping); err != nil {
+		slog.Error("unable to ping database", "error", err)
+		os.Exit(1)
+	}
+
+	migrationErr := runMigrations(startupCtx, pool, options)
+	if migrationErr != nil && startupSettings.StartupTimeout > 0 && dbstartup.IsTransientDatabaseError(migrationErr) {
+		slog.Warn("migration interrupted by database unavailability; retrying", "error", migrationErr)
+		migrationRetryOptions := startupSettings.RetryOptions()
+		migrationRetryOptions.ShouldRetry = dbstartup.IsTransientDatabaseError
+		migrationRetryOptions.AllowOperationPastTimeout = true
+		migrationRetryOptions.OnRetry = func(event dbstartup.RetryEvent) {
+			slog.Warn("database unavailable during migration retry",
+				"attempt", event.Attempt,
+				"retry_in", event.Delay,
+				"error", event.Err,
+			)
+		}
+		migrationErr = dbstartup.Retry(startupCtx, migrationRetryOptions, func(attemptCtx context.Context) error {
+			if err := pool.Ping(attemptCtx); err != nil {
+				return fmt.Errorf("ping database: %w", err)
+			}
+			return runMigrations(attemptCtx, pool, options)
+		})
+	}
+	if migrationErr != nil {
+		slog.Error("migration run failed", "error", migrationErr)
 		os.Exit(1)
 	}

@@ -10,6 +10,7 @@ import (
 	"time"

 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/multica-ai/multica/server/internal/dbstartup"
 )

 const (
@@ -52,10 +53,10 @@ const (
 //
 // pgx's own built-in default (max(4, NumCPU)) is intentionally NOT used as a
 // fallback — it is the value that caused the prod incident.
-func newDBPool(ctx context.Context, dbURL string) (*pgxpool.Pool, error) {
-	cfg, err := pgxpool.ParseConfig(dbURL)
+func newDBPool(ctx context.Context, dbURL string, connectTimeout time.Duration) (*pgxpool.Pool, error) {
+	cfg, err := dbstartup.ParsePoolConfig(dbURL, connectTimeout)
 	if err != nil {
-		return nil, fmt.Errorf("parse database url: %w", err)
+		return nil, err
 	}

 	urlParams := poolParamsFromURL(dbURL)
@@ -16,6 +16,7 @@ import (
 	"github.com/multica-ai/multica/server/internal/analytics"
 	"github.com/multica-ai/multica/server/internal/auth"
 	"github.com/multica-ai/multica/server/internal/daemonws"
+	"github.com/multica-ai/multica/server/internal/dbstartup"
 	"github.com/multica-ai/multica/server/internal/events"
 	"github.com/multica-ai/multica/server/internal/handler"
 	"github.com/multica-ai/multica/server/internal/logger"
@@ -322,21 +323,33 @@ func main() {
 		dbURL = "postgres://multica:multica@localhost:5432/multica?sslmode=disable"
 	}

-	// Connect to database
-	ctx := context.Background()
-	pool, err := newDBPool(ctx, dbURL)
+	startupSettings := dbstartup.SettingsFromEnv()
+	pool, err := newDBPool(context.Background(), dbURL, startupSettings.ConnectTimeout)
 	if err != nil {
 		slog.Error("unable to connect to database", "error", err)
 		os.Exit(1)
 	}
 	defer pool.Close()

-	if err := pool.Ping(ctx); err != nil {
+	startupCtx, stopStartup := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	retryOptions := startupSettings.RetryOptions()
+	retryOptions.ShouldRetry = dbstartup.IsTransientDatabaseError
+	retryOptions.OnRetry = func(event dbstartup.RetryEvent) {
+		slog.Warn("database unavailable during server startup; retrying",
+			"attempt", event.Attempt,
+			"retry_in", event.Delay,
+			"error", event.Err,
+		)
+	}
+	if err := dbstartup.Retry(startupCtx, retryOptions, pool.Ping); err != nil {
+		stopStartup()
 		slog.Error("unable to ping database", "error", err)
 		os.Exit(1)
 	}
+	stopStartup()
 	slog.Info("connected to database")
 	logPoolConfig(pool)
+	ctx := context.Background()

 	bus := events.New()
 	hub := realtime.NewHub()
@@ -0,0 +1,324 @@
+package dbstartup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math/rand/v2"
+	"net"
+	"net/url"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+const (
+	StartupTimeoutEnv = "MULTICA_DATABASE_STARTUP_TIMEOUT"
+	ConnectTimeoutEnv = "MULTICA_DATABASE_CONNECT_TIMEOUT"
+	// startupStartedAtEnv is set by the container entrypoint so the migrator
+	// and API server consume one shared retry budget. It is intentionally not
+	// a user-facing setting.
+	startupStartedAtEnv = "MULTICA_INTERNAL_DATABASE_STARTUP_STARTED_AT_UNIX"
+
+	DefaultStartupTimeout = 3 * time.Minute
+	DefaultConnectTimeout = 5 * time.Second
+	defaultInitialBackoff = time.Second
+	defaultMaxBackoff     = 30 * time.Second
+)
+
+var keywordConnectTimeoutPattern = regexp.MustCompile(`(?:^|\s)connect_timeout\s*=`)
+
+// Settings bounds database startup retries and individual connection attempts.
+type Settings struct {
+	StartupTimeout time.Duration
+	ConnectTimeout time.Duration
+
+	startupDeadline time.Time
+	now             func() time.Time
+}
+
+// SettingsFromEnv reads the shared startup settings used by the migrator and
+// API server. A zero startup timeout preserves an explicit fail-fast option.
+func SettingsFromEnv() Settings {
+	return settingsFromEnv(time.Now)
+}
+
+func settingsFromEnv(now func() time.Time) Settings {
+	startupTimeout := envDuration(StartupTimeoutEnv, DefaultStartupTimeout, true)
+	settings := Settings{
+		StartupTimeout: startupTimeout,
+		ConnectTimeout: envDuration(ConnectTimeoutEnv, DefaultConnectTimeout, false),
+		now:            now,
+	}
+	if startupTimeout <= 0 {
+		return settings
+	}
+
+	startedAt := now()
+	if raw := os.Getenv(startupStartedAtEnv); raw != "" {
+		unixSeconds, err := strconv.ParseInt(raw, 10, 64)
+		if err == nil && unixSeconds > 0 {
+			startedAt = time.Unix(unixSeconds, 0)
+		} else {
+			if err == nil {
+				err = errors.New("timestamp must be positive")
+			}
+			slog.Warn("invalid internal database startup timestamp, starting a new retry budget",
+				"name", startupStartedAtEnv,
+				"value", raw,
+				"error", err,
+			)
+		}
+	}
+	settings.startupDeadline = startedAt.Add(startupTimeout)
+	return settings
+}
+
+// ParsePoolConfig applies connectTimeout as a compatibility fallback. Native
+// pgx connect_timeout settings take precedence because pgx configures both its
+// whole-connection deadline and DialFunc from the same value.
+func ParsePoolConfig(databaseURL string, connectTimeout time.Duration) (*pgxpool.Config, error) {
+	cfg, err := pgxpool.ParseConfig(databaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse database url: %w", err)
+	}
+	if pgxConnectTimeoutConfigured(databaseURL, cfg.ConnConfig.ConnectTimeout) {
+		return cfg, nil
+	}
+	if connectTimeout <= 0 {
+		connectTimeout = DefaultConnectTimeout
+	}
+	cfg.ConnConfig.ConnectTimeout = connectTimeout
+	dialer := &net.Dialer{Timeout: connectTimeout}
+	cfg.ConnConfig.DialFunc = dialer.DialContext
+	return cfg, nil
+}
+
+func pgxConnectTimeoutConfigured(databaseURL string, parsedTimeout time.Duration) bool {
+	if parsedTimeout != 0 || os.Getenv("PGCONNECT_TIMEOUT") != "" {
+		return true
+	}
+
+	if strings.HasPrefix(databaseURL, "postgres://") || strings.HasPrefix(databaseURL, "postgresql://") {
+		parsedURL, err := url.Parse(databaseURL)
+		if err == nil {
+			query := parsedURL.Query()
+			_, hasConnectTimeout := query["connect_timeout"]
+			return hasConnectTimeout
+		}
+	}
+
+	return keywordConnectTimeoutPattern.MatchString(databaseURL)
+}
+
+// NewPool creates a startup pool with the shared connection timeout.
+func NewPool(ctx context.Context, databaseURL string, connectTimeout time.Duration) (*pgxpool.Pool, error) {
+	cfg, err := ParsePoolConfig(databaseURL, connectTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return pgxpool.NewWithConfig(ctx, cfg)
+}
+
+// RetryEvent describes a failed attempt before the next backoff begins.
+type RetryEvent struct {
+	Attempt int
+	Delay   time.Duration
+	Err     error
+}
+
+// RetryOptions controls a bounded exponential-backoff loop. Jitter and Sleep
+// are injectable so tests can cover the full sequence without wall-clock waits.
+type RetryOptions struct {
+	Timeout        time.Duration
+	InitialBackoff time.Duration
+	MaxBackoff     time.Duration
+	Jitter         func(time.Duration) time.Duration
+	Sleep          func(context.Context, time.Duration) error
+	ShouldRetry    func(error) bool
+	// AllowOperationPastTimeout keeps a long-running operation on the parent
+	// context while the timeout still bounds subsequent retries and backoffs.
+	AllowOperationPastTimeout bool
+	OnRetry                   func(RetryEvent)
+}
+
+// RetryOptions returns production defaults with the startup budget remaining
+// at the time of the call. Repeated phases therefore share one wall-clock
+// deadline, so successful work between retry phases also consumes the budget.
+// Once it expires, later phases make one fail-fast attempt instead of reserving
+// a new minimum window that could exceed the container-level retry bound.
+func (s Settings) RetryOptions() RetryOptions {
+	timeout := s.StartupTimeout
+	if timeout > 0 && !s.startupDeadline.IsZero() {
+		now := s.now
+		if now == nil {
+			now = time.Now
+		}
+		timeout = s.startupDeadline.Sub(now())
+		if timeout < 0 {
+			timeout = 0
+		}
+	}
+	return RetryOptions{
+		Timeout:        timeout,
+		InitialBackoff: defaultInitialBackoff,
+		MaxBackoff:     defaultMaxBackoff,
+	}
+}
+
+// Retry runs operation immediately and then retries transient failures with
+// jittered exponential backoff until it succeeds, the parent is cancelled, or
+// the configured startup budget expires. Timeout zero performs one attempt.
+func Retry(ctx context.Context, opts RetryOptions, operation func(context.Context) error) error {
+	if operation == nil {
+		return errors.New("database startup operation is nil")
+	}
+	if opts.Timeout <= 0 {
+		return operation(ctx)
+	}
+
+	retryCtx, cancel := context.WithTimeout(ctx, opts.Timeout)
+	defer cancel()
+
+	backoff := opts.InitialBackoff
+	if backoff <= 0 {
+		backoff = defaultInitialBackoff
+	}
+	maxBackoff := opts.MaxBackoff
+	if maxBackoff <= 0 {
+		maxBackoff = defaultMaxBackoff
+	}
+	if backoff > maxBackoff {
+		backoff = maxBackoff
+	}
+	jitter := opts.Jitter
+	if jitter == nil {
+		jitter = jitterDelay
+	}
+	sleep := opts.Sleep
+	if sleep == nil {
+		sleep = sleepContext
+	}
+
+	var lastErr error
+	for attempt := 1; ; attempt++ {
+		if err := retryCtx.Err(); err != nil {
+			return retryStopped(attempt-1, err, lastErr)
+		}
+
+		operationCtx := retryCtx
+		if opts.AllowOperationPastTimeout {
+			operationCtx = ctx
+		}
+		err := operation(operationCtx)
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if err := retryCtx.Err(); err != nil {
+			return retryStopped(attempt, err, lastErr)
+		}
+		if opts.ShouldRetry != nil && !opts.ShouldRetry(err) {
+			return err
+		}
+
+		delay := jitter(backoff)
+		if delay < 0 {
+			delay = 0
+		}
+		if opts.OnRetry != nil {
+			opts.OnRetry(RetryEvent{Attempt: attempt, Delay: delay, Err: err})
+		}
+		if err := sleep(retryCtx, delay); err != nil {
+			cause := retryCtx.Err()
+			if cause == nil {
+				cause = err
+			}
+			return retryStopped(attempt, cause, lastErr)
+		}
+
+		if backoff < maxBackoff {
+			backoff *= 2
+			if backoff <= 0 || backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+		}
+	}
+}
+
+func retryStopped(attempts int, cause, lastErr error) error {
+	if lastErr == nil {
+		return fmt.Errorf("database startup stopped before an attempt completed: %w", cause)
+	}
+	return fmt.Errorf("database startup failed after %d attempt(s): %w (last error: %w)", attempts, cause, lastErr)
+}
+
+func jitterDelay(delay time.Duration) time.Duration {
+	// Keep each delay within 80-100% so concurrent pods do not reconnect in
+	// lockstep and the configured maximum remains a hard upper bound.
+	return time.Duration(float64(delay) * (0.8 + rand.Float64()*0.2))
+}
+
+// IsTransientDatabaseError limits whole-migration retries to connection and
+// availability failures. SQL and migration-definition errors fail immediately
+// instead of consuming the startup retry budget.
+func IsTransientDatabaseError(err error) bool {
+	if err == nil || errors.Is(err, context.Canceled) {
+		return false
+	}
+
+	var pgErr *pgconn.PgError
+	if errors.As(err, &pgErr) {
+		return strings.HasPrefix(pgErr.Code, "08") ||
+			pgErr.Code == "57P01" || // admin_shutdown
+			pgErr.Code == "57P02" || // crash_shutdown
+			pgErr.Code == "57P03" || // cannot_connect_now
+			pgErr.Code == "53300" // too_many_connections
+	}
+
+	var connectErr *pgconn.ConnectError
+	if errors.As(err, &connectErr) {
+		return true
+	}
+	var networkErr net.Error
+	if errors.As(err, &networkErr) {
+		return true
+	}
+	return pgconn.SafeToRetry(err) || pgconn.Timeout(err)
+}
+
+func sleepContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func envDuration(name string, def time.Duration, allowZero bool) time.Duration {
+	raw := os.Getenv(name)
+	if raw == "" {
+		return def
+	}
+	value, err := time.ParseDuration(raw)
+	valid := err == nil && (value > 0 || allowZero && value == 0)
+	if !valid {
+		slog.Warn("invalid env var, using default",
+			"name", name,
+			"value", raw,
+			"default", def.String(),
+			"error", err,
+		)
+		return def
+	}
+	return value
+}
@@ -0,0 +1,481 @@
+package dbstartup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+func TestSettingsFromEnv(t *testing.T) {
+	t.Setenv(StartupTimeoutEnv, "45s")
+	t.Setenv(ConnectTimeoutEnv, "2s")
+
+	got := SettingsFromEnv()
+	if got.StartupTimeout != 45*time.Second {
+		t.Fatalf("StartupTimeout = %s, want 45s", got.StartupTimeout)
+	}
+	if got.ConnectTimeout != 2*time.Second {
+		t.Fatalf("ConnectTimeout = %s, want 2s", got.ConnectTimeout)
+	}
+}
+
+func TestSettingsFromEnvAllowsFailFastButKeepsConnectionTimeoutBounded(t *testing.T) {
+	t.Setenv(StartupTimeoutEnv, "0")
+	t.Setenv(ConnectTimeoutEnv, "0")
+
+	got := SettingsFromEnv()
+	if got.StartupTimeout != 0 {
+		t.Fatalf("StartupTimeout = %s, want 0", got.StartupTimeout)
+	}
+	if got.ConnectTimeout != DefaultConnectTimeout {
+		t.Fatalf("ConnectTimeout = %s, want %s", got.ConnectTimeout, DefaultConnectTimeout)
+	}
+}
+
+func TestSettingsFromEnvSharesEntrypointStartupBudget(t *testing.T) {
+	current := time.Unix(2_000_000_000, 0)
+	t.Setenv(StartupTimeoutEnv, "3m")
+	t.Setenv(startupStartedAtEnv, fmt.Sprintf("%d", current.Unix()))
+
+	settings := settingsFromEnv(func() time.Time { return current })
+	if got := settings.RetryOptions().Timeout; got != 3*time.Minute {
+		t.Fatalf("initial retry timeout = %s, want 3m", got)
+	}
+
+	current = current.Add(70 * time.Second)
+	if got := settings.RetryOptions().Timeout; got != 110*time.Second {
+		t.Fatalf("remaining retry timeout = %s, want 1m50s", got)
+	}
+
+	current = current.Add(2 * time.Minute)
+	if got := settings.RetryOptions().Timeout; got != 0 {
+		t.Fatalf("expired retry timeout = %s, want 0", got)
+	}
+}
+
+func TestParsePoolConfigAppliesConnectTimeout(t *testing.T) {
+	t.Setenv("PGCONNECT_TIMEOUT", "")
+	t.Setenv("PGSERVICE", "")
+
+	cfg, err := ParsePoolConfig("postgres://user:pass@localhost:5432/db?sslmode=disable", 7*time.Second)
+	if err != nil {
+		t.Fatalf("ParsePoolConfig: %v", err)
+	}
+	if cfg.ConnConfig.ConnectTimeout != 7*time.Second {
+		t.Fatalf("ConnectTimeout = %s, want 7s", cfg.ConnConfig.ConnectTimeout)
+	}
+}
+
+func TestParsePoolConfigPreservesNativeConnectTimeout(t *testing.T) {
+	t.Setenv("PGCONNECT_TIMEOUT", "")
+	t.Setenv("PGSERVICE", "")
+
+	tests := []struct {
+		name          string
+		databaseURL   string
+		multicaValue  time.Duration
+		wantPGXNative time.Duration
+	}{
+		{
+			name:          "URL longer than Multica fallback",
+			databaseURL:   "postgres://user:pass@localhost:5432/db?sslmode=disable&connect_timeout=30",
+			multicaValue:  5 * time.Second,
+			wantPGXNative: 30 * time.Second,
+		},
+		{
+			name:          "URL shorter than explicit Multica value",
+			databaseURL:   "postgres://user:pass@localhost:5432/db?sslmode=disable&connect_timeout=1",
+			multicaValue:  30 * time.Second,
+			wantPGXNative: time.Second,
+		},
+		{
+			name:          "URL explicitly disables timeout",
+			databaseURL:   "postgres://user:pass@localhost:5432/db?sslmode=disable&connect_timeout=0",
+			multicaValue:  5 * time.Second,
+			wantPGXNative: 0,
+		},
+		{
+			name:          "keyword value",
+			databaseURL:   "host=localhost port=5432 user=user password=pass dbname=db sslmode=disable connect_timeout=12",
+			multicaValue:  5 * time.Second,
+			wantPGXNative: 12 * time.Second,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg, err := ParsePoolConfig(tt.databaseURL, tt.multicaValue)
+			if err != nil {
+				t.Fatalf("ParsePoolConfig: %v", err)
+			}
+			if cfg.ConnConfig.ConnectTimeout != tt.wantPGXNative {
+				t.Fatalf("ConnectTimeout = %s, want native pgx value %s", cfg.ConnConfig.ConnectTimeout, tt.wantPGXNative)
+			}
+		})
+	}
+}
+
+func TestParsePoolConfigPreservesPGConnectTimeout(t *testing.T) {
+	t.Setenv("PGSERVICE", "")
+
+	tests := []struct {
+		value string
+		want  time.Duration
+	}{
+		{value: "17", want: 17 * time.Second},
+		{value: "0", want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.value, func(t *testing.T) {
+			t.Setenv("PGCONNECT_TIMEOUT", tt.value)
+			cfg, err := ParsePoolConfig("postgres://user:pass@localhost:5432/db?sslmode=disable", 5*time.Second)
+			if err != nil {
+				t.Fatalf("ParsePoolConfig: %v", err)
+			}
+			if cfg.ConnConfig.ConnectTimeout != tt.want {
+				t.Fatalf("ConnectTimeout = %s, want PGCONNECT_TIMEOUT value %s", cfg.ConnConfig.ConnectTimeout, tt.want)
+			}
+		})
+	}
+}
+
+func TestParsePoolConfigServiceConnectTimeout(t *testing.T) {
+	serviceFile := writePGServiceFile(t)
+	t.Setenv("PGCONNECT_TIMEOUT", "")
+	t.Setenv("PGSERVICEFILE", serviceFile)
+
+	tests := []struct {
+		name        string
+		serviceEnv  string
+		databaseURL string
+		want        time.Duration
+	}{
+		{
+			name:        "PGSERVICE without timeout",
+			serviceEnv:  "without_timeout",
+			databaseURL: "",
+			want:        7 * time.Second,
+		},
+		{
+			name:        "URL service without timeout",
+			databaseURL: fmt.Sprintf("postgres:///?servicefile=%s&service=without_timeout", url.QueryEscape(serviceFile)),
+			want:        7 * time.Second,
+		},
+		{
+			name:        "PGSERVICE with timeout",
+			serviceEnv:  "with_timeout",
+			databaseURL: "",
+			want:        13 * time.Second,
+		},
+		{
+			name:        "URL service with timeout",
+			databaseURL: fmt.Sprintf("postgres:///?servicefile=%s&service=with_timeout", url.QueryEscape(serviceFile)),
+			want:        13 * time.Second,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("PGSERVICE", tt.serviceEnv)
+			cfg, err := ParsePoolConfig(tt.databaseURL, 7*time.Second)
+			if err != nil {
+				t.Fatalf("ParsePoolConfig: %v", err)
+			}
+			if cfg.ConnConfig.ConnectTimeout != tt.want {
+				t.Fatalf("ConnectTimeout = %s, want %s", cfg.ConnConfig.ConnectTimeout, tt.want)
+			}
+		})
+	}
+}
+
+func writePGServiceFile(t *testing.T) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "pg_service.conf")
+	contents := []byte(`[without_timeout]
+host=localhost
+port=5432
+dbname=db
+user=user
+
+[with_timeout]
+host=localhost
+port=5432
+dbname=db
+user=user
+connect_timeout=13
+`)
+	if err := os.WriteFile(path, contents, 0o600); err != nil {
+		t.Fatalf("write service file: %v", err)
+	}
+	return path
+}
+
+func TestRetryRecoversWithCappedExponentialBackoff(t *testing.T) {
+	var attempts int
+	var delays []time.Duration
+	err := Retry(context.Background(), RetryOptions{
+		Timeout:        time.Minute,
+		InitialBackoff: time.Second,
+		MaxBackoff:     5 * time.Second,
+		Jitter:         func(delay time.Duration) time.Duration { return delay },
+		Sleep: func(_ context.Context, delay time.Duration) error {
+			delays = append(delays, delay)
+			return nil
+		},
+	}, func(context.Context) error {
+		attempts++
+		if attempts < 5 {
+			return errors.New("database unavailable")
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Retry: %v", err)
+	}
+	if attempts != 5 {
+		t.Fatalf("attempts = %d, want 5", attempts)
+	}
+	wantDelays := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 5 * time.Second}
+	if !reflect.DeepEqual(delays, wantDelays) {
+		t.Fatalf("delays = %v, want %v", delays, wantDelays)
+	}
+}
+
+func TestRetryStopsWhenParentIsCancelled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	var attempts int
+	err := Retry(ctx, RetryOptions{
+		Timeout: time.Minute,
+		Jitter:  func(delay time.Duration) time.Duration { return delay },
+		Sleep: func(context.Context, time.Duration) error {
+			cancel()
+			return context.Canceled
+		},
+	}, func(context.Context) error {
+		attempts++
+		return errors.New("database unavailable")
+	})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("Retry error = %v, want context.Canceled", err)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1", attempts)
+	}
+}
+
+func TestRetryStopsAtTimeout(t *testing.T) {
+	var attempts int
+	err := Retry(context.Background(), RetryOptions{
+		Timeout: 5 * time.Millisecond,
+		Jitter:  func(delay time.Duration) time.Duration { return delay },
+	}, func(context.Context) error {
+		attempts++
+		return errors.New("database unavailable")
+	})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("Retry error = %v, want context.DeadlineExceeded", err)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1", attempts)
+	}
+}
+
+func TestRetryTimeoutPreservesLastError(t *testing.T) {
+	wantErr := &pgconn.PgError{Code: "57P03", Message: "database is starting"}
+	err := Retry(context.Background(), RetryOptions{
+		Timeout: time.Minute,
+		Sleep: func(context.Context, time.Duration) error {
+			return context.DeadlineExceeded
+		},
+	}, func(context.Context) error {
+		return wantErr
+	})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("Retry error = %v, want context.DeadlineExceeded", err)
+	}
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("Retry error = %v, want wrapped last error %v", err, wantErr)
+	}
+}
+
+func TestRetryWithZeroTimeoutAttemptsOnce(t *testing.T) {
+	wantErr := errors.New("database unavailable")
+	var attempts int
+	err := Retry(context.Background(), RetryOptions{}, func(context.Context) error {
+		attempts++
+		return wantErr
+	})
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("Retry error = %v, want %v", err, wantErr)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1", attempts)
+	}
+}
+
+func TestRetryStopsImmediatelyForPermanentError(t *testing.T) {
+	wantErr := &pgconn.PgError{Code: "42601", Message: "syntax error"}
+	var attempts int
+	err := Retry(context.Background(), RetryOptions{
+		Timeout:     time.Minute,
+		ShouldRetry: IsTransientDatabaseError,
+	}, func(context.Context) error {
+		attempts++
+		return wantErr
+	})
+	if !errors.Is(err, wantErr) {
+		t.Fatalf("Retry error = %v, want %v", err, wantErr)
+	}
+	if attempts != 1 {
+		t.Fatalf("attempts = %d, want 1", attempts)
+	}
+}
+
+func TestRetryCanLeaveLongOperationOnParentContext(t *testing.T) {
+	err := Retry(context.Background(), RetryOptions{
+		Timeout:                   time.Minute,
+		AllowOperationPastTimeout: true,
+	}, func(ctx context.Context) error {
+		if _, ok := ctx.Deadline(); ok {
+			return errors.New("operation unexpectedly inherited retry deadline")
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Retry: %v", err)
+	}
+}
+
+func TestIsTransientDatabaseError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "connection exception", err: &pgconn.PgError{Code: "08006"}, want: true},
+		{name: "database starting", err: &pgconn.PgError{Code: "57P03"}, want: true},
+		{name: "too many connections", err: &pgconn.PgError{Code: "53300"}, want: true},
+		{name: "syntax error", err: &pgconn.PgError{Code: "42601"}, want: false},
+		{name: "connection deadline", err: context.DeadlineExceeded, want: true},
+		{name: "cancelled", err: context.Canceled, want: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsTransientDatabaseError(tt.err); got != tt.want {
+				t.Fatalf("IsTransientDatabaseError(%v) = %v, want %v", tt.err, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRetryRetriesConnectionHandshakeTimeout(t *testing.T) {
+	t.Setenv("PGCONNECT_TIMEOUT", "")
+	t.Setenv("PGSERVICE", "")
+
+	address := startStalledDatabaseListener(t)
+	pool, err := NewPool(
+		context.Background(),
+		fmt.Sprintf("postgres://user:pass@%s/db?sslmode=disable", address),
+		30*time.Millisecond,
+	)
+	if err != nil {
+		t.Fatalf("NewPool: %v", err)
+	}
+	defer pool.Close()
+
+	var attempts int
+	err = Retry(context.Background(), RetryOptions{
+		Timeout:        time.Second,
+		InitialBackoff: time.Millisecond,
+		MaxBackoff:     time.Millisecond,
+		Jitter:         func(delay time.Duration) time.Duration { return delay },
+		ShouldRetry:    IsTransientDatabaseError,
+	}, func(ctx context.Context) error {
+		attempts++
+		return pool.Ping(ctx)
+	})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("Retry error = %v, want context.DeadlineExceeded", err)
+	}
+	if attempts < 2 {
+		t.Fatalf("attempts = %d, want at least 2 for a connection handshake timeout", attempts)
+	}
+}
+
+func TestIsTransientDatabaseErrorFromRefusedConnection(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	address := listener.Addr().String()
+	if err := listener.Close(); err != nil {
+		t.Fatalf("close listener: %v", err)
+	}
+
+	pool, err := NewPool(
+		context.Background(),
+		fmt.Sprintf("postgres://user:pass@%s/db?sslmode=disable", address),
+		100*time.Millisecond,
+	)
+	if err != nil {
+		t.Fatalf("NewPool: %v", err)
+	}
+	defer pool.Close()
+
+	err = pool.Ping(context.Background())
+	if err == nil {
+		t.Fatal("Ping unexpectedly succeeded")
+	}
+	if !IsTransientDatabaseError(err) {
+		t.Fatalf("IsTransientDatabaseError(%v) = false, want true", err)
+	}
+}
+
+func startStalledDatabaseListener(t *testing.T) string {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+
+	var (
+		mu          sync.Mutex
+		connections []net.Conn
+	)
+	acceptDone := make(chan struct{})
+	go func() {
+		defer close(acceptDone)
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			mu.Lock()
+			connections = append(connections, conn)
+			mu.Unlock()
+		}
+	}()
+
+	t.Cleanup(func() {
+		_ = listener.Close()
+		<-acceptDone
+		mu.Lock()
+		defer mu.Unlock()
+		for _, conn := range connections {
+			_ = conn.Close()
+		}
+	})
+
+	return listener.Addr().String()
+}