Files
LobsterAI/scripts/setup-python-runtime.js
T

668 lines
22 KiB
JavaScript

#!/usr/bin/env node
/**
* Prepare bundled Windows Python runtime under resources/python-win.
*
* This script mirrors setup-mingit.js behavior:
* - Supports local offline archive via LOBSTERAI_PORTABLE_PYTHON_ARCHIVE
* - Supports optional mirror URL via LOBSTERAI_PORTABLE_PYTHON_URL
* - Can run cross-platform for Windows packaging
* - Bundles interpreter runtime only (no preinstalled skill dependencies)
*/
'use strict';
const fs = require('fs');
const path = require('path');
const { Readable } = require('stream');
const { pipeline } = require('stream/promises');
const { spawnSync } = require('child_process');
const extractZip = require('extract-zip');
const PROJECT_ROOT = path.resolve(__dirname, '..');
const OUTPUT_DIR = path.join(PROJECT_ROOT, 'resources', 'python-win');
const DEFAULT_ARCHIVE_PATH = path.join(PROJECT_ROOT, 'resources', 'python-win-runtime.zip');
const DEFAULT_WINDOWS_EMBED_PYTHON_VERSION = process.env.LOBSTERAI_WINDOWS_EMBED_PYTHON_VERSION || '3.11.9';
const DEFAULT_WINDOWS_EMBED_PYTHON_ZIP = `python-${DEFAULT_WINDOWS_EMBED_PYTHON_VERSION}-embed-amd64.zip`;
const DEFAULT_WINDOWS_EMBED_PYTHON_URL = process.env.LOBSTERAI_WINDOWS_EMBED_PYTHON_URL
|| `https://www.python.org/ftp/python/${DEFAULT_WINDOWS_EMBED_PYTHON_VERSION}/${DEFAULT_WINDOWS_EMBED_PYTHON_ZIP}`;
const DEFAULT_GET_PIP_URL = process.env.LOBSTERAI_WINDOWS_GET_PIP_URL || 'https://bootstrap.pypa.io/get-pip.py';
const DEFAULT_PIP_PYZ_URL = process.env.LOBSTERAI_WINDOWS_PIP_PYZ_URL || 'https://bootstrap.pypa.io/pip/pip.pyz';
const DEFAULT_RUNTIME_URL = DEFAULT_WINDOWS_EMBED_PYTHON_URL;
const REQUIRED_FILES = [
'python.exe',
'python3.exe',
];
const PIP_EXECUTABLE_CANDIDATES = [
path.join('Scripts', 'pip.exe'),
path.join('Scripts', 'pip3.exe'),
path.join('Scripts', 'pip.cmd'),
path.join('Scripts', 'pip3.cmd'),
path.join('Scripts', 'pip'),
path.join('Scripts', 'pip3'),
];
const PIP_RUNTIME_ARCHIVE_REL_PATH = path.join('tools', 'pip.pyz');
const PIP_MODULE_MAIN_REL_PATH = path.join('Lib', 'site-packages', 'pip', '__main__.py');
const PIP_MODULE_INIT_REL_PATH = path.join('Lib', 'site-packages', 'pip', '__init__.py');
function hasPipCommand(rootDir) {
return PIP_EXECUTABLE_CANDIDATES.some((relPath) => fs.existsSync(path.join(rootDir, relPath)));
}
function hasPipModule(rootDir) {
return fs.existsSync(path.join(rootDir, PIP_MODULE_MAIN_REL_PATH))
|| fs.existsSync(path.join(rootDir, PIP_MODULE_INIT_REL_PATH));
}
function parseArgs(argv) {
return {
required: argv.includes('--required'),
};
}
function resolveInputPath(input) {
if (typeof input !== 'string') return null;
const trimmed = input.trim();
if (!trimmed) return null;
return path.isAbsolute(trimmed) ? trimmed : path.resolve(process.cwd(), trimmed);
}
function isNonEmptyFile(filePath) {
try {
const stat = fs.statSync(filePath);
return stat.isFile() && stat.size > 0;
} catch {
return false;
}
}
function getDirSize(dir) {
let size = 0;
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
size += getDirSize(full);
} else {
size += fs.statSync(full).size;
}
}
return size;
}
function checkRuntimeHealth(rootDir, options = {}) {
const requirePython3Alias = options.requirePython3Alias !== false;
const requirePip = options.requirePip !== false;
const missing = [];
for (const relPath of REQUIRED_FILES) {
if (!requirePython3Alias && relPath === 'python3.exe') {
continue;
}
const fullPath = path.join(rootDir, relPath);
if (!fs.existsSync(fullPath)) {
missing.push(relPath);
}
}
if (requirePip) {
if (!hasPipCommand(rootDir)) {
missing.push('Scripts/pip.exe (or Scripts/pip3.exe/pip.cmd/pip3.cmd)');
}
if (!hasPipModule(rootDir)) {
missing.push(PIP_MODULE_MAIN_REL_PATH.replace(/\\/g, '/'));
}
}
return {
ok: missing.length === 0,
missing,
};
}
function writeFileIfChanged(filePath, content) {
try {
if (fs.existsSync(filePath) && fs.readFileSync(filePath, 'utf8') === content) {
return;
}
} catch {
// ignore stale read errors and rewrite below
}
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, content, 'utf8');
}
function createPipWrappers(rootDir) {
const scriptsDir = path.join(rootDir, 'Scripts');
const pipCmd = [
'@echo off',
'setlocal',
'set "PYROOT=%~dp0.."',
'"%PYROOT%\\python.exe" -m pip %*',
'',
].join('\r\n');
const pipSh = [
'#!/usr/bin/env bash',
'set -euo pipefail',
'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
'PYROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"',
'exec "${PYROOT}/python.exe" -m pip "$@"',
'',
].join('\n');
writeFileIfChanged(path.join(scriptsDir, 'pip.cmd'), pipCmd);
writeFileIfChanged(path.join(scriptsDir, 'pip3.cmd'), pipCmd);
writeFileIfChanged(path.join(scriptsDir, 'pip'), pipSh);
writeFileIfChanged(path.join(scriptsDir, 'pip3'), pipSh);
try {
fs.chmodSync(path.join(scriptsDir, 'pip'), 0o755);
fs.chmodSync(path.join(scriptsDir, 'pip3'), 0o755);
} catch {
// Ignore chmod failures on filesystems without POSIX modes.
}
}
function tryCopyPipFromHostPython(rootDir) {
const pythonCandidates = ['python3', 'python'];
for (const candidate of pythonCandidates) {
const probe = spawnSync(candidate, [
'-c',
[
'import importlib.util, json, pathlib',
"spec = importlib.util.find_spec('pip')",
'if spec is None or not spec.origin:',
" raise SystemExit(2)",
'pip_dir = pathlib.Path(spec.origin).resolve().parent',
'site_dir = pip_dir.parent',
"dist_info = [str(p) for p in site_dir.glob('pip-*.dist-info')]",
"print(json.dumps({'pip_dir': str(pip_dir), 'dist_info': dist_info}))",
].join('\n'),
], {
encoding: 'utf-8',
stdio: 'pipe',
timeout: 20_000,
});
if (probe.status !== 0 || !probe.stdout) {
continue;
}
let parsed;
try {
parsed = JSON.parse(probe.stdout.trim());
} catch {
continue;
}
if (!parsed || typeof parsed.pip_dir !== 'string' || !parsed.pip_dir) {
continue;
}
const pipDir = parsed.pip_dir;
if (!fs.existsSync(pipDir)) {
continue;
}
const targetSitePackages = path.join(rootDir, 'Lib', 'site-packages');
const targetPipDir = path.join(targetSitePackages, 'pip');
fs.mkdirSync(targetSitePackages, { recursive: true });
fs.cpSync(pipDir, targetPipDir, {
recursive: true,
force: true,
errorOnExist: false,
dereference: true,
});
if (Array.isArray(parsed.dist_info)) {
for (const entry of parsed.dist_info) {
if (typeof entry !== 'string' || !entry) continue;
if (!fs.existsSync(entry)) continue;
const targetEntry = path.join(targetSitePackages, path.basename(entry));
fs.cpSync(entry, targetEntry, {
recursive: true,
force: true,
errorOnExist: false,
dereference: true,
});
}
}
return { ok: true, source: candidate };
}
return { ok: false };
}
async function ensurePipPayload(rootDir, options = {}) {
const required = options.required !== false;
const existingPipHealth = checkRuntimeHealth(rootDir, { requirePip: true });
if (existingPipHealth.ok) {
return;
}
const copyResult = tryCopyPipFromHostPython(rootDir);
if (copyResult.ok) {
console.log(`[setup-python-runtime] Copied pip package from host ${copyResult.source}`);
createPipWrappers(rootDir);
const copiedHealth = checkRuntimeHealth(rootDir, { requirePip: true });
if (copiedHealth.ok) {
return;
}
}
const pipPyzPath = path.join(rootDir, PIP_RUNTIME_ARCHIVE_REL_PATH);
if (!isNonEmptyFile(pipPyzPath)) {
try {
console.log(`[setup-python-runtime] Downloading pip runtime from: ${DEFAULT_PIP_PYZ_URL}`);
await downloadArchive(DEFAULT_PIP_PYZ_URL, pipPyzPath);
const fileSizeKB = (fs.statSync(pipPyzPath).size / 1024).toFixed(0);
console.log(`[setup-python-runtime] Downloaded pip runtime (${fileSizeKB} KB): ${pipPyzPath}`);
} catch (error) {
if (required) {
throw new Error(
'Unable to obtain pip runtime archive (pip.pyz). '
+ 'Set LOBSTERAI_WINDOWS_PIP_PYZ_URL to a reachable mirror if needed. '
+ `Original error: ${error instanceof Error ? error.message : String(error)}`
);
}
console.warn(
'[setup-python-runtime] pip runtime archive is not available; continuing without pip. '
+ `Reason: ${error instanceof Error ? error.message : String(error)}`
);
return;
}
}
const pipModuleDir = path.join(rootDir, 'Lib', 'site-packages', 'pip');
const pipInitPath = path.join(pipModuleDir, '__init__.py');
const pipMainPath = path.join(pipModuleDir, '__main__.py');
const pipMain = [
'import pathlib',
'import runpy',
'import sys',
'',
'root = pathlib.Path(__file__).resolve().parents[3]',
"pip_pyz = root / 'tools' / 'pip.pyz'",
'if not pip_pyz.exists():',
" raise SystemExit(f'pip runtime archive missing: {pip_pyz}')",
"sys.argv[0] = 'pip'",
"runpy.run_path(str(pip_pyz), run_name='__main__')",
'',
].join('\n');
writeFileIfChanged(pipInitPath, '');
writeFileIfChanged(pipMainPath, pipMain);
createPipWrappers(rootDir);
const finalHealth = checkRuntimeHealth(rootDir, { requirePip: true });
if (!finalHealth.ok && required) {
throw new Error(`Failed to prepare pip payload. Missing: ${finalHealth.missing.join(', ')}`);
}
}
function ensurePython3Alias(rootDir) {
const pythonExe = path.join(rootDir, 'python.exe');
const python3Exe = path.join(rootDir, 'python3.exe');
if (fs.existsSync(python3Exe) || !fs.existsSync(pythonExe)) {
return;
}
fs.copyFileSync(pythonExe, python3Exe);
}
function findRuntimeRoot(baseDir) {
const directHealth = checkRuntimeHealth(baseDir, { requirePython3Alias: false, requirePip: false });
if (directHealth.ok) {
return baseDir;
}
const queue = [baseDir];
while (queue.length > 0) {
const current = queue.shift();
if (!current) continue;
const health = checkRuntimeHealth(current, { requirePython3Alias: false, requirePip: false });
if (health.ok) {
return current;
}
let entries = [];
try {
entries = fs.readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const entry of entries) {
if (!entry.isDirectory()) continue;
queue.push(path.join(current, entry.name));
}
}
return null;
}
async function downloadArchive(url, destination) {
const response = await fetch(url, { redirect: 'follow' });
if (!response.ok || !response.body) {
throw new Error(`Download failed (${response.status} ${response.statusText}) for ${url}`);
}
fs.mkdirSync(path.dirname(destination), { recursive: true });
const tmpFile = `${destination}.download`;
try {
const stream = fs.createWriteStream(tmpFile);
await pipeline(Readable.fromWeb(response.body), stream);
if (!isNonEmptyFile(tmpFile)) {
throw new Error('Downloaded archive is empty.');
}
fs.renameSync(tmpFile, destination);
} catch (error) {
try {
fs.rmSync(tmpFile, { force: true });
} catch {
// Ignore cleanup error.
}
throw error;
}
}
async function resolveArchive(required) {
const envArchive = resolveInputPath(process.env.LOBSTERAI_PORTABLE_PYTHON_ARCHIVE);
if (envArchive) {
if (!isNonEmptyFile(envArchive)) {
throw new Error(`LOBSTERAI_PORTABLE_PYTHON_ARCHIVE points to an invalid file: ${envArchive}`);
}
console.log(`[setup-python-runtime] Using local archive from LOBSTERAI_PORTABLE_PYTHON_ARCHIVE: ${envArchive}`);
return { archivePath: envArchive, source: 'env-archive' };
}
if (isNonEmptyFile(DEFAULT_ARCHIVE_PATH)) {
console.log(`[setup-python-runtime] Using cached archive: ${DEFAULT_ARCHIVE_PATH}`);
return { archivePath: DEFAULT_ARCHIVE_PATH, source: 'cache' };
}
const urlFromEnv = typeof process.env.LOBSTERAI_PORTABLE_PYTHON_URL === 'string'
? process.env.LOBSTERAI_PORTABLE_PYTHON_URL.trim()
: '';
const downloadUrl = urlFromEnv || DEFAULT_RUNTIME_URL;
if (!downloadUrl) {
if (required) {
throw new Error(
'Portable Python archive is not available. '
+ 'Set LOBSTERAI_PORTABLE_PYTHON_ARCHIVE to a local package or '
+ 'LOBSTERAI_PORTABLE_PYTHON_URL to a downloadable runtime archive URL.'
);
}
console.warn('[setup-python-runtime] Archive URL is not configured; skipping because --required is not set.');
return null;
}
try {
console.log(`[setup-python-runtime] Downloading runtime from: ${downloadUrl}`);
await downloadArchive(downloadUrl, DEFAULT_ARCHIVE_PATH);
const fileSizeMB = (fs.statSync(DEFAULT_ARCHIVE_PATH).size / 1024 / 1024).toFixed(1);
console.log(`[setup-python-runtime] Downloaded archive (${fileSizeMB} MB): ${DEFAULT_ARCHIVE_PATH}`);
return { archivePath: DEFAULT_ARCHIVE_PATH, source: 'download' };
} catch (error) {
if (required) {
throw new Error(
'Unable to obtain portable Python runtime archive. '
+ 'Set LOBSTERAI_PORTABLE_PYTHON_ARCHIVE to a local offline package or '
+ 'set LOBSTERAI_PORTABLE_PYTHON_URL to a reachable mirror. '
+ `Original error: ${error instanceof Error ? error.message : String(error)}`
);
}
console.warn(
'[setup-python-runtime] Runtime archive is not available; skip because --required is not set. '
+ `Reason: ${error instanceof Error ? error.message : String(error)}`
);
return null;
}
}
function copyRuntimeTree(sourceRoot, destRoot) {
if (fs.existsSync(destRoot)) {
fs.rmSync(destRoot, { recursive: true, force: true });
}
fs.mkdirSync(destRoot, { recursive: true });
fs.cpSync(sourceRoot, destRoot, {
recursive: true,
dereference: true,
force: true,
errorOnExist: false,
});
}
function runCommand(command, args, options = {}) {
const result = spawnSync(command, args, {
cwd: options.cwd,
encoding: 'utf-8',
stdio: 'pipe',
timeout: options.timeout || 5 * 60 * 1000,
env: options.env || process.env,
});
if (result.status !== 0) {
const detail = (result.stderr || result.stdout || '').trim();
throw new Error(`Command failed: ${command} ${args.join(' ')}${detail ? `\n${detail}` : ''}`);
}
}
function enableSitePackages(rootDir) {
const pthCandidates = fs.readdirSync(rootDir).filter((name) => name.endsWith('._pth'));
if (pthCandidates.length === 0) {
throw new Error('Could not find python _pth file in runtime directory.');
}
const pthPath = path.join(rootDir, pthCandidates[0]);
const raw = fs.readFileSync(pthPath, 'utf-8');
const lines = raw.split(/\r?\n/);
const updated = [];
let hasSitePackages = false;
let hasImportSite = false;
for (const line of lines) {
const trimmed = line.trim();
if (trimmed === 'import site' || trimmed === '#import site') {
updated.push('import site');
hasImportSite = true;
continue;
}
if (trimmed.toLowerCase() === 'lib\\site-packages' || trimmed.toLowerCase() === 'lib/site-packages') {
updated.push('Lib\\site-packages');
hasSitePackages = true;
continue;
}
updated.push(line);
}
if (!hasSitePackages) {
updated.push('Lib\\site-packages');
}
if (!hasImportSite) {
updated.push('import site');
}
fs.writeFileSync(pthPath, `${updated.join('\n').replace(/\n+$/g, '')}\n`, 'utf8');
}
async function bootstrapRuntimeOnWindows() {
if (process.platform !== 'win32') {
throw new Error('Windows bootstrap is only supported on Windows hosts.');
}
console.log('[setup-python-runtime] No prebuilt archive provided, bootstrapping runtime from python.org on Windows host...');
const tempRoot = fs.mkdtempSync(path.join(PROJECT_ROOT, 'tmp-python-bootstrap-'));
try {
const embedZipPath = path.join(tempRoot, DEFAULT_WINDOWS_EMBED_PYTHON_ZIP);
await downloadArchive(DEFAULT_WINDOWS_EMBED_PYTHON_URL, embedZipPath);
await extractArchiveToRuntime(embedZipPath);
enableSitePackages(OUTPUT_DIR);
ensurePython3Alias(OUTPUT_DIR);
const pythonExe = path.join(OUTPUT_DIR, 'python.exe');
if (!fs.existsSync(pythonExe)) {
throw new Error('python.exe not found after extraction.');
}
const pipExe = path.join(OUTPUT_DIR, 'Scripts', 'pip.exe');
if (!fs.existsSync(pipExe)) {
try {
const getPipPath = path.join(tempRoot, 'get-pip.py');
await downloadArchive(DEFAULT_GET_PIP_URL, getPipPath);
runCommand(pythonExe, [getPipPath], { timeout: 3 * 60 * 1000 });
} catch (error) {
console.warn(
'[setup-python-runtime] Failed to bootstrap pip during Windows-host preparation. '
+ `Python runtime remains usable. Reason: ${error instanceof Error ? error.message : String(error)}`
);
}
}
await ensurePipPayload(OUTPUT_DIR, { required: true });
const health = checkRuntimeHealth(OUTPUT_DIR, { requirePip: true });
if (!health.ok) {
throw new Error(`Bootstrapped runtime health check failed; missing: ${health.missing.join(', ')}`);
}
console.log('[setup-python-runtime] Windows bootstrap completed successfully');
} finally {
try {
fs.rmSync(tempRoot, { recursive: true, force: true });
} catch {
// Ignore cleanup errors.
}
}
}
async function extractArchiveToRuntime(archivePath) {
const tempRoot = fs.mkdtempSync(path.join(PROJECT_ROOT, 'tmp-python-runtime-'));
try {
await extractZip(archivePath, { dir: tempRoot });
const runtimeRoot = findRuntimeRoot(tempRoot);
if (!runtimeRoot) {
throw new Error('Could not locate python runtime root after extraction.');
}
copyRuntimeTree(runtimeRoot, OUTPUT_DIR);
enableSitePackages(OUTPUT_DIR);
ensurePython3Alias(OUTPUT_DIR);
await ensurePipPayload(OUTPUT_DIR, { required: true });
const health = checkRuntimeHealth(OUTPUT_DIR, { requirePip: true });
if (!health.ok) {
throw new Error(
`Runtime health check failed; missing: ${health.missing.join(', ')}. `
+ 'Please provide a valid Python runtime archive.'
);
}
} finally {
try {
fs.rmSync(tempRoot, { recursive: true, force: true });
} catch {
// Ignore cleanup errors.
}
}
}
function findPortablePythonExecutable(baseDir = OUTPUT_DIR) {
const candidates = [
path.join(baseDir, 'python.exe'),
path.join(baseDir, 'python3.exe'),
];
for (const candidate of candidates) {
if (fs.existsSync(candidate)) {
return candidate;
}
}
return null;
}
async function ensurePortablePythonRuntime(options = {}) {
const required = Boolean(options.required);
const shouldRun = process.platform === 'win32'
|| required
|| process.env.LOBSTERAI_SETUP_PYTHON_RUNTIME_FORCE === '1';
if (!shouldRun) {
console.log('[setup-python-runtime] Skip on non-Windows host (pass --required to force cross-platform preparation).');
return { ok: true, skipped: true, pythonPath: null };
}
const existingBaseHealth = checkRuntimeHealth(OUTPUT_DIR, { requirePip: false });
if (existingBaseHealth.ok) {
await ensurePipPayload(OUTPUT_DIR, { required: true });
const existingFullHealth = checkRuntimeHealth(OUTPUT_DIR, { requirePip: true });
if (existingFullHealth.ok) {
const pythonPath = findPortablePythonExecutable(OUTPUT_DIR);
console.log(`[setup-python-runtime] Runtime already prepared: ${pythonPath || OUTPUT_DIR}`);
return { ok: true, skipped: false, pythonPath };
}
console.warn(
'[setup-python-runtime] Existing runtime found but pip support is incomplete; '
+ `missing: ${existingFullHealth.missing.join(', ')}. Re-extracting runtime...`
);
}
const archive = await resolveArchive(required && process.platform !== 'win32');
if (archive) {
console.log(`[setup-python-runtime] Extracting runtime archive (${archive.source})...`);
try {
await extractArchiveToRuntime(archive.archivePath);
} catch (error) {
if (process.platform !== 'win32') {
throw error;
}
console.warn(
'[setup-python-runtime] Archive extraction or pip payload setup failed; '
+ `falling back to Windows bootstrap. Reason: ${error instanceof Error ? error.message : String(error)}`
);
await bootstrapRuntimeOnWindows();
}
} else if (process.platform === 'win32') {
await bootstrapRuntimeOnWindows();
} else if (required) {
throw new Error(
'Portable Python archive is not available for non-Windows host. '
+ 'Set LOBSTERAI_PORTABLE_PYTHON_ARCHIVE to a local package or '
+ 'LOBSTERAI_PORTABLE_PYTHON_URL to a downloadable runtime archive URL.'
);
} else {
return { ok: true, skipped: true, pythonPath: null };
}
const pythonPath = findPortablePythonExecutable(OUTPUT_DIR);
const finalHealth = checkRuntimeHealth(OUTPUT_DIR, { requirePip: true });
if (!finalHealth.ok) {
throw new Error(
'Portable Python runtime is missing required pip components after preparation: '
+ finalHealth.missing.join(', ')
);
}
const finalSize = getDirSize(OUTPUT_DIR);
console.log(`[setup-python-runtime] Portable Python runtime ready: ${pythonPath || OUTPUT_DIR}`);
console.log(`[setup-python-runtime] Total size: ~${(finalSize / 1024 / 1024).toFixed(1)} MB`);
return { ok: true, skipped: false, pythonPath };
}
async function main() {
const args = parseArgs(process.argv.slice(2));
await ensurePortablePythonRuntime({ required: args.required });
}
if (require.main === module) {
main().catch((error) => {
console.error('[setup-python-runtime] ERROR:', error instanceof Error ? error.message : String(error));
process.exit(1);
});
}
module.exports = {
ensurePortablePythonRuntime,
findPortablePythonExecutable,
checkRuntimeHealth,
};