feat(package): compress artifact size (#160)
Co-authored-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Haze <hazeone@users.noreply.github.com>
This commit is contained in:
Haze
@@ -10,73 +10,135 @@
|
||||
* Solution: This hook runs AFTER electron-builder finishes packing. It manually
|
||||
* copies build/openclaw/node_modules/ into the output resources directory,
|
||||
* bypassing electron-builder's glob filtering entirely.
|
||||
*
|
||||
* Additionally, it removes unnecessary files (type definitions, source maps, docs)
|
||||
* to reduce the number of files that need to be code-signed on macOS.
|
||||
*
|
||||
* Additionally it performs two rounds of cleanup:
|
||||
* 1. General cleanup — removes dev artifacts (type defs, source maps, docs,
|
||||
* test dirs) from both the openclaw root and its node_modules.
|
||||
* 2. Platform-specific cleanup — strips native binaries for non-target
|
||||
* platforms (koffi multi-platform prebuilds, @napi-rs/canvas, @img/sharp,
|
||||
* @mariozechner/clipboard).
|
||||
*/
|
||||
|
||||
const { cpSync, existsSync, readdirSync, rmSync, statSync } = require('fs');
|
||||
const { join } = require('path');
|
||||
|
||||
/**
|
||||
* Recursively remove unnecessary files to reduce code signing overhead
|
||||
*/
|
||||
// ── Arch helpers ─────────────────────────────────────────────────────────────
|
||||
// electron-builder Arch enum: 0=ia32, 1=x64, 2=armv7l, 3=arm64, 4=universal
|
||||
const ARCH_MAP = { 0: 'ia32', 1: 'x64', 2: 'armv7l', 3: 'arm64', 4: 'universal' };
|
||||
|
||||
function resolveArch(archEnum) {
|
||||
return ARCH_MAP[archEnum] || 'x64';
|
||||
}
|
||||
|
||||
// ── General cleanup ──────────────────────────────────────────────────────────
|
||||
|
||||
function cleanupUnnecessaryFiles(dir) {
|
||||
let removedCount = 0;
|
||||
|
||||
|
||||
const REMOVE_DIRS = new Set([
|
||||
'test', 'tests', '__tests__', '.github', 'examples', 'example',
|
||||
]);
|
||||
const REMOVE_FILE_EXTS = ['.d.ts', '.d.ts.map', '.js.map', '.mjs.map', '.ts.map', '.markdown'];
|
||||
const REMOVE_FILE_NAMES = new Set([
|
||||
'.DS_Store', 'README.md', 'CHANGELOG.md', 'LICENSE.md', 'CONTRIBUTING.md',
|
||||
'tsconfig.json', '.npmignore', '.eslintrc', '.prettierrc', '.editorconfig',
|
||||
]);
|
||||
|
||||
function walk(currentDir) {
|
||||
const entries = readdirSync(currentDir, { withFileTypes: true });
|
||||
|
||||
let entries;
|
||||
try { entries = readdirSync(currentDir, { withFileTypes: true }); } catch { return; }
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(currentDir, entry.name);
|
||||
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
// Remove entire test directories
|
||||
if (entry.name === 'test' || entry.name === 'tests' ||
|
||||
entry.name === '__tests__' || entry.name === '.github' ||
|
||||
entry.name === 'docs' || entry.name === 'examples') {
|
||||
try {
|
||||
rmSync(fullPath, { recursive: true, force: true });
|
||||
removedCount++;
|
||||
} catch (err) {
|
||||
// Ignore errors
|
||||
}
|
||||
if (REMOVE_DIRS.has(entry.name)) {
|
||||
try { rmSync(fullPath, { recursive: true, force: true }); removedCount++; } catch { /* */ }
|
||||
} else {
|
||||
walk(fullPath);
|
||||
}
|
||||
} else if (entry.isFile()) {
|
||||
const name = entry.name;
|
||||
// Remove unnecessary file types
|
||||
if (name.endsWith('.d.ts') || name.endsWith('.d.ts.map') ||
|
||||
name.endsWith('.js.map') || name.endsWith('.mjs.map') ||
|
||||
name.endsWith('.ts.map') || name === '.DS_Store' ||
|
||||
name === 'README.md' || name === 'CHANGELOG.md' ||
|
||||
name === 'LICENSE.md' || name === 'CONTRIBUTING.md' ||
|
||||
name.endsWith('.md.txt') || name.endsWith('.markdown') ||
|
||||
name === 'tsconfig.json' || name === '.npmignore' ||
|
||||
name === '.eslintrc' || name === '.prettierrc') {
|
||||
try {
|
||||
rmSync(fullPath, { force: true });
|
||||
removedCount++;
|
||||
} catch (err) {
|
||||
// Ignore errors
|
||||
}
|
||||
if (REMOVE_FILE_NAMES.has(name) || REMOVE_FILE_EXTS.some(e => name.endsWith(e))) {
|
||||
try { rmSync(fullPath, { force: true }); removedCount++; } catch { /* */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
walk(dir);
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
// ── Platform-specific: koffi ─────────────────────────────────────────────────
|
||||
// koffi ships 18 platform pre-builds under koffi/build/koffi/{platform}_{arch}/.
|
||||
// We only need the one matching the target.
|
||||
|
||||
function cleanupKoffi(nodeModulesDir, platform, arch) {
|
||||
const koffiDir = join(nodeModulesDir, 'koffi', 'build', 'koffi');
|
||||
if (!existsSync(koffiDir)) return 0;
|
||||
|
||||
const keepTarget = `${platform}_${arch}`;
|
||||
let removed = 0;
|
||||
for (const entry of readdirSync(koffiDir)) {
|
||||
if (entry !== keepTarget) {
|
||||
try { rmSync(join(koffiDir, entry), { recursive: true, force: true }); removed++; } catch { /* */ }
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
// ── Platform-specific: scoped native packages ────────────────────────────────
|
||||
// Packages like @napi-rs/canvas-darwin-arm64, @img/sharp-linux-x64, etc.
|
||||
// Only the variant matching the target platform should survive.
|
||||
|
||||
const PLATFORM_NATIVE_SCOPES = {
|
||||
'@napi-rs': /^canvas-(darwin|linux|win32)-(x64|arm64)/,
|
||||
'@img': /^sharp(?:-libvips)?-(darwin|linux|win32)-(x64|arm64)/,
|
||||
'@mariozechner': /^clipboard-(darwin|linux|win32)-(x64|arm64|universal)/,
|
||||
};
|
||||
|
||||
function cleanupNativePlatformPackages(nodeModulesDir, platform, arch) {
|
||||
let removed = 0;
|
||||
|
||||
for (const [scope, pattern] of Object.entries(PLATFORM_NATIVE_SCOPES)) {
|
||||
const scopeDir = join(nodeModulesDir, scope);
|
||||
if (!existsSync(scopeDir)) continue;
|
||||
|
||||
for (const entry of readdirSync(scopeDir)) {
|
||||
const match = entry.match(pattern);
|
||||
if (!match) continue; // not a platform-specific package, leave it
|
||||
|
||||
const pkgPlatform = match[1];
|
||||
const pkgArch = match[2];
|
||||
|
||||
const isMatch =
|
||||
pkgPlatform === platform &&
|
||||
(pkgArch === arch || pkgArch === 'universal');
|
||||
|
||||
if (!isMatch) {
|
||||
try {
|
||||
rmSync(join(scopeDir, entry), { recursive: true, force: true });
|
||||
removed++;
|
||||
} catch { /* */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
// ── Main hook ────────────────────────────────────────────────────────────────
|
||||
|
||||
exports.default = async function afterPack(context) {
|
||||
const appOutDir = context.appOutDir;
|
||||
const platform = context.electronPlatformName; // 'win32' | 'darwin' | 'linux'
|
||||
const arch = resolveArch(context.arch);
|
||||
|
||||
console.log(`[after-pack] Target: ${platform}/${arch}`);
|
||||
|
||||
const src = join(__dirname, '..', 'build', 'openclaw', 'node_modules');
|
||||
|
||||
// On macOS, resources live inside the .app bundle
|
||||
let resourcesDir;
|
||||
if (platform === 'darwin') {
|
||||
const appName = context.packager.appInfo.productFilename;
|
||||
@@ -85,23 +147,37 @@ exports.default = async function afterPack(context) {
|
||||
resourcesDir = join(appOutDir, 'resources');
|
||||
}
|
||||
|
||||
const dest = join(resourcesDir, 'openclaw', 'node_modules');
|
||||
const openclawRoot = join(resourcesDir, 'openclaw');
|
||||
const dest = join(openclawRoot, 'node_modules');
|
||||
|
||||
if (!existsSync(src)) {
|
||||
console.warn('[after-pack] ⚠️ build/openclaw/node_modules not found. Run "pnpm run bundle:openclaw" first.');
|
||||
console.warn('[after-pack] ⚠️ build/openclaw/node_modules not found. Run bundle-openclaw first.');
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Copy node_modules (electron-builder skips it due to .gitignore)
|
||||
const depCount = readdirSync(src, { withFileTypes: true })
|
||||
.filter(d => d.isDirectory() && d.name !== '.bin')
|
||||
.length;
|
||||
|
||||
console.log(`[after-pack] Copying ${depCount} openclaw dependencies to ${dest} ...`);
|
||||
cpSync(src, dest, { recursive: true });
|
||||
console.log('[after-pack] ✅ openclaw node_modules copied successfully.');
|
||||
|
||||
// Clean up unnecessary files to reduce code signing overhead (especially on macOS)
|
||||
console.log('[after-pack] 🧹 Cleaning up unnecessary files (type definitions, source maps, docs)...');
|
||||
const removedCount = cleanupUnnecessaryFiles(dest);
|
||||
console.log(`[after-pack] ✅ Removed ${removedCount} unnecessary files/directories.`);
|
||||
console.log('[after-pack] ✅ openclaw node_modules copied.');
|
||||
|
||||
// 2. General cleanup on the full openclaw directory (not just node_modules)
|
||||
console.log('[after-pack] 🧹 Cleaning up unnecessary files ...');
|
||||
const removedRoot = cleanupUnnecessaryFiles(openclawRoot);
|
||||
console.log(`[after-pack] ✅ Removed ${removedRoot} unnecessary files/directories.`);
|
||||
|
||||
// 3. Platform-specific: strip koffi non-target platform binaries
|
||||
const koffiRemoved = cleanupKoffi(dest, platform, arch);
|
||||
if (koffiRemoved > 0) {
|
||||
console.log(`[after-pack] ✅ koffi: removed ${koffiRemoved} non-target platform binaries (kept ${platform}_${arch}).`);
|
||||
}
|
||||
|
||||
// 4. Platform-specific: strip wrong-platform native packages
|
||||
const nativeRemoved = cleanupNativePlatformPackages(dest, platform, arch);
|
||||
if (nativeRemoved > 0) {
|
||||
console.log(`[after-pack] ✅ Removed ${nativeRemoved} non-target native platform packages.`);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -127,6 +127,14 @@ if (!openclawVirtualNM) {
|
||||
echo` Virtual store root: ${openclawVirtualNM}`;
|
||||
queue.push({ nodeModulesDir: openclawVirtualNM, skipPkg: 'openclaw' });
|
||||
|
||||
const SKIP_PACKAGES = new Set([
|
||||
'typescript',
|
||||
'playwright-core',
|
||||
'@playwright/test',
|
||||
]);
|
||||
const SKIP_SCOPES = ['@cloudflare/', '@types/'];
|
||||
let skippedDevCount = 0;
|
||||
|
||||
while (queue.length > 0) {
|
||||
const { nodeModulesDir, skipPkg } = queue.shift();
|
||||
const packages = listPackages(nodeModulesDir);
|
||||
@@ -135,6 +143,11 @@ while (queue.length > 0) {
|
||||
// Skip the package that owns this virtual store entry (it's the package itself, not a dep)
|
||||
if (name === skipPkg) continue;
|
||||
|
||||
if (SKIP_PACKAGES.has(name) || SKIP_SCOPES.some(s => name.startsWith(s))) {
|
||||
skippedDevCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
let realPath;
|
||||
try {
|
||||
realPath = fs.realpathSync(fullPath);
|
||||
@@ -156,6 +169,7 @@ while (queue.length > 0) {
|
||||
}
|
||||
|
||||
echo` Found ${collected.size} total packages (direct + transitive)`;
|
||||
echo` Skipped ${skippedDevCount} dev-only package references`;
|
||||
|
||||
// 5. Copy all collected packages into OUTPUT/node_modules/ (flat structure)
|
||||
//
|
||||
@@ -190,13 +204,160 @@ for (const [realPath, pkgName] of collected) {
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Verify the bundle
|
||||
// 6. Clean up the bundle to reduce package size
|
||||
//
|
||||
// This removes platform-agnostic waste: dev artifacts, docs, source maps,
|
||||
// type definitions, test directories, and known large unused subdirectories.
|
||||
// Platform-specific cleanup (e.g. koffi binaries) is handled in after-pack.cjs
|
||||
// which has access to the target platform/arch context.
|
||||
|
||||
function getDirSize(dir) {
|
||||
let total = 0;
|
||||
try {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) total += getDirSize(p);
|
||||
else if (entry.isFile()) total += fs.statSync(p).size;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return total;
|
||||
}
|
||||
|
||||
function formatSize(bytes) {
|
||||
if (bytes >= 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024 / 1024).toFixed(1)}G`;
|
||||
if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)}M`;
|
||||
if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)}K`;
|
||||
return `${bytes}B`;
|
||||
}
|
||||
|
||||
function rmSafe(target) {
|
||||
try {
|
||||
const stat = fs.lstatSync(target);
|
||||
if (stat.isDirectory()) fs.rmSync(target, { recursive: true, force: true });
|
||||
else fs.rmSync(target, { force: true });
|
||||
return true;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
function cleanupBundle(outputDir) {
|
||||
let removedCount = 0;
|
||||
const nm = path.join(outputDir, 'node_modules');
|
||||
const ext = path.join(outputDir, 'extensions');
|
||||
|
||||
// --- openclaw root junk ---
|
||||
for (const name of ['CHANGELOG.md', 'README.md']) {
|
||||
if (rmSafe(path.join(outputDir, name))) removedCount++;
|
||||
}
|
||||
|
||||
// docs/ is kept — contains prompt templates and other runtime-used prompts
|
||||
|
||||
// --- extensions: clean junk from source, aggressively clean nested node_modules ---
|
||||
// Extension source (.ts files) are runtime entry points — must be preserved.
|
||||
// Only nested node_modules/ inside extensions get the aggressive cleanup.
|
||||
if (fs.existsSync(ext)) {
|
||||
const JUNK_EXTS = new Set(['.prose', '.ignored_openclaw', '.keep']);
|
||||
const NM_REMOVE_DIRS = new Set([
|
||||
'test', 'tests', '__tests__', '.github', 'docs', 'examples', 'example',
|
||||
]);
|
||||
const NM_REMOVE_FILE_EXTS = ['.d.ts', '.d.ts.map', '.js.map', '.mjs.map', '.ts.map', '.markdown'];
|
||||
const NM_REMOVE_FILE_NAMES = new Set([
|
||||
'.DS_Store', 'README.md', 'CHANGELOG.md', 'LICENSE.md', 'CONTRIBUTING.md',
|
||||
'tsconfig.json', '.npmignore', '.eslintrc', '.prettierrc', '.editorconfig',
|
||||
]);
|
||||
|
||||
function walkExt(dir, insideNodeModules) {
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (insideNodeModules && NM_REMOVE_DIRS.has(entry.name)) {
|
||||
if (rmSafe(full)) removedCount++;
|
||||
} else {
|
||||
walkExt(full, insideNodeModules || entry.name === 'node_modules');
|
||||
}
|
||||
} else if (entry.isFile()) {
|
||||
if (insideNodeModules) {
|
||||
const name = entry.name;
|
||||
if (NM_REMOVE_FILE_NAMES.has(name) || NM_REMOVE_FILE_EXTS.some(e => name.endsWith(e))) {
|
||||
if (rmSafe(full)) removedCount++;
|
||||
}
|
||||
} else {
|
||||
if (JUNK_EXTS.has(path.extname(entry.name)) || entry.name.endsWith('.md')) {
|
||||
if (rmSafe(full)) removedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walkExt(ext, false);
|
||||
}
|
||||
|
||||
// --- node_modules: remove unnecessary file types and directories ---
|
||||
if (fs.existsSync(nm)) {
|
||||
const REMOVE_DIRS = new Set([
|
||||
'test', 'tests', '__tests__', '.github', 'docs', 'examples', 'example',
|
||||
]);
|
||||
const REMOVE_FILE_EXTS = ['.d.ts', '.d.ts.map', '.js.map', '.mjs.map', '.ts.map', '.markdown'];
|
||||
const REMOVE_FILE_NAMES = new Set([
|
||||
'.DS_Store', 'README.md', 'CHANGELOG.md', 'LICENSE.md', 'CONTRIBUTING.md',
|
||||
'tsconfig.json', '.npmignore', '.eslintrc', '.prettierrc', '.editorconfig',
|
||||
]);
|
||||
|
||||
function walkClean(dir) {
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (REMOVE_DIRS.has(entry.name)) {
|
||||
if (rmSafe(full)) removedCount++;
|
||||
} else {
|
||||
walkClean(full);
|
||||
}
|
||||
} else if (entry.isFile()) {
|
||||
const name = entry.name;
|
||||
if (REMOVE_FILE_NAMES.has(name) || REMOVE_FILE_EXTS.some(e => name.endsWith(e))) {
|
||||
if (rmSafe(full)) removedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walkClean(nm);
|
||||
}
|
||||
|
||||
// --- known large unused subdirectories ---
|
||||
const LARGE_REMOVALS = [
|
||||
'node_modules/pdfjs-dist/legacy',
|
||||
'node_modules/pdfjs-dist/types',
|
||||
'node_modules/node-llama-cpp/llama',
|
||||
'node_modules/koffi/src',
|
||||
'node_modules/koffi/vendor',
|
||||
'node_modules/koffi/doc',
|
||||
];
|
||||
for (const rel of LARGE_REMOVALS) {
|
||||
if (rmSafe(path.join(outputDir, rel))) removedCount++;
|
||||
}
|
||||
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
echo``;
|
||||
echo`🧹 Cleaning up bundle (removing dev artifacts, docs, source maps, type defs)...`;
|
||||
const sizeBefore = getDirSize(OUTPUT);
|
||||
const cleanedCount = cleanupBundle(OUTPUT);
|
||||
const sizeAfter = getDirSize(OUTPUT);
|
||||
echo` Removed ${cleanedCount} files/directories`;
|
||||
echo` Size: ${formatSize(sizeBefore)} → ${formatSize(sizeAfter)} (saved ${formatSize(sizeBefore - sizeAfter)})`;
|
||||
|
||||
// 7. Verify the bundle
|
||||
const entryExists = fs.existsSync(path.join(OUTPUT, 'openclaw.mjs'));
|
||||
const distExists = fs.existsSync(path.join(OUTPUT, 'dist', 'entry.js'));
|
||||
|
||||
echo``;
|
||||
echo`✅ Bundle complete: ${OUTPUT}`;
|
||||
echo` Unique packages copied: ${copiedCount}`;
|
||||
echo` Dev-only packages skipped: ${skippedDevCount}`;
|
||||
echo` Duplicate versions skipped: ${skippedDupes}`;
|
||||
echo` Total discovered: ${collected.size}`;
|
||||
echo` openclaw.mjs: ${entryExists ? '✓' : '✗'}`;
|
||||
|
||||
Reference in New Issue
Block a user