build: unify preinstalled skills bundling across dev/package/release and harden SignPath validation (#524)

2026-03-16 16:55:56 +08:00
parent f6128ed743
commit 4e3f3c83f6
10 changed files with 143 additions and 36 deletions
--- a/.github/workflows/package-win-manual.yml
+++ b/.github/workflows/package-win-manual.yml
@@ -52,7 +52,7 @@ jobs:
      - name: Build Windows package (no publish)
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pnpm run build:vite && pnpm exec zx scripts/bundle-openclaw.mjs && pnpm exec electron-builder --win --publish never
+        run: pnpm run package:win

      - name: Upload Windows Installer (x64)
        uses: actions/upload-artifact@v4
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -83,6 +83,19 @@ jobs:
        if: matrix.platform == 'win'
        run: pnpm run package:win

+      - name: Validate unsigned Windows artifacts before SignPath
+        if: matrix.platform == 'win'
+        shell: pwsh
+        run: |
+          $unsignedExeFiles = Get-ChildItem -Path "release" -Filter *.exe -File
+          if (-not $unsignedExeFiles) {
+            throw "No unsigned .exe files found in release/ before SignPath upload"
+          }
+          $unsignedCount = $unsignedExeFiles.Count
+          "UNSIGNED_EXE_COUNT=$unsignedCount" | Out-File -FilePath $env:GITHUB_ENV -Append
+          Write-Host "Found $unsignedCount unsigned .exe file(s):"
+          $unsignedExeFiles | ForEach-Object { Write-Host " - $($_.Name)" }
+
      - name: Upload unsigned Windows artifacts for SignPath
        if: matrix.platform == 'win'
        id: upload-unsigned-windows-artifact
@@ -109,14 +122,23 @@ jobs:
        if: matrix.platform == 'win'
        shell: pwsh
        run: |
+          Write-Host "SignPath GitHub artifact ID: ${{ steps.upload-unsigned-windows-artifact.outputs.artifact-id }}"
          $signedExeFiles = Get-ChildItem -Path "release/signed" -Filter *.exe -File -Recurse
          if (-not $signedExeFiles) {
            throw "No signed .exe files found in release/signed"
          }
+          $signedCount = $signedExeFiles.Count
+          if ($env:UNSIGNED_EXE_COUNT -and ($signedCount -ne [int]$env:UNSIGNED_EXE_COUNT)) {
+            throw "Signed .exe count ($signedCount) does not match unsigned count ($env:UNSIGNED_EXE_COUNT)"
+          }
          foreach ($file in $signedExeFiles) {
            Copy-Item -Path $file.FullName -Destination "release/$($file.Name)" -Force
          }
-          Write-Host "Signed executables copied to release/"
+          $finalExeFiles = Get-ChildItem -Path "release" -Filter *.exe -File
+          if ($env:UNSIGNED_EXE_COUNT -and ($finalExeFiles.Count -ne [int]$env:UNSIGNED_EXE_COUNT)) {
+            throw "Final release .exe count ($($finalExeFiles.Count)) does not match unsigned count ($env:UNSIGNED_EXE_COUNT)"
+          }
+          Write-Host "Signed executables copied to release/ ($($finalExeFiles.Count) file(s))"

      # Linux specific steps
      - name: Build Linux
--- a/.github/workflows/win-build-test.yml
+++ b/.github/workflows/win-build-test.yml
@@ -40,6 +40,18 @@ jobs:
      - name: Build Windows
        run: pnpm run package:win

+      - name: Validate unsigned Windows artifacts before SignPath
+        shell: pwsh
+        run: |
+          $unsignedExeFiles = Get-ChildItem -Path "release" -Filter *.exe -File
+          if (-not $unsignedExeFiles) {
+            throw "No unsigned .exe files found in release/ before SignPath upload"
+          }
+          $unsignedCount = $unsignedExeFiles.Count
+          "UNSIGNED_EXE_COUNT=$unsignedCount" | Out-File -FilePath $env:GITHUB_ENV -Append
+          Write-Host "Found $unsignedCount unsigned .exe file(s):"
+          $unsignedExeFiles | ForEach-Object { Write-Host " - $($_.Name)" }
+
      # Required by SignPath Trusted Build: artifact must exist on GitHub first.
      - name: Upload unsigned Windows artifacts for SignPath
        id: upload-unsigned-windows-artifact
@@ -64,14 +76,23 @@ jobs:
      - name: Replace unsigned executables with signed ones
        shell: pwsh
        run: |
+          Write-Host "SignPath GitHub artifact ID: ${{ steps.upload-unsigned-windows-artifact.outputs.artifact-id }}"
          $signedExeFiles = Get-ChildItem -Path "release/signed" -Filter *.exe -File -Recurse
          if (-not $signedExeFiles) {
            throw "No signed .exe files found in release/signed"
          }
+          $signedCount = $signedExeFiles.Count
+          if ($env:UNSIGNED_EXE_COUNT -and ($signedCount -ne [int]$env:UNSIGNED_EXE_COUNT)) {
+            throw "Signed .exe count ($signedCount) does not match unsigned count ($env:UNSIGNED_EXE_COUNT)"
+          }
          foreach ($file in $signedExeFiles) {
            Copy-Item -Path $file.FullName -Destination "release/$($file.Name)" -Force
          }
-          Write-Host "Signed executables copied to release/"
+          $finalExeFiles = Get-ChildItem -Path "release" -Filter *.exe -File
+          if ($env:UNSIGNED_EXE_COUNT -and ($finalExeFiles.Count -ne [int]$env:UNSIGNED_EXE_COUNT)) {
+            throw "Final release .exe count ($($finalExeFiles.Count)) does not match unsigned count ($env:UNSIGNED_EXE_COUNT)"
+          }
+          Write-Host "Signed executables copied to release/ ($($finalExeFiles.Count) file(s))"

      - name: Upload signed Windows artifacts
        uses: actions/upload-artifact@v4
--- a/README.ja-JP.md
+++ b/README.ja-JP.md
@@ -109,13 +109,12 @@ AIタスクを自動的に実行するようスケジュール設定できます

 ### 🧩 拡張可能なスキルシステム
 事前構築されたスキルでAIエージェントを拡張できます。統合スキルパネルからスキルの閲覧、インストール、管理が可能です。パッケージマネージャーは不要です。
-ClawX はドキュメント処理スキル（`pdf`、`xlsx`、`docx`、`pptx`）もフル内容で同梱し、起動時に管理スキルディレクトリ（既定 `~/.openclaw/skills`）へ自動配備し、初回インストール時に既定で有効化します。追加の同梱スキル（`find-skills`、`self-improving-agent`、`tavily-search`、`brave-web-search`、`bocha-skill`）も既定で有効化されますが、必要な API キーが未設定の場合は OpenClaw が実行時に設定エラーを表示します。  
+ClawX はドキュメント処理スキル（`pdf`、`xlsx`、`docx`、`pptx`）もフル内容で同梱し、起動時に管理スキルディレクトリ（既定 `~/.openclaw/skills`）へ自動配備し、初回インストール時に既定で有効化します。追加の同梱スキル（`find-skills`、`self-improving-agent`、`tavily-search`、`brave-web-search`）も既定で有効化されますが、必要な API キーが未設定の場合は OpenClaw が実行時に設定エラーを表示します。  
 Skills ページでは OpenClaw の複数ソース（管理ディレクトリ、workspace、追加スキルディレクトリ）から検出されたスキルを表示でき、各スキルの実際のパスを確認して実フォルダを直接開けます。

 主な検索スキルで必要な環境変数:
 - `BRAVE_SEARCH_API_KEY`: `brave-web-search` 用
 - `TAVILY_API_KEY`: `tavily-search` 用（上流ランタイムで OAuth 対応の場合あり）
- `BOCHA_API_KEY`: `bocha-skill` 用

 ### 🔐 セキュアなプロバイダー統合
 複数のAIプロバイダー（OpenAI、Anthropicなど）に接続でき、資格情報はシステムのネイティブキーチェーンに安全に保存されます。OpenAI は API キーとブラウザ OAuth（Codex サブスクリプション）の両方に対応しています。
@@ -308,7 +307,7 @@ AI を開発ワークフローに統合できます。エージェントを使
 ```bash
 # 開発
 pnpm run init             # 依存関係のインストール + uvのダウンロード
-pnpm dev                  # ホットリロードで起動
+pnpm dev                  # ホットリロードで起動（不足時は同梱スキルを自動準備）

 # コード品質
 pnpm lint                 # ESLintを実行
@@ -323,7 +322,7 @@ pnpm run comms:compare    # リプレイ指標をベースライン閾値と比
 # ビルド＆パッケージ
 pnpm run build:vite       # フロントエンドのみビルド
 pnpm build                # フルプロダクションビルド（パッケージアセット含む）
-pnpm package              # 現在のプラットフォーム向けにパッケージ化
+pnpm package              # 現在のプラットフォーム向けにパッケージ化（同梱プリインストールスキルを含む）
 pnpm package:mac          # macOS向けにパッケージ化
 pnpm package:win          # Windows向けにパッケージ化
 pnpm package:linux        # Linux向けにパッケージ化
--- a/README.md
+++ b/README.md
@@ -109,13 +109,12 @@ Schedule AI tasks to run automatically. Define triggers, set intervals, and let

 ### 🧩 Extensible Skill System
 Extend your AI agents with pre-built skills. Browse, install, and manage skills through the integrated skill panel—no package managers required.
-ClawX also pre-bundles full document-processing skills (`pdf`, `xlsx`, `docx`, `pptx`), deploys them automatically to the managed skills directory (default `~/.openclaw/skills`) on startup, and enables them by default on first install. Additional bundled skills (`find-skills`, `self-improving-agent`, `tavily-search`, `brave-web-search`, `bocha-skill`) are also enabled by default; if required API keys are missing, OpenClaw will surface configuration errors in runtime.  
+ClawX also pre-bundles full document-processing skills (`pdf`, `xlsx`, `docx`, `pptx`), deploys them automatically to the managed skills directory (default `~/.openclaw/skills`) on startup, and enables them by default on first install. Additional bundled skills (`find-skills`, `self-improving-agent`, `tavily-search`, `brave-web-search`) are also enabled by default; if required API keys are missing, OpenClaw will surface configuration errors in runtime.  
 The Skills page can display skills discovered from multiple OpenClaw sources (managed dir, workspace, and extra skill dirs), and now shows each skill's actual location so you can open the real folder directly.

 Environment variables for bundled search skills:
 - `BRAVE_SEARCH_API_KEY` for `brave-web-search`
 - `TAVILY_API_KEY` for `tavily-search` (OAuth may also be supported by upstream skill runtime)
- `BOCHA_API_KEY` for `bocha-skill`
 - `find-skills` and `self-improving-agent` do not require API keys

 ### 🔐 Secure Provider Integration
@@ -312,7 +311,7 @@ Chain multiple skills together to create sophisticated automation pipelines. Pro
 ```bash
 # Development
 pnpm run init             # Install dependencies + download uv
-pnpm dev                  # Start with hot reload
+pnpm dev                  # Start with hot reload (auto-prepares bundled skills if missing)

 # Quality
 pnpm lint                 # Run ESLint
@@ -327,7 +326,7 @@ pnpm run comms:compare    # Compare replay metrics against baseline thresholds
 # Build & Package
 pnpm run build:vite       # Build frontend only
 pnpm build                # Full production build (with packaging assets)
-pnpm package              # Package for current platform
+pnpm package              # Package for current platform (includes bundled preinstalled skills)
 pnpm package:mac          # Package for macOS
 pnpm package:win          # Package for Windows
 pnpm package:linux        # Package for Linux
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -110,13 +110,12 @@ ClawX 直接基于官方 **OpenClaw** 核心构建。无需单独安装，我们

 ### 🧩 可扩展技能系统
 通过预构建的技能扩展 AI 智能体的能力。在集成的技能面板中浏览、安装和管理技能——无需包管理器。
-ClawX 还会内置预装完整的文档处理技能（`pdf`、`xlsx`、`docx`、`pptx`），在启动时自动部署到托管技能目录（默认 `~/.openclaw/skills`），并在首次安装时默认启用。额外预装技能（`find-skills`、`self-improving-agent`、`tavily-search`、`brave-web-search`、`bocha-skill`）也会默认启用；若缺少必需的 API Key，OpenClaw 会在运行时给出配置错误提示。  
+ClawX 还会内置预装完整的文档处理技能（`pdf`、`xlsx`、`docx`、`pptx`），在启动时自动部署到托管技能目录（默认 `~/.openclaw/skills`），并在首次安装时默认启用。额外预装技能（`find-skills`、`self-improving-agent`、`tavily-search`、`brave-web-search`）也会默认启用；若缺少必需的 API Key，OpenClaw 会在运行时给出配置错误提示。  
 Skills 页面可展示来自多个 OpenClaw 来源的技能（托管目录、workspace、额外技能目录），并显示每个技能的实际路径，便于直接打开真实安装位置。

 重点搜索技能所需环境变量：
 - `BRAVE_SEARCH_API_KEY`：用于 `brave-web-search`
 - `TAVILY_API_KEY`：用于 `tavily-search`（上游运行时也可能支持 OAuth）
- `BOCHA_API_KEY`：用于 `bocha-skill`

 ### 🔐 安全的供应商集成
 连接多个 AI 供应商（OpenAI、Anthropic 等），凭证安全存储在系统原生密钥链中。OpenAI 同时支持 API Key 与浏览器 OAuth（Codex 订阅）登录。
@@ -312,7 +311,7 @@ ClawX 采用 **双进程 + Host API 统一接入架构**。渲染进程只调用
 ```bash
 # 开发
 pnpm run init             # 安装依赖并下载 uv
-pnpm dev                  # 以热重载模式启动
+pnpm dev                  # 以热重载模式启动（若缺失会自动准备预装技能包）

 # 代码质量
 pnpm lint                 # 运行 ESLint 检查
@@ -327,7 +326,7 @@ pnpm run comms:compare    # 将回放指标与基线阈值对比
 # 构建与打包
 pnpm run build:vite       # 仅构建前端
 pnpm build                # 完整生产构建（含打包资源）
-pnpm package              # 为当前平台打包
+pnpm package              # 为当前平台打包（包含预装技能资源）
 pnpm package:mac          # 为 macOS 打包
 pnpm package:win          # 为 Windows 打包
 pnpm package:linux        # 为 Linux 打包
--- a/package.json
+++ b/package.json
@@ -23,6 +23,7 @@
    "private": true,
    "scripts": {
        "init": "pnpm install && pnpm run uv:download",
+        "predev": "zx scripts/prepare-preinstalled-skills-dev.mjs",
        "dev": "vite",
        "build": "vite build && zx scripts/bundle-openclaw.mjs && zx scripts/bundle-openclaw-plugins.mjs && zx scripts/bundle-preinstalled-skills.mjs && electron-builder",
        "build:vite": "vite build",
@@ -40,7 +41,7 @@
        "uv:download:linux": "zx scripts/download-bundled-uv.mjs --platform=linux",
        "uv:download:all": "zx scripts/download-bundled-uv.mjs --all",
        "icons": "zx scripts/generate-icons.mjs",
-        "package": "vite build && zx scripts/bundle-openclaw.mjs && zx scripts/bundle-openclaw-plugins.mjs",
+        "package": "vite build && zx scripts/bundle-openclaw.mjs && zx scripts/bundle-openclaw-plugins.mjs && zx scripts/bundle-preinstalled-skills.mjs",
        "package:mac": "pnpm run package && electron-builder --mac --publish never",
        "package:win": "pnpm run package && electron-builder --win --publish never",
        "package:linux": "pnpm run package && electron-builder --linux --publish never",
--- a/resources/skills/preinstalled-manifest.json
+++ b/resources/skills/preinstalled-manifest.json
@@ -42,9 +42,9 @@
    },
    {
      "slug": "self-improving-agent",
-      "repo": "openclaw/skills",
-      "repoPath": "skills/pskoett/self-improving-agent",
-      "ref": "main",
+      "repo": "peterskoett/self-improving-agent",
+      "repoPath": ".",
+      "ref": "master",
      "version": "main",
      "autoEnable": true
    },
@@ -63,14 +63,6 @@
      "ref": "main",
      "version": "main",
      "autoEnable": true
-    },
-    {
-      "slug": "bocha-skill",
-      "repo": "openclaw/skills",
-      "repoPath": "skills/ypw757/bocha-skill",
-      "ref": "main",
-      "version": "main",
-      "autoEnable": true
    }
  ]
 }
--- a/scripts/bundle-preinstalled-skills.mjs
+++ b/scripts/bundle-preinstalled-skills.mjs
@@ -2,7 +2,7 @@

 import 'zx/globals';
 import { readFileSync, existsSync, mkdirSync, rmSync, cpSync, writeFileSync } from 'node:fs';
-import { join, dirname } from 'node:path';
+import { join, dirname, basename } from 'node:path';
 import { fileURLToPath } from 'node:url';

 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -43,18 +43,61 @@ function createRepoDirName(repo, ref) {
  return `${repo.replace(/[\\/]/g, '__')}__${ref.replace(/[^a-zA-Z0-9._-]/g, '_')}`;
 }

+function toGitPath(inputPath) {
+  if (process.platform !== 'win32') return inputPath;
+  // Git on Windows accepts forward slashes and avoids backslash escape quirks.
+  return inputPath.replace(/\\/g, '/');
+}
+
+function normalizeRepoPath(repoPath) {
+  return repoPath.replace(/\\/g, '/').replace(/^\/+/, '').replace(/\/+$/, '');
+}
+
+function shouldCopySkillFile(srcPath) {
+  const base = basename(srcPath);
+  if (base === '.git') return false;
+  if (base === '.subset.tar') return false;
+  return true;
+}
+
+async function extractArchive(archiveFileName, cwd) {
+  const prevCwd = $.cwd;
+  $.cwd = cwd;
+  try {
+    try {
+      await $`tar -xf ${archiveFileName}`;
+      return;
+    } catch (tarError) {
+      if (process.platform === 'win32') {
+        // Some Windows images expose bsdtar instead of tar.
+        await $`bsdtar -xf ${archiveFileName}`;
+        return;
+      }
+      throw tarError;
+    }
+  } finally {
+    $.cwd = prevCwd;
+  }
+}
+
 async function fetchSparseRepo(repo, ref, paths, checkoutDir) {
  const remote = `https://github.com/${repo}.git`;
  mkdirSync(checkoutDir, { recursive: true });
+  const gitCheckoutDir = toGitPath(checkoutDir);
+  const archiveFileName = '.subset.tar';
+  const archivePath = join(checkoutDir, archiveFileName);
+  const archivePaths = [...new Set(paths.map(normalizeRepoPath))];

-  await $`git init ${checkoutDir}`;
-  await $`git -C ${checkoutDir} remote add origin ${remote}`;
-  await $`git -C ${checkoutDir} sparse-checkout init --cone`;
-  await $`git -C ${checkoutDir} sparse-checkout set ${paths}`;
-  await $`git -C ${checkoutDir} fetch --depth 1 origin ${ref}`;
-  await $`git -C ${checkoutDir} checkout FETCH_HEAD`;
+  await $`git init ${gitCheckoutDir}`;
+  await $`git -C ${gitCheckoutDir} remote add origin ${remote}`;
+  await $`git -C ${gitCheckoutDir} fetch --depth 1 origin ${ref}`;
+  // Do not checkout working tree on Windows: upstream repos may contain
+  // Windows-invalid paths. Export only requested directories via git archive.
+  await $`git -C ${gitCheckoutDir} archive --format=tar --output ${archiveFileName} FETCH_HEAD ${archivePaths}`;
+  await extractArchive(archiveFileName, checkoutDir);
+  rmSync(archivePath, { force: true });

-  const commit = (await $`git -C ${checkoutDir} rev-parse HEAD`).stdout.trim();
+  const commit = (await $`git -C ${gitCheckoutDir} rev-parse FETCH_HEAD`).stdout.trim();
  return commit;
 }

@@ -89,7 +132,7 @@ for (const group of groups) {
    }

    rmSync(targetDir, { recursive: true, force: true });
-    cpSync(sourceDir, targetDir, { recursive: true, dereference: true });
+    cpSync(sourceDir, targetDir, { recursive: true, dereference: true, filter: shouldCopySkillFile });

    const skillManifest = join(targetDir, 'SKILL.md');
    if (!existsSync(skillManifest)) {
--- a/scripts/prepare-preinstalled-skills-dev.mjs
+++ b/scripts/prepare-preinstalled-skills-dev.mjs
@@ -0,0 +1,31 @@
+#!/usr/bin/env zx
+
+import 'zx/globals';
+import { existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = join(__dirname, '..');
+const lockPath = join(ROOT, 'build', 'preinstalled-skills', '.preinstalled-lock.json');
+const bundleScript = join(ROOT, 'scripts', 'bundle-preinstalled-skills.mjs');
+
+if (process.env.CLAWX_SKIP_PREINSTALLED_SKILLS_PREPARE === '1') {
+  echo`Skipping preinstalled skills prepare (CLAWX_SKIP_PREINSTALLED_SKILLS_PREPARE=1).`;
+  process.exit(0);
+}
+
+if (existsSync(lockPath)) {
+  echo`Preinstalled skills bundle already exists, skipping prepare.`;
+  process.exit(0);
+}
+
+echo`Preinstalled skills bundle missing, preparing for dev startup...`;
+
+try {
+  await $`zx ${bundleScript}`;
+} catch (error) {
+  // Dev startup should remain available even if network-based skill fetching fails.
+  echo`Warning: failed to prepare preinstalled skills for dev startup: ${error?.message || error}`;
+  process.exit(0);
+}