diff --git a/electron/api/routes/channels.ts b/electron/api/routes/channels.ts index da0223428..4708a8887 100644 --- a/electron/api/routes/channels.ts +++ b/electron/api/routes/channels.ts @@ -25,7 +25,9 @@ function scheduleGatewayChannelRestart(ctx: HostApiContext, reason: string): voi void reason; } -const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']); +// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky. +// GatewayManager.reload() already falls back to restart when reload is unhealthy. +const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'whatsapp']); function scheduleGatewayChannelSaveRefresh( ctx: HostApiContext, diff --git a/electron/gateway/manager.ts b/electron/gateway/manager.ts index 88eea6f56..fdf79a195 100644 --- a/electron/gateway/manager.ts +++ b/electron/gateway/manager.ts @@ -43,6 +43,12 @@ import { GatewayConnectionMonitor } from './connection-monitor'; import { GatewayLifecycleController, LifecycleSupersededError } from './lifecycle-controller'; import { launchGatewayProcess } from './process-launcher'; import { GatewayRestartController } from './restart-controller'; +import { GatewayRestartGovernor } from './restart-governor'; +import { + DEFAULT_GATEWAY_RELOAD_POLICY, + loadGatewayReloadPolicy, + type GatewayReloadPolicy, +} from './reload-policy'; import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr'; import { runGatewayStartupSequence } from './startup-orchestrator'; @@ -94,12 +100,15 @@ export class GatewayManager extends EventEmitter { private readonly connectionMonitor = new GatewayConnectionMonitor(); private readonly lifecycleController = new GatewayLifecycleController(); private readonly restartController = new GatewayRestartController(); + private readonly restartGovernor = new GatewayRestartGovernor(); private reloadDebounceTimer: NodeJS.Timeout | null = null; + private reloadPolicy: GatewayReloadPolicy = { ...DEFAULT_GATEWAY_RELOAD_POLICY }; + private reloadPolicyLoadedAt = 0; + private reloadPolicyRefreshPromise: Promise | null = null; private externalShutdownSupported: boolean | null = null; - private lastRestartAt = 0; private reconnectAttemptsTotal = 0; private reconnectSuccessTotal = 0; - private static readonly RESTART_COOLDOWN_MS = 2500; + private static readonly RELOAD_POLICY_REFRESH_MS = 15_000; constructor(config?: Partial) { super(); @@ -109,6 +118,9 @@ export class GatewayManager extends EventEmitter { this.emit('status', status); }, onTransition: (previousState, nextState) => { + if (nextState === 'running') { + this.restartGovernor.onRunning(); + } this.restartController.flushDeferredRestart( `status:${previousState}->${nextState}`, { @@ -186,6 +198,7 @@ export class GatewayManager extends EventEmitter { logger.info(`Gateway start requested (port=${this.status.port})`); this.lastSpawnSummary = null; this.shouldReconnect = true; + await this.refreshReloadPolicy(true); // Lazily load device identity (async file I/O + key generation). // Must happen before connect() which uses the identity for the handshake. @@ -353,18 +366,27 @@ export class GatewayManager extends EventEmitter { return; } - const now = Date.now(); - const sinceLastRestart = now - this.lastRestartAt; - if (sinceLastRestart < GatewayManager.RESTART_COOLDOWN_MS) { - logger.info( - `Gateway restart skipped due to cooldown (${sinceLastRestart}ms < ${GatewayManager.RESTART_COOLDOWN_MS}ms)`, + const decision = this.restartGovernor.decide(); + if (!decision.allow) { + const observability = this.restartGovernor.getObservability(); + logger.warn( + `[gateway-restart-governor] restart suppressed reason=${decision.reason} retryAfterMs=${decision.retryAfterMs} ` + + `suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`, ); + const props = { + reason: decision.reason, + retry_after_ms: decision.retryAfterMs, + gateway_restart_suppressed_total: observability.suppressed_total, + gateway_restart_executed_total: observability.executed_total, + gateway_restart_circuit_open_until: observability.circuit_open_until, + }; + trackMetric('gateway.restart.suppressed', props); + captureTelemetryEvent('gateway_restart_suppressed', props); return; } const pidBefore = this.status.pid; logger.info(`[gateway-refresh] mode=restart requested pidBefore=${pidBefore ?? 'n/a'}`); - this.lastRestartAt = now; this.restartInFlight = (async () => { await this.stop(); await this.start(); @@ -372,8 +394,18 @@ export class GatewayManager extends EventEmitter { try { await this.restartInFlight; + this.restartGovernor.recordExecuted(); + const observability = this.restartGovernor.getObservability(); + const props = { + gateway_restart_executed_total: observability.executed_total, + gateway_restart_suppressed_total: observability.suppressed_total, + gateway_restart_circuit_open_until: observability.circuit_open_until, + }; + trackMetric('gateway.restart.executed', props); + captureTelemetryEvent('gateway_restart_executed', props); logger.info( - `[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'}`, + `[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'} ` + + `suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`, ); } finally { this.restartInFlight = null; @@ -413,6 +445,16 @@ export class GatewayManager extends EventEmitter { * Falls back to restart on unsupported platforms or signaling failures. */ async reload(): Promise { + await this.refreshReloadPolicy(); + + if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') { + logger.info( + `[gateway-refresh] mode=reload result=policy_forced_restart policy=${this.reloadPolicy.mode}`, + ); + await this.restart(); + return; + } + if (this.restartController.isRestartDeferred({ state: this.status.state, startLock: this.startLock, @@ -481,17 +523,51 @@ export class GatewayManager extends EventEmitter { * Debounced reload — coalesces multiple rapid config-change events into one * in-process reload when possible. */ - debouncedReload(delayMs = 1200): void { + debouncedReload(delayMs?: number): void { + void this.refreshReloadPolicy(); + const effectiveDelay = delayMs ?? this.reloadPolicy.debounceMs; + if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') { + logger.debug( + `Gateway reload policy=${this.reloadPolicy.mode}; routing debouncedReload to debouncedRestart (${effectiveDelay}ms)`, + ); + this.debouncedRestart(effectiveDelay); + return; + } + if (this.reloadDebounceTimer) { clearTimeout(this.reloadDebounceTimer); } - logger.debug(`Gateway reload debounced (will fire in ${delayMs}ms)`); + logger.debug(`Gateway reload debounced (will fire in ${effectiveDelay}ms)`); this.reloadDebounceTimer = setTimeout(() => { this.reloadDebounceTimer = null; void this.reload().catch((err) => { logger.warn('Debounced Gateway reload failed:', err); }); - }, delayMs); + }, effectiveDelay); + } + + private async refreshReloadPolicy(force = false): Promise { + const now = Date.now(); + if (!force && now - this.reloadPolicyLoadedAt < GatewayManager.RELOAD_POLICY_REFRESH_MS) { + return; + } + + if (this.reloadPolicyRefreshPromise) { + await this.reloadPolicyRefreshPromise; + return; + } + + this.reloadPolicyRefreshPromise = (async () => { + const nextPolicy = await loadGatewayReloadPolicy(); + this.reloadPolicy = nextPolicy; + this.reloadPolicyLoadedAt = Date.now(); + })(); + + try { + await this.reloadPolicyRefreshPromise; + } finally { + this.reloadPolicyRefreshPromise = null; + } } /** diff --git a/electron/gateway/reload-policy.ts b/electron/gateway/reload-policy.ts new file mode 100644 index 000000000..67494d2bc --- /dev/null +++ b/electron/gateway/reload-policy.ts @@ -0,0 +1,63 @@ +import { readFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +export type GatewayReloadMode = 'hybrid' | 'reload' | 'restart' | 'off'; + +export type GatewayReloadPolicy = { + mode: GatewayReloadMode; + debounceMs: number; +}; + +export const DEFAULT_GATEWAY_RELOAD_POLICY: GatewayReloadPolicy = { + mode: 'hybrid', + debounceMs: 1200, +}; + +const OPENCLAW_CONFIG_PATH = join(homedir(), '.openclaw', 'openclaw.json'); +const MAX_DEBOUNCE_MS = 60_000; + +function normalizeMode(value: unknown): GatewayReloadMode { + if (value === 'off' || value === 'reload' || value === 'restart' || value === 'hybrid') { + return value; + } + return DEFAULT_GATEWAY_RELOAD_POLICY.mode; +} + +function normalizeDebounceMs(value: unknown): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return DEFAULT_GATEWAY_RELOAD_POLICY.debounceMs; + } + const rounded = Math.round(value); + if (rounded < 0) return 0; + if (rounded > MAX_DEBOUNCE_MS) return MAX_DEBOUNCE_MS; + return rounded; +} + +export function parseGatewayReloadPolicy(config: unknown): GatewayReloadPolicy { + if (!config || typeof config !== 'object') { + return { ...DEFAULT_GATEWAY_RELOAD_POLICY }; + } + const root = config as Record; + const gateway = (root.gateway && typeof root.gateway === 'object' + ? root.gateway + : {}) as Record; + const reload = (gateway.reload && typeof gateway.reload === 'object' + ? gateway.reload + : {}) as Record; + + return { + mode: normalizeMode(reload.mode), + debounceMs: normalizeDebounceMs(reload.debounceMs), + }; +} + +export async function loadGatewayReloadPolicy(): Promise { + try { + const raw = await readFile(OPENCLAW_CONFIG_PATH, 'utf-8'); + return parseGatewayReloadPolicy(JSON.parse(raw)); + } catch { + return { ...DEFAULT_GATEWAY_RELOAD_POLICY }; + } +} + diff --git a/electron/gateway/restart-governor.ts b/electron/gateway/restart-governor.ts new file mode 100644 index 000000000..1d5b98176 --- /dev/null +++ b/electron/gateway/restart-governor.ts @@ -0,0 +1,145 @@ +export type RestartDecision = + | { allow: true } + | { + allow: false; + reason: 'circuit_open' | 'budget_exceeded' | 'cooldown_active'; + retryAfterMs: number; + }; + +type RestartGovernorOptions = { + maxRestartsPerWindow: number; + windowMs: number; + baseCooldownMs: number; + maxCooldownMs: number; + circuitOpenMs: number; + stableResetMs: number; +}; + +const DEFAULT_OPTIONS: RestartGovernorOptions = { + maxRestartsPerWindow: 4, + windowMs: 10 * 60 * 1000, + baseCooldownMs: 2500, + maxCooldownMs: 2 * 60 * 1000, + circuitOpenMs: 10 * 60 * 1000, + stableResetMs: 2 * 60 * 1000, +}; + +export class GatewayRestartGovernor { + private readonly options: RestartGovernorOptions; + private restartTimestamps: number[] = []; + private circuitOpenUntil = 0; + private consecutiveRestarts = 0; + private lastRestartAt = 0; + private lastRunningAt = 0; + private suppressedTotal = 0; + private executedTotal = 0; + private static readonly MAX_COUNTER = Number.MAX_SAFE_INTEGER; + + constructor(options?: Partial) { + this.options = { ...DEFAULT_OPTIONS, ...options }; + } + + onRunning(now = Date.now()): void { + this.lastRunningAt = now; + } + + decide(now = Date.now()): RestartDecision { + this.pruneOld(now); + this.maybeResetConsecutive(now); + + if (now < this.circuitOpenUntil) { + this.suppressedTotal = this.incrementCounter(this.suppressedTotal); + return { + allow: false, + reason: 'circuit_open', + retryAfterMs: this.circuitOpenUntil - now, + }; + } + + if (this.restartTimestamps.length >= this.options.maxRestartsPerWindow) { + this.circuitOpenUntil = now + this.options.circuitOpenMs; + this.suppressedTotal = this.incrementCounter(this.suppressedTotal); + return { + allow: false, + reason: 'budget_exceeded', + retryAfterMs: this.options.circuitOpenMs, + }; + } + + const requiredCooldown = this.getCooldownMs(); + if (this.lastRestartAt > 0) { + const sinceLast = now - this.lastRestartAt; + if (sinceLast < requiredCooldown) { + this.suppressedTotal = this.incrementCounter(this.suppressedTotal); + return { + allow: false, + reason: 'cooldown_active', + retryAfterMs: requiredCooldown - sinceLast, + }; + } + } + + return { allow: true }; + } + + recordExecuted(now = Date.now()): void { + this.executedTotal = this.incrementCounter(this.executedTotal); + this.lastRestartAt = now; + this.consecutiveRestarts += 1; + this.restartTimestamps.push(now); + this.pruneOld(now); + } + + getCounters(): { executedTotal: number; suppressedTotal: number } { + return { + executedTotal: this.executedTotal, + suppressedTotal: this.suppressedTotal, + }; + } + + getObservability(): { + suppressed_total: number; + executed_total: number; + circuit_open_until: number; + } { + return { + suppressed_total: this.suppressedTotal, + executed_total: this.executedTotal, + circuit_open_until: this.circuitOpenUntil, + }; + } + + private getCooldownMs(): number { + const factor = Math.pow(2, Math.max(0, this.consecutiveRestarts)); + return Math.min(this.options.baseCooldownMs * factor, this.options.maxCooldownMs); + } + + private maybeResetConsecutive(now: number): void { + if (this.lastRunningAt <= 0) return; + if (now - this.lastRunningAt >= this.options.stableResetMs) { + this.consecutiveRestarts = 0; + } + } + + private pruneOld(now: number): void { + // Detect time rewind (system clock moved backwards) and clear all + // time-based guard state to avoid stale lockouts. + if (this.restartTimestamps.length > 0 && now < this.restartTimestamps[this.restartTimestamps.length - 1]) { + this.restartTimestamps = []; + this.circuitOpenUntil = 0; + this.lastRestartAt = 0; + this.lastRunningAt = 0; + this.consecutiveRestarts = 0; + return; + } + const threshold = now - this.options.windowMs; + while (this.restartTimestamps.length > 0 && this.restartTimestamps[0] < threshold) { + this.restartTimestamps.shift(); + } + } + + private incrementCounter(current: number): number { + if (current >= GatewayRestartGovernor.MAX_COUNTER) return 0; + return current + 1; + } +} diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index e16164eac..1a4945963 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -1355,7 +1355,9 @@ function registerGatewayHandlers( * For checking package status and channel configuration */ function registerOpenClawHandlers(gatewayManager: GatewayManager): void { - const forceRestartChannels = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']); + // Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky. + // GatewayManager.reload() already falls back to restart when reload is unhealthy. + const forceRestartChannels = new Set(['dingtalk', 'wecom', 'whatsapp']); const scheduleGatewayChannelRestart = (reason: string): void => { if (gatewayManager.getStatus().state !== 'stopped') { diff --git a/electron/utils/channel-config.ts b/electron/utils/channel-config.ts index 18b11aea4..8330f84b8 100644 --- a/electron/utils/channel-config.ts +++ b/electron/utils/channel-config.ts @@ -716,6 +716,65 @@ export interface ValidationResult { warnings: string[]; } +const DOCTOR_PARSER_FALLBACK_HINT = + 'Doctor output could not be confidently interpreted; falling back to local channel config checks.'; + +type DoctorValidationParseResult = { + errors: string[]; + warnings: string[]; + undetermined: boolean; +}; + +export function parseDoctorValidationOutput(channelType: string, output: string): DoctorValidationParseResult { + const errors: string[] = []; + const warnings: string[] = []; + const normalizedChannelType = channelType.toLowerCase(); + const normalizedOutput = output.trim(); + + if (!normalizedOutput) { + return { + errors, + warnings: [DOCTOR_PARSER_FALLBACK_HINT], + undetermined: true, + }; + } + + const lines = output + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const channelLines = lines.filter((line) => line.toLowerCase().includes(normalizedChannelType)); + let classifiedCount = 0; + + for (const line of channelLines) { + const lowerLine = line.toLowerCase(); + if (lowerLine.includes('error') || lowerLine.includes('unrecognized key')) { + errors.push(line); + classifiedCount += 1; + continue; + } + if (lowerLine.includes('warning')) { + warnings.push(line); + classifiedCount += 1; + } + } + + if (channelLines.length === 0 || classifiedCount === 0) { + warnings.push(DOCTOR_PARSER_FALLBACK_HINT); + return { + errors, + warnings, + undetermined: true, + }; + } + + return { + errors, + warnings, + undetermined: false, + }; +} + export interface CredentialValidationResult { valid: boolean; errors: string[]; @@ -853,34 +912,41 @@ export async function validateChannelConfig(channelType: string): Promise((resolve, reject) => { - exec( - `node openclaw.mjs doctor --json 2>&1`, - { - cwd: openclawPath, - encoding: 'utf-8', - timeout: 30000, - windowsHide: true, - }, - (err, stdout) => { - if (err) reject(err); - else resolve(stdout); - }, - ); - }); + const runDoctor = async (command: string): Promise => + await new Promise((resolve, reject) => { + exec( + command, + { + cwd: openclawPath, + encoding: 'utf-8', + timeout: 30000, + windowsHide: true, + }, + (err, stdout, stderr) => { + const combined = `${stdout || ''}${stderr || ''}`; + if (err) { + const next = new Error(combined || err.message); + reject(next); + return; + } + resolve(combined); + }, + ); + }); - const lines = output.split('\n'); - for (const line of lines) { - const lowerLine = line.toLowerCase(); - if (lowerLine.includes(channelType) && lowerLine.includes('error')) { - result.errors.push(line.trim()); - result.valid = false; - } else if (lowerLine.includes(channelType) && lowerLine.includes('warning')) { - result.warnings.push(line.trim()); - } else if (lowerLine.includes('unrecognized key') && lowerLine.includes(channelType)) { - result.errors.push(line.trim()); - result.valid = false; - } + const output = await runDoctor(`node openclaw.mjs doctor 2>&1`); + + const parsedDoctor = parseDoctorValidationOutput(channelType, output); + result.errors.push(...parsedDoctor.errors); + result.warnings.push(...parsedDoctor.warnings); + if (parsedDoctor.errors.length > 0) { + result.valid = false; + } + if (parsedDoctor.undetermined) { + logger.warn('Doctor output parsing fell back to local channel checks', { + channelType, + hint: DOCTOR_PARSER_FALLBACK_HINT, + }); } const config = await readOpenClawConfig(); diff --git a/electron/utils/device-oauth.ts b/electron/utils/device-oauth.ts index 4589f2fc7..9a9f93b1e 100644 --- a/electron/utils/device-oauth.ts +++ b/electron/utils/device-oauth.ts @@ -22,6 +22,7 @@ import { logger } from './logger'; import { saveProvider, getProvider, ProviderConfig } from './secure-storage'; import { getProviderDefaultModel } from './provider-registry'; import { isOpenClawPresent } from './paths'; +import { proxyAwareFetch } from './proxy-fetch'; import { loginMiniMaxPortalOAuth, type MiniMaxOAuthToken, @@ -47,6 +48,17 @@ class DeviceOAuthManager extends EventEmitter { private active: boolean = false; private mainWindow: BrowserWindow | null = null; + private async runWithProxyAwareFetch(task: () => Promise): Promise { + const originalFetch = globalThis.fetch; + globalThis.fetch = ((input: string | URL, init?: RequestInit) => + proxyAwareFetch(input, init)) as typeof fetch; + try { + return await task(); + } finally { + globalThis.fetch = originalFetch; + } + } + setWindow(window: BrowserWindow) { this.mainWindow = window; } @@ -109,7 +121,7 @@ class DeviceOAuthManager extends EventEmitter { } const provider = this.activeProvider!; - const token: MiniMaxOAuthToken = await loginMiniMaxPortalOAuth({ + const token: MiniMaxOAuthToken = await this.runWithProxyAwareFetch(() => loginMiniMaxPortalOAuth({ region, openUrl: async (url) => { logger.info(`[DeviceOAuth] MiniMax opening browser: ${url}`); @@ -133,7 +145,7 @@ class DeviceOAuthManager extends EventEmitter { update: (msg) => logger.info(`[DeviceOAuth] MiniMax progress: ${msg}`), stop: (msg) => logger.info(`[DeviceOAuth] MiniMax progress done: ${msg ?? ''}`), }, - }); + })); if (!this.active) return; @@ -159,7 +171,7 @@ class DeviceOAuthManager extends EventEmitter { } const provider = this.activeProvider!; - const token: QwenOAuthToken = await loginQwenPortalOAuth({ + const token: QwenOAuthToken = await this.runWithProxyAwareFetch(() => loginQwenPortalOAuth({ openUrl: async (url) => { logger.info(`[DeviceOAuth] Qwen opening browser: ${url}`); shell.openExternal(url).catch((err) => @@ -179,7 +191,7 @@ class DeviceOAuthManager extends EventEmitter { update: (msg) => logger.info(`[DeviceOAuth] Qwen progress: ${msg}`), stop: (msg) => logger.info(`[DeviceOAuth] Qwen progress done: ${msg ?? ''}`), }, - }); + })); if (!this.active) return; diff --git a/electron/utils/gemini-cli-oauth.ts b/electron/utils/gemini-cli-oauth.ts index 991c0d359..50081ffa5 100644 --- a/electron/utils/gemini-cli-oauth.ts +++ b/electron/utils/gemini-cli-oauth.ts @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, unlinkS import { createServer } from 'node:http'; import { delimiter, dirname, join } from 'node:path'; import { getClawXConfigDir } from './paths'; +import { proxyAwareFetch } from './proxy-fetch'; const CLIENT_ID_KEYS = ['OPENCLAW_GEMINI_OAUTH_CLIENT_ID', 'GEMINI_CLI_OAUTH_CLIENT_ID']; const CLIENT_SECRET_KEYS = [ @@ -243,7 +244,7 @@ async function installViaNpm(onProgress?: (msg: string) => void): Promise void): Promise { try { onProgress?.('Downloading Gemini OAuth helper...'); - const metaRes = await fetch('https://registry.npmjs.org/@google/gemini-cli-core/latest'); + const metaRes = await proxyAwareFetch('https://registry.npmjs.org/@google/gemini-cli-core/latest'); if (!metaRes.ok) { onProgress?.(`Failed to fetch Gemini package metadata: ${metaRes.status}`); return false; @@ -256,7 +257,7 @@ async function installViaDirectDownload(onProgress?: (msg: string) => void): Pro return false; } - const tarRes = await fetch(tarballUrl); + const tarRes = await proxyAwareFetch(tarballUrl); if (!tarRes.ok) { onProgress?.(`Failed to download Gemini package: ${tarRes.status}`); return false; @@ -440,7 +441,7 @@ async function waitForLocalCallback(params: { async function getUserEmail(accessToken: string): Promise { try { - const response = await fetch(USERINFO_URL, { + const response = await proxyAwareFetch(USERINFO_URL, { headers: { Authorization: `Bearer ${accessToken}` }, }); if (response.ok) { @@ -489,7 +490,7 @@ async function pollOperation( ): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> { for (let attempt = 0; attempt < 24; attempt += 1) { await new Promise((resolve) => setTimeout(resolve, 5000)); - const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers }); + const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers }); if (!response.ok) { continue; } @@ -530,7 +531,7 @@ async function discoverProject(accessToken: string): Promise { allowedTiers?: Array<{ id?: string; isDefault?: boolean }>; } = {}; - const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, { + const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, { method: 'POST', headers, body: JSON.stringify(loadBody), @@ -583,7 +584,7 @@ async function discoverProject(accessToken: string): Promise { (onboardBody.metadata as Record).duetProject = envProject; } - const onboardResponse = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, { + const onboardResponse = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, { method: 'POST', headers, body: JSON.stringify(onboardBody), @@ -638,7 +639,7 @@ async function exchangeCodeForTokens( body.set('client_secret', clientSecret); } - const response = await fetch(TOKEN_URL, { + const response = await proxyAwareFetch(TOKEN_URL, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: body.toString(), diff --git a/electron/utils/openai-codex-oauth.ts b/electron/utils/openai-codex-oauth.ts index 8c656e41a..119e23351 100644 --- a/electron/utils/openai-codex-oauth.ts +++ b/electron/utils/openai-codex-oauth.ts @@ -1,5 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { createServer } from 'node:http'; +import { proxyAwareFetch } from './proxy-fetch'; const CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann'; const AUTHORIZE_URL = 'https://auth.openai.com/oauth/authorize'; @@ -206,7 +207,7 @@ async function exchangeAuthorizationCode( code: string, verifier: string, ): Promise<{ access: string; refresh: string; expires: number }> { - const response = await fetch(TOKEN_URL, { + const response = await proxyAwareFetch(TOKEN_URL, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ diff --git a/electron/utils/openclaw-doctor.ts b/electron/utils/openclaw-doctor.ts index 66f6bed78..a5bd638bd 100644 --- a/electron/utils/openclaw-doctor.ts +++ b/electron/utils/openclaw-doctor.ts @@ -7,7 +7,7 @@ import { getUvMirrorEnv } from './uv-env'; const OPENCLAW_DOCTOR_TIMEOUT_MS = 60_000; const MAX_DOCTOR_OUTPUT_BYTES = 10 * 1024 * 1024; -const OPENCLAW_DOCTOR_ARGS = ['doctor', '--json']; +const OPENCLAW_DOCTOR_ARGS = ['doctor']; const OPENCLAW_DOCTOR_FIX_ARGS = ['doctor', '--fix', '--yes', '--non-interactive']; export type OpenClawDoctorMode = 'diagnose' | 'fix'; @@ -65,10 +65,12 @@ function getBundledBinPath(): string { : path.join(process.cwd(), 'resources', 'bin', target); } -async function runDoctorCommand(mode: OpenClawDoctorMode): Promise { +async function runDoctorCommandWithArgs( + mode: OpenClawDoctorMode, + args: string[], +): Promise { const openclawDir = getOpenClawDir(); const entryScript = getOpenClawEntryPath(); - const args = mode === 'fix' ? OPENCLAW_DOCTOR_FIX_ARGS : OPENCLAW_DOCTOR_ARGS; const command = `openclaw ${args.join(' ')}`; const startedAt = Date.now(); @@ -194,9 +196,9 @@ async function runDoctorCommand(mode: OpenClawDoctorMode): Promise { - return await runDoctorCommand('diagnose'); + return await runDoctorCommandWithArgs('diagnose', OPENCLAW_DOCTOR_ARGS); } export async function runOpenClawDoctorFix(): Promise { - return await runDoctorCommand('fix'); + return await runDoctorCommandWithArgs('fix', OPENCLAW_DOCTOR_FIX_ARGS); } diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 6551063e6..2e8b20726 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -220,7 +220,7 @@ "cmdUnavailable": "Command unavailable", "cmdCopied": "CLI command copied", "doctor": "OpenClaw Doctor", - "doctorDesc": "Run `openclaw doctor --json` and inspect the raw diagnostic output.", + "doctorDesc": "Run `openclaw doctor` and inspect the raw diagnostic output.", "runDoctor": "Run Doctor", "runDoctorFix": "Run Doctor Fix", "doctorSucceeded": "OpenClaw doctor completed", diff --git a/src/i18n/locales/ja/settings.json b/src/i18n/locales/ja/settings.json index 91d84e2cf..8f6ccc45a 100644 --- a/src/i18n/locales/ja/settings.json +++ b/src/i18n/locales/ja/settings.json @@ -217,7 +217,7 @@ "cmdUnavailable": "コマンドが利用できません", "cmdCopied": "CLI コマンドをコピーしました", "doctor": "OpenClaw Doctor", - "doctorDesc": "`openclaw doctor --json` を実行して診断の生出力を確認します。", + "doctorDesc": "`openclaw doctor` を実行して診断の生出力を確認します。", "runDoctor": "Doctor を実行", "runDoctorFix": "Doctor 修復を実行", "doctorSucceeded": "OpenClaw doctor が完了しました", diff --git a/src/i18n/locales/zh/settings.json b/src/i18n/locales/zh/settings.json index ee854a7f6..c2a90e51e 100644 --- a/src/i18n/locales/zh/settings.json +++ b/src/i18n/locales/zh/settings.json @@ -220,7 +220,7 @@ "cmdUnavailable": "命令不可用", "cmdCopied": "CLI 命令已复制", "doctor": "OpenClaw Doctor 诊断", - "doctorDesc": "运行 `openclaw doctor --json` 并查看原始诊断输出。", + "doctorDesc": "运行 `openclaw doctor` 并查看原始诊断输出。", "runDoctor": "运行 Doctor", "runDoctorFix": "运行 Doctor 并修复", "doctorSucceeded": "OpenClaw doctor 已完成", diff --git a/src/pages/Settings/index.tsx b/src/pages/Settings/index.tsx index 06c0901a1..5b694f1cf 100644 --- a/src/pages/Settings/index.tsx +++ b/src/pages/Settings/index.tsx @@ -169,7 +169,7 @@ export function Settings() { exitCode: null, stdout: '', stderr: '', - command: 'openclaw doctor --json', + command: 'openclaw doctor', cwd: '', durationMs: 0, error: message, diff --git a/tests/unit/channel-config.test.ts b/tests/unit/channel-config.test.ts index 6e9f961e3..e5ae3eefc 100644 --- a/tests/unit/channel-config.test.ts +++ b/tests/unit/channel-config.test.ts @@ -102,3 +102,38 @@ describe('channel credential normalization and duplicate checks', () => { ); }); }); + +describe('parseDoctorValidationOutput', () => { + it('extracts channel error and warning lines', async () => { + const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config'); + + const out = parseDoctorValidationOutput( + 'feishu', + 'feishu error: token invalid\nfeishu warning: fallback enabled\n', + ); + + expect(out.undetermined).toBe(false); + expect(out.errors).toEqual(['feishu error: token invalid']); + expect(out.warnings).toEqual(['feishu warning: fallback enabled']); + }); + + it('falls back with hint when output has no channel signal', async () => { + const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config'); + + const out = parseDoctorValidationOutput('feishu', 'all good, no channel details'); + + expect(out.undetermined).toBe(true); + expect(out.errors).toEqual([]); + expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true); + }); + + it('falls back with hint when output is empty', async () => { + const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config'); + + const out = parseDoctorValidationOutput('feishu', ' '); + + expect(out.undetermined).toBe(true); + expect(out.errors).toEqual([]); + expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true); + }); +}); diff --git a/tests/unit/gateway-manager-reload-policy-refresh.test.ts b/tests/unit/gateway-manager-reload-policy-refresh.test.ts new file mode 100644 index 000000000..e70f6b74c --- /dev/null +++ b/tests/unit/gateway-manager-reload-policy-refresh.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockLoadGatewayReloadPolicy } = vi.hoisted(() => ({ + mockLoadGatewayReloadPolicy: vi.fn(), +})); + +vi.mock('electron', () => ({ + app: { + getPath: () => '/tmp', + isPackaged: false, + }, + utilityProcess: { + fork: vi.fn(), + }, +})); + +vi.mock('@electron/gateway/reload-policy', async () => { + const actual = await vi.importActual( + '@electron/gateway/reload-policy', + ); + return { + ...actual, + loadGatewayReloadPolicy: (...args: unknown[]) => mockLoadGatewayReloadPolicy(...args), + }; +}); + +describe('GatewayManager refreshReloadPolicy', () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-03-15T00:00:00.000Z')); + }); + + it('deduplicates concurrent refresh calls', async () => { + const { GatewayManager } = await import('@electron/gateway/manager'); + let resolveLoad: ((value: { mode: 'reload'; debounceMs: number }) => void) | null = null; + mockLoadGatewayReloadPolicy.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + + const manager = new GatewayManager(); + const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise }) + .refreshReloadPolicy.bind(manager); + + const p1 = refresh(true); + const p2 = refresh(true); + + expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1); + + resolveLoad?.({ mode: 'reload', debounceMs: 1300 }); + await Promise.all([p1, p2]); + + expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({ + mode: 'reload', + debounceMs: 1300, + }); + }); + + it('hits TTL cache and skips refresh within window', async () => { + const { GatewayManager } = await import('@electron/gateway/manager'); + mockLoadGatewayReloadPolicy.mockResolvedValueOnce({ mode: 'restart', debounceMs: 2200 }); + + const manager = new GatewayManager(); + const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise }) + .refreshReloadPolicy.bind(manager); + + await refresh(); + expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1); + + vi.setSystemTime(new Date('2026-03-15T00:00:10.000Z')); + await refresh(); + + expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1); + }); + + it('refreshes immediately when force=true even within TTL', async () => { + const { GatewayManager } = await import('@electron/gateway/manager'); + mockLoadGatewayReloadPolicy + .mockResolvedValueOnce({ mode: 'hybrid', debounceMs: 1200 }) + .mockResolvedValueOnce({ mode: 'off', debounceMs: 9000 }); + + const manager = new GatewayManager(); + const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise }) + .refreshReloadPolicy.bind(manager); + + await refresh(); + expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1); + + vi.setSystemTime(new Date('2026-03-15T00:00:05.000Z')); + await refresh(true); + + expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(2); + expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({ + mode: 'off', + debounceMs: 9000, + }); + }); +}); diff --git a/tests/unit/gateway-reload-policy.test.ts b/tests/unit/gateway-reload-policy.test.ts new file mode 100644 index 000000000..1149f39e9 --- /dev/null +++ b/tests/unit/gateway-reload-policy.test.ts @@ -0,0 +1,150 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockReadFile } = vi.hoisted(() => ({ + mockReadFile: vi.fn(), +})); + +vi.mock('node:fs/promises', () => ({ + readFile: mockReadFile, + default: { + readFile: mockReadFile, + }, +})); + +import { + DEFAULT_GATEWAY_RELOAD_POLICY, + loadGatewayReloadPolicy, + parseGatewayReloadPolicy, +} from '@electron/gateway/reload-policy'; + +describe('parseGatewayReloadPolicy', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns defaults when config is missing', () => { + expect(parseGatewayReloadPolicy(undefined)).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + }); + + it('parses mode and debounce from gateway.reload', () => { + const result = parseGatewayReloadPolicy({ + gateway: { + reload: { + mode: 'off', + debounceMs: 3000, + }, + }, + }); + + expect(result).toEqual({ mode: 'off', debounceMs: 3000 }); + }); + + it('normalizes invalid mode and debounce bounds', () => { + const negative = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'invalid', debounceMs: -100 } }, + }); + expect(negative).toEqual({ + mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode, + debounceMs: 0, + }); + + const overMax = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'hybrid', debounceMs: 600_000 } }, + }); + expect(overMax).toEqual({ mode: 'hybrid', debounceMs: 60_000 }); + }); + + it('falls back to default mode for non-string or unknown mode values', () => { + const unknownString = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'HYBRID', debounceMs: 1200 } }, + }); + expect(unknownString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode); + + const nonString = parseGatewayReloadPolicy({ + gateway: { reload: { mode: { value: 'reload' }, debounceMs: 1200 } }, + }); + expect(nonString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode); + }); + + it('handles malformed gateway/reload shapes', () => { + const malformedGateway = parseGatewayReloadPolicy({ gateway: 'bad-shape' }); + expect(malformedGateway).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + + const malformedReload = parseGatewayReloadPolicy({ + gateway: { reload: ['bad-shape'] }, + }); + expect(malformedReload).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + }); + + it('normalizes debounce boundary and rounding behavior', () => { + const atMin = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'reload', debounceMs: 0 } }, + }); + expect(atMin).toEqual({ mode: 'reload', debounceMs: 0 }); + + const roundsUpToCap = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'reload', debounceMs: 60_000.5 } }, + }); + expect(roundsUpToCap).toEqual({ mode: 'reload', debounceMs: 60_000 }); + + const roundsDownAtCap = parseGatewayReloadPolicy({ + gateway: { reload: { mode: 'reload', debounceMs: 60_000.4 } }, + }); + expect(roundsDownAtCap).toEqual({ mode: 'reload', debounceMs: 60_000 }); + }); +}); + +describe('loadGatewayReloadPolicy', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns defaults when config read fails', async () => { + mockReadFile.mockRejectedValueOnce(new Error('EACCES')); + + await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + expect(mockReadFile).toHaveBeenCalledOnce(); + }); + + it('returns defaults when config JSON is malformed', async () => { + mockReadFile.mockResolvedValueOnce('{'); + + await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + }); + + it('returns defaults when config JSON has malformed shape', async () => { + mockReadFile.mockResolvedValueOnce( + JSON.stringify({ + gateway: { reload: ['malformed'] }, + }), + ); + + await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY); + }); + + it('loads config and applies invalid mode fallback', async () => { + mockReadFile.mockResolvedValueOnce( + JSON.stringify({ + gateway: { reload: { mode: 'unknown-mode', debounceMs: 1350 } }, + }), + ); + + await expect(loadGatewayReloadPolicy()).resolves.toEqual({ + mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode, + debounceMs: 1350, + }); + }); + + it('loads config and keeps debounce boundary values', async () => { + mockReadFile.mockResolvedValueOnce( + JSON.stringify({ + gateway: { reload: { mode: 'restart', debounceMs: 60_000 } }, + }), + ); + + await expect(loadGatewayReloadPolicy()).resolves.toEqual({ + mode: 'restart', + debounceMs: 60_000, + }); + }); +}); diff --git a/tests/unit/gateway-restart-governor.test.ts b/tests/unit/gateway-restart-governor.test.ts new file mode 100644 index 000000000..a1dfdfbfd --- /dev/null +++ b/tests/unit/gateway-restart-governor.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from 'vitest'; +import { GatewayRestartGovernor } from '@electron/gateway/restart-governor'; + +describe('GatewayRestartGovernor', () => { + it('suppresses restart during exponential cooldown window', () => { + const governor = new GatewayRestartGovernor({ + baseCooldownMs: 1000, + maxCooldownMs: 8000, + maxRestartsPerWindow: 10, + windowMs: 60000, + stableResetMs: 60000, + circuitOpenMs: 60000, + }); + + expect(governor.decide(1000).allow).toBe(true); + governor.recordExecuted(1000); + + const blocked = governor.decide(1500); + expect(blocked.allow).toBe(false); + expect(blocked.allow ? '' : blocked.reason).toBe('cooldown_active'); + expect(blocked.allow ? 0 : blocked.retryAfterMs).toBeGreaterThan(0); + + expect(governor.decide(3000).allow).toBe(true); + }); + + it('opens circuit after restart budget is exceeded', () => { + const governor = new GatewayRestartGovernor({ + maxRestartsPerWindow: 2, + windowMs: 60000, + baseCooldownMs: 0, + maxCooldownMs: 0, + stableResetMs: 120000, + circuitOpenMs: 30000, + }); + + expect(governor.decide(1000).allow).toBe(true); + governor.recordExecuted(1000); + expect(governor.decide(2000).allow).toBe(true); + governor.recordExecuted(2000); + + const budgetBlocked = governor.decide(3000); + expect(budgetBlocked.allow).toBe(false); + expect(budgetBlocked.allow ? '' : budgetBlocked.reason).toBe('budget_exceeded'); + + const circuitBlocked = governor.decide(4000); + expect(circuitBlocked.allow).toBe(false); + expect(circuitBlocked.allow ? '' : circuitBlocked.reason).toBe('circuit_open'); + + expect(governor.decide(62001).allow).toBe(true); + }); + + it('resets consecutive backoff after stable running period', () => { + const governor = new GatewayRestartGovernor({ + baseCooldownMs: 1000, + maxCooldownMs: 8000, + maxRestartsPerWindow: 10, + windowMs: 600000, + stableResetMs: 5000, + circuitOpenMs: 60000, + }); + + governor.recordExecuted(0); + governor.recordExecuted(1000); + const blockedBeforeStable = governor.decide(2500); + expect(blockedBeforeStable.allow).toBe(false); + expect(blockedBeforeStable.allow ? '' : blockedBeforeStable.reason).toBe('cooldown_active'); + + governor.onRunning(3000); + const allowedAfterStable = governor.decide(9000); + expect(allowedAfterStable.allow).toBe(true); + }); + + it('resets time-based state when clock moves backwards', () => { + const governor = new GatewayRestartGovernor({ + maxRestartsPerWindow: 2, + windowMs: 60000, + baseCooldownMs: 1000, + maxCooldownMs: 8000, + stableResetMs: 60000, + circuitOpenMs: 30000, + }); + + governor.recordExecuted(10_000); + governor.recordExecuted(11_000); + const blocked = governor.decide(11_500); + expect(blocked.allow).toBe(false); + + // Simulate clock rewind and verify stale guard state does not lock out restarts. + const afterRewind = governor.decide(9_000); + expect(afterRewind.allow).toBe(true); + }); + + it('wraps counters safely at MAX_SAFE_INTEGER', () => { + const governor = new GatewayRestartGovernor(); + (governor as unknown as { executedTotal: number; suppressedTotal: number }).executedTotal = Number.MAX_SAFE_INTEGER; + (governor as unknown as { executedTotal: number; suppressedTotal: number }).suppressedTotal = Number.MAX_SAFE_INTEGER; + + governor.recordExecuted(1000); + governor.decide(1000); + + expect(governor.getCounters()).toEqual({ + executedTotal: 0, + suppressedTotal: 0, + }); + }); +}); diff --git a/tests/unit/openclaw-doctor.test.ts b/tests/unit/openclaw-doctor.test.ts index fc0ef5345..3e95499a5 100644 --- a/tests/unit/openclaw-doctor.test.ts +++ b/tests/unit/openclaw-doctor.test.ts @@ -155,4 +155,23 @@ describe('openclaw doctor output handling', () => { expect(result.stdout).toBe('line-1\nline-2\n'); expect(result.stderr).toBe('warn-1\nwarn-2\n'); }); + + it('runs plain doctor command without --json', async () => { + const child = new MockUtilityChild(); + mockFork.mockReturnValue(child); + + const { runOpenClawDoctor } = await import('@electron/utils/openclaw-doctor'); + const resultPromise = runOpenClawDoctor(); + + await vi.waitFor(() => { + expect(mockFork).toHaveBeenCalledTimes(1); + }); + child.stdout.emit('data', Buffer.from('doctor ok\n')); + child.emit('exit', 0); + + const result = await resultPromise; + expect(result.success).toBe(true); + expect(result.command).toBe('openclaw doctor'); + expect(mockFork.mock.calls[0][1]).toEqual(['doctor']); + }); });