Stabilize gateway reload/restart behavior and remove doctor --json dependency (#504)

This commit is contained in:
Lingxuan Zuo
2026-03-16 09:47:04 +08:00
committed by GitHub
Unverified
parent 89bda3c7af
commit 7f3408559d
19 changed files with 843 additions and 62 deletions

View File

@@ -25,7 +25,9 @@ function scheduleGatewayChannelRestart(ctx: HostApiContext, reason: string): voi
void reason;
}
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'whatsapp']);
function scheduleGatewayChannelSaveRefresh(
ctx: HostApiContext,

View File

@@ -43,6 +43,12 @@ import { GatewayConnectionMonitor } from './connection-monitor';
import { GatewayLifecycleController, LifecycleSupersededError } from './lifecycle-controller';
import { launchGatewayProcess } from './process-launcher';
import { GatewayRestartController } from './restart-controller';
import { GatewayRestartGovernor } from './restart-governor';
import {
DEFAULT_GATEWAY_RELOAD_POLICY,
loadGatewayReloadPolicy,
type GatewayReloadPolicy,
} from './reload-policy';
import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr';
import { runGatewayStartupSequence } from './startup-orchestrator';
@@ -94,12 +100,15 @@ export class GatewayManager extends EventEmitter {
private readonly connectionMonitor = new GatewayConnectionMonitor();
private readonly lifecycleController = new GatewayLifecycleController();
private readonly restartController = new GatewayRestartController();
private readonly restartGovernor = new GatewayRestartGovernor();
private reloadDebounceTimer: NodeJS.Timeout | null = null;
private reloadPolicy: GatewayReloadPolicy = { ...DEFAULT_GATEWAY_RELOAD_POLICY };
private reloadPolicyLoadedAt = 0;
private reloadPolicyRefreshPromise: Promise<void> | null = null;
private externalShutdownSupported: boolean | null = null;
private lastRestartAt = 0;
private reconnectAttemptsTotal = 0;
private reconnectSuccessTotal = 0;
private static readonly RESTART_COOLDOWN_MS = 2500;
private static readonly RELOAD_POLICY_REFRESH_MS = 15_000;
constructor(config?: Partial<ReconnectConfig>) {
super();
@@ -109,6 +118,9 @@ export class GatewayManager extends EventEmitter {
this.emit('status', status);
},
onTransition: (previousState, nextState) => {
if (nextState === 'running') {
this.restartGovernor.onRunning();
}
this.restartController.flushDeferredRestart(
`status:${previousState}->${nextState}`,
{
@@ -186,6 +198,7 @@ export class GatewayManager extends EventEmitter {
logger.info(`Gateway start requested (port=${this.status.port})`);
this.lastSpawnSummary = null;
this.shouldReconnect = true;
await this.refreshReloadPolicy(true);
// Lazily load device identity (async file I/O + key generation).
// Must happen before connect() which uses the identity for the handshake.
@@ -353,18 +366,27 @@ export class GatewayManager extends EventEmitter {
return;
}
const now = Date.now();
const sinceLastRestart = now - this.lastRestartAt;
if (sinceLastRestart < GatewayManager.RESTART_COOLDOWN_MS) {
logger.info(
`Gateway restart skipped due to cooldown (${sinceLastRestart}ms < ${GatewayManager.RESTART_COOLDOWN_MS}ms)`,
const decision = this.restartGovernor.decide();
if (!decision.allow) {
const observability = this.restartGovernor.getObservability();
logger.warn(
`[gateway-restart-governor] restart suppressed reason=${decision.reason} retryAfterMs=${decision.retryAfterMs} ` +
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
);
const props = {
reason: decision.reason,
retry_after_ms: decision.retryAfterMs,
gateway_restart_suppressed_total: observability.suppressed_total,
gateway_restart_executed_total: observability.executed_total,
gateway_restart_circuit_open_until: observability.circuit_open_until,
};
trackMetric('gateway.restart.suppressed', props);
captureTelemetryEvent('gateway_restart_suppressed', props);
return;
}
const pidBefore = this.status.pid;
logger.info(`[gateway-refresh] mode=restart requested pidBefore=${pidBefore ?? 'n/a'}`);
this.lastRestartAt = now;
this.restartInFlight = (async () => {
await this.stop();
await this.start();
@@ -372,8 +394,18 @@ export class GatewayManager extends EventEmitter {
try {
await this.restartInFlight;
this.restartGovernor.recordExecuted();
const observability = this.restartGovernor.getObservability();
const props = {
gateway_restart_executed_total: observability.executed_total,
gateway_restart_suppressed_total: observability.suppressed_total,
gateway_restart_circuit_open_until: observability.circuit_open_until,
};
trackMetric('gateway.restart.executed', props);
captureTelemetryEvent('gateway_restart_executed', props);
logger.info(
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'}`,
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'} ` +
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
);
} finally {
this.restartInFlight = null;
@@ -413,6 +445,16 @@ export class GatewayManager extends EventEmitter {
* Falls back to restart on unsupported platforms or signaling failures.
*/
async reload(): Promise<void> {
await this.refreshReloadPolicy();
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
logger.info(
`[gateway-refresh] mode=reload result=policy_forced_restart policy=${this.reloadPolicy.mode}`,
);
await this.restart();
return;
}
if (this.restartController.isRestartDeferred({
state: this.status.state,
startLock: this.startLock,
@@ -481,17 +523,51 @@ export class GatewayManager extends EventEmitter {
* Debounced reload — coalesces multiple rapid config-change events into one
* in-process reload when possible.
*/
debouncedReload(delayMs = 1200): void {
debouncedReload(delayMs?: number): void {
void this.refreshReloadPolicy();
const effectiveDelay = delayMs ?? this.reloadPolicy.debounceMs;
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
logger.debug(
`Gateway reload policy=${this.reloadPolicy.mode}; routing debouncedReload to debouncedRestart (${effectiveDelay}ms)`,
);
this.debouncedRestart(effectiveDelay);
return;
}
if (this.reloadDebounceTimer) {
clearTimeout(this.reloadDebounceTimer);
}
logger.debug(`Gateway reload debounced (will fire in ${delayMs}ms)`);
logger.debug(`Gateway reload debounced (will fire in ${effectiveDelay}ms)`);
this.reloadDebounceTimer = setTimeout(() => {
this.reloadDebounceTimer = null;
void this.reload().catch((err) => {
logger.warn('Debounced Gateway reload failed:', err);
});
}, delayMs);
}, effectiveDelay);
}
private async refreshReloadPolicy(force = false): Promise<void> {
const now = Date.now();
if (!force && now - this.reloadPolicyLoadedAt < GatewayManager.RELOAD_POLICY_REFRESH_MS) {
return;
}
if (this.reloadPolicyRefreshPromise) {
await this.reloadPolicyRefreshPromise;
return;
}
this.reloadPolicyRefreshPromise = (async () => {
const nextPolicy = await loadGatewayReloadPolicy();
this.reloadPolicy = nextPolicy;
this.reloadPolicyLoadedAt = Date.now();
})();
try {
await this.reloadPolicyRefreshPromise;
} finally {
this.reloadPolicyRefreshPromise = null;
}
}
/**

View File

@@ -0,0 +1,63 @@
import { readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { join } from 'node:path';
export type GatewayReloadMode = 'hybrid' | 'reload' | 'restart' | 'off';
export type GatewayReloadPolicy = {
mode: GatewayReloadMode;
debounceMs: number;
};
export const DEFAULT_GATEWAY_RELOAD_POLICY: GatewayReloadPolicy = {
mode: 'hybrid',
debounceMs: 1200,
};
const OPENCLAW_CONFIG_PATH = join(homedir(), '.openclaw', 'openclaw.json');
const MAX_DEBOUNCE_MS = 60_000;
function normalizeMode(value: unknown): GatewayReloadMode {
if (value === 'off' || value === 'reload' || value === 'restart' || value === 'hybrid') {
return value;
}
return DEFAULT_GATEWAY_RELOAD_POLICY.mode;
}
function normalizeDebounceMs(value: unknown): number {
if (typeof value !== 'number' || !Number.isFinite(value)) {
return DEFAULT_GATEWAY_RELOAD_POLICY.debounceMs;
}
const rounded = Math.round(value);
if (rounded < 0) return 0;
if (rounded > MAX_DEBOUNCE_MS) return MAX_DEBOUNCE_MS;
return rounded;
}
export function parseGatewayReloadPolicy(config: unknown): GatewayReloadPolicy {
if (!config || typeof config !== 'object') {
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
}
const root = config as Record<string, unknown>;
const gateway = (root.gateway && typeof root.gateway === 'object'
? root.gateway
: {}) as Record<string, unknown>;
const reload = (gateway.reload && typeof gateway.reload === 'object'
? gateway.reload
: {}) as Record<string, unknown>;
return {
mode: normalizeMode(reload.mode),
debounceMs: normalizeDebounceMs(reload.debounceMs),
};
}
export async function loadGatewayReloadPolicy(): Promise<GatewayReloadPolicy> {
try {
const raw = await readFile(OPENCLAW_CONFIG_PATH, 'utf-8');
return parseGatewayReloadPolicy(JSON.parse(raw));
} catch {
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
}
}

View File

@@ -0,0 +1,145 @@
export type RestartDecision =
| { allow: true }
| {
allow: false;
reason: 'circuit_open' | 'budget_exceeded' | 'cooldown_active';
retryAfterMs: number;
};
type RestartGovernorOptions = {
maxRestartsPerWindow: number;
windowMs: number;
baseCooldownMs: number;
maxCooldownMs: number;
circuitOpenMs: number;
stableResetMs: number;
};
const DEFAULT_OPTIONS: RestartGovernorOptions = {
maxRestartsPerWindow: 4,
windowMs: 10 * 60 * 1000,
baseCooldownMs: 2500,
maxCooldownMs: 2 * 60 * 1000,
circuitOpenMs: 10 * 60 * 1000,
stableResetMs: 2 * 60 * 1000,
};
export class GatewayRestartGovernor {
private readonly options: RestartGovernorOptions;
private restartTimestamps: number[] = [];
private circuitOpenUntil = 0;
private consecutiveRestarts = 0;
private lastRestartAt = 0;
private lastRunningAt = 0;
private suppressedTotal = 0;
private executedTotal = 0;
private static readonly MAX_COUNTER = Number.MAX_SAFE_INTEGER;
constructor(options?: Partial<RestartGovernorOptions>) {
this.options = { ...DEFAULT_OPTIONS, ...options };
}
onRunning(now = Date.now()): void {
this.lastRunningAt = now;
}
decide(now = Date.now()): RestartDecision {
this.pruneOld(now);
this.maybeResetConsecutive(now);
if (now < this.circuitOpenUntil) {
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
return {
allow: false,
reason: 'circuit_open',
retryAfterMs: this.circuitOpenUntil - now,
};
}
if (this.restartTimestamps.length >= this.options.maxRestartsPerWindow) {
this.circuitOpenUntil = now + this.options.circuitOpenMs;
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
return {
allow: false,
reason: 'budget_exceeded',
retryAfterMs: this.options.circuitOpenMs,
};
}
const requiredCooldown = this.getCooldownMs();
if (this.lastRestartAt > 0) {
const sinceLast = now - this.lastRestartAt;
if (sinceLast < requiredCooldown) {
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
return {
allow: false,
reason: 'cooldown_active',
retryAfterMs: requiredCooldown - sinceLast,
};
}
}
return { allow: true };
}
recordExecuted(now = Date.now()): void {
this.executedTotal = this.incrementCounter(this.executedTotal);
this.lastRestartAt = now;
this.consecutiveRestarts += 1;
this.restartTimestamps.push(now);
this.pruneOld(now);
}
getCounters(): { executedTotal: number; suppressedTotal: number } {
return {
executedTotal: this.executedTotal,
suppressedTotal: this.suppressedTotal,
};
}
getObservability(): {
suppressed_total: number;
executed_total: number;
circuit_open_until: number;
} {
return {
suppressed_total: this.suppressedTotal,
executed_total: this.executedTotal,
circuit_open_until: this.circuitOpenUntil,
};
}
private getCooldownMs(): number {
const factor = Math.pow(2, Math.max(0, this.consecutiveRestarts));
return Math.min(this.options.baseCooldownMs * factor, this.options.maxCooldownMs);
}
private maybeResetConsecutive(now: number): void {
if (this.lastRunningAt <= 0) return;
if (now - this.lastRunningAt >= this.options.stableResetMs) {
this.consecutiveRestarts = 0;
}
}
private pruneOld(now: number): void {
// Detect time rewind (system clock moved backwards) and clear all
// time-based guard state to avoid stale lockouts.
if (this.restartTimestamps.length > 0 && now < this.restartTimestamps[this.restartTimestamps.length - 1]) {
this.restartTimestamps = [];
this.circuitOpenUntil = 0;
this.lastRestartAt = 0;
this.lastRunningAt = 0;
this.consecutiveRestarts = 0;
return;
}
const threshold = now - this.options.windowMs;
while (this.restartTimestamps.length > 0 && this.restartTimestamps[0] < threshold) {
this.restartTimestamps.shift();
}
}
private incrementCounter(current: number): number {
if (current >= GatewayRestartGovernor.MAX_COUNTER) return 0;
return current + 1;
}
}

View File

@@ -1355,7 +1355,9 @@ function registerGatewayHandlers(
* For checking package status and channel configuration
*/
function registerOpenClawHandlers(gatewayManager: GatewayManager): void {
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'whatsapp']);
const scheduleGatewayChannelRestart = (reason: string): void => {
if (gatewayManager.getStatus().state !== 'stopped') {

View File

@@ -716,6 +716,65 @@ export interface ValidationResult {
warnings: string[];
}
const DOCTOR_PARSER_FALLBACK_HINT =
'Doctor output could not be confidently interpreted; falling back to local channel config checks.';
type DoctorValidationParseResult = {
errors: string[];
warnings: string[];
undetermined: boolean;
};
export function parseDoctorValidationOutput(channelType: string, output: string): DoctorValidationParseResult {
const errors: string[] = [];
const warnings: string[] = [];
const normalizedChannelType = channelType.toLowerCase();
const normalizedOutput = output.trim();
if (!normalizedOutput) {
return {
errors,
warnings: [DOCTOR_PARSER_FALLBACK_HINT],
undetermined: true,
};
}
const lines = output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
const channelLines = lines.filter((line) => line.toLowerCase().includes(normalizedChannelType));
let classifiedCount = 0;
for (const line of channelLines) {
const lowerLine = line.toLowerCase();
if (lowerLine.includes('error') || lowerLine.includes('unrecognized key')) {
errors.push(line);
classifiedCount += 1;
continue;
}
if (lowerLine.includes('warning')) {
warnings.push(line);
classifiedCount += 1;
}
}
if (channelLines.length === 0 || classifiedCount === 0) {
warnings.push(DOCTOR_PARSER_FALLBACK_HINT);
return {
errors,
warnings,
undetermined: true,
};
}
return {
errors,
warnings,
undetermined: false,
};
}
export interface CredentialValidationResult {
valid: boolean;
errors: string[];
@@ -853,34 +912,41 @@ export async function validateChannelConfig(channelType: string): Promise<Valida
// Run openclaw doctor command to validate config (async to avoid
// blocking the main thread).
const output = await new Promise<string>((resolve, reject) => {
exec(
`node openclaw.mjs doctor --json 2>&1`,
{
cwd: openclawPath,
encoding: 'utf-8',
timeout: 30000,
windowsHide: true,
},
(err, stdout) => {
if (err) reject(err);
else resolve(stdout);
},
);
});
const runDoctor = async (command: string): Promise<string> =>
await new Promise<string>((resolve, reject) => {
exec(
command,
{
cwd: openclawPath,
encoding: 'utf-8',
timeout: 30000,
windowsHide: true,
},
(err, stdout, stderr) => {
const combined = `${stdout || ''}${stderr || ''}`;
if (err) {
const next = new Error(combined || err.message);
reject(next);
return;
}
resolve(combined);
},
);
});
const lines = output.split('\n');
for (const line of lines) {
const lowerLine = line.toLowerCase();
if (lowerLine.includes(channelType) && lowerLine.includes('error')) {
result.errors.push(line.trim());
result.valid = false;
} else if (lowerLine.includes(channelType) && lowerLine.includes('warning')) {
result.warnings.push(line.trim());
} else if (lowerLine.includes('unrecognized key') && lowerLine.includes(channelType)) {
result.errors.push(line.trim());
result.valid = false;
}
const output = await runDoctor(`node openclaw.mjs doctor 2>&1`);
const parsedDoctor = parseDoctorValidationOutput(channelType, output);
result.errors.push(...parsedDoctor.errors);
result.warnings.push(...parsedDoctor.warnings);
if (parsedDoctor.errors.length > 0) {
result.valid = false;
}
if (parsedDoctor.undetermined) {
logger.warn('Doctor output parsing fell back to local channel checks', {
channelType,
hint: DOCTOR_PARSER_FALLBACK_HINT,
});
}
const config = await readOpenClawConfig();

View File

@@ -22,6 +22,7 @@ import { logger } from './logger';
import { saveProvider, getProvider, ProviderConfig } from './secure-storage';
import { getProviderDefaultModel } from './provider-registry';
import { isOpenClawPresent } from './paths';
import { proxyAwareFetch } from './proxy-fetch';
import {
loginMiniMaxPortalOAuth,
type MiniMaxOAuthToken,
@@ -47,6 +48,17 @@ class DeviceOAuthManager extends EventEmitter {
private active: boolean = false;
private mainWindow: BrowserWindow | null = null;
private async runWithProxyAwareFetch<T>(task: () => Promise<T>): Promise<T> {
const originalFetch = globalThis.fetch;
globalThis.fetch = ((input: string | URL, init?: RequestInit) =>
proxyAwareFetch(input, init)) as typeof fetch;
try {
return await task();
} finally {
globalThis.fetch = originalFetch;
}
}
setWindow(window: BrowserWindow) {
this.mainWindow = window;
}
@@ -109,7 +121,7 @@ class DeviceOAuthManager extends EventEmitter {
}
const provider = this.activeProvider!;
const token: MiniMaxOAuthToken = await loginMiniMaxPortalOAuth({
const token: MiniMaxOAuthToken = await this.runWithProxyAwareFetch(() => loginMiniMaxPortalOAuth({
region,
openUrl: async (url) => {
logger.info(`[DeviceOAuth] MiniMax opening browser: ${url}`);
@@ -133,7 +145,7 @@ class DeviceOAuthManager extends EventEmitter {
update: (msg) => logger.info(`[DeviceOAuth] MiniMax progress: ${msg}`),
stop: (msg) => logger.info(`[DeviceOAuth] MiniMax progress done: ${msg ?? ''}`),
},
});
}));
if (!this.active) return;
@@ -159,7 +171,7 @@ class DeviceOAuthManager extends EventEmitter {
}
const provider = this.activeProvider!;
const token: QwenOAuthToken = await loginQwenPortalOAuth({
const token: QwenOAuthToken = await this.runWithProxyAwareFetch(() => loginQwenPortalOAuth({
openUrl: async (url) => {
logger.info(`[DeviceOAuth] Qwen opening browser: ${url}`);
shell.openExternal(url).catch((err) =>
@@ -179,7 +191,7 @@ class DeviceOAuthManager extends EventEmitter {
update: (msg) => logger.info(`[DeviceOAuth] Qwen progress: ${msg}`),
stop: (msg) => logger.info(`[DeviceOAuth] Qwen progress done: ${msg ?? ''}`),
},
});
}));
if (!this.active) return;

View File

@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, unlinkS
import { createServer } from 'node:http';
import { delimiter, dirname, join } from 'node:path';
import { getClawXConfigDir } from './paths';
import { proxyAwareFetch } from './proxy-fetch';
const CLIENT_ID_KEYS = ['OPENCLAW_GEMINI_OAUTH_CLIENT_ID', 'GEMINI_CLI_OAUTH_CLIENT_ID'];
const CLIENT_SECRET_KEYS = [
@@ -243,7 +244,7 @@ async function installViaNpm(onProgress?: (msg: string) => void): Promise<boolea
async function installViaDirectDownload(onProgress?: (msg: string) => void): Promise<boolean> {
try {
onProgress?.('Downloading Gemini OAuth helper...');
const metaRes = await fetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
const metaRes = await proxyAwareFetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
if (!metaRes.ok) {
onProgress?.(`Failed to fetch Gemini package metadata: ${metaRes.status}`);
return false;
@@ -256,7 +257,7 @@ async function installViaDirectDownload(onProgress?: (msg: string) => void): Pro
return false;
}
const tarRes = await fetch(tarballUrl);
const tarRes = await proxyAwareFetch(tarballUrl);
if (!tarRes.ok) {
onProgress?.(`Failed to download Gemini package: ${tarRes.status}`);
return false;
@@ -440,7 +441,7 @@ async function waitForLocalCallback(params: {
async function getUserEmail(accessToken: string): Promise<string | undefined> {
try {
const response = await fetch(USERINFO_URL, {
const response = await proxyAwareFetch(USERINFO_URL, {
headers: { Authorization: `Bearer ${accessToken}` },
});
if (response.ok) {
@@ -489,7 +490,7 @@ async function pollOperation(
): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> {
for (let attempt = 0; attempt < 24; attempt += 1) {
await new Promise((resolve) => setTimeout(resolve, 5000));
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
if (!response.ok) {
continue;
}
@@ -530,7 +531,7 @@ async function discoverProject(accessToken: string): Promise<string> {
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>;
} = {};
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
method: 'POST',
headers,
body: JSON.stringify(loadBody),
@@ -583,7 +584,7 @@ async function discoverProject(accessToken: string): Promise<string> {
(onboardBody.metadata as Record<string, unknown>).duetProject = envProject;
}
const onboardResponse = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
const onboardResponse = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
method: 'POST',
headers,
body: JSON.stringify(onboardBody),
@@ -638,7 +639,7 @@ async function exchangeCodeForTokens(
body.set('client_secret', clientSecret);
}
const response = await fetch(TOKEN_URL, {
const response = await proxyAwareFetch(TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: body.toString(),

View File

@@ -1,5 +1,6 @@
import { createHash, randomBytes } from 'node:crypto';
import { createServer } from 'node:http';
import { proxyAwareFetch } from './proxy-fetch';
const CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann';
const AUTHORIZE_URL = 'https://auth.openai.com/oauth/authorize';
@@ -206,7 +207,7 @@ async function exchangeAuthorizationCode(
code: string,
verifier: string,
): Promise<{ access: string; refresh: string; expires: number }> {
const response = await fetch(TOKEN_URL, {
const response = await proxyAwareFetch(TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({

View File

@@ -7,7 +7,7 @@ import { getUvMirrorEnv } from './uv-env';
const OPENCLAW_DOCTOR_TIMEOUT_MS = 60_000;
const MAX_DOCTOR_OUTPUT_BYTES = 10 * 1024 * 1024;
const OPENCLAW_DOCTOR_ARGS = ['doctor', '--json'];
const OPENCLAW_DOCTOR_ARGS = ['doctor'];
const OPENCLAW_DOCTOR_FIX_ARGS = ['doctor', '--fix', '--yes', '--non-interactive'];
export type OpenClawDoctorMode = 'diagnose' | 'fix';
@@ -65,10 +65,12 @@ function getBundledBinPath(): string {
: path.join(process.cwd(), 'resources', 'bin', target);
}
async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDoctorResult> {
async function runDoctorCommandWithArgs(
mode: OpenClawDoctorMode,
args: string[],
): Promise<OpenClawDoctorResult> {
const openclawDir = getOpenClawDir();
const entryScript = getOpenClawEntryPath();
const args = mode === 'fix' ? OPENCLAW_DOCTOR_FIX_ARGS : OPENCLAW_DOCTOR_ARGS;
const command = `openclaw ${args.join(' ')}`;
const startedAt = Date.now();
@@ -194,9 +196,9 @@ async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDocto
}
export async function runOpenClawDoctor(): Promise<OpenClawDoctorResult> {
return await runDoctorCommand('diagnose');
return await runDoctorCommandWithArgs('diagnose', OPENCLAW_DOCTOR_ARGS);
}
export async function runOpenClawDoctorFix(): Promise<OpenClawDoctorResult> {
return await runDoctorCommand('fix');
return await runDoctorCommandWithArgs('fix', OPENCLAW_DOCTOR_FIX_ARGS);
}

View File

@@ -220,7 +220,7 @@
"cmdUnavailable": "Command unavailable",
"cmdCopied": "CLI command copied",
"doctor": "OpenClaw Doctor",
"doctorDesc": "Run `openclaw doctor --json` and inspect the raw diagnostic output.",
"doctorDesc": "Run `openclaw doctor` and inspect the raw diagnostic output.",
"runDoctor": "Run Doctor",
"runDoctorFix": "Run Doctor Fix",
"doctorSucceeded": "OpenClaw doctor completed",

View File

@@ -217,7 +217,7 @@
"cmdUnavailable": "コマンドが利用できません",
"cmdCopied": "CLI コマンドをコピーしました",
"doctor": "OpenClaw Doctor",
"doctorDesc": "`openclaw doctor --json` を実行して診断の生出力を確認します。",
"doctorDesc": "`openclaw doctor` を実行して診断の生出力を確認します。",
"runDoctor": "Doctor を実行",
"runDoctorFix": "Doctor 修復を実行",
"doctorSucceeded": "OpenClaw doctor が完了しました",

View File

@@ -220,7 +220,7 @@
"cmdUnavailable": "命令不可用",
"cmdCopied": "CLI 命令已复制",
"doctor": "OpenClaw Doctor 诊断",
"doctorDesc": "运行 `openclaw doctor --json` 并查看原始诊断输出。",
"doctorDesc": "运行 `openclaw doctor` 并查看原始诊断输出。",
"runDoctor": "运行 Doctor",
"runDoctorFix": "运行 Doctor 并修复",
"doctorSucceeded": "OpenClaw doctor 已完成",

View File

@@ -169,7 +169,7 @@ export function Settings() {
exitCode: null,
stdout: '',
stderr: '',
command: 'openclaw doctor --json',
command: 'openclaw doctor',
cwd: '',
durationMs: 0,
error: message,

View File

@@ -102,3 +102,38 @@ describe('channel credential normalization and duplicate checks', () => {
);
});
});
describe('parseDoctorValidationOutput', () => {
it('extracts channel error and warning lines', async () => {
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
const out = parseDoctorValidationOutput(
'feishu',
'feishu error: token invalid\nfeishu warning: fallback enabled\n',
);
expect(out.undetermined).toBe(false);
expect(out.errors).toEqual(['feishu error: token invalid']);
expect(out.warnings).toEqual(['feishu warning: fallback enabled']);
});
it('falls back with hint when output has no channel signal', async () => {
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
const out = parseDoctorValidationOutput('feishu', 'all good, no channel details');
expect(out.undetermined).toBe(true);
expect(out.errors).toEqual([]);
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
});
it('falls back with hint when output is empty', async () => {
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
const out = parseDoctorValidationOutput('feishu', ' ');
expect(out.undetermined).toBe(true);
expect(out.errors).toEqual([]);
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
});
});

View File

@@ -0,0 +1,101 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { mockLoadGatewayReloadPolicy } = vi.hoisted(() => ({
mockLoadGatewayReloadPolicy: vi.fn(),
}));
vi.mock('electron', () => ({
app: {
getPath: () => '/tmp',
isPackaged: false,
},
utilityProcess: {
fork: vi.fn(),
},
}));
vi.mock('@electron/gateway/reload-policy', async () => {
const actual = await vi.importActual<typeof import('@electron/gateway/reload-policy')>(
'@electron/gateway/reload-policy',
);
return {
...actual,
loadGatewayReloadPolicy: (...args: unknown[]) => mockLoadGatewayReloadPolicy(...args),
};
});
describe('GatewayManager refreshReloadPolicy', () => {
beforeEach(() => {
vi.resetAllMocks();
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-03-15T00:00:00.000Z'));
});
it('deduplicates concurrent refresh calls', async () => {
const { GatewayManager } = await import('@electron/gateway/manager');
let resolveLoad: ((value: { mode: 'reload'; debounceMs: number }) => void) | null = null;
mockLoadGatewayReloadPolicy.mockImplementationOnce(
() =>
new Promise((resolve) => {
resolveLoad = resolve;
}),
);
const manager = new GatewayManager();
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
.refreshReloadPolicy.bind(manager);
const p1 = refresh(true);
const p2 = refresh(true);
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
resolveLoad?.({ mode: 'reload', debounceMs: 1300 });
await Promise.all([p1, p2]);
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
mode: 'reload',
debounceMs: 1300,
});
});
it('hits TTL cache and skips refresh within window', async () => {
const { GatewayManager } = await import('@electron/gateway/manager');
mockLoadGatewayReloadPolicy.mockResolvedValueOnce({ mode: 'restart', debounceMs: 2200 });
const manager = new GatewayManager();
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
.refreshReloadPolicy.bind(manager);
await refresh();
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
vi.setSystemTime(new Date('2026-03-15T00:00:10.000Z'));
await refresh();
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
});
it('refreshes immediately when force=true even within TTL', async () => {
const { GatewayManager } = await import('@electron/gateway/manager');
mockLoadGatewayReloadPolicy
.mockResolvedValueOnce({ mode: 'hybrid', debounceMs: 1200 })
.mockResolvedValueOnce({ mode: 'off', debounceMs: 9000 });
const manager = new GatewayManager();
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
.refreshReloadPolicy.bind(manager);
await refresh();
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
vi.setSystemTime(new Date('2026-03-15T00:00:05.000Z'));
await refresh(true);
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(2);
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
mode: 'off',
debounceMs: 9000,
});
});
});

View File

@@ -0,0 +1,150 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { mockReadFile } = vi.hoisted(() => ({
mockReadFile: vi.fn(),
}));
vi.mock('node:fs/promises', () => ({
readFile: mockReadFile,
default: {
readFile: mockReadFile,
},
}));
import {
DEFAULT_GATEWAY_RELOAD_POLICY,
loadGatewayReloadPolicy,
parseGatewayReloadPolicy,
} from '@electron/gateway/reload-policy';
describe('parseGatewayReloadPolicy', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('returns defaults when config is missing', () => {
expect(parseGatewayReloadPolicy(undefined)).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
});
it('parses mode and debounce from gateway.reload', () => {
const result = parseGatewayReloadPolicy({
gateway: {
reload: {
mode: 'off',
debounceMs: 3000,
},
},
});
expect(result).toEqual({ mode: 'off', debounceMs: 3000 });
});
it('normalizes invalid mode and debounce bounds', () => {
const negative = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'invalid', debounceMs: -100 } },
});
expect(negative).toEqual({
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
debounceMs: 0,
});
const overMax = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'hybrid', debounceMs: 600_000 } },
});
expect(overMax).toEqual({ mode: 'hybrid', debounceMs: 60_000 });
});
it('falls back to default mode for non-string or unknown mode values', () => {
const unknownString = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'HYBRID', debounceMs: 1200 } },
});
expect(unknownString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
const nonString = parseGatewayReloadPolicy({
gateway: { reload: { mode: { value: 'reload' }, debounceMs: 1200 } },
});
expect(nonString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
});
it('handles malformed gateway/reload shapes', () => {
const malformedGateway = parseGatewayReloadPolicy({ gateway: 'bad-shape' });
expect(malformedGateway).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
const malformedReload = parseGatewayReloadPolicy({
gateway: { reload: ['bad-shape'] },
});
expect(malformedReload).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
});
it('normalizes debounce boundary and rounding behavior', () => {
const atMin = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'reload', debounceMs: 0 } },
});
expect(atMin).toEqual({ mode: 'reload', debounceMs: 0 });
const roundsUpToCap = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'reload', debounceMs: 60_000.5 } },
});
expect(roundsUpToCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
const roundsDownAtCap = parseGatewayReloadPolicy({
gateway: { reload: { mode: 'reload', debounceMs: 60_000.4 } },
});
expect(roundsDownAtCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
});
});
describe('loadGatewayReloadPolicy', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('returns defaults when config read fails', async () => {
mockReadFile.mockRejectedValueOnce(new Error('EACCES'));
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
expect(mockReadFile).toHaveBeenCalledOnce();
});
it('returns defaults when config JSON is malformed', async () => {
mockReadFile.mockResolvedValueOnce('{');
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
});
it('returns defaults when config JSON has malformed shape', async () => {
mockReadFile.mockResolvedValueOnce(
JSON.stringify({
gateway: { reload: ['malformed'] },
}),
);
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
});
it('loads config and applies invalid mode fallback', async () => {
mockReadFile.mockResolvedValueOnce(
JSON.stringify({
gateway: { reload: { mode: 'unknown-mode', debounceMs: 1350 } },
}),
);
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
debounceMs: 1350,
});
});
it('loads config and keeps debounce boundary values', async () => {
mockReadFile.mockResolvedValueOnce(
JSON.stringify({
gateway: { reload: { mode: 'restart', debounceMs: 60_000 } },
}),
);
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
mode: 'restart',
debounceMs: 60_000,
});
});
});

View File

@@ -0,0 +1,106 @@
import { describe, expect, it } from 'vitest';
import { GatewayRestartGovernor } from '@electron/gateway/restart-governor';
describe('GatewayRestartGovernor', () => {
it('suppresses restart during exponential cooldown window', () => {
const governor = new GatewayRestartGovernor({
baseCooldownMs: 1000,
maxCooldownMs: 8000,
maxRestartsPerWindow: 10,
windowMs: 60000,
stableResetMs: 60000,
circuitOpenMs: 60000,
});
expect(governor.decide(1000).allow).toBe(true);
governor.recordExecuted(1000);
const blocked = governor.decide(1500);
expect(blocked.allow).toBe(false);
expect(blocked.allow ? '' : blocked.reason).toBe('cooldown_active');
expect(blocked.allow ? 0 : blocked.retryAfterMs).toBeGreaterThan(0);
expect(governor.decide(3000).allow).toBe(true);
});
it('opens circuit after restart budget is exceeded', () => {
const governor = new GatewayRestartGovernor({
maxRestartsPerWindow: 2,
windowMs: 60000,
baseCooldownMs: 0,
maxCooldownMs: 0,
stableResetMs: 120000,
circuitOpenMs: 30000,
});
expect(governor.decide(1000).allow).toBe(true);
governor.recordExecuted(1000);
expect(governor.decide(2000).allow).toBe(true);
governor.recordExecuted(2000);
const budgetBlocked = governor.decide(3000);
expect(budgetBlocked.allow).toBe(false);
expect(budgetBlocked.allow ? '' : budgetBlocked.reason).toBe('budget_exceeded');
const circuitBlocked = governor.decide(4000);
expect(circuitBlocked.allow).toBe(false);
expect(circuitBlocked.allow ? '' : circuitBlocked.reason).toBe('circuit_open');
expect(governor.decide(62001).allow).toBe(true);
});
it('resets consecutive backoff after stable running period', () => {
const governor = new GatewayRestartGovernor({
baseCooldownMs: 1000,
maxCooldownMs: 8000,
maxRestartsPerWindow: 10,
windowMs: 600000,
stableResetMs: 5000,
circuitOpenMs: 60000,
});
governor.recordExecuted(0);
governor.recordExecuted(1000);
const blockedBeforeStable = governor.decide(2500);
expect(blockedBeforeStable.allow).toBe(false);
expect(blockedBeforeStable.allow ? '' : blockedBeforeStable.reason).toBe('cooldown_active');
governor.onRunning(3000);
const allowedAfterStable = governor.decide(9000);
expect(allowedAfterStable.allow).toBe(true);
});
it('resets time-based state when clock moves backwards', () => {
const governor = new GatewayRestartGovernor({
maxRestartsPerWindow: 2,
windowMs: 60000,
baseCooldownMs: 1000,
maxCooldownMs: 8000,
stableResetMs: 60000,
circuitOpenMs: 30000,
});
governor.recordExecuted(10_000);
governor.recordExecuted(11_000);
const blocked = governor.decide(11_500);
expect(blocked.allow).toBe(false);
// Simulate clock rewind and verify stale guard state does not lock out restarts.
const afterRewind = governor.decide(9_000);
expect(afterRewind.allow).toBe(true);
});
it('wraps counters safely at MAX_SAFE_INTEGER', () => {
const governor = new GatewayRestartGovernor();
(governor as unknown as { executedTotal: number; suppressedTotal: number }).executedTotal = Number.MAX_SAFE_INTEGER;
(governor as unknown as { executedTotal: number; suppressedTotal: number }).suppressedTotal = Number.MAX_SAFE_INTEGER;
governor.recordExecuted(1000);
governor.decide(1000);
expect(governor.getCounters()).toEqual({
executedTotal: 0,
suppressedTotal: 0,
});
});
});

View File

@@ -155,4 +155,23 @@ describe('openclaw doctor output handling', () => {
expect(result.stdout).toBe('line-1\nline-2\n');
expect(result.stderr).toBe('warn-1\nwarn-2\n');
});
it('runs plain doctor command without --json', async () => {
const child = new MockUtilityChild();
mockFork.mockReturnValue(child);
const { runOpenClawDoctor } = await import('@electron/utils/openclaw-doctor');
const resultPromise = runOpenClawDoctor();
await vi.waitFor(() => {
expect(mockFork).toHaveBeenCalledTimes(1);
});
child.stdout.emit('data', Buffer.from('doctor ok\n'));
child.emit('exit', 0);
const result = await resultPromise;
expect(result.success).toBe(true);
expect(result.command).toBe('openclaw doctor');
expect(mockFork.mock.calls[0][1]).toEqual(['doctor']);
});
});