Stabilize gateway reload/restart behavior and remove doctor --json dependency (#504)
This commit is contained in:
committed by
GitHub
Unverified
parent
89bda3c7af
commit
7f3408559d
@@ -25,7 +25,9 @@ function scheduleGatewayChannelRestart(ctx: HostApiContext, reason: string): voi
|
||||
void reason;
|
||||
}
|
||||
|
||||
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
|
||||
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
|
||||
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
|
||||
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'whatsapp']);
|
||||
|
||||
function scheduleGatewayChannelSaveRefresh(
|
||||
ctx: HostApiContext,
|
||||
|
||||
@@ -43,6 +43,12 @@ import { GatewayConnectionMonitor } from './connection-monitor';
|
||||
import { GatewayLifecycleController, LifecycleSupersededError } from './lifecycle-controller';
|
||||
import { launchGatewayProcess } from './process-launcher';
|
||||
import { GatewayRestartController } from './restart-controller';
|
||||
import { GatewayRestartGovernor } from './restart-governor';
|
||||
import {
|
||||
DEFAULT_GATEWAY_RELOAD_POLICY,
|
||||
loadGatewayReloadPolicy,
|
||||
type GatewayReloadPolicy,
|
||||
} from './reload-policy';
|
||||
import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr';
|
||||
import { runGatewayStartupSequence } from './startup-orchestrator';
|
||||
|
||||
@@ -94,12 +100,15 @@ export class GatewayManager extends EventEmitter {
|
||||
private readonly connectionMonitor = new GatewayConnectionMonitor();
|
||||
private readonly lifecycleController = new GatewayLifecycleController();
|
||||
private readonly restartController = new GatewayRestartController();
|
||||
private readonly restartGovernor = new GatewayRestartGovernor();
|
||||
private reloadDebounceTimer: NodeJS.Timeout | null = null;
|
||||
private reloadPolicy: GatewayReloadPolicy = { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||
private reloadPolicyLoadedAt = 0;
|
||||
private reloadPolicyRefreshPromise: Promise<void> | null = null;
|
||||
private externalShutdownSupported: boolean | null = null;
|
||||
private lastRestartAt = 0;
|
||||
private reconnectAttemptsTotal = 0;
|
||||
private reconnectSuccessTotal = 0;
|
||||
private static readonly RESTART_COOLDOWN_MS = 2500;
|
||||
private static readonly RELOAD_POLICY_REFRESH_MS = 15_000;
|
||||
|
||||
constructor(config?: Partial<ReconnectConfig>) {
|
||||
super();
|
||||
@@ -109,6 +118,9 @@ export class GatewayManager extends EventEmitter {
|
||||
this.emit('status', status);
|
||||
},
|
||||
onTransition: (previousState, nextState) => {
|
||||
if (nextState === 'running') {
|
||||
this.restartGovernor.onRunning();
|
||||
}
|
||||
this.restartController.flushDeferredRestart(
|
||||
`status:${previousState}->${nextState}`,
|
||||
{
|
||||
@@ -186,6 +198,7 @@ export class GatewayManager extends EventEmitter {
|
||||
logger.info(`Gateway start requested (port=${this.status.port})`);
|
||||
this.lastSpawnSummary = null;
|
||||
this.shouldReconnect = true;
|
||||
await this.refreshReloadPolicy(true);
|
||||
|
||||
// Lazily load device identity (async file I/O + key generation).
|
||||
// Must happen before connect() which uses the identity for the handshake.
|
||||
@@ -353,18 +366,27 @@ export class GatewayManager extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const sinceLastRestart = now - this.lastRestartAt;
|
||||
if (sinceLastRestart < GatewayManager.RESTART_COOLDOWN_MS) {
|
||||
logger.info(
|
||||
`Gateway restart skipped due to cooldown (${sinceLastRestart}ms < ${GatewayManager.RESTART_COOLDOWN_MS}ms)`,
|
||||
const decision = this.restartGovernor.decide();
|
||||
if (!decision.allow) {
|
||||
const observability = this.restartGovernor.getObservability();
|
||||
logger.warn(
|
||||
`[gateway-restart-governor] restart suppressed reason=${decision.reason} retryAfterMs=${decision.retryAfterMs} ` +
|
||||
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
|
||||
);
|
||||
const props = {
|
||||
reason: decision.reason,
|
||||
retry_after_ms: decision.retryAfterMs,
|
||||
gateway_restart_suppressed_total: observability.suppressed_total,
|
||||
gateway_restart_executed_total: observability.executed_total,
|
||||
gateway_restart_circuit_open_until: observability.circuit_open_until,
|
||||
};
|
||||
trackMetric('gateway.restart.suppressed', props);
|
||||
captureTelemetryEvent('gateway_restart_suppressed', props);
|
||||
return;
|
||||
}
|
||||
|
||||
const pidBefore = this.status.pid;
|
||||
logger.info(`[gateway-refresh] mode=restart requested pidBefore=${pidBefore ?? 'n/a'}`);
|
||||
this.lastRestartAt = now;
|
||||
this.restartInFlight = (async () => {
|
||||
await this.stop();
|
||||
await this.start();
|
||||
@@ -372,8 +394,18 @@ export class GatewayManager extends EventEmitter {
|
||||
|
||||
try {
|
||||
await this.restartInFlight;
|
||||
this.restartGovernor.recordExecuted();
|
||||
const observability = this.restartGovernor.getObservability();
|
||||
const props = {
|
||||
gateway_restart_executed_total: observability.executed_total,
|
||||
gateway_restart_suppressed_total: observability.suppressed_total,
|
||||
gateway_restart_circuit_open_until: observability.circuit_open_until,
|
||||
};
|
||||
trackMetric('gateway.restart.executed', props);
|
||||
captureTelemetryEvent('gateway_restart_executed', props);
|
||||
logger.info(
|
||||
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'}`,
|
||||
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'} ` +
|
||||
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
|
||||
);
|
||||
} finally {
|
||||
this.restartInFlight = null;
|
||||
@@ -413,6 +445,16 @@ export class GatewayManager extends EventEmitter {
|
||||
* Falls back to restart on unsupported platforms or signaling failures.
|
||||
*/
|
||||
async reload(): Promise<void> {
|
||||
await this.refreshReloadPolicy();
|
||||
|
||||
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
|
||||
logger.info(
|
||||
`[gateway-refresh] mode=reload result=policy_forced_restart policy=${this.reloadPolicy.mode}`,
|
||||
);
|
||||
await this.restart();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.restartController.isRestartDeferred({
|
||||
state: this.status.state,
|
||||
startLock: this.startLock,
|
||||
@@ -481,17 +523,51 @@ export class GatewayManager extends EventEmitter {
|
||||
* Debounced reload — coalesces multiple rapid config-change events into one
|
||||
* in-process reload when possible.
|
||||
*/
|
||||
debouncedReload(delayMs = 1200): void {
|
||||
debouncedReload(delayMs?: number): void {
|
||||
void this.refreshReloadPolicy();
|
||||
const effectiveDelay = delayMs ?? this.reloadPolicy.debounceMs;
|
||||
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
|
||||
logger.debug(
|
||||
`Gateway reload policy=${this.reloadPolicy.mode}; routing debouncedReload to debouncedRestart (${effectiveDelay}ms)`,
|
||||
);
|
||||
this.debouncedRestart(effectiveDelay);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.reloadDebounceTimer) {
|
||||
clearTimeout(this.reloadDebounceTimer);
|
||||
}
|
||||
logger.debug(`Gateway reload debounced (will fire in ${delayMs}ms)`);
|
||||
logger.debug(`Gateway reload debounced (will fire in ${effectiveDelay}ms)`);
|
||||
this.reloadDebounceTimer = setTimeout(() => {
|
||||
this.reloadDebounceTimer = null;
|
||||
void this.reload().catch((err) => {
|
||||
logger.warn('Debounced Gateway reload failed:', err);
|
||||
});
|
||||
}, delayMs);
|
||||
}, effectiveDelay);
|
||||
}
|
||||
|
||||
private async refreshReloadPolicy(force = false): Promise<void> {
|
||||
const now = Date.now();
|
||||
if (!force && now - this.reloadPolicyLoadedAt < GatewayManager.RELOAD_POLICY_REFRESH_MS) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.reloadPolicyRefreshPromise) {
|
||||
await this.reloadPolicyRefreshPromise;
|
||||
return;
|
||||
}
|
||||
|
||||
this.reloadPolicyRefreshPromise = (async () => {
|
||||
const nextPolicy = await loadGatewayReloadPolicy();
|
||||
this.reloadPolicy = nextPolicy;
|
||||
this.reloadPolicyLoadedAt = Date.now();
|
||||
})();
|
||||
|
||||
try {
|
||||
await this.reloadPolicyRefreshPromise;
|
||||
} finally {
|
||||
this.reloadPolicyRefreshPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
63
electron/gateway/reload-policy.ts
Normal file
63
electron/gateway/reload-policy.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export type GatewayReloadMode = 'hybrid' | 'reload' | 'restart' | 'off';
|
||||
|
||||
export type GatewayReloadPolicy = {
|
||||
mode: GatewayReloadMode;
|
||||
debounceMs: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_GATEWAY_RELOAD_POLICY: GatewayReloadPolicy = {
|
||||
mode: 'hybrid',
|
||||
debounceMs: 1200,
|
||||
};
|
||||
|
||||
const OPENCLAW_CONFIG_PATH = join(homedir(), '.openclaw', 'openclaw.json');
|
||||
const MAX_DEBOUNCE_MS = 60_000;
|
||||
|
||||
function normalizeMode(value: unknown): GatewayReloadMode {
|
||||
if (value === 'off' || value === 'reload' || value === 'restart' || value === 'hybrid') {
|
||||
return value;
|
||||
}
|
||||
return DEFAULT_GATEWAY_RELOAD_POLICY.mode;
|
||||
}
|
||||
|
||||
function normalizeDebounceMs(value: unknown): number {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||
return DEFAULT_GATEWAY_RELOAD_POLICY.debounceMs;
|
||||
}
|
||||
const rounded = Math.round(value);
|
||||
if (rounded < 0) return 0;
|
||||
if (rounded > MAX_DEBOUNCE_MS) return MAX_DEBOUNCE_MS;
|
||||
return rounded;
|
||||
}
|
||||
|
||||
export function parseGatewayReloadPolicy(config: unknown): GatewayReloadPolicy {
|
||||
if (!config || typeof config !== 'object') {
|
||||
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||
}
|
||||
const root = config as Record<string, unknown>;
|
||||
const gateway = (root.gateway && typeof root.gateway === 'object'
|
||||
? root.gateway
|
||||
: {}) as Record<string, unknown>;
|
||||
const reload = (gateway.reload && typeof gateway.reload === 'object'
|
||||
? gateway.reload
|
||||
: {}) as Record<string, unknown>;
|
||||
|
||||
return {
|
||||
mode: normalizeMode(reload.mode),
|
||||
debounceMs: normalizeDebounceMs(reload.debounceMs),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadGatewayReloadPolicy(): Promise<GatewayReloadPolicy> {
|
||||
try {
|
||||
const raw = await readFile(OPENCLAW_CONFIG_PATH, 'utf-8');
|
||||
return parseGatewayReloadPolicy(JSON.parse(raw));
|
||||
} catch {
|
||||
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||
}
|
||||
}
|
||||
|
||||
145
electron/gateway/restart-governor.ts
Normal file
145
electron/gateway/restart-governor.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
export type RestartDecision =
|
||||
| { allow: true }
|
||||
| {
|
||||
allow: false;
|
||||
reason: 'circuit_open' | 'budget_exceeded' | 'cooldown_active';
|
||||
retryAfterMs: number;
|
||||
};
|
||||
|
||||
type RestartGovernorOptions = {
|
||||
maxRestartsPerWindow: number;
|
||||
windowMs: number;
|
||||
baseCooldownMs: number;
|
||||
maxCooldownMs: number;
|
||||
circuitOpenMs: number;
|
||||
stableResetMs: number;
|
||||
};
|
||||
|
||||
const DEFAULT_OPTIONS: RestartGovernorOptions = {
|
||||
maxRestartsPerWindow: 4,
|
||||
windowMs: 10 * 60 * 1000,
|
||||
baseCooldownMs: 2500,
|
||||
maxCooldownMs: 2 * 60 * 1000,
|
||||
circuitOpenMs: 10 * 60 * 1000,
|
||||
stableResetMs: 2 * 60 * 1000,
|
||||
};
|
||||
|
||||
export class GatewayRestartGovernor {
|
||||
private readonly options: RestartGovernorOptions;
|
||||
private restartTimestamps: number[] = [];
|
||||
private circuitOpenUntil = 0;
|
||||
private consecutiveRestarts = 0;
|
||||
private lastRestartAt = 0;
|
||||
private lastRunningAt = 0;
|
||||
private suppressedTotal = 0;
|
||||
private executedTotal = 0;
|
||||
private static readonly MAX_COUNTER = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
constructor(options?: Partial<RestartGovernorOptions>) {
|
||||
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||
}
|
||||
|
||||
onRunning(now = Date.now()): void {
|
||||
this.lastRunningAt = now;
|
||||
}
|
||||
|
||||
decide(now = Date.now()): RestartDecision {
|
||||
this.pruneOld(now);
|
||||
this.maybeResetConsecutive(now);
|
||||
|
||||
if (now < this.circuitOpenUntil) {
|
||||
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||
return {
|
||||
allow: false,
|
||||
reason: 'circuit_open',
|
||||
retryAfterMs: this.circuitOpenUntil - now,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.restartTimestamps.length >= this.options.maxRestartsPerWindow) {
|
||||
this.circuitOpenUntil = now + this.options.circuitOpenMs;
|
||||
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||
return {
|
||||
allow: false,
|
||||
reason: 'budget_exceeded',
|
||||
retryAfterMs: this.options.circuitOpenMs,
|
||||
};
|
||||
}
|
||||
|
||||
const requiredCooldown = this.getCooldownMs();
|
||||
if (this.lastRestartAt > 0) {
|
||||
const sinceLast = now - this.lastRestartAt;
|
||||
if (sinceLast < requiredCooldown) {
|
||||
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||
return {
|
||||
allow: false,
|
||||
reason: 'cooldown_active',
|
||||
retryAfterMs: requiredCooldown - sinceLast,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allow: true };
|
||||
}
|
||||
|
||||
recordExecuted(now = Date.now()): void {
|
||||
this.executedTotal = this.incrementCounter(this.executedTotal);
|
||||
this.lastRestartAt = now;
|
||||
this.consecutiveRestarts += 1;
|
||||
this.restartTimestamps.push(now);
|
||||
this.pruneOld(now);
|
||||
}
|
||||
|
||||
getCounters(): { executedTotal: number; suppressedTotal: number } {
|
||||
return {
|
||||
executedTotal: this.executedTotal,
|
||||
suppressedTotal: this.suppressedTotal,
|
||||
};
|
||||
}
|
||||
|
||||
getObservability(): {
|
||||
suppressed_total: number;
|
||||
executed_total: number;
|
||||
circuit_open_until: number;
|
||||
} {
|
||||
return {
|
||||
suppressed_total: this.suppressedTotal,
|
||||
executed_total: this.executedTotal,
|
||||
circuit_open_until: this.circuitOpenUntil,
|
||||
};
|
||||
}
|
||||
|
||||
private getCooldownMs(): number {
|
||||
const factor = Math.pow(2, Math.max(0, this.consecutiveRestarts));
|
||||
return Math.min(this.options.baseCooldownMs * factor, this.options.maxCooldownMs);
|
||||
}
|
||||
|
||||
private maybeResetConsecutive(now: number): void {
|
||||
if (this.lastRunningAt <= 0) return;
|
||||
if (now - this.lastRunningAt >= this.options.stableResetMs) {
|
||||
this.consecutiveRestarts = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private pruneOld(now: number): void {
|
||||
// Detect time rewind (system clock moved backwards) and clear all
|
||||
// time-based guard state to avoid stale lockouts.
|
||||
if (this.restartTimestamps.length > 0 && now < this.restartTimestamps[this.restartTimestamps.length - 1]) {
|
||||
this.restartTimestamps = [];
|
||||
this.circuitOpenUntil = 0;
|
||||
this.lastRestartAt = 0;
|
||||
this.lastRunningAt = 0;
|
||||
this.consecutiveRestarts = 0;
|
||||
return;
|
||||
}
|
||||
const threshold = now - this.options.windowMs;
|
||||
while (this.restartTimestamps.length > 0 && this.restartTimestamps[0] < threshold) {
|
||||
this.restartTimestamps.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private incrementCounter(current: number): number {
|
||||
if (current >= GatewayRestartGovernor.MAX_COUNTER) return 0;
|
||||
return current + 1;
|
||||
}
|
||||
}
|
||||
@@ -1355,7 +1355,9 @@ function registerGatewayHandlers(
|
||||
* For checking package status and channel configuration
|
||||
*/
|
||||
function registerOpenClawHandlers(gatewayManager: GatewayManager): void {
|
||||
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
|
||||
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
|
||||
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
|
||||
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'whatsapp']);
|
||||
|
||||
const scheduleGatewayChannelRestart = (reason: string): void => {
|
||||
if (gatewayManager.getStatus().state !== 'stopped') {
|
||||
|
||||
@@ -716,6 +716,65 @@ export interface ValidationResult {
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
const DOCTOR_PARSER_FALLBACK_HINT =
|
||||
'Doctor output could not be confidently interpreted; falling back to local channel config checks.';
|
||||
|
||||
type DoctorValidationParseResult = {
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
undetermined: boolean;
|
||||
};
|
||||
|
||||
export function parseDoctorValidationOutput(channelType: string, output: string): DoctorValidationParseResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const normalizedChannelType = channelType.toLowerCase();
|
||||
const normalizedOutput = output.trim();
|
||||
|
||||
if (!normalizedOutput) {
|
||||
return {
|
||||
errors,
|
||||
warnings: [DOCTOR_PARSER_FALLBACK_HINT],
|
||||
undetermined: true,
|
||||
};
|
||||
}
|
||||
|
||||
const lines = output
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
const channelLines = lines.filter((line) => line.toLowerCase().includes(normalizedChannelType));
|
||||
let classifiedCount = 0;
|
||||
|
||||
for (const line of channelLines) {
|
||||
const lowerLine = line.toLowerCase();
|
||||
if (lowerLine.includes('error') || lowerLine.includes('unrecognized key')) {
|
||||
errors.push(line);
|
||||
classifiedCount += 1;
|
||||
continue;
|
||||
}
|
||||
if (lowerLine.includes('warning')) {
|
||||
warnings.push(line);
|
||||
classifiedCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (channelLines.length === 0 || classifiedCount === 0) {
|
||||
warnings.push(DOCTOR_PARSER_FALLBACK_HINT);
|
||||
return {
|
||||
errors,
|
||||
warnings,
|
||||
undetermined: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
errors,
|
||||
warnings,
|
||||
undetermined: false,
|
||||
};
|
||||
}
|
||||
|
||||
export interface CredentialValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
@@ -853,34 +912,41 @@ export async function validateChannelConfig(channelType: string): Promise<Valida
|
||||
|
||||
// Run openclaw doctor command to validate config (async to avoid
|
||||
// blocking the main thread).
|
||||
const output = await new Promise<string>((resolve, reject) => {
|
||||
exec(
|
||||
`node openclaw.mjs doctor --json 2>&1`,
|
||||
{
|
||||
cwd: openclawPath,
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
windowsHide: true,
|
||||
},
|
||||
(err, stdout) => {
|
||||
if (err) reject(err);
|
||||
else resolve(stdout);
|
||||
},
|
||||
);
|
||||
});
|
||||
const runDoctor = async (command: string): Promise<string> =>
|
||||
await new Promise<string>((resolve, reject) => {
|
||||
exec(
|
||||
command,
|
||||
{
|
||||
cwd: openclawPath,
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
windowsHide: true,
|
||||
},
|
||||
(err, stdout, stderr) => {
|
||||
const combined = `${stdout || ''}${stderr || ''}`;
|
||||
if (err) {
|
||||
const next = new Error(combined || err.message);
|
||||
reject(next);
|
||||
return;
|
||||
}
|
||||
resolve(combined);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const lines = output.split('\n');
|
||||
for (const line of lines) {
|
||||
const lowerLine = line.toLowerCase();
|
||||
if (lowerLine.includes(channelType) && lowerLine.includes('error')) {
|
||||
result.errors.push(line.trim());
|
||||
result.valid = false;
|
||||
} else if (lowerLine.includes(channelType) && lowerLine.includes('warning')) {
|
||||
result.warnings.push(line.trim());
|
||||
} else if (lowerLine.includes('unrecognized key') && lowerLine.includes(channelType)) {
|
||||
result.errors.push(line.trim());
|
||||
result.valid = false;
|
||||
}
|
||||
const output = await runDoctor(`node openclaw.mjs doctor 2>&1`);
|
||||
|
||||
const parsedDoctor = parseDoctorValidationOutput(channelType, output);
|
||||
result.errors.push(...parsedDoctor.errors);
|
||||
result.warnings.push(...parsedDoctor.warnings);
|
||||
if (parsedDoctor.errors.length > 0) {
|
||||
result.valid = false;
|
||||
}
|
||||
if (parsedDoctor.undetermined) {
|
||||
logger.warn('Doctor output parsing fell back to local channel checks', {
|
||||
channelType,
|
||||
hint: DOCTOR_PARSER_FALLBACK_HINT,
|
||||
});
|
||||
}
|
||||
|
||||
const config = await readOpenClawConfig();
|
||||
|
||||
@@ -22,6 +22,7 @@ import { logger } from './logger';
|
||||
import { saveProvider, getProvider, ProviderConfig } from './secure-storage';
|
||||
import { getProviderDefaultModel } from './provider-registry';
|
||||
import { isOpenClawPresent } from './paths';
|
||||
import { proxyAwareFetch } from './proxy-fetch';
|
||||
import {
|
||||
loginMiniMaxPortalOAuth,
|
||||
type MiniMaxOAuthToken,
|
||||
@@ -47,6 +48,17 @@ class DeviceOAuthManager extends EventEmitter {
|
||||
private active: boolean = false;
|
||||
private mainWindow: BrowserWindow | null = null;
|
||||
|
||||
private async runWithProxyAwareFetch<T>(task: () => Promise<T>): Promise<T> {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = ((input: string | URL, init?: RequestInit) =>
|
||||
proxyAwareFetch(input, init)) as typeof fetch;
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
}
|
||||
|
||||
setWindow(window: BrowserWindow) {
|
||||
this.mainWindow = window;
|
||||
}
|
||||
@@ -109,7 +121,7 @@ class DeviceOAuthManager extends EventEmitter {
|
||||
}
|
||||
const provider = this.activeProvider!;
|
||||
|
||||
const token: MiniMaxOAuthToken = await loginMiniMaxPortalOAuth({
|
||||
const token: MiniMaxOAuthToken = await this.runWithProxyAwareFetch(() => loginMiniMaxPortalOAuth({
|
||||
region,
|
||||
openUrl: async (url) => {
|
||||
logger.info(`[DeviceOAuth] MiniMax opening browser: ${url}`);
|
||||
@@ -133,7 +145,7 @@ class DeviceOAuthManager extends EventEmitter {
|
||||
update: (msg) => logger.info(`[DeviceOAuth] MiniMax progress: ${msg}`),
|
||||
stop: (msg) => logger.info(`[DeviceOAuth] MiniMax progress done: ${msg ?? ''}`),
|
||||
},
|
||||
});
|
||||
}));
|
||||
|
||||
if (!this.active) return;
|
||||
|
||||
@@ -159,7 +171,7 @@ class DeviceOAuthManager extends EventEmitter {
|
||||
}
|
||||
const provider = this.activeProvider!;
|
||||
|
||||
const token: QwenOAuthToken = await loginQwenPortalOAuth({
|
||||
const token: QwenOAuthToken = await this.runWithProxyAwareFetch(() => loginQwenPortalOAuth({
|
||||
openUrl: async (url) => {
|
||||
logger.info(`[DeviceOAuth] Qwen opening browser: ${url}`);
|
||||
shell.openExternal(url).catch((err) =>
|
||||
@@ -179,7 +191,7 @@ class DeviceOAuthManager extends EventEmitter {
|
||||
update: (msg) => logger.info(`[DeviceOAuth] Qwen progress: ${msg}`),
|
||||
stop: (msg) => logger.info(`[DeviceOAuth] Qwen progress done: ${msg ?? ''}`),
|
||||
},
|
||||
});
|
||||
}));
|
||||
|
||||
if (!this.active) return;
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, unlinkS
|
||||
import { createServer } from 'node:http';
|
||||
import { delimiter, dirname, join } from 'node:path';
|
||||
import { getClawXConfigDir } from './paths';
|
||||
import { proxyAwareFetch } from './proxy-fetch';
|
||||
|
||||
const CLIENT_ID_KEYS = ['OPENCLAW_GEMINI_OAUTH_CLIENT_ID', 'GEMINI_CLI_OAUTH_CLIENT_ID'];
|
||||
const CLIENT_SECRET_KEYS = [
|
||||
@@ -243,7 +244,7 @@ async function installViaNpm(onProgress?: (msg: string) => void): Promise<boolea
|
||||
async function installViaDirectDownload(onProgress?: (msg: string) => void): Promise<boolean> {
|
||||
try {
|
||||
onProgress?.('Downloading Gemini OAuth helper...');
|
||||
const metaRes = await fetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
|
||||
const metaRes = await proxyAwareFetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
|
||||
if (!metaRes.ok) {
|
||||
onProgress?.(`Failed to fetch Gemini package metadata: ${metaRes.status}`);
|
||||
return false;
|
||||
@@ -256,7 +257,7 @@ async function installViaDirectDownload(onProgress?: (msg: string) => void): Pro
|
||||
return false;
|
||||
}
|
||||
|
||||
const tarRes = await fetch(tarballUrl);
|
||||
const tarRes = await proxyAwareFetch(tarballUrl);
|
||||
if (!tarRes.ok) {
|
||||
onProgress?.(`Failed to download Gemini package: ${tarRes.status}`);
|
||||
return false;
|
||||
@@ -440,7 +441,7 @@ async function waitForLocalCallback(params: {
|
||||
|
||||
async function getUserEmail(accessToken: string): Promise<string | undefined> {
|
||||
try {
|
||||
const response = await fetch(USERINFO_URL, {
|
||||
const response = await proxyAwareFetch(USERINFO_URL, {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
});
|
||||
if (response.ok) {
|
||||
@@ -489,7 +490,7 @@ async function pollOperation(
|
||||
): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> {
|
||||
for (let attempt = 0; attempt < 24; attempt += 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
|
||||
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
|
||||
if (!response.ok) {
|
||||
continue;
|
||||
}
|
||||
@@ -530,7 +531,7 @@ async function discoverProject(accessToken: string): Promise<string> {
|
||||
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>;
|
||||
} = {};
|
||||
|
||||
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
|
||||
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(loadBody),
|
||||
@@ -583,7 +584,7 @@ async function discoverProject(accessToken: string): Promise<string> {
|
||||
(onboardBody.metadata as Record<string, unknown>).duetProject = envProject;
|
||||
}
|
||||
|
||||
const onboardResponse = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
|
||||
const onboardResponse = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(onboardBody),
|
||||
@@ -638,7 +639,7 @@ async function exchangeCodeForTokens(
|
||||
body.set('client_secret', clientSecret);
|
||||
}
|
||||
|
||||
const response = await fetch(TOKEN_URL, {
|
||||
const response = await proxyAwareFetch(TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: body.toString(),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { createServer } from 'node:http';
|
||||
import { proxyAwareFetch } from './proxy-fetch';
|
||||
|
||||
const CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann';
|
||||
const AUTHORIZE_URL = 'https://auth.openai.com/oauth/authorize';
|
||||
@@ -206,7 +207,7 @@ async function exchangeAuthorizationCode(
|
||||
code: string,
|
||||
verifier: string,
|
||||
): Promise<{ access: string; refresh: string; expires: number }> {
|
||||
const response = await fetch(TOKEN_URL, {
|
||||
const response = await proxyAwareFetch(TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: new URLSearchParams({
|
||||
|
||||
@@ -7,7 +7,7 @@ import { getUvMirrorEnv } from './uv-env';
|
||||
|
||||
const OPENCLAW_DOCTOR_TIMEOUT_MS = 60_000;
|
||||
const MAX_DOCTOR_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||
const OPENCLAW_DOCTOR_ARGS = ['doctor', '--json'];
|
||||
const OPENCLAW_DOCTOR_ARGS = ['doctor'];
|
||||
const OPENCLAW_DOCTOR_FIX_ARGS = ['doctor', '--fix', '--yes', '--non-interactive'];
|
||||
|
||||
export type OpenClawDoctorMode = 'diagnose' | 'fix';
|
||||
@@ -65,10 +65,12 @@ function getBundledBinPath(): string {
|
||||
: path.join(process.cwd(), 'resources', 'bin', target);
|
||||
}
|
||||
|
||||
async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDoctorResult> {
|
||||
async function runDoctorCommandWithArgs(
|
||||
mode: OpenClawDoctorMode,
|
||||
args: string[],
|
||||
): Promise<OpenClawDoctorResult> {
|
||||
const openclawDir = getOpenClawDir();
|
||||
const entryScript = getOpenClawEntryPath();
|
||||
const args = mode === 'fix' ? OPENCLAW_DOCTOR_FIX_ARGS : OPENCLAW_DOCTOR_ARGS;
|
||||
const command = `openclaw ${args.join(' ')}`;
|
||||
const startedAt = Date.now();
|
||||
|
||||
@@ -194,9 +196,9 @@ async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDocto
|
||||
}
|
||||
|
||||
export async function runOpenClawDoctor(): Promise<OpenClawDoctorResult> {
|
||||
return await runDoctorCommand('diagnose');
|
||||
return await runDoctorCommandWithArgs('diagnose', OPENCLAW_DOCTOR_ARGS);
|
||||
}
|
||||
|
||||
export async function runOpenClawDoctorFix(): Promise<OpenClawDoctorResult> {
|
||||
return await runDoctorCommand('fix');
|
||||
return await runDoctorCommandWithArgs('fix', OPENCLAW_DOCTOR_FIX_ARGS);
|
||||
}
|
||||
|
||||
@@ -220,7 +220,7 @@
|
||||
"cmdUnavailable": "Command unavailable",
|
||||
"cmdCopied": "CLI command copied",
|
||||
"doctor": "OpenClaw Doctor",
|
||||
"doctorDesc": "Run `openclaw doctor --json` and inspect the raw diagnostic output.",
|
||||
"doctorDesc": "Run `openclaw doctor` and inspect the raw diagnostic output.",
|
||||
"runDoctor": "Run Doctor",
|
||||
"runDoctorFix": "Run Doctor Fix",
|
||||
"doctorSucceeded": "OpenClaw doctor completed",
|
||||
|
||||
@@ -217,7 +217,7 @@
|
||||
"cmdUnavailable": "コマンドが利用できません",
|
||||
"cmdCopied": "CLI コマンドをコピーしました",
|
||||
"doctor": "OpenClaw Doctor",
|
||||
"doctorDesc": "`openclaw doctor --json` を実行して診断の生出力を確認します。",
|
||||
"doctorDesc": "`openclaw doctor` を実行して診断の生出力を確認します。",
|
||||
"runDoctor": "Doctor を実行",
|
||||
"runDoctorFix": "Doctor 修復を実行",
|
||||
"doctorSucceeded": "OpenClaw doctor が完了しました",
|
||||
|
||||
@@ -220,7 +220,7 @@
|
||||
"cmdUnavailable": "命令不可用",
|
||||
"cmdCopied": "CLI 命令已复制",
|
||||
"doctor": "OpenClaw Doctor 诊断",
|
||||
"doctorDesc": "运行 `openclaw doctor --json` 并查看原始诊断输出。",
|
||||
"doctorDesc": "运行 `openclaw doctor` 并查看原始诊断输出。",
|
||||
"runDoctor": "运行 Doctor",
|
||||
"runDoctorFix": "运行 Doctor 并修复",
|
||||
"doctorSucceeded": "OpenClaw doctor 已完成",
|
||||
|
||||
@@ -169,7 +169,7 @@ export function Settings() {
|
||||
exitCode: null,
|
||||
stdout: '',
|
||||
stderr: '',
|
||||
command: 'openclaw doctor --json',
|
||||
command: 'openclaw doctor',
|
||||
cwd: '',
|
||||
durationMs: 0,
|
||||
error: message,
|
||||
|
||||
@@ -102,3 +102,38 @@ describe('channel credential normalization and duplicate checks', () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseDoctorValidationOutput', () => {
|
||||
it('extracts channel error and warning lines', async () => {
|
||||
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||
|
||||
const out = parseDoctorValidationOutput(
|
||||
'feishu',
|
||||
'feishu error: token invalid\nfeishu warning: fallback enabled\n',
|
||||
);
|
||||
|
||||
expect(out.undetermined).toBe(false);
|
||||
expect(out.errors).toEqual(['feishu error: token invalid']);
|
||||
expect(out.warnings).toEqual(['feishu warning: fallback enabled']);
|
||||
});
|
||||
|
||||
it('falls back with hint when output has no channel signal', async () => {
|
||||
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||
|
||||
const out = parseDoctorValidationOutput('feishu', 'all good, no channel details');
|
||||
|
||||
expect(out.undetermined).toBe(true);
|
||||
expect(out.errors).toEqual([]);
|
||||
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back with hint when output is empty', async () => {
|
||||
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||
|
||||
const out = parseDoctorValidationOutput('feishu', ' ');
|
||||
|
||||
expect(out.undetermined).toBe(true);
|
||||
expect(out.errors).toEqual([]);
|
||||
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
101
tests/unit/gateway-manager-reload-policy-refresh.test.ts
Normal file
101
tests/unit/gateway-manager-reload-policy-refresh.test.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const { mockLoadGatewayReloadPolicy } = vi.hoisted(() => ({
|
||||
mockLoadGatewayReloadPolicy: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: {
|
||||
getPath: () => '/tmp',
|
||||
isPackaged: false,
|
||||
},
|
||||
utilityProcess: {
|
||||
fork: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@electron/gateway/reload-policy', async () => {
|
||||
const actual = await vi.importActual<typeof import('@electron/gateway/reload-policy')>(
|
||||
'@electron/gateway/reload-policy',
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
loadGatewayReloadPolicy: (...args: unknown[]) => mockLoadGatewayReloadPolicy(...args),
|
||||
};
|
||||
});
|
||||
|
||||
describe('GatewayManager refreshReloadPolicy', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-03-15T00:00:00.000Z'));
|
||||
});
|
||||
|
||||
it('deduplicates concurrent refresh calls', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
let resolveLoad: ((value: { mode: 'reload'; debounceMs: number }) => void) | null = null;
|
||||
mockLoadGatewayReloadPolicy.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveLoad = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
const manager = new GatewayManager();
|
||||
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||
.refreshReloadPolicy.bind(manager);
|
||||
|
||||
const p1 = refresh(true);
|
||||
const p2 = refresh(true);
|
||||
|
||||
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||
|
||||
resolveLoad?.({ mode: 'reload', debounceMs: 1300 });
|
||||
await Promise.all([p1, p2]);
|
||||
|
||||
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
|
||||
mode: 'reload',
|
||||
debounceMs: 1300,
|
||||
});
|
||||
});
|
||||
|
||||
it('hits TTL cache and skips refresh within window', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
mockLoadGatewayReloadPolicy.mockResolvedValueOnce({ mode: 'restart', debounceMs: 2200 });
|
||||
|
||||
const manager = new GatewayManager();
|
||||
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||
.refreshReloadPolicy.bind(manager);
|
||||
|
||||
await refresh();
|
||||
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.setSystemTime(new Date('2026-03-15T00:00:10.000Z'));
|
||||
await refresh();
|
||||
|
||||
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('refreshes immediately when force=true even within TTL', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
mockLoadGatewayReloadPolicy
|
||||
.mockResolvedValueOnce({ mode: 'hybrid', debounceMs: 1200 })
|
||||
.mockResolvedValueOnce({ mode: 'off', debounceMs: 9000 });
|
||||
|
||||
const manager = new GatewayManager();
|
||||
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||
.refreshReloadPolicy.bind(manager);
|
||||
|
||||
await refresh();
|
||||
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.setSystemTime(new Date('2026-03-15T00:00:05.000Z'));
|
||||
await refresh(true);
|
||||
|
||||
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(2);
|
||||
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
|
||||
mode: 'off',
|
||||
debounceMs: 9000,
|
||||
});
|
||||
});
|
||||
});
|
||||
150
tests/unit/gateway-reload-policy.test.ts
Normal file
150
tests/unit/gateway-reload-policy.test.ts
Normal file
@@ -0,0 +1,150 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const { mockReadFile } = vi.hoisted(() => ({
|
||||
mockReadFile: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('node:fs/promises', () => ({
|
||||
readFile: mockReadFile,
|
||||
default: {
|
||||
readFile: mockReadFile,
|
||||
},
|
||||
}));
|
||||
|
||||
import {
|
||||
DEFAULT_GATEWAY_RELOAD_POLICY,
|
||||
loadGatewayReloadPolicy,
|
||||
parseGatewayReloadPolicy,
|
||||
} from '@electron/gateway/reload-policy';
|
||||
|
||||
describe('parseGatewayReloadPolicy', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('returns defaults when config is missing', () => {
|
||||
expect(parseGatewayReloadPolicy(undefined)).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
});
|
||||
|
||||
it('parses mode and debounce from gateway.reload', () => {
|
||||
const result = parseGatewayReloadPolicy({
|
||||
gateway: {
|
||||
reload: {
|
||||
mode: 'off',
|
||||
debounceMs: 3000,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toEqual({ mode: 'off', debounceMs: 3000 });
|
||||
});
|
||||
|
||||
it('normalizes invalid mode and debounce bounds', () => {
|
||||
const negative = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'invalid', debounceMs: -100 } },
|
||||
});
|
||||
expect(negative).toEqual({
|
||||
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
|
||||
debounceMs: 0,
|
||||
});
|
||||
|
||||
const overMax = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'hybrid', debounceMs: 600_000 } },
|
||||
});
|
||||
expect(overMax).toEqual({ mode: 'hybrid', debounceMs: 60_000 });
|
||||
});
|
||||
|
||||
it('falls back to default mode for non-string or unknown mode values', () => {
|
||||
const unknownString = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'HYBRID', debounceMs: 1200 } },
|
||||
});
|
||||
expect(unknownString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
|
||||
|
||||
const nonString = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: { value: 'reload' }, debounceMs: 1200 } },
|
||||
});
|
||||
expect(nonString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
|
||||
});
|
||||
|
||||
it('handles malformed gateway/reload shapes', () => {
|
||||
const malformedGateway = parseGatewayReloadPolicy({ gateway: 'bad-shape' });
|
||||
expect(malformedGateway).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
|
||||
const malformedReload = parseGatewayReloadPolicy({
|
||||
gateway: { reload: ['bad-shape'] },
|
||||
});
|
||||
expect(malformedReload).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
});
|
||||
|
||||
it('normalizes debounce boundary and rounding behavior', () => {
|
||||
const atMin = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'reload', debounceMs: 0 } },
|
||||
});
|
||||
expect(atMin).toEqual({ mode: 'reload', debounceMs: 0 });
|
||||
|
||||
const roundsUpToCap = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'reload', debounceMs: 60_000.5 } },
|
||||
});
|
||||
expect(roundsUpToCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
|
||||
|
||||
const roundsDownAtCap = parseGatewayReloadPolicy({
|
||||
gateway: { reload: { mode: 'reload', debounceMs: 60_000.4 } },
|
||||
});
|
||||
expect(roundsDownAtCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadGatewayReloadPolicy', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('returns defaults when config read fails', async () => {
|
||||
mockReadFile.mockRejectedValueOnce(new Error('EACCES'));
|
||||
|
||||
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
expect(mockReadFile).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('returns defaults when config JSON is malformed', async () => {
|
||||
mockReadFile.mockResolvedValueOnce('{');
|
||||
|
||||
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
});
|
||||
|
||||
it('returns defaults when config JSON has malformed shape', async () => {
|
||||
mockReadFile.mockResolvedValueOnce(
|
||||
JSON.stringify({
|
||||
gateway: { reload: ['malformed'] },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||
});
|
||||
|
||||
it('loads config and applies invalid mode fallback', async () => {
|
||||
mockReadFile.mockResolvedValueOnce(
|
||||
JSON.stringify({
|
||||
gateway: { reload: { mode: 'unknown-mode', debounceMs: 1350 } },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
|
||||
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
|
||||
debounceMs: 1350,
|
||||
});
|
||||
});
|
||||
|
||||
it('loads config and keeps debounce boundary values', async () => {
|
||||
mockReadFile.mockResolvedValueOnce(
|
||||
JSON.stringify({
|
||||
gateway: { reload: { mode: 'restart', debounceMs: 60_000 } },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
|
||||
mode: 'restart',
|
||||
debounceMs: 60_000,
|
||||
});
|
||||
});
|
||||
});
|
||||
106
tests/unit/gateway-restart-governor.test.ts
Normal file
106
tests/unit/gateway-restart-governor.test.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { GatewayRestartGovernor } from '@electron/gateway/restart-governor';
|
||||
|
||||
describe('GatewayRestartGovernor', () => {
|
||||
it('suppresses restart during exponential cooldown window', () => {
|
||||
const governor = new GatewayRestartGovernor({
|
||||
baseCooldownMs: 1000,
|
||||
maxCooldownMs: 8000,
|
||||
maxRestartsPerWindow: 10,
|
||||
windowMs: 60000,
|
||||
stableResetMs: 60000,
|
||||
circuitOpenMs: 60000,
|
||||
});
|
||||
|
||||
expect(governor.decide(1000).allow).toBe(true);
|
||||
governor.recordExecuted(1000);
|
||||
|
||||
const blocked = governor.decide(1500);
|
||||
expect(blocked.allow).toBe(false);
|
||||
expect(blocked.allow ? '' : blocked.reason).toBe('cooldown_active');
|
||||
expect(blocked.allow ? 0 : blocked.retryAfterMs).toBeGreaterThan(0);
|
||||
|
||||
expect(governor.decide(3000).allow).toBe(true);
|
||||
});
|
||||
|
||||
it('opens circuit after restart budget is exceeded', () => {
|
||||
const governor = new GatewayRestartGovernor({
|
||||
maxRestartsPerWindow: 2,
|
||||
windowMs: 60000,
|
||||
baseCooldownMs: 0,
|
||||
maxCooldownMs: 0,
|
||||
stableResetMs: 120000,
|
||||
circuitOpenMs: 30000,
|
||||
});
|
||||
|
||||
expect(governor.decide(1000).allow).toBe(true);
|
||||
governor.recordExecuted(1000);
|
||||
expect(governor.decide(2000).allow).toBe(true);
|
||||
governor.recordExecuted(2000);
|
||||
|
||||
const budgetBlocked = governor.decide(3000);
|
||||
expect(budgetBlocked.allow).toBe(false);
|
||||
expect(budgetBlocked.allow ? '' : budgetBlocked.reason).toBe('budget_exceeded');
|
||||
|
||||
const circuitBlocked = governor.decide(4000);
|
||||
expect(circuitBlocked.allow).toBe(false);
|
||||
expect(circuitBlocked.allow ? '' : circuitBlocked.reason).toBe('circuit_open');
|
||||
|
||||
expect(governor.decide(62001).allow).toBe(true);
|
||||
});
|
||||
|
||||
it('resets consecutive backoff after stable running period', () => {
|
||||
const governor = new GatewayRestartGovernor({
|
||||
baseCooldownMs: 1000,
|
||||
maxCooldownMs: 8000,
|
||||
maxRestartsPerWindow: 10,
|
||||
windowMs: 600000,
|
||||
stableResetMs: 5000,
|
||||
circuitOpenMs: 60000,
|
||||
});
|
||||
|
||||
governor.recordExecuted(0);
|
||||
governor.recordExecuted(1000);
|
||||
const blockedBeforeStable = governor.decide(2500);
|
||||
expect(blockedBeforeStable.allow).toBe(false);
|
||||
expect(blockedBeforeStable.allow ? '' : blockedBeforeStable.reason).toBe('cooldown_active');
|
||||
|
||||
governor.onRunning(3000);
|
||||
const allowedAfterStable = governor.decide(9000);
|
||||
expect(allowedAfterStable.allow).toBe(true);
|
||||
});
|
||||
|
||||
it('resets time-based state when clock moves backwards', () => {
|
||||
const governor = new GatewayRestartGovernor({
|
||||
maxRestartsPerWindow: 2,
|
||||
windowMs: 60000,
|
||||
baseCooldownMs: 1000,
|
||||
maxCooldownMs: 8000,
|
||||
stableResetMs: 60000,
|
||||
circuitOpenMs: 30000,
|
||||
});
|
||||
|
||||
governor.recordExecuted(10_000);
|
||||
governor.recordExecuted(11_000);
|
||||
const blocked = governor.decide(11_500);
|
||||
expect(blocked.allow).toBe(false);
|
||||
|
||||
// Simulate clock rewind and verify stale guard state does not lock out restarts.
|
||||
const afterRewind = governor.decide(9_000);
|
||||
expect(afterRewind.allow).toBe(true);
|
||||
});
|
||||
|
||||
it('wraps counters safely at MAX_SAFE_INTEGER', () => {
|
||||
const governor = new GatewayRestartGovernor();
|
||||
(governor as unknown as { executedTotal: number; suppressedTotal: number }).executedTotal = Number.MAX_SAFE_INTEGER;
|
||||
(governor as unknown as { executedTotal: number; suppressedTotal: number }).suppressedTotal = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
governor.recordExecuted(1000);
|
||||
governor.decide(1000);
|
||||
|
||||
expect(governor.getCounters()).toEqual({
|
||||
executedTotal: 0,
|
||||
suppressedTotal: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -155,4 +155,23 @@ describe('openclaw doctor output handling', () => {
|
||||
expect(result.stdout).toBe('line-1\nline-2\n');
|
||||
expect(result.stderr).toBe('warn-1\nwarn-2\n');
|
||||
});
|
||||
|
||||
it('runs plain doctor command without --json', async () => {
|
||||
const child = new MockUtilityChild();
|
||||
mockFork.mockReturnValue(child);
|
||||
|
||||
const { runOpenClawDoctor } = await import('@electron/utils/openclaw-doctor');
|
||||
const resultPromise = runOpenClawDoctor();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(mockFork).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
child.stdout.emit('data', Buffer.from('doctor ok\n'));
|
||||
child.emit('exit', 0);
|
||||
|
||||
const result = await resultPromise;
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.command).toBe('openclaw doctor');
|
||||
expect(mockFork.mock.calls[0][1]).toEqual(['doctor']);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user