Stabilize gateway reload/restart behavior and remove doctor --json dependency (#504)
This commit is contained in:
committed by
GitHub
Unverified
parent
89bda3c7af
commit
7f3408559d
@@ -25,7 +25,9 @@ function scheduleGatewayChannelRestart(ctx: HostApiContext, reason: string): voi
|
|||||||
void reason;
|
void reason;
|
||||||
}
|
}
|
||||||
|
|
||||||
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
|
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
|
||||||
|
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
|
||||||
|
const FORCE_RESTART_CHANNELS = new Set(['dingtalk', 'wecom', 'whatsapp']);
|
||||||
|
|
||||||
function scheduleGatewayChannelSaveRefresh(
|
function scheduleGatewayChannelSaveRefresh(
|
||||||
ctx: HostApiContext,
|
ctx: HostApiContext,
|
||||||
|
|||||||
@@ -43,6 +43,12 @@ import { GatewayConnectionMonitor } from './connection-monitor';
|
|||||||
import { GatewayLifecycleController, LifecycleSupersededError } from './lifecycle-controller';
|
import { GatewayLifecycleController, LifecycleSupersededError } from './lifecycle-controller';
|
||||||
import { launchGatewayProcess } from './process-launcher';
|
import { launchGatewayProcess } from './process-launcher';
|
||||||
import { GatewayRestartController } from './restart-controller';
|
import { GatewayRestartController } from './restart-controller';
|
||||||
|
import { GatewayRestartGovernor } from './restart-governor';
|
||||||
|
import {
|
||||||
|
DEFAULT_GATEWAY_RELOAD_POLICY,
|
||||||
|
loadGatewayReloadPolicy,
|
||||||
|
type GatewayReloadPolicy,
|
||||||
|
} from './reload-policy';
|
||||||
import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr';
|
import { classifyGatewayStderrMessage, recordGatewayStartupStderrLine } from './startup-stderr';
|
||||||
import { runGatewayStartupSequence } from './startup-orchestrator';
|
import { runGatewayStartupSequence } from './startup-orchestrator';
|
||||||
|
|
||||||
@@ -94,12 +100,15 @@ export class GatewayManager extends EventEmitter {
|
|||||||
private readonly connectionMonitor = new GatewayConnectionMonitor();
|
private readonly connectionMonitor = new GatewayConnectionMonitor();
|
||||||
private readonly lifecycleController = new GatewayLifecycleController();
|
private readonly lifecycleController = new GatewayLifecycleController();
|
||||||
private readonly restartController = new GatewayRestartController();
|
private readonly restartController = new GatewayRestartController();
|
||||||
|
private readonly restartGovernor = new GatewayRestartGovernor();
|
||||||
private reloadDebounceTimer: NodeJS.Timeout | null = null;
|
private reloadDebounceTimer: NodeJS.Timeout | null = null;
|
||||||
|
private reloadPolicy: GatewayReloadPolicy = { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||||
|
private reloadPolicyLoadedAt = 0;
|
||||||
|
private reloadPolicyRefreshPromise: Promise<void> | null = null;
|
||||||
private externalShutdownSupported: boolean | null = null;
|
private externalShutdownSupported: boolean | null = null;
|
||||||
private lastRestartAt = 0;
|
|
||||||
private reconnectAttemptsTotal = 0;
|
private reconnectAttemptsTotal = 0;
|
||||||
private reconnectSuccessTotal = 0;
|
private reconnectSuccessTotal = 0;
|
||||||
private static readonly RESTART_COOLDOWN_MS = 2500;
|
private static readonly RELOAD_POLICY_REFRESH_MS = 15_000;
|
||||||
|
|
||||||
constructor(config?: Partial<ReconnectConfig>) {
|
constructor(config?: Partial<ReconnectConfig>) {
|
||||||
super();
|
super();
|
||||||
@@ -109,6 +118,9 @@ export class GatewayManager extends EventEmitter {
|
|||||||
this.emit('status', status);
|
this.emit('status', status);
|
||||||
},
|
},
|
||||||
onTransition: (previousState, nextState) => {
|
onTransition: (previousState, nextState) => {
|
||||||
|
if (nextState === 'running') {
|
||||||
|
this.restartGovernor.onRunning();
|
||||||
|
}
|
||||||
this.restartController.flushDeferredRestart(
|
this.restartController.flushDeferredRestart(
|
||||||
`status:${previousState}->${nextState}`,
|
`status:${previousState}->${nextState}`,
|
||||||
{
|
{
|
||||||
@@ -186,6 +198,7 @@ export class GatewayManager extends EventEmitter {
|
|||||||
logger.info(`Gateway start requested (port=${this.status.port})`);
|
logger.info(`Gateway start requested (port=${this.status.port})`);
|
||||||
this.lastSpawnSummary = null;
|
this.lastSpawnSummary = null;
|
||||||
this.shouldReconnect = true;
|
this.shouldReconnect = true;
|
||||||
|
await this.refreshReloadPolicy(true);
|
||||||
|
|
||||||
// Lazily load device identity (async file I/O + key generation).
|
// Lazily load device identity (async file I/O + key generation).
|
||||||
// Must happen before connect() which uses the identity for the handshake.
|
// Must happen before connect() which uses the identity for the handshake.
|
||||||
@@ -353,18 +366,27 @@ export class GatewayManager extends EventEmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const now = Date.now();
|
const decision = this.restartGovernor.decide();
|
||||||
const sinceLastRestart = now - this.lastRestartAt;
|
if (!decision.allow) {
|
||||||
if (sinceLastRestart < GatewayManager.RESTART_COOLDOWN_MS) {
|
const observability = this.restartGovernor.getObservability();
|
||||||
logger.info(
|
logger.warn(
|
||||||
`Gateway restart skipped due to cooldown (${sinceLastRestart}ms < ${GatewayManager.RESTART_COOLDOWN_MS}ms)`,
|
`[gateway-restart-governor] restart suppressed reason=${decision.reason} retryAfterMs=${decision.retryAfterMs} ` +
|
||||||
|
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
|
||||||
);
|
);
|
||||||
|
const props = {
|
||||||
|
reason: decision.reason,
|
||||||
|
retry_after_ms: decision.retryAfterMs,
|
||||||
|
gateway_restart_suppressed_total: observability.suppressed_total,
|
||||||
|
gateway_restart_executed_total: observability.executed_total,
|
||||||
|
gateway_restart_circuit_open_until: observability.circuit_open_until,
|
||||||
|
};
|
||||||
|
trackMetric('gateway.restart.suppressed', props);
|
||||||
|
captureTelemetryEvent('gateway_restart_suppressed', props);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const pidBefore = this.status.pid;
|
const pidBefore = this.status.pid;
|
||||||
logger.info(`[gateway-refresh] mode=restart requested pidBefore=${pidBefore ?? 'n/a'}`);
|
logger.info(`[gateway-refresh] mode=restart requested pidBefore=${pidBefore ?? 'n/a'}`);
|
||||||
this.lastRestartAt = now;
|
|
||||||
this.restartInFlight = (async () => {
|
this.restartInFlight = (async () => {
|
||||||
await this.stop();
|
await this.stop();
|
||||||
await this.start();
|
await this.start();
|
||||||
@@ -372,8 +394,18 @@ export class GatewayManager extends EventEmitter {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await this.restartInFlight;
|
await this.restartInFlight;
|
||||||
|
this.restartGovernor.recordExecuted();
|
||||||
|
const observability = this.restartGovernor.getObservability();
|
||||||
|
const props = {
|
||||||
|
gateway_restart_executed_total: observability.executed_total,
|
||||||
|
gateway_restart_suppressed_total: observability.suppressed_total,
|
||||||
|
gateway_restart_circuit_open_until: observability.circuit_open_until,
|
||||||
|
};
|
||||||
|
trackMetric('gateway.restart.executed', props);
|
||||||
|
captureTelemetryEvent('gateway_restart_executed', props);
|
||||||
logger.info(
|
logger.info(
|
||||||
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'}`,
|
`[gateway-refresh] mode=restart result=applied pidBefore=${pidBefore ?? 'n/a'} pidAfter=${this.status.pid ?? 'n/a'} ` +
|
||||||
|
`suppressed=${observability.suppressed_total} executed=${observability.executed_total} circuitOpenUntil=${observability.circuit_open_until}`,
|
||||||
);
|
);
|
||||||
} finally {
|
} finally {
|
||||||
this.restartInFlight = null;
|
this.restartInFlight = null;
|
||||||
@@ -413,6 +445,16 @@ export class GatewayManager extends EventEmitter {
|
|||||||
* Falls back to restart on unsupported platforms or signaling failures.
|
* Falls back to restart on unsupported platforms or signaling failures.
|
||||||
*/
|
*/
|
||||||
async reload(): Promise<void> {
|
async reload(): Promise<void> {
|
||||||
|
await this.refreshReloadPolicy();
|
||||||
|
|
||||||
|
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
|
||||||
|
logger.info(
|
||||||
|
`[gateway-refresh] mode=reload result=policy_forced_restart policy=${this.reloadPolicy.mode}`,
|
||||||
|
);
|
||||||
|
await this.restart();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (this.restartController.isRestartDeferred({
|
if (this.restartController.isRestartDeferred({
|
||||||
state: this.status.state,
|
state: this.status.state,
|
||||||
startLock: this.startLock,
|
startLock: this.startLock,
|
||||||
@@ -481,17 +523,51 @@ export class GatewayManager extends EventEmitter {
|
|||||||
* Debounced reload — coalesces multiple rapid config-change events into one
|
* Debounced reload — coalesces multiple rapid config-change events into one
|
||||||
* in-process reload when possible.
|
* in-process reload when possible.
|
||||||
*/
|
*/
|
||||||
debouncedReload(delayMs = 1200): void {
|
debouncedReload(delayMs?: number): void {
|
||||||
|
void this.refreshReloadPolicy();
|
||||||
|
const effectiveDelay = delayMs ?? this.reloadPolicy.debounceMs;
|
||||||
|
if (this.reloadPolicy.mode === 'off' || this.reloadPolicy.mode === 'restart') {
|
||||||
|
logger.debug(
|
||||||
|
`Gateway reload policy=${this.reloadPolicy.mode}; routing debouncedReload to debouncedRestart (${effectiveDelay}ms)`,
|
||||||
|
);
|
||||||
|
this.debouncedRestart(effectiveDelay);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (this.reloadDebounceTimer) {
|
if (this.reloadDebounceTimer) {
|
||||||
clearTimeout(this.reloadDebounceTimer);
|
clearTimeout(this.reloadDebounceTimer);
|
||||||
}
|
}
|
||||||
logger.debug(`Gateway reload debounced (will fire in ${delayMs}ms)`);
|
logger.debug(`Gateway reload debounced (will fire in ${effectiveDelay}ms)`);
|
||||||
this.reloadDebounceTimer = setTimeout(() => {
|
this.reloadDebounceTimer = setTimeout(() => {
|
||||||
this.reloadDebounceTimer = null;
|
this.reloadDebounceTimer = null;
|
||||||
void this.reload().catch((err) => {
|
void this.reload().catch((err) => {
|
||||||
logger.warn('Debounced Gateway reload failed:', err);
|
logger.warn('Debounced Gateway reload failed:', err);
|
||||||
});
|
});
|
||||||
}, delayMs);
|
}, effectiveDelay);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async refreshReloadPolicy(force = false): Promise<void> {
|
||||||
|
const now = Date.now();
|
||||||
|
if (!force && now - this.reloadPolicyLoadedAt < GatewayManager.RELOAD_POLICY_REFRESH_MS) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.reloadPolicyRefreshPromise) {
|
||||||
|
await this.reloadPolicyRefreshPromise;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.reloadPolicyRefreshPromise = (async () => {
|
||||||
|
const nextPolicy = await loadGatewayReloadPolicy();
|
||||||
|
this.reloadPolicy = nextPolicy;
|
||||||
|
this.reloadPolicyLoadedAt = Date.now();
|
||||||
|
})();
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.reloadPolicyRefreshPromise;
|
||||||
|
} finally {
|
||||||
|
this.reloadPolicyRefreshPromise = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
63
electron/gateway/reload-policy.ts
Normal file
63
electron/gateway/reload-policy.ts
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { homedir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
export type GatewayReloadMode = 'hybrid' | 'reload' | 'restart' | 'off';
|
||||||
|
|
||||||
|
export type GatewayReloadPolicy = {
|
||||||
|
mode: GatewayReloadMode;
|
||||||
|
debounceMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_GATEWAY_RELOAD_POLICY: GatewayReloadPolicy = {
|
||||||
|
mode: 'hybrid',
|
||||||
|
debounceMs: 1200,
|
||||||
|
};
|
||||||
|
|
||||||
|
const OPENCLAW_CONFIG_PATH = join(homedir(), '.openclaw', 'openclaw.json');
|
||||||
|
const MAX_DEBOUNCE_MS = 60_000;
|
||||||
|
|
||||||
|
function normalizeMode(value: unknown): GatewayReloadMode {
|
||||||
|
if (value === 'off' || value === 'reload' || value === 'restart' || value === 'hybrid') {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
return DEFAULT_GATEWAY_RELOAD_POLICY.mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDebounceMs(value: unknown): number {
|
||||||
|
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
||||||
|
return DEFAULT_GATEWAY_RELOAD_POLICY.debounceMs;
|
||||||
|
}
|
||||||
|
const rounded = Math.round(value);
|
||||||
|
if (rounded < 0) return 0;
|
||||||
|
if (rounded > MAX_DEBOUNCE_MS) return MAX_DEBOUNCE_MS;
|
||||||
|
return rounded;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseGatewayReloadPolicy(config: unknown): GatewayReloadPolicy {
|
||||||
|
if (!config || typeof config !== 'object') {
|
||||||
|
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||||
|
}
|
||||||
|
const root = config as Record<string, unknown>;
|
||||||
|
const gateway = (root.gateway && typeof root.gateway === 'object'
|
||||||
|
? root.gateway
|
||||||
|
: {}) as Record<string, unknown>;
|
||||||
|
const reload = (gateway.reload && typeof gateway.reload === 'object'
|
||||||
|
? gateway.reload
|
||||||
|
: {}) as Record<string, unknown>;
|
||||||
|
|
||||||
|
return {
|
||||||
|
mode: normalizeMode(reload.mode),
|
||||||
|
debounceMs: normalizeDebounceMs(reload.debounceMs),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadGatewayReloadPolicy(): Promise<GatewayReloadPolicy> {
|
||||||
|
try {
|
||||||
|
const raw = await readFile(OPENCLAW_CONFIG_PATH, 'utf-8');
|
||||||
|
return parseGatewayReloadPolicy(JSON.parse(raw));
|
||||||
|
} catch {
|
||||||
|
return { ...DEFAULT_GATEWAY_RELOAD_POLICY };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
145
electron/gateway/restart-governor.ts
Normal file
145
electron/gateway/restart-governor.ts
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
export type RestartDecision =
|
||||||
|
| { allow: true }
|
||||||
|
| {
|
||||||
|
allow: false;
|
||||||
|
reason: 'circuit_open' | 'budget_exceeded' | 'cooldown_active';
|
||||||
|
retryAfterMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RestartGovernorOptions = {
|
||||||
|
maxRestartsPerWindow: number;
|
||||||
|
windowMs: number;
|
||||||
|
baseCooldownMs: number;
|
||||||
|
maxCooldownMs: number;
|
||||||
|
circuitOpenMs: number;
|
||||||
|
stableResetMs: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_OPTIONS: RestartGovernorOptions = {
|
||||||
|
maxRestartsPerWindow: 4,
|
||||||
|
windowMs: 10 * 60 * 1000,
|
||||||
|
baseCooldownMs: 2500,
|
||||||
|
maxCooldownMs: 2 * 60 * 1000,
|
||||||
|
circuitOpenMs: 10 * 60 * 1000,
|
||||||
|
stableResetMs: 2 * 60 * 1000,
|
||||||
|
};
|
||||||
|
|
||||||
|
export class GatewayRestartGovernor {
|
||||||
|
private readonly options: RestartGovernorOptions;
|
||||||
|
private restartTimestamps: number[] = [];
|
||||||
|
private circuitOpenUntil = 0;
|
||||||
|
private consecutiveRestarts = 0;
|
||||||
|
private lastRestartAt = 0;
|
||||||
|
private lastRunningAt = 0;
|
||||||
|
private suppressedTotal = 0;
|
||||||
|
private executedTotal = 0;
|
||||||
|
private static readonly MAX_COUNTER = Number.MAX_SAFE_INTEGER;
|
||||||
|
|
||||||
|
constructor(options?: Partial<RestartGovernorOptions>) {
|
||||||
|
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||||
|
}
|
||||||
|
|
||||||
|
onRunning(now = Date.now()): void {
|
||||||
|
this.lastRunningAt = now;
|
||||||
|
}
|
||||||
|
|
||||||
|
decide(now = Date.now()): RestartDecision {
|
||||||
|
this.pruneOld(now);
|
||||||
|
this.maybeResetConsecutive(now);
|
||||||
|
|
||||||
|
if (now < this.circuitOpenUntil) {
|
||||||
|
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||||
|
return {
|
||||||
|
allow: false,
|
||||||
|
reason: 'circuit_open',
|
||||||
|
retryAfterMs: this.circuitOpenUntil - now,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.restartTimestamps.length >= this.options.maxRestartsPerWindow) {
|
||||||
|
this.circuitOpenUntil = now + this.options.circuitOpenMs;
|
||||||
|
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||||
|
return {
|
||||||
|
allow: false,
|
||||||
|
reason: 'budget_exceeded',
|
||||||
|
retryAfterMs: this.options.circuitOpenMs,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const requiredCooldown = this.getCooldownMs();
|
||||||
|
if (this.lastRestartAt > 0) {
|
||||||
|
const sinceLast = now - this.lastRestartAt;
|
||||||
|
if (sinceLast < requiredCooldown) {
|
||||||
|
this.suppressedTotal = this.incrementCounter(this.suppressedTotal);
|
||||||
|
return {
|
||||||
|
allow: false,
|
||||||
|
reason: 'cooldown_active',
|
||||||
|
retryAfterMs: requiredCooldown - sinceLast,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { allow: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
recordExecuted(now = Date.now()): void {
|
||||||
|
this.executedTotal = this.incrementCounter(this.executedTotal);
|
||||||
|
this.lastRestartAt = now;
|
||||||
|
this.consecutiveRestarts += 1;
|
||||||
|
this.restartTimestamps.push(now);
|
||||||
|
this.pruneOld(now);
|
||||||
|
}
|
||||||
|
|
||||||
|
getCounters(): { executedTotal: number; suppressedTotal: number } {
|
||||||
|
return {
|
||||||
|
executedTotal: this.executedTotal,
|
||||||
|
suppressedTotal: this.suppressedTotal,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
getObservability(): {
|
||||||
|
suppressed_total: number;
|
||||||
|
executed_total: number;
|
||||||
|
circuit_open_until: number;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
suppressed_total: this.suppressedTotal,
|
||||||
|
executed_total: this.executedTotal,
|
||||||
|
circuit_open_until: this.circuitOpenUntil,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private getCooldownMs(): number {
|
||||||
|
const factor = Math.pow(2, Math.max(0, this.consecutiveRestarts));
|
||||||
|
return Math.min(this.options.baseCooldownMs * factor, this.options.maxCooldownMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
private maybeResetConsecutive(now: number): void {
|
||||||
|
if (this.lastRunningAt <= 0) return;
|
||||||
|
if (now - this.lastRunningAt >= this.options.stableResetMs) {
|
||||||
|
this.consecutiveRestarts = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private pruneOld(now: number): void {
|
||||||
|
// Detect time rewind (system clock moved backwards) and clear all
|
||||||
|
// time-based guard state to avoid stale lockouts.
|
||||||
|
if (this.restartTimestamps.length > 0 && now < this.restartTimestamps[this.restartTimestamps.length - 1]) {
|
||||||
|
this.restartTimestamps = [];
|
||||||
|
this.circuitOpenUntil = 0;
|
||||||
|
this.lastRestartAt = 0;
|
||||||
|
this.lastRunningAt = 0;
|
||||||
|
this.consecutiveRestarts = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const threshold = now - this.options.windowMs;
|
||||||
|
while (this.restartTimestamps.length > 0 && this.restartTimestamps[0] < threshold) {
|
||||||
|
this.restartTimestamps.shift();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private incrementCounter(current: number): number {
|
||||||
|
if (current >= GatewayRestartGovernor.MAX_COUNTER) return 0;
|
||||||
|
return current + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1355,7 +1355,9 @@ function registerGatewayHandlers(
|
|||||||
* For checking package status and channel configuration
|
* For checking package status and channel configuration
|
||||||
*/
|
*/
|
||||||
function registerOpenClawHandlers(gatewayManager: GatewayManager): void {
|
function registerOpenClawHandlers(gatewayManager: GatewayManager): void {
|
||||||
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'feishu', 'whatsapp']);
|
// Keep reload-first for feishu to avoid restart storms when channel auth/network is flaky.
|
||||||
|
// GatewayManager.reload() already falls back to restart when reload is unhealthy.
|
||||||
|
const forceRestartChannels = new Set(['dingtalk', 'wecom', 'whatsapp']);
|
||||||
|
|
||||||
const scheduleGatewayChannelRestart = (reason: string): void => {
|
const scheduleGatewayChannelRestart = (reason: string): void => {
|
||||||
if (gatewayManager.getStatus().state !== 'stopped') {
|
if (gatewayManager.getStatus().state !== 'stopped') {
|
||||||
|
|||||||
@@ -716,6 +716,65 @@ export interface ValidationResult {
|
|||||||
warnings: string[];
|
warnings: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const DOCTOR_PARSER_FALLBACK_HINT =
|
||||||
|
'Doctor output could not be confidently interpreted; falling back to local channel config checks.';
|
||||||
|
|
||||||
|
type DoctorValidationParseResult = {
|
||||||
|
errors: string[];
|
||||||
|
warnings: string[];
|
||||||
|
undetermined: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function parseDoctorValidationOutput(channelType: string, output: string): DoctorValidationParseResult {
|
||||||
|
const errors: string[] = [];
|
||||||
|
const warnings: string[] = [];
|
||||||
|
const normalizedChannelType = channelType.toLowerCase();
|
||||||
|
const normalizedOutput = output.trim();
|
||||||
|
|
||||||
|
if (!normalizedOutput) {
|
||||||
|
return {
|
||||||
|
errors,
|
||||||
|
warnings: [DOCTOR_PARSER_FALLBACK_HINT],
|
||||||
|
undetermined: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const lines = output
|
||||||
|
.split('\n')
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
const channelLines = lines.filter((line) => line.toLowerCase().includes(normalizedChannelType));
|
||||||
|
let classifiedCount = 0;
|
||||||
|
|
||||||
|
for (const line of channelLines) {
|
||||||
|
const lowerLine = line.toLowerCase();
|
||||||
|
if (lowerLine.includes('error') || lowerLine.includes('unrecognized key')) {
|
||||||
|
errors.push(line);
|
||||||
|
classifiedCount += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (lowerLine.includes('warning')) {
|
||||||
|
warnings.push(line);
|
||||||
|
classifiedCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (channelLines.length === 0 || classifiedCount === 0) {
|
||||||
|
warnings.push(DOCTOR_PARSER_FALLBACK_HINT);
|
||||||
|
return {
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
undetermined: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
errors,
|
||||||
|
warnings,
|
||||||
|
undetermined: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export interface CredentialValidationResult {
|
export interface CredentialValidationResult {
|
||||||
valid: boolean;
|
valid: boolean;
|
||||||
errors: string[];
|
errors: string[];
|
||||||
@@ -853,34 +912,41 @@ export async function validateChannelConfig(channelType: string): Promise<Valida
|
|||||||
|
|
||||||
// Run openclaw doctor command to validate config (async to avoid
|
// Run openclaw doctor command to validate config (async to avoid
|
||||||
// blocking the main thread).
|
// blocking the main thread).
|
||||||
const output = await new Promise<string>((resolve, reject) => {
|
const runDoctor = async (command: string): Promise<string> =>
|
||||||
exec(
|
await new Promise<string>((resolve, reject) => {
|
||||||
`node openclaw.mjs doctor --json 2>&1`,
|
exec(
|
||||||
{
|
command,
|
||||||
cwd: openclawPath,
|
{
|
||||||
encoding: 'utf-8',
|
cwd: openclawPath,
|
||||||
timeout: 30000,
|
encoding: 'utf-8',
|
||||||
windowsHide: true,
|
timeout: 30000,
|
||||||
},
|
windowsHide: true,
|
||||||
(err, stdout) => {
|
},
|
||||||
if (err) reject(err);
|
(err, stdout, stderr) => {
|
||||||
else resolve(stdout);
|
const combined = `${stdout || ''}${stderr || ''}`;
|
||||||
},
|
if (err) {
|
||||||
);
|
const next = new Error(combined || err.message);
|
||||||
});
|
reject(next);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
resolve(combined);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
const lines = output.split('\n');
|
const output = await runDoctor(`node openclaw.mjs doctor 2>&1`);
|
||||||
for (const line of lines) {
|
|
||||||
const lowerLine = line.toLowerCase();
|
const parsedDoctor = parseDoctorValidationOutput(channelType, output);
|
||||||
if (lowerLine.includes(channelType) && lowerLine.includes('error')) {
|
result.errors.push(...parsedDoctor.errors);
|
||||||
result.errors.push(line.trim());
|
result.warnings.push(...parsedDoctor.warnings);
|
||||||
result.valid = false;
|
if (parsedDoctor.errors.length > 0) {
|
||||||
} else if (lowerLine.includes(channelType) && lowerLine.includes('warning')) {
|
result.valid = false;
|
||||||
result.warnings.push(line.trim());
|
}
|
||||||
} else if (lowerLine.includes('unrecognized key') && lowerLine.includes(channelType)) {
|
if (parsedDoctor.undetermined) {
|
||||||
result.errors.push(line.trim());
|
logger.warn('Doctor output parsing fell back to local channel checks', {
|
||||||
result.valid = false;
|
channelType,
|
||||||
}
|
hint: DOCTOR_PARSER_FALLBACK_HINT,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const config = await readOpenClawConfig();
|
const config = await readOpenClawConfig();
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import { logger } from './logger';
|
|||||||
import { saveProvider, getProvider, ProviderConfig } from './secure-storage';
|
import { saveProvider, getProvider, ProviderConfig } from './secure-storage';
|
||||||
import { getProviderDefaultModel } from './provider-registry';
|
import { getProviderDefaultModel } from './provider-registry';
|
||||||
import { isOpenClawPresent } from './paths';
|
import { isOpenClawPresent } from './paths';
|
||||||
|
import { proxyAwareFetch } from './proxy-fetch';
|
||||||
import {
|
import {
|
||||||
loginMiniMaxPortalOAuth,
|
loginMiniMaxPortalOAuth,
|
||||||
type MiniMaxOAuthToken,
|
type MiniMaxOAuthToken,
|
||||||
@@ -47,6 +48,17 @@ class DeviceOAuthManager extends EventEmitter {
|
|||||||
private active: boolean = false;
|
private active: boolean = false;
|
||||||
private mainWindow: BrowserWindow | null = null;
|
private mainWindow: BrowserWindow | null = null;
|
||||||
|
|
||||||
|
private async runWithProxyAwareFetch<T>(task: () => Promise<T>): Promise<T> {
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
globalThis.fetch = ((input: string | URL, init?: RequestInit) =>
|
||||||
|
proxyAwareFetch(input, init)) as typeof fetch;
|
||||||
|
try {
|
||||||
|
return await task();
|
||||||
|
} finally {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
setWindow(window: BrowserWindow) {
|
setWindow(window: BrowserWindow) {
|
||||||
this.mainWindow = window;
|
this.mainWindow = window;
|
||||||
}
|
}
|
||||||
@@ -109,7 +121,7 @@ class DeviceOAuthManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
const provider = this.activeProvider!;
|
const provider = this.activeProvider!;
|
||||||
|
|
||||||
const token: MiniMaxOAuthToken = await loginMiniMaxPortalOAuth({
|
const token: MiniMaxOAuthToken = await this.runWithProxyAwareFetch(() => loginMiniMaxPortalOAuth({
|
||||||
region,
|
region,
|
||||||
openUrl: async (url) => {
|
openUrl: async (url) => {
|
||||||
logger.info(`[DeviceOAuth] MiniMax opening browser: ${url}`);
|
logger.info(`[DeviceOAuth] MiniMax opening browser: ${url}`);
|
||||||
@@ -133,7 +145,7 @@ class DeviceOAuthManager extends EventEmitter {
|
|||||||
update: (msg) => logger.info(`[DeviceOAuth] MiniMax progress: ${msg}`),
|
update: (msg) => logger.info(`[DeviceOAuth] MiniMax progress: ${msg}`),
|
||||||
stop: (msg) => logger.info(`[DeviceOAuth] MiniMax progress done: ${msg ?? ''}`),
|
stop: (msg) => logger.info(`[DeviceOAuth] MiniMax progress done: ${msg ?? ''}`),
|
||||||
},
|
},
|
||||||
});
|
}));
|
||||||
|
|
||||||
if (!this.active) return;
|
if (!this.active) return;
|
||||||
|
|
||||||
@@ -159,7 +171,7 @@ class DeviceOAuthManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
const provider = this.activeProvider!;
|
const provider = this.activeProvider!;
|
||||||
|
|
||||||
const token: QwenOAuthToken = await loginQwenPortalOAuth({
|
const token: QwenOAuthToken = await this.runWithProxyAwareFetch(() => loginQwenPortalOAuth({
|
||||||
openUrl: async (url) => {
|
openUrl: async (url) => {
|
||||||
logger.info(`[DeviceOAuth] Qwen opening browser: ${url}`);
|
logger.info(`[DeviceOAuth] Qwen opening browser: ${url}`);
|
||||||
shell.openExternal(url).catch((err) =>
|
shell.openExternal(url).catch((err) =>
|
||||||
@@ -179,7 +191,7 @@ class DeviceOAuthManager extends EventEmitter {
|
|||||||
update: (msg) => logger.info(`[DeviceOAuth] Qwen progress: ${msg}`),
|
update: (msg) => logger.info(`[DeviceOAuth] Qwen progress: ${msg}`),
|
||||||
stop: (msg) => logger.info(`[DeviceOAuth] Qwen progress done: ${msg ?? ''}`),
|
stop: (msg) => logger.info(`[DeviceOAuth] Qwen progress done: ${msg ?? ''}`),
|
||||||
},
|
},
|
||||||
});
|
}));
|
||||||
|
|
||||||
if (!this.active) return;
|
if (!this.active) return;
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, unlinkS
|
|||||||
import { createServer } from 'node:http';
|
import { createServer } from 'node:http';
|
||||||
import { delimiter, dirname, join } from 'node:path';
|
import { delimiter, dirname, join } from 'node:path';
|
||||||
import { getClawXConfigDir } from './paths';
|
import { getClawXConfigDir } from './paths';
|
||||||
|
import { proxyAwareFetch } from './proxy-fetch';
|
||||||
|
|
||||||
const CLIENT_ID_KEYS = ['OPENCLAW_GEMINI_OAUTH_CLIENT_ID', 'GEMINI_CLI_OAUTH_CLIENT_ID'];
|
const CLIENT_ID_KEYS = ['OPENCLAW_GEMINI_OAUTH_CLIENT_ID', 'GEMINI_CLI_OAUTH_CLIENT_ID'];
|
||||||
const CLIENT_SECRET_KEYS = [
|
const CLIENT_SECRET_KEYS = [
|
||||||
@@ -243,7 +244,7 @@ async function installViaNpm(onProgress?: (msg: string) => void): Promise<boolea
|
|||||||
async function installViaDirectDownload(onProgress?: (msg: string) => void): Promise<boolean> {
|
async function installViaDirectDownload(onProgress?: (msg: string) => void): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
onProgress?.('Downloading Gemini OAuth helper...');
|
onProgress?.('Downloading Gemini OAuth helper...');
|
||||||
const metaRes = await fetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
|
const metaRes = await proxyAwareFetch('https://registry.npmjs.org/@google/gemini-cli-core/latest');
|
||||||
if (!metaRes.ok) {
|
if (!metaRes.ok) {
|
||||||
onProgress?.(`Failed to fetch Gemini package metadata: ${metaRes.status}`);
|
onProgress?.(`Failed to fetch Gemini package metadata: ${metaRes.status}`);
|
||||||
return false;
|
return false;
|
||||||
@@ -256,7 +257,7 @@ async function installViaDirectDownload(onProgress?: (msg: string) => void): Pro
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const tarRes = await fetch(tarballUrl);
|
const tarRes = await proxyAwareFetch(tarballUrl);
|
||||||
if (!tarRes.ok) {
|
if (!tarRes.ok) {
|
||||||
onProgress?.(`Failed to download Gemini package: ${tarRes.status}`);
|
onProgress?.(`Failed to download Gemini package: ${tarRes.status}`);
|
||||||
return false;
|
return false;
|
||||||
@@ -440,7 +441,7 @@ async function waitForLocalCallback(params: {
|
|||||||
|
|
||||||
async function getUserEmail(accessToken: string): Promise<string | undefined> {
|
async function getUserEmail(accessToken: string): Promise<string | undefined> {
|
||||||
try {
|
try {
|
||||||
const response = await fetch(USERINFO_URL, {
|
const response = await proxyAwareFetch(USERINFO_URL, {
|
||||||
headers: { Authorization: `Bearer ${accessToken}` },
|
headers: { Authorization: `Bearer ${accessToken}` },
|
||||||
});
|
});
|
||||||
if (response.ok) {
|
if (response.ok) {
|
||||||
@@ -489,7 +490,7 @@ async function pollOperation(
|
|||||||
): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> {
|
): Promise<{ done?: boolean; response?: { cloudaicompanionProject?: { id?: string } } }> {
|
||||||
for (let attempt = 0; attempt < 24; attempt += 1) {
|
for (let attempt = 0; attempt < 24; attempt += 1) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||||
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
|
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal/${operationName}`, { headers });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -530,7 +531,7 @@ async function discoverProject(accessToken: string): Promise<string> {
|
|||||||
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>;
|
allowedTiers?: Array<{ id?: string; isDefault?: boolean }>;
|
||||||
} = {};
|
} = {};
|
||||||
|
|
||||||
const response = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
|
const response = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:loadCodeAssist`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers,
|
headers,
|
||||||
body: JSON.stringify(loadBody),
|
body: JSON.stringify(loadBody),
|
||||||
@@ -583,7 +584,7 @@ async function discoverProject(accessToken: string): Promise<string> {
|
|||||||
(onboardBody.metadata as Record<string, unknown>).duetProject = envProject;
|
(onboardBody.metadata as Record<string, unknown>).duetProject = envProject;
|
||||||
}
|
}
|
||||||
|
|
||||||
const onboardResponse = await fetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
|
const onboardResponse = await proxyAwareFetch(`${CODE_ASSIST_ENDPOINT}/v1internal:onboardUser`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers,
|
headers,
|
||||||
body: JSON.stringify(onboardBody),
|
body: JSON.stringify(onboardBody),
|
||||||
@@ -638,7 +639,7 @@ async function exchangeCodeForTokens(
|
|||||||
body.set('client_secret', clientSecret);
|
body.set('client_secret', clientSecret);
|
||||||
}
|
}
|
||||||
|
|
||||||
const response = await fetch(TOKEN_URL, {
|
const response = await proxyAwareFetch(TOKEN_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
body: body.toString(),
|
body: body.toString(),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { createHash, randomBytes } from 'node:crypto';
|
import { createHash, randomBytes } from 'node:crypto';
|
||||||
import { createServer } from 'node:http';
|
import { createServer } from 'node:http';
|
||||||
|
import { proxyAwareFetch } from './proxy-fetch';
|
||||||
|
|
||||||
const CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann';
|
const CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann';
|
||||||
const AUTHORIZE_URL = 'https://auth.openai.com/oauth/authorize';
|
const AUTHORIZE_URL = 'https://auth.openai.com/oauth/authorize';
|
||||||
@@ -206,7 +207,7 @@ async function exchangeAuthorizationCode(
|
|||||||
code: string,
|
code: string,
|
||||||
verifier: string,
|
verifier: string,
|
||||||
): Promise<{ access: string; refresh: string; expires: number }> {
|
): Promise<{ access: string; refresh: string; expires: number }> {
|
||||||
const response = await fetch(TOKEN_URL, {
|
const response = await proxyAwareFetch(TOKEN_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
body: new URLSearchParams({
|
body: new URLSearchParams({
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { getUvMirrorEnv } from './uv-env';
|
|||||||
|
|
||||||
const OPENCLAW_DOCTOR_TIMEOUT_MS = 60_000;
|
const OPENCLAW_DOCTOR_TIMEOUT_MS = 60_000;
|
||||||
const MAX_DOCTOR_OUTPUT_BYTES = 10 * 1024 * 1024;
|
const MAX_DOCTOR_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||||
const OPENCLAW_DOCTOR_ARGS = ['doctor', '--json'];
|
const OPENCLAW_DOCTOR_ARGS = ['doctor'];
|
||||||
const OPENCLAW_DOCTOR_FIX_ARGS = ['doctor', '--fix', '--yes', '--non-interactive'];
|
const OPENCLAW_DOCTOR_FIX_ARGS = ['doctor', '--fix', '--yes', '--non-interactive'];
|
||||||
|
|
||||||
export type OpenClawDoctorMode = 'diagnose' | 'fix';
|
export type OpenClawDoctorMode = 'diagnose' | 'fix';
|
||||||
@@ -65,10 +65,12 @@ function getBundledBinPath(): string {
|
|||||||
: path.join(process.cwd(), 'resources', 'bin', target);
|
: path.join(process.cwd(), 'resources', 'bin', target);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDoctorResult> {
|
async function runDoctorCommandWithArgs(
|
||||||
|
mode: OpenClawDoctorMode,
|
||||||
|
args: string[],
|
||||||
|
): Promise<OpenClawDoctorResult> {
|
||||||
const openclawDir = getOpenClawDir();
|
const openclawDir = getOpenClawDir();
|
||||||
const entryScript = getOpenClawEntryPath();
|
const entryScript = getOpenClawEntryPath();
|
||||||
const args = mode === 'fix' ? OPENCLAW_DOCTOR_FIX_ARGS : OPENCLAW_DOCTOR_ARGS;
|
|
||||||
const command = `openclaw ${args.join(' ')}`;
|
const command = `openclaw ${args.join(' ')}`;
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
|
|
||||||
@@ -194,9 +196,9 @@ async function runDoctorCommand(mode: OpenClawDoctorMode): Promise<OpenClawDocto
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function runOpenClawDoctor(): Promise<OpenClawDoctorResult> {
|
export async function runOpenClawDoctor(): Promise<OpenClawDoctorResult> {
|
||||||
return await runDoctorCommand('diagnose');
|
return await runDoctorCommandWithArgs('diagnose', OPENCLAW_DOCTOR_ARGS);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runOpenClawDoctorFix(): Promise<OpenClawDoctorResult> {
|
export async function runOpenClawDoctorFix(): Promise<OpenClawDoctorResult> {
|
||||||
return await runDoctorCommand('fix');
|
return await runDoctorCommandWithArgs('fix', OPENCLAW_DOCTOR_FIX_ARGS);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -220,7 +220,7 @@
|
|||||||
"cmdUnavailable": "Command unavailable",
|
"cmdUnavailable": "Command unavailable",
|
||||||
"cmdCopied": "CLI command copied",
|
"cmdCopied": "CLI command copied",
|
||||||
"doctor": "OpenClaw Doctor",
|
"doctor": "OpenClaw Doctor",
|
||||||
"doctorDesc": "Run `openclaw doctor --json` and inspect the raw diagnostic output.",
|
"doctorDesc": "Run `openclaw doctor` and inspect the raw diagnostic output.",
|
||||||
"runDoctor": "Run Doctor",
|
"runDoctor": "Run Doctor",
|
||||||
"runDoctorFix": "Run Doctor Fix",
|
"runDoctorFix": "Run Doctor Fix",
|
||||||
"doctorSucceeded": "OpenClaw doctor completed",
|
"doctorSucceeded": "OpenClaw doctor completed",
|
||||||
|
|||||||
@@ -217,7 +217,7 @@
|
|||||||
"cmdUnavailable": "コマンドが利用できません",
|
"cmdUnavailable": "コマンドが利用できません",
|
||||||
"cmdCopied": "CLI コマンドをコピーしました",
|
"cmdCopied": "CLI コマンドをコピーしました",
|
||||||
"doctor": "OpenClaw Doctor",
|
"doctor": "OpenClaw Doctor",
|
||||||
"doctorDesc": "`openclaw doctor --json` を実行して診断の生出力を確認します。",
|
"doctorDesc": "`openclaw doctor` を実行して診断の生出力を確認します。",
|
||||||
"runDoctor": "Doctor を実行",
|
"runDoctor": "Doctor を実行",
|
||||||
"runDoctorFix": "Doctor 修復を実行",
|
"runDoctorFix": "Doctor 修復を実行",
|
||||||
"doctorSucceeded": "OpenClaw doctor が完了しました",
|
"doctorSucceeded": "OpenClaw doctor が完了しました",
|
||||||
|
|||||||
@@ -220,7 +220,7 @@
|
|||||||
"cmdUnavailable": "命令不可用",
|
"cmdUnavailable": "命令不可用",
|
||||||
"cmdCopied": "CLI 命令已复制",
|
"cmdCopied": "CLI 命令已复制",
|
||||||
"doctor": "OpenClaw Doctor 诊断",
|
"doctor": "OpenClaw Doctor 诊断",
|
||||||
"doctorDesc": "运行 `openclaw doctor --json` 并查看原始诊断输出。",
|
"doctorDesc": "运行 `openclaw doctor` 并查看原始诊断输出。",
|
||||||
"runDoctor": "运行 Doctor",
|
"runDoctor": "运行 Doctor",
|
||||||
"runDoctorFix": "运行 Doctor 并修复",
|
"runDoctorFix": "运行 Doctor 并修复",
|
||||||
"doctorSucceeded": "OpenClaw doctor 已完成",
|
"doctorSucceeded": "OpenClaw doctor 已完成",
|
||||||
|
|||||||
@@ -169,7 +169,7 @@ export function Settings() {
|
|||||||
exitCode: null,
|
exitCode: null,
|
||||||
stdout: '',
|
stdout: '',
|
||||||
stderr: '',
|
stderr: '',
|
||||||
command: 'openclaw doctor --json',
|
command: 'openclaw doctor',
|
||||||
cwd: '',
|
cwd: '',
|
||||||
durationMs: 0,
|
durationMs: 0,
|
||||||
error: message,
|
error: message,
|
||||||
|
|||||||
@@ -102,3 +102,38 @@ describe('channel credential normalization and duplicate checks', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('parseDoctorValidationOutput', () => {
|
||||||
|
it('extracts channel error and warning lines', async () => {
|
||||||
|
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||||
|
|
||||||
|
const out = parseDoctorValidationOutput(
|
||||||
|
'feishu',
|
||||||
|
'feishu error: token invalid\nfeishu warning: fallback enabled\n',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(out.undetermined).toBe(false);
|
||||||
|
expect(out.errors).toEqual(['feishu error: token invalid']);
|
||||||
|
expect(out.warnings).toEqual(['feishu warning: fallback enabled']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back with hint when output has no channel signal', async () => {
|
||||||
|
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||||
|
|
||||||
|
const out = parseDoctorValidationOutput('feishu', 'all good, no channel details');
|
||||||
|
|
||||||
|
expect(out.undetermined).toBe(true);
|
||||||
|
expect(out.errors).toEqual([]);
|
||||||
|
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back with hint when output is empty', async () => {
|
||||||
|
const { parseDoctorValidationOutput } = await import('@electron/utils/channel-config');
|
||||||
|
|
||||||
|
const out = parseDoctorValidationOutput('feishu', ' ');
|
||||||
|
|
||||||
|
expect(out.undetermined).toBe(true);
|
||||||
|
expect(out.errors).toEqual([]);
|
||||||
|
expect(out.warnings.some((w) => w.includes('falling back to local channel config checks'))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
101
tests/unit/gateway-manager-reload-policy-refresh.test.ts
Normal file
101
tests/unit/gateway-manager-reload-policy-refresh.test.ts
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const { mockLoadGatewayReloadPolicy } = vi.hoisted(() => ({
|
||||||
|
mockLoadGatewayReloadPolicy: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('electron', () => ({
|
||||||
|
app: {
|
||||||
|
getPath: () => '/tmp',
|
||||||
|
isPackaged: false,
|
||||||
|
},
|
||||||
|
utilityProcess: {
|
||||||
|
fork: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('@electron/gateway/reload-policy', async () => {
|
||||||
|
const actual = await vi.importActual<typeof import('@electron/gateway/reload-policy')>(
|
||||||
|
'@electron/gateway/reload-policy',
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...actual,
|
||||||
|
loadGatewayReloadPolicy: (...args: unknown[]) => mockLoadGatewayReloadPolicy(...args),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('GatewayManager refreshReloadPolicy', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetAllMocks();
|
||||||
|
vi.useFakeTimers();
|
||||||
|
vi.setSystemTime(new Date('2026-03-15T00:00:00.000Z'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('deduplicates concurrent refresh calls', async () => {
|
||||||
|
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||||
|
let resolveLoad: ((value: { mode: 'reload'; debounceMs: number }) => void) | null = null;
|
||||||
|
mockLoadGatewayReloadPolicy.mockImplementationOnce(
|
||||||
|
() =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
resolveLoad = resolve;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const manager = new GatewayManager();
|
||||||
|
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||||
|
.refreshReloadPolicy.bind(manager);
|
||||||
|
|
||||||
|
const p1 = refresh(true);
|
||||||
|
const p2 = refresh(true);
|
||||||
|
|
||||||
|
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
resolveLoad?.({ mode: 'reload', debounceMs: 1300 });
|
||||||
|
await Promise.all([p1, p2]);
|
||||||
|
|
||||||
|
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
|
||||||
|
mode: 'reload',
|
||||||
|
debounceMs: 1300,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('hits TTL cache and skips refresh within window', async () => {
|
||||||
|
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||||
|
mockLoadGatewayReloadPolicy.mockResolvedValueOnce({ mode: 'restart', debounceMs: 2200 });
|
||||||
|
|
||||||
|
const manager = new GatewayManager();
|
||||||
|
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||||
|
.refreshReloadPolicy.bind(manager);
|
||||||
|
|
||||||
|
await refresh();
|
||||||
|
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
vi.setSystemTime(new Date('2026-03-15T00:00:10.000Z'));
|
||||||
|
await refresh();
|
||||||
|
|
||||||
|
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('refreshes immediately when force=true even within TTL', async () => {
|
||||||
|
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||||
|
mockLoadGatewayReloadPolicy
|
||||||
|
.mockResolvedValueOnce({ mode: 'hybrid', debounceMs: 1200 })
|
||||||
|
.mockResolvedValueOnce({ mode: 'off', debounceMs: 9000 });
|
||||||
|
|
||||||
|
const manager = new GatewayManager();
|
||||||
|
const refresh = (manager as unknown as { refreshReloadPolicy: (force?: boolean) => Promise<void> })
|
||||||
|
.refreshReloadPolicy.bind(manager);
|
||||||
|
|
||||||
|
await refresh();
|
||||||
|
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
vi.setSystemTime(new Date('2026-03-15T00:00:05.000Z'));
|
||||||
|
await refresh(true);
|
||||||
|
|
||||||
|
expect(mockLoadGatewayReloadPolicy).toHaveBeenCalledTimes(2);
|
||||||
|
expect((manager as unknown as { reloadPolicy: { mode: string; debounceMs: number } }).reloadPolicy).toEqual({
|
||||||
|
mode: 'off',
|
||||||
|
debounceMs: 9000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
150
tests/unit/gateway-reload-policy.test.ts
Normal file
150
tests/unit/gateway-reload-policy.test.ts
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const { mockReadFile } = vi.hoisted(() => ({
|
||||||
|
mockReadFile: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('node:fs/promises', () => ({
|
||||||
|
readFile: mockReadFile,
|
||||||
|
default: {
|
||||||
|
readFile: mockReadFile,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
import {
|
||||||
|
DEFAULT_GATEWAY_RELOAD_POLICY,
|
||||||
|
loadGatewayReloadPolicy,
|
||||||
|
parseGatewayReloadPolicy,
|
||||||
|
} from '@electron/gateway/reload-policy';
|
||||||
|
|
||||||
|
describe('parseGatewayReloadPolicy', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns defaults when config is missing', () => {
|
||||||
|
expect(parseGatewayReloadPolicy(undefined)).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses mode and debounce from gateway.reload', () => {
|
||||||
|
const result = parseGatewayReloadPolicy({
|
||||||
|
gateway: {
|
||||||
|
reload: {
|
||||||
|
mode: 'off',
|
||||||
|
debounceMs: 3000,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({ mode: 'off', debounceMs: 3000 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('normalizes invalid mode and debounce bounds', () => {
|
||||||
|
const negative = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'invalid', debounceMs: -100 } },
|
||||||
|
});
|
||||||
|
expect(negative).toEqual({
|
||||||
|
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
|
||||||
|
debounceMs: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const overMax = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'hybrid', debounceMs: 600_000 } },
|
||||||
|
});
|
||||||
|
expect(overMax).toEqual({ mode: 'hybrid', debounceMs: 60_000 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('falls back to default mode for non-string or unknown mode values', () => {
|
||||||
|
const unknownString = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'HYBRID', debounceMs: 1200 } },
|
||||||
|
});
|
||||||
|
expect(unknownString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
|
||||||
|
|
||||||
|
const nonString = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: { value: 'reload' }, debounceMs: 1200 } },
|
||||||
|
});
|
||||||
|
expect(nonString.mode).toBe(DEFAULT_GATEWAY_RELOAD_POLICY.mode);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('handles malformed gateway/reload shapes', () => {
|
||||||
|
const malformedGateway = parseGatewayReloadPolicy({ gateway: 'bad-shape' });
|
||||||
|
expect(malformedGateway).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
|
||||||
|
const malformedReload = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: ['bad-shape'] },
|
||||||
|
});
|
||||||
|
expect(malformedReload).toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('normalizes debounce boundary and rounding behavior', () => {
|
||||||
|
const atMin = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'reload', debounceMs: 0 } },
|
||||||
|
});
|
||||||
|
expect(atMin).toEqual({ mode: 'reload', debounceMs: 0 });
|
||||||
|
|
||||||
|
const roundsUpToCap = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'reload', debounceMs: 60_000.5 } },
|
||||||
|
});
|
||||||
|
expect(roundsUpToCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
|
||||||
|
|
||||||
|
const roundsDownAtCap = parseGatewayReloadPolicy({
|
||||||
|
gateway: { reload: { mode: 'reload', debounceMs: 60_000.4 } },
|
||||||
|
});
|
||||||
|
expect(roundsDownAtCap).toEqual({ mode: 'reload', debounceMs: 60_000 });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('loadGatewayReloadPolicy', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns defaults when config read fails', async () => {
|
||||||
|
mockReadFile.mockRejectedValueOnce(new Error('EACCES'));
|
||||||
|
|
||||||
|
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
expect(mockReadFile).toHaveBeenCalledOnce();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns defaults when config JSON is malformed', async () => {
|
||||||
|
mockReadFile.mockResolvedValueOnce('{');
|
||||||
|
|
||||||
|
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns defaults when config JSON has malformed shape', async () => {
|
||||||
|
mockReadFile.mockResolvedValueOnce(
|
||||||
|
JSON.stringify({
|
||||||
|
gateway: { reload: ['malformed'] },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(loadGatewayReloadPolicy()).resolves.toEqual(DEFAULT_GATEWAY_RELOAD_POLICY);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads config and applies invalid mode fallback', async () => {
|
||||||
|
mockReadFile.mockResolvedValueOnce(
|
||||||
|
JSON.stringify({
|
||||||
|
gateway: { reload: { mode: 'unknown-mode', debounceMs: 1350 } },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
|
||||||
|
mode: DEFAULT_GATEWAY_RELOAD_POLICY.mode,
|
||||||
|
debounceMs: 1350,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('loads config and keeps debounce boundary values', async () => {
|
||||||
|
mockReadFile.mockResolvedValueOnce(
|
||||||
|
JSON.stringify({
|
||||||
|
gateway: { reload: { mode: 'restart', debounceMs: 60_000 } },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(loadGatewayReloadPolicy()).resolves.toEqual({
|
||||||
|
mode: 'restart',
|
||||||
|
debounceMs: 60_000,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
106
tests/unit/gateway-restart-governor.test.ts
Normal file
106
tests/unit/gateway-restart-governor.test.ts
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { GatewayRestartGovernor } from '@electron/gateway/restart-governor';
|
||||||
|
|
||||||
|
describe('GatewayRestartGovernor', () => {
|
||||||
|
it('suppresses restart during exponential cooldown window', () => {
|
||||||
|
const governor = new GatewayRestartGovernor({
|
||||||
|
baseCooldownMs: 1000,
|
||||||
|
maxCooldownMs: 8000,
|
||||||
|
maxRestartsPerWindow: 10,
|
||||||
|
windowMs: 60000,
|
||||||
|
stableResetMs: 60000,
|
||||||
|
circuitOpenMs: 60000,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(governor.decide(1000).allow).toBe(true);
|
||||||
|
governor.recordExecuted(1000);
|
||||||
|
|
||||||
|
const blocked = governor.decide(1500);
|
||||||
|
expect(blocked.allow).toBe(false);
|
||||||
|
expect(blocked.allow ? '' : blocked.reason).toBe('cooldown_active');
|
||||||
|
expect(blocked.allow ? 0 : blocked.retryAfterMs).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
expect(governor.decide(3000).allow).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('opens circuit after restart budget is exceeded', () => {
|
||||||
|
const governor = new GatewayRestartGovernor({
|
||||||
|
maxRestartsPerWindow: 2,
|
||||||
|
windowMs: 60000,
|
||||||
|
baseCooldownMs: 0,
|
||||||
|
maxCooldownMs: 0,
|
||||||
|
stableResetMs: 120000,
|
||||||
|
circuitOpenMs: 30000,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(governor.decide(1000).allow).toBe(true);
|
||||||
|
governor.recordExecuted(1000);
|
||||||
|
expect(governor.decide(2000).allow).toBe(true);
|
||||||
|
governor.recordExecuted(2000);
|
||||||
|
|
||||||
|
const budgetBlocked = governor.decide(3000);
|
||||||
|
expect(budgetBlocked.allow).toBe(false);
|
||||||
|
expect(budgetBlocked.allow ? '' : budgetBlocked.reason).toBe('budget_exceeded');
|
||||||
|
|
||||||
|
const circuitBlocked = governor.decide(4000);
|
||||||
|
expect(circuitBlocked.allow).toBe(false);
|
||||||
|
expect(circuitBlocked.allow ? '' : circuitBlocked.reason).toBe('circuit_open');
|
||||||
|
|
||||||
|
expect(governor.decide(62001).allow).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resets consecutive backoff after stable running period', () => {
|
||||||
|
const governor = new GatewayRestartGovernor({
|
||||||
|
baseCooldownMs: 1000,
|
||||||
|
maxCooldownMs: 8000,
|
||||||
|
maxRestartsPerWindow: 10,
|
||||||
|
windowMs: 600000,
|
||||||
|
stableResetMs: 5000,
|
||||||
|
circuitOpenMs: 60000,
|
||||||
|
});
|
||||||
|
|
||||||
|
governor.recordExecuted(0);
|
||||||
|
governor.recordExecuted(1000);
|
||||||
|
const blockedBeforeStable = governor.decide(2500);
|
||||||
|
expect(blockedBeforeStable.allow).toBe(false);
|
||||||
|
expect(blockedBeforeStable.allow ? '' : blockedBeforeStable.reason).toBe('cooldown_active');
|
||||||
|
|
||||||
|
governor.onRunning(3000);
|
||||||
|
const allowedAfterStable = governor.decide(9000);
|
||||||
|
expect(allowedAfterStable.allow).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resets time-based state when clock moves backwards', () => {
|
||||||
|
const governor = new GatewayRestartGovernor({
|
||||||
|
maxRestartsPerWindow: 2,
|
||||||
|
windowMs: 60000,
|
||||||
|
baseCooldownMs: 1000,
|
||||||
|
maxCooldownMs: 8000,
|
||||||
|
stableResetMs: 60000,
|
||||||
|
circuitOpenMs: 30000,
|
||||||
|
});
|
||||||
|
|
||||||
|
governor.recordExecuted(10_000);
|
||||||
|
governor.recordExecuted(11_000);
|
||||||
|
const blocked = governor.decide(11_500);
|
||||||
|
expect(blocked.allow).toBe(false);
|
||||||
|
|
||||||
|
// Simulate clock rewind and verify stale guard state does not lock out restarts.
|
||||||
|
const afterRewind = governor.decide(9_000);
|
||||||
|
expect(afterRewind.allow).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('wraps counters safely at MAX_SAFE_INTEGER', () => {
|
||||||
|
const governor = new GatewayRestartGovernor();
|
||||||
|
(governor as unknown as { executedTotal: number; suppressedTotal: number }).executedTotal = Number.MAX_SAFE_INTEGER;
|
||||||
|
(governor as unknown as { executedTotal: number; suppressedTotal: number }).suppressedTotal = Number.MAX_SAFE_INTEGER;
|
||||||
|
|
||||||
|
governor.recordExecuted(1000);
|
||||||
|
governor.decide(1000);
|
||||||
|
|
||||||
|
expect(governor.getCounters()).toEqual({
|
||||||
|
executedTotal: 0,
|
||||||
|
suppressedTotal: 0,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -155,4 +155,23 @@ describe('openclaw doctor output handling', () => {
|
|||||||
expect(result.stdout).toBe('line-1\nline-2\n');
|
expect(result.stdout).toBe('line-1\nline-2\n');
|
||||||
expect(result.stderr).toBe('warn-1\nwarn-2\n');
|
expect(result.stderr).toBe('warn-1\nwarn-2\n');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('runs plain doctor command without --json', async () => {
|
||||||
|
const child = new MockUtilityChild();
|
||||||
|
mockFork.mockReturnValue(child);
|
||||||
|
|
||||||
|
const { runOpenClawDoctor } = await import('@electron/utils/openclaw-doctor');
|
||||||
|
const resultPromise = runOpenClawDoctor();
|
||||||
|
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(mockFork).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
child.stdout.emit('data', Buffer.from('doctor ok\n'));
|
||||||
|
child.emit('exit', 0);
|
||||||
|
|
||||||
|
const result = await resultPromise;
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.command).toBe('openclaw doctor');
|
||||||
|
expect(mockFork.mock.calls[0][1]).toEqual(['doctor']);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user