Add channel health diagnostics and gateway recovery fixes (#855)
This commit is contained in:
committed by
GitHub
Unverified
parent
6acd8acf5a
commit
1f39d1a8a7
@@ -33,7 +33,9 @@ import {
|
||||
import {
|
||||
computeChannelRuntimeStatus,
|
||||
pickChannelRuntimeStatus,
|
||||
type ChannelConnectionStatus,
|
||||
type ChannelRuntimeAccountSnapshot,
|
||||
type GatewayHealthState,
|
||||
} from '../../utils/channel-status';
|
||||
import {
|
||||
OPENCLAW_WECHAT_CHANNEL_TYPE,
|
||||
@@ -65,6 +67,8 @@ import {
|
||||
normalizeWhatsAppMessagingTarget,
|
||||
} from '../../utils/openclaw-sdk';
|
||||
import { logger } from '../../utils/logger';
|
||||
import { buildGatewayHealthSummary } from '../../utils/gateway-health';
|
||||
import type { GatewayHealthSummary } from '../../gateway/manager';
|
||||
|
||||
// listWhatsAppDirectory*FromConfig were removed from openclaw's public exports
|
||||
// in 2026.3.23-1. No-op stubs; WhatsApp target picker uses session discovery.
|
||||
@@ -405,7 +409,8 @@ interface ChannelAccountView {
|
||||
running: boolean;
|
||||
linked: boolean;
|
||||
lastError?: string;
|
||||
status: 'connected' | 'connecting' | 'disconnected' | 'error';
|
||||
status: ChannelConnectionStatus;
|
||||
statusReason?: string;
|
||||
isDefault: boolean;
|
||||
agentId?: string;
|
||||
}
|
||||
@@ -413,10 +418,34 @@ interface ChannelAccountView {
|
||||
interface ChannelAccountsView {
|
||||
channelType: string;
|
||||
defaultAccountId: string;
|
||||
status: 'connected' | 'connecting' | 'disconnected' | 'error';
|
||||
status: ChannelConnectionStatus;
|
||||
statusReason?: string;
|
||||
accounts: ChannelAccountView[];
|
||||
}
|
||||
|
||||
export function getChannelStatusDiagnostics(): {
|
||||
lastChannelsStatusOkAt?: number;
|
||||
lastChannelsStatusFailureAt?: number;
|
||||
} {
|
||||
return {
|
||||
lastChannelsStatusOkAt,
|
||||
lastChannelsStatusFailureAt,
|
||||
};
|
||||
}
|
||||
|
||||
function gatewayHealthStateForChannels(
|
||||
gatewayHealthState: GatewayHealthState,
|
||||
): GatewayHealthState | undefined {
|
||||
return gatewayHealthState === 'healthy' ? undefined : gatewayHealthState;
|
||||
}
|
||||
|
||||
function overlayStatusReason(
|
||||
gatewayHealth: GatewayHealthSummary,
|
||||
fallbackReason: string,
|
||||
): string {
|
||||
return gatewayHealth.reasons[0] || fallbackReason;
|
||||
}
|
||||
|
||||
function buildGatewayStatusSnapshot(status: GatewayChannelStatusPayload | null): string {
|
||||
if (!status?.channelAccounts) return 'none';
|
||||
const entries = Object.entries(status.channelAccounts);
|
||||
@@ -480,11 +509,13 @@ type DirectoryEntry = {
|
||||
const CHANNEL_TARGET_CACHE_TTL_MS = 60_000;
|
||||
const CHANNEL_TARGET_CACHE_ENABLED = process.env.VITEST !== 'true';
|
||||
const channelTargetCache = new Map<string, { expiresAt: number; targets: ChannelTargetOptionView[] }>();
|
||||
let lastChannelsStatusOkAt: number | undefined;
|
||||
let lastChannelsStatusFailureAt: number | undefined;
|
||||
|
||||
async function buildChannelAccountsView(
|
||||
export async function buildChannelAccountsView(
|
||||
ctx: HostApiContext,
|
||||
options?: { probe?: boolean },
|
||||
): Promise<ChannelAccountsView[]> {
|
||||
): Promise<{ channels: ChannelAccountsView[]; gatewayHealth: GatewayHealthSummary }> {
|
||||
const startedAt = Date.now();
|
||||
// Read config once and share across all sub-calls (was 5 readFile calls before).
|
||||
const openClawConfig = await readOpenClawConfig();
|
||||
@@ -507,17 +538,32 @@ async function buildChannelAccountsView(
|
||||
{ probe },
|
||||
probe ? 5000 : 8000,
|
||||
);
|
||||
lastChannelsStatusOkAt = Date.now();
|
||||
logger.info(
|
||||
`[channels.accounts] channels.status probe=${probe ? '1' : '0'} elapsedMs=${Date.now() - rpcStartedAt} snapshot=${buildGatewayStatusSnapshot(gatewayStatus)}`
|
||||
);
|
||||
} catch {
|
||||
const probe = options?.probe === true;
|
||||
lastChannelsStatusFailureAt = Date.now();
|
||||
logger.warn(
|
||||
`[channels.accounts] channels.status probe=${probe ? '1' : '0'} failed after ${Date.now() - startedAt}ms`
|
||||
);
|
||||
gatewayStatus = null;
|
||||
}
|
||||
|
||||
const gatewayDiagnostics = ctx.gatewayManager.getDiagnostics?.() ?? {
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
consecutiveRpcFailures: 0,
|
||||
};
|
||||
const gatewayHealth = buildGatewayHealthSummary({
|
||||
status: ctx.gatewayManager.getStatus(),
|
||||
diagnostics: gatewayDiagnostics,
|
||||
lastChannelsStatusOkAt,
|
||||
lastChannelsStatusFailureAt,
|
||||
platform: process.platform,
|
||||
});
|
||||
const gatewayHealthState = gatewayHealthStateForChannels(gatewayHealth.state);
|
||||
|
||||
const channelTypes = new Set<string>([
|
||||
...configuredChannels,
|
||||
...Object.keys(configuredAccounts),
|
||||
@@ -566,7 +612,9 @@ async function buildChannelAccountsView(
|
||||
const accounts: ChannelAccountView[] = accountIds.map((accountId) => {
|
||||
const runtime = runtimeAccounts.find((item) => item.accountId === accountId);
|
||||
const runtimeSnapshot: ChannelRuntimeAccountSnapshot = runtime ?? {};
|
||||
const status = computeChannelRuntimeStatus(runtimeSnapshot);
|
||||
const status = computeChannelRuntimeStatus(runtimeSnapshot, {
|
||||
gatewayHealthState,
|
||||
});
|
||||
return {
|
||||
accountId,
|
||||
name: runtime?.name || accountId,
|
||||
@@ -576,6 +624,11 @@ async function buildChannelAccountsView(
|
||||
linked: runtime?.linked === true,
|
||||
lastError: typeof runtime?.lastError === 'string' ? runtime.lastError : undefined,
|
||||
status,
|
||||
statusReason: status === 'degraded'
|
||||
? overlayStatusReason(gatewayHealth, 'gateway_degraded')
|
||||
: status === 'error'
|
||||
? 'runtime_error'
|
||||
: undefined,
|
||||
isDefault: accountId === defaultAccountId,
|
||||
agentId: agentsSnapshot.channelAccountOwners[`${rawChannelType}:${accountId}`],
|
||||
};
|
||||
@@ -585,10 +638,32 @@ async function buildChannelAccountsView(
|
||||
return left.accountId.localeCompare(right.accountId);
|
||||
});
|
||||
|
||||
const visibleAccountSnapshots: ChannelRuntimeAccountSnapshot[] = accounts.map((account) => ({
|
||||
connected: account.connected,
|
||||
running: account.running,
|
||||
linked: account.linked,
|
||||
lastError: account.lastError,
|
||||
}));
|
||||
const hasRuntimeError = visibleAccountSnapshots.some((account) => typeof account.lastError === 'string' && account.lastError.trim())
|
||||
|| Boolean(channelSummary?.error?.trim() || channelSummary?.lastError?.trim());
|
||||
const baseGroupStatus = pickChannelRuntimeStatus(visibleAccountSnapshots, channelSummary);
|
||||
const groupStatus = !gatewayStatus && ctx.gatewayManager.getStatus().state === 'running'
|
||||
? 'degraded'
|
||||
: gatewayHealthState && !hasRuntimeError && baseGroupStatus === 'connected'
|
||||
? 'degraded'
|
||||
: pickChannelRuntimeStatus(visibleAccountSnapshots, channelSummary, {
|
||||
gatewayHealthState,
|
||||
});
|
||||
|
||||
channels.push({
|
||||
channelType: uiChannelType,
|
||||
defaultAccountId,
|
||||
status: pickChannelRuntimeStatus(runtimeAccounts, channelSummary),
|
||||
status: groupStatus,
|
||||
statusReason: !gatewayStatus && ctx.gatewayManager.getStatus().state === 'running'
|
||||
? 'channels_status_timeout'
|
||||
: groupStatus === 'degraded'
|
||||
? overlayStatusReason(gatewayHealth, 'gateway_degraded')
|
||||
: undefined,
|
||||
accounts,
|
||||
});
|
||||
}
|
||||
@@ -597,7 +672,7 @@ async function buildChannelAccountsView(
|
||||
logger.info(
|
||||
`[channels.accounts] response probe=${options?.probe === true ? '1' : '0'} elapsedMs=${Date.now() - startedAt} view=${sorted.map((item) => `${item.channelType}:${item.status}`).join(',')}`
|
||||
);
|
||||
return sorted;
|
||||
return { channels: sorted, gatewayHealth };
|
||||
}
|
||||
|
||||
function buildChannelTargetLabel(baseLabel: string, value: string): string {
|
||||
@@ -1193,8 +1268,8 @@ export async function handleChannelRoutes(
|
||||
try {
|
||||
const probe = url.searchParams.get('probe') === '1';
|
||||
logger.info(`[channels.accounts] request probe=${probe ? '1' : '0'}`);
|
||||
const channels = await buildChannelAccountsView(ctx, { probe });
|
||||
sendJson(res, 200, { success: true, channels });
|
||||
const { channels, gatewayHealth } = await buildChannelAccountsView(ctx, { probe });
|
||||
sendJson(res, 200, { success: true, channels, gatewayHealth });
|
||||
} catch (error) {
|
||||
sendJson(res, 500, { success: false, error: String(error) });
|
||||
}
|
||||
|
||||
86
electron/api/routes/diagnostics.ts
Normal file
86
electron/api/routes/diagnostics.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import { open } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import type { IncomingMessage, ServerResponse } from 'http';
|
||||
import { logger } from '../../utils/logger';
|
||||
import { getOpenClawConfigDir } from '../../utils/paths';
|
||||
import { buildGatewayHealthSummary } from '../../utils/gateway-health';
|
||||
import type { HostApiContext } from '../context';
|
||||
import { sendJson } from '../route-utils';
|
||||
import { buildChannelAccountsView, getChannelStatusDiagnostics } from './channels';
|
||||
|
||||
const DEFAULT_TAIL_LINES = 200;
|
||||
|
||||
async function readTail(filePath: string, tailLines = DEFAULT_TAIL_LINES): Promise<string> {
|
||||
const safeTailLines = Math.max(1, Math.floor(tailLines));
|
||||
try {
|
||||
const file = await open(filePath, 'r');
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
if (stat.size === 0) return '';
|
||||
|
||||
const chunkSize = 64 * 1024;
|
||||
let position = stat.size;
|
||||
let content = '';
|
||||
let lineCount = 0;
|
||||
|
||||
while (position > 0 && lineCount <= safeTailLines) {
|
||||
const bytesToRead = Math.min(chunkSize, position);
|
||||
position -= bytesToRead;
|
||||
const buffer = Buffer.allocUnsafe(bytesToRead);
|
||||
const { bytesRead } = await file.read(buffer, 0, bytesToRead, position);
|
||||
content = `${buffer.subarray(0, bytesRead).toString('utf-8')}${content}`;
|
||||
lineCount = content.split('\n').length - 1;
|
||||
}
|
||||
|
||||
const lines = content.split('\n');
|
||||
return lines.length <= safeTailLines ? content : lines.slice(-safeTailLines).join('\n');
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleDiagnosticsRoutes(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
url: URL,
|
||||
ctx: HostApiContext,
|
||||
): Promise<boolean> {
|
||||
if (url.pathname === '/api/diagnostics/gateway-snapshot' && req.method === 'GET') {
|
||||
try {
|
||||
const { channels } = await buildChannelAccountsView(ctx, { probe: false });
|
||||
const diagnostics = ctx.gatewayManager.getDiagnostics?.() ?? {
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
consecutiveRpcFailures: 0,
|
||||
};
|
||||
const channelStatusDiagnostics = getChannelStatusDiagnostics();
|
||||
const gateway = {
|
||||
...ctx.gatewayManager.getStatus(),
|
||||
...buildGatewayHealthSummary({
|
||||
status: ctx.gatewayManager.getStatus(),
|
||||
diagnostics,
|
||||
lastChannelsStatusOkAt: channelStatusDiagnostics.lastChannelsStatusOkAt,
|
||||
lastChannelsStatusFailureAt: channelStatusDiagnostics.lastChannelsStatusFailureAt,
|
||||
platform: process.platform,
|
||||
}),
|
||||
};
|
||||
const openClawDir = getOpenClawConfigDir();
|
||||
sendJson(res, 200, {
|
||||
capturedAt: Date.now(),
|
||||
platform: process.platform,
|
||||
gateway,
|
||||
channels,
|
||||
clawxLogTail: await logger.readLogFile(DEFAULT_TAIL_LINES),
|
||||
gatewayLogTail: await readTail(join(openClawDir, 'logs', 'gateway.log')),
|
||||
gatewayErrLogTail: await readTail(join(openClawDir, 'logs', 'gateway.err.log')),
|
||||
});
|
||||
} catch (error) {
|
||||
sendJson(res, 500, { success: false, error: String(error) });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import { handleSkillRoutes } from './routes/skills';
|
||||
import { handleFileRoutes } from './routes/files';
|
||||
import { handleSessionRoutes } from './routes/sessions';
|
||||
import { handleCronRoutes } from './routes/cron';
|
||||
import { handleDiagnosticsRoutes } from './routes/diagnostics';
|
||||
import { sendJson, setCorsHeaders, requireJsonContentType } from './route-utils';
|
||||
|
||||
type RouteHandler = (
|
||||
@@ -35,6 +36,7 @@ const routeHandlers: RouteHandler[] = [
|
||||
handleFileRoutes,
|
||||
handleSessionRoutes,
|
||||
handleCronRoutes,
|
||||
handleDiagnosticsRoutes,
|
||||
handleLogRoutes,
|
||||
handleUsageRoutes,
|
||||
];
|
||||
|
||||
@@ -65,6 +65,40 @@ export interface GatewayStatus {
|
||||
gatewayReady?: boolean;
|
||||
}
|
||||
|
||||
export type GatewayHealthState = 'healthy' | 'degraded' | 'unresponsive';
|
||||
|
||||
export interface GatewayHealthSummary {
|
||||
state: GatewayHealthState;
|
||||
reasons: string[];
|
||||
consecutiveHeartbeatMisses: number;
|
||||
lastAliveAt?: number;
|
||||
lastRpcSuccessAt?: number;
|
||||
lastRpcFailureAt?: number;
|
||||
lastRpcFailureMethod?: string;
|
||||
lastChannelsStatusOkAt?: number;
|
||||
lastChannelsStatusFailureAt?: number;
|
||||
}
|
||||
|
||||
export interface GatewayDiagnosticsSnapshot {
|
||||
lastAliveAt?: number;
|
||||
lastRpcSuccessAt?: number;
|
||||
lastRpcFailureAt?: number;
|
||||
lastRpcFailureMethod?: string;
|
||||
lastHeartbeatTimeoutAt?: number;
|
||||
consecutiveHeartbeatMisses: number;
|
||||
lastSocketCloseAt?: number;
|
||||
lastSocketCloseCode?: number;
|
||||
consecutiveRpcFailures: number;
|
||||
}
|
||||
|
||||
function isTransportRpcFailure(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return message.includes('RPC timeout:')
|
||||
|| message.includes('Gateway not connected')
|
||||
|| message.includes('Gateway stopped')
|
||||
|| message.includes('Failed to send RPC request:');
|
||||
}
|
||||
|
||||
/**
|
||||
* Gateway Manager Events
|
||||
*/
|
||||
@@ -126,6 +160,10 @@ export class GatewayManager extends EventEmitter {
|
||||
/** Set by scheduleReconnect() before calling start() to signal auto-reconnect. */
|
||||
private isAutoReconnectStart = false;
|
||||
private gatewayReadyFallbackTimer: NodeJS.Timeout | null = null;
|
||||
private diagnostics: GatewayDiagnosticsSnapshot = {
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
consecutiveRpcFailures: 0,
|
||||
};
|
||||
|
||||
constructor(config?: Partial<ReconnectConfig>) {
|
||||
super();
|
||||
@@ -197,6 +235,10 @@ export class GatewayManager extends EventEmitter {
|
||||
return this.stateController.getStatus();
|
||||
}
|
||||
|
||||
getDiagnostics(): GatewayDiagnosticsSnapshot {
|
||||
return { ...this.diagnostics };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if Gateway is connected and ready
|
||||
*/
|
||||
@@ -413,6 +455,7 @@ export class GatewayManager extends EventEmitter {
|
||||
|
||||
this.restartController.resetDeferredRestart();
|
||||
this.isAutoReconnectStart = false;
|
||||
this.diagnostics.consecutiveHeartbeatMisses = 0;
|
||||
this.setStatus({ state: 'stopped', error: undefined, pid: undefined, connectedAt: undefined, uptime: undefined, gatewayReady: undefined });
|
||||
}
|
||||
|
||||
@@ -712,7 +755,7 @@ export class GatewayManager extends EventEmitter {
|
||||
* Uses OpenClaw protocol format: { type: "req", id: "...", method: "...", params: {...} }
|
||||
*/
|
||||
async rpc<T>(method: string, params?: unknown, timeoutMs = 30000): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
return await new Promise<T>((resolve, reject) => {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
||||
reject(new Error('Gateway not connected'));
|
||||
return;
|
||||
@@ -745,6 +788,14 @@ export class GatewayManager extends EventEmitter {
|
||||
} catch (error) {
|
||||
rejectPendingGatewayRequest(this.pendingRequests, id, new Error(`Failed to send RPC request: ${error}`));
|
||||
}
|
||||
}).then((result) => {
|
||||
this.recordRpcSuccess();
|
||||
return result;
|
||||
}).catch((error) => {
|
||||
if (isTransportRpcFailure(error)) {
|
||||
this.recordRpcFailure(method);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -782,6 +833,32 @@ export class GatewayManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
private recordGatewayAlive(): void {
|
||||
this.diagnostics.lastAliveAt = Date.now();
|
||||
this.diagnostics.consecutiveHeartbeatMisses = 0;
|
||||
}
|
||||
|
||||
private recordRpcSuccess(): void {
|
||||
this.diagnostics.lastRpcSuccessAt = Date.now();
|
||||
this.diagnostics.consecutiveRpcFailures = 0;
|
||||
}
|
||||
|
||||
private recordRpcFailure(method: string): void {
|
||||
this.diagnostics.lastRpcFailureAt = Date.now();
|
||||
this.diagnostics.lastRpcFailureMethod = method;
|
||||
this.diagnostics.consecutiveRpcFailures += 1;
|
||||
}
|
||||
|
||||
private recordHeartbeatTimeout(consecutiveMisses: number): void {
|
||||
this.diagnostics.lastHeartbeatTimeoutAt = Date.now();
|
||||
this.diagnostics.consecutiveHeartbeatMisses = consecutiveMisses;
|
||||
}
|
||||
|
||||
private recordSocketClose(code: number): void {
|
||||
this.diagnostics.lastSocketCloseAt = Date.now();
|
||||
this.diagnostics.lastSocketCloseCode = code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start Gateway process
|
||||
* Uses OpenClaw npm package from node_modules (dev) or resources (production)
|
||||
@@ -878,7 +955,9 @@ export class GatewayManager extends EventEmitter {
|
||||
this.ws = ws;
|
||||
ws.on('pong', () => {
|
||||
this.connectionMonitor.markAlive('pong');
|
||||
this.recordGatewayAlive();
|
||||
});
|
||||
this.recordGatewayAlive();
|
||||
this.setStatus({
|
||||
state: 'running',
|
||||
port,
|
||||
@@ -892,6 +971,8 @@ export class GatewayManager extends EventEmitter {
|
||||
},
|
||||
onCloseAfterHandshake: (closeCode) => {
|
||||
this.connectionMonitor.clear();
|
||||
this.recordSocketClose(closeCode);
|
||||
this.diagnostics.consecutiveHeartbeatMisses = 0;
|
||||
if (this.status.state === 'running') {
|
||||
this.setStatus({ state: 'stopped' });
|
||||
// On Windows, skip reconnect from WS close. The Gateway is a local
|
||||
@@ -916,6 +997,7 @@ export class GatewayManager extends EventEmitter {
|
||||
*/
|
||||
private handleMessage(message: unknown): void {
|
||||
this.connectionMonitor.markAlive('message');
|
||||
this.recordGatewayAlive();
|
||||
|
||||
if (typeof message !== 'object' || message === null) {
|
||||
logger.debug('Received non-object Gateway message');
|
||||
@@ -986,24 +1068,25 @@ export class GatewayManager extends EventEmitter {
|
||||
}
|
||||
},
|
||||
onHeartbeatTimeout: ({ consecutiveMisses, timeoutMs }) => {
|
||||
// Heartbeat timeout is observability-only. We intentionally do NOT
|
||||
// terminate the socket or trigger reconnection here because:
|
||||
//
|
||||
// 1. If the gateway process dies → child.on('exit') fires reliably.
|
||||
// 2. If the socket disconnects → ws.on('close') fires reliably.
|
||||
// 3. If the gateway event loop is blocked (skills scanning, GC,
|
||||
// antivirus) → pong is delayed but the process and connection
|
||||
// are still valid. Terminating the socket would cause a
|
||||
// cascading restart loop for no reason.
|
||||
//
|
||||
// The only scenario ping/pong could catch (silent half-open TCP on
|
||||
// localhost) is practically impossible. So we just log.
|
||||
this.recordHeartbeatTimeout(consecutiveMisses);
|
||||
const pid = this.process?.pid ?? 'unknown';
|
||||
const isWindows = process.platform === 'win32';
|
||||
const shouldAttemptRecovery = !isWindows && this.shouldReconnect && this.status.state === 'running';
|
||||
logger.warn(
|
||||
`Gateway heartbeat: ${consecutiveMisses} consecutive pong misses ` +
|
||||
`(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}). ` +
|
||||
`No action taken — relying on process exit and socket close events.`,
|
||||
`(timeout=${timeoutMs}ms, pid=${pid}, state=${this.status.state}, autoReconnect=${this.shouldReconnect}).`,
|
||||
);
|
||||
if (!shouldAttemptRecovery) {
|
||||
const reason = isWindows
|
||||
? 'platform=win32'
|
||||
: 'lifecycle is not in auto-recoverable running state';
|
||||
logger.warn(`Gateway heartbeat recovery skipped (${reason})`);
|
||||
return;
|
||||
}
|
||||
logger.warn('Gateway heartbeat recovery: restarting unresponsive gateway process');
|
||||
void this.restart().catch((error) => {
|
||||
logger.warn('Gateway heartbeat recovery failed:', error);
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export type ChannelConnectionStatus = 'connected' | 'connecting' | 'disconnected' | 'error';
|
||||
export type GatewayHealthState = 'healthy' | 'degraded' | 'unresponsive';
|
||||
export type ChannelConnectionStatus = 'connected' | 'connecting' | 'degraded' | 'disconnected' | 'error';
|
||||
|
||||
export interface ChannelRuntimeAccountSnapshot {
|
||||
connected?: boolean;
|
||||
@@ -19,6 +20,10 @@ export interface ChannelRuntimeSummarySnapshot {
|
||||
lastError?: string | null;
|
||||
}
|
||||
|
||||
export interface ChannelHealthOverlay {
|
||||
gatewayHealthState?: GatewayHealthState;
|
||||
}
|
||||
|
||||
const RECENT_ACTIVITY_MS = 10 * 60 * 1000;
|
||||
|
||||
function hasNonEmptyError(value: string | null | undefined): boolean {
|
||||
@@ -74,9 +79,11 @@ export function isChannelRuntimeConnected(
|
||||
|
||||
export function computeChannelRuntimeStatus(
|
||||
account: ChannelRuntimeAccountSnapshot,
|
||||
healthOverlay?: ChannelHealthOverlay,
|
||||
): ChannelConnectionStatus {
|
||||
if (isChannelRuntimeConnected(account)) return 'connected';
|
||||
if (hasChannelRuntimeError(account)) return 'error';
|
||||
if (healthOverlay?.gatewayHealthState && healthOverlay.gatewayHealthState !== 'healthy') return 'degraded';
|
||||
if (isChannelRuntimeConnected(account)) return 'connected';
|
||||
if (account.running === true) return 'connecting';
|
||||
return 'disconnected';
|
||||
}
|
||||
@@ -84,6 +91,7 @@ export function computeChannelRuntimeStatus(
|
||||
export function pickChannelRuntimeStatus(
|
||||
accounts: ChannelRuntimeAccountSnapshot[],
|
||||
summary?: ChannelRuntimeSummarySnapshot,
|
||||
healthOverlay?: ChannelHealthOverlay,
|
||||
): ChannelConnectionStatus {
|
||||
if (accounts.some((account) => isChannelRuntimeConnected(account))) {
|
||||
return 'connected';
|
||||
@@ -93,6 +101,10 @@ export function pickChannelRuntimeStatus(
|
||||
return 'error';
|
||||
}
|
||||
|
||||
if (healthOverlay?.gatewayHealthState && healthOverlay.gatewayHealthState !== 'healthy') {
|
||||
return 'degraded';
|
||||
}
|
||||
|
||||
if (accounts.some((account) => account.running === true)) {
|
||||
return 'connecting';
|
||||
}
|
||||
|
||||
81
electron/utils/gateway-health.ts
Normal file
81
electron/utils/gateway-health.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
import type {
|
||||
GatewayDiagnosticsSnapshot,
|
||||
GatewayHealthSummary,
|
||||
GatewayStatus,
|
||||
} from '../gateway/manager';
|
||||
|
||||
type BuildGatewayHealthSummaryOptions = {
|
||||
status: GatewayStatus;
|
||||
diagnostics: GatewayDiagnosticsSnapshot;
|
||||
lastChannelsStatusOkAt?: number;
|
||||
lastChannelsStatusFailureAt?: number;
|
||||
platform?: string;
|
||||
now?: number;
|
||||
};
|
||||
|
||||
const CHANNEL_STATUS_FAILURE_WINDOW_MS = 2 * 60_000;
|
||||
const HEARTBEAT_MISS_THRESHOLD_DEFAULT = 3;
|
||||
const HEARTBEAT_MISS_THRESHOLD_WIN = 5;
|
||||
|
||||
export function buildGatewayHealthSummary(
|
||||
options: BuildGatewayHealthSummaryOptions,
|
||||
): GatewayHealthSummary {
|
||||
const now = options.now ?? Date.now();
|
||||
const reasons = new Set<string>();
|
||||
const heartbeatThreshold = options.platform === 'win32'
|
||||
? HEARTBEAT_MISS_THRESHOLD_WIN
|
||||
: HEARTBEAT_MISS_THRESHOLD_DEFAULT;
|
||||
|
||||
const channelStatusFailureIsRecent =
|
||||
typeof options.lastChannelsStatusFailureAt === 'number'
|
||||
&& now - options.lastChannelsStatusFailureAt <= CHANNEL_STATUS_FAILURE_WINDOW_MS
|
||||
&& (
|
||||
typeof options.lastChannelsStatusOkAt !== 'number'
|
||||
|| options.lastChannelsStatusFailureAt > options.lastChannelsStatusOkAt
|
||||
);
|
||||
|
||||
if (options.status.state !== 'running') {
|
||||
reasons.add(options.status.state === 'error' ? 'gateway_error' : 'gateway_not_running');
|
||||
return {
|
||||
state: 'degraded',
|
||||
reasons: [...reasons],
|
||||
consecutiveHeartbeatMisses: options.diagnostics.consecutiveHeartbeatMisses,
|
||||
lastAliveAt: options.diagnostics.lastAliveAt,
|
||||
lastRpcSuccessAt: options.diagnostics.lastRpcSuccessAt,
|
||||
lastRpcFailureAt: options.diagnostics.lastRpcFailureAt,
|
||||
lastRpcFailureMethod: options.diagnostics.lastRpcFailureMethod,
|
||||
lastChannelsStatusOkAt: options.lastChannelsStatusOkAt,
|
||||
lastChannelsStatusFailureAt: options.lastChannelsStatusFailureAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (options.diagnostics.consecutiveHeartbeatMisses >= heartbeatThreshold) {
|
||||
reasons.add('gateway_unresponsive');
|
||||
} else if (options.diagnostics.consecutiveHeartbeatMisses > 0) {
|
||||
reasons.add('gateway_degraded');
|
||||
}
|
||||
|
||||
if (options.diagnostics.consecutiveRpcFailures > 0) {
|
||||
reasons.add('rpc_timeout');
|
||||
}
|
||||
|
||||
if (channelStatusFailureIsRecent) {
|
||||
reasons.add('channels_status_timeout');
|
||||
}
|
||||
|
||||
return {
|
||||
state: reasons.has('gateway_unresponsive')
|
||||
? 'unresponsive'
|
||||
: reasons.size > 0
|
||||
? 'degraded'
|
||||
: 'healthy',
|
||||
reasons: [...reasons],
|
||||
consecutiveHeartbeatMisses: options.diagnostics.consecutiveHeartbeatMisses,
|
||||
lastAliveAt: options.diagnostics.lastAliveAt,
|
||||
lastRpcSuccessAt: options.diagnostics.lastRpcSuccessAt,
|
||||
lastRpcFailureAt: options.diagnostics.lastRpcFailureAt,
|
||||
lastRpcFailureMethod: options.diagnostics.lastRpcFailureMethod,
|
||||
lastChannelsStatusOkAt: options.lastChannelsStatusOkAt,
|
||||
lastChannelsStatusFailureAt: options.lastChannelsStatusFailureAt,
|
||||
};
|
||||
}
|
||||
@@ -1634,6 +1634,51 @@ export async function sanitizeOpenClawConfig(): Promise<void> {
|
||||
pluginsObj.allow = allowArr;
|
||||
}
|
||||
|
||||
// ── acpx legacy config/install cleanup ─────────────────────
|
||||
// Older OpenClaw releases allowed plugins.entries.acpx.config.command
|
||||
// and expectedVersion overrides. Current bundled acpx schema rejects
|
||||
// them, which causes the Gateway to fail validation before startup.
|
||||
// Strip those keys and drop stale installs metadata that still points
|
||||
// at an older bundled OpenClaw tree so the current bundled plugin can
|
||||
// be re-registered cleanly.
|
||||
const acpxEntry = isPlainRecord(pEntries.acpx) ? pEntries.acpx as Record<string, unknown> : null;
|
||||
const acpxConfig = acpxEntry && isPlainRecord(acpxEntry.config)
|
||||
? acpxEntry.config as Record<string, unknown>
|
||||
: null;
|
||||
if (acpxConfig) {
|
||||
for (const legacyKey of ['command', 'expectedVersion'] as const) {
|
||||
if (legacyKey in acpxConfig) {
|
||||
delete acpxConfig[legacyKey];
|
||||
modified = true;
|
||||
console.log(`[sanitize] Removed legacy plugins.entries.acpx.config.${legacyKey}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const installs = isPlainRecord(pluginsObj.installs) ? pluginsObj.installs as Record<string, unknown> : null;
|
||||
const acpxInstall = installs && isPlainRecord(installs.acpx) ? installs.acpx as Record<string, unknown> : null;
|
||||
if (acpxInstall) {
|
||||
const currentBundledAcpxDir = join(getOpenClawResolvedDir(), 'dist', 'extensions', 'acpx').replace(/\\/g, '/');
|
||||
const sourcePath = typeof acpxInstall.sourcePath === 'string' ? acpxInstall.sourcePath : '';
|
||||
const installPath = typeof acpxInstall.installPath === 'string' ? acpxInstall.installPath : '';
|
||||
const normalizedSourcePath = sourcePath.replace(/\\/g, '/');
|
||||
const normalizedInstallPath = installPath.replace(/\\/g, '/');
|
||||
const pointsAtDifferentBundledTree = [normalizedSourcePath, normalizedInstallPath].some(
|
||||
(candidate) => candidate.includes('/node_modules/.pnpm/openclaw@') && candidate !== currentBundledAcpxDir,
|
||||
);
|
||||
const pointsAtMissingPath = (sourcePath && !(await fileExists(sourcePath)))
|
||||
|| (installPath && !(await fileExists(installPath)));
|
||||
|
||||
if (pointsAtDifferentBundledTree || pointsAtMissingPath) {
|
||||
delete installs.acpx;
|
||||
if (Object.keys(installs).length === 0) {
|
||||
delete pluginsObj.installs;
|
||||
}
|
||||
modified = true;
|
||||
console.log('[sanitize] Removed stale plugins.installs.acpx metadata');
|
||||
}
|
||||
}
|
||||
|
||||
const installedFeishuId = await resolveInstalledFeishuPluginId();
|
||||
const configuredFeishuId =
|
||||
FEISHU_PLUGIN_ID_CANDIDATES.find((id) => allowArr.includes(id))
|
||||
|
||||
Reference in New Issue
Block a user