Add channel health diagnostics and gateway recovery fixes (#855)
This commit is contained in:
committed by
GitHub
Unverified
parent
6acd8acf5a
commit
1f39d1a8a7
@@ -175,6 +175,7 @@ describe('handleChannelRoutes', () => {
|
||||
gatewayManager: {
|
||||
rpc,
|
||||
getStatus: () => ({ state: 'running' }),
|
||||
getDiagnostics: () => ({ consecutiveHeartbeatMisses: 0, consecutiveRpcFailures: 0 }),
|
||||
debouncedReload: vi.fn(),
|
||||
debouncedRestart: vi.fn(),
|
||||
},
|
||||
@@ -921,6 +922,145 @@ describe('handleChannelRoutes', () => {
|
||||
expect(feishu?.accounts.map((entry) => entry.accountId)).toEqual(['default']);
|
||||
});
|
||||
|
||||
it('returns degraded channel health when channels.status times out while gateway is still running', async () => {
|
||||
listConfiguredChannelsMock.mockResolvedValue(['feishu']);
|
||||
listConfiguredChannelAccountsMock.mockResolvedValue({
|
||||
feishu: {
|
||||
defaultAccountId: 'default',
|
||||
accountIds: ['default'],
|
||||
},
|
||||
});
|
||||
readOpenClawConfigMock.mockResolvedValue({
|
||||
channels: {
|
||||
feishu: {
|
||||
defaultAccount: 'default',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rpc = vi.fn().mockRejectedValue(new Error('RPC timeout: channels.status'));
|
||||
|
||||
const { handleChannelRoutes } = await import('@electron/api/routes/channels');
|
||||
await handleChannelRoutes(
|
||||
{ method: 'GET' } as IncomingMessage,
|
||||
{} as ServerResponse,
|
||||
new URL('http://127.0.0.1:13210/api/channels/accounts'),
|
||||
{
|
||||
gatewayManager: {
|
||||
rpc,
|
||||
getStatus: () => ({ state: 'running' }),
|
||||
getDiagnostics: () => ({ consecutiveHeartbeatMisses: 0, consecutiveRpcFailures: 0 }),
|
||||
debouncedReload: vi.fn(),
|
||||
debouncedRestart: vi.fn(),
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(sendJsonMock).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
200,
|
||||
expect.objectContaining({
|
||||
success: true,
|
||||
gatewayHealth: expect.objectContaining({
|
||||
state: 'degraded',
|
||||
reasons: expect.arrayContaining(['channels_status_timeout']),
|
||||
}),
|
||||
channels: [
|
||||
expect.objectContaining({
|
||||
channelType: 'feishu',
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
accounts: [
|
||||
expect.objectContaining({
|
||||
accountId: 'default',
|
||||
status: 'degraded',
|
||||
}),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps channel degraded when only filtered stale runtime accounts carry lastError', async () => {
|
||||
listConfiguredChannelsMock.mockResolvedValue(['feishu']);
|
||||
listConfiguredChannelAccountsMock.mockResolvedValue({
|
||||
feishu: {
|
||||
defaultAccountId: 'default',
|
||||
accountIds: ['default'],
|
||||
},
|
||||
});
|
||||
readOpenClawConfigMock.mockResolvedValue({
|
||||
channels: {
|
||||
feishu: {
|
||||
defaultAccount: 'default',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rpc = vi.fn().mockResolvedValue({
|
||||
channels: {
|
||||
feishu: {
|
||||
configured: true,
|
||||
},
|
||||
},
|
||||
channelAccounts: {
|
||||
feishu: [
|
||||
{
|
||||
accountId: 'default',
|
||||
configured: true,
|
||||
connected: true,
|
||||
running: true,
|
||||
linked: false,
|
||||
},
|
||||
{
|
||||
accountId: '2',
|
||||
configured: false,
|
||||
connected: false,
|
||||
running: false,
|
||||
lastError: 'stale runtime session',
|
||||
},
|
||||
],
|
||||
},
|
||||
channelDefaultAccountId: {
|
||||
feishu: 'default',
|
||||
},
|
||||
});
|
||||
|
||||
const { handleChannelRoutes } = await import('@electron/api/routes/channels');
|
||||
await handleChannelRoutes(
|
||||
{ method: 'GET' } as IncomingMessage,
|
||||
{} as ServerResponse,
|
||||
new URL('http://127.0.0.1:13210/api/channels/accounts'),
|
||||
{
|
||||
gatewayManager: {
|
||||
rpc,
|
||||
getStatus: () => ({ state: 'running' }),
|
||||
getDiagnostics: () => ({ consecutiveHeartbeatMisses: 1, consecutiveRpcFailures: 0 }),
|
||||
debouncedReload: vi.fn(),
|
||||
debouncedRestart: vi.fn(),
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(sendJsonMock).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
200,
|
||||
expect.objectContaining({
|
||||
success: true,
|
||||
channels: [
|
||||
expect.objectContaining({
|
||||
channelType: 'feishu',
|
||||
status: 'degraded',
|
||||
accounts: [
|
||||
expect.objectContaining({ accountId: 'default', status: 'degraded' }),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('lists known QQ Bot targets for a configured account', async () => {
|
||||
const knownUsersPath = join(testOpenClawConfigDir, 'qqbot', 'data');
|
||||
mkdirSync(knownUsersPath, { recursive: true });
|
||||
|
||||
@@ -63,4 +63,42 @@ describe('channel runtime status helpers', () => {
|
||||
),
|
||||
).toBe('error');
|
||||
});
|
||||
|
||||
it('returns degraded when gateway health is degraded', () => {
|
||||
expect(
|
||||
computeChannelRuntimeStatus(
|
||||
{ running: true, connected: false, linked: false },
|
||||
{ gatewayHealthState: 'degraded' },
|
||||
),
|
||||
).toBe('degraded');
|
||||
});
|
||||
|
||||
it('keeps runtime error higher priority than degraded overlay', () => {
|
||||
expect(
|
||||
computeChannelRuntimeStatus(
|
||||
{ running: true, lastError: 'bot token invalid' },
|
||||
{ gatewayHealthState: 'degraded' },
|
||||
),
|
||||
).toBe('error');
|
||||
});
|
||||
|
||||
it('degrades channel summary when gateway health is degraded', () => {
|
||||
expect(
|
||||
pickChannelRuntimeStatus(
|
||||
[{ connected: false, running: false }],
|
||||
undefined,
|
||||
{ gatewayHealthState: 'degraded' },
|
||||
),
|
||||
).toBe('degraded');
|
||||
});
|
||||
|
||||
it('keeps summary error higher priority than degraded gateway health', () => {
|
||||
expect(
|
||||
pickChannelRuntimeStatus(
|
||||
[{ connected: false, running: false }],
|
||||
{ error: 'channel bootstrap failed' },
|
||||
{ gatewayHealthState: 'degraded' },
|
||||
),
|
||||
).toBe('error');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -51,11 +51,22 @@ function createDeferred<T>() {
|
||||
describe('Channels page status refresh', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
Object.defineProperty(globalThis.navigator, 'clipboard', {
|
||||
value: {
|
||||
writeText: vi.fn(),
|
||||
},
|
||||
configurable: true,
|
||||
});
|
||||
gatewayState.status = { state: 'running', port: 18789 };
|
||||
hostApiFetchMock.mockImplementation(async (path: string) => {
|
||||
if (path === '/api/channels/accounts') {
|
||||
return {
|
||||
success: true,
|
||||
gatewayHealth: {
|
||||
state: 'healthy',
|
||||
reasons: [],
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
},
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
@@ -384,4 +395,263 @@ describe('Channels page status refresh', () => {
|
||||
expect(appIdInput).toHaveValue('cli_test_app');
|
||||
expect(appSecretInput).toHaveValue('secret_test_value');
|
||||
});
|
||||
|
||||
it('shows degraded gateway banner and copies diagnostics snapshot', async () => {
|
||||
subscribeHostEventMock.mockImplementation(() => vi.fn());
|
||||
const writeTextMock = vi.mocked(navigator.clipboard.writeText);
|
||||
|
||||
hostApiFetchMock.mockImplementation(async (path: string, init?: { method?: string }) => {
|
||||
if (path === '/api/channels/accounts') {
|
||||
return {
|
||||
success: true,
|
||||
gatewayHealth: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
defaultAccountId: 'default',
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
accounts: [
|
||||
{
|
||||
accountId: 'default',
|
||||
name: 'Primary Account',
|
||||
configured: true,
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
isDefault: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
if (path === '/api/agents') {
|
||||
return {
|
||||
success: true,
|
||||
agents: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (path === '/api/diagnostics/gateway-snapshot') {
|
||||
return {
|
||||
capturedAt: 123,
|
||||
platform: 'darwin',
|
||||
gateway: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [],
|
||||
clawxLogTail: 'clawx',
|
||||
gatewayLogTail: 'gateway',
|
||||
gatewayErrLogTail: '',
|
||||
};
|
||||
}
|
||||
|
||||
if (path === '/api/gateway/restart' && init?.method === 'POST') {
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected host API path: ${path}`);
|
||||
});
|
||||
|
||||
render(<Channels />);
|
||||
|
||||
expect(await screen.findByTestId('channels-health-banner')).toBeInTheDocument();
|
||||
expect(screen.getByText('health.state.degraded')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-copy-diagnostics'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(hostApiFetchMock).toHaveBeenCalledWith('/api/diagnostics/gateway-snapshot');
|
||||
expect(writeTextMock).toHaveBeenCalledWith(expect.stringContaining('"platform": "darwin"'));
|
||||
});
|
||||
});
|
||||
|
||||
it('surfaces diagnostics fetch failure payloads instead of caching them as snapshots', async () => {
|
||||
subscribeHostEventMock.mockImplementation(() => vi.fn());
|
||||
|
||||
hostApiFetchMock.mockImplementation(async (path: string) => {
|
||||
if (path === '/api/channels/accounts') {
|
||||
return {
|
||||
success: true,
|
||||
gatewayHealth: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
defaultAccountId: 'default',
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
accounts: [
|
||||
{
|
||||
accountId: 'default',
|
||||
name: 'Primary Account',
|
||||
configured: true,
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
isDefault: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (path === '/api/agents') {
|
||||
return { success: true, agents: [] };
|
||||
}
|
||||
if (path === '/api/diagnostics/gateway-snapshot') {
|
||||
return { success: false, error: 'snapshot failed' };
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected host API path: ${path}`);
|
||||
});
|
||||
|
||||
render(<Channels />);
|
||||
expect(await screen.findByTestId('channels-health-banner')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-toggle-diagnostics'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toastErrorMock).toHaveBeenCalledWith('health.diagnosticsCopyFailed');
|
||||
});
|
||||
expect(screen.queryByTestId('channels-diagnostics')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows restart failure when gateway restart returns success=false', async () => {
|
||||
subscribeHostEventMock.mockImplementation(() => vi.fn());
|
||||
|
||||
hostApiFetchMock.mockImplementation(async (path: string, init?: { method?: string }) => {
|
||||
if (path === '/api/channels/accounts') {
|
||||
return {
|
||||
success: true,
|
||||
gatewayHealth: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
defaultAccountId: 'default',
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
accounts: [
|
||||
{
|
||||
accountId: 'default',
|
||||
name: 'Primary Account',
|
||||
configured: true,
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
isDefault: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (path === '/api/agents') {
|
||||
return { success: true, agents: [] };
|
||||
}
|
||||
if (path === '/api/gateway/restart' && init?.method === 'POST') {
|
||||
return { success: false, error: 'restart failed' };
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected host API path: ${path}`);
|
||||
});
|
||||
|
||||
render(<Channels />);
|
||||
expect(await screen.findByTestId('channels-health-banner')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-restart-gateway'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toastErrorMock).toHaveBeenCalledWith('health.restartFailed');
|
||||
});
|
||||
expect(toastSuccessMock).not.toHaveBeenCalledWith('health.restartTriggered');
|
||||
});
|
||||
|
||||
it('refetches diagnostics snapshot every time the diagnostics panel is reopened', async () => {
|
||||
subscribeHostEventMock.mockImplementation(() => vi.fn());
|
||||
|
||||
let diagnosticsFetchCount = 0;
|
||||
hostApiFetchMock.mockImplementation(async (path: string) => {
|
||||
if (path === '/api/channels/accounts') {
|
||||
return {
|
||||
success: true,
|
||||
gatewayHealth: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
defaultAccountId: 'default',
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
accounts: [
|
||||
{
|
||||
accountId: 'default',
|
||||
name: 'Primary Account',
|
||||
configured: true,
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
isDefault: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (path === '/api/agents') {
|
||||
return { success: true, agents: [] };
|
||||
}
|
||||
if (path === '/api/diagnostics/gateway-snapshot') {
|
||||
diagnosticsFetchCount += 1;
|
||||
return {
|
||||
capturedAt: diagnosticsFetchCount,
|
||||
platform: 'darwin',
|
||||
gateway: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
channels: [],
|
||||
clawxLogTail: `clawx-${diagnosticsFetchCount}`,
|
||||
gatewayLogTail: 'gateway',
|
||||
gatewayErrLogTail: '',
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected host API path: ${path}`);
|
||||
});
|
||||
|
||||
render(<Channels />);
|
||||
|
||||
expect(await screen.findByTestId('channels-health-banner')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-toggle-diagnostics'));
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('channels-diagnostics')).toHaveTextContent('"capturedAt": 1');
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-toggle-diagnostics'));
|
||||
expect(screen.queryByTestId('channels-diagnostics')).not.toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId('channels-toggle-diagnostics'));
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('channels-diagnostics')).toHaveTextContent('"capturedAt": 2');
|
||||
});
|
||||
|
||||
expect(diagnosticsFetchCount).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
166
tests/unit/diagnostics-routes.test.ts
Normal file
166
tests/unit/diagnostics-routes.test.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import type { IncomingMessage, ServerResponse } from 'http';
|
||||
|
||||
const buildChannelAccountsViewMock = vi.fn();
|
||||
const getChannelStatusDiagnosticsMock = vi.fn();
|
||||
const sendJsonMock = vi.fn();
|
||||
const readLogFileMock = vi.fn();
|
||||
|
||||
const testOpenClawConfigDir = join(tmpdir(), 'clawx-tests', 'diagnostics-routes-openclaw');
|
||||
|
||||
vi.mock('@electron/api/routes/channels', () => ({
|
||||
buildChannelAccountsView: (...args: unknown[]) => buildChannelAccountsViewMock(...args),
|
||||
getChannelStatusDiagnostics: (...args: unknown[]) => getChannelStatusDiagnosticsMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('@electron/api/route-utils', () => ({
|
||||
sendJson: (...args: unknown[]) => sendJsonMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock('@electron/utils/logger', () => ({
|
||||
logger: {
|
||||
readLogFile: (...args: unknown[]) => readLogFileMock(...args),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@electron/utils/paths', () => ({
|
||||
getOpenClawConfigDir: () => testOpenClawConfigDir,
|
||||
}));
|
||||
|
||||
describe('handleDiagnosticsRoutes', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
rmSync(testOpenClawConfigDir, { recursive: true, force: true });
|
||||
mkdirSync(join(testOpenClawConfigDir, 'logs'), { recursive: true });
|
||||
buildChannelAccountsViewMock.mockResolvedValue({
|
||||
channels: [
|
||||
{
|
||||
channelType: 'feishu',
|
||||
defaultAccountId: 'default',
|
||||
status: 'degraded',
|
||||
accounts: [
|
||||
{
|
||||
accountId: 'default',
|
||||
name: 'Primary Account',
|
||||
configured: true,
|
||||
status: 'degraded',
|
||||
statusReason: 'channels_status_timeout',
|
||||
isDefault: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
gatewayHealth: {
|
||||
state: 'degraded',
|
||||
reasons: ['channels_status_timeout'],
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
},
|
||||
});
|
||||
getChannelStatusDiagnosticsMock.mockReturnValue({
|
||||
lastChannelsStatusOkAt: 100,
|
||||
lastChannelsStatusFailureAt: 200,
|
||||
});
|
||||
readLogFileMock.mockResolvedValue('clawx-log-tail');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(testOpenClawConfigDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns diagnostics snapshot with channel view and tailed logs', async () => {
|
||||
writeFileSync(join(testOpenClawConfigDir, 'logs', 'gateway.log'), 'gateway-line-1\ngateway-line-2\n');
|
||||
|
||||
const { handleDiagnosticsRoutes } = await import('@electron/api/routes/diagnostics');
|
||||
const handled = await handleDiagnosticsRoutes(
|
||||
{ method: 'GET' } as IncomingMessage,
|
||||
{} as ServerResponse,
|
||||
new URL('http://127.0.0.1:13210/api/diagnostics/gateway-snapshot'),
|
||||
{
|
||||
gatewayManager: {
|
||||
getStatus: () => ({ state: 'running', port: 18789, connectedAt: 50 }),
|
||||
getDiagnostics: () => ({
|
||||
lastAliveAt: 60,
|
||||
lastRpcSuccessAt: 70,
|
||||
consecutiveHeartbeatMisses: 1,
|
||||
consecutiveRpcFailures: 0,
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
expect(handled).toBe(true);
|
||||
const payload = sendJsonMock.mock.calls.at(-1)?.[2] as {
|
||||
platform?: string;
|
||||
channels?: Array<{ channelType: string; status: string }>;
|
||||
clawxLogTail?: string;
|
||||
gatewayLogTail?: string;
|
||||
gatewayErrLogTail?: string;
|
||||
gateway?: { state?: string; reasons?: string[] };
|
||||
};
|
||||
expect(payload.platform).toBe(process.platform);
|
||||
expect(payload.channels).toEqual([
|
||||
expect.objectContaining({
|
||||
channelType: 'feishu',
|
||||
status: 'degraded',
|
||||
}),
|
||||
]);
|
||||
expect(payload.clawxLogTail).toBe('clawx-log-tail');
|
||||
expect(payload.gatewayLogTail).toContain('gateway-line-1');
|
||||
expect(payload.gatewayErrLogTail).toBe('');
|
||||
expect(payload.gateway?.state).toBe('degraded');
|
||||
expect(payload.gateway?.reasons).toEqual(expect.arrayContaining(['gateway_degraded']));
|
||||
});
|
||||
|
||||
it('returns empty gateway log tails when log files are missing', async () => {
|
||||
const { handleDiagnosticsRoutes } = await import('@electron/api/routes/diagnostics');
|
||||
await handleDiagnosticsRoutes(
|
||||
{ method: 'GET' } as IncomingMessage,
|
||||
{} as ServerResponse,
|
||||
new URL('http://127.0.0.1:13210/api/diagnostics/gateway-snapshot'),
|
||||
{
|
||||
gatewayManager: {
|
||||
getStatus: () => ({ state: 'running', port: 18789 }),
|
||||
getDiagnostics: () => ({
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
consecutiveRpcFailures: 0,
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
const payload = sendJsonMock.mock.calls.at(-1)?.[2] as {
|
||||
gatewayLogTail?: string;
|
||||
gatewayErrLogTail?: string;
|
||||
};
|
||||
expect(payload.gatewayLogTail).toBe('');
|
||||
expect(payload.gatewayErrLogTail).toBe('');
|
||||
});
|
||||
|
||||
it('reads tailed logs without leaking unread buffer bytes', async () => {
|
||||
writeFileSync(join(testOpenClawConfigDir, 'logs', 'gateway.log'), 'only-one-line');
|
||||
|
||||
const { handleDiagnosticsRoutes } = await import('@electron/api/routes/diagnostics');
|
||||
await handleDiagnosticsRoutes(
|
||||
{ method: 'GET' } as IncomingMessage,
|
||||
{} as ServerResponse,
|
||||
new URL('http://127.0.0.1:13210/api/diagnostics/gateway-snapshot'),
|
||||
{
|
||||
gatewayManager: {
|
||||
getStatus: () => ({ state: 'running', port: 18789 }),
|
||||
getDiagnostics: () => ({
|
||||
consecutiveHeartbeatMisses: 0,
|
||||
consecutiveRpcFailures: 0,
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
);
|
||||
|
||||
const payload = sendJsonMock.mock.calls.at(-1)?.[2] as {
|
||||
gatewayLogTail?: string;
|
||||
};
|
||||
expect(payload.gatewayLogTail).toBe('only-one-line');
|
||||
});
|
||||
});
|
||||
165
tests/unit/gateway-manager-diagnostics.test.ts
Normal file
165
tests/unit/gateway-manager-diagnostics.test.ts
Normal file
@@ -0,0 +1,165 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: {
|
||||
getPath: () => '/tmp',
|
||||
isPackaged: false,
|
||||
},
|
||||
utilityProcess: {
|
||||
fork: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@electron/utils/logger', () => ({
|
||||
logger: {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('GatewayManager diagnostics', () => {
|
||||
const originalPlatform = process.platform;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-04-14T00:00:00.000Z'));
|
||||
Object.defineProperty(process, 'platform', { value: originalPlatform });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
Object.defineProperty(process, 'platform', { value: originalPlatform });
|
||||
});
|
||||
|
||||
it('updates diagnostics on gateway message, rpc success/timeout, and socket close', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
const ws = {
|
||||
readyState: 1,
|
||||
send: vi.fn(),
|
||||
ping: vi.fn(),
|
||||
terminate: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
|
||||
(manager as unknown as { ws: typeof ws }).ws = ws;
|
||||
|
||||
(manager as unknown as { handleMessage: (message: unknown) => void }).handleMessage({
|
||||
type: 'event',
|
||||
event: 'gateway.ready',
|
||||
payload: {},
|
||||
});
|
||||
expect(manager.getDiagnostics().lastAliveAt).toBe(Date.now());
|
||||
|
||||
const successPromise = manager.rpc<{ ok: boolean }>('chat.history', {}, 1000);
|
||||
const successRequestId = Array.from(
|
||||
(manager as unknown as { pendingRequests: Map<string, unknown> }).pendingRequests.keys(),
|
||||
)[0];
|
||||
(manager as unknown as { handleMessage: (message: unknown) => void }).handleMessage({
|
||||
type: 'res',
|
||||
id: successRequestId,
|
||||
ok: true,
|
||||
payload: { ok: true },
|
||||
});
|
||||
await expect(successPromise).resolves.toEqual({ ok: true });
|
||||
expect(manager.getDiagnostics().lastRpcSuccessAt).toBe(Date.now());
|
||||
expect(manager.getDiagnostics().consecutiveRpcFailures).toBe(0);
|
||||
|
||||
const failurePromise = manager.rpc('chat.history', {}, 1000);
|
||||
vi.advanceTimersByTime(1001);
|
||||
await expect(failurePromise).rejects.toThrow('RPC timeout: chat.history');
|
||||
|
||||
const diagnostics = manager.getDiagnostics();
|
||||
expect(diagnostics.lastRpcFailureAt).toBe(Date.now());
|
||||
expect(diagnostics.lastRpcFailureMethod).toBe('chat.history');
|
||||
expect(diagnostics.consecutiveRpcFailures).toBe(1);
|
||||
|
||||
(manager as unknown as { recordSocketClose: (code: number) => void }).recordSocketClose(1006);
|
||||
expect(manager.getDiagnostics().lastSocketCloseAt).toBe(Date.now());
|
||||
expect(manager.getDiagnostics().lastSocketCloseCode).toBe(1006);
|
||||
});
|
||||
|
||||
it('does not count gateway-declared rpc errors as transport failures', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const { buildGatewayHealthSummary } = await import('@electron/utils/gateway-health');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
const ws = {
|
||||
readyState: 1,
|
||||
send: vi.fn(),
|
||||
ping: vi.fn(),
|
||||
terminate: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
|
||||
(manager as unknown as { ws: typeof ws }).ws = ws;
|
||||
(manager as unknown as { status: { state: string; port: number } }).status = {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
|
||||
const failurePromise = manager.rpc('channels.status', {}, 1000);
|
||||
const failureRequestId = Array.from(
|
||||
(manager as unknown as { pendingRequests: Map<string, unknown> }).pendingRequests.keys(),
|
||||
)[0];
|
||||
(manager as unknown as { handleMessage: (message: unknown) => void }).handleMessage({
|
||||
type: 'res',
|
||||
id: failureRequestId,
|
||||
ok: false,
|
||||
error: { message: 'channel unavailable' },
|
||||
});
|
||||
await expect(failurePromise).rejects.toThrow('channel unavailable');
|
||||
|
||||
expect(manager.getDiagnostics().consecutiveRpcFailures).toBe(0);
|
||||
|
||||
const health = buildGatewayHealthSummary({
|
||||
status: manager.getStatus(),
|
||||
diagnostics: manager.getDiagnostics(),
|
||||
platform: process.platform,
|
||||
});
|
||||
expect(health.reasons).not.toContain('rpc_timeout');
|
||||
});
|
||||
|
||||
it('keeps windows heartbeat recovery disabled while diagnostics degrade', async () => {
|
||||
Object.defineProperty(process, 'platform', { value: 'win32' });
|
||||
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const { buildGatewayHealthSummary } = await import('@electron/utils/gateway-health');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
const ws = {
|
||||
readyState: 1,
|
||||
send: vi.fn(),
|
||||
ping: vi.fn(),
|
||||
terminate: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
|
||||
(manager as unknown as { ws: typeof ws }).ws = ws;
|
||||
(manager as unknown as { shouldReconnect: boolean }).shouldReconnect = true;
|
||||
(manager as unknown as { status: { state: string; port: number } }).status = {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue();
|
||||
|
||||
(manager as unknown as { startPing: () => void }).startPing();
|
||||
vi.advanceTimersByTime(400_000);
|
||||
|
||||
expect(restartSpy).not.toHaveBeenCalled();
|
||||
|
||||
const health = buildGatewayHealthSummary({
|
||||
status: manager.getStatus(),
|
||||
diagnostics: manager.getDiagnostics(),
|
||||
platform: 'win32',
|
||||
});
|
||||
expect(health.state).not.toBe('healthy');
|
||||
|
||||
(manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear();
|
||||
});
|
||||
});
|
||||
@@ -11,12 +11,22 @@ vi.mock('electron', () => ({
|
||||
}));
|
||||
|
||||
describe('GatewayManager heartbeat recovery', () => {
|
||||
const originalPlatform = process.platform;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date('2026-03-19T00:00:00.000Z'));
|
||||
Object.defineProperty(process, 'platform', { value: originalPlatform });
|
||||
});
|
||||
|
||||
it('logs warning but does NOT terminate socket after consecutive heartbeat misses', async () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
Object.defineProperty(process, 'platform', { value: originalPlatform });
|
||||
});
|
||||
|
||||
it('restarts after consecutive heartbeat misses reach threshold', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
@@ -33,20 +43,20 @@ describe('GatewayManager heartbeat recovery', () => {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue();
|
||||
|
||||
(manager as unknown as { startPing: () => void }).startPing();
|
||||
|
||||
vi.advanceTimersByTime(120_000);
|
||||
|
||||
expect(ws.ping).toHaveBeenCalledTimes(3);
|
||||
// Heartbeat timeout is now observability-only — socket should NOT be terminated.
|
||||
// Process liveness is detected via child.on('exit'), socket disconnects via ws.on('close').
|
||||
expect(ws.terminate).not.toHaveBeenCalled();
|
||||
expect(restartSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
(manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear();
|
||||
});
|
||||
|
||||
it('does not terminate when heartbeat is recovered by incoming messages', async () => {
|
||||
it('does not restart when heartbeat is recovered by incoming messages', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
@@ -63,6 +73,7 @@ describe('GatewayManager heartbeat recovery', () => {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue();
|
||||
|
||||
(manager as unknown as { startPing: () => void }).startPing();
|
||||
|
||||
@@ -75,6 +86,65 @@ describe('GatewayManager heartbeat recovery', () => {
|
||||
vi.advanceTimersByTime(30_000); // miss #2 + ping #5
|
||||
|
||||
expect(ws.terminate).not.toHaveBeenCalled();
|
||||
expect(restartSpy).not.toHaveBeenCalled();
|
||||
|
||||
(manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear();
|
||||
});
|
||||
|
||||
it('skips heartbeat recovery when auto-reconnect is disabled', async () => {
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
const ws = {
|
||||
readyState: 1,
|
||||
ping: vi.fn(),
|
||||
terminate: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
|
||||
(manager as unknown as { ws: typeof ws }).ws = ws;
|
||||
(manager as unknown as { shouldReconnect: boolean }).shouldReconnect = false;
|
||||
(manager as unknown as { status: { state: string; port: number } }).status = {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue();
|
||||
|
||||
(manager as unknown as { startPing: () => void }).startPing();
|
||||
|
||||
vi.advanceTimersByTime(120_000);
|
||||
|
||||
expect(restartSpy).not.toHaveBeenCalled();
|
||||
|
||||
(manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear();
|
||||
});
|
||||
|
||||
it('keeps heartbeat recovery disabled on windows', async () => {
|
||||
Object.defineProperty(process, 'platform', { value: 'win32' });
|
||||
|
||||
const { GatewayManager } = await import('@electron/gateway/manager');
|
||||
const manager = new GatewayManager();
|
||||
|
||||
const ws = {
|
||||
readyState: 1,
|
||||
ping: vi.fn(),
|
||||
terminate: vi.fn(),
|
||||
on: vi.fn(),
|
||||
};
|
||||
|
||||
(manager as unknown as { ws: typeof ws }).ws = ws;
|
||||
(manager as unknown as { shouldReconnect: boolean }).shouldReconnect = true;
|
||||
(manager as unknown as { status: { state: string; port: number } }).status = {
|
||||
state: 'running',
|
||||
port: 18789,
|
||||
};
|
||||
const restartSpy = vi.spyOn(manager, 'restart').mockResolvedValue();
|
||||
|
||||
(manager as unknown as { startPing: () => void }).startPing();
|
||||
|
||||
vi.advanceTimersByTime(400_000);
|
||||
|
||||
expect(restartSpy).not.toHaveBeenCalled();
|
||||
|
||||
(manager as unknown as { connectionMonitor: { clear: () => void } }).connectionMonitor.clear();
|
||||
});
|
||||
|
||||
@@ -128,6 +128,56 @@ async function sanitizeConfig(
|
||||
: {}
|
||||
) as Record<string, unknown>;
|
||||
|
||||
const acpxEntry = (entries.acpx && typeof entries.acpx === 'object' && !Array.isArray(entries.acpx))
|
||||
? { ...(entries.acpx as Record<string, unknown>) }
|
||||
: null;
|
||||
const acpxConfig = (acpxEntry?.config && typeof acpxEntry.config === 'object' && !Array.isArray(acpxEntry.config))
|
||||
? { ...(acpxEntry.config as Record<string, unknown>) }
|
||||
: null;
|
||||
if (acpxConfig) {
|
||||
for (const legacyKey of ['command', 'expectedVersion'] as const) {
|
||||
if (legacyKey in acpxConfig) {
|
||||
delete acpxConfig[legacyKey];
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
acpxEntry!.config = acpxConfig;
|
||||
entries.acpx = acpxEntry!;
|
||||
pluginsObj.entries = entries;
|
||||
}
|
||||
|
||||
const installs = (
|
||||
pluginsObj.installs && typeof pluginsObj.installs === 'object' && !Array.isArray(pluginsObj.installs)
|
||||
? { ...(pluginsObj.installs as Record<string, unknown>) }
|
||||
: {}
|
||||
) as Record<string, unknown>;
|
||||
const acpxInstall = (installs.acpx && typeof installs.acpx === 'object' && !Array.isArray(installs.acpx))
|
||||
? installs.acpx as Record<string, unknown>
|
||||
: null;
|
||||
if (acpxInstall) {
|
||||
const currentBundledAcpxDir = join(tempDir, 'node_modules', 'openclaw', 'dist', 'extensions', 'acpx').replace(/\\/g, '/');
|
||||
const sourcePath = typeof acpxInstall.sourcePath === 'string' ? acpxInstall.sourcePath : '';
|
||||
const installPath = typeof acpxInstall.installPath === 'string' ? acpxInstall.installPath : '';
|
||||
const normalizedSourcePath = sourcePath.replace(/\\/g, '/');
|
||||
const normalizedInstallPath = installPath.replace(/\\/g, '/');
|
||||
const pointsAtDifferentBundledTree = [normalizedSourcePath, normalizedInstallPath].some(
|
||||
(candidate) => candidate.includes('/node_modules/.pnpm/openclaw@') && candidate !== currentBundledAcpxDir,
|
||||
);
|
||||
const pointsAtMissingPath = (sourcePath && !(await fileExists(sourcePath)))
|
||||
|| (installPath && !(await fileExists(installPath)));
|
||||
|
||||
if (pointsAtDifferentBundledTree || pointsAtMissingPath) {
|
||||
delete installs.acpx;
|
||||
modified = true;
|
||||
}
|
||||
|
||||
if (Object.keys(installs).length > 0) {
|
||||
pluginsObj.installs = installs;
|
||||
} else {
|
||||
delete pluginsObj.installs;
|
||||
}
|
||||
}
|
||||
|
||||
if ('whatsapp' in entries) {
|
||||
delete entries.whatsapp;
|
||||
pluginsObj.entries = entries;
|
||||
@@ -625,6 +675,49 @@ describe('sanitizeOpenClawConfig (blocklist approach)', () => {
|
||||
expect(load.paths).toEqual(['relative/plugin-path', './another-relative']);
|
||||
});
|
||||
|
||||
it('removes legacy acpx overrides and stale bundled install metadata', async () => {
|
||||
await writeConfig({
|
||||
plugins: {
|
||||
entries: {
|
||||
acpx: {
|
||||
enabled: true,
|
||||
config: {
|
||||
permissionMode: 'approve-all',
|
||||
nonInteractivePermissions: 'fail',
|
||||
command: '/Users/example/project/node_modules/.pnpm/openclaw@2026.4.1/node_modules/openclaw/dist/extensions/acpx/node_modules/acpx/dist/cli.js',
|
||||
expectedVersion: 'any',
|
||||
pluginToolsMcpBridge: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
installs: {
|
||||
acpx: {
|
||||
source: 'path',
|
||||
spec: 'acpx',
|
||||
sourcePath: '/Users/example/project/node_modules/.pnpm/openclaw@2026.4.1/node_modules/openclaw/dist/extensions/acpx',
|
||||
installPath: '/Users/example/project/node_modules/.pnpm/openclaw@2026.4.1/node_modules/openclaw/dist/extensions/acpx',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const modified = await sanitizeConfig(configPath);
|
||||
expect(modified).toBe(true);
|
||||
|
||||
const result = await readConfig();
|
||||
const plugins = result.plugins as Record<string, unknown>;
|
||||
const entries = plugins.entries as Record<string, unknown>;
|
||||
const acpx = entries.acpx as Record<string, unknown>;
|
||||
const acpxConfig = acpx.config as Record<string, unknown>;
|
||||
|
||||
expect(acpxConfig).toEqual({
|
||||
permissionMode: 'approve-all',
|
||||
nonInteractivePermissions: 'fail',
|
||||
pluginToolsMcpBridge: true,
|
||||
});
|
||||
expect(plugins).not.toHaveProperty('installs');
|
||||
});
|
||||
|
||||
it('does nothing when plugins.load.paths contains only valid paths', async () => {
|
||||
const original = {
|
||||
plugins: {
|
||||
|
||||
Reference in New Issue
Block a user