123 lines
4.0 KiB
JavaScript
123 lines
4.0 KiB
JavaScript
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import process from 'node:process';
|
|
|
|
const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..', '..');
|
|
const CURRENT_FILE = path.join(ROOT, 'artifacts/comms/current-metrics.json');
|
|
const BASELINE_FILE = path.join(ROOT, 'scripts/comms/baseline/metrics.baseline.json');
|
|
const OUTPUT_DIR = path.join(ROOT, 'artifacts/comms');
|
|
const REPORT_FILE = path.join(OUTPUT_DIR, 'compare-report.md');
|
|
|
|
const HARD_THRESHOLDS = {
|
|
duplicate_event_rate: 0.005,
|
|
event_fanout_ratio: 1.2,
|
|
history_inflight_max: 1,
|
|
rpc_timeout_rate: 0.01,
|
|
message_loss_count: 0,
|
|
message_order_violation_count: 0,
|
|
};
|
|
|
|
const RELATIVE_THRESHOLDS = {
|
|
history_load_qps: 0.10,
|
|
rpc_p95_ms: 0.15,
|
|
};
|
|
|
|
const REQUIRED_SCENARIOS = [
|
|
'gateway-restart-during-run',
|
|
'happy-path-chat',
|
|
'history-overlap-guard',
|
|
'invalid-config-patch-recovered',
|
|
'multi-agent-channel-switch',
|
|
'network-degraded',
|
|
];
|
|
|
|
function ratioDelta(current, baseline) {
|
|
if (!Number.isFinite(baseline) || baseline === 0) return current === 0 ? 0 : Infinity;
|
|
return (current - baseline) / baseline;
|
|
}
|
|
|
|
function fmtPercent(value) {
|
|
return `${(value * 100).toFixed(2)}%`;
|
|
}
|
|
|
|
function fmtNumber(value) {
|
|
return Number.isFinite(value) ? Number(value).toFixed(4) : String(value);
|
|
}
|
|
|
|
export function evaluateReport(current, baseline) {
|
|
const c = current.aggregate ?? {};
|
|
const b = baseline.aggregate ?? {};
|
|
const scenarios = current.scenarios ?? {};
|
|
const failures = [];
|
|
const rows = [];
|
|
|
|
for (const scenario of REQUIRED_SCENARIOS) {
|
|
if (!scenarios[scenario]) {
|
|
failures.push(`missing scenario: ${scenario}`);
|
|
rows.push(`| scenario:${scenario} | missing | required | FAIL |`);
|
|
continue;
|
|
}
|
|
const scenarioMetrics = scenarios[scenario];
|
|
for (const [metric, threshold] of Object.entries(HARD_THRESHOLDS)) {
|
|
const cv = Number(scenarioMetrics[metric] ?? 0);
|
|
const pass = cv <= threshold;
|
|
if (!pass) failures.push(`scenario:${scenario} ${metric}=${cv} > ${threshold}`);
|
|
rows.push(`| ${scenario}.${metric} | ${fmtNumber(cv)} | <= ${threshold} | ${pass ? 'PASS' : 'FAIL'} |`);
|
|
}
|
|
}
|
|
|
|
for (const [metric, threshold] of Object.entries(HARD_THRESHOLDS)) {
|
|
const cv = Number(c[metric] ?? 0);
|
|
const pass = cv <= threshold;
|
|
if (!pass) failures.push(`${metric}=${cv} > ${threshold}`);
|
|
rows.push(`| ${metric} | ${fmtNumber(cv)} | <= ${threshold} | ${pass ? 'PASS' : 'FAIL'} |`);
|
|
}
|
|
|
|
for (const [metric, maxIncrease] of Object.entries(RELATIVE_THRESHOLDS)) {
|
|
const cv = Number(c[metric] ?? 0);
|
|
const bv = Number(b[metric] ?? 0);
|
|
const delta = ratioDelta(cv, bv);
|
|
const pass = delta <= maxIncrease;
|
|
if (!pass) failures.push(`${metric} delta=${delta} > ${maxIncrease}`);
|
|
rows.push(`| ${metric} | ${fmtNumber(cv)} (baseline ${fmtNumber(bv)}) | delta <= ${fmtPercent(maxIncrease)} | ${pass ? 'PASS' : 'FAIL'} (${fmtPercent(delta)}) |`);
|
|
}
|
|
|
|
return { failures, rows };
|
|
}
|
|
|
|
export async function main() {
|
|
const current = JSON.parse(await readFile(CURRENT_FILE, 'utf8'));
|
|
const baseline = JSON.parse(await readFile(BASELINE_FILE, 'utf8'));
|
|
const { failures, rows } = evaluateReport(current, baseline);
|
|
|
|
const report = [
|
|
'# Comms Regression Report',
|
|
'',
|
|
`- Generated at: ${new Date().toISOString()}`,
|
|
`- Result: ${failures.length === 0 ? 'PASS' : 'FAIL'}`,
|
|
'',
|
|
'| Metric | Current | Threshold | Status |',
|
|
'|---|---:|---:|---|',
|
|
...rows,
|
|
'',
|
|
].join('\n');
|
|
|
|
await mkdir(OUTPUT_DIR, { recursive: true });
|
|
await writeFile(REPORT_FILE, report);
|
|
console.log(report);
|
|
console.log(`\nWrote comparison report to ${REPORT_FILE}`);
|
|
|
|
if (failures.length > 0) {
|
|
console.error('\nThreshold failures:\n- ' + failures.join('\n- '));
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
const isEntrypoint = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);
|
|
if (isEntrypoint) {
|
|
main().catch((error) => {
|
|
console.error('[comms:compare] failed:', error);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|