Optimize gateway comms reload behavior and strengthen regression coverage (#496)

scripts/comms/compare.mjs

@@ -0,0 +1,122 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import process from 'node:process';
+
+const ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..', '..');
+const CURRENT_FILE = path.join(ROOT, 'artifacts/comms/current-metrics.json');
+const BASELINE_FILE = path.join(ROOT, 'scripts/comms/baseline/metrics.baseline.json');
+const OUTPUT_DIR = path.join(ROOT, 'artifacts/comms');
+const REPORT_FILE = path.join(OUTPUT_DIR, 'compare-report.md');
+
+const HARD_THRESHOLDS = {
+  duplicate_event_rate: 0.005,
+  event_fanout_ratio: 1.2,
+  history_inflight_max: 1,
+  rpc_timeout_rate: 0.01,
+  message_loss_count: 0,
+  message_order_violation_count: 0,
+};
+
+const RELATIVE_THRESHOLDS = {
+  history_load_qps: 0.10,
+  rpc_p95_ms: 0.15,
+};
+
+const REQUIRED_SCENARIOS = [
+  'gateway-restart-during-run',
+  'happy-path-chat',
+  'history-overlap-guard',
+  'invalid-config-patch-recovered',
+  'multi-agent-channel-switch',
+  'network-degraded',
+];
+
+function ratioDelta(current, baseline) {
+  if (!Number.isFinite(baseline) || baseline === 0) return current === 0 ? 0 : Infinity;
+  return (current - baseline) / baseline;
+}
+
+function fmtPercent(value) {
+  return `${(value * 100).toFixed(2)}%`;
+}
+
+function fmtNumber(value) {
+  return Number.isFinite(value) ? Number(value).toFixed(4) : String(value);
+}
+
+export function evaluateReport(current, baseline) {
+  const c = current.aggregate ?? {};
+  const b = baseline.aggregate ?? {};
+  const scenarios = current.scenarios ?? {};
+  const failures = [];
+  const rows = [];
+
+  for (const scenario of REQUIRED_SCENARIOS) {
+    if (!scenarios[scenario]) {
+      failures.push(`missing scenario: ${scenario}`);
+      rows.push(`| scenario:${scenario} | missing | required | FAIL |`);
+      continue;
+    }
+    const scenarioMetrics = scenarios[scenario];
+    for (const [metric, threshold] of Object.entries(HARD_THRESHOLDS)) {
+      const cv = Number(scenarioMetrics[metric] ?? 0);
+      const pass = cv <= threshold;
+      if (!pass) failures.push(`scenario:${scenario} ${metric}=${cv} > ${threshold}`);
+      rows.push(`| ${scenario}.${metric} | ${fmtNumber(cv)} | <= ${threshold} | ${pass ? 'PASS' : 'FAIL'} |`);
+    }
+  }
+
+  for (const [metric, threshold] of Object.entries(HARD_THRESHOLDS)) {
+    const cv = Number(c[metric] ?? 0);
+    const pass = cv <= threshold;
+    if (!pass) failures.push(`${metric}=${cv} > ${threshold}`);
+    rows.push(`| ${metric} | ${fmtNumber(cv)} | <= ${threshold} | ${pass ? 'PASS' : 'FAIL'} |`);
+  }
+
+  for (const [metric, maxIncrease] of Object.entries(RELATIVE_THRESHOLDS)) {
+    const cv = Number(c[metric] ?? 0);
+    const bv = Number(b[metric] ?? 0);
+    const delta = ratioDelta(cv, bv);
+    const pass = delta <= maxIncrease;
+    if (!pass) failures.push(`${metric} delta=${delta} > ${maxIncrease}`);
+    rows.push(`| ${metric} | ${fmtNumber(cv)} (baseline ${fmtNumber(bv)}) | delta <= ${fmtPercent(maxIncrease)} | ${pass ? 'PASS' : 'FAIL'} (${fmtPercent(delta)}) |`);
+  }
+
+  return { failures, rows };
+}
+
+export async function main() {
+  const current = JSON.parse(await readFile(CURRENT_FILE, 'utf8'));
+  const baseline = JSON.parse(await readFile(BASELINE_FILE, 'utf8'));
+  const { failures, rows } = evaluateReport(current, baseline);
+
+  const report = [
+    '# Comms Regression Report',
+    '',
+    `- Generated at: ${new Date().toISOString()}`,
+    `- Result: ${failures.length === 0 ? 'PASS' : 'FAIL'}`,
+    '',
+    '| Metric | Current | Threshold | Status |',
+    '|---|---:|---:|---|',
+    ...rows,
+    '',
+  ].join('\n');
+
+  await mkdir(OUTPUT_DIR, { recursive: true });
+  await writeFile(REPORT_FILE, report);
+  console.log(report);
+  console.log(`\nWrote comparison report to ${REPORT_FILE}`);
+
+  if (failures.length > 0) {
+    console.error('\nThreshold failures:\n- ' + failures.join('\n- '));
+    process.exitCode = 1;
+  }
+}
+
+const isEntrypoint = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);
+if (isEntrypoint) {
+  main().catch((error) => {
+    console.error('[comms:compare] failed:', error);
+    process.exitCode = 1;
+  });
+}