feat: FX quorum-across-providers variance guard

docs/runbooks/README.md

@@ -21,6 +21,7 @@ New alerts MUST be added to this table before their PR merges (validated by CI).
 | `fx_provider_health_score` (degraded/demoted) | FX provider health score drops below threshold | [`fx-provider-health-scoring.md`](../fx-provider-health-scoring.md) | Backend Platform | Check provider latency/error rate; failover to secondary provider if demoted |
 | `provider_demoted` | FX provider automatically demoted | [`fx-provider-health-scoring.md`](../fx-provider-health-scoring.md) | Backend Platform | Verify secondary provider is healthy; investigate root cause; re-promote after fix |
 | `provider_degraded` | FX provider health score degraded | [`fx-provider-health-scoring.md`](../fx-provider-health-scoring.md) | Backend Platform | Monitor provider recovery; no immediate action unless persists > 15 min |
+| `fx_quorum_failed_total` | FX providers diverged beyond quorum tolerance; run blocked | [`fx-quorum-variance-guard.md`](../fx-quorum-variance-guard.md) | Backend Platform | Inspect divergent rates in pager payload; disable bad provider; page treasury if payouts mid-flight |
 | `outbox_saturation_alerts` | Outbox pending records exceed saturation threshold | [`outbox-lag-saturation-alerts.md`](../outbox-lag-saturation-alerts.md) | Backend Platform | Scale outbox consumer; verify downstream webhook endpoints; clear backpressure |
 | `migration_failed` | Database migration failed during deploy | [`payout-reconciliation.md`](payout-reconciliation.md) | Backend Platform | Check migration logs; rollback migration; notify team before retrying |
 | `migration_rolled_back` | Migration auto-rolled back after failure | [`payout-reconciliation.md`](payout-reconciliation.md) | Backend Platform | Verify schema is at previous version; investigate failure cause |

scripts/validate-alert-mappings.ts

@@ -26,6 +26,7 @@ export const KNOWN_ALERTS: AlertEntry[] = [
   { name: 'fx_provider_health_score', source: 'src/services/providerHealthScorer.ts' },
   { name: 'provider_demoted', source: 'docs/fx-provider-health-scoring.md' },
   { name: 'provider_degraded', source: 'docs/fx-provider-health-scoring.md' },
+  { name: 'fx_quorum_failed_total', source: 'src/services/fxQuorumEvaluator.ts' },
   { name: 'MultiRegionFailover', source: 'docs/runbooks/multi-region-failover.md' },
   { name: 'contract_upgrade_auto_rollback', source: 'docs/contract-upgrade-auto-rollback.md' },
   { name: 'migration_failed', source: 'src/db/migrations/safety/monitoring.ts' },

src/index.ts

@@ -1080,6 +1080,36 @@ if (require.main === module && env.NODE_ENV !== "test") {
     console.log("[server] PayoutDriftDetector started");
   }
 
+  // FX quorum variance guard (#704) — opt-in via FX_QUORUM_ENABLED=true.
+  // When enabled with at least one provider env stub, the router enforces K-of-N.
+  if (process.env.FX_QUORUM_ENABLED === 'true') {
+    try {
+      const { bootstrapFxQuorumRouter } = require('./services/fxQuorumBootstrap');
+      const { InMemoryRateProvider } = require('./services/fxConversionEngine');
+      // Placeholder providers for boot smoke; production injects real RateProviders.
+      const primary = new InMemoryRateProvider();
+      const secondary = new InMemoryRateProvider();
+      const boot = bootstrapFxQuorumRouter({
+        providers: [
+          { id: 'primary', provider: primary },
+          { id: 'secondary', provider: secondary },
+        ],
+        metrics: metricsCollector,
+        quorum: {
+          k: parseInt(process.env.FX_QUORUM_K || '2', 10),
+          tolerance: parseFloat(process.env.FX_QUORUM_TOLERANCE || '0.005'),
+        },
+      });
+      (global as any).__fxQuorumRouter = boot.router;
+      console.log('[server] FX quorum router bootstrapped', {
+        k: boot.evaluator.getConfig().k,
+        tolerance: boot.evaluator.getConfig().tolerance,
+      });
+    } catch (err) {
+      console.error('[server] FX quorum bootstrap failed', err);
+    }
+  }
+
   // --- Hot-path services (only for "api" and "all" roles) ---
 
   if (roleConfig.webhookQueue) {

src/services/fxQuorumBootstrap.test.ts

@@ -0,0 +1,49 @@
+import { bootstrapFxQuorumRouter } from './fxQuorumBootstrap';
+import { InMemoryRateProvider } from './fxConversionEngine';
+import { FxQuorumFailedError } from './fxQuorumEvaluator';
+
+function makeProvider(mid: string): InMemoryRateProvider {
+  const p = new InMemoryRateProvider();
+  p.setRateFromValues('USD/EUR', mid, mid, mid);
+  return p;
+}
+
+describe('bootstrapFxQuorumRouter', () => {
+  it('rejects empty provider lists', () => {
+    expect(() => bootstrapFxQuorumRouter({ providers: [] })).toThrow(/at least one/);
+  });
+
+  it('returns a quorum-enforcing router that pages on divergence', async () => {
+    const paged: unknown[] = [];
+    const { router } = bootstrapFxQuorumRouter({
+      providers: [
+        { id: 'a', provider: makeProvider('1.0000') },
+        { id: 'b', provider: makeProvider('1.2000') },
+      ],
+      quorum: { k: 2, tolerance: 0.005 },
+      pager: (f) => {
+        paged.push(f);
+      },
+    });
+
+    await expect(router.getRate('USD', 'EUR')).rejects.toBeInstanceOf(FxQuorumFailedError);
+    await new Promise((r) => setTimeout(r, 10));
+    expect(paged).toHaveLength(1);
+  });
+
+  it('meets quorum when one provider is down and N-1 agree', async () => {
+    const down = new InMemoryRateProvider(); // no rates → null
+    const { router } = bootstrapFxQuorumRouter({
+      providers: [
+        { id: 'a', provider: makeProvider('1.0000') },
+        { id: 'b', provider: makeProvider('1.0010') },
+        { id: 'c', provider: down },
+      ],
+      quorum: { k: 2, tolerance: 0.01, minValidProviders: 2 },
+    });
+
+    const rate = await router.getRate('USD', 'EUR');
+    expect(rate).not.toBeNull();
+    expect(Number(rate!.mid.toString())).toBeCloseTo(1.0005, 3);
+  });
+});

src/services/fxQuorumBootstrap.ts

@@ -0,0 +1,110 @@
+/**
+ * FX Quorum runtime bootstrap (#704).
+ *
+ * Constructs an `FxProviderRouter` in quorum mode with paging + audit so
+ * production no longer silently falls back to first-healthy-wins.
+ */
+
+import { Logger, globalLogger } from '../lib/logger';
+import { MetricsCollector } from '../lib/metrics';
+import { SecurityAuditRepository } from '../security/types';
+import { RateProvider } from './fxConversionEngine';
+import {
+  FxQuorumAlerting,
+  FxQuorumAssessment,
+  FxQuorumConfig,
+  FxQuorumEvaluator,
+  FxQuorumPageSink,
+} from './fxQuorumEvaluator';
+import { DEFAULT_FX_QUORUM_CONFIG } from './tenantSettingsService';
+import {
+  FxProviderRouter,
+  ProviderHealthScorer,
+  ScoredRateProvider,
+} from './providerHealthScorer';
+
+export interface FxQuorumBootstrapOptions {
+  /** Upstream rate providers (declaration order = priority). */
+  providers: Array<{ id: string; provider: RateProvider }>;
+  /** Quorum knobs; defaults to platform defaults (k=2, tolerance=0.5%). */
+  quorum?: Partial<FxQuorumConfig>;
+  metrics?: MetricsCollector;
+  logger?: Logger;
+  auditRepo?: SecurityAuditRepository;
+  /**
+   * Optional pager sink. When omitted, failures are still logged + counted;
+   * operators should wire PagerDuty here in production.
+   */
+  pager?: FxQuorumPageSink;
+  /** Tenant id used for audit annotations (optional). */
+  tenantId?: string;
+  actorId?: string;
+}
+
+export interface FxQuorumBootstrapResult {
+  router: FxProviderRouter;
+  scorer: ProviderHealthScorer;
+  evaluator: FxQuorumEvaluator;
+  alerting: FxQuorumAlerting;
+}
+
+/**
+ * Build a quorum-enforcing FX router ready for `FxConversionEngine`.
+ *
+ * @throws if fewer than one provider is supplied, or if k > n.
+ */
+export function bootstrapFxQuorumRouter(
+  options: FxQuorumBootstrapOptions
+): FxQuorumBootstrapResult {
+  if (!options.providers.length) {
+    throw new Error('bootstrapFxQuorumRouter: at least one provider is required');
+  }
+
+  const logger = options.logger ?? globalLogger;
+  const metrics = options.metrics;
+  const scorer = new ProviderHealthScorer({}, metrics, logger);
+
+  const scored = options.providers.map(
+    ({ id, provider }) => new ScoredRateProvider(id, provider, scorer)
+  );
+
+  const quorumConfig: FxQuorumConfig = {
+    ...DEFAULT_FX_QUORUM_CONFIG,
+    ...options.quorum,
+  };
+
+  const defaultPager: FxQuorumPageSink = (failure: FxQuorumAssessment) => {
+    logger.error('fx.quorum.failed — paging ops', {
+      alert: 'fx_quorum_failed_total',
+      pair: failure.pair,
+      k: failure.k,
+      valid: failure.valid,
+      inConsensus: failure.inConsensus,
+      divergent: failure.divergent,
+      runbook: 'docs/fx-quorum-variance-guard.md',
+    });
+  };
+
+  const alerting = new FxQuorumAlerting(
+    options.pager ?? defaultPager,
+    options.auditRepo,
+    { tenantId: options.tenantId, actorId: options.actorId }
+  );
+
+  const evaluator = new FxQuorumEvaluator(quorumConfig, {
+    metrics,
+    logger,
+    pager: (failure) => alerting.handle(failure),
+  });
+
+  const router = new FxProviderRouter(scored, scorer, evaluator);
+
+  logger.info('FX quorum router bootstrapped', {
+    providerCount: scored.length,
+    k: quorumConfig.k,
+    tolerance: quorumConfig.tolerance,
+    reference: quorumConfig.reference ?? 'median',
+  });
+
+  return { router, scorer, evaluator, alerting };
+}