From fa6be0b292e9d4491b2ce1308b9030760b693fa6 Mon Sep 17 00:00:00 2001 From: laurentketterle-hub Date: Sun, 2 Aug 2026 06:59:10 +0200 Subject: [PATCH 1/2] docs: add scheduler jitter/anti-thundering-herd architecture --- docs/architecture/scheduler-jitter.md | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/architecture/scheduler-jitter.md diff --git a/docs/architecture/scheduler-jitter.md b/docs/architecture/scheduler-jitter.md new file mode 100644 index 00000000..57465bf8 --- /dev/null +++ b/docs/architecture/scheduler-jitter.md @@ -0,0 +1,58 @@ +# Scheduler Jitter / Anti-Thundering-Herd + +## Problem +When multiple offering distribution ticks are scheduled at the same wall-clock time, all workers fire simultaneously, causing: +- Database connection pool exhaustion +- Rate-limit contention on external APIs +- Uneven system load with burst/starve pattern + +## Solution: Jittered Scheduling + +### Implementation +```typescript +interface SchedulerConfig { + /** Base cron expression for the tick */ + cronExpression: string + /** Maximum random delay in milliseconds (default: 5000) */ + maxJitterMs: number + /** Minimum spacing between ticks in ms to avoid herd (default: 1000) */ + minSpacingMs: number +} + +class JitteredScheduler { + private offeringTicks: Map + private jitterProvider: () => number // injectable for testing + + constructor(config: SchedulerConfig) { + this.jitterProvider = () => Math.floor(Math.random() * config.maxJitterMs) + } + + async scheduleTick(offeringId: string, handler: () => Promise): Promise { + const jitter = this.jitterProvider() + setTimeout(async () => { + await handler() + }, jitter) + } + + async scheduleMultiOffering(offeringIds: string[], handler: (id: string) => Promise): Promise { + // Stagger ticks to avoid thundering herd + for (let i = 0; i < offeringIds.length; i++) { + const stagger = i * this.config.minSpacingMs + const jitter = this.jitterProvider() + setTimeout(async () => { + await handler(offeringIds[i]) + }, stagger + jitter) + } + } +} +``` + +### Benefits +- **Reduced DB contention**: Staggered writes avoid lock contention +- **Predictable load**: Even CPU/memory utilization across tick windows +- **Resilience**: Jitter prevents cascading failures from simultaneous retries + +### Testing +- **Deterministic jitter**: Inject fixed jitter provider for reproducible tests +- **Concurrency stress**: Test with 100+ offerings at same cron tick +- **Recovery**: Verify jittered retry on failure From e49a3150c59db00296212f9cb221139194bcd5fe Mon Sep 17 00:00:00 2001 From: laurentketterle-hub Date: Sun, 2 Aug 2026 06:59:11 +0200 Subject: [PATCH 2/2] docs: add scheduler jitter test plan --- docs/testing/scheduler-jitter-test-plan.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/testing/scheduler-jitter-test-plan.md diff --git a/docs/testing/scheduler-jitter-test-plan.md b/docs/testing/scheduler-jitter-test-plan.md new file mode 100644 index 00000000..927f735b --- /dev/null +++ b/docs/testing/scheduler-jitter-test-plan.md @@ -0,0 +1,17 @@ +# Scheduler Jitter Test Plan + +## Unit Tests +- [ ] `JitteredScheduler.scheduleTick()` respects maxJitterMs bound +- [ ] `JitteredScheduler.scheduleMultiOffering()` respects minSpacingMs between ticks +- [ ] Jitter provider is injectable for deterministic testing +- [ ] Negative jitter values are clamped to 0 +- [ ] Zero maxJitterMs produces immediate execution (no delay) + +## Integration Tests +- [ ] 50 simultaneous offering ticks with 5s maxJitter complete within maxJitterMs + tolerance +- [ ] No two ticks fire within minSpacingMs of each other +- [ ] Database connection pool never exceeds max during jittered batch + +## Stress Tests +- [ ] 200 offerings at same cron tick complete without connection exhaustion +- [ ] Jittered scheduler recovers from handler failure without affecting other ticks