From 73689ccf7f156cd4ef370024e2c3670530a6a4f7 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 13:19:06 -0500 Subject: [PATCH 01/11] fix(ensindexer): make EnsDbWriterWorker hot-reload safe Ponder's dev-mode hot reload re-executes the API entry file but caches transitively-imported modules, leaving module-level singleton state stale. On reload, startEnsDbWriterWorker() would throw "EnsDbWriterWorker has already been initialized" and kill the process. - local-ponder-context.ts is now a reactive proxy that re-reads globalThis.PONDER_COMMON.apiShutdown (and .shutdown) on every access. Ponder kills and replaces these managers on each reload (ponder/src/bin/commands/dev.ts:95-101), so cached references go stale immediately. - startEnsDbWriterWorker() is reset-aware: awaits any prior worker's stop(), registers cleanup via apiShutdown.add(), and ignores AbortError in the .run().catch() path so shutdown does not kill the process. - EnsDbWriterWorker.stop() is async and awaits any in-flight snapshot upsert, so Ponder's shutdown sequence can wait on it. - PonderClient/LocalPonderClient accept an optional getAbortSignal() getter, invoked at fetch time, so HTTP requests use the current signal instead of a captured-at-construct reference that goes stale per reload. Resolves #1432. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ensdb-writer-worker.ts | 23 +++-- .../src/lib/ensdb-writer-worker/singleton.ts | 74 +++++++++++--- .../ensindexer/src/lib/local-ponder-client.ts | 3 + .../src/lib/local-ponder-context.ts | 97 ++++++++++++++++++- packages/ponder-sdk/src/client.ts | 18 +++- .../ponder-sdk/src/local-ponder-client.ts | 4 +- 6 files changed, 189 insertions(+), 30 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index 38aa6d12de..064bdd125d 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -37,6 +37,12 @@ export class EnsDbWriterWorker { */ private indexingStatusInterval: ReturnType | null = null; + /** + * Tracks the most recently launched snapshot upsert so that {@link stop} + * can wait for any in-flight work to settle before returning. + */ + private inFlightSnapshot: Promise | undefined; + /** * ENSDb Client instance used by the worker to interact with ENSDb. */ @@ -121,10 +127,9 @@ export class EnsDbWriterWorker { }); // Task 3: recurring upsert of Indexing Status Snapshot into ENSDb. - this.indexingStatusInterval = setInterval( - () => this.upsertIndexingStatusSnapshot(), - secondsToMilliseconds(INDEXING_STATUS_RECORD_UPDATE_INTERVAL), - ); + this.indexingStatusInterval = setInterval(() => { + this.inFlightSnapshot = this.upsertIndexingStatusSnapshot(); + }, secondsToMilliseconds(INDEXING_STATUS_RECORD_UPDATE_INTERVAL)); } /** @@ -137,13 +142,19 @@ export class EnsDbWriterWorker { /** * Stop the ENSDb Writer Worker * - * Stops all recurring tasks in the worker. + * Stops all recurring tasks in the worker and waits for any in-flight + * snapshot upsert to settle. Safe to call when not running. */ - public stop(): void { + public async stop(): Promise { if (this.indexingStatusInterval) { clearInterval(this.indexingStatusInterval); this.indexingStatusInterval = null; } + if (this.inFlightSnapshot) { + // Errors are already logged inside upsertIndexingStatusSnapshot; swallow here. + await this.inFlightSnapshot.catch(() => {}); + this.inFlightSnapshot = undefined; + } } /** diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index 22fd6a5e9b..c0d9873abd 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -1,40 +1,86 @@ import { ensDbClient } from "@/lib/ensdb/singleton"; import { indexingStatusBuilder } from "@/lib/indexing-status-builder/singleton"; import { localPonderClient } from "@/lib/local-ponder-client"; +import { localPonderContext } from "@/lib/local-ponder-context"; import { logger } from "@/lib/logger"; import { publicConfigBuilder } from "@/lib/public-config-builder/singleton"; import { EnsDbWriterWorker } from "./ensdb-writer-worker"; -let ensDbWriterWorker: EnsDbWriterWorker; +let ensDbWriterWorker: EnsDbWriterWorker | undefined; + +function isAbortError(error: unknown): boolean { + return error instanceof Error && error.name === "AbortError"; +} /** - * Starts the EnsDbWriterWorker in a new asynchronous context. + * Start (or restart) the EnsDbWriterWorker. * - * The worker will run indefinitely until it is stopped via {@link EnsDbWriterWorker.stop}, - * for example in response to a process termination signal or an internal error, at - * which point it will attempt to gracefully shut down. + * Called from `apps/ensindexer/ponder/src/api/index.ts` on every Ponder + * API exec. Ponder re-executes the API entry file on hot reload, but this + * module is cached by vite-node, so module-level state survives across + * reloads. This function therefore must: * - * @throws Error if the worker is already running when this function is called. + * 1. Be idempotent — treat a re-call as "the previous instance is dead, + * replace it" rather than throwing. + * 2. Re-bind reload-scoped resources (e.g. `apiShutdown`) fresh from + * `localPonderContext` on every call. Never hoist them to module + * scope. See `local-ponder-context.ts` for the staleness contract. */ -export function startEnsDbWriterWorker() { - if (typeof ensDbWriterWorker !== "undefined") { - throw new Error("EnsDbWriterWorker has already been initialized"); +export async function startEnsDbWriterWorker(): Promise { + // Defensively reset any prior instance. The apiShutdown.add() callback + // from the previous API exec is the primary cleanup path on hot reload; + // this is a safety net for cases where the callback didn't run (e.g. + // unexpected shutdown ordering). + if (ensDbWriterWorker) { + await ensDbWriterWorker.stop(); + ensDbWriterWorker = undefined; } - ensDbWriterWorker = new EnsDbWriterWorker( + const worker = new EnsDbWriterWorker( ensDbClient, publicConfigBuilder, indexingStatusBuilder, localPonderClient, ); + ensDbWriterWorker = worker; + + // Read apiShutdown FRESH from the reactive context. Ponder kills and + // replaces this on every dev-mode hot reload, so this read MUST happen + // inside the function call (not at module scope). + const apiShutdown = localPonderContext.apiShutdown; - ensDbWriterWorker + apiShutdown.add(async () => { + logger.info({ + msg: "Stopping EnsDbWriterWorker due to API shutdown", + module: "EnsDbWriterWorker", + }); + await worker.stop(); + if (ensDbWriterWorker === worker) { + ensDbWriterWorker = undefined; + } + }); + + worker .run() // Handle any uncaught errors from the worker - .catch((error) => { - // Abort the worker on error to trigger cleanup - ensDbWriterWorker.stop(); + .catch(async (error) => { + // If Ponder has begun shutting down our API instance (hot reload or + // graceful shutdown), the abort propagates through in-flight fetches + // as an AbortError. Treat that as a clean stop, not a worker failure. + if (apiShutdown.abortController.signal.aborted || isAbortError(error)) { + logger.info({ + msg: "EnsDbWriterWorker stopped due to API shutdown", + module: "EnsDbWriterWorker", + }); + return; + } + + // Real worker error — clean up and trigger non-zero exit. + await worker.stop(); + if (ensDbWriterWorker === worker) { + ensDbWriterWorker = undefined; + } logger.error({ msg: "EnsDbWriterWorker encountered an error", diff --git a/apps/ensindexer/src/lib/local-ponder-client.ts b/apps/ensindexer/src/lib/local-ponder-client.ts index fcfe293a92..93edf5d366 100644 --- a/apps/ensindexer/src/lib/local-ponder-client.ts +++ b/apps/ensindexer/src/lib/local-ponder-client.ts @@ -17,4 +17,7 @@ export const localPonderClient = new LocalPonderClient( indexedBlockranges, publicClients, localPonderContext, + // Reload-scoped: read fresh on every fetch via the reactive proxy. See + // local-ponder-context.ts for the staleness contract. + () => localPonderContext.apiShutdown.abortController.signal, ); diff --git a/apps/ensindexer/src/lib/local-ponder-context.ts b/apps/ensindexer/src/lib/local-ponder-context.ts index ed66a40883..7e4db6047e 100644 --- a/apps/ensindexer/src/lib/local-ponder-context.ts +++ b/apps/ensindexer/src/lib/local-ponder-context.ts @@ -1,14 +1,101 @@ import { deserializePonderAppContext, type PonderAppContext } from "@ensnode/ponder-sdk"; +/** + * Local Ponder Context — reactive wrapper over Ponder's runtime globals. + * + * Why this is a Proxy and not an eagerly-deserialized object: + * + * Ponder's dev mode hot-reloads the API entry file by re-executing it via + * vite-node. On every indexing-file change, Ponder ALSO kills and replaces + * `common.shutdown` and `common.apiShutdown` on `globalThis.PONDER_COMMON` + * (see `ponder/src/bin/commands/dev.ts:95-101`). Modules in our API-side + * dependency graph (this file included) are NOT re-evaluated when only an + * indexing file changes — vite-node only invalidates the changed file's + * dep tree. So any value cached in a module-level closure during the + * original boot becomes stale on the very next reload. + * + * Stable fields (`command`, `localPonderAppUrl`, `logger`) are validated + * once and memoized — Ponder does not mutate `options` or `logger` on + * reload. Reload-scoped fields (`apiShutdown`, `shutdown`) MUST be re-read + * from `globalThis.PONDER_COMMON` on every access. + * + * Contract for callers: NEVER cache reload-scoped fields in a module-level + * closure or capture them in a constructor argument. Always reach for them + * via `localPonderContext.` from code that runs per-reload (e.g. the + * API entry file or per-request handlers). If you need an `AbortSignal` + * across calls, store a getter (`() => localPonderContext.apiShutdown + * .abortController.signal`), not the signal itself. + */ + if (!globalThis.PONDER_COMMON) { throw new Error("PONDER_COMMON must be defined by Ponder at runtime as a global variable."); } /** - * Local Ponder app context + * Ponder shutdown manager runtime shape. * - * Represents the {@link PonderAppContext} object provided by Ponder runtime to - * the local Ponder app. Useful for accessing internal Ponder app configuration - * and utilities such as the logger. + * Mirrors `ponder/src/internal/shutdown.ts` — the object Ponder publishes + * on `globalThis.PONDER_COMMON.{shutdown,apiShutdown}`. */ -export const localPonderContext = deserializePonderAppContext(globalThis.PONDER_COMMON); +export interface PonderAppShutdownManager { + add: (callback: () => undefined | Promise) => void; + isKilled: boolean; + abortController: AbortController; +} + +function isPonderAppShutdownManager(value: unknown): value is PonderAppShutdownManager { + if (typeof value !== "object" || value === null) return false; + const obj = value as Record; + return ( + typeof obj.add === "function" && + typeof obj.isKilled === "boolean" && + obj.abortController instanceof AbortController + ); +} + +function readShutdownManager(field: "apiShutdown" | "shutdown"): PonderAppShutdownManager { + const raw = (globalThis.PONDER_COMMON as Record | undefined)?.[field]; + if (!isPonderAppShutdownManager(raw)) { + throw new Error(`globalThis.PONDER_COMMON.${field} is not a valid Ponder shutdown manager.`); + } + return raw; +} + +let cachedStableContext: PonderAppContext | undefined; +function getStableContext(): PonderAppContext { + if (!cachedStableContext) { + if (!globalThis.PONDER_COMMON) { + throw new Error("PONDER_COMMON must be defined by Ponder at runtime as a global variable."); + } + cachedStableContext = deserializePonderAppContext(globalThis.PONDER_COMMON); + } + return cachedStableContext; +} + +/** + * Local Ponder Context. + * + * Combines stable {@link PonderAppContext} fields with reload-scoped + * shutdown managers. See module-level comment for the staleness contract. + */ +export interface LocalPonderContext extends PonderAppContext { + /** + * The current `apiShutdown` manager. RELOAD-SCOPED — identity changes + * every API hot-reload. Always read fresh; never cache. + */ + readonly apiShutdown: PonderAppShutdownManager; + + /** + * The current `shutdown` manager. RELOAD-SCOPED — identity changes + * every indexing hot-reload. Always read fresh; never cache. + */ + readonly shutdown: PonderAppShutdownManager; +} + +export const localPonderContext: LocalPonderContext = new Proxy({} as LocalPonderContext, { + get(_target, prop) { + if (prop === "apiShutdown") return readShutdownManager("apiShutdown"); + if (prop === "shutdown") return readShutdownManager("shutdown"); + return getStableContext()[prop as keyof PonderAppContext]; + }, +}); diff --git a/packages/ponder-sdk/src/client.ts b/packages/ponder-sdk/src/client.ts index 84d740f75a..af6243f6fd 100644 --- a/packages/ponder-sdk/src/client.ts +++ b/packages/ponder-sdk/src/client.ts @@ -5,9 +5,19 @@ import type { PonderIndexingStatus } from "./indexing-status"; /** * PonderClient for fetching data from Ponder apps. + * + * The optional `getAbortSignal` is invoked at fetch time so each request + * uses the current `AbortSignal`. Passing a getter (instead of a captured + * `AbortSignal`) is required for consumers that need to track signals + * which change identity over the client's lifetime — e.g. signals derived + * from Ponder's `apiShutdown` manager, which Ponder kills and replaces on + * every dev-mode hot reload. */ export class PonderClient { - constructor(private readonly baseUrl: URL) {} + constructor( + private readonly baseUrl: URL, + private readonly getAbortSignal?: () => AbortSignal | undefined, + ) {} /** * Check Ponder Health @@ -18,7 +28,7 @@ export class PonderClient { */ async health(): Promise { const requestUrl = new URL("/health", this.baseUrl); - const response = await fetch(requestUrl); + const response = await fetch(requestUrl, { signal: this.getAbortSignal?.() }); if (!response.ok) { throw new Error( @@ -35,7 +45,7 @@ export class PonderClient { */ async metrics(): Promise { const requestUrl = new URL("/metrics", this.baseUrl); - const response = await fetch(requestUrl); + const response = await fetch(requestUrl, { signal: this.getAbortSignal?.() }); if (!response.ok) { throw new Error( @@ -56,7 +66,7 @@ export class PonderClient { */ async status(): Promise { const requestUrl = new URL("/status", this.baseUrl); - const response = await fetch(requestUrl); + const response = await fetch(requestUrl, { signal: this.getAbortSignal?.() }); if (!response.ok) { throw new Error( diff --git a/packages/ponder-sdk/src/local-ponder-client.ts b/packages/ponder-sdk/src/local-ponder-client.ts index 0e8e786c5e..d6745d4259 100644 --- a/packages/ponder-sdk/src/local-ponder-client.ts +++ b/packages/ponder-sdk/src/local-ponder-client.ts @@ -82,14 +82,16 @@ export class LocalPonderClient extends PonderClient { * @param ponderPublicClients All cached public clients provided by the local Ponder app * (may include non-indexed chains). * @param ponderAppContext The internal context of the local Ponder app. + * @param getAbortSignal Optional getter invoked at fetch time to attach an `AbortSignal` to outgoing HTTP requests. Use a getter (not a captured signal) when the underlying signal can change identity over time — e.g. across Ponder dev-mode hot reloads. */ constructor( indexedChainIds: Set, indexedBlockranges: Map, ponderPublicClients: Record, ponderAppContext: PonderAppContext, + getAbortSignal?: () => AbortSignal | undefined, ) { - super(ponderAppContext.localPonderAppUrl); + super(ponderAppContext.localPonderAppUrl, getAbortSignal); this.indexedChainIds = indexedChainIds; From 31604d98fc47bf5336a85e6c0a40a656703161b2 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 13:34:58 -0500 Subject: [PATCH 02/11] add changeset Co-Authored-By: Claude Opus 4.6 (1M context) --- .changeset/fix-ensindexer-hot-reload.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/fix-ensindexer-hot-reload.md diff --git a/.changeset/fix-ensindexer-hot-reload.md b/.changeset/fix-ensindexer-hot-reload.md new file mode 100644 index 0000000000..0d7354b2d4 --- /dev/null +++ b/.changeset/fix-ensindexer-hot-reload.md @@ -0,0 +1,6 @@ +--- +"ensindexer": patch +"@ensnode/ponder-sdk": patch +--- + +ENSIndexer in dev mode no longer crashes during hot reloading due to EnsDbWriterWorker failure. From d80e3d4f7c4584bfe1d958fc100d3dad29c4a0ae Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 13:54:19 -0500 Subject: [PATCH 03/11] fix(ensindexer): address PR review feedback - await worker.stop() in all ensdb-writer-worker tests (stop() is async) - skip overlapping snapshot upserts via inFlightSnapshot guard so a slow upsert can't pile up concurrent ENSDb writes; stop() awaits the single in-flight upsert deterministically - thread an optional AbortSignal into worker.run() and check between await steps so a hot reload during run() startup bails before the recurring interval is scheduled (closes the startup race) - extract gracefulShutdown helper in singleton.ts shared by the apiShutdown.add() callback and the .run().catch() paths - add PonderClient unit tests asserting the getAbortSignal getter is invoked per-fetch, returns fresh identity across calls, and cancels in-flight fetches when aborted Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ensdb-writer-worker.test.ts | 18 ++--- .../ensdb-writer-worker.ts | 23 +++++- .../src/lib/ensdb-writer-worker/singleton.ts | 47 +++++------ packages/ponder-sdk/src/client.test.ts | 78 +++++++++++++++++++ 4 files changed, 131 insertions(+), 35 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts index ba0f0bee5b..944250e2db 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts @@ -78,7 +78,7 @@ describe("EnsDbWriterWorker", () => { ); // cleanup - worker.stop(); + await worker.stop(); }); it("throws when stored config is incompatible", async () => { @@ -129,7 +129,7 @@ describe("EnsDbWriterWorker", () => { expect(ensDbClient.upsertEnsIndexerPublicConfig).toHaveBeenCalledWith(mockPublicConfig); // cleanup - worker.stop(); + await worker.stop(); }); it("throws error when worker is already running", async () => { @@ -143,7 +143,7 @@ describe("EnsDbWriterWorker", () => { await expect(worker.run()).rejects.toThrow("EnsDbWriterWorker is already running"); // cleanup - worker.stop(); + await worker.stop(); }); it("throws error when config fetch fails", async () => { @@ -193,7 +193,7 @@ describe("EnsDbWriterWorker", () => { expect(publicConfigBuilder.getPublicConfig).toHaveBeenCalledTimes(1); // cleanup - worker.stop(); + await worker.stop(); }); it("calls pRetry for config fetch with retry logic", async () => { @@ -213,7 +213,7 @@ describe("EnsDbWriterWorker", () => { expect(ensDbClient.upsertEnsIndexerPublicConfig).toHaveBeenCalledWith(mockPublicConfig); // cleanup - worker.stop(); + await worker.stop(); }); }); @@ -230,7 +230,7 @@ describe("EnsDbWriterWorker", () => { const callCountBeforeStop = upsertIndexingStatusSnapshot.mock.calls.length; - worker.stop(); + await worker.stop(); // advance time after stop await vi.advanceTimersByTimeAsync(2000); @@ -255,7 +255,7 @@ describe("EnsDbWriterWorker", () => { expect(worker.isRunning).toBe(true); // act - stop worker - worker.stop(); + await worker.stop(); // assert - not running after stop expect(worker.isRunning).toBe(false); @@ -303,7 +303,7 @@ describe("EnsDbWriterWorker", () => { expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledWith(crossChainSnapshot); // cleanup - worker.stop(); + await worker.stop(); }); it("recovers from errors and continues upserting snapshots", async () => { @@ -361,7 +361,7 @@ describe("EnsDbWriterWorker", () => { expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(3); // cleanup - worker.stop(); + await worker.stop(); }); }); }); diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index 064bdd125d..ce46bf08cd 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -93,22 +93,32 @@ export class EnsDbWriterWorker { * 3) A recurring attempt to upsert serialized representation of * {@link CrossChainIndexingStatusSnapshot} into ENSDb. * + * @param signal Optional AbortSignal that, if aborted, causes `run` to bail + * between its async setup steps. Use this to prevent the worker + * from finishing initialization (and starting the recurring + * interval) after the surrounding API instance has begun + * shutting down. * @throws Error if the worker is already running, or * if the in-memory ENSIndexer Public Config could not be fetched, or - * if the in-memory ENSIndexer Public Config is incompatible with the stored config in ENSDb. + * if the in-memory ENSIndexer Public Config is incompatible with the stored config in ENSDb, or + * if `signal` is aborted before the recurring interval is scheduled. */ - public async run(): Promise { + public async run(signal?: AbortSignal): Promise { // Do not allow multiple concurrent runs of the worker if (this.isRunning) { throw new Error("EnsDbWriterWorker is already running"); } + signal?.throwIfAborted(); + // Fetch data required for task 1 and task 2. const inMemoryConfig = await this.getValidatedEnsIndexerPublicConfig(); + signal?.throwIfAborted(); // Task 1: upsert ENSDb version into ENSDb. logger.debug({ msg: "Upserting ENSDb version", module: "EnsDbWriterWorker" }); await this.ensDbClient.upsertEnsDbVersion(inMemoryConfig.versionInfo.ensDb); + signal?.throwIfAborted(); logger.info({ msg: "Upserted ENSDb version", ensDbVersion: inMemoryConfig.versionInfo.ensDb, @@ -121,14 +131,21 @@ export class EnsDbWriterWorker { module: "EnsDbWriterWorker", }); await this.ensDbClient.upsertEnsIndexerPublicConfig(inMemoryConfig); + signal?.throwIfAborted(); logger.info({ msg: "Upserted ENSIndexer public config", module: "EnsDbWriterWorker", }); // Task 3: recurring upsert of Indexing Status Snapshot into ENSDb. + // Skip overlapping ticks so a slow upsert can't pile up concurrent + // ENSDb writes. With skip-overlap there is at most one in-flight + // upsert at a time, which `stop()` then has a single promise to await. this.indexingStatusInterval = setInterval(() => { - this.inFlightSnapshot = this.upsertIndexingStatusSnapshot(); + if (this.inFlightSnapshot) return; + this.inFlightSnapshot = this.upsertIndexingStatusSnapshot().finally(() => { + this.inFlightSnapshot = undefined; + }); }, secondsToMilliseconds(INDEXING_STATUS_RECORD_UPDATE_INTERVAL)); } diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index c0d9873abd..53cd437db6 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -13,6 +13,21 @@ function isAbortError(error: unknown): boolean { return error instanceof Error && error.name === "AbortError"; } +/** + * Stop the given worker (if it is still the active singleton) and clear the + * singleton reference. Safe to call multiple times. + */ +async function gracefulShutdown(worker: EnsDbWriterWorker, reason: string): Promise { + logger.info({ + msg: `Stopping EnsDbWriterWorker: ${reason}`, + module: "EnsDbWriterWorker", + }); + await worker.stop(); + if (ensDbWriterWorker === worker) { + ensDbWriterWorker = undefined; + } +} + /** * Start (or restart) the EnsDbWriterWorker. * @@ -33,8 +48,7 @@ export async function startEnsDbWriterWorker(): Promise { // this is a safety net for cases where the callback didn't run (e.g. // unexpected shutdown ordering). if (ensDbWriterWorker) { - await ensDbWriterWorker.stop(); - ensDbWriterWorker = undefined; + await gracefulShutdown(ensDbWriterWorker, "stale instance from previous API exec"); } const worker = new EnsDbWriterWorker( @@ -49,38 +63,25 @@ export async function startEnsDbWriterWorker(): Promise { // replaces this on every dev-mode hot reload, so this read MUST happen // inside the function call (not at module scope). const apiShutdown = localPonderContext.apiShutdown; + const abortSignal = apiShutdown.abortController.signal; - apiShutdown.add(async () => { - logger.info({ - msg: "Stopping EnsDbWriterWorker due to API shutdown", - module: "EnsDbWriterWorker", - }); - await worker.stop(); - if (ensDbWriterWorker === worker) { - ensDbWriterWorker = undefined; - } - }); + apiShutdown.add(() => gracefulShutdown(worker, "API shutdown")); worker - .run() + .run(abortSignal) // Handle any uncaught errors from the worker .catch(async (error) => { // If Ponder has begun shutting down our API instance (hot reload or // graceful shutdown), the abort propagates through in-flight fetches - // as an AbortError. Treat that as a clean stop, not a worker failure. - if (apiShutdown.abortController.signal.aborted || isAbortError(error)) { - logger.info({ - msg: "EnsDbWriterWorker stopped due to API shutdown", - module: "EnsDbWriterWorker", - }); + // (or `signal.throwIfAborted()`) as an AbortError. Treat that as a + // clean stop, not a worker failure. + if (abortSignal.aborted || isAbortError(error)) { + await gracefulShutdown(worker, "API shutdown (run aborted)"); return; } // Real worker error — clean up and trigger non-zero exit. - await worker.stop(); - if (ensDbWriterWorker === worker) { - ensDbWriterWorker = undefined; - } + await gracefulShutdown(worker, "uncaught error"); logger.error({ msg: "EnsDbWriterWorker encountered an error", diff --git a/packages/ponder-sdk/src/client.test.ts b/packages/ponder-sdk/src/client.test.ts index 3b4bd73d44..043c243bb9 100644 --- a/packages/ponder-sdk/src/client.test.ts +++ b/packages/ponder-sdk/src/client.test.ts @@ -322,4 +322,82 @@ describe("Ponder Client", () => { }); }); }); + + describe("getAbortSignal getter", () => { + it("invokes getAbortSignal on every fetch and forwards the signal", async () => { + // Arrange + mockFetch.mockResolvedValue(new Response(null, { status: 200 })); + const signal = new AbortController().signal; + const getAbortSignal = vi.fn(() => signal); + const ponderClient = new PonderClient(new URL("http://localhost:3000"), getAbortSignal); + + // Act + await ponderClient.health(); + await ponderClient.health(); + + // Assert + expect(getAbortSignal).toHaveBeenCalledTimes(2); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch).toHaveBeenNthCalledWith(1, expect.any(URL), { signal }); + expect(mockFetch).toHaveBeenNthCalledWith(2, expect.any(URL), { signal }); + }); + + it("re-reads the signal between fetches so callers get fresh identity", async () => { + // Arrange + mockFetch.mockResolvedValue(new Response(null, { status: 200 })); + const firstSignal = new AbortController().signal; + const secondSignal = new AbortController().signal; + const getAbortSignal = vi + .fn<() => AbortSignal | undefined>() + .mockReturnValueOnce(firstSignal) + .mockReturnValueOnce(secondSignal); + const ponderClient = new PonderClient(new URL("http://localhost:3000"), getAbortSignal); + + // Act + await ponderClient.health(); + await ponderClient.health(); + + // Assert + expect(mockFetch).toHaveBeenNthCalledWith(1, expect.any(URL), { signal: firstSignal }); + expect(mockFetch).toHaveBeenNthCalledWith(2, expect.any(URL), { signal: secondSignal }); + }); + + it("aborting the signal cancels in-flight fetches", async () => { + // Arrange + const abortController = new AbortController(); + mockFetch.mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + }), + ); + const ponderClient = new PonderClient( + new URL("http://localhost:3000"), + () => abortController.signal, + ); + + // Act + const pending = ponderClient.health(); + abortController.abort(); + + // Assert + await expect(pending).rejects.toThrowError(/aborted/); + }); + + it("treats getAbortSignal as optional (undefined signal → no abort)", async () => { + // Arrange + mockFetch.mockResolvedValueOnce(new Response(null, { status: 200 })); + const ponderClient = new PonderClient(new URL("http://localhost:3000")); + + // Act + await ponderClient.health(); + + // Assert + expect(mockFetch).toHaveBeenCalledWith(expect.any(URL), { signal: undefined }); + }); + }); }); From 1617314fdaaf9e5235a22e2187a356f8980b4ba0 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 14:01:27 -0500 Subject: [PATCH 04/11] refactor(ensindexer): cleanups from simplify pass - move PonderAppShutdownManager type + isPonderAppShutdownManager guard to @ensnode/ponder-sdk where they sit alongside PonderAppContext, since they mirror a Ponder runtime type. local-ponder-context.ts now imports them instead of redeclaring. - introduce AbortSignalGetter type alias in @ensnode/ponder-sdk so PonderClient and LocalPonderClient share one definition instead of three inline spellings. - isAbortError: check error.name structurally instead of via instanceof Error so DOMException AbortErrors (the actual rejection type from fetch aborts) are caught. - localPonderContext Proxy: ignore symbol property reads so well-known symbols (Symbol.toPrimitive, etc.) don't blow past the keyof cast. - trim a duplicate staleness-contract comment in local-ponder-client.ts. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/singleton.ts | 8 ++++- .../ensindexer/src/lib/local-ponder-client.ts | 3 +- .../src/lib/local-ponder-context.ts | 30 +++++-------------- packages/ponder-sdk/src/client.ts | 15 ++++++---- .../ponder-sdk/src/local-ponder-client.ts | 6 ++-- packages/ponder-sdk/src/ponder-app-context.ts | 25 ++++++++++++++++ 6 files changed, 52 insertions(+), 35 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index 53cd437db6..f5be927c46 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -10,7 +10,13 @@ import { EnsDbWriterWorker } from "./ensdb-writer-worker"; let ensDbWriterWorker: EnsDbWriterWorker | undefined; function isAbortError(error: unknown): boolean { - return error instanceof Error && error.name === "AbortError"; + // `fetch` aborts reject with a `DOMException` whose `name === "AbortError"`, + // which is not always `instanceof Error` across runtimes. Check by name. + return ( + typeof error === "object" && + error !== null && + (error as { name?: unknown }).name === "AbortError" + ); } /** diff --git a/apps/ensindexer/src/lib/local-ponder-client.ts b/apps/ensindexer/src/lib/local-ponder-client.ts index 93edf5d366..b9a711b4ad 100644 --- a/apps/ensindexer/src/lib/local-ponder-client.ts +++ b/apps/ensindexer/src/lib/local-ponder-client.ts @@ -17,7 +17,6 @@ export const localPonderClient = new LocalPonderClient( indexedBlockranges, publicClients, localPonderContext, - // Reload-scoped: read fresh on every fetch via the reactive proxy. See - // local-ponder-context.ts for the staleness contract. + // See local-ponder-context.ts for the staleness contract. () => localPonderContext.apiShutdown.abortController.signal, ); diff --git a/apps/ensindexer/src/lib/local-ponder-context.ts b/apps/ensindexer/src/lib/local-ponder-context.ts index 7e4db6047e..01fded5683 100644 --- a/apps/ensindexer/src/lib/local-ponder-context.ts +++ b/apps/ensindexer/src/lib/local-ponder-context.ts @@ -1,4 +1,9 @@ -import { deserializePonderAppContext, type PonderAppContext } from "@ensnode/ponder-sdk"; +import { + deserializePonderAppContext, + isPonderAppShutdownManager, + type PonderAppContext, + type PonderAppShutdownManager, +} from "@ensnode/ponder-sdk"; /** * Local Ponder Context — reactive wrapper over Ponder's runtime globals. @@ -31,28 +36,6 @@ if (!globalThis.PONDER_COMMON) { throw new Error("PONDER_COMMON must be defined by Ponder at runtime as a global variable."); } -/** - * Ponder shutdown manager runtime shape. - * - * Mirrors `ponder/src/internal/shutdown.ts` — the object Ponder publishes - * on `globalThis.PONDER_COMMON.{shutdown,apiShutdown}`. - */ -export interface PonderAppShutdownManager { - add: (callback: () => undefined | Promise) => void; - isKilled: boolean; - abortController: AbortController; -} - -function isPonderAppShutdownManager(value: unknown): value is PonderAppShutdownManager { - if (typeof value !== "object" || value === null) return false; - const obj = value as Record; - return ( - typeof obj.add === "function" && - typeof obj.isKilled === "boolean" && - obj.abortController instanceof AbortController - ); -} - function readShutdownManager(field: "apiShutdown" | "shutdown"): PonderAppShutdownManager { const raw = (globalThis.PONDER_COMMON as Record | undefined)?.[field]; if (!isPonderAppShutdownManager(raw)) { @@ -94,6 +77,7 @@ export interface LocalPonderContext extends PonderAppContext { export const localPonderContext: LocalPonderContext = new Proxy({} as LocalPonderContext, { get(_target, prop) { + if (typeof prop === "symbol") return undefined; if (prop === "apiShutdown") return readShutdownManager("apiShutdown"); if (prop === "shutdown") return readShutdownManager("shutdown"); return getStableContext()[prop as keyof PonderAppContext]; diff --git a/packages/ponder-sdk/src/client.ts b/packages/ponder-sdk/src/client.ts index af6243f6fd..6de4e30bb8 100644 --- a/packages/ponder-sdk/src/client.ts +++ b/packages/ponder-sdk/src/client.ts @@ -4,19 +4,22 @@ import type { PonderIndexingMetrics } from "./indexing-metrics"; import type { PonderIndexingStatus } from "./indexing-status"; /** - * PonderClient for fetching data from Ponder apps. + * Returns the current `AbortSignal` to attach to outgoing requests. * - * The optional `getAbortSignal` is invoked at fetch time so each request - * uses the current `AbortSignal`. Passing a getter (instead of a captured - * `AbortSignal`) is required for consumers that need to track signals - * which change identity over the client's lifetime — e.g. signals derived + * Consumers must use a getter (not a captured `AbortSignal`) when the + * underlying signal can change identity over time — e.g. signals derived * from Ponder's `apiShutdown` manager, which Ponder kills and replaces on * every dev-mode hot reload. */ +export type AbortSignalGetter = () => AbortSignal | undefined; + +/** + * PonderClient for fetching data from Ponder apps. + */ export class PonderClient { constructor( private readonly baseUrl: URL, - private readonly getAbortSignal?: () => AbortSignal | undefined, + private readonly getAbortSignal?: AbortSignalGetter, ) {} /** diff --git a/packages/ponder-sdk/src/local-ponder-client.ts b/packages/ponder-sdk/src/local-ponder-client.ts index d6745d4259..375509c676 100644 --- a/packages/ponder-sdk/src/local-ponder-client.ts +++ b/packages/ponder-sdk/src/local-ponder-client.ts @@ -1,7 +1,7 @@ import type { BlockNumberRangeWithStartBlock } from "./blockrange"; import type { CachedPublicClient } from "./cached-public-client"; import type { ChainId, ChainIdString } from "./chains"; -import { PonderClient } from "./client"; +import { type AbortSignalGetter, PonderClient } from "./client"; import { deserializeChainId } from "./deserialize/chains"; import { type ChainIndexingMetrics, @@ -82,14 +82,14 @@ export class LocalPonderClient extends PonderClient { * @param ponderPublicClients All cached public clients provided by the local Ponder app * (may include non-indexed chains). * @param ponderAppContext The internal context of the local Ponder app. - * @param getAbortSignal Optional getter invoked at fetch time to attach an `AbortSignal` to outgoing HTTP requests. Use a getter (not a captured signal) when the underlying signal can change identity over time — e.g. across Ponder dev-mode hot reloads. + * @param getAbortSignal Optional {@link AbortSignalGetter} invoked at fetch time. See its docs for the staleness contract. */ constructor( indexedChainIds: Set, indexedBlockranges: Map, ponderPublicClients: Record, ponderAppContext: PonderAppContext, - getAbortSignal?: () => AbortSignal | undefined, + getAbortSignal?: AbortSignalGetter, ) { super(ponderAppContext.localPonderAppUrl, getAbortSignal); diff --git a/packages/ponder-sdk/src/ponder-app-context.ts b/packages/ponder-sdk/src/ponder-app-context.ts index 4e560f1aa8..da6fee0cd2 100644 --- a/packages/ponder-sdk/src/ponder-app-context.ts +++ b/packages/ponder-sdk/src/ponder-app-context.ts @@ -12,6 +12,31 @@ export const PonderAppCommands = { export type PonderAppCommand = (typeof PonderAppCommands)[keyof typeof PonderAppCommands]; +/** + * Ponder shutdown manager runtime shape. + * + * Mirrors `ponder/src/internal/shutdown.ts` — the object Ponder publishes + * on `globalThis.PONDER_COMMON.{shutdown,apiShutdown}`. Reload-scoped: + * Ponder kills and replaces these on every dev-mode hot reload, so + * consumers must always read the current instance fresh and never cache + * a captured reference. + */ +export interface PonderAppShutdownManager { + add: (callback: () => undefined | Promise) => void; + isKilled: boolean; + abortController: AbortController; +} + +export function isPonderAppShutdownManager(value: unknown): value is PonderAppShutdownManager { + if (typeof value !== "object" || value === null) return false; + const obj = value as Record; + return ( + typeof obj.add === "function" && + typeof obj.isKilled === "boolean" && + obj.abortController instanceof AbortController + ); +} + /** * Ponder app context * From 46b4f4d84e9dcd67aa5ab746db9ead5e78d300f1 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 16:40:09 -0500 Subject: [PATCH 05/11] fix(ensindexer): tighten singleton catch path and widen shutdown callback type - Change the catch path's abort discrimination from `||` to `&&`. Both conditions must be true for a clean-stop bail: the captured shutdown signal is aborted AND the error is an AbortError. Either alone could mask a real failure (a non-AbortError after Ponder killed the signal, or an AbortError from a cross-contamination race that didn't actually signal our worker to stop). Flagged by greptile and copilot reviews. - Drop the rethrow at the end of the catch handler. The promise is fire-and-forget, so a rethrow becomes an unhandled rejection rather than reaching the outer .catch in api/index.ts. process.exitCode is already set, which is sufficient to fail the process. Flagged by copilot review. - Widen `PonderAppShutdownManager.add` callback type from `() => undefined | Promise` to `() => unknown` so void- returning callbacks (`() => { foo(); }`) compile, matching Ponder's actual runtime contract. Flagged by copilot review. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/singleton.ts | 18 +++++++++++------- packages/ponder-sdk/src/ponder-app-context.ts | 6 +++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index f5be927c46..ca47751572 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -77,11 +77,14 @@ export async function startEnsDbWriterWorker(): Promise { .run(abortSignal) // Handle any uncaught errors from the worker .catch(async (error) => { - // If Ponder has begun shutting down our API instance (hot reload or - // graceful shutdown), the abort propagates through in-flight fetches - // (or `signal.throwIfAborted()`) as an AbortError. Treat that as a - // clean stop, not a worker failure. - if (abortSignal.aborted || isAbortError(error)) { + // Treat as a clean stop only when BOTH the captured shutdown signal + // is aborted AND the error is an AbortError. Either condition alone + // could mask a real failure: a non-AbortError thrown after Ponder + // killed the signal is still a bug worth surfacing, and an + // AbortError without our signal aborted means it came from + // somewhere else (e.g. a reactive-getter race) and shouldn't be + // silently swallowed. + if (abortSignal.aborted && isAbortError(error)) { await gracefulShutdown(worker, "API shutdown (run aborted)"); return; } @@ -94,8 +97,9 @@ export async function startEnsDbWriterWorker(): Promise { error, }); - // Re-throw the error to ensure the application shuts down with a non-zero exit code. + // Set a non-zero exit code so the process terminates with failure. + // Don't rethrow — this catch handler is on a fire-and-forget promise, + // so a rethrow becomes an unhandled rejection. process.exitCode = 1; - throw error; }); } diff --git a/packages/ponder-sdk/src/ponder-app-context.ts b/packages/ponder-sdk/src/ponder-app-context.ts index da6fee0cd2..51f3eb2a97 100644 --- a/packages/ponder-sdk/src/ponder-app-context.ts +++ b/packages/ponder-sdk/src/ponder-app-context.ts @@ -22,7 +22,11 @@ export type PonderAppCommand = (typeof PonderAppCommands)[keyof typeof PonderApp * a captured reference. */ export interface PonderAppShutdownManager { - add: (callback: () => undefined | Promise) => void; + // Ponder awaits the callback's return value internally, so any value + // (sync or async) is accepted. Typed as `unknown` to keep callbacks + // that return `void` assignable without a Biome `noConfusingVoidType` + // violation. + add: (callback: () => unknown) => void; isKilled: boolean; abortController: AbortController; } From 7bb950f765ba7aa2a9b3ca00bc42aa410c69f2af Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 16:53:29 -0500 Subject: [PATCH 06/11] fix(ensindexer): cancel in-progress run on stop, fail-fast on real worker errors - Add internal `stopRequested` flag on EnsDbWriterWorker. `stop()` sets it; `run()` checks it (alongside the optional external AbortSignal) between every async setup step. Closes a race where `stop()` returned while `run()` was mid-startup, letting the stale run() arm a new recurring interval after stop. Surfaced by coderabbit. - Replace `process.exitCode = 1` with `process.exit(1)` in the singleton catch path for real worker errors. The worker is a startup invariant for the API layer; setting exitCode alone left ensindexer half-alive on fatal errors. Can't rethrow because the catch is on a fire-and- forget promise (would become an unhandled rejection rather than reaching api/index.ts), so process.exit(1) is the explicit fail-fast. Surfaced by coderabbit. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ensdb-writer-worker.ts | 35 +++++++++++++++---- .../src/lib/ensdb-writer-worker/singleton.ts | 13 ++++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index ce46bf08cd..86a3993681 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -43,6 +43,15 @@ export class EnsDbWriterWorker { */ private inFlightSnapshot: Promise | undefined; + /** + * Set by {@link stop} to signal an in-progress {@link run} that it should + * bail before scheduling the recurring interval. Required because the + * external `signal` parameter to `run` may not be aborted in every + * cancellation path (e.g. a defensive caller-driven `stop()` in singleton + * cleanup). + */ + private stopRequested = false; + /** * ENSDb Client instance used by the worker to interact with ENSDb. */ @@ -109,16 +118,17 @@ export class EnsDbWriterWorker { throw new Error("EnsDbWriterWorker is already running"); } - signal?.throwIfAborted(); + this.stopRequested = false; + this.checkCancellation(signal); // Fetch data required for task 1 and task 2. const inMemoryConfig = await this.getValidatedEnsIndexerPublicConfig(); - signal?.throwIfAborted(); + this.checkCancellation(signal); // Task 1: upsert ENSDb version into ENSDb. logger.debug({ msg: "Upserting ENSDb version", module: "EnsDbWriterWorker" }); await this.ensDbClient.upsertEnsDbVersion(inMemoryConfig.versionInfo.ensDb); - signal?.throwIfAborted(); + this.checkCancellation(signal); logger.info({ msg: "Upserted ENSDb version", ensDbVersion: inMemoryConfig.versionInfo.ensDb, @@ -131,7 +141,7 @@ export class EnsDbWriterWorker { module: "EnsDbWriterWorker", }); await this.ensDbClient.upsertEnsIndexerPublicConfig(inMemoryConfig); - signal?.throwIfAborted(); + this.checkCancellation(signal); logger.info({ msg: "Upserted ENSIndexer public config", module: "EnsDbWriterWorker", @@ -159,10 +169,12 @@ export class EnsDbWriterWorker { /** * Stop the ENSDb Writer Worker * - * Stops all recurring tasks in the worker and waits for any in-flight - * snapshot upsert to settle. Safe to call when not running. + * Cancels any in-progress {@link run} startup, stops all recurring tasks, + * and waits for any in-flight snapshot upsert to settle. Safe to call when + * not running. */ public async stop(): Promise { + this.stopRequested = true; if (this.indexingStatusInterval) { clearInterval(this.indexingStatusInterval); this.indexingStatusInterval = null; @@ -174,6 +186,17 @@ export class EnsDbWriterWorker { } } + /** + * Throw an `AbortError` if cancellation has been requested either via the + * caller's `AbortSignal` or via an internal {@link stop} call. + */ + private checkCancellation(signal?: AbortSignal): void { + signal?.throwIfAborted(); + if (this.stopRequested) { + throw new DOMException("Worker stop requested", "AbortError"); + } + } + /** * Get validated ENSIndexer Public Config object for the ENSDb Writer Worker. * diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index ca47751572..bd3770000a 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -89,7 +89,13 @@ export async function startEnsDbWriterWorker(): Promise { return; } - // Real worker error — clean up and trigger non-zero exit. + // Real worker error — clean up and fail-fast. The worker is a startup + // invariant for the API layer; leaving the process half-alive (just + // setting `process.exitCode`) would let ensindexer keep serving with a + // dead writer. We can't rethrow because this `.catch()` is on a + // fire-and-forget promise, so a rethrow becomes an unhandled rejection + // instead of reaching a top-level handler — call `process.exit(1)` to + // terminate immediately. await gracefulShutdown(worker, "uncaught error"); logger.error({ @@ -97,9 +103,6 @@ export async function startEnsDbWriterWorker(): Promise { error, }); - // Set a non-zero exit code so the process terminates with failure. - // Don't rethrow — this catch handler is on a fire-and-forget promise, - // so a rethrow becomes an unhandled rejection. - process.exitCode = 1; + process.exit(1); }); } From f57ae1dd9c14bfe72cc10d9d5c0567ce198fa0de Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 17:05:31 -0500 Subject: [PATCH 07/11] fix(ensindexer): treat superseded worker as intentional stop in catch path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the defensive stale-instance cleanup inside startEnsDbWriterWorker calls worker.stop() on the previous worker, the old run() throws an AbortError via the new stopRequested guard — but the captured abortSignal may not be aborted (the safety net runs precisely when Ponder's own apiShutdown.add() callback didn't fire). The catch path now also treats `ensDbWriterWorker !== worker` as a clean-stop signal, so the supersession path doesn't get misclassified as a fatal error and call process.exit(1). isAbortError() is still required so unrelated failures are not silently swallowed. Surfaced by copilot review. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/singleton.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index bd3770000a..14618e9767 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -77,14 +77,17 @@ export async function startEnsDbWriterWorker(): Promise { .run(abortSignal) // Handle any uncaught errors from the worker .catch(async (error) => { - // Treat as a clean stop only when BOTH the captured shutdown signal - // is aborted AND the error is an AbortError. Either condition alone - // could mask a real failure: a non-AbortError thrown after Ponder - // killed the signal is still a bug worth surfacing, and an - // AbortError without our signal aborted means it came from - // somewhere else (e.g. a reactive-getter race) and shouldn't be - // silently swallowed. - if (abortSignal.aborted && isAbortError(error)) { + // Treat as a clean stop when the error is an AbortError AND the + // worker was intentionally stopped — either because Ponder aborted + // the captured shutdown signal, or because this worker has been + // superseded in the singleton (the defensive stale-instance + // cleanup path inside `startEnsDbWriterWorker` calls + // `worker.stop()` on the old worker, which causes its `run()` to + // throw AbortError without `abortSignal.aborted` necessarily + // being true). Requiring `isAbortError(error)` keeps unrelated + // failures from being silently swallowed. + const intentionallyStopped = abortSignal.aborted || ensDbWriterWorker !== worker; + if (intentionallyStopped && isAbortError(error)) { await gracefulShutdown(worker, "API shutdown (run aborted)"); return; } From 23070cf3a202ebf6e8805e7a4d7bcfc4f8917228 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 17:17:00 -0500 Subject: [PATCH 08/11] docs(ensindexer): explain DOMException AbortError choice in checkCancellation Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/ensdb-writer-worker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index 86a3993681..b548b2e163 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -193,6 +193,10 @@ export class EnsDbWriterWorker { private checkCancellation(signal?: AbortSignal): void { signal?.throwIfAborted(); if (this.stopRequested) { + // Match what `AbortSignal.throwIfAborted()` throws — a DOMException + // with `name === "AbortError"` — so callers' `isAbortError` checks + // treat both abort sources (external signal and internal stop) + // identically. throw new DOMException("Worker stop requested", "AbortError"); } } From cb59db068ec5ad8192363c5735788615869a6048 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 17:25:07 -0500 Subject: [PATCH 09/11] refactor(ensindexer): replace localPonderContext Proxy with explicit getters The Proxy was technically correct but call-site ergonomics hid the staleness contract: `localPonderContext.apiShutdown` looked like an ordinary property access, masking that it was actually a fresh read from globalThis. A captured `const sig = localPonderContext.apiShutdown .signal` looked innocent but was a footgun. Split the context into two shapes: - `localPonderContext` is back to an eager `const` for the stable fields (command, localPonderAppUrl, logger). Ponder doesn't mutate these, and the original code shape already matched this. - Reload-scoped fields are now plain functions: `getApiShutdown()` and `getShutdown()`. The function-call form makes it visible at every call site that the value is freshly read each call. A captured `getApiShutdown().signal` obviously caches a function result, which reads as a bug. Drops the Proxy, the symbol-prop guard, the LocalPonderContext interface, the prop-as-keyof cast, and the cachedStableContext memoization helper. Net: 19 fewer lines, simpler shape, better ergonomics. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/singleton.ts | 6 +- .../ensindexer/src/lib/local-ponder-client.ts | 4 +- .../src/lib/local-ponder-context.ts | 75 +++++++------------ 3 files changed, 33 insertions(+), 52 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index 14618e9767..51c08b2765 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -1,7 +1,7 @@ import { ensDbClient } from "@/lib/ensdb/singleton"; import { indexingStatusBuilder } from "@/lib/indexing-status-builder/singleton"; import { localPonderClient } from "@/lib/local-ponder-client"; -import { localPonderContext } from "@/lib/local-ponder-context"; +import { getApiShutdown } from "@/lib/local-ponder-context"; import { logger } from "@/lib/logger"; import { publicConfigBuilder } from "@/lib/public-config-builder/singleton"; @@ -65,10 +65,10 @@ export async function startEnsDbWriterWorker(): Promise { ); ensDbWriterWorker = worker; - // Read apiShutdown FRESH from the reactive context. Ponder kills and + // Read apiShutdown FRESH via getApiShutdown(). Ponder kills and // replaces this on every dev-mode hot reload, so this read MUST happen // inside the function call (not at module scope). - const apiShutdown = localPonderContext.apiShutdown; + const apiShutdown = getApiShutdown(); const abortSignal = apiShutdown.abortController.signal; apiShutdown.add(() => gracefulShutdown(worker, "API shutdown")); diff --git a/apps/ensindexer/src/lib/local-ponder-client.ts b/apps/ensindexer/src/lib/local-ponder-client.ts index b9a711b4ad..6cd1538646 100644 --- a/apps/ensindexer/src/lib/local-ponder-client.ts +++ b/apps/ensindexer/src/lib/local-ponder-client.ts @@ -7,7 +7,7 @@ import { LocalPonderClient } from "@ensnode/ponder-sdk"; import { getPluginsAllDatasourceNames } from "@/lib/plugin-helpers"; -import { localPonderContext } from "./local-ponder-context"; +import { getApiShutdown, localPonderContext } from "./local-ponder-context"; const pluginsAllDatasourceNames = getPluginsAllDatasourceNames(config.plugins); const indexedBlockranges = buildIndexedBlockranges(config.namespace, pluginsAllDatasourceNames); @@ -18,5 +18,5 @@ export const localPonderClient = new LocalPonderClient( publicClients, localPonderContext, // See local-ponder-context.ts for the staleness contract. - () => localPonderContext.apiShutdown.abortController.signal, + () => getApiShutdown().abortController.signal, ); diff --git a/apps/ensindexer/src/lib/local-ponder-context.ts b/apps/ensindexer/src/lib/local-ponder-context.ts index 01fded5683..ce8b6d35a0 100644 --- a/apps/ensindexer/src/lib/local-ponder-context.ts +++ b/apps/ensindexer/src/lib/local-ponder-context.ts @@ -1,14 +1,13 @@ import { deserializePonderAppContext, isPonderAppShutdownManager, - type PonderAppContext, type PonderAppShutdownManager, } from "@ensnode/ponder-sdk"; /** - * Local Ponder Context — reactive wrapper over Ponder's runtime globals. + * Local Ponder Context — wrappers over Ponder's runtime globals. * - * Why this is a Proxy and not an eagerly-deserialized object: + * Stable vs reload-scoped fields: * * Ponder's dev mode hot-reloads the API entry file by re-executing it via * vite-node. On every indexing-file change, Ponder ALSO kills and replaces @@ -19,23 +18,29 @@ import { * dep tree. So any value cached in a module-level closure during the * original boot becomes stale on the very next reload. * - * Stable fields (`command`, `localPonderAppUrl`, `logger`) are validated - * once and memoized — Ponder does not mutate `options` or `logger` on - * reload. Reload-scoped fields (`apiShutdown`, `shutdown`) MUST be re-read - * from `globalThis.PONDER_COMMON` on every access. + * Stable fields (`command`, `localPonderAppUrl`, `logger`) are eagerly + * deserialized once at module load — Ponder does not mutate `options` or + * `logger` on reload. Reload-scoped fields are exposed as FUNCTIONS + * ({@link getApiShutdown}, {@link getShutdown}) rather than properties on + * the context object so that every call site is forced to re-read fresh + * from `globalThis.PONDER_COMMON`. The function call form makes the + * staleness contract visible at the call site — a captured `const sig = + * getApiShutdown().abortController.signal` is obviously caching a + * function result, whereas a captured `localPonderContext.apiShutdown` + * would have looked like an innocent property access. * - * Contract for callers: NEVER cache reload-scoped fields in a module-level - * closure or capture them in a constructor argument. Always reach for them - * via `localPonderContext.` from code that runs per-reload (e.g. the - * API entry file or per-request handlers). If you need an `AbortSignal` - * across calls, store a getter (`() => localPonderContext.apiShutdown - * .abortController.signal`), not the signal itself. + * Contract for callers: NEVER cache the return value of `getApiShutdown` + * or `getShutdown` (or any field on it) in a module-level closure or a + * constructor argument. If you need to attach a listener or read the + * signal across calls, store the GETTER function, not its return value. */ if (!globalThis.PONDER_COMMON) { throw new Error("PONDER_COMMON must be defined by Ponder at runtime as a global variable."); } +export const localPonderContext = deserializePonderAppContext(globalThis.PONDER_COMMON); + function readShutdownManager(field: "apiShutdown" | "shutdown"): PonderAppShutdownManager { const raw = (globalThis.PONDER_COMMON as Record | undefined)?.[field]; if (!isPonderAppShutdownManager(raw)) { @@ -44,42 +49,18 @@ function readShutdownManager(field: "apiShutdown" | "shutdown"): PonderAppShutdo return raw; } -let cachedStableContext: PonderAppContext | undefined; -function getStableContext(): PonderAppContext { - if (!cachedStableContext) { - if (!globalThis.PONDER_COMMON) { - throw new Error("PONDER_COMMON must be defined by Ponder at runtime as a global variable."); - } - cachedStableContext = deserializePonderAppContext(globalThis.PONDER_COMMON); - } - return cachedStableContext; +/** + * Returns the current `apiShutdown` manager. RELOAD-SCOPED — Ponder + * replaces this on every API hot reload. Always call fresh; never cache. + */ +export function getApiShutdown(): PonderAppShutdownManager { + return readShutdownManager("apiShutdown"); } /** - * Local Ponder Context. - * - * Combines stable {@link PonderAppContext} fields with reload-scoped - * shutdown managers. See module-level comment for the staleness contract. + * Returns the current `shutdown` manager. RELOAD-SCOPED — Ponder + * replaces this on every indexing hot reload. Always call fresh; never cache. */ -export interface LocalPonderContext extends PonderAppContext { - /** - * The current `apiShutdown` manager. RELOAD-SCOPED — identity changes - * every API hot-reload. Always read fresh; never cache. - */ - readonly apiShutdown: PonderAppShutdownManager; - - /** - * The current `shutdown` manager. RELOAD-SCOPED — identity changes - * every indexing hot-reload. Always read fresh; never cache. - */ - readonly shutdown: PonderAppShutdownManager; +export function getShutdown(): PonderAppShutdownManager { + return readShutdownManager("shutdown"); } - -export const localPonderContext: LocalPonderContext = new Proxy({} as LocalPonderContext, { - get(_target, prop) { - if (typeof prop === "symbol") return undefined; - if (prop === "apiShutdown") return readShutdownManager("apiShutdown"); - if (prop === "shutdown") return readShutdownManager("shutdown"); - return getStableContext()[prop as keyof PonderAppContext]; - }, -}); From 8702d0df9114f56e3f74a9038119659e0c823e02 Mon Sep 17 00:00:00 2001 From: shrugs Date: Wed, 15 Apr 2026 17:45:12 -0500 Subject: [PATCH 10/11] docs(ensindexer): document AbortError DOMException in run() JSDoc Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/lib/ensdb-writer-worker/ensdb-writer-worker.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts index b548b2e163..f044bcaf09 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts @@ -109,8 +109,9 @@ export class EnsDbWriterWorker { * shutting down. * @throws Error if the worker is already running, or * if the in-memory ENSIndexer Public Config could not be fetched, or - * if the in-memory ENSIndexer Public Config is incompatible with the stored config in ENSDb, or - * if `signal` is aborted before the recurring interval is scheduled. + * if the in-memory ENSIndexer Public Config is incompatible with the stored config in ENSDb. + * @throws DOMException with `name === "AbortError"` if `signal` is aborted + * or if {@link stop} is called before the recurring interval is scheduled. */ public async run(signal?: AbortSignal): Promise { // Do not allow multiple concurrent runs of the worker From 60b269e4d754cb573dd712d5e8d39cddcba07c5d Mon Sep 17 00:00:00 2001 From: shrugs Date: Thu, 16 Apr 2026 09:30:06 -0500 Subject: [PATCH 11/11] fix(ensindexer): clear singleton before awaiting stop, add cancellation tests - In gracefulShutdown(), clear ensDbWriterWorker before awaiting worker.stop() so the catch discriminator immediately sees the worker as superseded. Prevents misclassifying a stop-driven AbortError as fatal when the run().catch fires before the singleton was cleared. - Add test: stop() during run() startup rejects with AbortError and never arms the recurring interval. - Add test: overlapping snapshot ticks are skipped when a prior upsert is still in flight, and resume once it settles. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../ensdb-writer-worker.test.ts | 65 +++++++++++++++++++ .../src/lib/ensdb-writer-worker/singleton.ts | 10 ++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts index 944250e2db..da56acf234 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts @@ -364,4 +364,69 @@ describe("EnsDbWriterWorker", () => { await worker.stop(); }); }); + + describe("cancellation - stopRequested and skip-overlap", () => { + it("stop() during run() startup causes run() to reject with AbortError", async () => { + // arrange — make upsertEnsDbVersion block until we release it + let resolveUpsert!: () => void; + const upsertPromise = new Promise((resolve) => { + resolveUpsert = resolve; + }); + const ensDbClient = createMockEnsDbWriter({ + upsertEnsDbVersion: vi.fn().mockReturnValue(upsertPromise), + }); + const worker = createMockEnsDbWriterWorker({ ensDbClient }); + + // act — start run, then stop while it's blocked on upsertEnsDbVersion + const runPromise = worker.run(); + await worker.stop(); + resolveUpsert(); + + // assert — run rejects with AbortError + await expect(runPromise).rejects.toThrow("Worker stop requested"); + + // the interval was never armed + expect(worker.isRunning).toBe(false); + }); + + it("skips overlapping snapshot ticks when a prior upsert is still in flight", async () => { + // arrange — make upsertIndexingStatusSnapshot block until released + let resolveSnapshot!: () => void; + const snapshotPromise = new Promise((resolve) => { + resolveSnapshot = resolve; + }); + const omnichainSnapshot = createMockOmnichainSnapshot(); + const crossChainSnapshot = createMockCrossChainSnapshot({ omnichainSnapshot }); + vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(crossChainSnapshot); + + const ensDbClient = createMockEnsDbWriter({ + upsertIndexingStatusSnapshot: vi.fn().mockReturnValueOnce(snapshotPromise), + }); + const worker = createMockEnsDbWriterWorker({ + ensDbClient, + indexingStatusBuilder: createMockIndexingStatusBuilder(omnichainSnapshot), + }); + + await worker.run(); + + // act — first tick starts the slow upsert + await vi.advanceTimersByTimeAsync(1000); + expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(1); + + // second + third ticks should be skipped (still in flight) + await vi.advanceTimersByTimeAsync(2000); + expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(1); + + // release the slow upsert + resolveSnapshot(); + await vi.advanceTimersByTimeAsync(0); + + // next tick should fire again + await vi.advanceTimersByTimeAsync(1000); + expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(2); + + // cleanup + await worker.stop(); + }); + }); }); diff --git a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts index 51c08b2765..f9622207e8 100644 --- a/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts +++ b/apps/ensindexer/src/lib/ensdb-writer-worker/singleton.ts @@ -20,18 +20,22 @@ function isAbortError(error: unknown): boolean { } /** - * Stop the given worker (if it is still the active singleton) and clear the - * singleton reference. Safe to call multiple times. + * Stop the given worker and, if it is still the active singleton, clear the + * singleton reference BEFORE awaiting shutdown. Clearing first ensures that + * if `stop()` causes an in-progress `run()` to reject with AbortError, the + * catch discriminator immediately sees `ensDbWriterWorker !== worker` and + * classifies it as an intentional stop rather than a fatal error. + * Safe to call multiple times. */ async function gracefulShutdown(worker: EnsDbWriterWorker, reason: string): Promise { logger.info({ msg: `Stopping EnsDbWriterWorker: ${reason}`, module: "EnsDbWriterWorker", }); - await worker.stop(); if (ensDbWriterWorker === worker) { ensDbWriterWorker = undefined; } + await worker.stop(); } /**