Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
4b125d0
Introduce `IndexingMetadataContext` data model to ENSNode SDK
tk-o Apr 24, 2026
dcad533
Create `init*` functions for running event handlers preconditions
tk-o Apr 24, 2026
a5a8bd6
Execute `migrateEnsNodeSchema` from inside of `initIndexingSetup` fun…
tk-o Apr 24, 2026
c57c2ac
Use `EnsIndexerStackInfo` for `stackInfo` field in `IndexingMetadataC…
tk-o Apr 24, 2026
a28b0de
Add `getIndexingMetadataContext` method to `EnsDbReader` class
tk-o Apr 24, 2026
e91bd13
Add `upsertIndexingMetadataContext` method to `EnsDbWriter` class
tk-o Apr 24, 2026
8892dd5
Update `EnsDbWriterWorker` class to serve a new limited role
tk-o Apr 24, 2026
ee1190e
Fix ponder build issue
tk-o Apr 24, 2026
4954e21
Fix typos
tk-o Apr 24, 2026
2f8532e
Implement full tasks sequence for `initIndexingOnchainEvents`
tk-o Apr 25, 2026
361e99d
Merge `initIndexingSetup` function into `initIndexingOnchainEvents`
tk-o Apr 25, 2026
7595a41
Update unit tests
tk-o Apr 25, 2026
1902bfb
Use `getIndexingMetadataContext` for all ENSNode Metadata reads from …
tk-o Apr 25, 2026
6bda3d6
Merge remote-tracking branch 'origin/main' into feat/indexing-metadat…
tk-o Apr 25, 2026
e3ddda0
Simplify `EnsDbReader` class and `EnsDbWriter` class
tk-o Apr 25, 2026
4684fb4
Fix tests
tk-o Apr 25, 2026
1ff960e
Apply AI PR feedback
tk-o Apr 26, 2026
41060d0
Simplify `initIndexingOnchainEvents` logic
tk-o Apr 26, 2026
e1d6d04
Update unit tests
tk-o Apr 26, 2026
98e6c45
Simplify `initIndexingOnchainEvents` function
tk-o Apr 26, 2026
eebe386
Simplify logic in ENSIndexer HTTP endpoints
tk-o Apr 26, 2026
11711ca
Improve naming
tk-o Apr 26, 2026
89c974a
Make indexing status cache and stack info cache for ENSApi to use the…
tk-o Apr 27, 2026
292ed35
Merge remote-tracking branch 'origin/main' into feat/indexing-metadat…
tk-o Apr 27, 2026
159c4ff
Improve code docs
tk-o Apr 27, 2026
b36418b
Create a mock file for config.schema.ts
tk-o Apr 27, 2026
f3355ef
Integrate ENSDb health check and readiness check into `initIndexingOn…
tk-o Apr 27, 2026
40961af
Update code docs
tk-o Apr 27, 2026
6c5691d
Apply AI PR feedback
tk-o Apr 27, 2026
39c7d82
Merge remote-tracking branch 'origin/main' into feat/indexing-metadat…
tk-o Apr 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
277 changes: 277 additions & 0 deletions PR-1997-description.md

Large diffs are not rendered by default.

81 changes: 49 additions & 32 deletions apps/ensapi/src/cache/indexing-status.cache.ts
Original file line number Diff line number Diff line change
@@ -1,50 +1,67 @@
import { EnsNodeMetadataKeys } from "@ensnode/ensdb-sdk";
import { type CrossChainIndexingStatusSnapshot, SWRCache } from "@ensnode/ensnode-sdk";
import {
type CrossChainIndexingStatusSnapshot,
IndexingMetadataContextStatusCodes,
SWRCache,
} from "@ensnode/ensnode-sdk";

import { ensDbClient } from "@/lib/ensdb/singleton";
import { lazyProxy } from "@/lib/lazy";
import { makeLogger } from "@/lib/logger";

const logger = makeLogger("indexing-status.cache");

export type IndexingStatusCache = SWRCache<CrossChainIndexingStatusSnapshot>;

// lazyProxy defers construction until first use so that this module can be
// imported without env vars being present (e.g. during OpenAPI generation).
// SWRCache with proactivelyInitialize:true starts background polling immediately
// on construction, which would trigger ensDbClient before env vars are available.
export const indexingStatusCache = lazyProxy<SWRCache<CrossChainIndexingStatusSnapshot>>(
/**
* Cache for {@link CrossChainIndexingStatusSnapshot}, which is loaded
* from ENSDb on demand. The cached value is expected to be updated
* very frequently, following the update frequency of
* {@link IndexingMetadataContextInitialized.indexingStatus} in ENSDb.
* Therefore, the cache is configured with a very short TTL and
* proactive revalidation interval to ensure that the cached value is
* as fresh as possible.
*/
export const indexingStatusCache = lazyProxy<IndexingStatusCache>(
() =>
new SWRCache<CrossChainIndexingStatusSnapshot>({
fn: async (_cachedResult) =>
ensDbClient
.getIndexingStatusSnapshot() // get the latest indexing status snapshot
.then((snapshot) => {
if (snapshot === undefined) {
// An indexing status snapshot has not been found in ENSDb yet.
// This might happen during application startup, i.e. when ENSDb
// has not yet been populated with the first snapshot.
// Therefore, throw an error to trigger the subsequent `.catch` handler.
throw new Error("Indexing Status snapshot not found in ENSDb yet.");
}
fn: async function loadIndexingStatusSnapshot() {
try {
const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext();

if (
indexingMetadataContext.statusCode !== IndexingMetadataContextStatusCodes.Initialized
) {
// The IndexingMetadataContext has not been initialized in ENSDb yet.
// This might happen during application startup, i.e. when ENSDb
// has not yet been populated with the IndexingMetadataContext record.
// Therefore, throw an error to trigger the subsequent catch handler.
throw new Error("Indexing Metadata Context was uninitialized in ENSDb.");
}

// The indexing status snapshot has been fetched and successfully validated for caching.
// Therefore, return it so that this current invocation of `readCache` will:
// - Replace the currently cached value (if any) with this new value.
// - Return this non-null value.
return snapshot;
})
.catch((error) => {
// Either the indexing status snapshot fetch failed, or the indexing status snapshot was not found in ENSDb yet.
// Therefore, throw an error so that this current invocation of `readCache` will:
// - Reject the newly fetched response (if any) such that it won't be cached.
// - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value.
logger.error(
error,
`Error occurred while loading Indexing Status snapshot record from ENSNode Metadata table in ENSDb. ` +
`Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.EnsIndexerIndexingStatus}"). ` +
`The cached indexing status snapshot (if any) will not be updated.`,
);
throw error;
}),
// The CrossChainIndexingStatusSnapshot has been successfully loaded for caching.
// Therefore, return it so that this current invocation of `readCache` will:
// - Replace the currently cached value (if any) with this new value.
// - Return this non-null value.
return indexingMetadataContext.indexingStatus;
} catch (error) {
// IndexingMetadataContext was uninitialized in ENSDb.
// Therefore, throw an error so that this current invocation of `readCache` will:
// - Reject the newly fetched response (if any) such that it won't be cached.
// - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value.
Comment thread
tk-o marked this conversation as resolved.
logger.error(
error,
`Error occurred while loading Indexing Metadata Context record from ENSNode Metadata table in ENSDb. ` +
`Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.IndexingMetadataContext}"). ` +
`The cached indexing status snapshot (if any) will not be updated.`,
);
throw error;
}
},
// We need to refresh the indexing status cache very frequently.
// ENSDb won't have issues handling this frequency of queries.
ttl: 1, // 1 second
Expand Down
106 changes: 65 additions & 41 deletions apps/ensapi/src/cache/stack-info.cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,76 +2,100 @@ import config from "@/config";

import { minutesToSeconds } from "date-fns";

import { EnsNodeMetadataKeys } from "@ensnode/ensdb-sdk";
import {
buildEnsNodeStackInfo,
type CachedResult,
type EnsNodeStackInfo,
type IndexingMetadataContextInitialized,
IndexingMetadataContextStatusCodes,
SWRCache,
} from "@ensnode/ensnode-sdk";
Comment thread
tk-o marked this conversation as resolved.

import { buildEnsApiPublicConfig } from "@/config/config.schema";
import { ensDbClient } from "@/lib/ensdb/singleton";
import { lazyProxy } from "@/lib/lazy";
import logger from "@/lib/logger";

/**
* Loads the ENSNode stack info, either from cache if available,
* or by building it from the public configs of ENSApi and ENSDb.
*
* The ENSNode Stack Info object is considered immutable for
* the lifecycle of an ENSApi process instance, so once it is successfully
* loaded, it will be cached indefinitely.
*/
async function loadEnsNodeStackInfo(
cachedResult?: CachedResult<EnsNodeStackInfo>,
): Promise<EnsNodeStackInfo> {
if (cachedResult && !(cachedResult.result instanceof Error)) {
return cachedResult.result;
}

const ensApiPublicConfig = buildEnsApiPublicConfig(config);
const ensDbPublicConfig = await ensDbClient.buildEnsDbPublicConfig();
const ensIndexerPublicConfig = ensApiPublicConfig.ensIndexerPublicConfig;
const ensRainbowPublicConfig = ensIndexerPublicConfig.ensRainbowPublicConfig;

return buildEnsNodeStackInfo(
ensApiPublicConfig,
ensDbPublicConfig,
ensIndexerPublicConfig,
ensRainbowPublicConfig,
);
}
export type EnsNodeStackInfoCache = SWRCache<EnsNodeStackInfo>;

// lazyProxy defers construction until first use so that this module can be
// imported without env vars being present (e.g. during OpenAPI generation).
// SWRCache with proactivelyInitialize:true starts background polling immediately
// on construction, which would trigger ensDbClient before env vars are available.
/**
* Cache for ENSNode stack info
* Once successfully loaded, the ENSNode Stack Info is cached indefinitely and
* never revalidated. This ensures the JSON is only fetched once during
* the application lifecycle.
* Cache for {@link EnsNodeStackInfo}, which is loaded from ENSDb on demand.
* Once successfully loaded, the {@link EnsNodeStackInfo} is cached and kept up-to-date
* by proactive revalidation, since the {@link EnsNodeStackInfo} might change during
* the lifecycle of the ENSApi instance, for example, when
* {@link IndexingMetadataContextInitialized.stackInfo} is updated in ENSDb.
* This is unlikely to happen at all, and if it does happen, it is likely to be
* very infrequent. However, proactive revalidation ensures that if such changes do happen,
* the cached value will be updated in a reasonable time frame without requiring
* a restart of the ENSApi application.
*
* Configuration:
* - ttl: Infinity - Never expires once cached
* - errorTtl: 1 minute - If loading fails, retry on next access after 1 minute
* - proactiveRevalidationInterval: undefined - No proactive revalidation
* - ttl: 1 minute - Allow cached value to be fresh for up to 1 minute.
* - errorTtl: 1 minute - If loading fails, retry on next access after 1 minute.
* - proactiveRevalidationInterval: 5 minutes - Refresh the cached value every 5 minutes.
* - proactivelyInitialize: true - Load immediately on startup
*/
export const stackInfoCache = lazyProxy(
export const stackInfoCache = lazyProxy<EnsNodeStackInfoCache>(
() =>
/**
* Cache for ENSNode stack info
*
* Once initialized successfully, this cache will always return
* the same stack info for the lifecycle of the ENSApi instance.
* the same {@link EnsNodeStackInfo} for the lifecycle of the ENSApi instance.
*
* If initialization fails, it will keep retrying on access until it succeeds, which is desirable because the stack info is critical for the functioning of the application and we want to recover from transient initialization failures without requiring a restart.
* If initialization fails, it will keep retrying on access until it succeeds,
* which is desirable because the {@link EnsNodeStackInfo} is critical for the functioning of the application and we want to recover from transient initialization failures without requiring a restart.
*/
new SWRCache<EnsNodeStackInfo>({
fn: loadEnsNodeStackInfo,
ttl: Number.POSITIVE_INFINITY,
fn: async function loadEnsNodeStackInfo() {
try {
const indexingMetadataContext = await ensDbClient.getIndexingMetadataContext();

if (
indexingMetadataContext.statusCode !== IndexingMetadataContextStatusCodes.Initialized
) {
// The IndexingMetadataContext has not been initialized in ENSDb yet.
// This might happen during application startup, i.e. when ENSDb
// has not yet been populated with the IndexingMetadataContext record.
// Therefore, throw an error to trigger the subsequent catch handler.
throw new Error("Indexing Metadata Context was uninitialized in ENSDb.");
}

const ensIndexerStackInfo = indexingMetadataContext.stackInfo;
const ensNodeStackInfo = buildEnsNodeStackInfo(
buildEnsApiPublicConfig(config, ensIndexerStackInfo.ensIndexer),
ensIndexerStackInfo.ensDb,
ensIndexerStackInfo.ensIndexer,
ensIndexerStackInfo.ensRainbow,
);

// The EnsNodeStackInfo has been successfully built for caching.
// Therefore, return it so that this current invocation of `readCache` will:
// - Replace the currently cached value (if any) with this new value.
// - Return this non-null value.
return ensNodeStackInfo;
} catch (error) {
// IndexingMetadataContext was uninitialized in ENSDb.
// Therefore, throw an error so that this current invocation of `readCache` will:
// - Reject the newly fetched response (if any) such that it won't be cached.
// - Return the most recently cached value from prior invocations, or `null` if no prior invocation successfully cached a value.
Comment thread
tk-o marked this conversation as resolved.
logger.error(
error,
`Error occurred while loading Indexing Metadata Context record from ENSNode Metadata table in ENSDb. ` +
`Where clause applied: ("ensIndexerSchemaName" = "${ensDbClient.ensIndexerSchemaName}", "key" = "${EnsNodeMetadataKeys.IndexingMetadataContext}"). ` +
`The cached EnsNodeStackInfo object (if any) will not be updated.`,
);

throw error;
}
},
ttl: minutesToSeconds(1),
errorTtl: minutesToSeconds(1),
proactiveRevalidationInterval: undefined,
proactiveRevalidationInterval: minutesToSeconds(5),
proactivelyInitialize: true,
}),
);
108 changes: 108 additions & 0 deletions apps/ensapi/src/config/config.schema.mock.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import packageJson from "@/../package.json" with { type: "json" };

import {
ChainIndexingStatusIds,
CrossChainIndexingStrategyIds,
deserializeIndexingMetadataContext,
type EnsRainbowPublicConfig,
type IndexingMetadataContextInitialized,
IndexingMetadataContextStatusCodes,
OmnichainIndexingStatusIds,
PluginName,
RangeTypeIds,
type SerializedCrossChainIndexingStatusSnapshot,
type SerializedEnsDbPublicConfig,
type SerializedEnsIndexerPublicConfig,
type SerializedEnsIndexerStackInfo,
type SerializedIndexingMetadataContextInitialized,
} from "@ensnode/ensnode-sdk";

import type { EnsApiEnvironment } from "@/config/environment";

export const VALID_RPC_URL = "https://eth-sepolia.g.alchemy.com/v2/1234";

export const ENSDB_PUBLIC_CONFIG = {
versionInfo: {
postgresql: "17.4",
},
} satisfies SerializedEnsDbPublicConfig;

export const ENSINDEXER_PUBLIC_CONFIG = {
namespace: "mainnet",
ensIndexerSchemaName: "ensindexer_0",
ensRainbowPublicConfig: {
serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 },
versionInfo: {
ensRainbow: packageJson.version,
},
},
indexedChainIds: [1],
isSubgraphCompatible: false,
clientLabelSet: { labelSetId: "subgraph", labelSetVersion: 0 },
plugins: [PluginName.Subgraph],
versionInfo: {
ensDb: packageJson.version,
ensIndexer: packageJson.version,
ensNormalize: "1.11.1",
ponder: "0.8.0",
},
} satisfies SerializedEnsIndexerPublicConfig;

const ENSRAINBOW_PUBLIC_CONFIG = {
serverLabelSet: { labelSetId: "subgraph", highestLabelSetVersion: 0 },
versionInfo: {
ensRainbow: packageJson.version,
},
} satisfies EnsRainbowPublicConfig;

export const INDEXING_STATUS = {
strategy: CrossChainIndexingStrategyIds.Omnichain,
slowestChainIndexingCursor: 1777147427,
snapshotTime: 1777147440,
omnichainSnapshot: {
omnichainStatus: OmnichainIndexingStatusIds.Following,
chains: {
"1": {
chainStatus: ChainIndexingStatusIds.Following,
config: {
rangeType: RangeTypeIds.LeftBounded,
startBlock: {
timestamp: 1489165544,
number: 3327417,
},
},
latestIndexedBlock: {
timestamp: 1777147427,
number: 24959286,
},
latestKnownBlock: {
timestamp: 1777147427,
number: 24959286,
},
},
},
omnichainIndexingCursor: 1777147427,
},
} satisfies SerializedCrossChainIndexingStatusSnapshot;

export const ENSINDEXER_STACK_INFO = {
ensDb: ENSDB_PUBLIC_CONFIG,
ensIndexer: ENSINDEXER_PUBLIC_CONFIG,
ensRainbow: ENSRAINBOW_PUBLIC_CONFIG,
} satisfies SerializedEnsIndexerStackInfo;

export const INDEXING_METADATA_CONTEXT = {
statusCode: IndexingMetadataContextStatusCodes.Initialized,
indexingStatus: INDEXING_STATUS,
stackInfo: ENSINDEXER_STACK_INFO,
} satisfies SerializedIndexingMetadataContextInitialized;

export const indexingMetadataContextInitialized = deserializeIndexingMetadataContext(
INDEXING_METADATA_CONTEXT,
) as IndexingMetadataContextInitialized;

export const BASE_ENV = {
ENSDB_URL: "postgresql://user:password@localhost:5432/mydb",
ENSINDEXER_SCHEMA_NAME: "ensindexer_0",
RPC_URL_1: VALID_RPC_URL,
} satisfies EnsApiEnvironment;
Loading
Loading