feat: add decode layer, anomaly recording and chain-upgrade tracking [redesign(03)]

db/compat.sql

@@ -1,44 +1,41 @@
+-- Everything in this file is something `schema.graphql` cannot express. Plain and composite
+-- indexes are declared with `@index` / `@compositeIndexes` in the schema instead, so that the
+-- index set has one source of truth. Each block below says why it has to live here.
+
+-- Generated columns. SubQuery writes the JSON payloads as text; these expose them as JSONB for
+-- the query layer. There is no directive for a generated column.
 ALTER TABLE events
 ADD COLUMN IF NOT EXISTS attributes JSONB GENERATED ALWAYS AS (attributes_txt::jsonb) STORED NULL;
 
 ALTER TABLE extrinsics
 ADD COLUMN IF NOT EXISTS params JSONB GENERATED ALWAYS AS (params_txt::jsonb) STORED NULL;
 
+-- A plain `datetime` index an older deployment may have left behind. The expression index below
+-- is what serves the queries that used it.
 DROP INDEX IF EXISTS data_block_datetime;
 
-CREATE UNIQUE INDEX IF NOT EXISTS data_block_id ON blocks (block_id);
-CREATE UNIQUE INDEX IF NOT EXISTS data_block_hash ON blocks (hash);
+-- Expression index. `@index` indexes a column; this one indexes the value cast to a
+-- second-resolution timestamp, which is what the query layer compares against.
 CREATE INDEX IF NOT EXISTS data_block_datetime_timestamp ON blocks (((datetime)::timestamp(0) without time zone));
-CREATE INDEX IF NOT EXISTS data_block_parent_hash ON blocks (parent_hash);
 
+-- Unique composite indexes. `@compositeIndexes` declares a composite index but has no `unique`
+-- argument, so uniqueness across two columns can only be stated here.
 CREATE UNIQUE INDEX IF NOT EXISTS data_extrinsic_id ON extrinsics (block_id, extrinsic_idx);
-CREATE INDEX IF NOT EXISTS data_extrinsic_block_id ON extrinsics (block_id);
-CREATE INDEX IF NOT EXISTS data_extrinsic_extrinsic_idx ON extrinsics (extrinsic_idx);
-CREATE INDEX IF NOT EXISTS data_extrinsic_call_id ON extrinsics (call_id);
-CREATE INDEX IF NOT EXISTS data_extrinsic_address ON extrinsics (address);
-CREATE INDEX IF NOT EXISTS data_extrinsic_module_id ON extrinsics (module_id);
-CREATE INDEX IF NOT EXISTS data_extrinsic_signed ON extrinsics (signed);
-
 CREATE UNIQUE INDEX IF NOT EXISTS data_event_id ON events (block_id, event_idx);
-CREATE INDEX IF NOT EXISTS data_event_block_id ON events (block_id);
-CREATE INDEX IF NOT EXISTS data_event_event_idx ON events (event_idx);
-CREATE INDEX IF NOT EXISTS data_event_extrinsic_idx ON events (extrinsic_idx);
-CREATE INDEX IF NOT EXISTS data_event_module_id ON events (module_id);
-CREATE INDEX IF NOT EXISTS data_event_event_id ON events (event_id);
+
+-- Expression indexes over the event argument columns. Each is indexed on its first 100
+-- characters to keep the entry inside Postgres' btree row limit, which no directive can say.
 CREATE INDEX IF NOT EXISTS data_event_event_arg_0 ON events (left(event_arg_0, 100));
 CREATE INDEX IF NOT EXISTS data_event_event_arg_1 ON events (left(event_arg_1, 100));
 CREATE INDEX IF NOT EXISTS data_event_event_arg_2 ON events (left(event_arg_2, 100));
 CREATE INDEX IF NOT EXISTS data_event_event_arg_3 ON events (left(event_arg_3, 100));
-CREATE INDEX IF NOT EXISTS data_event_claim_type ON events (claim_type);
-CREATE INDEX IF NOT EXISTS data_event_claim_scope ON events (claim_scope);
-CREATE INDEX IF NOT EXISTS data_event_claim_issuer ON events (claim_issuer);
-CREATE INDEX IF NOT EXISTS data_event_corporate_action_ticker ON events (corporate_action_ticker);
-CREATE INDEX IF NOT EXISTS data_event_fundraiser_offering_asset ON events (fundraiser_offering_asset);
-CREATE INDEX IF NOT EXISTS data_event_spec_version_id ON events (spec_version_id);
-CREATE INDEX IF NOT EXISTS data_event_module_id_event_id ON events (module_id, event_id);
 CREATE INDEX IF NOT EXISTS data_event_module_id_event_id_event_arg_2 ON events (module_id, event_id, left(event_arg_2, 100));
+
+-- JSONB path index, over the generated column above. Neither the path expression nor the column
+-- it reads exists in `schema.graphql`.
 CREATE INDEX IF NOT EXISTS data_event_transfer_from ON events (trim( '"' from attributes #>> '{2,value,did}'));
 
+-- Legacy views, dropped if an older deployment left them behind.
 DROP VIEW IF EXISTS data_block;
 DROP VIEW IF EXISTS data_event;
 DROP VIEW IF EXISTS data_extrinsic;

docker/docker-entrypoint.sh

@@ -19,8 +19,6 @@ fi
 
 npm run build ## This creates the project.yaml file
 
-(npm run sql || (sleep 3 && kill "$$")) &
-
 npm run migrations
 
 # Allow configuring node memory. It should be no more than 75% of available RAM.
@@ -31,4 +29,16 @@ NODE_OPTIONS=--max_old_space_size="$NODE_SPACE" \
 	/bin/run --disable-historical=false \
 	--db-schema=public "$@" &
 child=$!
+
+# `db/compat.sql` adds generated columns and expression indexes to tables the node creates, so it
+# has to run after the node has built the schema. It used to run in the background with a
+# `kill "$$"` on failure, which raced the node and reported nothing useful when it lost. It waits
+# for the schema itself, so it is run here in the foreground and its failure stops the container.
+if ! npm run sql; then
+	echo "Failed to apply db/compat.sql; stopping the indexer" >&2
+	kill -TERM "$child" 2>/dev/null
+	wait "$child" || true
+	exit 1
+fi
+
 wait "$child"

package.json

@@ -19,6 +19,7 @@
     "codegen": "./node_modules/.bin/subql codegen",
     "typecheck": "yarn codegen && tsc --noEmit -p tsconfig.test.json",
     "check-handlers": "ts-node scripts/check-handlers.ts",
+    "sync-metadata": "ts-node scripts/sync-metadata.ts",
     "commit": "npx git-cz",
     "start:docker": "docker compose pull && docker compose up --remove-orphans",
     "stop:docker": "docker compose down -v",

schema.graphql

@@ -1773,8 +1773,8 @@ type Block @entity {
   id: ID!
   blockId: Int! @index(unique: true)
   parentId: Int!
-  hash: String! # a hash value
-  parentHash: String!
+  hash: String! @index(unique: true) # a hash value
+  parentHash: String! @index(unique: false)
   stateRoot: String!
   extrinsicsRoot: String!
   countExtrinsics: Int!
@@ -1789,6 +1789,27 @@ type Block @entity {
   extrinsics: [Extrinsic] @derivedFrom(field: "block")
 }
 
+"""
+A runtime upgrade the indexer observed, one row per spec version.
+
+Written from `system.CodeUpdated`, which the runtime emits for every upgrade. Two things read
+it: upgrade detection itself, which was module level mutable state and so held a separate copy
+per worker thread, and any handler that needs the block a spec version started at.
+"""
+type ChainUpgrade @entity {
+  "Zero padded spec version, so lexical order matches numeric order"
+  id: ID!
+  specVersionId: Int! @index(unique: true)
+  """
+  Transaction version the upgrade shipped. A bump here means the call encoding changed, which is
+  what distinguishes a major upgrade from a routine one
+  """
+  transactionVersion: Int!
+  "The block `system.CodeUpdated` was emitted in"
+  firstBlock: Block!
+  datetime: Date!
+}
+
 """
 Represents external data included into the chain. Virtually all user actions, as well as runtime operations are extrinsics
 
@@ -1797,15 +1818,15 @@ Usually extrinsics are signed. When the block author includes data, e.g. `timest
 type Extrinsic @entity {
   id: ID!
   block: Block! @index(unique: false)
-  extrinsicIdx: Int!
+  extrinsicIdx: Int! @index(unique: false)
   extrinsicLength: Int!
-  signed: Int!
+  signed: Int! @index(unique: false)
   "`signedbyAddress` is now deprecated in favour of `signed`"
   signedbyAddress: Int! @deprecated(reason: "use `signed` instead.")
-  address: String
-  moduleId: ModuleIdEnum!
+  address: String @index(unique: false)
+  moduleId: ModuleIdEnum! @index
   moduleIdText: String!
-  callId: CallIdEnum!
+  callId: CallIdEnum! @index
   callIdText: String!
   paramsTxt: String!
   success: Int! @index(unique: false)
@@ -1898,31 +1919,78 @@ type EvmAccountMapping @entity {
 """
 Information of a chain state transition on. For most use cases a more specific entity should be queried
 """
-type Event @entity {
+type Event @entity @compositeIndexes(fields: [["moduleId", "eventId"]]) {
   id: ID!
-  block: Block!
-  eventIdx: Int!
-  extrinsicIdx: Int
-  specVersionId: Int!
-  moduleId: ModuleIdEnum!
+  block: Block! @index(unique: false)
+  eventIdx: Int! @index(unique: false)
+  extrinsicIdx: Int @index(unique: false)
+  specVersionId: Int! @index(unique: false)
+  moduleId: ModuleIdEnum! @index
   moduleIdText: String!
-  eventId: EventIdEnum!
+  eventId: EventIdEnum! @index
   eventIdText: String!
   attributesTxt: String!
+  "Indexed in `db/compat.sql` as `left(event_arg_0, 100)`, which no directive can express"
   eventArg_0: String
   eventArg_1: String
   eventArg_2: String
   eventArg_3: String
-  claimType: String
-  claimScope: String
-  claimIssuer: String
+  claimType: String @index(unique: false)
+  claimScope: String @index(unique: false)
+  claimIssuer: String @index(unique: false)
   claimExpiry: String
-  corporateActionTicker: String
-  fundraiserOfferingAsset: String
+  corporateActionTicker: String @index(unique: false)
+  fundraiserOfferingAsset: String @index(unique: false)
   transferTo: String
   extrinsic: Extrinsic
 }
 
+"""
+What the indexer could not decode or resolve.
+
+Every value here names a specific failure the indexer would otherwise have swallowed - a
+decoder that did not match the block's shape, an enum value the schema does not know, a
+referenced entity that was never written.
+"""
+enum AnomalyKind {
+  "An event carried a different number of parameters than the decoder registered for its spec version declares"
+  ArityMismatch
+  "A named field lookup found no field of that name in the block's own metadata"
+  FieldNotFound
+  "No legacy decoder covers the spec version of the block the event was emitted in"
+  NoDecoderForSpecVersion
+  "A chain value did not match any member of the schema enum it maps to, and fell back to the `Unknown` member"
+  UnknownEnumValue
+  "A handler resolved a relation to an entity that does not exist in the index"
+  MissingReferencedEntity
+  "An accumulated balance disagreed with the same balance read from chain state"
+  BalanceReconciliationDrift
+  "A handler threw. The index is missing whatever that handler would have written"
+  HandlerError
+}
+
+"""
+Recorded whenever the indexer could not decode or resolve something.
+
+This table is a defect list, not a log. Every row is either a genuine chain oddity worth
+documenting or an indexer bug worth fixing, so a full resync that leaves it empty is the
+acceptance signal. Query `(kind, moduleId, eventId)` distinct to review them.
+"""
+type IndexerAnomaly @entity {
+  "`blockId/eventIdx/seq`, each part zero padded so lexical order matches emission order"
+  id: ID!
+  kind: AnomalyKind! @index
+  "The pallet the event came from, where the anomaly is attributable to one"
+  moduleId: ModuleIdEnum @index
+  "The event the anomaly is attributable to, where there is one"
+  eventId: EventIdEnum @index
+  "What was expected against what was seen - an arity, a field name, an entity id"
+  detail: String!
+  specVersionId: Int! @index
+  block: Block!
+  createdAt: Date!
+}
+
 """
 Represents which permissions an account may have over an Identity's assets
 """
@@ -2163,7 +2231,8 @@ type Authorization @entity {
   toKey: String @index(unique: false)
   data: String
   expiry: Date @index(unique: false)
-  status: AuthorizationStatusEnum!
+  "Indexed because `repairAuthorizationsAfterUpgrade` reads every pending row, and `store.getByFields` refuses to filter on a column with no index"
+  status: AuthorizationStatusEnum! @index
   createdBlock: Block!
   updatedBlock: Block!
   createdEvent: Event!