fix(gateway): reconcile durable storage on admission

Upgrade supported QuotaTracker, IdempotencyStore, and RateLimiter SQLite schemas atomically.
Reconcile when each object wakes, preserve validated rows, and retain the guarded maintenance path.

Author: iamjr15

apps/gateway-worker/README.md

@@ -88,34 +88,18 @@ request is made while either RLS transaction is open.
 Gateway emits `first_byok_key_added` after the first successful provider-key save
 and accepts authenticated `/v1/user-events` activation pings from the real web UI.
 
-Production releases use `CHEATCODE_RELEASE_GATE` as a fail-closed deployment
-barrier. The deploy operation first publishes the final gateway bundle with the
-gate set to `closed`; every public route, including `/health`, returns a
-non-cacheable `503`. Agent and webhooks are then deployed with their own gates
-set to `draining`; the gateway health body proves both service-bound downstream
-SHAs and gates. The release drains AgentRun and every webhook, ops, and
-resource-deletion Workflow before redeploying both services `closed` and allowing
-DDL. In steady state, the public 200 `/health` response also fails closed unless
-both downstream services report `open` at the gateway's exact release SHA. After closed reconciliation,
-contractions, and Vercel promotion, agent and webhooks reopen first and gateway
-opens last. Internal lifecycle work reaches quota state through the webhooks
-Worker's direct cross-Worker Durable Object binding, so gateway has no maintenance
-bypass route during the closed window.
-
-If a barrier step fails, the deploy operation re-deploys and verifies all three
-writer gates closed before stopping. If recovery cannot be verified, writer state
-is reported as unconfirmed and requires immediate inspection. Once closed,
-recover by rerunning the complete deployment from the same immutable commit. If
-that release cannot continue, keep the gateway closed and dispatch a reviewed,
-forward-compatible `stage-closed` release that explicitly names the superseded
-closed SHA. Never recover a schema contraction by deploying older code or bypass
-convergence by flipping the gate in the dashboard.
-
-The HTTP barrier stops new public work, and the draining agent/webhook gates fence
-new admissions while pinned Workflow and Durable Object continuations finish. The
-coordinated release drains relational AgentRun state and every retained writer
-Workflow before moving those services to `closed` and running DDL. Durable Object schema changes use
-explicit in-place reconciliation; the gate alone is not an atomic migration.
+Production deploys bind an immutable `CHEATCODE_RELEASE_SHA` into every affected
+Worker. Forward-compatible Durable Object storage changes reconcile transactionally
+when an existing object is first admitted, before the operation reaches an await.
+This keeps dormant objects deployable without a fleet-wide maintenance pass while
+preserving every validated source row. The signed internal storage route and
+`CHEATCODE_RELEASE_GATE=closed` remain available for explicit bulk verification
+and schema contractions that require a coordinated maintenance window.
+
+`CHEATCODE_RELEASE_GATE=open` is the steady state. A closed gateway rejects public
+work, and agent/webhook draining or closed gates fence new writer admissions.
+The gate is an operational barrier; SQLite reconciliation itself is performed by
+the object's synchronous transaction and verified against the exact schema.
 
 `IdempotencyStore` owns one exact SQLite table shape in its stable namespace and
 reconciles dormant objects to that shape when they are next activated. Run

apps/gateway-worker/src/durable-objects/idempotency-storage.ts

@@ -2,6 +2,7 @@ import {
   assertExactSqliteSchema,
   assertSqliteRowCountPreserved,
   type ExpectedSqliteObject,
+  reconcileExactSqliteStorage,
   setCurrentSqliteStorageVersion,
 } from "@cheatcode/durable-storage";
 
@@ -46,17 +47,29 @@ const IDEMPOTENCY_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [
   },
 ];
 
-/** Reconciles every dormant object to the one current persisted schema. */
-export function initializeIdempotencyStorage(ctx: DurableObjectState): void {
+function initializeIdempotencyStorage(ctx: DurableObjectState): void {
   normalizeIdempotencyStorage(ctx, false);
   assertIdempotencyStorage(ctx);
 }
 
+/** Opens an existing object on the exact schema before serving its next request. */
+export function ensureIdempotencyStorage(ctx: DurableObjectState): void {
+  if (!hasIdempotencyStorage(ctx)) {
+    initializeIdempotencyStorage(ctx);
+    return;
+  }
+  reconcileExactSqliteStorage(
+    "reconcile",
+    () => assertIdempotencyStorage(ctx),
+    () => reconcileIdempotencyStorage(ctx),
+  );
+}
+
 export function hasIdempotencyStorage(ctx: DurableObjectState): boolean {
   return tableColumns(ctx, "idempotency_entry").length > 0;
 }
 
-/** One-shot cutover normalizer; a later release removes this force-rebuild entrypoint. */
+/** Rebuilds a supported predecessor schema while preserving every stored row. */
 export function reconcileIdempotencyStorage(ctx: DurableObjectState): void {
   normalizeIdempotencyStorage(ctx, true);
   assertExactSqliteSchema(ctx, IDEMPOTENCY_STORAGE_SCHEMA);

apps/gateway-worker/src/durable-objects/idempotency.ts

@@ -18,8 +18,8 @@ import {
 } from "./idempotency-contract";
 import {
   assertIdempotencyStorage,
+  ensureIdempotencyStorage,
   hasIdempotencyStorage,
-  initializeIdempotencyStorage,
   reconcileIdempotencyStorage,
 } from "./idempotency-storage";
 import {
@@ -92,7 +92,7 @@ export class IdempotencyStore extends DurableObject<IdempotencyEnv> {
       await rearmClosedGatewayDurableObjectAlarm(this.ctx);
       return;
     }
-    this.isStorageInitialized = true;
+    this.ensureStorage();
     this.deleteExpired(Date.now());
     await this.scheduleNextAlarm();
   }
@@ -186,11 +186,7 @@ export class IdempotencyStore extends DurableObject<IdempotencyEnv> {
     if (this.isStorageInitialized) {
       return;
     }
-    if (hasIdempotencyStorage(this.ctx)) {
-      assertIdempotencyStorage(this.ctx);
-    } else {
-      initializeIdempotencyStorage(this.ctx);
-    }
+    ensureIdempotencyStorage(this.ctx);
     this.isStorageInitialized = true;
   }
 }

apps/gateway-worker/src/durable-objects/quota-tracker-storage.ts

@@ -2,6 +2,7 @@ import {
   assertExactSqliteSchema,
   assertSqliteRowCountPreserved,
   type ExpectedSqliteObject,
+  reconcileExactSqliteStorage,
   setCurrentSqliteStorageVersion,
 } from "@cheatcode/durable-storage";
 import { QUOTA_FEATURES } from "@cheatcode/types/quota";
@@ -75,7 +76,27 @@ const QUOTA_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [
   },
 ];
 
-/** Force-normalizes all quota tables after the release barrier has drained every caller. */
+/**
+ * Opens quota storage and transactionally upgrades an older supported schema.
+ *
+ * Durable Object input gates serialize this synchronous admission path. The
+ * guarded maintenance route still calls the same reconciler for planned bulk
+ * verification, but ordinary forward-compatible releases cannot strand a
+ * dormant user object on its previous schema.
+ */
+export function ensureQuotaTrackerStorage(ctx: DurableObjectState): void {
+  if (!hasQuotaTrackerStorage(ctx)) {
+    initializeQuotaTrackerStorage(ctx);
+    return;
+  }
+  reconcileExactSqliteStorage(
+    "reconcile",
+    () => assertQuotaTrackerStorage(ctx),
+    () => reconcileQuotaTrackerStorage(ctx),
+  );
+}
+
+/** Force-normalizes all quota tables while preserving every valid source row. */
 export function reconcileQuotaTrackerStorage(ctx: DurableObjectState): void {
   ensureSourceTables(ctx);
   const limitColumns = ctx.storage.sql.exec("PRAGMA table_info(limit_override)").toArray();
@@ -94,7 +115,7 @@ export function assertQuotaTrackerStorage(ctx: DurableObjectState): void {
   assertExactSqliteSchema(ctx, QUOTA_STORAGE_SCHEMA);
 }
 
-export function initializeQuotaTrackerStorage(ctx: DurableObjectState): void {
+function initializeQuotaTrackerStorage(ctx: DurableObjectState): void {
   ensureSourceTables(ctx);
   ctx.storage.sql.exec(USAGE_EVENT_INDEX_SQL.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS"));
   ctx.storage.sql.exec(

apps/gateway-worker/src/durable-objects/quota-tracker.ts

@@ -33,8 +33,8 @@ import {
 } from "./quota-tracker-contract";
 import {
   assertQuotaTrackerStorage,
+  ensureQuotaTrackerStorage,
   hasQuotaTrackerStorage,
-  initializeQuotaTrackerStorage,
   reconcileQuotaTrackerStorage,
 } from "./quota-tracker-storage";
 import {
@@ -326,7 +326,7 @@ export class QuotaTracker extends DurableObject<QuotaTrackerEnv> {
       await rearmClosedGatewayDurableObjectAlarm(this.ctx);
       return;
     }
-    this.isStorageInitialized = true;
+    this.ensureStorage();
     this.ctx.storage.sql.exec(
       "DELETE FROM counter WHERE updated_at < ?",
       Date.now() - QUOTA_TRACKER_RETENTION_MS,
@@ -347,11 +347,7 @@ export class QuotaTracker extends DurableObject<QuotaTrackerEnv> {
     if (this.isStorageInitialized) {
       return;
     }
-    if (hasQuotaTrackerStorage(this.ctx)) {
-      assertQuotaTrackerStorage(this.ctx);
-    } else {
-      initializeQuotaTrackerStorage(this.ctx);
-    }
+    ensureQuotaTrackerStorage(this.ctx);
     this.isStorageInitialized = true;
   }
 

apps/gateway-worker/src/durable-objects/rate-limiter-storage.ts

@@ -2,6 +2,7 @@ import {
   assertExactSqliteSchema,
   assertSqliteRowCountPreserved,
   type ExpectedSqliteObject,
+  reconcileExactSqliteStorage,
   setCurrentSqliteStorageVersion,
 } from "@cheatcode/durable-storage";
 
@@ -19,12 +20,25 @@ const RATE_LIMITER_STORAGE_SCHEMA: readonly ExpectedSqliteObject[] = [
   { name: "bucket", sql: BUCKET_TABLE_SQL, tableName: "bucket", type: "table" },
 ];
 
-export function initializeRateLimiterStorage(ctx: DurableObjectState): void {
+function initializeRateLimiterStorage(ctx: DurableObjectState): void {
   ensureRateLimiterTable(ctx);
   setCurrentSqliteStorageVersion(ctx);
   assertRateLimiterStorage(ctx);
 }
 
+/** Opens an existing object on the exact schema before serving its next request. */
+export function ensureRateLimiterStorage(ctx: DurableObjectState): void {
+  if (!hasRateLimiterStorage(ctx)) {
+    initializeRateLimiterStorage(ctx);
+    return;
+  }
+  reconcileExactSqliteStorage(
+    "reconcile",
+    () => assertRateLimiterStorage(ctx),
+    () => reconcileRateLimiterStorage(ctx),
+  );
+}
+
 export function hasRateLimiterStorage(ctx: DurableObjectState): boolean {
   return ctx.storage.sql.exec("PRAGMA table_info(bucket)").toArray().length > 0;
 }

apps/gateway-worker/src/durable-objects/rate-limiter.ts

@@ -16,8 +16,8 @@ import {
 } from "./rate-limit-contract";
 import {
   assertRateLimiterStorage,
+  ensureRateLimiterStorage,
   hasRateLimiterStorage,
-  initializeRateLimiterStorage,
   reconcileRateLimiterStorage,
 } from "./rate-limiter-storage";
 import {
@@ -126,7 +126,7 @@ export class RateLimiter extends DurableObject<RateLimiterEnv> {
       await rearmClosedGatewayDurableObjectAlarm(this.ctx);
       return;
     }
-    this.isStorageInitialized = true;
+    this.ensureStorage();
     this.ctx.storage.sql.exec(
       "DELETE FROM bucket WHERE last_refill_ms < ?",
       Date.now() - RATE_LIMITER_RETENTION_MS,
@@ -143,11 +143,7 @@ export class RateLimiter extends DurableObject<RateLimiterEnv> {
     if (this.isStorageInitialized) {
       return;
     }
-    if (hasRateLimiterStorage(this.ctx)) {
-      assertRateLimiterStorage(this.ctx);
-    } else {
-      initializeRateLimiterStorage(this.ctx);
-    }
+    ensureRateLimiterStorage(this.ctx);
     this.isStorageInitialized = true;
   }