Fix/1763 settlement scheduler persist retry count

backend/src/markets/entities/market.entity.ts

@@ -126,6 +126,11 @@ export class Market {
   @Min(0)
   grace_period_seconds: number;
 
+  @Column({ type: 'int', default: 0 })
+  @IsNumber()
+  @Min(0)
+  settlement_attempt_count: number;
+
   @Column({ default: true })
   @IsBoolean()
   is_public: boolean;

backend/src/markets/market-settlement.scheduler.spec.ts

@@ -40,6 +40,7 @@ describe('MarketSettlementScheduler', () => {
       resolution_proposed_at: new Date(currentNow.getTime() - 90_000_000),
       grace_period_seconds: 86400,
       is_resolved: false,
+      settlement_attempt_count: 0,
       ...overrides,
     }) as Market;
 
@@ -157,7 +158,10 @@ describe('MarketSettlementScheduler', () => {
     expect(queryRunners[0].manager.update).toHaveBeenCalledWith(
       Market,
       { id: 'market-1' },
-      { settlement_state: MarketSettlementState.SETTLING },
+      {
+        settlement_state: MarketSettlementState.SETTLING,
+        settlement_attempt_count: 1,
+      },
     );
     // Finalize transaction: market SETTLED, attempt RESOLVED.
     expect(queryRunners[1].manager.update).toHaveBeenCalledWith(
@@ -355,4 +359,86 @@ describe('MarketSettlementScheduler', () => {
 
     await expect(scheduler.handleSettlement()).resolves.not.toThrow();
   });
+
+  it('increments the persisted settlement_attempt_count on every claim', async () => {
+    const market = makeMarket({ settlement_attempt_count: 2 });
+    mockProposedCandidates([market]);
+    lockResultsQueue = [true, true];
+
+    const queryRunners: ReturnType<typeof makeQueryRunner>[] = [];
+    dataSource.createQueryRunner = jest.fn(() => {
+      const qr = makeQueryRunner();
+      qr.manager.findOne.mockResolvedValue(market);
+      queryRunners.push(qr);
+      return qr;
+    }) as unknown as jest.Mocked<DataSource>['createQueryRunner'];
+
+    await scheduler.settleEligibleMarkets();
+
+    expect(queryRunners[0].manager.update).toHaveBeenCalledWith(
+      Market,
+      { id: 'market-1' },
+      expect.objectContaining({ settlement_attempt_count: 3 }),
+    );
+  });
+
+  it('refuses to claim a market that has exhausted its persisted retry budget, even after a restart', async () => {
+    // No in-memory retry-queue entry (simulating a fresh process), but the
+    // persisted counter already recorded MAX_RETRY_ATTEMPTS failures.
+    const exhaustedMarket = makeMarket({ settlement_attempt_count: 5 });
+    mockProposedCandidates([exhaustedMarket]);
+
+    dataSource.createQueryRunner = jest.fn(() => {
+      const qr = makeQueryRunner();
+      qr.manager.findOne.mockResolvedValue(exhaustedMarket);
+      return qr;
+    }) as unknown as jest.Mocked<DataSource>['createQueryRunner'];
+
+    const settled = await scheduler.settleEligibleMarkets();
+
+    expect(settled).toBe(0);
+    expect(sorobanService.resolveMarket).not.toHaveBeenCalled();
+  });
+
+  it('routes a manual retry through the same transactional claim/finalize path as a fresh sweep', async () => {
+    // First sweep: the on-chain call fails, landing the market in the
+    // dead-letter queue with attempts=0.
+    const market = makeMarket();
+    mockProposedCandidates([market]);
+    lockResultsQueue = [true];
+    dataSource.createQueryRunner = jest.fn(() => {
+      const qr = makeQueryRunner();
+      qr.manager.findOne.mockResolvedValue(market);
+      return qr;
+    }) as unknown as jest.Mocked<DataSource>['createQueryRunner'];
+    sorobanService.resolveMarket.mockRejectedValueOnce(new Error('RPC down'));
+
+    await scheduler.settleEligibleMarkets();
+    expect(scheduler.getDeadLetterQueue()).toHaveLength(1);
+
+    // Manual retry (admin endpoint): should go through claimMarketForSettlement
+    // again (advisory lock + attempt row + persisted counter), not a bare
+    // resolveMarket call.
+    sorobanService.resolveMarket.mockResolvedValueOnce(undefined);
+    marketsRepository.findOne.mockResolvedValue(market);
+    lockResultsQueue = [true, true]; // claim, then finalize
+
+    const queryRunners: ReturnType<typeof makeQueryRunner>[] = [];
+    dataSource.createQueryRunner = jest.fn(() => {
+      const qr = makeQueryRunner();
+      qr.manager.findOne.mockResolvedValue(market);
+      queryRunners.push(qr);
+      return qr;
+    }) as unknown as jest.Mocked<DataSource>['createQueryRunner'];
+
+    const settled = await scheduler.retrySettlement('market-1');
+
+    expect(settled).toBe(true);
+    expect(scheduler.getDeadLetterQueue()).toHaveLength(0);
+    expect(queryRunners[0].manager.update).toHaveBeenCalledWith(
+      Market,
+      { id: 'market-1' },
+      expect.objectContaining({ settlement_attempt_count: 1 }),
+    );
+  });
 });

backend/src/markets/market-settlement.scheduler.ts

@@ -243,20 +243,33 @@ export class MarketSettlementScheduler {
         return null;
       }
 
+      if (fresh.settlement_attempt_count >= this.MAX_RETRY_ATTEMPTS) {
+        await queryRunner.rollbackTransaction();
+        this.logger.error(
+          `Market ${market.id} has exhausted its persisted retry budget (${fresh.settlement_attempt_count} attempts), skipping`,
+        );
+        return null;
+      }
+
       const attempt = queryRunner.manager.create(SettlementAttempt, {
         market_id: fresh.id,
         status: SettlementAttemptStatus.RESOLVING,
         proposed_outcome: fresh.proposed_outcome,
       });
       const savedAttempt = await queryRunner.manager.save(attempt);
 
-      if (fresh.settlement_state !== MarketSettlementState.SETTLING) {
-        await queryRunner.manager.update(
-          Market,
-          { id: fresh.id },
-          { settlement_state: MarketSettlementState.SETTLING },
-        );
-      }
+      // Persisted alongside the in-memory retry queue so the attempt count
+      // survives a process restart instead of resetting to zero.
+      await queryRunner.manager.update(
+        Market,
+        { id: fresh.id },
+        {
+          ...(fresh.settlement_state !== MarketSettlementState.SETTLING
+            ? { settlement_state: MarketSettlementState.SETTLING }
+            : {}),
+          settlement_attempt_count: fresh.settlement_attempt_count + 1,
+        },
+      );
 
       await queryRunner.commitTransaction();
       return {
@@ -364,38 +377,25 @@ export class MarketSettlementScheduler {
   }
 
   /**
-   * Settle market with retry tracking
+   * Settle market with retry tracking. Reuses the same claim/finalize
+   * transaction path as a fresh sweep (advisory lock, settlement-attempt
+   * row, persisted attempt counter) so a retry is never a second,
+   * untracked code path around the DB.
    */
   private async settleMarketWithRetry(
     market: Market,
     retryInfo: SettlementRetryInfo,
   ): Promise<boolean> {
-    try {
-      await this.sorobanService.resolveMarket(
-        market.on_chain_market_id,
-        market.proposed_outcome as string,
-      );
-
-      await this.webhookDispatcher.emit('market.settled', {
-        id: market.id,
-        on_chain_market_id: market.on_chain_market_id,
-        resolved_outcome: market.proposed_outcome,
-        settled_at: new Date(),
-      });
-
+    const settled = await this.settleMarket(market);
+    if (settled) {
       return true;
-    } catch (err) {
-      const errorMsg = err instanceof Error ? err.message : 'Unknown error';
-      this.logger.warn(
-        `Retry attempt ${retryInfo.attempts + 1} failed for market ${market.id}: ${errorMsg}`,
-      );
+    }
 
-      retryInfo.attempts++;
-      retryInfo.lastError = errorMsg;
-      retryInfo.nextRetryAt = this.calculateNextRetry(retryInfo.attempts);
+    retryInfo.attempts++;
+    retryInfo.lastError = 'Settlement attempt failed, see logs';
+    retryInfo.nextRetryAt = this.calculateNextRetry(retryInfo.attempts);
 
-      return false;
-    }
+    return false;
   }
 
   /**

backend/src/markets/public-markets.controller.spec.ts

@@ -28,6 +28,7 @@ describe('PublicMarketsController', () => {
     proposed_outcome: null,
     resolution_proposed_at: null,
     grace_period_seconds: 86400,
+    settlement_attempt_count: 0,
     total_pool_stroops: '1000',
     participant_count: 2,
     featured_at: null,

backend/src/migrations/1787900100000-AddSettlementAttemptCountToMarkets.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
+
+export class AddSettlementAttemptCountToMarkets1787900100000 implements MigrationInterface {
+  name = 'AddSettlementAttemptCountToMarkets1787900100000';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.addColumn(
+      'markets',
+      new TableColumn({
+        name: 'settlement_attempt_count',
+        type: 'int',
+        default: 0,
+        isNullable: false,
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropColumn('markets', 'settlement_attempt_count');
+  }
+}

backend/test/markets.e2e-spec.ts

@@ -71,6 +71,7 @@ describe('Markets (e2e)', () => {
     proposed_outcome: null,
     resolution_proposed_at: null,
     grace_period_seconds: 86400,
+    settlement_attempt_count: 0,
   };
 
   const mockResolvedMarket: Market = {