diff --git a/backend/src/markets/entities/market.entity.ts b/backend/src/markets/entities/market.entity.ts index ef73e4f2..2a50f0b2 100644 --- a/backend/src/markets/entities/market.entity.ts +++ b/backend/src/markets/entities/market.entity.ts @@ -126,6 +126,11 @@ export class Market { @Min(0) grace_period_seconds: number; + @Column({ type: 'int', default: 0 }) + @IsNumber() + @Min(0) + settlement_attempt_count: number; + @Column({ default: true }) @IsBoolean() is_public: boolean; diff --git a/backend/src/markets/market-settlement.scheduler.spec.ts b/backend/src/markets/market-settlement.scheduler.spec.ts index a553d98e..64950b9e 100644 --- a/backend/src/markets/market-settlement.scheduler.spec.ts +++ b/backend/src/markets/market-settlement.scheduler.spec.ts @@ -40,6 +40,7 @@ describe('MarketSettlementScheduler', () => { resolution_proposed_at: new Date(currentNow.getTime() - 90_000_000), grace_period_seconds: 86400, is_resolved: false, + settlement_attempt_count: 0, ...overrides, }) as Market; @@ -157,7 +158,10 @@ describe('MarketSettlementScheduler', () => { expect(queryRunners[0].manager.update).toHaveBeenCalledWith( Market, { id: 'market-1' }, - { settlement_state: MarketSettlementState.SETTLING }, + { + settlement_state: MarketSettlementState.SETTLING, + settlement_attempt_count: 1, + }, ); // Finalize transaction: market SETTLED, attempt RESOLVED. expect(queryRunners[1].manager.update).toHaveBeenCalledWith( @@ -355,4 +359,86 @@ describe('MarketSettlementScheduler', () => { await expect(scheduler.handleSettlement()).resolves.not.toThrow(); }); + + it('increments the persisted settlement_attempt_count on every claim', async () => { + const market = makeMarket({ settlement_attempt_count: 2 }); + mockProposedCandidates([market]); + lockResultsQueue = [true, true]; + + const queryRunners: ReturnType[] = []; + dataSource.createQueryRunner = jest.fn(() => { + const qr = makeQueryRunner(); + qr.manager.findOne.mockResolvedValue(market); + queryRunners.push(qr); + return qr; + }) as unknown as jest.Mocked['createQueryRunner']; + + await scheduler.settleEligibleMarkets(); + + expect(queryRunners[0].manager.update).toHaveBeenCalledWith( + Market, + { id: 'market-1' }, + expect.objectContaining({ settlement_attempt_count: 3 }), + ); + }); + + it('refuses to claim a market that has exhausted its persisted retry budget, even after a restart', async () => { + // No in-memory retry-queue entry (simulating a fresh process), but the + // persisted counter already recorded MAX_RETRY_ATTEMPTS failures. + const exhaustedMarket = makeMarket({ settlement_attempt_count: 5 }); + mockProposedCandidates([exhaustedMarket]); + + dataSource.createQueryRunner = jest.fn(() => { + const qr = makeQueryRunner(); + qr.manager.findOne.mockResolvedValue(exhaustedMarket); + return qr; + }) as unknown as jest.Mocked['createQueryRunner']; + + const settled = await scheduler.settleEligibleMarkets(); + + expect(settled).toBe(0); + expect(sorobanService.resolveMarket).not.toHaveBeenCalled(); + }); + + it('routes a manual retry through the same transactional claim/finalize path as a fresh sweep', async () => { + // First sweep: the on-chain call fails, landing the market in the + // dead-letter queue with attempts=0. + const market = makeMarket(); + mockProposedCandidates([market]); + lockResultsQueue = [true]; + dataSource.createQueryRunner = jest.fn(() => { + const qr = makeQueryRunner(); + qr.manager.findOne.mockResolvedValue(market); + return qr; + }) as unknown as jest.Mocked['createQueryRunner']; + sorobanService.resolveMarket.mockRejectedValueOnce(new Error('RPC down')); + + await scheduler.settleEligibleMarkets(); + expect(scheduler.getDeadLetterQueue()).toHaveLength(1); + + // Manual retry (admin endpoint): should go through claimMarketForSettlement + // again (advisory lock + attempt row + persisted counter), not a bare + // resolveMarket call. + sorobanService.resolveMarket.mockResolvedValueOnce(undefined); + marketsRepository.findOne.mockResolvedValue(market); + lockResultsQueue = [true, true]; // claim, then finalize + + const queryRunners: ReturnType[] = []; + dataSource.createQueryRunner = jest.fn(() => { + const qr = makeQueryRunner(); + qr.manager.findOne.mockResolvedValue(market); + queryRunners.push(qr); + return qr; + }) as unknown as jest.Mocked['createQueryRunner']; + + const settled = await scheduler.retrySettlement('market-1'); + + expect(settled).toBe(true); + expect(scheduler.getDeadLetterQueue()).toHaveLength(0); + expect(queryRunners[0].manager.update).toHaveBeenCalledWith( + Market, + { id: 'market-1' }, + expect.objectContaining({ settlement_attempt_count: 1 }), + ); + }); }); diff --git a/backend/src/markets/market-settlement.scheduler.ts b/backend/src/markets/market-settlement.scheduler.ts index f3c6b1d3..01ec679c 100644 --- a/backend/src/markets/market-settlement.scheduler.ts +++ b/backend/src/markets/market-settlement.scheduler.ts @@ -243,6 +243,14 @@ export class MarketSettlementScheduler { return null; } + if (fresh.settlement_attempt_count >= this.MAX_RETRY_ATTEMPTS) { + await queryRunner.rollbackTransaction(); + this.logger.error( + `Market ${market.id} has exhausted its persisted retry budget (${fresh.settlement_attempt_count} attempts), skipping`, + ); + return null; + } + const attempt = queryRunner.manager.create(SettlementAttempt, { market_id: fresh.id, status: SettlementAttemptStatus.RESOLVING, @@ -250,13 +258,18 @@ export class MarketSettlementScheduler { }); const savedAttempt = await queryRunner.manager.save(attempt); - if (fresh.settlement_state !== MarketSettlementState.SETTLING) { - await queryRunner.manager.update( - Market, - { id: fresh.id }, - { settlement_state: MarketSettlementState.SETTLING }, - ); - } + // Persisted alongside the in-memory retry queue so the attempt count + // survives a process restart instead of resetting to zero. + await queryRunner.manager.update( + Market, + { id: fresh.id }, + { + ...(fresh.settlement_state !== MarketSettlementState.SETTLING + ? { settlement_state: MarketSettlementState.SETTLING } + : {}), + settlement_attempt_count: fresh.settlement_attempt_count + 1, + }, + ); await queryRunner.commitTransaction(); return { @@ -364,38 +377,25 @@ export class MarketSettlementScheduler { } /** - * Settle market with retry tracking + * Settle market with retry tracking. Reuses the same claim/finalize + * transaction path as a fresh sweep (advisory lock, settlement-attempt + * row, persisted attempt counter) so a retry is never a second, + * untracked code path around the DB. */ private async settleMarketWithRetry( market: Market, retryInfo: SettlementRetryInfo, ): Promise { - try { - await this.sorobanService.resolveMarket( - market.on_chain_market_id, - market.proposed_outcome as string, - ); - - await this.webhookDispatcher.emit('market.settled', { - id: market.id, - on_chain_market_id: market.on_chain_market_id, - resolved_outcome: market.proposed_outcome, - settled_at: new Date(), - }); - + const settled = await this.settleMarket(market); + if (settled) { return true; - } catch (err) { - const errorMsg = err instanceof Error ? err.message : 'Unknown error'; - this.logger.warn( - `Retry attempt ${retryInfo.attempts + 1} failed for market ${market.id}: ${errorMsg}`, - ); + } - retryInfo.attempts++; - retryInfo.lastError = errorMsg; - retryInfo.nextRetryAt = this.calculateNextRetry(retryInfo.attempts); + retryInfo.attempts++; + retryInfo.lastError = 'Settlement attempt failed, see logs'; + retryInfo.nextRetryAt = this.calculateNextRetry(retryInfo.attempts); - return false; - } + return false; } /** diff --git a/backend/src/markets/public-markets.controller.spec.ts b/backend/src/markets/public-markets.controller.spec.ts index 65690fd9..1b2e2503 100644 --- a/backend/src/markets/public-markets.controller.spec.ts +++ b/backend/src/markets/public-markets.controller.spec.ts @@ -28,6 +28,7 @@ describe('PublicMarketsController', () => { proposed_outcome: null, resolution_proposed_at: null, grace_period_seconds: 86400, + settlement_attempt_count: 0, total_pool_stroops: '1000', participant_count: 2, featured_at: null, diff --git a/backend/src/migrations/1787900100000-AddSettlementAttemptCountToMarkets.ts b/backend/src/migrations/1787900100000-AddSettlementAttemptCountToMarkets.ts new file mode 100644 index 00000000..0f6bd884 --- /dev/null +++ b/backend/src/migrations/1787900100000-AddSettlementAttemptCountToMarkets.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; + +export class AddSettlementAttemptCountToMarkets1787900100000 implements MigrationInterface { + name = 'AddSettlementAttemptCountToMarkets1787900100000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn( + 'markets', + new TableColumn({ + name: 'settlement_attempt_count', + type: 'int', + default: 0, + isNullable: false, + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn('markets', 'settlement_attempt_count'); + } +} diff --git a/backend/test/markets.e2e-spec.ts b/backend/test/markets.e2e-spec.ts index eade182b..5b71198e 100644 --- a/backend/test/markets.e2e-spec.ts +++ b/backend/test/markets.e2e-spec.ts @@ -71,6 +71,7 @@ describe('Markets (e2e)', () => { proposed_outcome: null, resolution_proposed_at: null, grace_period_seconds: 86400, + settlement_attempt_count: 0, }; const mockResolvedMarket: Market = {