diff --git a/README.md b/README.md index 5d6f9cc8..954dc023 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ newer Bee versions is not recommended and may not work. Stay up to date by joini - [2. Hardcoded postage stamp](#2-hardcoded-postage-stamp) - [3. Autobuy postage stamps](#3-autobuy-postage-stamps) - [4. Extends stamps TTL](#4-extends-stamps-ttl) + - [5. Extends stamps capacity](#5-extends-stamps-capacity) - [Enable authentication](#enable-authentication) - [Environment variables](#environment-variables) - [Curl](#curl) @@ -56,6 +57,9 @@ The proxy can manage postage stamps for you in 4 modes of operation: 4. It can extend the TTL of a stamp that is about to expire. To enable this, set `POSTAGE_EXTENDSTTL=true`, provide `POSTAGE_AMOUNT`, `POSTAGE_DEPTH` with the desired amount to use and `POSTAGE_TTL_MIN` above with a number above or equal to 60. +5. It can extends the postage stamp capacity to those that are about to be fulfill. To enable this, set + `POSTAGE_EXTENDS_CAPACITY=true`. You can also set the env variable `POSTAGE_USAGE_THRESHOLD=0.7` to determine + the maximum usage level to check if the stamp needs to be extended In modes 1, 2 and 3, the proxy can be configured to require authentication secret to forward the requests. Use the `AUTH_SECRET` environment variable to enable it. @@ -95,7 +99,7 @@ npm run start ```sh export POSTAGE_DEPTH=20 export POSTAGE_AMOUNT=1000000 -export BEE_DEBUG_API_URL=http://localhost:1635 +export BEE_DEBUG_API_URL=http://localhost:1635 (optional) npm run start ``` @@ -107,7 +111,15 @@ export POSTAGE_EXTENDSTTL=true export POSTAGE_TTL_MIN=60 export POSTAGE_DEPTH=20 export POSTAGE_AMOUNT=1000000 -export BEE_DEBUG_API_URL=http://localhost:1635 +export BEE_DEBUG_API_URL=http://localhost:1635 (optional) + +npm run start +``` + +#### 5. Extends stamps capacity + +```sh +export POSTAGE_EXTENDS_CAPACITY=true npm run start ``` @@ -149,6 +161,8 @@ npm run start | REMOVE_PIN_HEADER | true | Removes swarm-pin header on all proxy requests. | | `LOG_LEVEL` | info | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`) | | POSTAGE_EXTENDSTTL | false | Enables extends TTL feature. Works along with POSTAGE_AMOUNT | +| POSTAGE_EXTENDS_CAPACITY | false | Enables extending stamp capacity +feature. | | EXPOSE_HASHED_IDENTITY | false | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes | | REUPLOAD_PERIOD | undefined | How frequently are the pinned content checked to be reuploaded. | diff --git a/src/config.ts b/src/config.ts index 6230b410..5b1765db 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,5 @@ +import { isInteger, assertInteger, assertDecimal } from './utils' + export interface AppConfig { beeApiUrl: string beeDebugApiUrl: string @@ -14,32 +16,36 @@ export interface ServerConfig { port: number } -interface StampsConfigHardcoded { +export interface StampsConfigHardcoded { mode: 'hardcoded' stamp: string } -export interface StampsConfigExtends { - mode: 'extendsTTL' + +export interface StampsConfigAutobuy { + mode: 'autobuy' ttlMin: number depth: number amount: string + usageThreshold: number refreshPeriod: number + usageMax: number beeDebugApiUrl: string } -export interface ContentConfigReupload { - beeApiUrl: string - refreshPeriod: number -} - -export interface StampsConfigAutobuy { - mode: 'autobuy' +export interface StampsConfigExtends { + mode: 'extends' + enableTtl: boolean + enableCapacity: boolean + ttlMin: number depth: number amount: string - beeDebugApiUrl: string usageThreshold: number - usageMax: number - ttlMin: number + refreshPeriod: number + beeDebugApiUrl: string +} + +export interface ContentConfigReupload { + beeApiUrl: string refreshPeriod: number } @@ -80,6 +86,7 @@ export type EnvironmentVariables = Partial<{ POSTAGE_REFRESH_PERIOD: string POSTAGE_EXTENDSTTL: string REUPLOAD_PERIOD: string + POSTAGE_EXTENDS_CAPACITY: string }> export const SUPPORTED_LEVELS = ['critical', 'error', 'warn', 'info', 'verbose', 'debug'] as const @@ -138,45 +145,44 @@ export function getStampsConfig({ POSTAGE_TTL_MIN, POSTAGE_REFRESH_PERIOD, POSTAGE_EXTENDSTTL, + POSTAGE_EXTENDS_CAPACITY, }: EnvironmentVariables = {}): StampsConfig | undefined { + if (POSTAGE_REFRESH_PERIOD) { + assertInteger(POSTAGE_REFRESH_PERIOD) + } + const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD) const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL // Start in hardcoded mode if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP } // Start autobuy - else if (!POSTAGE_EXTENDSTTL && POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { - return { - mode: 'autobuy', - depth: Number(POSTAGE_DEPTH), - amount: POSTAGE_AMOUNT, - usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), - usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), - ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), + else if (POSTAGE_EXTENDSTTL === 'true' || POSTAGE_EXTENDS_CAPACITY === 'true') { + return createExtendsStampsConfig( + POSTAGE_EXTENDSTTL, + POSTAGE_EXTENDS_CAPACITY, + POSTAGE_AMOUNT, + POSTAGE_USAGE_THRESHOLD, + POSTAGE_TTL_MIN, + POSTAGE_DEPTH, refreshPeriod, beeDebugApiUrl, - } - } else if ( - POSTAGE_EXTENDSTTL === 'true' && - POSTAGE_AMOUNT && - POSTAGE_DEPTH && - Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE - ) { - return { - mode: 'extendsTTL', - depth: Number(POSTAGE_DEPTH), - ttlMin: Number(POSTAGE_TTL_MIN), - amount: POSTAGE_AMOUNT, + ) + } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT) { + return createAutobuyStampsConfig( + POSTAGE_DEPTH, + POSTAGE_AMOUNT, + POSTAGE_USAGE_THRESHOLD, + POSTAGE_USAGE_MAX, + POSTAGE_TTL_MIN, refreshPeriod, beeDebugApiUrl, - } + ) } // Missing one of the variables needed for the autobuy or extends TTL - else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) { + else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN) { throw new Error( - `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} ${ - POSTAGE_EXTENDSTTL === 'true' ? 'at least 60 seconds ' : '' - }or BEE_DEBUG_API_URL=${BEE_DEBUG_API_URL} for the feature to work`, + `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} for the feature to work`, ) } @@ -184,6 +190,84 @@ export function getStampsConfig({ return undefined } +export function createAutobuyStampsConfig( + POSTAGE_DEPTH: string, + POSTAGE_AMOUNT: string, + POSTAGE_USAGE_THRESHOLD: string | undefined, + POSTAGE_USAGE_MAX: string | undefined, + POSTAGE_TTL_MIN: string | undefined, + refreshPeriod: number, + beeDebugApiUrl: string, +): StampsConfigAutobuy { + // Missing one of the variables needed for the autobuy + if (!isInteger(POSTAGE_DEPTH) || !isInteger(POSTAGE_AMOUNT)) { + throw new Error( + `config: please provide valid values for POSTAGE_DEPTH, POSTAGE_AMOUNT for the autobuy feature to work. + Current state are POSTAGE_DEPTH=${POSTAGE_DEPTH} and POSTAGE_AMOUNT=${POSTAGE_AMOUNT}`, + ) + } + + return { + mode: 'autobuy', + depth: Number(POSTAGE_DEPTH), + amount: POSTAGE_AMOUNT, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), + usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), + ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), + refreshPeriod, + beeDebugApiUrl, + } +} + +export function createExtendsStampsConfig( + POSTAGE_EXTENDSTTL: string | undefined, + POSTAGE_EXTENDS_CAPACITY: string | undefined, + POSTAGE_AMOUNT: string | undefined, + POSTAGE_USAGE_THRESHOLD: string | undefined, + POSTAGE_TTL_MIN: string | undefined, + POSTAGE_DEPTH: string | undefined, + refreshPeriod: number, + beeDebugApiUrl: string, +): StampsConfigExtends { + if ( + POSTAGE_EXTENDSTTL === 'true' && + (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE || + !isInteger(POSTAGE_AMOUNT) || + (POSTAGE_TTL_MIN && !isInteger(POSTAGE_TTL_MIN)) || + (POSTAGE_DEPTH && !isInteger(POSTAGE_DEPTH))) + ) { + throw new Error( + `config: to extends stamps TTL please provide POSTAGE_TTL_MIN bigger than ${MINIMAL_EXTENDS_TTL_VALUE}, valid values for + POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN}, + POSTAGE_AMOUNT=${POSTAGE_AMOUNT} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`, + ) + } + + if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !assertDecimal(POSTAGE_USAGE_THRESHOLD)) { + throw new Error( + `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is + POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`, + ) + } + + const amount = POSTAGE_AMOUNT || '0' + const usageThreshold = Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD) + const ttlMin = Number(POSTAGE_TTL_MIN) + const depth = Number(POSTAGE_DEPTH) + + return { + mode: 'extends', + enableTtl: POSTAGE_EXTENDSTTL === 'true', + enableCapacity: POSTAGE_EXTENDS_CAPACITY === 'true', + depth, + ttlMin, + amount, + usageThreshold, + refreshPeriod, + beeDebugApiUrl, + } +} + export function getContentConfig({ BEE_API_URL, REUPLOAD_PERIOD }: EnvironmentVariables = {}): ContentConfig | false { if (!REUPLOAD_PERIOD) { return false diff --git a/src/content.ts b/src/content.ts index b0c80058..b0b0fd1e 100644 --- a/src/content.ts +++ b/src/content.ts @@ -14,40 +14,36 @@ export class ContentManager { private interval?: ReturnType private isReuploading = false - public async attemptRefreshContentReupload(beeApi: Bee): Promise { - try { - await this.refreshContentReupload(beeApi) - } catch (error) { - logger.error('content reupload job failed', error) - } - } - public async refreshContentReupload(beeApi: Bee): Promise { - const pins = await beeApi.getAllPins() + try { + const pins = await beeApi.getAllPins() - if (!pins.length) { - logger.info(`no pins found`) + if (!pins.length) { + logger.info(`no pins found`) - return - } + return + } - logger.info(`checking pinned content (${pins.length} pins)`) - for (const pin of pins) { - const isRetrievable = await beeApi.isReferenceRetrievable(pin) - logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`) + logger.info(`checking pinned content (${pins.length} pins)`) + for (const pin of pins) { + const isRetrievable = await beeApi.isReferenceRetrievable(pin) + logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`) - if (!isRetrievable && !this.isReuploading) { - this.isReuploading = true - try { - logger.debug(`reuploading pinned content: ${pin}`) - await beeApi.reuploadPinnedData(pin) - contentReuploadCounter.inc() - logger.info(`pinned content reuploaded: ${pin}`) - } catch (error) { - logger.error('failed to reupload pinned content', error) + if (!isRetrievable && !this.isReuploading) { + this.isReuploading = true + try { + logger.debug(`reuploading pinned content: ${pin}`) + await beeApi.reuploadPinnedData(pin) + contentReuploadCounter.inc() + logger.info(`pinned content reuploaded: ${pin}`) + } catch (error) { + logger.error('failed to reupload pinned content', error) + } + this.isReuploading = false } - this.isReuploading = false } + } catch (error) { + logger.error('content reupload job failed', error) } } @@ -55,7 +51,8 @@ export class ContentManager { * Start the manager that checks for pinned content availability and reuploads the data if needed. */ start(config: ContentConfig): void { - const refreshContent = async () => this.attemptRefreshContentReupload(new Bee(config.beeApiUrl)) + const bee = new Bee(config.beeApiUrl) + const refreshContent = async () => this.refreshContentReupload(bee) this.stop() refreshContent() diff --git a/src/index.ts b/src/index.ts index 02137b08..e7ef4238 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,10 +2,13 @@ import { Application } from 'express' import { createApp } from './server' -import { StampsManager } from './stamps' +import { AutoBuyStampsManager, ExtendsStampManager } from './stamps' import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config' import { logger, subscribeLogServerRequests } from './logger' import { ContentManager } from './content' +import type { StampsManager } from './stamps' +import { BeeDebug } from '@ethersphere/bee-js' +import { BaseStampManager } from './stamps/base' async function main() { // Configuration @@ -28,9 +31,22 @@ async function main() { if (stampsConfig) { logger.debug('stamps config', stampsConfig) - const stampManager = new StampsManager() - logger.info('starting postage stamp manager') - stampManager.start(stampsConfig) + let stampManager: StampsManager + const { mode } = stampsConfig + + if (mode === 'hardcoded') { + stampManager = new BaseStampManager() + stampManager.start(stampsConfig) + logger.info('starting hardcoded postage stamp manager') + } else if (mode === 'autobuy') { + logger.info('starting autobuy postage stamp manager') + stampManager = new AutoBuyStampsManager(new BeeDebug(stampsConfig.beeDebugApiUrl)) + stampManager.start(stampsConfig, async () => (stampManager as AutoBuyStampsManager).refreshStamps(stampsConfig)) + } else { + logger.info('starting extends postage stamp manager') + stampManager = new ExtendsStampManager(new BeeDebug(stampsConfig.beeDebugApiUrl)) + stampManager.start(stampsConfig, async () => (stampManager as ExtendsStampManager).refreshStamps(stampsConfig)) + } logger.info('starting the proxy') app = createApp(appConfig, stampManager) } else { diff --git a/src/readiness.ts b/src/readiness.ts index 17248e59..d147847e 100644 --- a/src/readiness.ts +++ b/src/readiness.ts @@ -1,7 +1,7 @@ import { Bee, BeeDebug, Utils } from '@ethersphere/bee-js' import { ERROR_NO_STAMP, READINESS_TIMEOUT_MS } from './config' import { logger } from './logger' -import { StampsManager } from './stamps' +import type { StampsManager } from './stamps' import { getErrorMessage } from './utils' const MAX_CHUNK_SIZE = 4096 @@ -37,7 +37,7 @@ export async function checkReadiness( async function tryUploadingSingleChunk(bee: Bee, stampsManager: StampsManager): Promise { const chunk = makeChunk() try { - await bee.uploadChunk(stampsManager.postageStamp, chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) + await bee.uploadChunk(stampsManager.postageStamp(), chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) return ReadinessStatus.OK } catch (error) { diff --git a/src/server.ts b/src/server.ts index 918cfa86..f7a84ef8 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,8 +7,8 @@ import { fetchBeeIdentity, getHashedIdentity, HASHED_IDENTITY_HEADER } from './i import { logger } from './logger' import { register } from './metrics' import { checkReadiness, ReadinessStatus } from './readiness' -import type { StampsManager } from './stamps' import { getErrorMessage } from './utils' +import type { StampsManager } from './stamps/base' const SWARM_STAMP_HEADER = 'swarm-postage-batch-id' @@ -136,7 +136,7 @@ export const createApp = ( if (stampManager) { proxyReq.removeHeader(SWARM_STAMP_HEADER) try { - proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp) + proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp()) } catch (error) { logger.error('proxy failure', error) diff --git a/src/stamps.ts b/src/stamps.ts deleted file mode 100644 index f519ec9c..00000000 --- a/src/stamps.ts +++ /dev/null @@ -1,326 +0,0 @@ -import { BeeDebug, PostageBatch, BatchId } from '@ethersphere/bee-js' -import client from 'prom-client' -import { ERROR_NO_STAMP, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from './config' -import { logger } from './logger' -import { register } from './metrics' -import { waitForStampUsable } from './utils' - -interface Options { - timeout?: number -} - -const stampPurchaseCounter = new client.Counter({ - name: 'stamp_purchase_counter', - help: 'How many stamps were purchased', -}) -register.registerMetric(stampPurchaseCounter) - -const stampPurchaseFailedCounter = new client.Counter({ - name: 'stamp_purchase_failed_counter', - help: 'How many stamps failed to be purchased', -}) -register.registerMetric(stampPurchaseFailedCounter) - -const stampCheckCounter = new client.Counter({ - name: 'stamp_check_counter', - help: 'How many times were stamps retrieved from server', -}) -register.registerMetric(stampCheckCounter) - -const stampGetCounter = new client.Counter({ - name: 'stamp_get_counter', - help: 'How many times was get postageStamp called', -}) -register.registerMetric(stampGetCounter) - -const stampGetErrorCounter = new client.Counter({ - name: 'stamp_get_error_counter', - help: 'How many times was get postageStamp called and there was no valid postage stamp', -}) -register.registerMetric(stampGetErrorCounter) - -const stampTtlGauge = new client.Gauge({ - name: 'stamp_ttl_gauge', - help: 'TTL on the selected automanaged stamp', -}) -register.registerMetric(stampTtlGauge) - -const stampUsageGauge = new client.Gauge({ - name: 'stamp_usage_gauge', - help: 'Usage on the selected automanaged stamp', -}) -register.registerMetric(stampUsageGauge) - -const stampUsableCountGauge = new client.Gauge({ - name: 'stamp_usable_count_gauge', - help: 'How many stamps exist on the bee node that can be used', -}) -register.registerMetric(stampUsableCountGauge) - -/** - * Calculate usage of a given postage stamp - * - * @param stamp Postage stamp which usage should be determined - */ -export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): number { - return utilization / Math.pow(2, depth - bucketDepth) -} - -/** - * Filter the stamps and only return those that are usable, have correct amount, depth, are not close to beying maxUsage or close to expire - * - * @param stamps Postage stamps to be filtered - * @param depth Postage stamps depth - * @param amount Postage stamps amount - * @param maxUsage Maximal usage of the stamp to be usable by the proxy - * @param minTTL Minimal TTL of the stamp to be usable by the proxy - * - * @returns Filtered stamps soltered by usage - */ -export function filterUsableStampsAutobuy( - stamps: PostageBatch[], - depth: number, - amount: string, - maxUsage: number, - minTTL: number, -): PostageBatch[] { - const usableStamps = stamps - // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable && s.depth === depth && s.amount === amount && getUsage(s) < maxUsage && s.batchTTL > minTTL) - // sort the stamps by usage - .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) - - // return the all usable stamp sorted by usage - return usableStamps -} - -/** - * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL - * - * @param stamps Postage stamps to be filtered - * - * @returns Filtered stamps soltered by usage - */ -export function filterUsableStampsExtends(stamps: PostageBatch[]): PostageBatch[] { - const usableStamps = stamps - // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable) - // sort the stamps by usage - .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) - - // return the all usable stamp sorted by usage - return usableStamps -} - -/** - * Buy new postage stamp and wait until it is usable - * - * @param depth Postage stamps depth - * @param amount Postage stamps amount - * @param beeDebug Connection to debug endpoint for checking/buying stamps - * @param options - * timeout (optional) How long should the system wait for the stamp to be usable in ms, default to 10000 - * - * @returns Newly bought postage stamp - */ -export async function buyNewStamp( - depth: number, - amount: string, - beeDebug: BeeDebug, -): Promise<{ batchId: BatchId; stamp: PostageBatch }> { - logger.info('buying new stamp') - const batchId = await beeDebug.createPostageBatch(amount, depth, { waitForUsable: false }) - await waitForStampUsable(beeDebug, batchId) - stampPurchaseCounter.inc() - - const stamp = await beeDebug.getPostageBatch(batchId) - logger.info('successfully bought new stamp', { stamp }) - - return { batchId, stamp } -} - -export async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { - await beeDebug.topUpBatch(postageBatchId, amount) - const stamp = await beeDebug.getPostageBatch(postageBatchId) - - return stamp -} - -export class StampsManager { - private stamp?: string - private usableStamps?: PostageBatch[] - private interval?: ReturnType - private isBuyingStamp?: boolean = false - private extendingStamps: string[] = [] - - /** - * Get postage stamp that should be replaced in a the proxy request header - * - * @return Postage stamp that should be used by the proxy - * - * @throws Error if there is no postage stamp - */ - get postageStamp(): string { - stampGetCounter.inc() - - if (this.stamp) { - const stamp = this.stamp - logger.info('using hardcoded stamp', { stamp }) - - return stamp - } - - if (this.usableStamps && this.usableStamps[0]) { - const stamp = this.usableStamps[0] - logger.info('using autobought stamp', { stamp }) - - return stamp.batchID - } - - stampGetErrorCounter.inc() - throw new Error(ERROR_NO_STAMP) - } - - /** - * Refresh stamps from the bee node and if needed buy new stamp - * - * @param config Stamps config - * @param beeDebug Connection to debug endpoint for checking/buying stamps - */ - public async refreshStampsAutobuy(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { - try { - stampCheckCounter.inc() - logger.info('checking postage stamps') - const stamps = await beeDebug.getAllPostageBatch() - logger.debug('retrieved stamps', stamps) - - const { depth, amount, usageMax, usageThreshold, ttlMin } = config - - // Get all usable stamps sorted by usage from most used to least - this.usableStamps = filterUsableStampsAutobuy(stamps, depth, amount, usageMax, ttlMin) - const leastUsed = this.usableStamps[this.usableStamps.length - 1] - const mostUsed = this.usableStamps[0] - - stampTtlGauge.set(mostUsed ? mostUsed.batchTTL : 0) - stampUsageGauge.set(mostUsed ? getUsage(mostUsed) : 0) - stampUsableCountGauge.set(this.usableStamps.length) - - // Check if the least used stamps is starting to get full and if so purchase new stamp - if (!this.isBuyingStamp && (!leastUsed || getUsage(leastUsed) > usageThreshold)) { - this.isBuyingStamp = true - try { - const { stamp } = await buyNewStamp(depth, amount, beeDebug) - - // Add the bought postage stamp - this.usableStamps.push(stamp) - stampUsableCountGauge.set(this.usableStamps.length) - } catch (e) { - logger.error('failed to buy postage stamp', e) - stampPurchaseFailedCounter.inc() - } finally { - this.isBuyingStamp = false - } - } - } catch (e) { - logger.error('failed to refresh postage stamp', e) - } - } - - public async refreshStampsExtends(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { - stampCheckCounter.inc() - logger.info('checking postage stamps') - - try { - const stamps = await beeDebug.getAllPostageBatch() - logger.debug('retrieved stamps', stamps) - - const { amount, ttlMin, depth } = config - - // Get all usable stamps sorted by usage from most used to least - this.usableStamps = filterUsableStampsExtends(stamps) - - if (!this.isBuyingStamp) { - if (this.usableStamps.length === 0) { - this.isBuyingStamp = true - try { - const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) - - // Add the bought postage stamp - this.usableStamps.push(newStamp) - } finally { - this.isBuyingStamp = false - } - } else { - await this.verifyUsableStamps(beeDebug, ttlMin, config, amount) - } - } - } catch (e) { - logger.error('failed to refresh on extends postage stamps', e) - } - } - - async verifyUsableStamps( - beeDebug: BeeDebug, - ttlMin: number, - config: StampsConfigAutobuy | StampsConfigExtends, - amount: string, - ) { - for (let i = 0; i < this.usableStamps!.length; i++) { - const stamp = this.usableStamps![i] - - const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 - - if (stamp.batchTTL < minTimeThreshold && !this.extendingStamps.includes(stamp.batchID)) { - this.extendingStamps.push(stamp.batchID) - logger.info(`extending postage stamp ${stamp.batchID}`) - - try { - const stampRes = await topUpStamp(beeDebug, stamp.batchID, amount) - - setTimeout(() => this.completeTopUp(stampRes), 60000) - } catch (e: any) { - // error that indicate that 2 stamps are trying to be extended at the same time. Comes out as a warning - const errorStampIndex = this.extendingStamps.indexOf(stamp.batchID) - this.extendingStamps.splice(errorStampIndex, 1) - logger.error('failed to topup postage stamp', e) - } - } - } - } - - completeTopUp(stamp: PostageBatch) { - logger.info('successfully extended postage stamp', { stamp }) - // remove stamps from extending stamps array - const stampIndex = this.extendingStamps.findIndex(id => stamp.batchID === id) - this.extendingStamps.splice(stampIndex, 1) - } - - /** - * Start the manager in either hardcoded or autobuy mode - */ - async start(config: StampsConfig): Promise { - // Hardcoded stamp mode - if (config.mode === 'hardcoded') this.stamp = config.stamp - // Autobuy or ExtendsTTL mode - else { - let refreshStamps: () => Promise - - if (config.mode === 'autobuy') { - refreshStamps = async () => this.refreshStampsAutobuy(config, new BeeDebug(config.beeDebugApiUrl)) - } else { - refreshStamps = async () => this.refreshStampsExtends(config, new BeeDebug(config.beeDebugApiUrl)) - } - this.stop() - await refreshStamps() - - this.interval = setInterval(refreshStamps, config.refreshPeriod) - } - } - - stop(): void { - if (this.interval) { - clearInterval(this.interval) - this.interval = undefined - } - } -} diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts new file mode 100644 index 00000000..6652cc38 --- /dev/null +++ b/src/stamps/autobuy.ts @@ -0,0 +1,95 @@ +import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { buyNewStamp, getUsage } from '../utils' +import { logger } from '../logger' +import { StampsConfig, StampsConfigAutobuy } from '../config' +import { + stampCheckCounter, + stampPurchaseFailedCounter, + stampTtlGauge, + stampUsableCountGauge, + stampUsageGauge, +} from './counters' +import { BaseStampManager, StampsManager } from './base' + +/** + * Filter the stamps and only return those that are usable, have correct amount, depth, are not close to beying maxUsage or close to expire + * + * @param stamps Postage stamps to be filtered + * @param depth Postage stamps depth + * @param amount Postage stamps amount + * @param maxUsage Maximal usage of the stamp to be usable by the proxy + * @param minTTL Minimal TTL of the stamp to be usable by the proxy + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsAutobuy( + stamps: PostageBatch[], + depth: number, + amount: string, + maxUsage: number, + minTTL: number, +): PostageBatch[] { + const usableStamps = stamps + // filter to get stamps that have the right depth, amount and are not fully used or expired + .filter(s => s.usable && s.depth === depth && s.amount === amount && getUsage(s) < maxUsage && s.batchTTL > minTTL) + // sort the stamps by usage + .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps +} + +export class AutoBuyStampsManager extends BaseStampManager implements StampsManager { + private isBuyingStamp?: boolean = false + private beeDebug: BeeDebug + + constructor(beeDebug: BeeDebug) { + super() + this.beeDebug = beeDebug + } + + /** + * Refresh stamps from the bee node and if needed buy new stamp + * + * @param config Stamps config + * @param beeDebug Connection to debug endpoint for checking/buying stamps + */ + public async refreshStamps(config: StampsConfig) { + try { + stampCheckCounter.inc() + logger.info('checking postage stamps') + const stamps = await this.beeDebug.getAllPostageBatch() + logger.debug('retrieved stamps', stamps) + + const { depth, amount, usageMax, usageThreshold, ttlMin } = config as StampsConfigAutobuy + + // Get all usable stamps sorted by usage from most used to least + this.usableStamps = filterUsableStampsAutobuy(stamps, depth, amount, usageMax, ttlMin) + const leastUsed = this.usableStamps[this.usableStamps.length - 1] + const mostUsed = this.usableStamps[0] + + stampTtlGauge.set(mostUsed ? mostUsed.batchTTL : 0) + stampUsageGauge.set(mostUsed ? getUsage(mostUsed) : 0) + stampUsableCountGauge.set(this.usableStamps.length) + + // Check if the least used stamps is starting to get full and if so purchase new stamp + if (!this.isBuyingStamp && (!leastUsed || getUsage(leastUsed) > usageThreshold)) { + this.isBuyingStamp = true + try { + const { stamp } = await buyNewStamp(depth, amount, this.beeDebug) + + // Add the bought postage stamp + this.usableStamps.push(stamp) + stampUsableCountGauge.set(this.usableStamps.length) + } catch (e) { + logger.error('failed to buy postage stamp', e) + stampPurchaseFailedCounter.inc() + } finally { + this.isBuyingStamp = false + } + } + } catch (e) { + logger.error('failed to refresh postage stamp', e) + } + } +} diff --git a/src/stamps/base.ts b/src/stamps/base.ts new file mode 100644 index 00000000..b1406062 --- /dev/null +++ b/src/stamps/base.ts @@ -0,0 +1,70 @@ +import { PostageBatch } from '@ethersphere/bee-js' +import { stampGetCounter, stampGetErrorCounter } from './counters' +import { logger } from '../logger' +import { ERROR_NO_STAMP, StampsConfig } from '../config' + +export interface StampsManager { + start: (config: StampsConfig, refreshStamps?: () => Promise) => Promise + stop: () => void + postageStamp: () => string +} + +export class BaseStampManager { + private interval?: ReturnType + public stamp?: string + public usableStamps?: PostageBatch[] + + constructor(stamp?: string) { + this.stamp = stamp + } + + /** + * Get postage stamp that should be replaced in a the proxy request header + * + * @return Postage stamp that should be used by the proxy + * + * @throws Error if there is no postage stamp + */ + postageStamp(): string { + stampGetCounter.inc() + + if (this.stamp) { + const stamp = this.stamp + logger.info('using hardcoded stamp', { stamp }) + + return stamp + } + + if (this.usableStamps && this.usableStamps[0]) { + const stamp = this.usableStamps[0] + logger.info('using autobought stamp', { stamp }) + + return stamp.batchID + } + + stampGetErrorCounter.inc() + throw new Error(ERROR_NO_STAMP) + } + + /** + * Start the manager in either hardcoded, autobuy or extends mode + */ + async start(config: StampsConfig, refreshStamps?: (config: StampsConfig) => Promise) { + this.stop() + + if (refreshStamps && (config.mode === 'autobuy' || config.mode === 'extends')) { + refreshStamps(config) + + this.interval = setInterval(async () => refreshStamps(config), config.refreshPeriod) + } else if (config.mode === 'hardcoded') { + this.stamp = config.stamp + } + } + + stop(): void { + if (this.interval) { + clearInterval(this.interval) + this.interval = undefined + } + } +} diff --git a/src/stamps/counters.ts b/src/stamps/counters.ts new file mode 100644 index 00000000..84fc5a03 --- /dev/null +++ b/src/stamps/counters.ts @@ -0,0 +1,50 @@ +import client from 'prom-client' +import { register } from '../metrics' + +export const stampPurchaseCounter = new client.Counter({ + name: 'stamp_purchase_counter', + help: 'How many stamps were purchased', +}) +register.registerMetric(stampPurchaseCounter) + +export const stampCheckCounter = new client.Counter({ + name: 'stamp_check_counter', + help: 'How many times were stamps retrieved from server', +}) +register.registerMetric(stampCheckCounter) + +export const stampGetCounter = new client.Counter({ + name: 'stamp_get_counter', + help: 'How many times was get postageStamp called', +}) +register.registerMetric(stampGetCounter) + +export const stampGetErrorCounter = new client.Counter({ + name: 'stamp_get_error_counter', + help: 'How many times was get postageStamp called and there was no valid postage stamp', +}) +register.registerMetric(stampGetErrorCounter) + +export const stampPurchaseFailedCounter = new client.Counter({ + name: 'stamp_purchase_failed_counter', + help: 'How many stamps failed to be purchased', +}) +register.registerMetric(stampPurchaseFailedCounter) + +export const stampTtlGauge = new client.Gauge({ + name: 'stamp_ttl_gauge', + help: 'TTL on the selected automanaged stamp', +}) +register.registerMetric(stampTtlGauge) + +export const stampUsageGauge = new client.Gauge({ + name: 'stamp_usage_gauge', + help: 'Usage on the selected automanaged stamp', +}) +register.registerMetric(stampUsageGauge) + +export const stampUsableCountGauge = new client.Gauge({ + name: 'stamp_usable_count_gauge', + help: 'How many stamps exist on the bee node that can be used', +}) +register.registerMetric(stampUsableCountGauge) diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts new file mode 100644 index 00000000..a1360e36 --- /dev/null +++ b/src/stamps/extends.ts @@ -0,0 +1,156 @@ +import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { buyNewStamp, getUsage } from '../utils' +import { logger } from '../logger' +import { stampCheckCounter } from './counters' +import { BaseStampManager, StampsManager } from './base' +import { StampsConfig, StampsConfigExtends } from '../config' + +/** + * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL + * + * @param stamps Postage stamps to be filtered + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { + const usableStamps = stamps + // filter to get usable stamps + .filter(s => s.usable) + // sort the stamps by TTL + .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps +} + +/** + * Filter the stamps and only return those that are usable and sort by usage in a increasing order + * + * @param stamps Postage stamps to be filtered + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsExtendsCapacity(stamps: PostageBatch[], usageThreshold: number): PostageBatch[] { + if (usageThreshold > 0) { + const usableStamps = stamps + // filter to get stamps that have been used over the usageThreshold set + .filter(s => s.usable && getUsage(s) > usageThreshold) + // sort the stamps by usage + .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps + } + + return [] +} + +async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { + await beeDebug.topUpBatch(postageBatchId, amount) + const stamp = await beeDebug.getPostageBatch(postageBatchId) + + return stamp +} + +async function extendsCapacity(extendManager: ExtendsStampManager, beeDebug: BeeDebug, stamp: PostageBatch) { + const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) + setTimeout(() => extendManager.completeTopUp('capacity', stampRes), 60000) + await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) + logger.info(`capacity extended for stamp ${stamp.batchID}`) +} + +export class ExtendsStampManager extends BaseStampManager implements StampsManager { + private isBuyingStamp?: boolean = false + private topingUpStamps: string[] = [] + private beeDebug: BeeDebug + + constructor(beeDebug: BeeDebug) { + super() + this.beeDebug = beeDebug + } + + completeTopUp(extendsTypeFeature: 'ttl' | 'capacity', stamp: PostageBatch) { + if (extendsTypeFeature === 'ttl') { + logger.info('successfully postage stamp TTL extended', { stamp }) + } else { + logger.info('successfully postage stamp capacity extended', { stamp }) + } + // remove stamps from extending stamps array + const stampIndex = this.topingUpStamps.findIndex(id => stamp.batchID === id) + this.topingUpStamps.splice(stampIndex, 1) + } + + async verifyUsableStamps(beeDebug: BeeDebug, ttlMin: number, amount: string) { + for (const stamp of this.usableStamps!) { + if (!this.topingUpStamps.includes(stamp.batchID) && amount !== '0') { + this.topingUpStamps.push(stamp.batchID) + logger.info(`extending postage stamp TTL ${stamp.batchID}`) + + try { + const stampRes = await topUpStamp(beeDebug, stamp.batchID, amount) + + setTimeout(() => this.completeTopUp('ttl', stampRes), 60000) + } catch (e: any) { + // error that indicate that 2 stamps are trying to be extended at the same time. Comes out as a warning + const errorStampIndex = this.topingUpStamps.indexOf(stamp.batchID) + this.topingUpStamps.splice(errorStampIndex, 1) + logger.error('failed to topup postage stamp', e) + } + } + } + } + + public async refreshStamps(config: StampsConfig) { + stampCheckCounter.inc() + logger.info('checking postage stamps') + + let stamps: PostageBatch[] = [] + + try { + stamps = await this.beeDebug.getAllPostageBatch() + } catch (e) { + logger.error(`There's been an error getting postage batches: ${e}`) + } + + const { depth, amount, ttlMin, refreshPeriod, usageThreshold, enableTtl, enableCapacity } = + config as StampsConfigExtends + + // Get all usable stamps sorted by usage from most used to least + if (!this.isBuyingStamp && enableTtl && stamps.length > 0) { + const usableStampsSortByTTL = filterUsableStampsExtendsTTL(stamps) + + if (usableStampsSortByTTL.length === 0) { + this.isBuyingStamp = true + try { + const { stamp: newStamp } = await buyNewStamp(depth, amount, this.beeDebug) + + // Add the bought postage stamp + usableStampsSortByTTL.push(newStamp) + } finally { + this.isBuyingStamp = false + } + } else { + const minTimeThreshold = ttlMin + refreshPeriod / 1000 + this.usableStamps = usableStampsSortByTTL.filter(s => s.batchTTL < minTimeThreshold) + await this.verifyUsableStamps(this.beeDebug, ttlMin, amount) + } + } + + if (enableCapacity && stamps.length > 0) { + const usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) + + for (const stamp of usableStampsExtendsCapacity) { + try { + if (!this.topingUpStamps.includes(stamp.batchID)) { + logger.info(`extending stamp capacity: ${stamp.batchID}`) + + this.topingUpStamps.push(stamp.batchID) + extendsCapacity(this, this.beeDebug, stamp) + } + } catch (err) { + logger.error('failed to extend stamp capacity', err) + } + } + } + } +} diff --git a/src/stamps/index.ts b/src/stamps/index.ts new file mode 100644 index 00000000..0c422e93 --- /dev/null +++ b/src/stamps/index.ts @@ -0,0 +1,3 @@ +export { AutoBuyStampsManager } from './autobuy' +export { ExtendsStampManager } from './extends' +export { StampsManager } from './base' diff --git a/src/utils.ts b/src/utils.ts index 9f9c8f4e..a2890325 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,6 @@ -import { BeeDebug } from '@ethersphere/bee-js' +import { BatchId, BeeDebug, NumberString, PostageBatch } from '@ethersphere/bee-js' +import { logger } from './logger' +import { stampPurchaseCounter } from './stamps/counters' /** * Sleep for N miliseconds @@ -38,3 +40,70 @@ export async function waitForStampUsable(beeDebug: BeeDebug, batchId: string): P } throw Error(`Stamp not found/usable: ${batchId}`) } + +/** + * Calculate usage of a given postage stamp + * + * @param stamp Postage stamp which usage should be determined + */ +export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): number { + return utilization / Math.pow(2, depth - bucketDepth) +} + +/** + * Validate if the value is an integer + * @param value + * @returns boolean + */ +export function isInteger(value: unknown): value is number | string | NumberString { + return ( + (typeof value === 'string' && /^-?(0|[1-9][0-9]*)$/g.test(value)) || + (typeof value === 'number' && + value > Number.MIN_SAFE_INTEGER && + value < Number.MAX_SAFE_INTEGER && + Number.isInteger(value)) + ) +} + +export function assertInteger(value: unknown, name = 'value'): asserts value is number | NumberString { + if (!isInteger(value)) throw new TypeError(`${name} is not a valid integer`) +} + +export function assertDecimal(value: unknown, name = 'value'): boolean { + if (typeof value === 'number') { + value = value.toString() + } + + if (typeof value !== 'string' || (typeof value === 'string' && !/^\d*\.?\d*$/g.test(value))) { + throw new TypeError(`${name} is not a valid decimal`) + } + + return true +} + +/** + * Buy new postage stamp and wait until it is usable + * + * @param depth Postage stamps depth + * @param amount Postage stamps amount + * @param beeDebug Connection to debug endpoint for checking/buying stamps + * @param options + * timeout (optional) How long should the system wait for the stamp to be usable in ms, default to 10000 + * + * @returns Newly bought postage stamp + */ +export async function buyNewStamp( + depth: number, + amount: string, + beeDebug: BeeDebug, +): Promise<{ batchId: BatchId; stamp: PostageBatch }> { + logger.info('buying new stamp') + const batchId = await beeDebug.createPostageBatch(amount, depth, { waitForUsable: false }) + await waitForStampUsable(beeDebug, batchId) + stampPurchaseCounter.inc() + + const stamp = await beeDebug.getPostageBatch(batchId) + logger.info('successfully bought new stamp', { stamp }) + + return { batchId, stamp } +} diff --git a/test/config.spec.ts b/test/config.spec.ts index 41d35af5..9b3d1bbf 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -1,3 +1,4 @@ +import { sleep } from '../src/utils' import { DEFAULT_BEE_DEBUG_API_URL, DEFAULT_HOSTNAME, @@ -14,8 +15,9 @@ import { } from '../src/config' describe('getAppConfig', () => { - it('should return default values', () => { + it('should return default values', async () => { const config = getAppConfig() + await sleep(1_000) expect(config.beeApiUrl).toEqual(DEFAULT_BEE_API_URL) expect(config.authorization).toBeUndefined() }) @@ -66,7 +68,7 @@ describe('getStampsConfig', () => { const POSTAGE_AMOUNT = '100' const POSTAGE_DEPTH = '20' const BEE_DEBUG_API_URL = 'http://localhost:1635' - const POSTAGE_USAGE_THRESHOLD = '0.6' + const POSTAGE_USAGE_THRESHOLD = '0.7' const POSTAGE_USAGE_MAX = '0.8' const POSTAGE_TTL_MIN = '200' const POSTAGE_REFRESH_PERIOD = '60000' @@ -144,12 +146,15 @@ describe('getStampsConfig', () => { POSTAGE_TTL_MIN, }, output: { - mode: 'extendsTTL', + mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD), beeDebugApiUrl: BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), + enableTtl: true, + enableCapacity: false, }, }, { @@ -162,12 +167,15 @@ describe('getStampsConfig', () => { POSTAGE_TTL_MIN, }, output: { - mode: 'extendsTTL', + mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD), beeDebugApiUrl: BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), + enableTtl: true, + enableCapacity: false, }, }, ] @@ -180,12 +188,10 @@ describe('getStampsConfig', () => { }) const throwValues: EnvironmentVariables[] = [ - { BEE_DEBUG_API_URL }, { POSTAGE_DEPTH }, { POSTAGE_AMOUNT }, { BEE_DEBUG_API_URL, POSTAGE_DEPTH }, { BEE_DEBUG_API_URL, POSTAGE_AMOUNT }, - { POSTAGE_DEPTH, POSTAGE_AMOUNT }, ] throwValues.forEach(v => { @@ -193,11 +199,7 @@ describe('getStampsConfig', () => { expect(() => { getStampsConfig(v) }).toThrowError( - `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${ - v.POSTAGE_TTL_MIN - } ${v.POSTAGE_EXTENDSTTL === 'true' ? 'at least 60 seconds ' : ''}or BEE_DEBUG_API_URL=${ - v.BEE_DEBUG_API_URL - } for the feature to work`, + `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${v.POSTAGE_TTL_MIN} for the feature to work`, ) }) }) diff --git a/test/readiness.spec.ts b/test/readiness.spec.ts index 9c69b05d..93ba7114 100644 --- a/test/readiness.spec.ts +++ b/test/readiness.spec.ts @@ -1,6 +1,6 @@ import request from 'supertest' import { createApp } from '../src/server' -import { StampsManager } from '../src/stamps' +import { BaseStampManager } from '../src/stamps/base' describe('readiness', () => { test('should be ready without stamp management', async () => { @@ -13,8 +13,7 @@ describe('readiness', () => { }) test('should be ready with stamp management', async () => { - const stampManager = new StampsManager() - await stampManager.start({ mode: 'hardcoded', stamp: process.env.BEE_POSTAGE as string }) + const stampManager = new BaseStampManager(process.env.BEE_POSTAGE as string) const app = createApp( { beeApiUrl: 'http://localhost:1633', diff --git a/test/server.spec.ts b/test/server.spec.ts index 8d0d1e07..6306b6a9 100644 --- a/test/server.spec.ts +++ b/test/server.spec.ts @@ -5,8 +5,8 @@ import type { Server } from 'http' import { DEFAULT_BEE_DEBUG_API_URL } from '../src/config' import { bee, getPostageBatch, makeCollectionFromFS } from './utils' -import { StampsManager } from '../src/stamps' import { createHeaderCheckMockServer } from './header-check.mockserver' +import { BaseStampManager } from '../src/stamps/base' const beeApiUrl = process.env.BEE_API_URL || 'http://localhost:1633' const beeApiUrlWrong = process.env.BEE_API_URL_WRONG || 'http://localhost:2021' @@ -36,8 +36,7 @@ beforeAll(async () => { beeProxy = new Bee(`http://localhost:${port}`) const stamp = getPostageBatch() - const stampManager = new StampsManager() - await stampManager.start({ mode: 'hardcoded', stamp }) + const stampManager = new BaseStampManager(stamp) const appWithStamp = createApp({ beeApiUrl, beeDebugApiUrl: DEFAULT_BEE_DEBUG_API_URL }, stampManager) proxyWithStamp = await new Promise((resolve, _reject) => { const server = appWithStamp.listen(async () => resolve(server)) diff --git a/test/stamps.mockserver.ts b/test/stamps.mockserver.ts index c2643162..962568f5 100644 --- a/test/stamps.mockserver.ts +++ b/test/stamps.mockserver.ts @@ -59,6 +59,12 @@ export async function createStampMockServer(db: StampDB): Promise { res.send({ batchID: stamp.batchID }) }) + app.patch('/stamps/dilute/:batchId/:depth', (req, res) => { + const stamp = db.get(req.params.batchId as BatchId) + stamp.depth = Number(req.params.depth) + res.send({ batchID: stamp.batchID }) + }) + return new Promise(resolve => { const server = app.listen(() => { resolve(server) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index ffa51f67..f524f64c 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,11 +1,16 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' -import { StampsManager, getUsage, buyNewStamp, topUpStamp, filterUsableStampsAutobuy } from '../src/stamps' -import { getStampsConfig } from '../src/config' -import { sleep } from '../src/utils' +import { buyNewStamp, getUsage, sleep } from '../src/utils' +import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' - +import { + ExtendsStampManager, + filterUsableStampsExtendsCapacity, + filterUsableStampsExtendsTTL, +} from '../src/stamps/extends' +import { AutoBuyStampsManager, filterUsableStampsAutobuy } from '../src/stamps/autobuy' +import { BaseStampManager } from '../src/stamps/base' interface AddressInfo { address: string family: string @@ -32,6 +37,7 @@ afterEach(() => { const defaultAmount = '1000000' const defaultDepth = 20 const defaultTTL = Number(defaultAmount) +const defaultUsageThreshold = 0.7 const defaultStamp: PostageBatch = { batchID: genRandomHex(64) as BatchId, utilization: 0, @@ -59,46 +65,46 @@ const buildStamp = (overwrites: Partial) => { describe('postageStamp', () => { it('should return correct hardcoded single postage stamp', async () => { const stamp = '0000000000000000000000000000000000000000000000000000000000000000' - const stampManager = new StampsManager() - await stampManager.start(getStampsConfig({ POSTAGE_STAMP: stamp })!) - expect(stampManager.postageStamp).toEqual(stamp) + const stampManager = new BaseStampManager(stamp) + expect(stampManager.postageStamp()).toEqual(stamp) }) it('should return existing stamp', async () => { const stamp = buildStamp({ utilization: 0 }) db.add(stamp) - const manager = new StampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + }) as StampsConfigAutobuy + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) + await sleep(1_000) expect(db.toArray().length).toEqual(1) - const batchId = manager.postageStamp + const batchId = manager.postageStamp() expect(batchId).toBe(stamp.batchID) manager.stop() await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) it('should start without any postage stamp and create new one', async () => { - const manager = new StampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) - await sleep(1_000) + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + }) as StampsConfigAutobuy + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) + + await sleep(4_000) expect(db.toArray().length).toEqual(1) - expect(manager.postageStamp).toEqual(db.toArray()[0].batchID) + const batchId = manager.postageStamp() + expect(batchId).toEqual(db.toArray()[0].batchID) manager.stop() await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) @@ -106,21 +112,20 @@ describe('postageStamp', () => { it('should create additional stamp if existing is starting to get full', async () => { const stamp = buildStamp({ utilization: 14 }) db.add(stamp) + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + }) as StampsConfigAutobuy - const manager = new StampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(1_000) expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).toEqual(stamp.batchID) + expect(manager.postageStamp()).toEqual(stamp.batchID) manager.stop() await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) @@ -128,27 +133,26 @@ describe('postageStamp', () => { it('should create additional stamp if existing stamp usage increases', async () => { const stamp = buildStamp({ utilization: 5 }) db.add(stamp) + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + POSTAGE_USAGE_MAX: '0.8', + BEE_DEBUG_API_URL: url, + }) as StampsConfigAutobuy - const manager = new StampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - POSTAGE_USAGE_MAX: '0.8', - BEE_DEBUG_API_URL: url, - })!, - ) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(200) expect(db.toArray().length).toEqual(1) - expect(manager.postageStamp).toEqual(stamp.batchID) + expect(manager.postageStamp()).toEqual(stamp.batchID) stamp.utilization = 15 await sleep(500) expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).not.toEqual(stamp.batchID) + expect(manager.postageStamp()).not.toEqual(stamp.batchID) manager.stop() await sleep(1500) // Needed as there could be the wait for posage stamp usable process in progress }) @@ -230,14 +234,95 @@ describe('filterUsableStamps', () => { }) }) -describe('topUpStamp', () => { - it('should extend stamp stamp and await for it to extend others', async () => { - const beeDebug = new BeeDebug(url) - const stamp = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) +describe('extendsStampsTTL', () => { + let stampsConfig: StampsConfig + + beforeEach(() => { + stampsConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_TTL_MIN: defaultTTL.toString(), + POSTAGE_USAGE_THRESHOLD: defaultUsageThreshold.toString(), + POSTAGE_REFRESH_PERIOD: '200', + POSTAGE_EXTENDSTTL: 'true', + BEE_DEBUG_API_URL: url, + }) as StampsConfig + }) + + it('should not find any usable stamps', async () => { + const stamps = [ + buildStamp({ utilization: 7, batchTTL: 200_000, usable: false }), + buildStamp({ utilization: 8, batchTTL: 150_000, usable: false }), + buildStamp({ utilization: 10, batchTTL: 120_000, usable: false }), + ] + + const { ttlMin, refreshPeriod } = stampsConfig as StampsConfigExtends + const minTimeThreshold = ttlMin + refreshPeriod / 1000 + const usableStampsSortByTTL = filterUsableStampsExtendsTTL(stamps) + const res = usableStampsSortByTTL.filter(s => s.batchTTL < minTimeThreshold) + expect(0).toEqual(res.length) + }) + + it('should extend stamp ttl and await for it to extend others', async () => { + const stamp = buildStamp({ amount: defaultAmount, utilization: 10, batchTTL: 120_000, usable: true }) + db.add(stamp) + + const extendManager = new ExtendsStampManager(new BeeDebug(url)) + extendManager.start(stampsConfig, async () => (extendManager as ExtendsStampManager).refreshStamps(stampsConfig)) const extendAmount = '100' - await topUpStamp(beeDebug, stamp.batchId, extendAmount) - const stampExtended = await beeDebug.getPostageBatch(stamp.batchId) + stamp.amount += extendAmount + await sleep(500) + + const [stampExtended] = db.toArray() expect(Number(stampExtended.amount)).toBeGreaterThan(Number(defaultAmount)) + extendManager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + }) +}) + +describe('extendsStampsCapacity', () => { + let stampsConfig: StampsConfig + + beforeEach(() => { + stampsConfig = getStampsConfig({ + POSTAGE_USAGE_THRESHOLD: defaultUsageThreshold.toString(), + POSTAGE_EXTENDS_CAPACITY: 'true', + BEE_DEBUG_API_URL: url, + POSTAGE_REFRESH_PERIOD: '100', + }) as StampsConfig + }) + it('should not find any usable stamps', async () => { + const stamps = [ + buildStamp({ utilization: 14, depth: 20, bucketDepth: 16, usable: false }), + buildStamp({ utilization: 8, depth: 20, bucketDepth: 16, usable: true }), + buildStamp({ utilization: 10, depth: 20, bucketDepth: 16, usable: true }), + ] + const res = filterUsableStampsExtendsCapacity(stamps, defaultUsageThreshold) + expect(0).toEqual(res.length) + }) + + it('should extend stamps capacity', async () => { + const stamp = buildStamp({ + amount: defaultAmount, + utilization: 10, + batchTTL: 120_000, + depth: defaultDepth, + usable: true, + bucketDepth: defaultDepth - 1, + }) + db.add(stamp) + const extendManager = new ExtendsStampManager(new BeeDebug(url)) + extendManager.start(stampsConfig, async () => (extendManager as ExtendsStampManager).refreshStamps(stampsConfig)) + + await sleep(200) + + stamp.utilization += 20 + await sleep(1000) + + const [stampExtended] = db.toArray() + expect(stampExtended.depth).toBeGreaterThan(defaultDepth) + extendManager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) })