From e90e1df0a9adabea81f8be2ba782bffe408d71f1 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Wed, 19 Oct 2022 13:09:30 -0500 Subject: [PATCH 01/18] feat: extends capacity changes --- README.md | 14 ++++++ src/config.ts | 38 ++++++++++------- src/content.ts | 53 +++++++++++------------ src/stamps.ts | 90 ++++++++++++++++++++++++++++++--------- test/config.spec.ts | 6 ++- test/stamps.mockserver.ts | 6 +++ test/stamps.spec.ts | 13 ++++++ 7 files changed, 155 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 5d6f9cc8..9f223994 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ newer Bee versions is not recommended and may not work. Stay up to date by joini - [2. Hardcoded postage stamp](#2-hardcoded-postage-stamp) - [3. Autobuy postage stamps](#3-autobuy-postage-stamps) - [4. Extends stamps TTL](#4-extends-stamps-ttl) + - [5. Extends stamps capacity](#5-extends-stamps-capacity) - [Enable authentication](#enable-authentication) - [Environment variables](#environment-variables) - [Curl](#curl) @@ -56,6 +57,9 @@ The proxy can manage postage stamps for you in 4 modes of operation: 4. It can extend the TTL of a stamp that is about to expire. To enable this, set `POSTAGE_EXTENDSTTL=true`, provide `POSTAGE_AMOUNT`, `POSTAGE_DEPTH` with the desired amount to use and `POSTAGE_TTL_MIN` above with a number above or equal to 60. +5. It can extends the postage stamp capacity to those that are about to be fulfill. To enable this, set + `POSTAGE_EXTENDS_CAPACITY=true`. You can also set the env variable `POSTAGE_USAGE_THRESHOLD=0.7` to determine + the maximum usage level to check if the stamp needs to be extended In modes 1, 2 and 3, the proxy can be configured to require authentication secret to forward the requests. Use the `AUTH_SECRET` environment variable to enable it. @@ -112,6 +116,14 @@ export BEE_DEBUG_API_URL=http://localhost:1635 npm run start ``` +#### 5. Extends stamps capacity + +```sh +export POSTAGE_EXTENDS_CAPACITY=true + +npm run start +``` + #### Reupload pinned content ```sh @@ -149,6 +161,8 @@ npm run start | REMOVE_PIN_HEADER | true | Removes swarm-pin header on all proxy requests. | | `LOG_LEVEL` | info | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`) | | POSTAGE_EXTENDSTTL | false | Enables extends TTL feature. Works along with POSTAGE_AMOUNT | +| POSTAGE_EXTENDS_CAPACITY | false | Enables extending stamp capacity +feature. | | EXPOSE_HASHED_IDENTITY | false | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes | | REUPLOAD_PERIOD | undefined | How frequently are the pinned content checked to be reuploaded. | diff --git a/src/config.ts b/src/config.ts index 6230b410..78e5ebe0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -18,11 +18,13 @@ interface StampsConfigHardcoded { mode: 'hardcoded' stamp: string } + export interface StampsConfigExtends { - mode: 'extendsTTL' + mode: 'extends' ttlMin: number depth: number amount: string + usageThreshold: number refreshPeriod: number beeDebugApiUrl: string } @@ -80,6 +82,7 @@ export type EnvironmentVariables = Partial<{ POSTAGE_REFRESH_PERIOD: string POSTAGE_EXTENDSTTL: string REUPLOAD_PERIOD: string + POSTAGE_EXTENDS_CAPACITY: string }> export const SUPPORTED_LEVELS = ['critical', 'error', 'warn', 'info', 'verbose', 'debug'] as const @@ -138,6 +141,7 @@ export function getStampsConfig({ POSTAGE_TTL_MIN, POSTAGE_REFRESH_PERIOD, POSTAGE_EXTENDSTTL, + POSTAGE_EXTENDS_CAPACITY, }: EnvironmentVariables = {}): StampsConfig | undefined { const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD) const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL @@ -145,28 +149,32 @@ export function getStampsConfig({ // Start in hardcoded mode if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP } // Start autobuy - else if (!POSTAGE_EXTENDSTTL && POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { + else if ( + (POSTAGE_EXTENDSTTL === 'true' && + POSTAGE_AMOUNT && + POSTAGE_DEPTH && + Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE) || + POSTAGE_EXTENDS_CAPACITY === 'true' + ) { return { - mode: 'autobuy', + mode: 'extends', depth: Number(POSTAGE_DEPTH), - amount: POSTAGE_AMOUNT, - usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), - usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), - ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), + ttlMin: Number(POSTAGE_TTL_MIN), + amount: POSTAGE_AMOUNT ?? '0', + usageThreshold: Number( + POSTAGE_USAGE_THRESHOLD || (POSTAGE_EXTENDS_CAPACITY === 'true' ? DEFAULT_POSTAGE_USAGE_THRESHOLD : '0'), + ), refreshPeriod, beeDebugApiUrl, } - } else if ( - POSTAGE_EXTENDSTTL === 'true' && - POSTAGE_AMOUNT && - POSTAGE_DEPTH && - Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE - ) { + } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { return { - mode: 'extendsTTL', + mode: 'autobuy', depth: Number(POSTAGE_DEPTH), - ttlMin: Number(POSTAGE_TTL_MIN), amount: POSTAGE_AMOUNT, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), + usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), + ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), refreshPeriod, beeDebugApiUrl, } diff --git a/src/content.ts b/src/content.ts index b0c80058..fb670904 100644 --- a/src/content.ts +++ b/src/content.ts @@ -14,40 +14,36 @@ export class ContentManager { private interval?: ReturnType private isReuploading = false - public async attemptRefreshContentReupload(beeApi: Bee): Promise { - try { - await this.refreshContentReupload(beeApi) - } catch (error) { - logger.error('content reupload job failed', error) - } - } - public async refreshContentReupload(beeApi: Bee): Promise { - const pins = await beeApi.getAllPins() + try{ + const pins = await beeApi.getAllPins() - if (!pins.length) { - logger.info(`no pins found`) + if (!pins.length) { + logger.info(`no pins found`) - return - } + return + } - logger.info(`checking pinned content (${pins.length} pins)`) - for (const pin of pins) { - const isRetrievable = await beeApi.isReferenceRetrievable(pin) - logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`) + logger.info(`checking pinned content (${pins.length} pins)`) + for (const pin of pins) { + const isRetrievable = await beeApi.isReferenceRetrievable(pin) + logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`) - if (!isRetrievable && !this.isReuploading) { - this.isReuploading = true - try { - logger.debug(`reuploading pinned content: ${pin}`) - await beeApi.reuploadPinnedData(pin) - contentReuploadCounter.inc() - logger.info(`pinned content reuploaded: ${pin}`) - } catch (error) { - logger.error('failed to reupload pinned content', error) + if (!isRetrievable && !this.isReuploading) { + this.isReuploading = true + try { + logger.debug(`reuploading pinned content: ${pin}`) + await beeApi.reuploadPinnedData(pin) + contentReuploadCounter.inc() + logger.info(`pinned content reuploaded: ${pin}`) + } catch (error) { + logger.error('failed to reupload pinned content', error) + } + this.isReuploading = false } - this.isReuploading = false } + } catch (error) { + logger.error('content reupload job failed', error) } } @@ -55,7 +51,8 @@ export class ContentManager { * Start the manager that checks for pinned content availability and reuploads the data if needed. */ start(config: ContentConfig): void { - const refreshContent = async () => this.attemptRefreshContentReupload(new Bee(config.beeApiUrl)) + const bee = new Bee(config.beeApiUrl) + const refreshContent = async () => this.refreshContentReupload(bee) this.stop() refreshContent() diff --git a/src/stamps.ts b/src/stamps.ts index f519ec9c..ffdfe732 100644 --- a/src/stamps.ts +++ b/src/stamps.ts @@ -5,10 +5,6 @@ import { logger } from './logger' import { register } from './metrics' import { waitForStampUsable } from './utils' -interface Options { - timeout?: number -} - const stampPurchaseCounter = new client.Counter({ name: 'stamp_purchase_counter', help: 'How many stamps were purchased', @@ -101,7 +97,7 @@ export function filterUsableStampsAutobuy( * * @returns Filtered stamps soltered by usage */ -export function filterUsableStampsExtends(stamps: PostageBatch[]): PostageBatch[] { +export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { const usableStamps = stamps // filter to get stamps that have the right depth, amount and are not fully used or expired .filter(s => s.usable) @@ -112,6 +108,28 @@ export function filterUsableStampsExtends(stamps: PostageBatch[]): PostageBatch[ return usableStamps } +/** + * Filter the stamps and only return those that are usable and sort by usage in a increasing order + * + * @param stamps Postage stamps to be filtered + * + * @returns Filtered stamps soltered by usage + */ + export function filterUsableStampsExtendsCapacity(stamps: PostageBatch[], usageThreshold: number): PostageBatch[] { + if (usageThreshold > 0) { + const usableStamps = stamps + // filter to get stamps that have been used over the usageThreshold set + .filter(s => s.usable && getUsage(s) > usageThreshold) + // sort the stamps by usage + .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps + } + + return [] +} + /** * Buy new postage stamp and wait until it is usable * @@ -151,7 +169,7 @@ export class StampsManager { private usableStamps?: PostageBatch[] private interval?: ReturnType private isBuyingStamp?: boolean = false - private extendingStamps: string[] = [] + private topingUpStamps: string[] = [] /** * Get postage stamp that should be replaced in a the proxy request header @@ -227,6 +245,8 @@ export class StampsManager { } public async refreshStampsExtends(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { + let usableStampsExtendsTTL: PostageBatch[] = [] + let usableStampsExtendsCapacity: PostageBatch[] = [] stampCheckCounter.inc() logger.info('checking postage stamps') @@ -234,19 +254,21 @@ export class StampsManager { const stamps = await beeDebug.getAllPostageBatch() logger.debug('retrieved stamps', stamps) - const { amount, ttlMin, depth } = config + const { depth, amount, ttlMin, usageThreshold } = config // Get all usable stamps sorted by usage from most used to least - this.usableStamps = filterUsableStampsExtends(stamps) + usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps) if (!this.isBuyingStamp) { - if (this.usableStamps.length === 0) { + const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 + + if (!isNaN(depth) && usableStampsExtendsTTL.length === 0) { this.isBuyingStamp = true try { const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) // Add the bought postage stamp - this.usableStamps.push(newStamp) + usableStampsExtendsTTL.push(newStamp) } finally { this.isBuyingStamp = false } @@ -254,6 +276,24 @@ export class StampsManager { await this.verifyUsableStamps(beeDebug, ttlMin, config, amount) } } + + usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) + + for (const stamp of usableStampsExtendsCapacity) { + try { + if (!this.topingUpStamps.includes(stamp.batchID)) { + logger.info(`extending stamp capacity: ${stamp.batchID}`) + + this.topingUpStamps.push(stamp.batchID) + const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) + setTimeout(() => this.completeTopUp('capacity', stampRes), 60000) + await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) + logger.info(`capacity extended for stamp ${stamp.batchID}`) + } + } catch (err) { + logger.error('failed to extend stamp capacity', err) + } + } } catch (e) { logger.error('failed to refresh on extends postage stamps', e) } @@ -270,29 +310,38 @@ export class StampsManager { const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 - if (stamp.batchTTL < minTimeThreshold && !this.extendingStamps.includes(stamp.batchID)) { - this.extendingStamps.push(stamp.batchID) - logger.info(`extending postage stamp ${stamp.batchID}`) + if ( + !isNaN(minTimeThreshold) && + stamp.batchTTL < minTimeThreshold && + !this.topingUpStamps.includes(stamp.batchID) && + amount !== '0' + ) { + this.topingUpStamps.push(stamp.batchID) + logger.info(`extending postage stamp TTL ${stamp.batchID}`) try { const stampRes = await topUpStamp(beeDebug, stamp.batchID, amount) - setTimeout(() => this.completeTopUp(stampRes), 60000) + setTimeout(() => this.completeTopUp('ttl', stampRes), 60000) } catch (e: any) { // error that indicate that 2 stamps are trying to be extended at the same time. Comes out as a warning - const errorStampIndex = this.extendingStamps.indexOf(stamp.batchID) - this.extendingStamps.splice(errorStampIndex, 1) + const errorStampIndex = this.topingUpStamps.indexOf(stamp.batchID) + this.topingUpStamps.splice(errorStampIndex, 1) logger.error('failed to topup postage stamp', e) } } } } - completeTopUp(stamp: PostageBatch) { - logger.info('successfully extended postage stamp', { stamp }) + completeTopUp(extendsTypeFeature: 'ttl' | 'capacity',stamp: PostageBatch) { + if (extendsTypeFeature === 'ttl') { + logger.info('successfully postage stamp TTL extended', { stamp }) + } else { + logger.info('successfully postage stamp capacity extended', { stamp }) + } // remove stamps from extending stamps array - const stampIndex = this.extendingStamps.findIndex(id => stamp.batchID === id) - this.extendingStamps.splice(stampIndex, 1) + const stampIndex = this.topingUpStamps.findIndex(id => stamp.batchID === id) + this.topingUpStamps.splice(stampIndex, 1) } /** @@ -310,6 +359,7 @@ export class StampsManager { } else { refreshStamps = async () => this.refreshStampsExtends(config, new BeeDebug(config.beeDebugApiUrl)) } + this.stop() await refreshStamps() diff --git a/test/config.spec.ts b/test/config.spec.ts index 41d35af5..a1a95466 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -144,9 +144,10 @@ describe('getStampsConfig', () => { POSTAGE_TTL_MIN, }, output: { - mode: 'extendsTTL', + mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, + usageThreshold: 0, beeDebugApiUrl: BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), @@ -162,9 +163,10 @@ describe('getStampsConfig', () => { POSTAGE_TTL_MIN, }, output: { - mode: 'extendsTTL', + mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, + usageThreshold: 0, beeDebugApiUrl: BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), diff --git a/test/stamps.mockserver.ts b/test/stamps.mockserver.ts index c2643162..962568f5 100644 --- a/test/stamps.mockserver.ts +++ b/test/stamps.mockserver.ts @@ -59,6 +59,12 @@ export async function createStampMockServer(db: StampDB): Promise { res.send({ batchID: stamp.batchID }) }) + app.patch('/stamps/dilute/:batchId/:depth', (req, res) => { + const stamp = db.get(req.params.batchId as BatchId) + stamp.depth = Number(req.params.depth) + res.send({ batchID: stamp.batchID }) + }) + return new Promise(resolve => { const server = app.listen(() => { resolve(server) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index ffa51f67..0a5e4e66 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -241,3 +241,16 @@ describe('topUpStamp', () => { expect(Number(stampExtended.amount)).toBeGreaterThan(Number(defaultAmount)) }) }) + +describe('extendsCapacity', () => { + it('should extend stamps capacity', async () => { + const beeDebug = new BeeDebug(url) + + const stamp = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) + + await topUpStamp(beeDebug, stamp.batchId, (Number(stamp.stamp.amount) * 2).toString()) + await beeDebug.diluteBatch(stamp.batchId, stamp.stamp.depth + 1) + const stampExtended = await beeDebug.getPostageBatch(stamp.batchId) + expect(stampExtended.depth).toBeGreaterThan(defaultDepth) + }) +}) From d83db7257455481fc9ce9649c16a8f1e132dac2d Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Sun, 23 Oct 2022 22:58:53 -0500 Subject: [PATCH 02/18] fix: refactor the stamps features --- src/config.ts | 21 ++- src/content.ts | 2 +- src/index.ts | 24 ++- src/readiness.ts | 9 +- src/server.ts | 4 +- src/stamps.ts | 376 ---------------------------------------- src/stamps/autobuy.ts | 101 +++++++++++ src/stamps/base.ts | 45 +++++ src/stamps/counters.ts | 50 ++++++ src/stamps/extends.ts | 163 +++++++++++++++++ src/stamps/hardcoded.ts | 12 ++ src/stamps/index.ts | 7 + src/utils.ts | 60 ++++++- test/readiness.spec.ts | 4 +- test/server.spec.ts | 4 +- test/stamps.spec.ts | 16 +- 16 files changed, 494 insertions(+), 404 deletions(-) delete mode 100644 src/stamps.ts create mode 100644 src/stamps/autobuy.ts create mode 100644 src/stamps/base.ts create mode 100644 src/stamps/counters.ts create mode 100644 src/stamps/extends.ts create mode 100644 src/stamps/hardcoded.ts create mode 100644 src/stamps/index.ts diff --git a/src/config.ts b/src/config.ts index 78e5ebe0..28a1d98f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,5 @@ +import { getAmount, getDepth, getTTLMin, getUsageThreshold } from './utils' + export interface AppConfig { beeApiUrl: string beeDebugApiUrl: string @@ -156,14 +158,21 @@ export function getStampsConfig({ Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE) || POSTAGE_EXTENDS_CAPACITY === 'true' ) { + const amount = getAmount(POSTAGE_AMOUNT) + const usageThreshold = getUsageThreshold( + POSTAGE_USAGE_THRESHOLD, + POSTAGE_EXTENDS_CAPACITY === 'true', + DEFAULT_POSTAGE_USAGE_THRESHOLD, + ) + const ttlMin = getTTLMin(POSTAGE_TTL_MIN) + const depth = getDepth(POSTAGE_DEPTH) + return { mode: 'extends', - depth: Number(POSTAGE_DEPTH), - ttlMin: Number(POSTAGE_TTL_MIN), - amount: POSTAGE_AMOUNT ?? '0', - usageThreshold: Number( - POSTAGE_USAGE_THRESHOLD || (POSTAGE_EXTENDS_CAPACITY === 'true' ? DEFAULT_POSTAGE_USAGE_THRESHOLD : '0'), - ), + depth, + ttlMin, + amount, + usageThreshold, refreshPeriod, beeDebugApiUrl, } diff --git a/src/content.ts b/src/content.ts index fb670904..b0b0fd1e 100644 --- a/src/content.ts +++ b/src/content.ts @@ -15,7 +15,7 @@ export class ContentManager { private isReuploading = false public async refreshContentReupload(beeApi: Bee): Promise { - try{ + try { const pins = await beeApi.getAllPins() if (!pins.length) { diff --git a/src/index.ts b/src/index.ts index 02137b08..359976a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,7 +2,7 @@ import { Application } from 'express' import { createApp } from './server' -import { StampsManager } from './stamps' +import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config' import { logger, subscribeLogServerRequests } from './logger' import { ContentManager } from './content' @@ -28,9 +28,25 @@ async function main() { if (stampsConfig) { logger.debug('stamps config', stampsConfig) - const stampManager = new StampsManager() - logger.info('starting postage stamp manager') - stampManager.start(stampsConfig) + let stampManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager + const { mode } = stampsConfig + + if (mode === 'hardcoded') { + stampManager = new HardcodedStampsManager() + + logger.info('starting hardcoded postage stamp manager') + stampManager.start(stampsConfig) + } else if (mode === 'autobuy') { + stampManager = new AutoBuyStampsManager() + + logger.info('starting autobuy postage stamp manager') + stampManager.start(stampsConfig) + } else { + stampManager = new ExtendsStampManager() + + logger.info('starting extends postage stamp manager') + stampManager.start(stampsConfig) + } logger.info('starting the proxy') app = createApp(appConfig, stampManager) } else { diff --git a/src/readiness.ts b/src/readiness.ts index 17248e59..6bbf9083 100644 --- a/src/readiness.ts +++ b/src/readiness.ts @@ -1,7 +1,7 @@ import { Bee, BeeDebug, Utils } from '@ethersphere/bee-js' import { ERROR_NO_STAMP, READINESS_TIMEOUT_MS } from './config' import { logger } from './logger' -import { StampsManager } from './stamps' +import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' import { getErrorMessage } from './utils' const MAX_CHUNK_SIZE = 4096 @@ -16,7 +16,7 @@ export enum ReadinessStatus { export async function checkReadiness( bee: Bee, beeDebug: BeeDebug, - stampManager?: StampsManager, + stampManager?: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager, ): Promise { if (stampManager) { const ready = await tryUploadingSingleChunk(bee, stampManager) @@ -34,7 +34,10 @@ export async function checkReadiness( } } -async function tryUploadingSingleChunk(bee: Bee, stampsManager: StampsManager): Promise { +async function tryUploadingSingleChunk( + bee: Bee, + stampsManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager, +): Promise { const chunk = makeChunk() try { await bee.uploadChunk(stampsManager.postageStamp, chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) diff --git a/src/server.ts b/src/server.ts index 918cfa86..5e242314 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,7 +7,7 @@ import { fetchBeeIdentity, getHashedIdentity, HASHED_IDENTITY_HEADER } from './i import { logger } from './logger' import { register } from './metrics' import { checkReadiness, ReadinessStatus } from './readiness' -import type { StampsManager } from './stamps' +import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' import { getErrorMessage } from './utils' const SWARM_STAMP_HEADER = 'swarm-postage-batch-id' @@ -32,7 +32,7 @@ export const createApp = ( removePinHeader, exposeHashedIdentity, }: AppConfig, - stampManager?: StampsManager, + stampManager?: AutoBuyStampsManager | ExtendsStampManager | HardcodedStampsManager, ): Application => { const commonOptions: Options = { target: beeApiUrl, diff --git a/src/stamps.ts b/src/stamps.ts deleted file mode 100644 index ffdfe732..00000000 --- a/src/stamps.ts +++ /dev/null @@ -1,376 +0,0 @@ -import { BeeDebug, PostageBatch, BatchId } from '@ethersphere/bee-js' -import client from 'prom-client' -import { ERROR_NO_STAMP, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from './config' -import { logger } from './logger' -import { register } from './metrics' -import { waitForStampUsable } from './utils' - -const stampPurchaseCounter = new client.Counter({ - name: 'stamp_purchase_counter', - help: 'How many stamps were purchased', -}) -register.registerMetric(stampPurchaseCounter) - -const stampPurchaseFailedCounter = new client.Counter({ - name: 'stamp_purchase_failed_counter', - help: 'How many stamps failed to be purchased', -}) -register.registerMetric(stampPurchaseFailedCounter) - -const stampCheckCounter = new client.Counter({ - name: 'stamp_check_counter', - help: 'How many times were stamps retrieved from server', -}) -register.registerMetric(stampCheckCounter) - -const stampGetCounter = new client.Counter({ - name: 'stamp_get_counter', - help: 'How many times was get postageStamp called', -}) -register.registerMetric(stampGetCounter) - -const stampGetErrorCounter = new client.Counter({ - name: 'stamp_get_error_counter', - help: 'How many times was get postageStamp called and there was no valid postage stamp', -}) -register.registerMetric(stampGetErrorCounter) - -const stampTtlGauge = new client.Gauge({ - name: 'stamp_ttl_gauge', - help: 'TTL on the selected automanaged stamp', -}) -register.registerMetric(stampTtlGauge) - -const stampUsageGauge = new client.Gauge({ - name: 'stamp_usage_gauge', - help: 'Usage on the selected automanaged stamp', -}) -register.registerMetric(stampUsageGauge) - -const stampUsableCountGauge = new client.Gauge({ - name: 'stamp_usable_count_gauge', - help: 'How many stamps exist on the bee node that can be used', -}) -register.registerMetric(stampUsableCountGauge) - -/** - * Calculate usage of a given postage stamp - * - * @param stamp Postage stamp which usage should be determined - */ -export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): number { - return utilization / Math.pow(2, depth - bucketDepth) -} - -/** - * Filter the stamps and only return those that are usable, have correct amount, depth, are not close to beying maxUsage or close to expire - * - * @param stamps Postage stamps to be filtered - * @param depth Postage stamps depth - * @param amount Postage stamps amount - * @param maxUsage Maximal usage of the stamp to be usable by the proxy - * @param minTTL Minimal TTL of the stamp to be usable by the proxy - * - * @returns Filtered stamps soltered by usage - */ -export function filterUsableStampsAutobuy( - stamps: PostageBatch[], - depth: number, - amount: string, - maxUsage: number, - minTTL: number, -): PostageBatch[] { - const usableStamps = stamps - // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable && s.depth === depth && s.amount === amount && getUsage(s) < maxUsage && s.batchTTL > minTTL) - // sort the stamps by usage - .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) - - // return the all usable stamp sorted by usage - return usableStamps -} - -/** - * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL - * - * @param stamps Postage stamps to be filtered - * - * @returns Filtered stamps soltered by usage - */ -export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { - const usableStamps = stamps - // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable) - // sort the stamps by usage - .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) - - // return the all usable stamp sorted by usage - return usableStamps -} - -/** - * Filter the stamps and only return those that are usable and sort by usage in a increasing order - * - * @param stamps Postage stamps to be filtered - * - * @returns Filtered stamps soltered by usage - */ - export function filterUsableStampsExtendsCapacity(stamps: PostageBatch[], usageThreshold: number): PostageBatch[] { - if (usageThreshold > 0) { - const usableStamps = stamps - // filter to get stamps that have been used over the usageThreshold set - .filter(s => s.usable && getUsage(s) > usageThreshold) - // sort the stamps by usage - .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) - - // return the all usable stamp sorted by usage - return usableStamps - } - - return [] -} - -/** - * Buy new postage stamp and wait until it is usable - * - * @param depth Postage stamps depth - * @param amount Postage stamps amount - * @param beeDebug Connection to debug endpoint for checking/buying stamps - * @param options - * timeout (optional) How long should the system wait for the stamp to be usable in ms, default to 10000 - * - * @returns Newly bought postage stamp - */ -export async function buyNewStamp( - depth: number, - amount: string, - beeDebug: BeeDebug, -): Promise<{ batchId: BatchId; stamp: PostageBatch }> { - logger.info('buying new stamp') - const batchId = await beeDebug.createPostageBatch(amount, depth, { waitForUsable: false }) - await waitForStampUsable(beeDebug, batchId) - stampPurchaseCounter.inc() - - const stamp = await beeDebug.getPostageBatch(batchId) - logger.info('successfully bought new stamp', { stamp }) - - return { batchId, stamp } -} - -export async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { - await beeDebug.topUpBatch(postageBatchId, amount) - const stamp = await beeDebug.getPostageBatch(postageBatchId) - - return stamp -} - -export class StampsManager { - private stamp?: string - private usableStamps?: PostageBatch[] - private interval?: ReturnType - private isBuyingStamp?: boolean = false - private topingUpStamps: string[] = [] - - /** - * Get postage stamp that should be replaced in a the proxy request header - * - * @return Postage stamp that should be used by the proxy - * - * @throws Error if there is no postage stamp - */ - get postageStamp(): string { - stampGetCounter.inc() - - if (this.stamp) { - const stamp = this.stamp - logger.info('using hardcoded stamp', { stamp }) - - return stamp - } - - if (this.usableStamps && this.usableStamps[0]) { - const stamp = this.usableStamps[0] - logger.info('using autobought stamp', { stamp }) - - return stamp.batchID - } - - stampGetErrorCounter.inc() - throw new Error(ERROR_NO_STAMP) - } - - /** - * Refresh stamps from the bee node and if needed buy new stamp - * - * @param config Stamps config - * @param beeDebug Connection to debug endpoint for checking/buying stamps - */ - public async refreshStampsAutobuy(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { - try { - stampCheckCounter.inc() - logger.info('checking postage stamps') - const stamps = await beeDebug.getAllPostageBatch() - logger.debug('retrieved stamps', stamps) - - const { depth, amount, usageMax, usageThreshold, ttlMin } = config - - // Get all usable stamps sorted by usage from most used to least - this.usableStamps = filterUsableStampsAutobuy(stamps, depth, amount, usageMax, ttlMin) - const leastUsed = this.usableStamps[this.usableStamps.length - 1] - const mostUsed = this.usableStamps[0] - - stampTtlGauge.set(mostUsed ? mostUsed.batchTTL : 0) - stampUsageGauge.set(mostUsed ? getUsage(mostUsed) : 0) - stampUsableCountGauge.set(this.usableStamps.length) - - // Check if the least used stamps is starting to get full and if so purchase new stamp - if (!this.isBuyingStamp && (!leastUsed || getUsage(leastUsed) > usageThreshold)) { - this.isBuyingStamp = true - try { - const { stamp } = await buyNewStamp(depth, amount, beeDebug) - - // Add the bought postage stamp - this.usableStamps.push(stamp) - stampUsableCountGauge.set(this.usableStamps.length) - } catch (e) { - logger.error('failed to buy postage stamp', e) - stampPurchaseFailedCounter.inc() - } finally { - this.isBuyingStamp = false - } - } - } catch (e) { - logger.error('failed to refresh postage stamp', e) - } - } - - public async refreshStampsExtends(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { - let usableStampsExtendsTTL: PostageBatch[] = [] - let usableStampsExtendsCapacity: PostageBatch[] = [] - stampCheckCounter.inc() - logger.info('checking postage stamps') - - try { - const stamps = await beeDebug.getAllPostageBatch() - logger.debug('retrieved stamps', stamps) - - const { depth, amount, ttlMin, usageThreshold } = config - - // Get all usable stamps sorted by usage from most used to least - usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps) - - if (!this.isBuyingStamp) { - const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 - - if (!isNaN(depth) && usableStampsExtendsTTL.length === 0) { - this.isBuyingStamp = true - try { - const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) - - // Add the bought postage stamp - usableStampsExtendsTTL.push(newStamp) - } finally { - this.isBuyingStamp = false - } - } else { - await this.verifyUsableStamps(beeDebug, ttlMin, config, amount) - } - } - - usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) - - for (const stamp of usableStampsExtendsCapacity) { - try { - if (!this.topingUpStamps.includes(stamp.batchID)) { - logger.info(`extending stamp capacity: ${stamp.batchID}`) - - this.topingUpStamps.push(stamp.batchID) - const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) - setTimeout(() => this.completeTopUp('capacity', stampRes), 60000) - await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) - logger.info(`capacity extended for stamp ${stamp.batchID}`) - } - } catch (err) { - logger.error('failed to extend stamp capacity', err) - } - } - } catch (e) { - logger.error('failed to refresh on extends postage stamps', e) - } - } - - async verifyUsableStamps( - beeDebug: BeeDebug, - ttlMin: number, - config: StampsConfigAutobuy | StampsConfigExtends, - amount: string, - ) { - for (let i = 0; i < this.usableStamps!.length; i++) { - const stamp = this.usableStamps![i] - - const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 - - if ( - !isNaN(minTimeThreshold) && - stamp.batchTTL < minTimeThreshold && - !this.topingUpStamps.includes(stamp.batchID) && - amount !== '0' - ) { - this.topingUpStamps.push(stamp.batchID) - logger.info(`extending postage stamp TTL ${stamp.batchID}`) - - try { - const stampRes = await topUpStamp(beeDebug, stamp.batchID, amount) - - setTimeout(() => this.completeTopUp('ttl', stampRes), 60000) - } catch (e: any) { - // error that indicate that 2 stamps are trying to be extended at the same time. Comes out as a warning - const errorStampIndex = this.topingUpStamps.indexOf(stamp.batchID) - this.topingUpStamps.splice(errorStampIndex, 1) - logger.error('failed to topup postage stamp', e) - } - } - } - } - - completeTopUp(extendsTypeFeature: 'ttl' | 'capacity',stamp: PostageBatch) { - if (extendsTypeFeature === 'ttl') { - logger.info('successfully postage stamp TTL extended', { stamp }) - } else { - logger.info('successfully postage stamp capacity extended', { stamp }) - } - // remove stamps from extending stamps array - const stampIndex = this.topingUpStamps.findIndex(id => stamp.batchID === id) - this.topingUpStamps.splice(stampIndex, 1) - } - - /** - * Start the manager in either hardcoded or autobuy mode - */ - async start(config: StampsConfig): Promise { - // Hardcoded stamp mode - if (config.mode === 'hardcoded') this.stamp = config.stamp - // Autobuy or ExtendsTTL mode - else { - let refreshStamps: () => Promise - - if (config.mode === 'autobuy') { - refreshStamps = async () => this.refreshStampsAutobuy(config, new BeeDebug(config.beeDebugApiUrl)) - } else { - refreshStamps = async () => this.refreshStampsExtends(config, new BeeDebug(config.beeDebugApiUrl)) - } - - this.stop() - await refreshStamps() - - this.interval = setInterval(refreshStamps, config.refreshPeriod) - } - } - - stop(): void { - if (this.interval) { - clearInterval(this.interval) - this.interval = undefined - } - } -} diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts new file mode 100644 index 00000000..5c62e42f --- /dev/null +++ b/src/stamps/autobuy.ts @@ -0,0 +1,101 @@ +import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { buyNewStamp, getUsage } from '../utils' +import { logger } from '../logger' +import { StampsConfigAutobuy } from '../config' +import { + stampCheckCounter, + stampPurchaseFailedCounter, + stampTtlGauge, + stampUsableCountGauge, + stampUsageGauge, +} from './counters' +import { BaseStampManager } from './base' + +/** + * Filter the stamps and only return those that are usable, have correct amount, depth, are not close to beying maxUsage or close to expire + * + * @param stamps Postage stamps to be filtered + * @param depth Postage stamps depth + * @param amount Postage stamps amount + * @param maxUsage Maximal usage of the stamp to be usable by the proxy + * @param minTTL Minimal TTL of the stamp to be usable by the proxy + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsAutobuy( + stamps: PostageBatch[], + depth: number, + amount: string, + maxUsage: number, + minTTL: number, +): PostageBatch[] { + const usableStamps = stamps + // filter to get stamps that have the right depth, amount and are not fully used or expired + .filter(s => s.usable && s.depth === depth && s.amount === amount && getUsage(s) < maxUsage && s.batchTTL > minTTL) + // sort the stamps by usage + .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps +} + +export class AutoBuyStampsManager extends BaseStampManager { + public isBuyingStamp?: boolean = false + /** + * Start the manager in either hardcoded or autobuy mode + */ + async start(config: StampsConfigAutobuy): Promise { + // Autobuy mode + const refreshStamps = async () => this.refreshStampsAutobuy(config, new BeeDebug(config.beeDebugApiUrl)) + + this.stop() + await refreshStamps() + + this.interval = setInterval(refreshStamps, config.refreshPeriod) + } + + /** + * Refresh stamps from the bee node and if needed buy new stamp + * + * @param config Stamps config + * @param beeDebug Connection to debug endpoint for checking/buying stamps + */ + public async refreshStampsAutobuy(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { + try { + stampCheckCounter.inc() + logger.info('checking postage stamps') + const stamps = await beeDebug.getAllPostageBatch() + logger.debug('retrieved stamps', stamps) + + const { depth, amount, usageMax, usageThreshold, ttlMin } = config + + // Get all usable stamps sorted by usage from most used to least + this.usableStamps = filterUsableStampsAutobuy(stamps, depth, amount, usageMax, ttlMin) + const leastUsed = this.usableStamps[this.usableStamps.length - 1] + const mostUsed = this.usableStamps[0] + + stampTtlGauge.set(mostUsed ? mostUsed.batchTTL : 0) + stampUsageGauge.set(mostUsed ? getUsage(mostUsed) : 0) + stampUsableCountGauge.set(this.usableStamps.length) + + // Check if the least used stamps is starting to get full and if so purchase new stamp + if (!this.isBuyingStamp && (!leastUsed || getUsage(leastUsed) > usageThreshold)) { + this.isBuyingStamp = true + try { + const { stamp } = await buyNewStamp(depth, amount, beeDebug) + + // Add the bought postage stamp + this.usableStamps.push(stamp) + stampUsableCountGauge.set(this.usableStamps.length) + } catch (e) { + logger.error('failed to buy postage stamp', e) + stampPurchaseFailedCounter.inc() + } finally { + this.isBuyingStamp = false + } + } + } catch (e) { + logger.error('failed to refresh postage stamp', e) + } + } +} diff --git a/src/stamps/base.ts b/src/stamps/base.ts new file mode 100644 index 00000000..bac30ab3 --- /dev/null +++ b/src/stamps/base.ts @@ -0,0 +1,45 @@ +import { PostageBatch } from '@ethersphere/bee-js' +import { stampGetCounter, stampGetErrorCounter } from './counters' +import { logger } from '../logger' +import { ERROR_NO_STAMP } from '../config' + +export class BaseStampManager { + public stamp?: string + public interval?: ReturnType + public usableStamps?: PostageBatch[] + + /** + * Get postage stamp that should be replaced in a the proxy request header + * + * @return Postage stamp that should be used by the proxy + * + * @throws Error if there is no postage stamp + */ + get postageStamp(): string { + stampGetCounter.inc() + + if (this.stamp) { + const stamp = this.stamp + logger.info('using hardcoded stamp', { stamp }) + + return stamp + } + + if (this.usableStamps && this.usableStamps[0]) { + const stamp = this.usableStamps[0] + logger.info('using autobought stamp', { stamp }) + + return stamp.batchID + } + + stampGetErrorCounter.inc() + throw new Error(ERROR_NO_STAMP) + } + + stop(): void { + if (this.interval) { + clearInterval(this.interval) + this.interval = undefined + } + } +} diff --git a/src/stamps/counters.ts b/src/stamps/counters.ts new file mode 100644 index 00000000..84fc5a03 --- /dev/null +++ b/src/stamps/counters.ts @@ -0,0 +1,50 @@ +import client from 'prom-client' +import { register } from '../metrics' + +export const stampPurchaseCounter = new client.Counter({ + name: 'stamp_purchase_counter', + help: 'How many stamps were purchased', +}) +register.registerMetric(stampPurchaseCounter) + +export const stampCheckCounter = new client.Counter({ + name: 'stamp_check_counter', + help: 'How many times were stamps retrieved from server', +}) +register.registerMetric(stampCheckCounter) + +export const stampGetCounter = new client.Counter({ + name: 'stamp_get_counter', + help: 'How many times was get postageStamp called', +}) +register.registerMetric(stampGetCounter) + +export const stampGetErrorCounter = new client.Counter({ + name: 'stamp_get_error_counter', + help: 'How many times was get postageStamp called and there was no valid postage stamp', +}) +register.registerMetric(stampGetErrorCounter) + +export const stampPurchaseFailedCounter = new client.Counter({ + name: 'stamp_purchase_failed_counter', + help: 'How many stamps failed to be purchased', +}) +register.registerMetric(stampPurchaseFailedCounter) + +export const stampTtlGauge = new client.Gauge({ + name: 'stamp_ttl_gauge', + help: 'TTL on the selected automanaged stamp', +}) +register.registerMetric(stampTtlGauge) + +export const stampUsageGauge = new client.Gauge({ + name: 'stamp_usage_gauge', + help: 'Usage on the selected automanaged stamp', +}) +register.registerMetric(stampUsageGauge) + +export const stampUsableCountGauge = new client.Gauge({ + name: 'stamp_usable_count_gauge', + help: 'How many stamps exist on the bee node that can be used', +}) +register.registerMetric(stampUsableCountGauge) diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts new file mode 100644 index 00000000..f133a60e --- /dev/null +++ b/src/stamps/extends.ts @@ -0,0 +1,163 @@ +import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { StampsConfigExtends } from '../config' +import { buyNewStamp, getUsage } from '../utils' +import { logger } from '../logger' +import { stampCheckCounter } from './counters' +import { BaseStampManager } from './base' + +/** + * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL + * + * @param stamps Postage stamps to be filtered + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { + const usableStamps = stamps + // filter to get stamps that have the right depth, amount and are not fully used or expired + .filter(s => s.usable) + // sort the stamps by usage + .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps +} + +/** + * Filter the stamps and only return those that are usable and sort by usage in a increasing order + * + * @param stamps Postage stamps to be filtered + * + * @returns Filtered stamps soltered by usage + */ +export function filterUsableStampsExtendsCapacity(stamps: PostageBatch[], usageThreshold: number): PostageBatch[] { + if (usageThreshold > 0) { + const usableStamps = stamps + // filter to get stamps that have been used over the usageThreshold set + .filter(s => s.usable && getUsage(s) > usageThreshold) + // sort the stamps by usage + .sort((a, b) => (getUsage(a) < getUsage(b) ? 1 : -1)) + + // return the all usable stamp sorted by usage + return usableStamps + } + + return [] +} + +export async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { + await beeDebug.topUpBatch(postageBatchId, amount) + const stamp = await beeDebug.getPostageBatch(postageBatchId) + + return stamp +} + +export class ExtendsStampManager extends BaseStampManager { + private isBuyingStamp?: boolean = false + private topingUpStamps: string[] = [] + /** + * Start the manager in either hardcoded or autobuy mode + */ + async start(config: StampsConfigExtends): Promise { + // Extends mode + const refreshStamps = async () => this.refreshStampsExtends(config, new BeeDebug(config.beeDebugApiUrl)) + + this.stop() + await refreshStamps() + + this.interval = setInterval(refreshStamps, config.refreshPeriod) + } + + completeTopUp(extendsTypeFeature: 'ttl' | 'capacity', stamp: PostageBatch) { + if (extendsTypeFeature === 'ttl') { + logger.info('successfully postage stamp TTL extended', { stamp }) + } else { + logger.info('successfully postage stamp capacity extended', { stamp }) + } + // remove stamps from extending stamps array + const stampIndex = this.topingUpStamps.findIndex(id => stamp.batchID === id) + this.topingUpStamps.splice(stampIndex, 1) + } + + async verifyUsableStamps(beeDebug: BeeDebug, ttlMin: number, config: StampsConfigExtends, amount: string) { + for (let i = 0; i < this.usableStamps!.length; i++) { + const stamp = this.usableStamps![i] + + const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 + + if ( + ttlMin > 0 && + stamp.batchTTL < minTimeThreshold && + !this.topingUpStamps.includes(stamp.batchID) && + amount !== '0' + ) { + this.topingUpStamps.push(stamp.batchID) + logger.info(`extending postage stamp TTL ${stamp.batchID}`) + + try { + const stampRes = await topUpStamp(beeDebug, stamp.batchID, amount) + + setTimeout(() => this.completeTopUp('ttl', stampRes), 60000) + } catch (e: any) { + // error that indicate that 2 stamps are trying to be extended at the same time. Comes out as a warning + const errorStampIndex = this.topingUpStamps.indexOf(stamp.batchID) + this.topingUpStamps.splice(errorStampIndex, 1) + logger.error('failed to topup postage stamp', e) + } + } + } + } + + public async refreshStampsExtends(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { + let usableStampsExtendsTTL: PostageBatch[] = [] + let usableStampsExtendsCapacity: PostageBatch[] = [] + stampCheckCounter.inc() + logger.info('checking postage stamps') + + try { + const stamps = await beeDebug.getAllPostageBatch() + + const { depth, amount, ttlMin, usageThreshold } = config + + // Get all usable stamps sorted by usage from most used to least + usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps) + + if (!this.isBuyingStamp) { + if (depth > 0 && usableStampsExtendsTTL.length === 0) { + this.isBuyingStamp = true + try { + const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) + + // Add the bought postage stamp + usableStampsExtendsTTL.push(newStamp) + } finally { + this.isBuyingStamp = false + } + } else { + this.usableStamps = usableStampsExtendsTTL + await this.verifyUsableStamps(beeDebug, ttlMin, config, amount) + } + } + + usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) + + for (const stamp of usableStampsExtendsCapacity) { + try { + if (!this.topingUpStamps.includes(stamp.batchID)) { + logger.info(`extending stamp capacity: ${stamp.batchID}`) + + this.topingUpStamps.push(stamp.batchID) + const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) + setTimeout(() => this.completeTopUp('capacity', stampRes), 60000) + await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) + logger.info(`capacity extended for stamp ${stamp.batchID}`) + } + } catch (err) { + logger.error('failed to extend stamp capacity', err) + } + } + } catch (e) { + logger.error('failed to refresh on extends postage stamps', e) + } + } +} diff --git a/src/stamps/hardcoded.ts b/src/stamps/hardcoded.ts new file mode 100644 index 00000000..fcca28c0 --- /dev/null +++ b/src/stamps/hardcoded.ts @@ -0,0 +1,12 @@ +import { StampsConfig } from '../config' +import { BaseStampManager } from './base' + +export class HardcodedStampsManager extends BaseStampManager { + /** + * Start the manager in either hardcoded or autobuy mode + */ + async start(config: StampsConfig): Promise { + // Hardcoded stamp mode + if (config.mode === 'hardcoded') this.stamp = config.stamp + } +} diff --git a/src/stamps/index.ts b/src/stamps/index.ts new file mode 100644 index 00000000..32215126 --- /dev/null +++ b/src/stamps/index.ts @@ -0,0 +1,7 @@ +import { HardcodedStampsManager } from './hardcoded' +import { AutoBuyStampsManager } from './autobuy' +import { ExtendsStampManager } from './extends' + +export { HardcodedStampsManager } +export { AutoBuyStampsManager } +export { ExtendsStampManager } diff --git a/src/utils.ts b/src/utils.ts index 9f9c8f4e..17a0acb4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,6 @@ -import { BeeDebug } from '@ethersphere/bee-js' +import { BatchId, BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { logger } from './logger' +import { stampPurchaseCounter } from './stamps/counters' /** * Sleep for N miliseconds @@ -38,3 +40,59 @@ export async function waitForStampUsable(beeDebug: BeeDebug, batchId: string): P } throw Error(`Stamp not found/usable: ${batchId}`) } + +/** + * Calculate usage of a given postage stamp + * + * @param stamp Postage stamp which usage should be determined + */ +export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): number { + return utilization / Math.pow(2, depth - bucketDepth) +} + +export function getUsageThreshold( + postageUsageThreshold: string | undefined, + isExtendsCapacity: boolean, + defaultUsageThreshold: number, +): number { + return Number(postageUsageThreshold || (isExtendsCapacity ? defaultUsageThreshold : '0')) +} + +export function getAmount(postageAmount: string | undefined): string { + return postageAmount ?? '0' +} + +export function getTTLMin(postageTTLMin: string | undefined): number { + return Number(postageTTLMin || '0') +} + +export function getDepth(postageDepth: string | undefined): number { + return Number(postageDepth || '0') +} + +/** + * Buy new postage stamp and wait until it is usable + * + * @param depth Postage stamps depth + * @param amount Postage stamps amount + * @param beeDebug Connection to debug endpoint for checking/buying stamps + * @param options + * timeout (optional) How long should the system wait for the stamp to be usable in ms, default to 10000 + * + * @returns Newly bought postage stamp + */ +export async function buyNewStamp( + depth: number, + amount: string, + beeDebug: BeeDebug, +): Promise<{ batchId: BatchId; stamp: PostageBatch }> { + logger.info('buying new stamp') + const batchId = await beeDebug.createPostageBatch(amount, depth, { waitForUsable: false }) + await waitForStampUsable(beeDebug, batchId) + stampPurchaseCounter.inc() + + const stamp = await beeDebug.getPostageBatch(batchId) + logger.info('successfully bought new stamp', { stamp }) + + return { batchId, stamp } +} diff --git a/test/readiness.spec.ts b/test/readiness.spec.ts index 9c69b05d..0b090d16 100644 --- a/test/readiness.spec.ts +++ b/test/readiness.spec.ts @@ -1,6 +1,6 @@ import request from 'supertest' import { createApp } from '../src/server' -import { StampsManager } from '../src/stamps' +import { HardcodedStampsManager } from '../src/stamps' describe('readiness', () => { test('should be ready without stamp management', async () => { @@ -13,7 +13,7 @@ describe('readiness', () => { }) test('should be ready with stamp management', async () => { - const stampManager = new StampsManager() + const stampManager = new HardcodedStampsManager() await stampManager.start({ mode: 'hardcoded', stamp: process.env.BEE_POSTAGE as string }) const app = createApp( { diff --git a/test/server.spec.ts b/test/server.spec.ts index 8d0d1e07..39d4294d 100644 --- a/test/server.spec.ts +++ b/test/server.spec.ts @@ -5,7 +5,7 @@ import type { Server } from 'http' import { DEFAULT_BEE_DEBUG_API_URL } from '../src/config' import { bee, getPostageBatch, makeCollectionFromFS } from './utils' -import { StampsManager } from '../src/stamps' +import { HardcodedStampsManager } from '../src/stamps' import { createHeaderCheckMockServer } from './header-check.mockserver' const beeApiUrl = process.env.BEE_API_URL || 'http://localhost:1633' @@ -36,7 +36,7 @@ beforeAll(async () => { beeProxy = new Bee(`http://localhost:${port}`) const stamp = getPostageBatch() - const stampManager = new StampsManager() + const stampManager = new HardcodedStampsManager() await stampManager.start({ mode: 'hardcoded', stamp }) const appWithStamp = createApp({ beeApiUrl, beeDebugApiUrl: DEFAULT_BEE_DEBUG_API_URL }, stampManager) proxyWithStamp = await new Promise((resolve, _reject) => { diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 0a5e4e66..028d38a2 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,10 +1,12 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' -import { StampsManager, getUsage, buyNewStamp, topUpStamp, filterUsableStampsAutobuy } from '../src/stamps' import { getStampsConfig } from '../src/config' -import { sleep } from '../src/utils' +import { buyNewStamp, getUsage, sleep } from '../src/utils' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' +import { topUpStamp } from '../src/stamps/extends' +import { filterUsableStampsAutobuy } from '../src/stamps/autobuy' +import { HardcodedStampsManager } from '../src/stamps' interface AddressInfo { address: string @@ -59,7 +61,7 @@ const buildStamp = (overwrites: Partial) => { describe('postageStamp', () => { it('should return correct hardcoded single postage stamp', async () => { const stamp = '0000000000000000000000000000000000000000000000000000000000000000' - const stampManager = new StampsManager() + const stampManager = new HardcodedStampsManager() await stampManager.start(getStampsConfig({ POSTAGE_STAMP: stamp })!) expect(stampManager.postageStamp).toEqual(stamp) }) @@ -67,7 +69,7 @@ describe('postageStamp', () => { it('should return existing stamp', async () => { const stamp = buildStamp({ utilization: 0 }) db.add(stamp) - const manager = new StampsManager() + const manager = new HardcodedStampsManager() await manager.start( getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), @@ -86,7 +88,7 @@ describe('postageStamp', () => { }) it('should start without any postage stamp and create new one', async () => { - const manager = new StampsManager() + const manager = new HardcodedStampsManager() await manager.start( getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), @@ -107,7 +109,7 @@ describe('postageStamp', () => { const stamp = buildStamp({ utilization: 14 }) db.add(stamp) - const manager = new StampsManager() + const manager = new HardcodedStampsManager() await manager.start( getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), @@ -129,7 +131,7 @@ describe('postageStamp', () => { const stamp = buildStamp({ utilization: 5 }) db.add(stamp) - const manager = new StampsManager() + const manager = new HardcodedStampsManager() await manager.start( getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), From 9b68f1eead841d9f10deda15a31f2de6a3a95dd6 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Mon, 24 Oct 2022 14:50:58 -0500 Subject: [PATCH 03/18] fix: testing logger fixed suppoted levels --- src/logger.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/logger.ts b/src/logger.ts index b9a859d8..3fd0f8ec 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -2,12 +2,16 @@ import { createLogger, format, transports, Logger, Logform } from 'winston' import requestStats from 'request-stats' import type { Server } from 'http' -import { SupportedLevels, logLevel, SUPPORTED_LEVELS } from './config' +import { SupportedLevels, logLevel } from './config' -const supportedLevels: Record = SUPPORTED_LEVELS.reduce( - (acc, cur, idx) => ({ ...acc, [cur]: idx }), - {} as Record, -) +const supportedLevels: Record = [ + 'critical', + 'error', + 'warn', + 'info', + 'verbose', + 'debug', +].reduce((acc, cur, idx) => ({ ...acc, [cur]: idx }), {} as Record) export const logger: Logger = createLogger({ // To see more detailed errors, change this to 'debug' From ab624cfc67e7c0032c9869650d701ad9d407fe6e Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Mon, 24 Oct 2022 15:50:31 -0500 Subject: [PATCH 04/18] fix: rearranged the imports on unit test for the logger to be readable --- src/logger.ts | 14 ++--- test/config.spec.ts | 4 +- test/stamps.spec.ts | 150 +++++++++++++++++++++++--------------------- 3 files changed, 87 insertions(+), 81 deletions(-) diff --git a/src/logger.ts b/src/logger.ts index 3fd0f8ec..b9a859d8 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -2,16 +2,12 @@ import { createLogger, format, transports, Logger, Logform } from 'winston' import requestStats from 'request-stats' import type { Server } from 'http' -import { SupportedLevels, logLevel } from './config' +import { SupportedLevels, logLevel, SUPPORTED_LEVELS } from './config' -const supportedLevels: Record = [ - 'critical', - 'error', - 'warn', - 'info', - 'verbose', - 'debug', -].reduce((acc, cur, idx) => ({ ...acc, [cur]: idx }), {} as Record) +const supportedLevels: Record = SUPPORTED_LEVELS.reduce( + (acc, cur, idx) => ({ ...acc, [cur]: idx }), + {} as Record, +) export const logger: Logger = createLogger({ // To see more detailed errors, change this to 'debug' diff --git a/test/config.spec.ts b/test/config.spec.ts index a1a95466..8e6eebfb 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -1,3 +1,4 @@ +import { sleep } from '../src/utils' import { DEFAULT_BEE_DEBUG_API_URL, DEFAULT_HOSTNAME, @@ -14,8 +15,9 @@ import { } from '../src/config' describe('getAppConfig', () => { - it('should return default values', () => { + it('should return default values', async () => { const config = getAppConfig() + await sleep(1_000) expect(config.beeApiUrl).toEqual(DEFAULT_BEE_API_URL) expect(config.authorization).toBeUndefined() }) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 028d38a2..250618ca 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,11 +1,11 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' -import { getStampsConfig } from '../src/config' import { buyNewStamp, getUsage, sleep } from '../src/utils' +import { getStampsConfig } from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' import { topUpStamp } from '../src/stamps/extends' -import { filterUsableStampsAutobuy } from '../src/stamps/autobuy' +import { AutoBuyStampsManager, filterUsableStampsAutobuy } from '../src/stamps/autobuy' import { HardcodedStampsManager } from '../src/stamps' interface AddressInfo { @@ -69,90 +69,98 @@ describe('postageStamp', () => { it('should return existing stamp', async () => { const stamp = buildStamp({ utilization: 0 }) db.add(stamp) - const manager = new HardcodedStampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) - await sleep(1_000) - expect(db.toArray().length).toEqual(1) - - const batchId = manager.postageStamp - expect(batchId).toBe(stamp.batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + const manager = new AutoBuyStampsManager() + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + })! + + if (stampConfig.mode === 'autobuy') { + await manager.start(stampConfig) + await sleep(1_000) + expect(db.toArray().length).toEqual(1) + + const batchId = manager.postageStamp + expect(batchId).toBe(stamp.batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + } }) it('should start without any postage stamp and create new one', async () => { - const manager = new HardcodedStampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) - await sleep(1_000) - expect(db.toArray().length).toEqual(1) - - expect(manager.postageStamp).toEqual(db.toArray()[0].batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + const manager = new AutoBuyStampsManager() + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + })! + + if (stampConfig.mode === 'autobuy') { + await manager.start(stampConfig) + await sleep(1_000) + expect(db.toArray().length).toEqual(1) + + expect(manager.postageStamp).toEqual(db.toArray()[0].batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + } }) it('should create additional stamp if existing is starting to get full', async () => { const stamp = buildStamp({ utilization: 14 }) db.add(stamp) - const manager = new HardcodedStampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - BEE_DEBUG_API_URL: url, - })!, - ) - - await sleep(1_000) - - expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).toEqual(stamp.batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + const manager = new AutoBuyStampsManager() + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + BEE_DEBUG_API_URL: url, + })! + + if (stampConfig.mode === 'autobuy') { + await manager.start(stampConfig) + + await sleep(1_000) + + expect(db.toArray().length).toEqual(2) + expect(manager.postageStamp).toEqual(stamp.batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress + } }) it('should create additional stamp if existing stamp usage increases', async () => { const stamp = buildStamp({ utilization: 5 }) db.add(stamp) - const manager = new HardcodedStampsManager() - await manager.start( - getStampsConfig({ - POSTAGE_DEPTH: defaultDepth.toString(), - POSTAGE_AMOUNT: defaultAmount.toString(), - POSTAGE_REFRESH_PERIOD: '200', - POSTAGE_USAGE_MAX: '0.8', - BEE_DEBUG_API_URL: url, - })!, - ) - - await sleep(200) - expect(db.toArray().length).toEqual(1) - expect(manager.postageStamp).toEqual(stamp.batchID) - - stamp.utilization = 15 - await sleep(500) - - expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).not.toEqual(stamp.batchID) - manager.stop() - await sleep(1500) // Needed as there could be the wait for posage stamp usable process in progress + const manager = new AutoBuyStampsManager() + const stampConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_REFRESH_PERIOD: '200', + POSTAGE_USAGE_MAX: '0.8', + BEE_DEBUG_API_URL: url, + })! + + if (stampConfig.mode === 'autobuy') { + await manager.start(stampConfig) + + await sleep(200) + expect(db.toArray().length).toEqual(1) + expect(manager.postageStamp).toEqual(stamp.batchID) + + stamp.utilization = 15 + await sleep(500) + + expect(db.toArray().length).toEqual(2) + expect(manager.postageStamp).not.toEqual(stamp.batchID) + manager.stop() + await sleep(1500) // Needed as there could be the wait for posage stamp usable process in progress + } }) }) From ddbdaeca21d488e376c919070c718718e484ebba Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Tue, 1 Nov 2022 21:46:10 -0500 Subject: [PATCH 05/18] fix: comments addressed and validation on env vars made --- src/config.ts | 104 +++++++++++++++++-------- src/index.ts | 8 +- src/readiness.ts | 2 +- src/server.ts | 2 +- src/stamps/autobuy.ts | 23 +++--- src/stamps/base.ts | 27 ++++++- src/stamps/extends.ts | 59 ++++++-------- src/stamps/hardcoded.ts | 4 +- src/stamps/index.ts | 11 +-- src/utils.ts | 42 ++++++---- test/config.spec.ts | 6 +- test/stamps.spec.ts | 167 +++++++++++++++++++++++++--------------- 12 files changed, 272 insertions(+), 183 deletions(-) diff --git a/src/config.ts b/src/config.ts index 28a1d98f..5a46e139 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { getAmount, getDepth, getTTLMin, getUsageThreshold } from './utils' +import { isInteger, isBoolean } from './utils' export interface AppConfig { beeApiUrl: string @@ -21,8 +21,21 @@ interface StampsConfigHardcoded { stamp: string } +export interface StampsConfigAutobuy { + mode: 'autobuy' + ttlMin: number + depth: number + amount: string + usageThreshold: number + refreshPeriod: number + usageMax: number + beeDebugApiUrl: string +} + export interface StampsConfigExtends { mode: 'extends' + ttl: boolean + capacity: boolean ttlMin: number depth: number amount: string @@ -36,17 +49,6 @@ export interface ContentConfigReupload { refreshPeriod: number } -export interface StampsConfigAutobuy { - mode: 'autobuy' - depth: number - amount: string - beeDebugApiUrl: string - usageThreshold: number - usageMax: number - ttlMin: number - refreshPeriod: number -} - export type StampsConfig = StampsConfigHardcoded | StampsConfigAutobuy | StampsConfigExtends export type ContentConfig = ContentConfigReupload @@ -152,30 +154,19 @@ export function getStampsConfig({ if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP } // Start autobuy else if ( - (POSTAGE_EXTENDSTTL === 'true' && - POSTAGE_AMOUNT && - POSTAGE_DEPTH && - Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE) || - POSTAGE_EXTENDS_CAPACITY === 'true' + (isBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true') || + (isBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true') ) { - const amount = getAmount(POSTAGE_AMOUNT) - const usageThreshold = getUsageThreshold( + return createExtendsStampsConfig( + POSTAGE_EXTENDSTTL, + POSTAGE_EXTENDS_CAPACITY, + POSTAGE_AMOUNT, POSTAGE_USAGE_THRESHOLD, - POSTAGE_EXTENDS_CAPACITY === 'true', - DEFAULT_POSTAGE_USAGE_THRESHOLD, - ) - const ttlMin = getTTLMin(POSTAGE_TTL_MIN) - const depth = getDepth(POSTAGE_DEPTH) - - return { - mode: 'extends', - depth, - ttlMin, - amount, - usageThreshold, + POSTAGE_TTL_MIN, + POSTAGE_DEPTH, refreshPeriod, beeDebugApiUrl, - } + ) } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { return { mode: 'autobuy', @@ -201,6 +192,55 @@ export function getStampsConfig({ return undefined } +export function createExtendsStampsConfig( + POSTAGE_EXTENDSTTL: string | undefined, + POSTAGE_EXTENDS_CAPACITY: string | undefined, + POSTAGE_AMOUNT: string | undefined, + POSTAGE_USAGE_THRESHOLD: string | undefined, + POSTAGE_TTL_MIN: string | undefined, + POSTAGE_DEPTH: string | undefined, + refreshPeriod: number, + beeDebugApiUrl: string, +): StampsConfigExtends { + if ( + POSTAGE_EXTENDSTTL === 'true' && + (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE || + !isInteger(POSTAGE_AMOUNT) || + (POSTAGE_TTL_MIN && !isInteger(POSTAGE_TTL_MIN)) || + (POSTAGE_DEPTH && !isInteger(POSTAGE_DEPTH))) + ) { + throw new Error( + `config: to extends stamps TTL please provide POSTAGE_TTL_MIN bigger than ${MINIMAL_EXTENDS_TTL_VALUE}, valid values for + POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are + POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`, + ) + } + + if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !isInteger(POSTAGE_USAGE_THRESHOLD)) { + throw new Error( + `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is + POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`, + ) + } + + const amount = POSTAGE_AMOUNT || '0' + const usageThreshold = Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD) + const ttlMin = Number(POSTAGE_TTL_MIN) + const depth = Number(POSTAGE_DEPTH) + + return { + mode: 'extends', + ttl: POSTAGE_EXTENDSTTL === 'true', + capacity: POSTAGE_EXTENDS_CAPACITY === 'true', + depth, + ttlMin, + amount, + usageThreshold, + refreshPeriod, + beeDebugApiUrl, + } +} + export function getContentConfig({ BEE_API_URL, REUPLOAD_PERIOD }: EnvironmentVariables = {}): ContentConfig | false { if (!REUPLOAD_PERIOD) { return false diff --git a/src/index.ts b/src/index.ts index 359976a2..b21c5604 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,15 +37,11 @@ async function main() { logger.info('starting hardcoded postage stamp manager') stampManager.start(stampsConfig) } else if (mode === 'autobuy') { - stampManager = new AutoBuyStampsManager() - logger.info('starting autobuy postage stamp manager') - stampManager.start(stampsConfig) + stampManager = new AutoBuyStampsManager(stampsConfig) } else { - stampManager = new ExtendsStampManager() - logger.info('starting extends postage stamp manager') - stampManager.start(stampsConfig) + stampManager = new ExtendsStampManager(stampsConfig) } logger.info('starting the proxy') app = createApp(appConfig, stampManager) diff --git a/src/readiness.ts b/src/readiness.ts index 6bbf9083..72459ddd 100644 --- a/src/readiness.ts +++ b/src/readiness.ts @@ -40,7 +40,7 @@ async function tryUploadingSingleChunk( ): Promise { const chunk = makeChunk() try { - await bee.uploadChunk(stampsManager.postageStamp, chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) + await bee.uploadChunk(stampsManager.postageStamp(), chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) return ReadinessStatus.OK } catch (error) { diff --git a/src/server.ts b/src/server.ts index 5e242314..070219f4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -136,7 +136,7 @@ export const createApp = ( if (stampManager) { proxyReq.removeHeader(SWARM_STAMP_HEADER) try { - proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp) + proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp()) } catch (error) { logger.error('proxy failure', error) diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts index 5c62e42f..444cd1a9 100644 --- a/src/stamps/autobuy.ts +++ b/src/stamps/autobuy.ts @@ -9,7 +9,7 @@ import { stampUsableCountGauge, stampUsageGauge, } from './counters' -import { BaseStampManager } from './base' +import { BaseStampManager, StampsManager } from './base' /** * Filter the stamps and only return those that are usable, have correct amount, depth, are not close to beying maxUsage or close to expire @@ -39,19 +39,14 @@ export function filterUsableStampsAutobuy( return usableStamps } -export class AutoBuyStampsManager extends BaseStampManager { - public isBuyingStamp?: boolean = false - /** - * Start the manager in either hardcoded or autobuy mode - */ - async start(config: StampsConfigAutobuy): Promise { - // Autobuy mode - const refreshStamps = async () => this.refreshStampsAutobuy(config, new BeeDebug(config.beeDebugApiUrl)) +export class AutoBuyStampsManager extends BaseStampManager implements StampsManager { + private isBuyingStamp?: boolean = false - this.stop() - await refreshStamps() - - this.interval = setInterval(refreshStamps, config.refreshPeriod) + constructor(config: StampsConfigAutobuy) { + super() + // Autobuy mode + const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) + this.startFeature(config, refreshStamps) } /** @@ -60,7 +55,7 @@ export class AutoBuyStampsManager extends BaseStampManager { * @param config Stamps config * @param beeDebug Connection to debug endpoint for checking/buying stamps */ - public async refreshStampsAutobuy(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { + private async refreshStamps(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { try { stampCheckCounter.inc() logger.info('checking postage stamps') diff --git a/src/stamps/base.ts b/src/stamps/base.ts index bac30ab3..39547136 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -1,11 +1,17 @@ import { PostageBatch } from '@ethersphere/bee-js' import { stampGetCounter, stampGetErrorCounter } from './counters' import { logger } from '../logger' -import { ERROR_NO_STAMP } from '../config' +import { ERROR_NO_STAMP, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../config' -export class BaseStampManager { +export interface StampsManager { + start?: (config: StampsConfig) => Promise + stop: () => void + postageStamp: () => string +} + +export class BaseStampManager implements StampsManager { + private interval?: ReturnType public stamp?: string - public interval?: ReturnType public usableStamps?: PostageBatch[] /** @@ -15,7 +21,7 @@ export class BaseStampManager { * * @throws Error if there is no postage stamp */ - get postageStamp(): string { + postageStamp(): string { stampGetCounter.inc() if (this.stamp) { @@ -36,6 +42,19 @@ export class BaseStampManager { throw new Error(ERROR_NO_STAMP) } + /** + * Start the manager in either hardcoded or autobuy mode + */ + public async startFeature( + config: StampsConfigAutobuy | StampsConfigExtends, + refreshStamps: () => void, + ): Promise { + this.stop() + refreshStamps() + + this.interval = setInterval(refreshStamps, config.refreshPeriod) + } + stop(): void { if (this.interval) { clearInterval(this.interval) diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index f133a60e..62088fc7 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -3,7 +3,7 @@ import { StampsConfigExtends } from '../config' import { buyNewStamp, getUsage } from '../utils' import { logger } from '../logger' import { stampCheckCounter } from './counters' -import { BaseStampManager } from './base' +import { BaseStampManager, StampsManager } from './base' /** * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL @@ -12,10 +12,10 @@ import { BaseStampManager } from './base' * * @returns Filtered stamps soltered by usage */ -export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { +export function filterUsableStampsExtendsTTL(stamps: PostageBatch[], minTimeThreshold: number): PostageBatch[] { const usableStamps = stamps // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable) + .filter(s => s.usable && s.batchTTL < minTimeThreshold) // sort the stamps by usage .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) @@ -52,20 +52,22 @@ export async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amo return stamp } -export class ExtendsStampManager extends BaseStampManager { +export async function extendsCapacity(extendManager: ExtendsStampManager, beeDebug: BeeDebug, stamp: PostageBatch) { + const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) + setTimeout(() => extendManager.completeTopUp('capacity', stampRes), 60000) + await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) + logger.info(`capacity extended for stamp ${stamp.batchID}`) +} + +export class ExtendsStampManager extends BaseStampManager implements StampsManager { private isBuyingStamp?: boolean = false private topingUpStamps: string[] = [] - /** - * Start the manager in either hardcoded or autobuy mode - */ - async start(config: StampsConfigExtends): Promise { - // Extends mode - const refreshStamps = async () => this.refreshStampsExtends(config, new BeeDebug(config.beeDebugApiUrl)) - - this.stop() - await refreshStamps() - this.interval = setInterval(refreshStamps, config.refreshPeriod) + constructor(config: StampsConfigExtends) { + super() + // Autobuy mode + const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) + this.startFeature(config, refreshStamps) } completeTopUp(extendsTypeFeature: 'ttl' | 'capacity', stamp: PostageBatch) { @@ -79,18 +81,9 @@ export class ExtendsStampManager extends BaseStampManager { this.topingUpStamps.splice(stampIndex, 1) } - async verifyUsableStamps(beeDebug: BeeDebug, ttlMin: number, config: StampsConfigExtends, amount: string) { - for (let i = 0; i < this.usableStamps!.length; i++) { - const stamp = this.usableStamps![i] - - const minTimeThreshold = ttlMin + config.refreshPeriod / 1000 - - if ( - ttlMin > 0 && - stamp.batchTTL < minTimeThreshold && - !this.topingUpStamps.includes(stamp.batchID) && - amount !== '0' - ) { + async verifyUsableStamps(beeDebug: BeeDebug, ttlMin: number, amount: string) { + for (const stamp of this.usableStamps!) { + if (ttlMin > 0 && !this.topingUpStamps.includes(stamp.batchID) && amount !== '0') { this.topingUpStamps.push(stamp.batchID) logger.info(`extending postage stamp TTL ${stamp.batchID}`) @@ -108,7 +101,7 @@ export class ExtendsStampManager extends BaseStampManager { } } - public async refreshStampsExtends(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { + private async refreshStamps(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { let usableStampsExtendsTTL: PostageBatch[] = [] let usableStampsExtendsCapacity: PostageBatch[] = [] stampCheckCounter.inc() @@ -117,10 +110,11 @@ export class ExtendsStampManager extends BaseStampManager { try { const stamps = await beeDebug.getAllPostageBatch() - const { depth, amount, ttlMin, usageThreshold } = config + const { depth, amount, ttlMin, refreshPeriod, usageThreshold } = config // Get all usable stamps sorted by usage from most used to least - usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps) + const minTimeThreshold = ttlMin + refreshPeriod / 1000 + usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) if (!this.isBuyingStamp) { if (depth > 0 && usableStampsExtendsTTL.length === 0) { @@ -135,7 +129,7 @@ export class ExtendsStampManager extends BaseStampManager { } } else { this.usableStamps = usableStampsExtendsTTL - await this.verifyUsableStamps(beeDebug, ttlMin, config, amount) + await this.verifyUsableStamps(beeDebug, ttlMin, amount) } } @@ -147,10 +141,7 @@ export class ExtendsStampManager extends BaseStampManager { logger.info(`extending stamp capacity: ${stamp.batchID}`) this.topingUpStamps.push(stamp.batchID) - const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) - setTimeout(() => this.completeTopUp('capacity', stampRes), 60000) - await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) - logger.info(`capacity extended for stamp ${stamp.batchID}`) + extendsCapacity(this, beeDebug, stamp) } } catch (err) { logger.error('failed to extend stamp capacity', err) diff --git a/src/stamps/hardcoded.ts b/src/stamps/hardcoded.ts index fcca28c0..cfeb7de1 100644 --- a/src/stamps/hardcoded.ts +++ b/src/stamps/hardcoded.ts @@ -1,7 +1,7 @@ import { StampsConfig } from '../config' -import { BaseStampManager } from './base' +import { BaseStampManager, StampsManager } from './base' -export class HardcodedStampsManager extends BaseStampManager { +export class HardcodedStampsManager extends BaseStampManager implements StampsManager { /** * Start the manager in either hardcoded or autobuy mode */ diff --git a/src/stamps/index.ts b/src/stamps/index.ts index 32215126..9035736d 100644 --- a/src/stamps/index.ts +++ b/src/stamps/index.ts @@ -1,7 +1,4 @@ -import { HardcodedStampsManager } from './hardcoded' -import { AutoBuyStampsManager } from './autobuy' -import { ExtendsStampManager } from './extends' - -export { HardcodedStampsManager } -export { AutoBuyStampsManager } -export { ExtendsStampManager } +export { HardcodedStampsManager } from './hardcoded' +export { AutoBuyStampsManager } from './autobuy' +export { ExtendsStampManager } from './extends' +export { StampsManager } from './base' diff --git a/src/utils.ts b/src/utils.ts index 17a0acb4..e6ef0b78 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,4 +1,4 @@ -import { BatchId, BeeDebug, PostageBatch } from '@ethersphere/bee-js' +import { BatchId, BeeDebug, NumberString, PostageBatch } from '@ethersphere/bee-js' import { logger } from './logger' import { stampPurchaseCounter } from './stamps/counters' @@ -50,24 +50,32 @@ export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): num return utilization / Math.pow(2, depth - bucketDepth) } -export function getUsageThreshold( - postageUsageThreshold: string | undefined, - isExtendsCapacity: boolean, - defaultUsageThreshold: number, -): number { - return Number(postageUsageThreshold || (isExtendsCapacity ? defaultUsageThreshold : '0')) -} - -export function getAmount(postageAmount: string | undefined): string { - return postageAmount ?? '0' -} - -export function getTTLMin(postageTTLMin: string | undefined): number { - return Number(postageTTLMin || '0') +/** + * Validate if the value is an integer + * @param value + * @returns boolean + */ +export function isBoolean(value: unknown): boolean { + return ( + typeof value === 'boolean' || + (typeof value === 'string' && (value === 'true' || value === 'false')) || + (typeof value === 'number' && (value === 1 || value === 0)) + ) } -export function getDepth(postageDepth: string | undefined): number { - return Number(postageDepth || '0') +/** + * Validate if the value is an integer + * @param value + * @returns boolean + */ +export function isInteger(value: unknown): value is number | string | NumberString { + return ( + typeof value === 'string' || + (typeof value === 'number' && + value > Number.MIN_SAFE_INTEGER && + value < Number.MAX_SAFE_INTEGER && + Number.isInteger(value)) + ) } /** diff --git a/test/config.spec.ts b/test/config.spec.ts index 8e6eebfb..db0c36d1 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -68,7 +68,7 @@ describe('getStampsConfig', () => { const POSTAGE_AMOUNT = '100' const POSTAGE_DEPTH = '20' const BEE_DEBUG_API_URL = 'http://localhost:1635' - const POSTAGE_USAGE_THRESHOLD = '0.6' + const POSTAGE_USAGE_THRESHOLD = '0.7' const POSTAGE_USAGE_MAX = '0.8' const POSTAGE_TTL_MIN = '200' const POSTAGE_REFRESH_PERIOD = '60000' @@ -149,7 +149,7 @@ describe('getStampsConfig', () => { mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, - usageThreshold: 0, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD), beeDebugApiUrl: BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), @@ -168,7 +168,7 @@ describe('getStampsConfig', () => { mode: 'extends', depth: Number(POSTAGE_DEPTH), amount: POSTAGE_AMOUNT, - usageThreshold: 0, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD), beeDebugApiUrl: BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 250618ca..0a0c5298 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,10 +1,16 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' import { buyNewStamp, getUsage, sleep } from '../src/utils' -import { getStampsConfig } from '../src/config' +import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' -import { topUpStamp } from '../src/stamps/extends' +import { + extendsCapacity, + ExtendsStampManager, + filterUsableStampsExtendsCapacity, + filterUsableStampsExtendsTTL, + topUpStamp, +} from '../src/stamps/extends' import { AutoBuyStampsManager, filterUsableStampsAutobuy } from '../src/stamps/autobuy' import { HardcodedStampsManager } from '../src/stamps' @@ -34,6 +40,7 @@ afterEach(() => { const defaultAmount = '1000000' const defaultDepth = 20 const defaultTTL = Number(defaultAmount) +const defaultUsageThreshold = 0.7 const defaultStamp: PostageBatch = { batchID: genRandomHex(64) as BatchId, utilization: 0, @@ -63,104 +70,91 @@ describe('postageStamp', () => { const stamp = '0000000000000000000000000000000000000000000000000000000000000000' const stampManager = new HardcodedStampsManager() await stampManager.start(getStampsConfig({ POSTAGE_STAMP: stamp })!) - expect(stampManager.postageStamp).toEqual(stamp) + expect(stampManager.postageStamp()).toEqual(stamp) }) it('should return existing stamp', async () => { const stamp = buildStamp({ utilization: 0 }) db.add(stamp) - const manager = new AutoBuyStampsManager() const stampConfig = getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), POSTAGE_AMOUNT: defaultAmount.toString(), POSTAGE_REFRESH_PERIOD: '200', BEE_DEBUG_API_URL: url, - })! - - if (stampConfig.mode === 'autobuy') { - await manager.start(stampConfig) - await sleep(1_000) - expect(db.toArray().length).toEqual(1) - - const batchId = manager.postageStamp - expect(batchId).toBe(stamp.batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress - } + }) as StampsConfigAutobuy + const manager = new AutoBuyStampsManager(stampConfig) + + await sleep(1_000) + expect(db.toArray().length).toEqual(1) + + const batchId = manager.postageStamp() + expect(batchId).toBe(stamp.batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) it('should start without any postage stamp and create new one', async () => { - const manager = new AutoBuyStampsManager() const stampConfig = getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), POSTAGE_AMOUNT: defaultAmount.toString(), POSTAGE_REFRESH_PERIOD: '200', BEE_DEBUG_API_URL: url, - })! + }) as StampsConfigAutobuy + const manager = new AutoBuyStampsManager(stampConfig) - if (stampConfig.mode === 'autobuy') { - await manager.start(stampConfig) - await sleep(1_000) - expect(db.toArray().length).toEqual(1) + await sleep(4_000) + expect(db.toArray().length).toEqual(1) - expect(manager.postageStamp).toEqual(db.toArray()[0].batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress - } + const batchId = manager.postageStamp() + expect(batchId).toEqual(db.toArray()[0].batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) it('should create additional stamp if existing is starting to get full', async () => { const stamp = buildStamp({ utilization: 14 }) db.add(stamp) - - const manager = new AutoBuyStampsManager() const stampConfig = getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), POSTAGE_AMOUNT: defaultAmount.toString(), POSTAGE_REFRESH_PERIOD: '200', BEE_DEBUG_API_URL: url, - })! + }) as StampsConfigAutobuy - if (stampConfig.mode === 'autobuy') { - await manager.start(stampConfig) + const manager = new AutoBuyStampsManager(stampConfig) - await sleep(1_000) + await sleep(1_000) - expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).toEqual(stamp.batchID) - manager.stop() - await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress - } + expect(db.toArray().length).toEqual(2) + expect(manager.postageStamp()).toEqual(stamp.batchID) + manager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) it('should create additional stamp if existing stamp usage increases', async () => { const stamp = buildStamp({ utilization: 5 }) db.add(stamp) - - const manager = new AutoBuyStampsManager() const stampConfig = getStampsConfig({ POSTAGE_DEPTH: defaultDepth.toString(), POSTAGE_AMOUNT: defaultAmount.toString(), POSTAGE_REFRESH_PERIOD: '200', POSTAGE_USAGE_MAX: '0.8', BEE_DEBUG_API_URL: url, - })! + }) as StampsConfigAutobuy - if (stampConfig.mode === 'autobuy') { - await manager.start(stampConfig) + const manager = new AutoBuyStampsManager(stampConfig) - await sleep(200) - expect(db.toArray().length).toEqual(1) - expect(manager.postageStamp).toEqual(stamp.batchID) + await sleep(200) + expect(db.toArray().length).toEqual(1) + expect(manager.postageStamp()).toEqual(stamp.batchID) - stamp.utilization = 15 - await sleep(500) + stamp.utilization = 15 + await sleep(500) - expect(db.toArray().length).toEqual(2) - expect(manager.postageStamp).not.toEqual(stamp.batchID) - manager.stop() - await sleep(1500) // Needed as there could be the wait for posage stamp usable process in progress - } + expect(db.toArray().length).toEqual(2) + expect(manager.postageStamp()).not.toEqual(stamp.batchID) + manager.stop() + await sleep(1500) // Needed as there could be the wait for posage stamp usable process in progress }) }) @@ -240,27 +234,76 @@ describe('filterUsableStamps', () => { }) }) -describe('topUpStamp', () => { - it('should extend stamp stamp and await for it to extend others', async () => { +describe('extendsStampsTTL', () => { + let stampsConfig: StampsConfig + + beforeEach(() => { + stampsConfig = getStampsConfig({ + POSTAGE_DEPTH: defaultDepth.toString(), + POSTAGE_AMOUNT: defaultAmount.toString(), + POSTAGE_TTL_MIN: defaultTTL.toString(), + POSTAGE_USAGE_THRESHOLD: defaultUsageThreshold.toString(), + POSTAGE_REFRESH_PERIOD: '200', + POSTAGE_EXTENDSTTL: 'true', + BEE_DEBUG_API_URL: url, + }) as StampsConfig + }) + + it('should not find any usable stamps', async () => { + const stamps = [ + buildStamp({ utilization: 7, batchTTL: 200_000, usable: false }), + buildStamp({ utilization: 8, batchTTL: 150_000, usable: false }), + buildStamp({ utilization: 10, batchTTL: 120_000, usable: false }), + ] + + const { ttlMin, refreshPeriod } = stampsConfig as StampsConfigExtends + const minTimeThreshold = ttlMin + refreshPeriod / 1000 + const res = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) + expect(0).toEqual(res.length) + }) + + it('should extend stamp ttl and await for it to extend others', async () => { const beeDebug = new BeeDebug(url) - const stamp = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) + const stampId = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) const extendAmount = '100' - await topUpStamp(beeDebug, stamp.batchId, extendAmount) - const stampExtended = await beeDebug.getPostageBatch(stamp.batchId) + await topUpStamp(beeDebug, stampId.batchId, extendAmount) + const stampExtended = await beeDebug.getPostageBatch(stampId.batchId) expect(Number(stampExtended.amount)).toBeGreaterThan(Number(defaultAmount)) }) }) -describe('extendsCapacity', () => { +describe('extendsStampsCapacity', () => { + let stampsConfig: StampsConfig + + beforeEach(() => { + stampsConfig = getStampsConfig({ + POSTAGE_USAGE_THRESHOLD: defaultUsageThreshold.toString(), + POSTAGE_EXTENDS_CAPACITY: 'true', + BEE_DEBUG_API_URL: url, + }) as StampsConfig + }) + it('should not find any usable stamps', async () => { + const stamps = [ + buildStamp({ utilization: 14, depth: 20, bucketDepth: 16, usable: false }), + buildStamp({ utilization: 8, depth: 20, bucketDepth: 16, usable: true }), + buildStamp({ utilization: 10, depth: 20, bucketDepth: 16, usable: true }), + ] + const res = filterUsableStampsExtendsCapacity(stamps, defaultUsageThreshold) + expect(0).toEqual(res.length) + }) + it('should extend stamps capacity', async () => { + const extendManager = new ExtendsStampManager(stampsConfig as StampsConfigExtends) const beeDebug = new BeeDebug(url) - const stamp = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) + const stampId = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) - await topUpStamp(beeDebug, stamp.batchId, (Number(stamp.stamp.amount) * 2).toString()) - await beeDebug.diluteBatch(stamp.batchId, stamp.stamp.depth + 1) - const stampExtended = await beeDebug.getPostageBatch(stamp.batchId) + await extendsCapacity(extendManager, beeDebug, stampId.stamp) + await sleep(1_000) + const stampExtended = await beeDebug.getPostageBatch(stampId.batchId) expect(stampExtended.depth).toBeGreaterThan(defaultDepth) + extendManager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) }) From a4198e50abfca3d73d82c298aec032574e480d9c Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Tue, 1 Nov 2022 21:51:11 -0500 Subject: [PATCH 06/18] fix: unit test with all properties for extends config --- test/config.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/config.spec.ts b/test/config.spec.ts index db0c36d1..ed65e30a 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -153,6 +153,8 @@ describe('getStampsConfig', () => { beeDebugApiUrl: BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), + ttl: true, + capacity: false, }, }, { @@ -172,6 +174,8 @@ describe('getStampsConfig', () => { beeDebugApiUrl: BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), + ttl: true, + capacity: false, }, }, ] From 643896948c97fa5dd89caa0452eb7b70b753aeb7 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Tue, 1 Nov 2022 21:57:02 -0500 Subject: [PATCH 07/18] fix: introduce the capacity and ttl indicators --- src/stamps/extends.ts | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index 62088fc7..e77daea0 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -107,32 +107,32 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag stampCheckCounter.inc() logger.info('checking postage stamps') - try { - const stamps = await beeDebug.getAllPostageBatch() + const stamps = await beeDebug.getAllPostageBatch() - const { depth, amount, ttlMin, refreshPeriod, usageThreshold } = config + const { depth, amount, ttlMin, refreshPeriod, usageThreshold, ttl, capacity } = config - // Get all usable stamps sorted by usage from most used to least + // Get all usable stamps sorted by usage from most used to least + if (!this.isBuyingStamp && ttl) { const minTimeThreshold = ttlMin + refreshPeriod / 1000 usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) - if (!this.isBuyingStamp) { - if (depth > 0 && usableStampsExtendsTTL.length === 0) { - this.isBuyingStamp = true - try { - const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) + if (depth > 0 && usableStampsExtendsTTL.length === 0) { + this.isBuyingStamp = true + try { + const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) - // Add the bought postage stamp - usableStampsExtendsTTL.push(newStamp) - } finally { - this.isBuyingStamp = false - } - } else { - this.usableStamps = usableStampsExtendsTTL - await this.verifyUsableStamps(beeDebug, ttlMin, amount) + // Add the bought postage stamp + usableStampsExtendsTTL.push(newStamp) + } finally { + this.isBuyingStamp = false } + } else { + this.usableStamps = usableStampsExtendsTTL + await this.verifyUsableStamps(beeDebug, ttlMin, amount) } + } + if (capacity) { usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) for (const stamp of usableStampsExtendsCapacity) { @@ -147,8 +147,6 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag logger.error('failed to extend stamp capacity', err) } } - } catch (e) { - logger.error('failed to refresh on extends postage stamps', e) } } } From e3bd7523a2f424bfeece085c13232ef1848a048a Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Tue, 1 Nov 2022 22:00:49 -0500 Subject: [PATCH 08/18] fix: better naming on vars that enables ttl and capacity extends --- src/config.ts | 8 ++++---- src/stamps/extends.ts | 6 +++--- test/config.spec.ts | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/config.ts b/src/config.ts index 5a46e139..f9066517 100644 --- a/src/config.ts +++ b/src/config.ts @@ -34,8 +34,8 @@ export interface StampsConfigAutobuy { export interface StampsConfigExtends { mode: 'extends' - ttl: boolean - capacity: boolean + enableTtl: boolean + enableCapacity: boolean ttlMin: number depth: number amount: string @@ -230,8 +230,8 @@ export function createExtendsStampsConfig( return { mode: 'extends', - ttl: POSTAGE_EXTENDSTTL === 'true', - capacity: POSTAGE_EXTENDS_CAPACITY === 'true', + enableTtl: POSTAGE_EXTENDSTTL === 'true', + enableCapacity: POSTAGE_EXTENDS_CAPACITY === 'true', depth, ttlMin, amount, diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index e77daea0..3b8e764a 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -109,10 +109,10 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag const stamps = await beeDebug.getAllPostageBatch() - const { depth, amount, ttlMin, refreshPeriod, usageThreshold, ttl, capacity } = config + const { depth, amount, ttlMin, refreshPeriod, usageThreshold, enableTtl, enableCapacity } = config // Get all usable stamps sorted by usage from most used to least - if (!this.isBuyingStamp && ttl) { + if (!this.isBuyingStamp && enableTtl) { const minTimeThreshold = ttlMin + refreshPeriod / 1000 usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) @@ -132,7 +132,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag } } - if (capacity) { + if (enableCapacity) { usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) for (const stamp of usableStampsExtendsCapacity) { diff --git a/test/config.spec.ts b/test/config.spec.ts index ed65e30a..0d82507e 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -153,8 +153,8 @@ describe('getStampsConfig', () => { beeDebugApiUrl: BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), - ttl: true, - capacity: false, + enableTtl: true, + enableCapacity: false, }, }, { @@ -174,8 +174,8 @@ describe('getStampsConfig', () => { beeDebugApiUrl: BEE_DEBUG_API_URL, ttlMin: Number(POSTAGE_TTL_MIN), refreshPeriod: Number(POSTAGE_REFRESH_PERIOD), - ttl: true, - capacity: false, + enableTtl: true, + enableCapacity: false, }, }, ] From 3279a1f5a892e230f5e7c4c23354c1d73a4a8524 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Fri, 4 Nov 2022 18:22:10 -0500 Subject: [PATCH 09/18] fix: addressed all comments and some additional fixes --- src/config.ts | 65 ++++++++++++++++++++++++++++++------------- src/stamps/autobuy.ts | 2 +- src/stamps/base.ts | 15 +++++----- src/stamps/extends.ts | 22 +++++++-------- src/utils.ts | 21 +++++++++----- 5 files changed, 78 insertions(+), 47 deletions(-) diff --git a/src/config.ts b/src/config.ts index f9066517..d3e554fa 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { isInteger, isBoolean } from './utils' +import { isInteger, assertBoolean, assertInteger } from './utils' export interface AppConfig { beeApiUrl: string @@ -147,6 +147,7 @@ export function getStampsConfig({ POSTAGE_EXTENDSTTL, POSTAGE_EXTENDS_CAPACITY, }: EnvironmentVariables = {}): StampsConfig | undefined { + assertInteger(POSTAGE_REFRESH_PERIOD) const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD) const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL @@ -154,8 +155,8 @@ export function getStampsConfig({ if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP } // Start autobuy else if ( - (isBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true') || - (isBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true') + (assertBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true') || + (assertBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true') ) { return createExtendsStampsConfig( POSTAGE_EXTENDSTTL, @@ -168,23 +169,14 @@ export function getStampsConfig({ beeDebugApiUrl, ) } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { - return { - mode: 'autobuy', - depth: Number(POSTAGE_DEPTH), - amount: POSTAGE_AMOUNT, - usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), - usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), - ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), + return createAutobuyStampsConfig( + POSTAGE_DEPTH, + POSTAGE_AMOUNT, + POSTAGE_USAGE_THRESHOLD, + POSTAGE_USAGE_MAX, + POSTAGE_TTL_MIN, refreshPeriod, beeDebugApiUrl, - } - } - // Missing one of the variables needed for the autobuy or extends TTL - else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) { - throw new Error( - `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} ${ - POSTAGE_EXTENDSTTL === 'true' ? 'at least 60 seconds ' : '' - }or BEE_DEBUG_API_URL=${BEE_DEBUG_API_URL} for the feature to work`, ) } @@ -192,6 +184,35 @@ export function getStampsConfig({ return undefined } +export function createAutobuyStampsConfig( + POSTAGE_DEPTH: string, + POSTAGE_AMOUNT: string, + POSTAGE_USAGE_THRESHOLD: string | undefined, + POSTAGE_USAGE_MAX: string | undefined, + POSTAGE_TTL_MIN: string | undefined, + refreshPeriod: number, + beeDebugApiUrl: string, +): StampsConfigAutobuy { + // Missing one of the variables needed for the autobuy + if (!isInteger(POSTAGE_DEPTH) || !isInteger(POSTAGE_AMOUNT)) { + throw new Error( + `config: please provide valid values for POSTAGE_DEPTH, POSTAGE_AMOUNT for the autobuy feature to work. + Current state are POSTAGE_DEPTH=${POSTAGE_DEPTH} and POSTAGE_AMOUNT=${POSTAGE_AMOUNT}`, + ) + } + + return { + mode: 'autobuy', + depth: Number(POSTAGE_DEPTH), + amount: POSTAGE_AMOUNT, + usageThreshold: Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD), + usageMax: Number(POSTAGE_USAGE_MAX || DEFAULT_POSTAGE_USAGE_MAX), + ttlMin: Number(POSTAGE_TTL_MIN || (refreshPeriod / 1000) * 5), + refreshPeriod, + beeDebugApiUrl, + } +} + export function createExtendsStampsConfig( POSTAGE_EXTENDSTTL: string | undefined, POSTAGE_EXTENDS_CAPACITY: string | undefined, @@ -203,6 +224,7 @@ export function createExtendsStampsConfig( beeDebugApiUrl: string, ): StampsConfigExtends { if ( + assertBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true' && (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE || !isInteger(POSTAGE_AMOUNT) || @@ -216,7 +238,12 @@ export function createExtendsStampsConfig( ) } - if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !isInteger(POSTAGE_USAGE_THRESHOLD)) { + if ( + assertBoolean(POSTAGE_EXTENDS_CAPACITY) && + POSTAGE_EXTENDS_CAPACITY === 'true' && + POSTAGE_USAGE_THRESHOLD && + !isInteger(POSTAGE_USAGE_THRESHOLD) + ) { throw new Error( `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`, diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts index 444cd1a9..d92f1a59 100644 --- a/src/stamps/autobuy.ts +++ b/src/stamps/autobuy.ts @@ -46,7 +46,7 @@ export class AutoBuyStampsManager extends BaseStampManager implements StampsMana super() // Autobuy mode const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) - this.startFeature(config, refreshStamps) + this.start(config, refreshStamps) } /** diff --git a/src/stamps/base.ts b/src/stamps/base.ts index 39547136..1bf21ac1 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -1,15 +1,15 @@ import { PostageBatch } from '@ethersphere/bee-js' import { stampGetCounter, stampGetErrorCounter } from './counters' import { logger } from '../logger' -import { ERROR_NO_STAMP, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../config' +import { ERROR_NO_STAMP, StampsConfig } from '../config' export interface StampsManager { - start?: (config: StampsConfig) => Promise + start: (config: StampsConfig, refreshStamps: () => void) => Promise stop: () => void postageStamp: () => string } -export class BaseStampManager implements StampsManager { +export class BaseStampManager { private interval?: ReturnType public stamp?: string public usableStamps?: PostageBatch[] @@ -45,14 +45,13 @@ export class BaseStampManager implements StampsManager { /** * Start the manager in either hardcoded or autobuy mode */ - public async startFeature( - config: StampsConfigAutobuy | StampsConfigExtends, - refreshStamps: () => void, - ): Promise { + async start(config: StampsConfig, refreshStamps: () => void): Promise { this.stop() refreshStamps() - this.interval = setInterval(refreshStamps, config.refreshPeriod) + if (config.mode === 'autobuy' || config.mode === 'extends') { + this.interval = setInterval(refreshStamps, config.refreshPeriod) + } } stop(): void { diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index 3b8e764a..2bafebc5 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -12,10 +12,10 @@ import { BaseStampManager, StampsManager } from './base' * * @returns Filtered stamps soltered by usage */ -export function filterUsableStampsExtendsTTL(stamps: PostageBatch[], minTimeThreshold: number): PostageBatch[] { +export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { const usableStamps = stamps // filter to get stamps that have the right depth, amount and are not fully used or expired - .filter(s => s.usable && s.batchTTL < minTimeThreshold) + .filter(s => s.usable) // sort the stamps by usage .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) @@ -67,7 +67,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag super() // Autobuy mode const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) - this.startFeature(config, refreshStamps) + this.start(config, refreshStamps) } completeTopUp(extendsTypeFeature: 'ttl' | 'capacity', stamp: PostageBatch) { @@ -83,7 +83,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag async verifyUsableStamps(beeDebug: BeeDebug, ttlMin: number, amount: string) { for (const stamp of this.usableStamps!) { - if (ttlMin > 0 && !this.topingUpStamps.includes(stamp.batchID) && amount !== '0') { + if (!this.topingUpStamps.includes(stamp.batchID) && amount !== '0') { this.topingUpStamps.push(stamp.batchID) logger.info(`extending postage stamp TTL ${stamp.batchID}`) @@ -102,8 +102,6 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag } private async refreshStamps(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { - let usableStampsExtendsTTL: PostageBatch[] = [] - let usableStampsExtendsCapacity: PostageBatch[] = [] stampCheckCounter.inc() logger.info('checking postage stamps') @@ -113,27 +111,27 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag // Get all usable stamps sorted by usage from most used to least if (!this.isBuyingStamp && enableTtl) { - const minTimeThreshold = ttlMin + refreshPeriod / 1000 - usableStampsExtendsTTL = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) + const usableStampsSortByTTL = filterUsableStampsExtendsTTL(stamps) - if (depth > 0 && usableStampsExtendsTTL.length === 0) { + if (usableStampsSortByTTL.length === 0) { this.isBuyingStamp = true try { const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) // Add the bought postage stamp - usableStampsExtendsTTL.push(newStamp) + usableStampsSortByTTL.push(newStamp) } finally { this.isBuyingStamp = false } } else { - this.usableStamps = usableStampsExtendsTTL + const minTimeThreshold = ttlMin + refreshPeriod / 1000 + this.usableStamps = usableStampsSortByTTL.filter(s => s.batchTTL < minTimeThreshold) await this.verifyUsableStamps(beeDebug, ttlMin, amount) } } if (enableCapacity) { - usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) + const usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) for (const stamp of usableStampsExtendsCapacity) { try { diff --git a/src/utils.ts b/src/utils.ts index e6ef0b78..6dc422f0 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -55,12 +55,15 @@ export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): num * @param value * @returns boolean */ -export function isBoolean(value: unknown): boolean { - return ( - typeof value === 'boolean' || - (typeof value === 'string' && (value === 'true' || value === 'false')) || - (typeof value === 'number' && (value === 1 || value === 0)) - ) +export function assertBoolean(value: unknown): boolean { + if ( + (typeof value === 'string' && value !== 'true' && value !== 'false') || + (typeof value === 'number' && value !== 1 && value !== 0) + ) { + throw Error(`Invalid value pass as boolean: ${value}`) + } + + return true } /** @@ -70,7 +73,7 @@ export function isBoolean(value: unknown): boolean { */ export function isInteger(value: unknown): value is number | string | NumberString { return ( - typeof value === 'string' || + (typeof value === 'string' && /^-?(0|[1-9][0-9]*)$/g.test(value)) || (typeof value === 'number' && value > Number.MIN_SAFE_INTEGER && value < Number.MAX_SAFE_INTEGER && @@ -78,6 +81,10 @@ export function isInteger(value: unknown): value is number | string | NumberStri ) } +export function assertInteger(value: unknown, name = 'value'): asserts value is number | NumberString { + if (!isInteger(value)) throw new TypeError(`${name} is not integer`) +} + /** * Buy new postage stamp and wait until it is usable * From 76b09d240fd5723ffaf326465be63bb54985e605 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Fri, 4 Nov 2022 19:03:36 -0500 Subject: [PATCH 10/18] fix: some unit test oriented fixes --- src/config.ts | 16 +++++++++++++--- src/utils.ts | 14 +++++++++++++- test/config.spec.ts | 7 ++----- test/stamps.spec.ts | 3 ++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/config.ts b/src/config.ts index d3e554fa..bb08d194 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { isInteger, assertBoolean, assertInteger } from './utils' +import { isInteger, assertBoolean, assertInteger, assertDecimal } from './utils' export interface AppConfig { beeApiUrl: string @@ -147,7 +147,10 @@ export function getStampsConfig({ POSTAGE_EXTENDSTTL, POSTAGE_EXTENDS_CAPACITY, }: EnvironmentVariables = {}): StampsConfig | undefined { - assertInteger(POSTAGE_REFRESH_PERIOD) + if (POSTAGE_REFRESH_PERIOD) { + assertInteger(POSTAGE_REFRESH_PERIOD) + } + const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD) const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL @@ -179,6 +182,13 @@ export function getStampsConfig({ beeDebugApiUrl, ) } + // Missing one of the variables needed for the autobuy or extends TTL + else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) { + throw new Error( + `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} + or BEE_DEBUG_API_URL=${BEE_DEBUG_API_URL} for the feature to work`, + ) + } // Stamps rewrite is disabled return undefined @@ -242,7 +252,7 @@ export function createExtendsStampsConfig( assertBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && - !isInteger(POSTAGE_USAGE_THRESHOLD) + !assertDecimal(POSTAGE_USAGE_THRESHOLD) ) { throw new Error( `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is diff --git a/src/utils.ts b/src/utils.ts index 6dc422f0..6dcfee44 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -82,7 +82,19 @@ export function isInteger(value: unknown): value is number | string | NumberStri } export function assertInteger(value: unknown, name = 'value'): asserts value is number | NumberString { - if (!isInteger(value)) throw new TypeError(`${name} is not integer`) + if (!isInteger(value)) throw new TypeError(`${name} is not a valid integer`) +} + +export function assertDecimal(value: unknown, name = 'value'): boolean { + if (typeof value === 'number') { + value = value.toString() + } + + if (typeof value !== 'string' || (typeof value === 'string' && !/^[-+]?[0]+\.[0-9]+$/g.test(value))) { + throw new TypeError(`${name} is not a valid decimal`) + } + + return true } /** diff --git a/test/config.spec.ts b/test/config.spec.ts index 0d82507e..8f0959ce 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -201,11 +201,8 @@ describe('getStampsConfig', () => { expect(() => { getStampsConfig(v) }).toThrowError( - `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${ - v.POSTAGE_TTL_MIN - } ${v.POSTAGE_EXTENDSTTL === 'true' ? 'at least 60 seconds ' : ''}or BEE_DEBUG_API_URL=${ - v.BEE_DEBUG_API_URL - } for the feature to work`, + `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${v.POSTAGE_TTL_MIN} + or BEE_DEBUG_API_URL=${v.BEE_DEBUG_API_URL} for the feature to work`, ) }) }) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 0a0c5298..b6424f4f 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -258,7 +258,8 @@ describe('extendsStampsTTL', () => { const { ttlMin, refreshPeriod } = stampsConfig as StampsConfigExtends const minTimeThreshold = ttlMin + refreshPeriod / 1000 - const res = filterUsableStampsExtendsTTL(stamps, minTimeThreshold) + const usableStampsSortByTTL = filterUsableStampsExtendsTTL(stamps) + const res = usableStampsSortByTTL.filter(s => s.batchTTL < minTimeThreshold) expect(0).toEqual(res.length) }) From 9598a4064910b7fd06a393e40e39342f43b2e8ed Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Sat, 5 Nov 2022 02:50:17 -0500 Subject: [PATCH 11/18] fix: fixes unit test related --- README.md | 4 ++-- src/config.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9f223994..954dc023 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ npm run start ```sh export POSTAGE_DEPTH=20 export POSTAGE_AMOUNT=1000000 -export BEE_DEBUG_API_URL=http://localhost:1635 +export BEE_DEBUG_API_URL=http://localhost:1635 (optional) npm run start ``` @@ -111,7 +111,7 @@ export POSTAGE_EXTENDSTTL=true export POSTAGE_TTL_MIN=60 export POSTAGE_DEPTH=20 export POSTAGE_AMOUNT=1000000 -export BEE_DEBUG_API_URL=http://localhost:1635 +export BEE_DEBUG_API_URL=http://localhost:1635 (optional) npm run start ``` diff --git a/src/config.ts b/src/config.ts index bb08d194..e9c9c585 100644 --- a/src/config.ts +++ b/src/config.ts @@ -171,7 +171,7 @@ export function getStampsConfig({ refreshPeriod, beeDebugApiUrl, ) - } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) { + } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT) { return createAutobuyStampsConfig( POSTAGE_DEPTH, POSTAGE_AMOUNT, @@ -183,10 +183,10 @@ export function getStampsConfig({ ) } // Missing one of the variables needed for the autobuy or extends TTL - else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) { + else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN) { throw new Error( - `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} - or BEE_DEBUG_API_URL=${BEE_DEBUG_API_URL} for the feature to work`, + `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} + for the feature to work`, ) } @@ -243,8 +243,8 @@ export function createExtendsStampsConfig( ) { throw new Error( `config: to extends stamps TTL please provide POSTAGE_TTL_MIN bigger than ${MINIMAL_EXTENDS_TTL_VALUE}, valid values for - POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are - POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`, + POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN}, + POSTAGE_AMOUNT=${POSTAGE_AMOUNT} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`, ) } From 86de7f8342dcd0d28d2be745ee75f04480329060 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Sat, 5 Nov 2022 03:06:49 -0500 Subject: [PATCH 12/18] fix: unit test of the env variables update --- src/config.ts | 3 +-- test/config.spec.ts | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index e9c9c585..53e8b4f3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -185,8 +185,7 @@ export function getStampsConfig({ // Missing one of the variables needed for the autobuy or extends TTL else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN) { throw new Error( - `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} - for the feature to work`, + `config: please provide POSTAGE_DEPTH=${POSTAGE_DEPTH}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} for the feature to work`, ) } diff --git a/test/config.spec.ts b/test/config.spec.ts index 8f0959ce..9b3d1bbf 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -188,12 +188,10 @@ describe('getStampsConfig', () => { }) const throwValues: EnvironmentVariables[] = [ - { BEE_DEBUG_API_URL }, { POSTAGE_DEPTH }, { POSTAGE_AMOUNT }, { BEE_DEBUG_API_URL, POSTAGE_DEPTH }, { BEE_DEBUG_API_URL, POSTAGE_AMOUNT }, - { POSTAGE_DEPTH, POSTAGE_AMOUNT }, ] throwValues.forEach(v => { @@ -201,8 +199,7 @@ describe('getStampsConfig', () => { expect(() => { getStampsConfig(v) }).toThrowError( - `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${v.POSTAGE_TTL_MIN} - or BEE_DEBUG_API_URL=${v.BEE_DEBUG_API_URL} for the feature to work`, + `config: please provide POSTAGE_DEPTH=${v.POSTAGE_DEPTH}, POSTAGE_AMOUNT=${v.POSTAGE_AMOUNT} or POSTAGE_TTL_MIN=${v.POSTAGE_TTL_MIN} for the feature to work`, ) }) }) From dee12830a9ca4159c7001c72b78132495dcf85b5 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Fri, 11 Nov 2022 22:37:46 -0500 Subject: [PATCH 13/18] fix: all comments on PR addressed --- src/config.ts | 17 ++++------------- src/index.ts | 4 +--- src/server.ts | 4 ++-- src/stamps/autobuy.ts | 2 +- src/stamps/base.ts | 11 +++++++---- src/stamps/extends.ts | 14 ++++++++++---- src/stamps/hardcoded.ts | 11 ++++------- src/utils.ts | 16 ---------------- test/readiness.spec.ts | 3 +-- test/server.spec.ts | 3 +-- test/stamps.spec.ts | 6 +++--- 11 files changed, 34 insertions(+), 57 deletions(-) diff --git a/src/config.ts b/src/config.ts index 53e8b4f3..5b1765db 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { isInteger, assertBoolean, assertInteger, assertDecimal } from './utils' +import { isInteger, assertInteger, assertDecimal } from './utils' export interface AppConfig { beeApiUrl: string @@ -16,7 +16,7 @@ export interface ServerConfig { port: number } -interface StampsConfigHardcoded { +export interface StampsConfigHardcoded { mode: 'hardcoded' stamp: string } @@ -157,10 +157,7 @@ export function getStampsConfig({ // Start in hardcoded mode if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP } // Start autobuy - else if ( - (assertBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true') || - (assertBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true') - ) { + else if (POSTAGE_EXTENDSTTL === 'true' || POSTAGE_EXTENDS_CAPACITY === 'true') { return createExtendsStampsConfig( POSTAGE_EXTENDSTTL, POSTAGE_EXTENDS_CAPACITY, @@ -233,7 +230,6 @@ export function createExtendsStampsConfig( beeDebugApiUrl: string, ): StampsConfigExtends { if ( - assertBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true' && (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE || !isInteger(POSTAGE_AMOUNT) || @@ -247,12 +243,7 @@ export function createExtendsStampsConfig( ) } - if ( - assertBoolean(POSTAGE_EXTENDS_CAPACITY) && - POSTAGE_EXTENDS_CAPACITY === 'true' && - POSTAGE_USAGE_THRESHOLD && - !assertDecimal(POSTAGE_USAGE_THRESHOLD) - ) { + if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !assertDecimal(POSTAGE_USAGE_THRESHOLD)) { throw new Error( `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`, diff --git a/src/index.ts b/src/index.ts index b21c5604..70866c89 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,10 +32,8 @@ async function main() { const { mode } = stampsConfig if (mode === 'hardcoded') { - stampManager = new HardcodedStampsManager() - + stampManager = new HardcodedStampsManager(stampsConfig) logger.info('starting hardcoded postage stamp manager') - stampManager.start(stampsConfig) } else if (mode === 'autobuy') { logger.info('starting autobuy postage stamp manager') stampManager = new AutoBuyStampsManager(stampsConfig) diff --git a/src/server.ts b/src/server.ts index 070219f4..018ab4ae 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,8 +7,8 @@ import { fetchBeeIdentity, getHashedIdentity, HASHED_IDENTITY_HEADER } from './i import { logger } from './logger' import { register } from './metrics' import { checkReadiness, ReadinessStatus } from './readiness' -import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' import { getErrorMessage } from './utils' +import type { BaseStampManager } from './stamps/base' const SWARM_STAMP_HEADER = 'swarm-postage-batch-id' @@ -32,7 +32,7 @@ export const createApp = ( removePinHeader, exposeHashedIdentity, }: AppConfig, - stampManager?: AutoBuyStampsManager | ExtendsStampManager | HardcodedStampsManager, + stampManager?: BaseStampManager, ): Application => { const commonOptions: Options = { target: beeApiUrl, diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts index d92f1a59..8fbfa69e 100644 --- a/src/stamps/autobuy.ts +++ b/src/stamps/autobuy.ts @@ -44,7 +44,7 @@ export class AutoBuyStampsManager extends BaseStampManager implements StampsMana constructor(config: StampsConfigAutobuy) { super() - // Autobuy mode + const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) this.start(config, refreshStamps) } diff --git a/src/stamps/base.ts b/src/stamps/base.ts index 1bf21ac1..a2090030 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -43,14 +43,17 @@ export class BaseStampManager { } /** - * Start the manager in either hardcoded or autobuy mode + * Start the manager in either hardcoded, autobuy or extends mode */ - async start(config: StampsConfig, refreshStamps: () => void): Promise { + async start(config: StampsConfig, refreshStamps?: () => void): Promise { this.stop() - refreshStamps() - if (config.mode === 'autobuy' || config.mode === 'extends') { + if (refreshStamps && (config.mode === 'autobuy' || config.mode === 'extends')) { + refreshStamps() + this.interval = setInterval(refreshStamps, config.refreshPeriod) + } else if (config.mode === 'hardcoded') { + this.stamp = config.stamp } } diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index 2bafebc5..0c9587d3 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -65,7 +65,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag constructor(config: StampsConfigExtends) { super() - // Autobuy mode + const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) this.start(config, refreshStamps) } @@ -105,12 +105,18 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag stampCheckCounter.inc() logger.info('checking postage stamps') - const stamps = await beeDebug.getAllPostageBatch() + let stamps: PostageBatch[] = [] + + try { + stamps = await beeDebug.getAllPostageBatch() + } catch (e) { + logger.error(`There's been an error getting postage batches: ${e}`) + } const { depth, amount, ttlMin, refreshPeriod, usageThreshold, enableTtl, enableCapacity } = config // Get all usable stamps sorted by usage from most used to least - if (!this.isBuyingStamp && enableTtl) { + if (!this.isBuyingStamp && enableTtl && stamps.length > 0) { const usableStampsSortByTTL = filterUsableStampsExtendsTTL(stamps) if (usableStampsSortByTTL.length === 0) { @@ -130,7 +136,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag } } - if (enableCapacity) { + if (enableCapacity && stamps.length > 0) { const usableStampsExtendsCapacity = filterUsableStampsExtendsCapacity(stamps, usageThreshold) for (const stamp of usableStampsExtendsCapacity) { diff --git a/src/stamps/hardcoded.ts b/src/stamps/hardcoded.ts index cfeb7de1..5c4a4479 100644 --- a/src/stamps/hardcoded.ts +++ b/src/stamps/hardcoded.ts @@ -1,12 +1,9 @@ -import { StampsConfig } from '../config' +import { StampsConfigHardcoded } from '../config' import { BaseStampManager, StampsManager } from './base' export class HardcodedStampsManager extends BaseStampManager implements StampsManager { - /** - * Start the manager in either hardcoded or autobuy mode - */ - async start(config: StampsConfig): Promise { - // Hardcoded stamp mode - if (config.mode === 'hardcoded') this.stamp = config.stamp + constructor(config: StampsConfigHardcoded) { + super() + this.start(config) } } diff --git a/src/utils.ts b/src/utils.ts index 6dcfee44..3cbd1a48 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -50,22 +50,6 @@ export function getUsage({ utilization, depth, bucketDepth }: PostageBatch): num return utilization / Math.pow(2, depth - bucketDepth) } -/** - * Validate if the value is an integer - * @param value - * @returns boolean - */ -export function assertBoolean(value: unknown): boolean { - if ( - (typeof value === 'string' && value !== 'true' && value !== 'false') || - (typeof value === 'number' && value !== 1 && value !== 0) - ) { - throw Error(`Invalid value pass as boolean: ${value}`) - } - - return true -} - /** * Validate if the value is an integer * @param value diff --git a/test/readiness.spec.ts b/test/readiness.spec.ts index 0b090d16..162df29a 100644 --- a/test/readiness.spec.ts +++ b/test/readiness.spec.ts @@ -13,8 +13,7 @@ describe('readiness', () => { }) test('should be ready with stamp management', async () => { - const stampManager = new HardcodedStampsManager() - await stampManager.start({ mode: 'hardcoded', stamp: process.env.BEE_POSTAGE as string }) + const stampManager = new HardcodedStampsManager({ mode: 'hardcoded', stamp: process.env.BEE_POSTAGE as string }) const app = createApp( { beeApiUrl: 'http://localhost:1633', diff --git a/test/server.spec.ts b/test/server.spec.ts index 39d4294d..13cf0ac5 100644 --- a/test/server.spec.ts +++ b/test/server.spec.ts @@ -36,8 +36,7 @@ beforeAll(async () => { beeProxy = new Bee(`http://localhost:${port}`) const stamp = getPostageBatch() - const stampManager = new HardcodedStampsManager() - await stampManager.start({ mode: 'hardcoded', stamp }) + const stampManager = new HardcodedStampsManager({ mode: 'hardcoded', stamp }) const appWithStamp = createApp({ beeApiUrl, beeDebugApiUrl: DEFAULT_BEE_DEBUG_API_URL }, stampManager) proxyWithStamp = await new Promise((resolve, _reject) => { const server = appWithStamp.listen(async () => resolve(server)) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index b6424f4f..1a59f264 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,7 +1,7 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' import { buyNewStamp, getUsage, sleep } from '../src/utils' -import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../src/config' +import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends, StampsConfigHardcoded } from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' import { @@ -68,8 +68,8 @@ const buildStamp = (overwrites: Partial) => { describe('postageStamp', () => { it('should return correct hardcoded single postage stamp', async () => { const stamp = '0000000000000000000000000000000000000000000000000000000000000000' - const stampManager = new HardcodedStampsManager() - await stampManager.start(getStampsConfig({ POSTAGE_STAMP: stamp })!) + const config = getStampsConfig({ POSTAGE_STAMP: stamp })! as StampsConfigHardcoded + const stampManager = new HardcodedStampsManager(config) expect(stampManager.postageStamp()).toEqual(stamp) }) From d3244b25c353ad28544f666d09a19d74c81aa0e1 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Fri, 11 Nov 2022 22:44:09 -0500 Subject: [PATCH 14/18] fix: minor prettier error on unit test --- test/stamps.spec.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 1a59f264..15fbfa17 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,7 +1,13 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' import { buyNewStamp, getUsage, sleep } from '../src/utils' -import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends, StampsConfigHardcoded } from '../src/config' +import { + getStampsConfig, + StampsConfig, + StampsConfigAutobuy, + StampsConfigExtends, + StampsConfigHardcoded, +} from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' import { From c5e08d478b59a50214bf326c4db8698e40cd2dde Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Tue, 15 Nov 2022 22:48:27 -0500 Subject: [PATCH 15/18] fix: removal of .start from constructors --- src/index.ts | 8 ++++++-- src/stamps/autobuy.ts | 17 ++++++++--------- src/stamps/base.ts | 8 ++++---- src/stamps/extends.ts | 22 +++++++++++----------- test/stamps.spec.ts | 17 +++++++++++------ 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/index.ts b/src/index.ts index 70866c89..8194c64c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } fro import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config' import { logger, subscribeLogServerRequests } from './logger' import { ContentManager } from './content' +import { BeeDebug } from '@ethersphere/bee-js' async function main() { // Configuration @@ -33,13 +34,16 @@ async function main() { if (mode === 'hardcoded') { stampManager = new HardcodedStampsManager(stampsConfig) + stampManager.start(stampsConfig) logger.info('starting hardcoded postage stamp manager') } else if (mode === 'autobuy') { logger.info('starting autobuy postage stamp manager') - stampManager = new AutoBuyStampsManager(stampsConfig) + stampManager = new AutoBuyStampsManager(new BeeDebug(stampsConfig.beeDebugApiUrl)) + stampManager.start(stampsConfig, async () => (stampManager as AutoBuyStampsManager).refreshStamps(stampsConfig)) } else { logger.info('starting extends postage stamp manager') - stampManager = new ExtendsStampManager(stampsConfig) + stampManager = new ExtendsStampManager(new BeeDebug(stampsConfig.beeDebugApiUrl)) + stampManager.start(stampsConfig, async () => (stampManager as ExtendsStampManager).refreshStamps(stampsConfig)) } logger.info('starting the proxy') app = createApp(appConfig, stampManager) diff --git a/src/stamps/autobuy.ts b/src/stamps/autobuy.ts index 8fbfa69e..6652cc38 100644 --- a/src/stamps/autobuy.ts +++ b/src/stamps/autobuy.ts @@ -1,7 +1,7 @@ import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' import { buyNewStamp, getUsage } from '../utils' import { logger } from '../logger' -import { StampsConfigAutobuy } from '../config' +import { StampsConfig, StampsConfigAutobuy } from '../config' import { stampCheckCounter, stampPurchaseFailedCounter, @@ -41,12 +41,11 @@ export function filterUsableStampsAutobuy( export class AutoBuyStampsManager extends BaseStampManager implements StampsManager { private isBuyingStamp?: boolean = false + private beeDebug: BeeDebug - constructor(config: StampsConfigAutobuy) { + constructor(beeDebug: BeeDebug) { super() - - const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) - this.start(config, refreshStamps) + this.beeDebug = beeDebug } /** @@ -55,14 +54,14 @@ export class AutoBuyStampsManager extends BaseStampManager implements StampsMana * @param config Stamps config * @param beeDebug Connection to debug endpoint for checking/buying stamps */ - private async refreshStamps(config: StampsConfigAutobuy, beeDebug: BeeDebug): Promise { + public async refreshStamps(config: StampsConfig) { try { stampCheckCounter.inc() logger.info('checking postage stamps') - const stamps = await beeDebug.getAllPostageBatch() + const stamps = await this.beeDebug.getAllPostageBatch() logger.debug('retrieved stamps', stamps) - const { depth, amount, usageMax, usageThreshold, ttlMin } = config + const { depth, amount, usageMax, usageThreshold, ttlMin } = config as StampsConfigAutobuy // Get all usable stamps sorted by usage from most used to least this.usableStamps = filterUsableStampsAutobuy(stamps, depth, amount, usageMax, ttlMin) @@ -77,7 +76,7 @@ export class AutoBuyStampsManager extends BaseStampManager implements StampsMana if (!this.isBuyingStamp && (!leastUsed || getUsage(leastUsed) > usageThreshold)) { this.isBuyingStamp = true try { - const { stamp } = await buyNewStamp(depth, amount, beeDebug) + const { stamp } = await buyNewStamp(depth, amount, this.beeDebug) // Add the bought postage stamp this.usableStamps.push(stamp) diff --git a/src/stamps/base.ts b/src/stamps/base.ts index a2090030..5922a753 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -4,7 +4,7 @@ import { logger } from '../logger' import { ERROR_NO_STAMP, StampsConfig } from '../config' export interface StampsManager { - start: (config: StampsConfig, refreshStamps: () => void) => Promise + start: (config: StampsConfig, refreshStamps: () => Promise) => Promise stop: () => void postageStamp: () => string } @@ -45,13 +45,13 @@ export class BaseStampManager { /** * Start the manager in either hardcoded, autobuy or extends mode */ - async start(config: StampsConfig, refreshStamps?: () => void): Promise { + async start(config: StampsConfig, refreshStamps?: (config: StampsConfig) => Promise) { this.stop() if (refreshStamps && (config.mode === 'autobuy' || config.mode === 'extends')) { - refreshStamps() + refreshStamps(config) - this.interval = setInterval(refreshStamps, config.refreshPeriod) + this.interval = setInterval(async () => refreshStamps, config.refreshPeriod) } else if (config.mode === 'hardcoded') { this.stamp = config.stamp } diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index 0c9587d3..876890fb 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -1,9 +1,9 @@ import { BeeDebug, PostageBatch } from '@ethersphere/bee-js' -import { StampsConfigExtends } from '../config' import { buyNewStamp, getUsage } from '../utils' import { logger } from '../logger' import { stampCheckCounter } from './counters' import { BaseStampManager, StampsManager } from './base' +import { StampsConfig, StampsConfigExtends } from '../config' /** * Filter the stamps and only return those that are usable and sort by from closer to farer expire TTL @@ -62,12 +62,11 @@ export async function extendsCapacity(extendManager: ExtendsStampManager, beeDeb export class ExtendsStampManager extends BaseStampManager implements StampsManager { private isBuyingStamp?: boolean = false private topingUpStamps: string[] = [] + private beeDebug: BeeDebug - constructor(config: StampsConfigExtends) { + constructor(beeDebug: BeeDebug) { super() - - const refreshStamps = async () => this.refreshStamps(config, new BeeDebug(config.beeDebugApiUrl)) - this.start(config, refreshStamps) + this.beeDebug = beeDebug } completeTopUp(extendsTypeFeature: 'ttl' | 'capacity', stamp: PostageBatch) { @@ -101,19 +100,20 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag } } - private async refreshStamps(config: StampsConfigExtends, beeDebug: BeeDebug): Promise { + public async refreshStamps(config: StampsConfig) { stampCheckCounter.inc() logger.info('checking postage stamps') let stamps: PostageBatch[] = [] try { - stamps = await beeDebug.getAllPostageBatch() + stamps = await this.beeDebug.getAllPostageBatch() } catch (e) { logger.error(`There's been an error getting postage batches: ${e}`) } - const { depth, amount, ttlMin, refreshPeriod, usageThreshold, enableTtl, enableCapacity } = config + const { depth, amount, ttlMin, refreshPeriod, usageThreshold, enableTtl, enableCapacity } = + config as StampsConfigExtends // Get all usable stamps sorted by usage from most used to least if (!this.isBuyingStamp && enableTtl && stamps.length > 0) { @@ -122,7 +122,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag if (usableStampsSortByTTL.length === 0) { this.isBuyingStamp = true try { - const { stamp: newStamp } = await buyNewStamp(depth, amount, beeDebug) + const { stamp: newStamp } = await buyNewStamp(depth, amount, this.beeDebug) // Add the bought postage stamp usableStampsSortByTTL.push(newStamp) @@ -132,7 +132,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag } else { const minTimeThreshold = ttlMin + refreshPeriod / 1000 this.usableStamps = usableStampsSortByTTL.filter(s => s.batchTTL < minTimeThreshold) - await this.verifyUsableStamps(beeDebug, ttlMin, amount) + await this.verifyUsableStamps(this.beeDebug, ttlMin, amount) } } @@ -145,7 +145,7 @@ export class ExtendsStampManager extends BaseStampManager implements StampsManag logger.info(`extending stamp capacity: ${stamp.batchID}`) this.topingUpStamps.push(stamp.batchID) - extendsCapacity(this, beeDebug, stamp) + extendsCapacity(this, this.beeDebug, stamp) } } catch (err) { logger.error('failed to extend stamp capacity', err) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index 15fbfa17..ebcf6ebf 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -88,7 +88,8 @@ describe('postageStamp', () => { POSTAGE_REFRESH_PERIOD: '200', BEE_DEBUG_API_URL: url, }) as StampsConfigAutobuy - const manager = new AutoBuyStampsManager(stampConfig) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(1_000) expect(db.toArray().length).toEqual(1) @@ -106,7 +107,8 @@ describe('postageStamp', () => { POSTAGE_REFRESH_PERIOD: '200', BEE_DEBUG_API_URL: url, }) as StampsConfigAutobuy - const manager = new AutoBuyStampsManager(stampConfig) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(4_000) expect(db.toArray().length).toEqual(1) @@ -127,7 +129,8 @@ describe('postageStamp', () => { BEE_DEBUG_API_URL: url, }) as StampsConfigAutobuy - const manager = new AutoBuyStampsManager(stampConfig) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(1_000) @@ -148,14 +151,15 @@ describe('postageStamp', () => { BEE_DEBUG_API_URL: url, }) as StampsConfigAutobuy - const manager = new AutoBuyStampsManager(stampConfig) + const manager = new AutoBuyStampsManager(new BeeDebug(url)) + manager.start(stampConfig, async () => (manager as AutoBuyStampsManager).refreshStamps(stampConfig)) await sleep(200) expect(db.toArray().length).toEqual(1) expect(manager.postageStamp()).toEqual(stamp.batchID) stamp.utilization = 15 - await sleep(500) + await sleep(2000) expect(db.toArray().length).toEqual(2) expect(manager.postageStamp()).not.toEqual(stamp.batchID) @@ -301,8 +305,9 @@ describe('extendsStampsCapacity', () => { }) it('should extend stamps capacity', async () => { - const extendManager = new ExtendsStampManager(stampsConfig as StampsConfigExtends) const beeDebug = new BeeDebug(url) + const extendManager = new ExtendsStampManager(beeDebug) + extendManager.start(stampsConfig, async () => (extendManager as ExtendsStampManager).refreshStamps(stampsConfig)) const stampId = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) From ea692382382d7d366c5fb5bfe0de4dcb3b946577 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Wed, 16 Nov 2022 19:44:59 -0500 Subject: [PATCH 16/18] fix: setInterval callback need to a litle clean up --- src/stamps/base.ts | 2 +- test/stamps.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stamps/base.ts b/src/stamps/base.ts index 5922a753..5d01d06e 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -51,7 +51,7 @@ export class BaseStampManager { if (refreshStamps && (config.mode === 'autobuy' || config.mode === 'extends')) { refreshStamps(config) - this.interval = setInterval(async () => refreshStamps, config.refreshPeriod) + this.interval = setInterval(async () => refreshStamps(config), config.refreshPeriod) } else if (config.mode === 'hardcoded') { this.stamp = config.stamp } diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index ebcf6ebf..d1241652 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -159,7 +159,7 @@ describe('postageStamp', () => { expect(manager.postageStamp()).toEqual(stamp.batchID) stamp.utilization = 15 - await sleep(2000) + await sleep(500) expect(db.toArray().length).toEqual(2) expect(manager.postageStamp()).not.toEqual(stamp.batchID) From e6076d7959385360e3a18e624db47509484199d0 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Wed, 16 Nov 2022 23:21:07 -0500 Subject: [PATCH 17/18] fix: several comments addressed and unit test modifications --- src/index.ts | 8 ++++--- src/readiness.ts | 9 +++----- src/server.ts | 4 ++-- src/stamps/base.ts | 6 ++++- src/stamps/extends.ts | 8 +++---- src/stamps/hardcoded.ts | 9 -------- src/stamps/index.ts | 1 - src/utils.ts | 2 +- test/readiness.spec.ts | 4 ++-- test/server.spec.ts | 4 ++-- test/stamps.spec.ts | 51 ++++++++++++++++++++++------------------- 11 files changed, 52 insertions(+), 54 deletions(-) delete mode 100644 src/stamps/hardcoded.ts diff --git a/src/index.ts b/src/index.ts index 8194c64c..e7ef4238 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,11 +2,13 @@ import { Application } from 'express' import { createApp } from './server' -import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' +import { AutoBuyStampsManager, ExtendsStampManager } from './stamps' import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config' import { logger, subscribeLogServerRequests } from './logger' import { ContentManager } from './content' +import type { StampsManager } from './stamps' import { BeeDebug } from '@ethersphere/bee-js' +import { BaseStampManager } from './stamps/base' async function main() { // Configuration @@ -29,11 +31,11 @@ async function main() { if (stampsConfig) { logger.debug('stamps config', stampsConfig) - let stampManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager + let stampManager: StampsManager const { mode } = stampsConfig if (mode === 'hardcoded') { - stampManager = new HardcodedStampsManager(stampsConfig) + stampManager = new BaseStampManager() stampManager.start(stampsConfig) logger.info('starting hardcoded postage stamp manager') } else if (mode === 'autobuy') { diff --git a/src/readiness.ts b/src/readiness.ts index 72459ddd..d147847e 100644 --- a/src/readiness.ts +++ b/src/readiness.ts @@ -1,7 +1,7 @@ import { Bee, BeeDebug, Utils } from '@ethersphere/bee-js' import { ERROR_NO_STAMP, READINESS_TIMEOUT_MS } from './config' import { logger } from './logger' -import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps' +import type { StampsManager } from './stamps' import { getErrorMessage } from './utils' const MAX_CHUNK_SIZE = 4096 @@ -16,7 +16,7 @@ export enum ReadinessStatus { export async function checkReadiness( bee: Bee, beeDebug: BeeDebug, - stampManager?: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager, + stampManager?: StampsManager, ): Promise { if (stampManager) { const ready = await tryUploadingSingleChunk(bee, stampManager) @@ -34,10 +34,7 @@ export async function checkReadiness( } } -async function tryUploadingSingleChunk( - bee: Bee, - stampsManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager, -): Promise { +async function tryUploadingSingleChunk(bee: Bee, stampsManager: StampsManager): Promise { const chunk = makeChunk() try { await bee.uploadChunk(stampsManager.postageStamp(), chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true }) diff --git a/src/server.ts b/src/server.ts index 018ab4ae..f7a84ef8 100644 --- a/src/server.ts +++ b/src/server.ts @@ -8,7 +8,7 @@ import { logger } from './logger' import { register } from './metrics' import { checkReadiness, ReadinessStatus } from './readiness' import { getErrorMessage } from './utils' -import type { BaseStampManager } from './stamps/base' +import type { StampsManager } from './stamps/base' const SWARM_STAMP_HEADER = 'swarm-postage-batch-id' @@ -32,7 +32,7 @@ export const createApp = ( removePinHeader, exposeHashedIdentity, }: AppConfig, - stampManager?: BaseStampManager, + stampManager?: StampsManager, ): Application => { const commonOptions: Options = { target: beeApiUrl, diff --git a/src/stamps/base.ts b/src/stamps/base.ts index 5d01d06e..b1406062 100644 --- a/src/stamps/base.ts +++ b/src/stamps/base.ts @@ -4,7 +4,7 @@ import { logger } from '../logger' import { ERROR_NO_STAMP, StampsConfig } from '../config' export interface StampsManager { - start: (config: StampsConfig, refreshStamps: () => Promise) => Promise + start: (config: StampsConfig, refreshStamps?: () => Promise) => Promise stop: () => void postageStamp: () => string } @@ -14,6 +14,10 @@ export class BaseStampManager { public stamp?: string public usableStamps?: PostageBatch[] + constructor(stamp?: string) { + this.stamp = stamp + } + /** * Get postage stamp that should be replaced in a the proxy request header * diff --git a/src/stamps/extends.ts b/src/stamps/extends.ts index 876890fb..a1360e36 100644 --- a/src/stamps/extends.ts +++ b/src/stamps/extends.ts @@ -14,9 +14,9 @@ import { StampsConfig, StampsConfigExtends } from '../config' */ export function filterUsableStampsExtendsTTL(stamps: PostageBatch[]): PostageBatch[] { const usableStamps = stamps - // filter to get stamps that have the right depth, amount and are not fully used or expired + // filter to get usable stamps .filter(s => s.usable) - // sort the stamps by usage + // sort the stamps by TTL .sort((a, b) => (a.batchTTL > b.batchTTL ? 1 : -1)) // return the all usable stamp sorted by usage @@ -45,14 +45,14 @@ export function filterUsableStampsExtendsCapacity(stamps: PostageBatch[], usageT return [] } -export async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { +async function topUpStamp(beeDebug: BeeDebug, postageBatchId: string, amount: string): Promise { await beeDebug.topUpBatch(postageBatchId, amount) const stamp = await beeDebug.getPostageBatch(postageBatchId) return stamp } -export async function extendsCapacity(extendManager: ExtendsStampManager, beeDebug: BeeDebug, stamp: PostageBatch) { +async function extendsCapacity(extendManager: ExtendsStampManager, beeDebug: BeeDebug, stamp: PostageBatch) { const stampRes = await topUpStamp(beeDebug, stamp.batchID, BigInt(stamp.amount).toString()) setTimeout(() => extendManager.completeTopUp('capacity', stampRes), 60000) await beeDebug.diluteBatch(stamp.batchID, stamp.depth + 1) diff --git a/src/stamps/hardcoded.ts b/src/stamps/hardcoded.ts deleted file mode 100644 index 5c4a4479..00000000 --- a/src/stamps/hardcoded.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { StampsConfigHardcoded } from '../config' -import { BaseStampManager, StampsManager } from './base' - -export class HardcodedStampsManager extends BaseStampManager implements StampsManager { - constructor(config: StampsConfigHardcoded) { - super() - this.start(config) - } -} diff --git a/src/stamps/index.ts b/src/stamps/index.ts index 9035736d..0c422e93 100644 --- a/src/stamps/index.ts +++ b/src/stamps/index.ts @@ -1,4 +1,3 @@ -export { HardcodedStampsManager } from './hardcoded' export { AutoBuyStampsManager } from './autobuy' export { ExtendsStampManager } from './extends' export { StampsManager } from './base' diff --git a/src/utils.ts b/src/utils.ts index 3cbd1a48..a2890325 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -74,7 +74,7 @@ export function assertDecimal(value: unknown, name = 'value'): boolean { value = value.toString() } - if (typeof value !== 'string' || (typeof value === 'string' && !/^[-+]?[0]+\.[0-9]+$/g.test(value))) { + if (typeof value !== 'string' || (typeof value === 'string' && !/^\d*\.?\d*$/g.test(value))) { throw new TypeError(`${name} is not a valid decimal`) } diff --git a/test/readiness.spec.ts b/test/readiness.spec.ts index 162df29a..93ba7114 100644 --- a/test/readiness.spec.ts +++ b/test/readiness.spec.ts @@ -1,6 +1,6 @@ import request from 'supertest' import { createApp } from '../src/server' -import { HardcodedStampsManager } from '../src/stamps' +import { BaseStampManager } from '../src/stamps/base' describe('readiness', () => { test('should be ready without stamp management', async () => { @@ -13,7 +13,7 @@ describe('readiness', () => { }) test('should be ready with stamp management', async () => { - const stampManager = new HardcodedStampsManager({ mode: 'hardcoded', stamp: process.env.BEE_POSTAGE as string }) + const stampManager = new BaseStampManager(process.env.BEE_POSTAGE as string) const app = createApp( { beeApiUrl: 'http://localhost:1633', diff --git a/test/server.spec.ts b/test/server.spec.ts index 13cf0ac5..6306b6a9 100644 --- a/test/server.spec.ts +++ b/test/server.spec.ts @@ -5,8 +5,8 @@ import type { Server } from 'http' import { DEFAULT_BEE_DEBUG_API_URL } from '../src/config' import { bee, getPostageBatch, makeCollectionFromFS } from './utils' -import { HardcodedStampsManager } from '../src/stamps' import { createHeaderCheckMockServer } from './header-check.mockserver' +import { BaseStampManager } from '../src/stamps/base' const beeApiUrl = process.env.BEE_API_URL || 'http://localhost:1633' const beeApiUrlWrong = process.env.BEE_API_URL_WRONG || 'http://localhost:2021' @@ -36,7 +36,7 @@ beforeAll(async () => { beeProxy = new Bee(`http://localhost:${port}`) const stamp = getPostageBatch() - const stampManager = new HardcodedStampsManager({ mode: 'hardcoded', stamp }) + const stampManager = new BaseStampManager(stamp) const appWithStamp = createApp({ beeApiUrl, beeDebugApiUrl: DEFAULT_BEE_DEBUG_API_URL }, stampManager) proxyWithStamp = await new Promise((resolve, _reject) => { const server = appWithStamp.listen(async () => resolve(server)) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index d1241652..c4781b9d 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -1,25 +1,16 @@ import type { Server } from 'http' import { BeeDebug, BatchId, PostageBatch } from '@ethersphere/bee-js' import { buyNewStamp, getUsage, sleep } from '../src/utils' -import { - getStampsConfig, - StampsConfig, - StampsConfigAutobuy, - StampsConfigExtends, - StampsConfigHardcoded, -} from '../src/config' +import { getStampsConfig, StampsConfig, StampsConfigAutobuy, StampsConfigExtends } from '../src/config' import { createStampMockServer, StampDB } from './stamps.mockserver' import { genRandomHex } from './utils' import { - extendsCapacity, ExtendsStampManager, filterUsableStampsExtendsCapacity, filterUsableStampsExtendsTTL, - topUpStamp, } from '../src/stamps/extends' import { AutoBuyStampsManager, filterUsableStampsAutobuy } from '../src/stamps/autobuy' -import { HardcodedStampsManager } from '../src/stamps' - +import { BaseStampManager } from '../src/stamps/base' interface AddressInfo { address: string family: string @@ -74,8 +65,7 @@ const buildStamp = (overwrites: Partial) => { describe('postageStamp', () => { it('should return correct hardcoded single postage stamp', async () => { const stamp = '0000000000000000000000000000000000000000000000000000000000000000' - const config = getStampsConfig({ POSTAGE_STAMP: stamp })! as StampsConfigHardcoded - const stampManager = new HardcodedStampsManager(config) + const stampManager = new BaseStampManager(stamp) expect(stampManager.postageStamp()).toEqual(stamp) }) @@ -274,12 +264,17 @@ describe('extendsStampsTTL', () => { }) it('should extend stamp ttl and await for it to extend others', async () => { - const beeDebug = new BeeDebug(url) - const stampId = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) + const stamp = buildStamp({ amount: defaultAmount, utilization: 10, batchTTL: 120_000, usable: true }) + db.add(stamp) + + const extendManager = new ExtendsStampManager(new BeeDebug(url)) + extendManager.start(stampsConfig, async () => (extendManager as ExtendsStampManager).refreshStamps(stampsConfig)) const extendAmount = '100' - await topUpStamp(beeDebug, stampId.batchId, extendAmount) - const stampExtended = await beeDebug.getPostageBatch(stampId.batchId) + stamp.amount += extendAmount + await sleep(500) + + const [stampExtended] = db.toArray() expect(Number(stampExtended.amount)).toBeGreaterThan(Number(defaultAmount)) }) }) @@ -292,6 +287,7 @@ describe('extendsStampsCapacity', () => { POSTAGE_USAGE_THRESHOLD: defaultUsageThreshold.toString(), POSTAGE_EXTENDS_CAPACITY: 'true', BEE_DEBUG_API_URL: url, + POSTAGE_REFRESH_PERIOD: '100', }) as StampsConfig }) it('should not find any usable stamps', async () => { @@ -305,15 +301,24 @@ describe('extendsStampsCapacity', () => { }) it('should extend stamps capacity', async () => { - const beeDebug = new BeeDebug(url) - const extendManager = new ExtendsStampManager(beeDebug) + const stamp = buildStamp({ + amount: defaultAmount, + utilization: 10, + batchTTL: 120_000, + depth: defaultDepth, + usable: true, + bucketDepth: defaultDepth - 1, + }) + db.add(stamp) + const extendManager = new ExtendsStampManager(new BeeDebug(url)) extendManager.start(stampsConfig, async () => (extendManager as ExtendsStampManager).refreshStamps(stampsConfig)) - const stampId = await buyNewStamp(defaultDepth, defaultAmount, beeDebug) + await sleep(200) - await extendsCapacity(extendManager, beeDebug, stampId.stamp) - await sleep(1_000) - const stampExtended = await beeDebug.getPostageBatch(stampId.batchId) + stamp.utilization += 20 + await sleep(1000) + + const [stampExtended] = db.toArray() expect(stampExtended.depth).toBeGreaterThan(defaultDepth) extendManager.stop() await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress From 5219318c9cca7762ac14b01ae8d9179284f81cd2 Mon Sep 17 00:00:00 2001 From: Luis Ricardo Sanchez Date: Thu, 17 Nov 2022 09:58:23 -0500 Subject: [PATCH 18/18] fix: include .stop on unit test --- test/stamps.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/stamps.spec.ts b/test/stamps.spec.ts index c4781b9d..f524f64c 100644 --- a/test/stamps.spec.ts +++ b/test/stamps.spec.ts @@ -276,6 +276,8 @@ describe('extendsStampsTTL', () => { const [stampExtended] = db.toArray() expect(Number(stampExtended.amount)).toBeGreaterThan(Number(defaultAmount)) + extendManager.stop() + await sleep(250) // Needed as there could be the wait for posage stamp usable process in progress }) })