Feature/extends capacity refactor

README.md

@@ -30,6 +30,7 @@ newer Bee versions is not recommended and may not work. Stay up to date by joini
     - [2. Hardcoded postage stamp](#2-hardcoded-postage-stamp)
     - [3. Autobuy postage stamps](#3-autobuy-postage-stamps)
     - [4. Extends stamps TTL](#4-extends-stamps-ttl)
+    - [5. Extends stamps capacity](#5-extends-stamps-capacity)
     - [Enable authentication](#enable-authentication)
   - [Environment variables](#environment-variables)
   - [Curl](#curl)
@@ -56,6 +57,9 @@ The proxy can manage postage stamps for you in 4 modes of operation:
 4. It can extend the TTL of a stamp that is about to expire. To enable this, set `POSTAGE_EXTENDSTTL=true`,
    provide `POSTAGE_AMOUNT`, `POSTAGE_DEPTH` with the desired amount to use and `POSTAGE_TTL_MIN` above with
    a number above or equal to 60.
+5. It can extends the postage stamp capacity to those that are about to be fulfill. To enable this, set
+   `POSTAGE_EXTENDS_CAPACITY=true`. You can also set the env variable `POSTAGE_USAGE_THRESHOLD=0.7` to determine
+   the maximum usage level to check if the stamp needs to be extended
 
 In modes 1, 2 and 3, the proxy can be configured to require authentication secret to forward the requests. Use the
 `AUTH_SECRET` environment variable to enable it.
@@ -112,6 +116,14 @@ export BEE_DEBUG_API_URL=http://localhost:1635
 npm run start
 ```
 
+#### 5. Extends stamps capacity
+
+```sh
+export POSTAGE_EXTENDS_CAPACITY=true
+
+npm run start
+```
+
 #### Reupload pinned content
 
 ```sh
@@ -149,6 +161,8 @@ npm run start
 | REMOVE_PIN_HEADER       | true                        | Removes swarm-pin header on all proxy requests.                                                            |
 | `LOG_LEVEL`             | info                        | Log level that is outputted (values: `critical`, `error`, `warn`, `info`, `verbose`, `debug`)              |
 | POSTAGE_EXTENDSTTL      | false                       | Enables extends TTL feature. Works along with POSTAGE_AMOUNT                                               |
+| POSTAGE_EXTENDS_CAPACITY | false                       | Enables extending stamp capacity
+feature.                                                |
 | EXPOSE_HASHED_IDENTITY  | false                       | Exposes `x-bee-node` header, which is the hashed identity of the Bee node for identification purposes      |
 | REUPLOAD_PERIOD         | undefined                   | How frequently are the pinned content checked to be reuploaded.                                            |
 

src/config.ts

@@ -1,3 +1,5 @@
+import { isInteger, isBoolean } from './utils'
+
 export interface AppConfig {
   beeApiUrl: string
   beeDebugApiUrl: string
@@ -18,28 +20,32 @@ interface StampsConfigHardcoded {
   mode: 'hardcoded'
   stamp: string
 }
-export interface StampsConfigExtends {
-  mode: 'extendsTTL'
+
+export interface StampsConfigAutobuy {
+  mode: 'autobuy'
   ttlMin: number
   depth: number
   amount: string
+  usageThreshold: number
   refreshPeriod: number
+  usageMax: number
   beeDebugApiUrl: string
 }
 
-export interface ContentConfigReupload {
-  beeApiUrl: string
-  refreshPeriod: number
-}
-
-export interface StampsConfigAutobuy {
-  mode: 'autobuy'
+export interface StampsConfigExtends {
+  mode: 'extends'
+  enableTtl: boolean
+  enableCapacity: boolean
+  ttlMin: number
   depth: number
   amount: string
-  beeDebugApiUrl: string
   usageThreshold: number
-  usageMax: number
-  ttlMin: number
+  refreshPeriod: number
+  beeDebugApiUrl: string
+}
+
+export interface ContentConfigReupload {
+  beeApiUrl: string
   refreshPeriod: number
 }
 
@@ -80,6 +86,7 @@ export type EnvironmentVariables = Partial<{
   POSTAGE_REFRESH_PERIOD: string
   POSTAGE_EXTENDSTTL: string
   REUPLOAD_PERIOD: string
+  POSTAGE_EXTENDS_CAPACITY: string
 }>
 
 export const SUPPORTED_LEVELS = ['critical', 'error', 'warn', 'info', 'verbose', 'debug'] as const
@@ -138,14 +145,29 @@ export function getStampsConfig({
   POSTAGE_TTL_MIN,
   POSTAGE_REFRESH_PERIOD,
   POSTAGE_EXTENDSTTL,
+  POSTAGE_EXTENDS_CAPACITY,
 }: EnvironmentVariables = {}): StampsConfig | undefined {
   const refreshPeriod = Number(POSTAGE_REFRESH_PERIOD || DEFAULT_POSTAGE_REFRESH_PERIOD)
   const beeDebugApiUrl = BEE_DEBUG_API_URL || DEFAULT_BEE_DEBUG_API_URL
 
   // Start in hardcoded mode
   if (POSTAGE_STAMP) return { mode: 'hardcoded', stamp: POSTAGE_STAMP }
   // Start autobuy
-  else if (!POSTAGE_EXTENDSTTL && POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) {
+  else if (
+    (isBoolean(POSTAGE_EXTENDSTTL) && POSTAGE_EXTENDSTTL === 'true') ||
+    (isBoolean(POSTAGE_EXTENDS_CAPACITY) && POSTAGE_EXTENDS_CAPACITY === 'true')
+  ) {
+    return createExtendsStampsConfig(
+      POSTAGE_EXTENDSTTL,
+      POSTAGE_EXTENDS_CAPACITY,
+      POSTAGE_AMOUNT,
+      POSTAGE_USAGE_THRESHOLD,
+      POSTAGE_TTL_MIN,
+      POSTAGE_DEPTH,
+      refreshPeriod,
+      beeDebugApiUrl,
+    )
+  } else if (POSTAGE_DEPTH && POSTAGE_AMOUNT && BEE_DEBUG_API_URL) {
     return {
       mode: 'autobuy',
       depth: Number(POSTAGE_DEPTH),
@@ -156,20 +178,6 @@ export function getStampsConfig({
       refreshPeriod,
       beeDebugApiUrl,
     }
-  } else if (
-    POSTAGE_EXTENDSTTL === 'true' &&
-    POSTAGE_AMOUNT &&
-    POSTAGE_DEPTH &&
-    Number(POSTAGE_TTL_MIN) >= MINIMAL_EXTENDS_TTL_VALUE
-  ) {
-    return {
-      mode: 'extendsTTL',
-      depth: Number(POSTAGE_DEPTH),
-      ttlMin: Number(POSTAGE_TTL_MIN),
-      amount: POSTAGE_AMOUNT,
-      refreshPeriod,
-      beeDebugApiUrl,
-    }
   }
   // Missing one of the variables needed for the autobuy or extends TTL
   else if (POSTAGE_DEPTH || POSTAGE_AMOUNT || POSTAGE_TTL_MIN || BEE_DEBUG_API_URL) {
@@ -184,6 +192,55 @@ export function getStampsConfig({
   return undefined
 }
 
+export function createExtendsStampsConfig(
+  POSTAGE_EXTENDSTTL: string | undefined,
+  POSTAGE_EXTENDS_CAPACITY: string | undefined,
+  POSTAGE_AMOUNT: string | undefined,
+  POSTAGE_USAGE_THRESHOLD: string | undefined,
+  POSTAGE_TTL_MIN: string | undefined,
+  POSTAGE_DEPTH: string | undefined,
+  refreshPeriod: number,
+  beeDebugApiUrl: string,
+): StampsConfigExtends {
+  if (
+    POSTAGE_EXTENDSTTL === 'true' &&
+    (Number(POSTAGE_TTL_MIN) < MINIMAL_EXTENDS_TTL_VALUE ||
+      !isInteger(POSTAGE_AMOUNT) ||
+      (POSTAGE_TTL_MIN && !isInteger(POSTAGE_TTL_MIN)) ||
+      (POSTAGE_DEPTH && !isInteger(POSTAGE_DEPTH)))
+  ) {
+    throw new Error(
+      `config: to extends stamps TTL please provide POSTAGE_TTL_MIN bigger than ${MINIMAL_EXTENDS_TTL_VALUE}, valid values for
+      POSTAGE_AMOUNT, POSTAGE_TTL_MIN, POSTAGE_DEPTH. Current states are
+      POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN}, POSTAGE_AMOUNT=${POSTAGE_AMOUNT}, POSTAGE_TTL_MIN=${POSTAGE_TTL_MIN} and POSTAGE_DEPTH=${POSTAGE_DEPTH}`,
+    )
+  }
+
+  if (POSTAGE_EXTENDS_CAPACITY === 'true' && POSTAGE_USAGE_THRESHOLD && !isInteger(POSTAGE_USAGE_THRESHOLD)) {
+    throw new Error(
+      `config: to extends capacity please provide valid number for POSTAGE_USAGE_THRESHOLD. Current states is
+      POSTAGE_USAGE_THRESHOLD=${POSTAGE_USAGE_THRESHOLD}`,
+    )
+  }
+
+  const amount = POSTAGE_AMOUNT || '0'
+  const usageThreshold = Number(POSTAGE_USAGE_THRESHOLD || DEFAULT_POSTAGE_USAGE_THRESHOLD)
+  const ttlMin = Number(POSTAGE_TTL_MIN)
+  const depth = Number(POSTAGE_DEPTH)
+
+  return {
+    mode: 'extends',
+    enableTtl: POSTAGE_EXTENDSTTL === 'true',
+    enableCapacity: POSTAGE_EXTENDS_CAPACITY === 'true',
+    depth,
+    ttlMin,
+    amount,
+    usageThreshold,
+    refreshPeriod,
+    beeDebugApiUrl,
+  }
+}
+
 export function getContentConfig({ BEE_API_URL, REUPLOAD_PERIOD }: EnvironmentVariables = {}): ContentConfig | false {
   if (!REUPLOAD_PERIOD) {
     return false

src/content.ts

@@ -14,48 +14,45 @@ export class ContentManager {
   private interval?: ReturnType<typeof setInterval>
   private isReuploading = false
 
-  public async attemptRefreshContentReupload(beeApi: Bee): Promise<void> {
-    try {
-      await this.refreshContentReupload(beeApi)
-    } catch (error) {
-      logger.error('content reupload job failed', error)
-    }
-  }
-
   public async refreshContentReupload(beeApi: Bee): Promise<void> {
-    const pins = await beeApi.getAllPins()
+    try {
+      const pins = await beeApi.getAllPins()
 
-    if (!pins.length) {
-      logger.info(`no pins found`)
+      if (!pins.length) {
+        logger.info(`no pins found`)
 
-      return
-    }
+        return
+      }
 
-    logger.info(`checking pinned content (${pins.length} pins)`)
-    for (const pin of pins) {
-      const isRetrievable = await beeApi.isReferenceRetrievable(pin)
-      logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
+      logger.info(`checking pinned content (${pins.length} pins)`)
+      for (const pin of pins) {
+        const isRetrievable = await beeApi.isReferenceRetrievable(pin)
+        logger.debug(`pin ${pin} is ${isRetrievable ? 'retrievable' : 'not retrievable'}`)
 
-      if (!isRetrievable && !this.isReuploading) {
-        this.isReuploading = true
-        try {
-          logger.debug(`reuploading pinned content: ${pin}`)
-          await beeApi.reuploadPinnedData(pin)
-          contentReuploadCounter.inc()
-          logger.info(`pinned content reuploaded: ${pin}`)
-        } catch (error) {
-          logger.error('failed to reupload pinned content', error)
+        if (!isRetrievable && !this.isReuploading) {
+          this.isReuploading = true
+          try {
+            logger.debug(`reuploading pinned content: ${pin}`)
+            await beeApi.reuploadPinnedData(pin)
+            contentReuploadCounter.inc()
+            logger.info(`pinned content reuploaded: ${pin}`)
+          } catch (error) {
+            logger.error('failed to reupload pinned content', error)
+          }
+          this.isReuploading = false
         }
-        this.isReuploading = false
       }
+    } catch (error) {
+      logger.error('content reupload job failed', error)
     }
   }
 
   /**
    * Start the manager that checks for pinned content availability and reuploads the data if needed.
    */
   start(config: ContentConfig): void {
-    const refreshContent = async () => this.attemptRefreshContentReupload(new Bee(config.beeApiUrl))
+    const bee = new Bee(config.beeApiUrl)
+    const refreshContent = async () => this.refreshContentReupload(bee)
     this.stop()
     refreshContent()
 

src/index.ts

@@ -2,7 +2,7 @@
 import { Application } from 'express'
 
 import { createApp } from './server'
-import { StampsManager } from './stamps'
+import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps'
 import { getAppConfig, getServerConfig, getStampsConfig, EnvironmentVariables, getContentConfig } from './config'
 import { logger, subscribeLogServerRequests } from './logger'
 import { ContentManager } from './content'
@@ -28,9 +28,21 @@ async function main() {
 
   if (stampsConfig) {
     logger.debug('stamps config', stampsConfig)
-    const stampManager = new StampsManager()
-    logger.info('starting postage stamp manager')
-    stampManager.start(stampsConfig)
+    let stampManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager
+    const { mode } = stampsConfig
+
+    if (mode === 'hardcoded') {
+      stampManager = new HardcodedStampsManager()
+
+      logger.info('starting hardcoded postage stamp manager')
+      stampManager.start(stampsConfig)
+    } else if (mode === 'autobuy') {
+      logger.info('starting autobuy postage stamp manager')
+      stampManager = new AutoBuyStampsManager(stampsConfig)
+    } else {
+      logger.info('starting extends postage stamp manager')
+      stampManager = new ExtendsStampManager(stampsConfig)
+    }
     logger.info('starting the proxy')
     app = createApp(appConfig, stampManager)
   } else {

src/readiness.ts

@@ -1,7 +1,7 @@
 import { Bee, BeeDebug, Utils } from '@ethersphere/bee-js'
 import { ERROR_NO_STAMP, READINESS_TIMEOUT_MS } from './config'
 import { logger } from './logger'
-import { StampsManager } from './stamps'
+import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps'
 import { getErrorMessage } from './utils'
 
 const MAX_CHUNK_SIZE = 4096
@@ -16,7 +16,7 @@ export enum ReadinessStatus {
 export async function checkReadiness(
   bee: Bee,
   beeDebug: BeeDebug,
-  stampManager?: StampsManager,
+  stampManager?: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager,
 ): Promise<ReadinessStatus> {
   if (stampManager) {
     const ready = await tryUploadingSingleChunk(bee, stampManager)
@@ -34,10 +34,13 @@ export async function checkReadiness(
   }
 }
 
-async function tryUploadingSingleChunk(bee: Bee, stampsManager: StampsManager): Promise<ReadinessStatus> {
+async function tryUploadingSingleChunk(
+  bee: Bee,
+  stampsManager: HardcodedStampsManager | AutoBuyStampsManager | ExtendsStampManager,
+): Promise<ReadinessStatus> {
   const chunk = makeChunk()
   try {
-    await bee.uploadChunk(stampsManager.postageStamp, chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true })
+    await bee.uploadChunk(stampsManager.postageStamp(), chunk, { timeout: READINESS_TIMEOUT_MS, deferred: true })
 
     return ReadinessStatus.OK
   } catch (error) {

src/server.ts

@@ -7,7 +7,7 @@ import { fetchBeeIdentity, getHashedIdentity, HASHED_IDENTITY_HEADER } from './i
 import { logger } from './logger'
 import { register } from './metrics'
 import { checkReadiness, ReadinessStatus } from './readiness'
-import type { StampsManager } from './stamps'
+import { AutoBuyStampsManager, ExtendsStampManager, HardcodedStampsManager } from './stamps'
 import { getErrorMessage } from './utils'
 
 const SWARM_STAMP_HEADER = 'swarm-postage-batch-id'
@@ -32,7 +32,7 @@ export const createApp = (
     removePinHeader,
     exposeHashedIdentity,
   }: AppConfig,
-  stampManager?: StampsManager,
+  stampManager?: AutoBuyStampsManager | ExtendsStampManager | HardcodedStampsManager,
 ): Application => {
   const commonOptions: Options = {
     target: beeApiUrl,
@@ -136,7 +136,7 @@ export const createApp = (
     if (stampManager) {
       proxyReq.removeHeader(SWARM_STAMP_HEADER)
       try {
-        proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp)
+        proxyReq.setHeader(SWARM_STAMP_HEADER, stampManager.postageStamp())
       } catch (error) {
         logger.error('proxy failure', error)