Merge pull request #152 from ty-everett/codex/overlay-monitor-arc-callback

Add Overlay Express monitor and ARC callbacks

Author: sirdeggen

infra/overlay-server/package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@bsv/overlay": "^2.0.3",
     "@bsv/overlay-discovery-services": "^2.0.3",
-    "@bsv/overlay-express": "^2.2.1",
+    "@bsv/overlay-express": "^2.3.0",
     "@bsv/overlay-topics": "^1.0.1",
     "@bsv/sdk": "^2.1.3",
     "dotenv": "^17.2.4",

packages/overlays/overlay-express/CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file. The format
 ## Table of Contents
 
 - [Unreleased](#unreleased)
+- [2.3.0](#230-2026-05-28)
 - [0.7.11](#0711-2025-08-13)
 
 ## [Unreleased]
@@ -29,6 +30,18 @@ All notable changes to this project will be documented in this file. The format
 
 ---
 
+## [2.3.0] - 2026-05-28
+
+### Added
+- Added `OverlayMonitor`, a reusable worker for probing Overlay Express `/lookup`
+  responses and reporting BEEF proof-shape bloat.
+
+### Fixed
+- ARC broadcasts now include an explicit `/arc-ingest` callback URL, with optional
+  callback-token validation for deployments that configure one.
+
+---
+
 ## [0.7.11] - 2025-08-13
 
 ### Added
@@ -62,4 +75,4 @@ Replace `X.X.X` with the new version number and `YYYY-MM-DD` with the release da
 - 
 ```
 
-Use this template as the starting point for each new version. Always update the "Unreleased" section with changes as they're implemented, and then move them under the new version header when that version is released.
+Use this template as the starting point for each new version. Always update the "Unreleased" section with changes as they're implemented, and then move them under the new version header when that version is released.

packages/overlays/overlay-express/README.md

@@ -116,6 +116,39 @@ server.registerHealthCheck({
 
 The janitor service also understands the richer `/health` response format, so existing SHIP/SLAP health validation remains compatible.
 
+### Overlay Monitor
+
+`OverlayMonitor` provides a reusable worker for monitoring Overlay Express lookup behavior. It posts configured `/lookup` probes to any Overlay Express deployment, measures response size, parses returned BEEF, and reports whether responsive output transactions have direct Merkle proofs or are being served with deeper proof ancestry.
+
+This is intended to be run by deployments as a long-running monitor process or by cluster scheduling. It is not tied to a specific deployment platform.
+
+```typescript
+import { OverlayMonitor } from '@bsv/overlay-express'
+
+const monitor = new OverlayMonitor({
+  intervalMs: 24 * 60 * 60 * 1000,
+  targets: [
+    {
+      name: 'example-overlay',
+      baseUrl: 'https://overlay.example',
+      probes: [
+        {
+          name: 'example-topic',
+          service: 'ls_example',
+          query: { topic: 'tm_example' },
+          maxOutputs: 20
+        }
+      ]
+    }
+  ],
+  onReport: async report => {
+    console.log(JSON.stringify(report.summary))
+  }
+})
+
+monitor.start()
+```
+
 ## License
 
 The license for the code in this repository is the Open BSV License. Refer to [LICENSE.txt](./LICENSE.txt) for the license text.

packages/overlays/overlay-express/mod.ts

@@ -11,3 +11,16 @@ export {
 export { BanService, type BannedRecord } from './src/BanService.js'
 export { BanAwareLookupWrapper } from './src/BanAwareLookupWrapper.js'
 export { JanitorService, type JanitorConfig, type JanitorReport, type HostHealthResult } from './src/JanitorService.js'
+export {
+  OverlayMonitor,
+  analyzeOverlayLookupResponse,
+  type OverlayLookupOutputSummary,
+  type OverlayLookupProbe,
+  type OverlayLookupProbeResult,
+  type OverlayMonitorConfig,
+  type OverlayMonitorLogger,
+  type OverlayMonitorReport,
+  type OverlayMonitorTarget,
+  type OverlayMonitorThresholds,
+  type OverlayMonitorWarning
+} from './src/OverlayMonitor.js'

packages/overlays/overlay-express/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bsv/overlay-express",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "type": "module",
   "description": "BSV Blockchain Overlay Express",
   "main": "dist/cjs/mod.js",

packages/overlays/overlay-express/src/OverlayExpress.ts

@@ -189,6 +189,9 @@ export default class OverlayExpress {
   // ARC API Key
   arcApiKey: string | undefined = undefined
 
+  // Optional ARC callback token for /arc-ingest notifications
+  arcCallbackToken: string | undefined = undefined
+
   // Verbose request logging
   verboseRequestLogging: boolean = false
 
@@ -372,6 +375,15 @@ export default class OverlayExpress {
     this.logger.log(chalk.blue('ARC API key has been configured.'))
   }
 
+  /**
+   * Configures the ARC callback token expected by /arc-ingest.
+   * @param token - The token ARC should present when posting callback notifications.
+   */
+  configureArcCallbackToken (token: string): void {
+    this.arcCallbackToken = token
+    this.logger.log(chalk.blue('ARC callback token has been configured.'))
+  }
+
   /**
    * Enables or disables GASP synchronization (high-level setting).
    * This is a broad toggle that can be overridden or customized through syncConfiguration.
@@ -581,7 +593,11 @@ export default class OverlayExpress {
   private buildBroadcaster (): Broadcaster | undefined {
     if (typeof this.arcApiKey !== 'string') return undefined
     const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com'
-    return new ARC(arcUrl, { apiKey: this.arcApiKey })
+    return new ARC(arcUrl, {
+      apiKey: this.arcApiKey,
+      callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
+      callbackToken: this.arcCallbackToken
+    })
   }
 
   /** Resolve the SLAP trackers from config or network defaults. */
@@ -1253,6 +1269,18 @@ export default class OverlayExpress {
       this.app.post('/arc-ingest', (req, res) => {
         ; (async () => {
           try {
+            if (typeof this.arcCallbackToken === 'string' && this.arcCallbackToken.length > 0) {
+              const authorization = req.headers.authorization
+              const headerToken = Array.isArray(authorization) ? authorization[0] : authorization
+              const xCallbackToken = req.headers['x-callback-token']
+              const callbackToken = Array.isArray(xCallbackToken) ? xCallbackToken[0] : xCallbackToken
+              const bearerToken = typeof headerToken === 'string' && headerToken.startsWith('Bearer ')
+                ? headerToken.slice('Bearer '.length)
+                : headerToken
+              if (bearerToken !== this.arcCallbackToken && callbackToken !== this.arcCallbackToken) {
+                return res.status(401).json({ status: 'error', message: 'Unauthorized callback' })
+              }
+            }
             const { txid, merklePath: merklePathHex, blockHeight } = req.body
             const merklePath = MerklePath.fromHex(merklePathHex)
             await engine.handleNewMerkleProof(txid, merklePath, blockHeight)