Wait for soroban ledger ingestion at chain head instead of failing fatally on -32600

ahmdssi · ahmdssi · commit 64f7af711d83 · 2026-06-12T18:52:06.000+02:00
diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Crash at the chain head when the soroban endpoint has not yet ingested the target ledger: JSON-RPC -32600 `startLedger must be within the ledger range` is now treated as a transient condition and the fetch waits for ingestion (bounded by the new `sorobanIngestWaitSeconds` endpoint config, default 600s) instead of failing fatally after retries
 
 ## [6.2.0] - 2026-01-21
 ### Changed
diff --git a/packages/node/src/stellar/api.stellar.spec.ts b/packages/node/src/stellar/api.stellar.spec.ts
@@ -1,9 +1,15 @@
 // Copyright 2020-2025 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import { delay } from '@subql/node-core';
 import { StellarApi } from './api.stellar';
 import { SorobanServer } from './soroban.server';
 
+jest.mock('@subql/node-core', () => ({
+  ...jest.requireActual('@subql/node-core'),
+  delay: jest.fn(() => Promise.resolve()),
+}));
+
 const HTTP_ENDPOINT = 'https://horizon-futurenet.stellar.org';
 const SOROBAN_ENDPOINT = 'https://rpc-futurenet.stellar.org';
 
@@ -129,3 +135,217 @@ describe('StellarApi', () => {
     expect(tx?.operations[3].events.length).toEqual(0);
   });
 });
+
+describe('StellarApi soroban ingestion lag', () => {
+  const mockedDelay = delay as unknown as jest.Mock;
+
+  const rangeError = (lo: number, hi: number) => ({
+    code: -32600,
+    message: `startLedger must be within the ledger range: ${lo} - ${hi}`,
+  });
+
+  const makeApi = (soroban: any, waitSeconds?: number) =>
+    new StellarApi(HTTP_ENDPOINT, soroban as SorobanServer, {
+      sorobanIngestWaitSeconds: waitSeconds,
+    });
+
+  beforeEach(() => {
+    mockedDelay.mockClear();
+  });
+
+  it('waits for the soroban endpoint to ingest the ledger then recovers', async () => {
+    let calls = 0;
+    const soroban = {
+      getEvents: jest.fn(({ startLedger }: { startLedger: number }) => {
+        calls++;
+        if (calls <= 2) return Promise.reject(rangeError(100, startLedger - 1));
+        return Promise.resolve({
+          events: [
+            { ledger: startLedger, id: 'e1', operationIndex: 0, txHash: 't1' },
+          ],
+          latestLedger: startLedger,
+        });
+      }),
+    };
+    const api = makeApi(soroban);
+
+    const events = await (api as any).getEventsWhenIngested(200);
+
+    expect(soroban.getEvents).toHaveBeenCalledTimes(3);
+    expect(mockedDelay).toHaveBeenCalledTimes(2);
+    expect(events).toHaveLength(1);
+    expect(events[0].id).toEqual('e1');
+  });
+
+  it('rethrows immediately when the ledger is below the retention window', async () => {
+    const soroban = {
+      getEvents: jest.fn(() => Promise.reject(rangeError(1000, 2000))),
+      getLatestLedger: jest.fn(() => Promise.resolve({ sequence: 2000 })),
+    };
+    const api = makeApi(soroban);
+
+    await expect((api as any).getEventsWhenIngested(500)).rejects.toMatchObject(
+      {
+        code: -32600,
+      },
+    );
+    expect(soroban.getEvents).toHaveBeenCalledTimes(1);
+    expect(mockedDelay).not.toHaveBeenCalled();
+  });
+
+  it('keeps the explanatory error for the legacy oldest-ledger message', async () => {
+    const soroban = {
+      getEvents: jest.fn(() =>
+        Promise.reject(new Error('start is before oldest ledger')),
+      ),
+    };
+    const api = makeApi(soroban);
+
+    await expect((api as any).getEventsWhenIngested(500)).rejects.toThrow(
+      'older than the oldest ledger',
+    );
+    expect(mockedDelay).not.toHaveBeenCalled();
+  });
+
+  it('treats the legacy after-newest-ledger message as transient', async () => {
+    let calls = 0;
+    const soroban = {
+      getEvents: jest.fn(({ startLedger }: { startLedger: number }) => {
+        calls++;
+        if (calls === 1) {
+          return Promise.reject(new Error('start is after newest ledger'));
+        }
+        return Promise.resolve({ events: [], latestLedger: startLedger });
+      }),
+    };
+    const api = makeApi(soroban);
+
+    const events = await (api as any).getEventsWhenIngested(200);
+
+    expect(events).toEqual([]);
+    expect(soroban.getEvents).toHaveBeenCalledTimes(2);
+  });
+
+  it('falls back to getLatestLedger for a -32600 with unknown wording', async () => {
+    let calls = 0;
+    const soroban = {
+      getEvents: jest.fn(({ startLedger }: { startLedger: number }) => {
+        calls++;
+        if (calls === 1) {
+          return Promise.reject({
+            code: -32600,
+            message: `startLedger ${startLedger} exceeds latest ledger`,
+          });
+        }
+        return Promise.resolve({ events: [], latestLedger: startLedger });
+      }),
+      getLatestLedger: jest.fn(() => Promise.resolve({ sequence: 199 })),
+    };
+    const api = makeApi(soroban);
+
+    const events = await (api as any).getEventsWhenIngested(200);
+
+    expect(events).toEqual([]);
+    expect(soroban.getEvents).toHaveBeenCalledTimes(2);
+    expect(soroban.getLatestLedger).toHaveBeenCalled();
+  });
+
+  it('rethrows a genuine -32600 when the soroban head is ahead of the sequence', async () => {
+    const soroban = {
+      getEvents: jest.fn(() =>
+        Promise.reject({ code: -32600, message: 'some other invalid request' }),
+      ),
+      getLatestLedger: jest.fn(() => Promise.resolve({ sequence: 500 })),
+    };
+    const api = makeApi(soroban);
+
+    await expect((api as any).getEventsWhenIngested(200)).rejects.toMatchObject(
+      {
+        message: 'some other invalid request',
+      },
+    );
+    expect(soroban.getEvents).toHaveBeenCalledTimes(1);
+    expect(mockedDelay).not.toHaveBeenCalled();
+  });
+
+  it('rethrows once the wait deadline is exhausted', async () => {
+    const soroban = {
+      getEvents: jest.fn(({ startLedger }: { startLedger: number }) =>
+        Promise.reject(rangeError(100, startLedger - 1)),
+      ),
+    };
+    const api = makeApi(soroban, 0);
+
+    await expect((api as any).getEventsWhenIngested(200)).rejects.toMatchObject(
+      {
+        code: -32600,
+      },
+    );
+    expect(soroban.getEvents).toHaveBeenCalledTimes(1);
+    expect(mockedDelay).not.toHaveBeenCalled();
+  });
+
+  it('wires the wait loop into fetchAndWrapLedger', async () => {
+    let calls = 0;
+    const soroban = {
+      getEvents: jest.fn(({ startLedger }: { startLedger: number }) => {
+        calls++;
+        if (calls === 1) {
+          return Promise.reject(rangeError(100, startLedger - 1));
+        }
+        return Promise.resolve({
+          events: [
+            { ledger: startLedger, id: 'e9', operationIndex: 0, txHash: 't1' },
+          ],
+          latestLedger: startLedger,
+        });
+      }),
+    };
+    const api = makeApi(soroban);
+
+    const emptyPage: any = {
+      records: [],
+      next: () => Promise.resolve(emptyPage),
+    };
+    (api as any).stellarClient = {
+      ledgers: () => ({
+        ledger: () => ({
+          call: () => Promise.resolve({ sequence: 300, hash: 'abc' }),
+        }),
+      }),
+      transactions: () => ({
+        forLedger: () => ({
+          limit: () => ({ call: () => Promise.resolve(emptyPage) }),
+        }),
+      }),
+      operations: () => ({
+        forLedger: () => ({
+          limit: () => ({
+            call: () =>
+              Promise.resolve({
+                records: [
+                  {
+                    type: 'invoke_host_function',
+                    id: '1',
+                    transaction_hash: 't1',
+                  },
+                ],
+                next: () => Promise.resolve(emptyPage),
+              }),
+          }),
+        }),
+      }),
+      effects: () => ({
+        forLedger: () => ({
+          limit: () => ({ call: () => Promise.resolve(emptyPage) }),
+        }),
+      }),
+    };
+
+    const block = await (api as any).fetchAndWrapLedger(300);
+
+    expect(soroban.getEvents).toHaveBeenCalledTimes(2);
+    expect(block.block.events).toHaveLength(1);
+    expect(block.block.events[0].id).toEqual('e9');
+  });
+});
diff --git a/packages/node/src/stellar/api.stellar.ts b/packages/node/src/stellar/api.stellar.ts
@@ -3,7 +3,7 @@
 
 import assert from 'assert';
 import { Horizon, rpc } from '@stellar/stellar-sdk';
-import { getLogger, IBlock } from '@subql/node-core';
+import { delay, getLogger, IBlock } from '@subql/node-core';
 import {
   SorobanEvent,
   StellarBlock,
@@ -27,6 +27,7 @@ export class StellarApi {
 
   private chainId?: string;
   private pageLimit = DEFAULT_PAGE_SIZE;
+  private sorobanIngestWaitSeconds: number;
 
   constructor(
     private endpoint: string,
@@ -35,6 +36,7 @@ export class StellarApi {
   ) {
     const { hostname, protocol, searchParams } = new URL(this.endpoint);
     this.pageLimit = config?.pageLimit || this.pageLimit;
+    this.sorobanIngestWaitSeconds = config?.sorobanIngestWaitSeconds ?? 600;
 
     const protocolStr = protocol.replace(':', '');
 
@@ -150,6 +152,80 @@ export class StellarApi {
     return effects;
   }
 
+  // The target height comes from Horizon while events are fetched from a separate
+  // soroban endpoint; with hosted load-balanced RPCs the serving backend can lag the
+  // target by several ledgers, in which case stellar-rpc rejects getEvents with
+  // JSON-RPC -32600 'startLedger must be within the ledger range: X - Y' (legacy
+  // soroban-rpc: 'start is after newest ledger'). The ledger exists, it just is not
+  // ingested yet: a transient condition, not an invalid request.
+  private isLedgerNotYetIngestedError(e: any, sequence: number): boolean {
+    const message = e?.message;
+    if (typeof message !== 'string') {
+      return false;
+    }
+    if (message === 'start is after newest ledger') {
+      return true;
+    }
+    const range =
+      /startLedger must be within the ledger range: (\d+) - (\d+)/.exec(
+        message,
+      );
+    return range !== null && sequence > Number(range[2]);
+  }
+
+  // Fallback when a -32600 carries an unrecognized wording: compare the requested
+  // sequence against the soroban endpoint's own head instead of parsing the message.
+  private async isSequenceAheadOfSorobanHead(
+    e: any,
+    sequence: number,
+  ): Promise<boolean> {
+    if (e?.code !== -32600) {
+      return false;
+    }
+    try {
+      const latest = await this.sorobanClient.getLatestLedger();
+      return sequence > latest.sequence;
+    } catch {
+      return false;
+    }
+  }
+
+  private async getEventsWhenIngested(
+    sequence: number,
+  ): Promise<SorobanEvent[]> {
+    const deadline = Date.now() + this.sorobanIngestWaitSeconds * 1000;
+    for (let attempt = 1; ; attempt++) {
+      try {
+        return await this.getAndWrapEvents(sequence);
+      } catch (e: any) {
+        if (e?.message === 'start is before oldest ledger') {
+          throw new Error(`The requested events for ledger number ${sequence} is not available on the current soroban node.
+                This is because you're trying to access a ledger that is older than the oldest ledger stored in this node.
+                To resolve this issue, you can either:
+                1. Increase the start ledger to a more recent one, or
+                2. Connect to a different node that might have a longer history of ledgers.`);
+        }
+        const notYetIngested =
+          this.isLedgerNotYetIngestedError(e, sequence) ||
+          (await this.isSequenceAheadOfSorobanHead(e, sequence));
+        if (!notYetIngested || Date.now() >= deadline) {
+          if (e?.code === -32600) {
+            logger.warn(
+              `Giving up on events for ledger ${sequence} after JSON-RPC -32600: ${e.message}`,
+            );
+          }
+          throw e;
+        }
+        if (attempt === 1 || attempt % 10 === 0) {
+          logger.warn(
+            `Events for ledger ${sequence} are not yet ingested by the soroban endpoint ("${e.message}"), waiting for it to catch up (attempt ${attempt})`,
+          );
+        }
+        await delay(6);
+      }
+    }
+  }
+
   async getAndWrapEvents(height: number): Promise<SorobanEvent[]> {
     const { events: events } = await this.sorobanClient.getEvents({
       startLedger: height,
@@ -306,27 +382,7 @@ export class StellarApi {
     );
 
     if (this.sorobanClient && hasInvokeHostFunctionOp) {
-      try {
-        eventsForSequence = await this.getAndWrapEvents(sequence);
-      } catch (e: any) {
-        if (e.message === 'start is after newest ledger') {
-          const latestLedger = (await this.sorobanClient.getLatestLedger())
-            .sequence;
-          throw new Error(`The requested events for ledger number ${sequence} is not available on the current soroban node.
-                This is because you're trying to access a ledger that is after the latest ledger number ${latestLedger} stored in this node.
-                To resolve this issue, please check you endpoint node start height`);
-        }
-
-        if (e.message === 'start is before oldest ledger') {
-          throw new Error(`The requested events for ledger number ${sequence} is not available on the current soroban node.
-                This is because you're trying to access a ledger that is older than the oldest ledger stored in this node.
-                To resolve this issue, you can either:
-                1. Increase the start ledger to a more recent one, or
-                2. Connect to a different node that might have a longer history of ledgers.`);
-        }
-
-        throw e;
-      }
+      eventsForSequence = await this.getEventsWhenIngested(sequence);
     }
 
     const wrappedLedger: StellarBlock = {
diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- `sorobanIngestWaitSeconds` option to `IStellarEndpointConfig` to bound how long the node waits for the soroban endpoint to ingest a ledger at the chain head
 
 ## [5.2.0] - 2026-01-21
 ### Changed
diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts
