[etherscan] Fix pagination for accounts with many transactions

src/plugins/etherscan/__tests__/converters/append-transactions.test.ts

@@ -0,0 +1,37 @@
+import { appendTransactions } from '../../common/converters'
+import type { Transaction } from '../../../../types/zenmoney'
+
+const transaction: Transaction = {
+  hold: null,
+  date: new Date('2026-01-01T00:00:00.000Z'),
+  movements: [{
+    id: '1',
+    account: { id: 'account' },
+    invoice: null,
+    sum: 1,
+    fee: 0
+  }],
+  merchant: null,
+  comment: null
+}
+
+describe('appendTransactions', () => {
+  it('appends large transaction batches without argument spreading', () => {
+    const target: Transaction[] = []
+    const transactions: Transaction[] = Array.from(
+      { length: 200000 },
+      (_, index): Transaction => ({
+        ...transaction,
+        movements: [{
+          ...transaction.movements[0],
+          id: String(index)
+        }]
+      })
+    )
+
+    appendTransactions(target, transactions)
+
+    expect(target).toHaveLength(transactions.length)
+    expect(target[199999].movements[0].id).toBe('199999')
+  })
+})

src/plugins/etherscan/__tests__/converters/token-transactions.test.ts

@@ -12,6 +12,7 @@ const baseTransaction: TokenTransaction = {
   blockNumber: '1',
   timeStamp: '1658608646',
   hash: '0x90bb0dcbe8fa38387145aa17d6ad99f57da91d4c6d4b65b5f7cf56454f73234b',
+  logIndex: '1',
   nonce: '1',
   blockHash: '0x1',
   from: '0x1111111111111111111111111111111111111111',

src/plugins/etherscan/__tests__/converters/transaction/transaction.test.ts

@@ -3,6 +3,7 @@ import { EthereumTransaction } from '../../../ether/types'
 
 const OWN_ACCOUNT = 'ACCOUNT_ID'
 const baseMock: EthereumTransaction = {
+  blockNumber: '1',
   timeStamp: '1658608646',
   hash: '0x90bb0dcbe8fa38387145aa17d6ad99f57da91d4c6d4b65b5f7cf56454f73234b',
   from: '1',

src/plugins/etherscan/common/converters.ts

@@ -1,5 +1,14 @@
 import { Transaction } from '../../../types/zenmoney'
 
+export function appendTransactions (
+  target: Transaction[],
+  transactions: Transaction[]
+): void {
+  for (const transaction of transactions) {
+    target.push(transaction)
+  }
+}
+
 function canBeMergedAsTransfer (left: Transaction, right: Transaction): boolean {
   const leftMovement = left.movements[0]
   const rightMovement = right.movements[0]

src/plugins/etherscan/common/pagination.test.ts

@@ -0,0 +1,179 @@
+import {
+  type BlockTransaction,
+  type FetchPageOptions,
+  fetchPaginatedTransactions
+} from './pagination'
+
+interface TestTransaction extends BlockTransaction {
+  id: string
+}
+
+function buildTransactions (
+  count: number,
+  blockNumber: number,
+  idPrefix: string
+): TestTransaction[] {
+  return Array.from({ length: count }, (_, index) => ({
+    id: `${idPrefix}-${index}`,
+    blockNumber: String(blockNumber)
+  }))
+}
+
+describe('fetchPaginatedTransactions', () => {
+  it('uses page only inside the result window and then advances endBlock', async () => {
+    const requests: FetchPageOptions[] = []
+
+    await fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async (options) => {
+        requests.push(options)
+
+        if (options.endBlock === 100) {
+          return buildTransactions(
+            1000,
+            options.page === 10 ? 90 : 100,
+            `cursor-100-page-${options.page}`
+          )
+        }
+
+        return buildTransactions(1, 80, 'cursor-90-page-1')
+      }
+    })
+
+    expect(requests).toHaveLength(11)
+    expect(requests[9]).toEqual({
+      startBlock: 1,
+      endBlock: 100,
+      page: 10,
+      offset: 1000
+    })
+    expect(requests[10]).toEqual({
+      startBlock: 1,
+      endBlock: 90,
+      page: 1,
+      offset: 1000
+    })
+  })
+
+  it('deduplicates transactions from the overlapped cursor block', async () => {
+    const transactions = await fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async (options) => {
+        if (options.endBlock === 100) {
+          const pageTransactions = buildTransactions(
+            1000,
+            options.page === 10 ? 90 : 100,
+            `cursor-100-page-${options.page}`
+          )
+
+          if (options.page === 10) {
+            pageTransactions[999] = {
+              id: 'overlap',
+              blockNumber: '90'
+            }
+          }
+
+          return pageTransactions
+        }
+
+        return [
+          {
+            id: 'overlap',
+            blockNumber: '90'
+          },
+          {
+            id: 'new',
+            blockNumber: '80'
+          }
+        ]
+      }
+    })
+
+    expect(transactions.filter((transaction) => transaction.id === 'overlap')).toHaveLength(1)
+    expect(transactions.some((transaction) => transaction.id === 'new')).toBe(true)
+  })
+
+  it('fails when a single cursor block fills the whole result window', async () => {
+    await expect(fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async (options) => buildTransactions(
+        1000,
+        100,
+        `page-${options.page}`
+      )
+    })).rejects.toThrow('Cannot safely progress')
+  })
+
+  it('advances the cursor after a result-window error when a safe last block exists', async () => {
+    const requests: FetchPageOptions[] = []
+
+    await fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async (options) => {
+        requests.push(options)
+
+        if (options.endBlock === 100) {
+          if (options.page === 10) {
+            throw new Error(
+              'Result window is too large, PageNo x Offset size must be less than or equal to 10000'
+            )
+          }
+
+          return buildTransactions(
+            1000,
+            options.page === 9 ? 90 : 100,
+            `cursor-100-page-${options.page}`
+          )
+        }
+
+        return buildTransactions(1, 80, 'cursor-90-page-1')
+      }
+    })
+
+    expect(requests[9]).toEqual({
+      startBlock: 1,
+      endBlock: 100,
+      page: 10,
+      offset: 1000
+    })
+    expect(requests[10]).toEqual({
+      startBlock: 1,
+      endBlock: 90,
+      page: 1,
+      offset: 1000
+    })
+  })
+
+  it('fails with a clear error when a pagination error happens before any cursor exists', async () => {
+    await expect(fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async () => {
+        throw new Error('Query Timeout occured')
+      }
+    })).rejects.toThrow(
+      'Cannot safely progress after Etherscan pagination error for block range 1..100'
+    )
+  })
+
+  it('fails on invalid block numbers', async () => {
+    await expect(fetchPaginatedTransactions<TestTransaction>({
+      startBlock: 1,
+      endBlock: 100,
+      getKey: (transaction) => transaction.id,
+      fetchPage: async () => [{
+        id: 'invalid',
+        blockNumber: 'invalid'
+      }]
+    })).rejects.toThrow('Invalid blockNumber')
+  })
+})

src/plugins/etherscan/common/pagination.ts

@@ -0,0 +1,130 @@
+const PAGE_SIZE = 1000
+const RESULT_WINDOW_LIMIT = 10000
+const MAX_PAGES = RESULT_WINDOW_LIMIT / PAGE_SIZE
+const RESULT_WINDOW_ERROR_PATTERN = /result window is too large|pageno\s*x\s*offset/i
+const QUERY_TIMEOUT_ERROR_PATTERN = /query timeout/i
+
+export interface BlockTransaction {
+  blockNumber: string
+}
+
+export interface FetchPageOptions {
+  startBlock: number
+  endBlock: number
+  page: number
+  offset: number
+}
+
+interface FetchPaginatedTransactionsOptions<TTransaction extends BlockTransaction> {
+  startBlock: number
+  endBlock: number
+  fetchPage: (options: FetchPageOptions) => Promise<TTransaction[]>
+  getKey: (transaction: TTransaction) => string
+}
+
+function parseBlockNumber (transaction: BlockTransaction): number {
+  const blockNumber = Number(transaction.blockNumber)
+
+  if (!Number.isSafeInteger(blockNumber)) {
+    throw new Error(`Invalid blockNumber from API: ${transaction.blockNumber}`)
+  }
+
+  return blockNumber
+}
+
+function getErrorMessage (error: unknown): string {
+  if (error instanceof Error) {
+    return error.message
+  }
+
+  return String(error)
+}
+
+function isRecoverableCursorError (error: unknown): boolean {
+  const message = getErrorMessage(error)
+
+  return RESULT_WINDOW_ERROR_PATTERN.test(message) ||
+    QUERY_TIMEOUT_ERROR_PATTERN.test(message)
+}
+
+export async function fetchPaginatedTransactions<TTransaction extends BlockTransaction> ({
+  startBlock,
+  endBlock,
+  fetchPage,
+  getKey
+}: FetchPaginatedTransactionsOptions<TTransaction>): Promise<TTransaction[]> {
+  let cursor = endBlock
+  const transactions: TTransaction[] = []
+  const seen = new Set<string>()
+
+  while (cursor >= startBlock) {
+    let lastBlock: number | null = null
+    let exhausted = false
+
+    for (let page = 1; page <= MAX_PAGES; page++) {
+      let pageTransactions: TTransaction[]
+
+      try {
+        pageTransactions = await fetchPage({
+          startBlock,
+          endBlock: cursor,
+          page,
+          offset: PAGE_SIZE
+        })
+      } catch (error) {
+        if (!isRecoverableCursorError(error)) {
+          throw error
+        }
+
+        if (lastBlock === null) {
+          throw new Error(
+            'Cannot safely progress after Etherscan pagination error for ' +
+            `block range ${startBlock}..${cursor}: ${getErrorMessage(error)}`
+          )
+        }
+
+        break
+      }
+
+      if (pageTransactions.length === 0) {
+        exhausted = true
+        break
+      }
+
+      for (const transaction of pageTransactions) {
+        const key = getKey(transaction)
+
+        if (!seen.has(key)) {
+          seen.add(key)
+          transactions.push(transaction)
+        }
+      }
+
+      lastBlock = parseBlockNumber(pageTransactions[pageTransactions.length - 1])
+
+      if (pageTransactions.length < PAGE_SIZE) {
+        exhausted = true
+        break
+      }
+    }
+
+    if (exhausted || lastBlock === null) {
+      break
+    }
+
+    if (lastBlock > cursor) {
+      throw new Error('API returned rows in non-monotonic block order')
+    }
+
+    if (lastBlock === cursor) {
+      throw new Error(
+        `Cannot safely progress: block ${cursor} alone fills the ` +
+        `${RESULT_WINDOW_LIMIT} result window`
+      )
+    }
+
+    cursor = lastBlock
+  }
+
+  return transactions
+}