test(xdr): source the real-traffic corpus from both Horizon and RPC

Author: quietbits

scripts/corpus-fixtures.ts

@@ -0,0 +1,133 @@
+// Shared plumbing for the two corpus refreshers, `refresh-horizon-corpus.ts`
+// and `refresh-rpc-corpus.ts`.
+//
+// The validation helpers exist because of a real incident: Horizon quietly
+// stopped serving `result_meta_xdr`, the refresher wrote `undefined` into the
+// fixture, and it surfaced much later as a wall of unreadable test failures.
+// Anything a corpus depends on goes through these, so a field disappearing
+// upstream fails at fetch time and names the field.
+import { writeFileSync, mkdirSync } from "node:fs";
+import { resolve } from "node:path";
+
+export interface CorpusFile<T> {
+  source: string;
+  fetchedAt: string;
+  records: T[];
+}
+
+export function requireString(
+  value: unknown,
+  field: string,
+  where: string,
+): string {
+  // Trimmed, so a whitespace-only value fails here rather than surviving into
+  // the fixture and failing later at base64 decode. Internal whitespace is
+  // left alone: `Buffer.from(x, "base64")` tolerates it, so wrapped base64 is
+  // still legitimate.
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(
+      `${where}: response has no \`${field}\`. The corpus needs it — check ` +
+        `whether this endpoint still returns the field before refreshing.`,
+    );
+  }
+  return value;
+}
+
+// `JSON.stringify` renders NaN and Infinity as `null`, which is the opposite of
+// helpful in a diagnostic, so numbers are stringified directly.
+function describe(value: unknown): string {
+  return typeof value === "number" ? String(value) : JSON.stringify(value);
+}
+
+// Only used for ledger sequences, which are always positive integers — `0`,
+// negatives and fractions all mean the response was not what we expected.
+export function requireNumber(
+  value: unknown,
+  field: string,
+  where: string,
+): number {
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
+    throw new Error(
+      `${where}: response has no positive integer \`${field}\` ` +
+        `(got ${describe(value)}).`,
+    );
+  }
+  return value;
+}
+
+// `COUNT` comes from the environment, so every malformed value has to fail
+// here. It used to flow straight into `Number()`: `COUNT=abc` became `NaN`, the
+// `records.length < COUNT` fetch loops never ran, an empty corpus was written,
+// and the test file skipped every corpus describe on a zero exit code.
+export function readCount(raw: string | undefined, fallback: number): number {
+  const count = Number(raw ?? fallback);
+  if (!Number.isInteger(count) || count < 1) {
+    throw new Error(
+      `COUNT must be a positive integer, got ${JSON.stringify(raw)}.`,
+    );
+  }
+  return count;
+}
+
+export function writeCorpus<T>(
+  outDir: string,
+  filename: string,
+  payload: CorpusFile<T>,
+): void {
+  // An empty corpus is never a legitimate refresh result, and writing one is
+  // silently destructive: `corpus_round_trip.test.ts` skips a section whose
+  // corpus has no records, so replacing a populated fixture with an empty one
+  // converts real coverage into a green run. Refuse rather than overwrite.
+  if (payload.records.length === 0) {
+    throw new Error(
+      `${filename}: refusing to write an empty corpus — the fetch returned no ` +
+        `records, and an empty fixture would make the tests skip silently.`,
+    );
+  }
+
+  mkdirSync(outDir, { recursive: true });
+  writeFileSync(
+    resolve(outDir, filename),
+    JSON.stringify(payload, null, 2) + "\n",
+  );
+}
+
+export function reportCount(got: number, want: number, kind: string): void {
+  console.log(`Wrote ${got} ${kind} records.`);
+  if (got < want) {
+    console.warn(`  ! wanted ${want}; only ${got} were available.`);
+  }
+}
+
+// The corpus is only as good as its envelope-type spread, and that is the one
+// thing a fetch cannot guarantee — print it so a lopsided sample is visible
+// before it gets committed.
+export function reportEnvelopeSpread(envelopesBase64: string[]): void {
+  // EnvelopeType discriminants, per the XDR union.
+  const names = new Map([
+    [0, "envelopeTypeTxV0"],
+    [2, "envelopeTypeTx"],
+    [5, "envelopeTypeTxFeeBump"],
+  ]);
+  const counts = new Map<string, number>();
+  for (const b64 of envelopesBase64) {
+    const buf = Buffer.from(b64, "base64");
+    // A truncated or non-base64 value carries no discriminant. Count it rather
+    // than letting `readUInt32BE` throw a bare RangeError — this runs after the
+    // fixture is already on disk, so aborting here would leave a written file
+    // behind a failed run.
+    if (buf.length < 4) {
+      counts.set("unparseable", (counts.get("unparseable") ?? 0) + 1);
+      continue;
+    }
+    // The discriminant is the first 4 bytes of the XDR union.
+    const kind = buf.readUInt32BE(0);
+    const name = names.get(kind) ?? `unknown(${kind})`;
+    counts.set(name, (counts.get(name) ?? 0) + 1);
+  }
+  const spread = [...counts]
+    .sort((a, b) => b[1] - a[1])
+    .map(([name, n]) => `${name}=${n}`)
+    .join(", ");
+  console.log(`  envelope spread: ${spread || "(none)"}`);
+}

scripts/refresh-horizon-corpus.ts

@@ -1,19 +1,27 @@
-// One-shot fetcher that populates `test/fixtures/horizon-corpus/` with
-// real envelope/operation/ledger XDR captured from horizon mainnet.
+// One-shot fetcher that populates `test/fixtures/horizon-corpus/` with real
+// envelope/result/fee-meta/ledger XDR captured from Horizon mainnet.
 //
 // Usage:
 //   pnpm tsx scripts/refresh-horizon-corpus.ts
+//   HORIZON=... COUNT=100 pnpm tsx scripts/refresh-horizon-corpus.ts
 //
 // Refresh quarterly or when a new Stellar protocol version ships. The
 // resulting JSON files are checked into the repo and consumed by
-// `test/unit/base/xdr/corpus_round_trip.test.ts` — that test asserts every
+// `test/unit/xdr/corpus_round_trip.test.ts` — that test asserts every
 // fixture round-trips losslessly through both the new and legacy XDR
 // runtimes, so any encoding regression that affects shapes the network
 // actually produces will fail loudly.
 //
-// We pull from a few endpoints to hit a broad surface:
-//   - /transactions          → TransactionEnvelope (covers ops, memos, signers)
-//   - /ledgers/{seq}          → LedgerHeader (covers nested ledger entry types)
+// We pull from two endpoints to hit a broad surface:
+//   - /transactions          → TransactionEnvelope (covers ops, memos,
+//                              signers), TransactionResult, and OperationMeta
+//                              via `fee_meta_xdr`
+//   - /ledgers               → LedgerHeader (covers nested ledger entry types)
+//
+// Note on transaction meta: SDF removed `result_meta_xdr` from the hosted
+// Horizon API and points callers at Stellar RPC instead, so there is no
+// `TransactionMeta` here. That coverage lives in the RPC corpus — see
+// `scripts/refresh-rpc-corpus.ts`. Don't try to reintroduce it from Horizon.
 //
 // We don't decode the bytes here — just snapshot the on-wire form. The
 // test does the decode/re-encode validation.
@@ -23,80 +31,82 @@
 // either sample a larger range, or hand-pick a mix of `envelopeTypeTx`,
 // `envelopeTypeTxV0`, and `envelopeTypeTxFeeBump` records by querying
 // historical ranges that included Soroban contract calls, classic
-// payments, and fee-bumped batches.
-import { writeFileSync, mkdirSync } from "node:fs";
+// payments, and fee-bumped batches. The run prints the spread it got.
 import { resolve } from "node:path";
 
+import {
+  readCount,
+  requireNumber,
+  requireString,
+  reportCount,
+  reportEnvelopeSpread,
+  writeCorpus,
+} from "./corpus-fixtures.js";
+
 const HORIZON = process.env.HORIZON ?? "https://horizon.stellar.org";
 const OUT_DIR = resolve(import.meta.dirname, "../test/fixtures/horizon-corpus");
-const COUNT = Number(process.env.COUNT ?? 50);
+const COUNT = readCount(process.env.COUNT, 50);
 
-interface CorpusFile<T> {
-  source: string;
-  fetchedAt: string;
-  records: T[];
+// Horizon caps a single page at 200 records.
+const PAGE_LIMIT = Math.min(COUNT, 200);
+
+interface HorizonPage {
+  _embedded?: { records?: Array<Record<string, unknown>> };
 }
 
-async function getJson(url: string): Promise<unknown> {
+async function getJson(url: string): Promise<HorizonPage> {
   const r = await fetch(url);
   if (!r.ok) throw new Error(`${url} → ${r.status} ${r.statusText}`);
-  return r.json();
+  return (await r.json()) as HorizonPage;
 }
 
 async function snapshotTransactions(): Promise<void> {
-  const url = `${HORIZON}/transactions?limit=${Math.min(COUNT, 200)}&order=desc&include_failed=false`;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const json = (await getJson(url)) as any;
-  const records = json._embedded.records.map(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (r: any) => ({
-      hash: r.hash,
-      envelope_xdr: r.envelope_xdr,
-      result_xdr: r.result_xdr,
-      result_meta_xdr: r.result_meta_xdr,
-    }),
-  );
-  writeCorpus("transactions.json", {
+  const url = `${HORIZON}/transactions?limit=${PAGE_LIMIT}&order=desc&include_failed=false`;
+  const records = ((await getJson(url))._embedded?.records ?? []).map((r) => {
+    const where = `transaction ${String(r.hash ?? "<no hash>")}`;
+    return {
+      hash: requireString(r.hash, "hash", where),
+      envelope_xdr: requireString(r.envelope_xdr, "envelope_xdr", where),
+      result_xdr: requireString(r.result_xdr, "result_xdr", where),
+      fee_meta_xdr: requireString(r.fee_meta_xdr, "fee_meta_xdr", where),
+    };
+  });
+
+  writeCorpus(OUT_DIR, "transactions.json", {
     source: url,
     fetchedAt: new Date().toISOString(),
     records,
   });
-  console.log(`Wrote ${records.length} transaction records.`);
+  reportCount(records.length, COUNT, "transaction");
+  reportEnvelopeSpread(records.map((r) => r.envelope_xdr));
 }
 
 async function snapshotLedgers(): Promise<void> {
-  const url = `${HORIZON}/ledgers?limit=${Math.min(COUNT, 200)}&order=desc`;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const json = (await getJson(url)) as any;
-  const records = json._embedded.records.map(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (r: any) => ({
-      sequence: r.sequence,
-      header_xdr: r.header_xdr,
-    }),
-  );
-  writeCorpus("ledgers.json", {
+  const url = `${HORIZON}/ledgers?limit=${PAGE_LIMIT}&order=desc`;
+  const records = ((await getJson(url))._embedded?.records ?? []).map((r) => {
+    const where = `ledger ${String(r.sequence ?? "<no sequence>")}`;
+    return {
+      sequence: requireNumber(r.sequence, "sequence", where),
+      header_xdr: requireString(r.header_xdr, "header_xdr", where),
+    };
+  });
+
+  writeCorpus(OUT_DIR, "ledgers.json", {
     source: url,
     fetchedAt: new Date().toISOString(),
     records,
   });
-  console.log(`Wrote ${records.length} ledger records.`);
-}
-
-function writeCorpus<T>(filename: string, payload: CorpusFile<T>): void {
-  mkdirSync(OUT_DIR, { recursive: true });
-  writeFileSync(
-    resolve(OUT_DIR, filename),
-    JSON.stringify(payload, null, 2) + "\n",
-  );
+  reportCount(records.length, COUNT, "ledger");
 }
 
 async function main(): Promise<void> {
   console.log(`Fetching from ${HORIZON}, COUNT=${COUNT}`);
   await snapshotTransactions();
   await snapshotLedgers();
   console.log(`\nCorpus written to ${OUT_DIR}`);
-  console.log(`Run \`pnpm exec vitest run test/unit/base/xdr/corpus_round_trip.test.ts\` to validate.`);
+  console.log(
+    `Run \`pnpm exec vitest run test/unit/xdr/corpus_round_trip.test.ts\` to validate.`,
+  );
 }
 
 main().catch((err) => {