33 * them to test/fixtures/soroban-events-real/<symbol>.json in the shape expected
44 * by the pipeline's transform_2 output.
55 *
6- * Invoked via `npm run fixtures:refresh`. Not part of CI — requires network.
7- * Re-run when the event schema changes or a new event symbol is introduced.
6+ * Usage:
7+ * npm run fixtures:refresh — wipe & replace all real fixtures
8+ * npm run fixtures:pull — fetch only events newer than what we have
9+ *
10+ * Not part of CI — requires network.
811 *
912 * The script only fetches events actually emitted by the current root. Soroban
1013 * testnet RPC only retains ~10-20k ledgers of event history, so we scan a
@@ -72,26 +75,24 @@ async function fetchFor(contractId: string, startLedger: number) {
7275 return resp . events ;
7376}
7477
75- async function main ( ) : Promise < void > {
76- const server = new SorobanRpc . Server ( RPC_URL ) ;
77- const latest = await server . getLatestLedger ( ) ;
78- const startLedger = Math . max ( latest . sequence - LOOKBACK_LEDGERS , 1 ) ;
79-
80- console . log (
81- `fetching events for ${ CURRENT_ROOT } ${ startLedger } ${ latest . sequence } ,
82- ) ;
83-
84- const events = await fetchFor ( CURRENT_ROOT , startLedger ) ;
85- console . log ( `decoded ${ events . length } ) ;
86- if ( events . length === 0 ) {
87- console . log ( 'no events for the current root in the retention window' ) ;
88- console . log ( 'fixtures will be empty; real-data test suite will skip' ) ;
89- }
78+ interface FixtureRow {
79+ id : string ;
80+ transaction_hash : string ;
81+ ledger_sequence : number ;
82+ created_at : string ;
83+ command : string ;
84+ channel : string ;
85+ emitter_contract_id : string ;
86+ data : string ;
87+ topics : string ;
88+ }
9089
91- const bySymbol : Record < string , unknown [ ] > = { } ;
92- const toScVal = ( raw : unknown ) : xdr . ScVal =>
93- raw instanceof xdr . ScVal ? raw : xdr . ScVal . fromXDR ( raw as string , 'base64' ) ;
90+ const toScVal = ( raw : unknown ) : xdr . ScVal =>
91+ raw instanceof xdr . ScVal ? raw : xdr . ScVal . fromXDR ( raw as string , 'base64' ) ;
9492
93+ /** Convert raw RPC events into fixture rows grouped by command symbol. */
94+ function eventsToRows ( events : Awaited < ReturnType < typeof fetchFor > > ) : Record < string , FixtureRow [ ] > {
95+ const bySymbol : Record < string , FixtureRow [ ] > = { } ;
9596 for ( const ev of events ) {
9697 const topicsJson = ( ev . topic as unknown [ ] ) . map ( ( t ) => scvalToGoldsky ( toScVal ( t ) ) ) ;
9798 const dataJson = scvalToGoldsky ( toScVal ( ev . value ) ) ;
@@ -101,7 +102,7 @@ async function main(): Promise<void> {
101102 const rawContractId = ( ev as { contractId ?: unknown } ) . contractId ;
102103 const emitter =
103104 typeof rawContractId === 'string' ? rawContractId : String ( rawContractId ?? '' ) ;
104- const row = {
105+ const row : FixtureRow = {
105106 id : ev . id ,
106107 transaction_hash : ev . txHash ,
107108 ledger_sequence : Number ( ev . ledger ) ,
@@ -114,10 +115,75 @@ async function main(): Promise<void> {
114115 } ;
115116 ( bySymbol [ symbol ] ??= [ ] ) . push ( row ) ;
116117 }
118+ return bySymbol ;
119+ }
120+
121+ /** Read existing fixture files and return the highest ledger_sequence seen. */
122+ function loadExistingFixtures ( ) : { bySymbol : Record < string , FixtureRow [ ] > ; maxLedger : number } {
123+ const bySymbol : Record < string , FixtureRow [ ] > = { } ;
124+ let maxLedger = 0 ;
125+ if ( ! fs . existsSync ( FIXTURE_DIR ) ) return { bySymbol, maxLedger } ;
126+ for ( const file of fs . readdirSync ( FIXTURE_DIR ) . filter ( ( f ) => f . endsWith ( '.json' ) ) ) {
127+ const symbol = path . basename ( file , '.json' ) ;
128+ const rows : FixtureRow [ ] = JSON . parse ( fs . readFileSync ( path . join ( FIXTURE_DIR , file ) , 'utf8' ) ) ;
129+ bySymbol [ symbol ] = rows ;
130+ for ( const r of rows ) {
131+ if ( r . ledger_sequence > maxLedger ) maxLedger = r . ledger_sequence ;
132+ }
133+ }
134+ return { bySymbol, maxLedger } ;
135+ }
136+
137+ /** Extract sub-registry contract IDs from sub_reg fixture rows. */
138+ function extractSubRegistryIds ( rows : FixtureRow [ ] ) : string [ ] {
139+ const ids : string [ ] = [ ] ;
140+ for ( const row of rows ) {
141+ const data = JSON . parse ( row . data ) as {
142+ map ?: { key : { symbol ?: string } ; val : { address ?: string } } [ ] ;
143+ } ;
144+ for ( const entry of data . map ?? [ ] ) {
145+ if ( entry . key . symbol === 'contract_id' && entry . val . address ) {
146+ ids . push ( entry . val . address ) ;
147+ }
148+ }
149+ }
150+ return ids ;
151+ }
117152
153+ /** Fetch events for the root contract and all known sub-registries. */
154+ async function fetchAllContracts (
155+ startLedger : number ,
156+ existingSubRegRows ?: FixtureRow [ ] ,
157+ ) : Promise < Awaited < ReturnType < typeof fetchFor > > > {
158+ // Fetch root events first.
159+ const rootEvents = await fetchFor ( CURRENT_ROOT , startLedger ) ;
160+ console . log ( ` root: ${ rootEvents . length } ) ;
161+
162+ // Discover sub-registry contract IDs from both freshly fetched and
163+ // previously existing sub_reg rows.
164+ const freshRows = eventsToRows ( rootEvents ) ;
165+ const allSubRegRows = [
166+ ...( freshRows [ 'sub_reg' ] ?? [ ] ) ,
167+ ...( existingSubRegRows ?? [ ] ) ,
168+ ] ;
169+ const subIds = [ ...new Set ( extractSubRegistryIds ( allSubRegRows ) ) ]
170+ . filter ( ( id ) => id !== CURRENT_ROOT ) ;
171+
172+ // Fetch events from each sub-registry in parallel.
173+ const subResults = await Promise . all (
174+ subIds . map ( async ( id ) => {
175+ const events = await fetchFor ( id , startLedger ) ;
176+ console . log ( ` sub-registry ${ id } ${ events . length } ) ;
177+ return events ;
178+ } ) ,
179+ ) ;
180+
181+ return [ ...rootEvents , ...subResults . flat ( ) ] ;
182+ }
183+
184+ /** Write grouped fixture rows to disk. */
185+ function writeFixtures ( bySymbol : Record < string , FixtureRow [ ] > ) : void {
118186 fs . mkdirSync ( FIXTURE_DIR , { recursive : true } ) ;
119- // Clear stale per-symbol files so a shrinking event set doesn't leave
120- // leftovers from a previous refresh.
121187 for ( const existing of fs . readdirSync ( FIXTURE_DIR ) ) {
122188 if ( existing . endsWith ( '.json' ) ) {
123189 fs . unlinkSync ( path . join ( FIXTURE_DIR , existing ) ) ;
@@ -130,6 +196,78 @@ async function main(): Promise<void> {
130196 }
131197}
132198
199+ /** Wipe and replace: fetch everything in the lookback window. */
200+ async function refresh ( ) : Promise < void > {
201+ const server = new SorobanRpc . Server ( RPC_URL ) ;
202+ const latest = await server . getLatestLedger ( ) ;
203+ const startLedger = Math . max ( latest . sequence - LOOKBACK_LEDGERS , 1 ) ;
204+
205+ console . log (
206+ `fetching events from ledger ${ startLedger } ${ latest . sequence } ,
207+ ) ;
208+
209+ const events = await fetchAllContracts ( startLedger ) ;
210+ console . log ( `decoded ${ events . length } ) ;
211+ if ( events . length === 0 ) {
212+ console . log ( 'no events in the retention window' ) ;
213+ console . log ( 'fixtures will be empty; real-data test suite will skip' ) ;
214+ }
215+
216+ writeFixtures ( eventsToRows ( events ) ) ;
217+ }
218+
219+ /** Incremental pull: keep existing fixtures and append only newer events. */
220+ async function pull ( ) : Promise < void > {
221+ const { bySymbol : existing , maxLedger } = loadExistingFixtures ( ) ;
222+ const existingIds = new Set < string > ( ) ;
223+ for ( const rows of Object . values ( existing ) ) {
224+ for ( const r of rows ) existingIds . add ( r . id ) ;
225+ }
226+
227+ const server = new SorobanRpc . Server ( RPC_URL ) ;
228+ const latest = await server . getLatestLedger ( ) ;
229+ // Start one ledger after the last one we already have, but never older
230+ // than the RPC retention window (the RPC silently returns nothing if the
231+ // startLedger is outside its ~10k-ledger window).
232+ const retentionFloor = Math . max ( latest . sequence - LOOKBACK_LEDGERS , 1 ) ;
233+ const startLedger = maxLedger > 0
234+ ? Math . max ( maxLedger + 1 , retentionFloor )
235+ : retentionFloor ;
236+
237+ console . log (
238+ `pulling new events from ledger ${ startLedger } ${ latest . sequence } +
239+ ( maxLedger > 0 ? ` (existing max ledger: ${ maxLedger } : ' (no existing fixtures, full fetch)' ) ,
240+ ) ;
241+
242+ const events = await fetchAllContracts ( startLedger , existing [ 'sub_reg' ] ) ;
243+ const newRows = eventsToRows ( events ) ;
244+
245+ // Deduplicate by event id and merge into existing fixtures.
246+ let added = 0 ;
247+ for ( const [ symbol , rows ] of Object . entries ( newRows ) ) {
248+ if ( ! existing [ symbol ] ) existing [ symbol ] = [ ] ;
249+ for ( const row of rows ) {
250+ if ( ! existingIds . has ( row . id ) ) {
251+ existing [ symbol ] . push ( row ) ;
252+ existingIds . add ( row . id ) ;
253+ added ++ ;
254+ }
255+ }
256+ }
257+
258+ console . log ( `fetched ${ events . length } ${ added } ) ;
259+ writeFixtures ( existing ) ;
260+ }
261+
262+ async function main ( ) : Promise < void > {
263+ const mode = process . argv [ 2 ] ?? 'refresh' ;
264+ if ( mode === 'pull' ) {
265+ await pull ( ) ;
266+ } else {
267+ await refresh ( ) ;
268+ }
269+ }
270+
133271main ( ) . catch ( ( err ) => {
134272 console . error ( err ) ;
135273 process . exit ( 1 ) ;
0 commit comments