@@ -24,7 +24,6 @@ import (
2424 "encoding/hex"
2525 "encoding/json"
2626 "fmt"
27- "io/fs"
2827 "path/filepath"
2928 "slices"
3029 "strings"
@@ -102,7 +101,9 @@ func (m *filesystemMerging) writeACHFile(ctx context.Context, xfer incoming.ACHF
102101 if err := ach .NewWriter (& buf ).Write (xfer .File ); err != nil {
103102 return err
104103 }
105- path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.ach" , xfer .FileID ))
104+
105+ fileID := strings .TrimSuffix (xfer .FileID , ".ach" )
106+ path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.ach" , fileID ))
106107 if err := m .storage .WriteFile (path , buf .Bytes ()); err != nil {
107108 return err
108109 }
@@ -116,7 +117,8 @@ func (m *filesystemMerging) writeACHFile(ctx context.Context, xfer incoming.ACHF
116117 "shardKey" : log .String (xfer .ShardKey ),
117118 }).Logf ("ERROR encoding ValidateOpts: %v" , err )
118119 }
119- path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.json" , xfer .FileID ))
120+
121+ path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.json" , fileID ))
120122 if err := m .storage .WriteFile (path , buf .Bytes ()); err != nil {
121123 m .logger .Warn ().With (log.Fields {
122124 "fileID" : log .String (xfer .FileID ),
@@ -129,7 +131,8 @@ func (m *filesystemMerging) writeACHFile(ctx context.Context, xfer incoming.ACHF
129131}
130132
131133func (m * filesystemMerging ) HandleCancel (ctx context.Context , cancel incoming.CancelACHFile ) (incoming.FileCancellationResponse , error ) {
132- path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.ach" , cancel .FileID ))
134+ fileID := strings .TrimSuffix (cancel .FileID , ".ach" )
135+ path := filepath .Join ("mergable" , m .shard .Name , fmt .Sprintf ("%s.ach" , fileID ))
133136
134137 // Check if the file exists already
135138 file , _ := m .storage .Open (path )
@@ -335,7 +338,7 @@ func (m *filesystemMerging) WithEachMerged(ctx context.Context, f func(context.C
335338 attribute .Int ("achgateway.successful_remote_writes" , successfulRemoteWrites ),
336339 )
337340
338- // Build a mapping of BatchHeader + EntryDetail from dir
341+ // Build a mapping of BatchHeader + EntryDetail from dir (input files)
339342 mappings , err := m .buildDirMapping (dir , canceledFiles )
340343 if err != nil {
341344 el .Add (err )
@@ -395,79 +398,87 @@ func fileAcceptor(canceledFiles []string) func(string) ach.FileAcceptance {
395398 }
396399}
397400
401+ // buildDirMapping computes a tree of the input files and their entries together so that we can quickly find
402+ // where they were merged into.
398403func (m * filesystemMerging ) buildDirMapping (dir string , canceledFiles []string ) (* treemap.TreeMap [string , string ], error ) {
399404 tree := treemap .New [string , string ]()
400405
406+ fds , err := m .storage .ReadDir (dir )
407+ if err != nil {
408+ return nil , err
409+ }
410+
401411 acceptor := fileAcceptor (canceledFiles )
402412
403- err := fs .WalkDir (m .storage , dir , func (path string , d fs.DirEntry , err error ) error {
404- if err != nil {
405- if strings .Contains (err .Error (), "is a directory" ) {
406- return nil
407- }
408- return err
409- }
410- if d .IsDir () {
411- return nil // skip directories
412- }
413+ for i := range fds {
414+ path := fds [i ].Name ()
413415
414- if strings . Contains ( path , "uploaded" ) || strings . HasSuffix ( path , ".json" ) {
415- // Skip /uploaded/ as we're only interested in the input files.
416- // Skip .json files as they contain ValidateOpts
417- return nil
416+ // Ignore directories as ReadDir continues inside of them.
417+ // .json files contain ValidateOpts which we can skip
418+ if fds [ i ]. IsDir () || strings . HasSuffix ( path , ".json" ) {
419+ continue
418420 }
419421
420422 // Skip the file if merging would have skipped it
421423 if acceptor (path ) == ach .SkipFile {
422- return nil
424+ continue
423425 }
424426
425- fd , err : =m .storage . Open ( path )
427+ err = m .accumulateMappings ( tree , filepath . Join ( dir , path ) )
426428 if err != nil {
427- return fmt .Errorf ("opening %s failed: %w" , path , err )
429+ return nil , fmt .Errorf ("accumulating mappings from %s failed: %w" , path , err )
428430 }
429- defer fd .Close ()
430-
431- // Check for validate opts
432- validateOptsPath := strings .TrimSuffix (path , filepath .Ext (path )) + ".json"
433- var validateOpts * ach.ValidateOpts
434- if optsFD , err := m .storage .Open (validateOptsPath ); err == nil {
435- if optsFD != nil {
436- defer optsFD .Close ()
437- }
431+ }
438432
439- err = json .NewDecoder (optsFD ).Decode (& validateOpts )
440- if err != nil {
441- return fmt .Errorf ("reading %s as validate opts failed: %w" , validateOptsPath , err )
442- }
443- }
433+ return tree , nil
434+ }
444435
445- rdr := ach .NewReader (fd )
446- if validateOpts != nil {
447- rdr .SetValidation (validateOpts )
436+ func (m * filesystemMerging ) accumulateMappings (tree * treemap.TreeMap [string , string ], path string ) error {
437+ fd , err := m .storage .Open (path )
438+ if err != nil {
439+ return fmt .Errorf ("opening %s failed: %w" , path , err )
440+ }
441+ defer fd .Close ()
442+
443+ // Check for validate opts
444+ validateOptsPath := strings .TrimSuffix (path , filepath .Ext (path )) + ".json"
445+ var validateOpts * ach.ValidateOpts
446+ if optsFD , err := m .storage .Open (validateOptsPath ); err == nil {
447+ if optsFD != nil {
448+ defer optsFD .Close ()
448449 }
449450
450- file , err := rdr . Read ( )
451+ err = json . NewDecoder ( optsFD ). Decode ( & validateOpts )
451452 if err != nil {
452- return fmt .Errorf ("reading %s failed: %w" , path , err )
453+ return fmt .Errorf ("reading %s as validate opts failed: %w" , validateOptsPath , err )
453454 }
455+ }
454456
455- _ , filename := filepath .Split (path )
457+ rdr := ach .NewReader (fd )
458+ if validateOpts != nil {
459+ rdr .SetValidation (validateOpts )
460+ }
461+
462+ file , err := rdr .Read ()
463+ if err != nil {
464+ return fmt .Errorf ("reading %s failed: %w" , path , err )
465+ }
456466
457- // Add each BatchHeader and Entry to the map
458- for i := range file .Batches {
459- bh := file .Batches [i ].GetHeader ().String ()
467+ _ , filename := filepath .Split (path )
460468
461- entries := file .Batches [i ].GetEntries ()
462- for m := range entries {
463- tree .Set (makeKey (bh , entries [m ]), filename )
464- }
465- }
469+ // Add each BatchHeader and Entry to the map
470+ for i := range file .Batches {
471+ bh := file .Batches [i ].GetHeader ().String ()
466472
467- return nil
468- })
473+ entries := file .Batches [i ].GetEntries ()
469474
470- return tree , err
475+ for m := range entries {
476+ key := makeKey (bh , entries [m ])
477+ tree .Set (key , filename )
478+ }
479+ }
480+
481+ return nil
471482}
472483
473484func makeKey (bh string , entry * ach.EntryDetail ) string {
0 commit comments