Skip to content

Commit b6c9f49

Browse files
adamdecafadamdecaf
authored
Merge pull request #227 from moov-io/pipeline-chunk-files-benchmark
Pipeline chunk files benchmark
2 parents dc6e821 + 8365ce6 commit b6c9f49

5 files changed

Lines changed: 121 additions & 34 deletions

File tree

‎go.mod‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ require (
1313
github.com/gorilla/mux v1.8.1
1414
github.com/hashicorp/go-retryablehttp v0.7.5
1515
github.com/jlaffaye/ftp v0.2.0
16-
github.com/moov-io/ach v1.33.4
16+
github.com/moov-io/ach v1.35.0
1717
github.com/moov-io/base v0.48.5
1818
github.com/moov-io/cryptfs v0.7.1
1919
github.com/moov-io/go-ftp v0.3.2
2020
github.com/moov-io/go-sftp v0.13.3
2121
github.com/ory/dockertest/v3 v3.9.1
2222
github.com/ory/mail/v3 v3.0.0
2323
github.com/pkg/errors v0.9.1
24-
github.com/prometheus/client_golang v1.18.0
24+
github.com/prometheus/client_golang v1.19.0
2525
github.com/rickar/cal/v2 v2.1.13
2626
github.com/robfig/cron/v3 v3.0.1
2727
github.com/sethvargo/go-retry v0.2.4
@@ -33,7 +33,7 @@ require (
3333
gocloud.dev v0.35.0
3434
gocloud.dev/pubsub/kafkapubsub v0.35.0
3535
goftp.io/server v0.4.1
36-
golang.org/x/crypto v0.18.0
36+
golang.org/x/crypto v0.19.0
3737
golang.org/x/exp v0.0.0-20231219160207-73b9e39aefca
3838
golang.org/x/sync v0.5.0
3939
golang.org/x/text v0.14.0
@@ -144,7 +144,6 @@ require (
144144
github.com/kylelemons/godebug v1.1.0 // indirect
145145
github.com/magiconair/properties v1.8.7 // indirect
146146
github.com/markbates/pkger v0.17.1 // indirect
147-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
148147
github.com/minio/md5-simd v1.1.2 // indirect
149148
github.com/minio/minio-go/v6 v6.0.57 // indirect
150149
github.com/minio/sha256-simd v1.0.0 // indirect
@@ -163,7 +162,7 @@ require (
163162
github.com/pkg/sftp v1.13.6 // indirect
164163
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
165164
github.com/prometheus/client_model v0.5.0 // indirect
166-
github.com/prometheus/common v0.45.0 // indirect
165+
github.com/prometheus/common v0.48.0 // indirect
167166
github.com/prometheus/procfs v0.12.0 // indirect
168167
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
169168
github.com/ryanuber/go-glob v1.0.0 // indirect
@@ -190,9 +189,9 @@ require (
190189
go.uber.org/atomic v1.11.0 // indirect
191190
go.uber.org/multierr v1.11.0 // indirect
192191
golang.org/x/mod v0.14.0 // indirect
193-
golang.org/x/net v0.19.0 // indirect
194-
golang.org/x/oauth2 v0.15.0 // indirect
195-
golang.org/x/sys v0.16.0 // indirect
192+
golang.org/x/net v0.21.0 // indirect
193+
golang.org/x/oauth2 v0.16.0 // indirect
194+
golang.org/x/sys v0.17.0 // indirect
196195
golang.org/x/time v0.5.0 // indirect
197196
golang.org/x/tools v0.16.0 // indirect
198197
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

‎go.sum‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -344,8 +344,6 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
344344
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
345345
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
346346
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
347-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
348-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
349347
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
350348
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
351349
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
@@ -368,8 +366,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
368366
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
369367
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
370368
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
371-
github.com/moov-io/ach v1.33.4 h1:6haoD+2H+rSeLBkpGndOdpNnSnM8g/EkgppX5T+2hDA=
372-
github.com/moov-io/ach v1.33.4/go.mod h1:liEoB/PlHQoYJq+V+O/izTzTebEDkAM4waGYVcIIKc4=
369+
github.com/moov-io/ach v1.35.0 h1:SBFgGYO+GwucaZJMVCk9PW5JEp/TqLR2zcP7UTZYABU=
370+
github.com/moov-io/ach v1.35.0/go.mod h1:0WUah/tGNDi3iLk9IX2AkSQkDXTrf7RJdMcLDYy/h3c=
373371
github.com/moov-io/base v0.48.5 h1:QaTyTo6eFFFV35R9l/GdePQN40IJti9knD5hqdWPnnM=
374372
github.com/moov-io/base v0.48.5/go.mod h1:D5ZV9COV/qtCjTQuYpq7gGInCk64AhOQI6UY4kt4Rq8=
375373
github.com/moov-io/cryptfs v0.7.1 h1:aT79fcxvX3vLJ25GMFlsl6WtPLijZ3VWMaRSq8OJu4M=
@@ -409,13 +407,13 @@ github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Q
409407
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
410408
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
411409
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
412-
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
413-
github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
410+
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
411+
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
414412
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
415413
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
416414
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
417-
github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
418-
github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
415+
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
416+
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
419417
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
420418
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
421419
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
@@ -532,8 +530,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw
532530
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
533531
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
534532
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
535-
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
536-
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
533+
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
534+
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
537535
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
538536
golang.org/x/exp v0.0.0-20231219160207-73b9e39aefca h1:+xQfFu/HO/82Wwg4zuJ5xiLp0yaOLJjBGnuafXp85YQ=
539537
golang.org/x/exp v0.0.0-20231219160207-73b9e39aefca/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
@@ -566,11 +564,11 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
566564
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
567565
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
568566
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
569-
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
570-
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
567+
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
568+
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
571569
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
572-
golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
573-
golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
570+
golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
571+
golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
574572
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
575573
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
576574
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -605,16 +603,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
605603
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
606604
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
607605
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
608-
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
609-
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
606+
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
607+
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
610608
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
611609
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
612610
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
613611
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
614612
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
615613
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
616-
golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
617-
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
614+
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
615+
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
618616
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
619617
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
620618
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

‎internal/incoming/odfi/processor_test.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"os"
2424
"path/filepath"
25+
"runtime"
2526
"testing"
2627
"time"
2728

@@ -39,7 +40,10 @@ import (
3940

4041
func TestProcessor(t *testing.T) {
4142
dir := t.TempDir()
42-
err := os.WriteFile(filepath.Join(dir, "invalid.ach"), []byte("invalid-ach-file"), 0600)
43+
44+
data, err := os.ReadFile(filepath.Join("testdata", "return-no-batch-controls.ach"))
45+
require.NoError(t, err)
46+
err = os.WriteFile(filepath.Join(dir, "invalid.ach"), data, 0600)
4347
require.NoError(t, err)
4448

4549
proc := &MockProcessor{}
@@ -60,7 +64,11 @@ func TestProcessor(t *testing.T) {
6064

6165
require.NotNil(t, proc.HandledFile)
6266
require.NotNil(t, proc.HandledFile.ACHFile)
63-
require.Equal(t, "7ffdca32898fc89e5e680d0a01e9e1c2a1cd2717", proc.HandledFile.ACHFile.ID)
67+
if runtime.GOOS == "windows" {
68+
require.Equal(t, "41bcece862c7cb3e74ca96b828fe57458db85199", proc.HandledFile.ACHFile.ID)
69+
} else {
70+
require.Equal(t, "36b5cb975ef810e23f82d976219c176b70d785b2", proc.HandledFile.ACHFile.ID)
71+
}
6472

6573
// Real world file
6674
path := filepath.Join("..", "..", "..", "testdata", "HMBRAD_ACHEXPORT_1001_08_19_2022_09_10")

‎internal/pipeline/merging.go‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -416,36 +416,54 @@ func (m *filesystemMerging) chunkFilesTogether(ctx context.Context, indices []in
416416
if err != nil {
417417
return nil, err
418418
}
419+
span.AddEvent("files-read")
420+
419421
merged, err := ach.MergeFilesWith(files.ACHFiles, conditions)
420422
if err != nil {
421423
return nil, err
422424
}
423-
return determineMergeDestinations(files, merged), nil
425+
span.AddEvent("merged-files")
426+
427+
out, err := determineMergeDestinations(files, merged), nil
428+
if err != nil {
429+
return nil, err
430+
}
431+
return out, nil
424432
}
425433

426434
var input namedFiles
427-
out := make([]*ach.File, 0, len(indices))
435+
mergeParts := make([]*ach.File, 0, len(indices))
428436
for i := 0; i < len(indices)-1; i += 0 {
429437
files, err := m.readFiles(matches[indices[i]:indices[i+1]]) // need to keep filename around
430438
if err != nil {
431439
return nil, err
432440
}
441+
span.AddEvent(fmt.Sprintf("files-read-idx-%d", i))
442+
433443
input.Names = append(input.Names, files.Names...)
434444
input.ACHFiles = append(input.ACHFiles, files.ACHFiles...)
435445

436446
fs, err := ach.MergeFilesWith(files.ACHFiles, conditions)
437447
if err != nil {
438448
return nil, err
439449
}
450+
span.AddEvent(fmt.Sprintf("merged-files-idx-%d", i))
451+
440452
i += 1
441-
out = append(out, fs...)
453+
mergeParts = append(mergeParts, fs...)
454+
}
455+
456+
merged, err := ach.MergeFilesWith(mergeParts, conditions)
457+
if err != nil {
458+
return nil, err
442459
}
460+
span.AddEvent("final-merge-files")
443461

444-
merged, err := ach.MergeFilesWith(out, conditions)
462+
out, err := determineMergeDestinations(input, merged), nil
445463
if err != nil {
446464
return nil, err
447465
}
448-
return determineMergeDestinations(input, merged), nil
466+
return out, nil
449467
}
450468

451469
// determineMergeDestinations will compare the input ACH files against the merged files to determine

‎internal/pipeline/merging_test.go‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"os"
2626
"path/filepath"
2727
"slices"
28+
"strconv"
2829
"testing"
2930

3031
"github.com/moov-io/ach"
@@ -128,7 +129,70 @@ func TestMerging_chunkFilesTogether(t *testing.T) {
128129
require.Len(t, merged[0].ACHFile.Batches[0].GetEntries(), 2)
129130
}
130131

131-
func read(t *testing.T, where string) *ach.File {
132+
func Benchmark_chunkFilesTogether(b *testing.B) {
133+
// Advice: Run this with "-benchtime 100x" to specify a fixed number of iterations
134+
b.StopTimer()
135+
136+
example := read(b, filepath.Join("testdata", "ppd-debit.ach"))
137+
initialTraceNumber, err := strconv.ParseInt(example.Batches[0].GetEntries()[0].TraceNumber, 10, 64)
138+
if err != nil {
139+
b.Fatal(err)
140+
}
141+
142+
// Setup pending directory
143+
dir := b.TempDir()
144+
iterations := b.N * 10_000
145+
groupSize := 250
146+
indices := makeIndices(iterations, iterations/groupSize)
147+
var matches []string
148+
for i := 0; i < iterations; i++ {
149+
file := ach.NewFile()
150+
file.Header = example.Header
151+
152+
exampleBatch := example.Batches[0]
153+
batch, err := ach.NewBatch(exampleBatch.GetHeader())
154+
require.NoError(b, err)
155+
entry := *exampleBatch.GetEntries()[0]
156+
entry.SetTraceNumber(exampleBatch.GetHeader().ODFIIdentification, int(initialTraceNumber)+i)
157+
batch.AddEntry(&entry)
158+
require.NoError(b, batch.Create())
159+
160+
file.AddBatch(batch)
161+
require.NoError(b, file.Create())
162+
163+
var buf bytes.Buffer
164+
err = ach.NewWriter(&buf).Write(file)
165+
require.NoError(b, err)
166+
167+
filename := fmt.Sprintf("%d.ach", i)
168+
matches = append(matches, filename)
169+
err = os.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0600)
170+
require.NoError(b, err)
171+
}
172+
b.Logf("merged %d files", iterations)
173+
174+
fs, err := storage.NewFilesystem(dir)
175+
require.NoError(b, err)
176+
177+
m := &filesystemMerging{
178+
logger: log.NewTestLogger(),
179+
storage: fs,
180+
}
181+
182+
b.Run("run", func(b *testing.B) {
183+
b.StartTimer()
184+
185+
conditions := ach.Conditions{
186+
MaxLines: 100_000,
187+
MaxDollarAmount: 50_000_000,
188+
}
189+
merged, err := m.chunkFilesTogether(context.Background(), indices, matches, conditions)
190+
require.NoError(b, err)
191+
b.Logf("%d files after merge", len(merged))
192+
})
193+
}
194+
195+
func read(t testing.TB, where string) *ach.File {
132196
t.Helper()
133197

134198
file, err := ach.ReadFile(where)

0 commit comments

Comments
 (0)