Skip to content

Commit 4c6ed7d

Browse files
mrodmjsoriano
mrodm
and
jsoriano
authored
Add SQL storage indexer (beta/technical preview) (#1313)
Added new SQL storage indexer to store all the package information in a SQL database (currently SQLite) instead of memory. The Indexer interface has been updated to include a Close method, so database connections can be closed at the end. This new indexer makes use of two databases so updates can be performed faster. Updating the unused and empty database before doing the actual swap in the service. Moreover, it has been added a cache LRU for the search endpoint. Search responses are one of the most expensive requests for the EPR service, specially when there are several packages in the response. Even if there is a CDN in front of the service, this could help to improve performance. During this process, code that was private in public modules has been moved to internal modules too. --------- Co-authored-by: Jaime Soriano Pastor <jaime.soriano@elastic.co>
1 parent 31c4eb7 commit 4c6ed7d

46 files changed

Lines changed: 604568 additions & 245 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/scripts/test-epr-gcs.sh‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,31 @@ export FAKE_GCS_SERVER_VERSION
8888
export SEARCH_INDEX_PATH="../storage/testdata/search-index-all-full.json"
8989
NUMBER_OF_PACKAGES=$( jq -r '.packages | length' "${SEARCH_INDEX_PATH}" )
9090

91-
echo "--- Running EPR service with fakce GCS server running within EPR"
91+
echo "--- Running EPR service with fakce GCS server running within EPR - storage indexer enabled"
9292

93+
export ENABLE_STORAGE_INDEXER=true
94+
export ENABLE_SQL_STORAGE_INDEXER=false
9395
test_services "${DOCKER_COMPOSE_EPR_PATH}"
9496

97+
echo "--- Running EPR service with fakce GCS server running within EPR - SQL storage indexer enabled"
9598

96-
echo "--- Running EPR and fake GCS server as services"
99+
export ENABLE_STORAGE_INDEXER=false
100+
export ENABLE_SQL_STORAGE_INDEXER=true
101+
test_services "${DOCKER_COMPOSE_EPR_PATH}"
102+
103+
104+
echo "--- Running EPR and fake GCS server as services - storage indexer enabled"
105+
106+
./setup_bucket_gcs.sh -b "${BUCKET_NAME}" -c 1 -i ../storage/testdata/search-index-all-full.json -p "${LOCAL_BUCKET_PATH}"
107+
108+
export ENABLE_STORAGE_INDEXER=true
109+
export ENABLE_SQL_STORAGE_INDEXER=false
110+
test_services "${DOCKER_COMPOSE_EPR_GCS_PATH}"
111+
112+
echo "--- Running EPR and fake GCS server as services - SQL storage indexer enabled"
97113

98114
./setup_bucket_gcs.sh -b "${BUCKET_NAME}" -c 1 -i ../storage/testdata/search-index-all-full.json -p "${LOCAL_BUCKET_PATH}"
99115

116+
export ENABLE_STORAGE_INDEXER=false
117+
export ENABLE_SQL_STORAGE_INDEXER=true
100118
test_services "${DOCKER_COMPOSE_EPR_GCS_PATH}"

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1919

2020
* Change license from Elastic License to Elastic License 2.0. [#1298](https://github.com/elastic/package-registry/pull/1298)
2121
* Add APM spans for proxy requests and storage indexer Get calls. [#1306](https://github.com/elastic/package-registry/pull/1306)
22+
* Add SQL storage indexer in technical preview. [#1313](https://github.com/elastic/package-registry/pull/1313)
2223

2324
### Deprecated
2425

‎categories_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ func TestCategoriesWithProxyMode(t *testing.T) {
4040
defer webServer.Close()
4141

4242
indexerProxy := packages.NewFileSystemIndexer(testLogger, "./testdata/second_package_path")
43+
defer indexerProxy.Close(context.Background())
44+
4345
err := indexerProxy.Init(context.Background())
4446
require.NoError(t, err)
4547

‎dev/docker-compose-epr-gcs.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ services:
2121
environment:
2222
- STORAGE_EMULATOR_HOST=fakegcsserver:4443
2323
- EPR_STORAGE_INDEXER_BUCKET_INTERNAL=gs://${BUCKET_NAME:-fake-package-storage-internal}
24-
- EPR_FEATURE_STORAGE_INDEXER=true
24+
- EPR_FEATURE_STORAGE_INDEXER=${ENABLE_STORAGE_INDEXER:-false}
25+
- EPR_FEATURE_SQL_STORAGE_INDEXER=${ENABLE_SQL_STORAGE_INDEXER:-false}
2526
- EPR_DISABLE_PACKAGE_VALIDATION=true
2627
- EPR_ADDRESS=0.0.0.0:8080
2728
# - EPR_LOG_LEVEL=debug

‎dev/docker-compose-epr.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ services:
1414
- STORAGE_EMULATOR_HOST=localhost:4443
1515
- EPR_EMULATOR_INDEX_PATH=/testdata/search-index-all.json
1616
- EPR_STORAGE_INDEXER_BUCKET_INTERNAL=gs://fake-package-storage-internal
17-
- EPR_FEATURE_STORAGE_INDEXER=true
17+
- EPR_FEATURE_STORAGE_INDEXER=${ENABLE_STORAGE_INDEXER:-false}
18+
- EPR_FEATURE_SQL_STORAGE_INDEXER=${ENABLE_SQL_STORAGE_INDEXER:-false}
1819
- EPR_DISABLE_PACKAGE_VALIDATION=true
1920
- EPR_ADDRESS=0.0.0.0:8080
2021
# - EPR_LOG_LEVEL=debug

‎dev/launch_epr_service_storage_indexer.sh‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ CURRENT_DIR="$(pwd)"
1111
SCRIPT_DIR="$( cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
1212

1313
usage() {
14-
echo "$0 [-b <bucket_name>] [-p <epr_address>] [-e <emulator_address>] [-i <index_path>] [-c <config_path>] [-h]"
14+
echo "$0 [-b <bucket_name>] [-p <epr_address>] [-e <emulator_address>] [-i <index_path>] [-c <config_path>] [-s] [-C] [-h]"
1515
echo -e "\t-b <bucket_name>: Bucket name. Default: example"
1616
echo -e "\t-p <epr_address>: Address of the package registry service. Default: localhost:8080"
1717
echo -e "\t-e <emulator_address>: Address of the emulator host (fake GCS server). Default: localhost:4443"
1818
echo -e "\t-i <index_path>: Path to the search index JSON. Default: \"\""
1919
echo -e "\t\t\tIf set, the bucket name will be ignored (-b parameter) and Package Registry will use its default development bucket gs://fake-package-storage-internal"
2020
echo -e "\t-c <config_path>: Path to the configurastion file. Default: \"\""
21+
echo -e "\t-s : Enable SQL Storage indexer. By default Storage Indexer is enabled."
22+
echo -e "\t-C : Enable Search Cache. Just supported with SQL Storage indexer. By default Search Cache is disabled."
2123
echo -e "\t-h: Show this message"
2224
}
2325

@@ -26,8 +28,10 @@ ADDRESS="localhost:8080"
2628
INDEX_PATH=""
2729
EMULATOR_HOST="localhost:4443"
2830
CONFIG_PATH="${SCRIPT_DIR}/../config.yml"
31+
ENABLE_STORAGE_SQL_INDEXER=0
32+
ENABLE_SEARCH_CACHE=0
2933

30-
while getopts ":b:p:i:e:c:h" o; do
34+
while getopts ":b:p:i:e:c:sCh" o; do
3135
case "${o}" in
3236
b)
3337
BUCKET_NAME="${OPTARG}"
@@ -44,6 +48,12 @@ while getopts ":b:p:i:e:c:h" o; do
4448
c)
4549
CONFIG_PATH="${OPTARG}"
4650
;;
51+
s)
52+
ENABLE_STORAGE_SQL_INDEXER=1
53+
;;
54+
C)
55+
ENABLE_SEARCH_CACHE=1
56+
;;
4757
h)
4858
usage
4959
exit 0
@@ -78,7 +88,17 @@ else
7888
export EPR_STORAGE_INDEXER_BUCKET_INTERNAL="gs://${BUCKET_NAME}"
7989
fi
8090

81-
export EPR_FEATURE_STORAGE_INDEXER="true"
91+
if [[ "${ENABLE_STORAGE_SQL_INDEXER}" == 0 ]]; then
92+
export EPR_FEATURE_STORAGE_INDEXER="true"
93+
export EPR_FEATURE_SQL_STORAGE_INDEXER="false"
94+
else
95+
export EPR_FEATURE_SQL_STORAGE_INDEXER="true"
96+
export EPR_FEATURE_STORAGE_INDEXER="false"
97+
fi
98+
99+
if [[ "${ENABLE_SEARCH_CACHE}" == 1 ]]; then
100+
export EPR_FEATURE_ENABLE_SEARCH_CACHE="true"
101+
fi
82102

83103
export EPR_DISABLE_PACKAGE_VALIDATION="true"
84104
export EPR_ADDRESS="${ADDRESS}"

‎go.mod‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ require (
1313
github.com/fsouza/fake-gcs-server v1.52.2
1414
github.com/gorilla/mux v1.8.1
1515
github.com/hashicorp/go-retryablehttp v0.7.8
16+
github.com/hashicorp/golang-lru/v2 v2.0.7
1617
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
1718
github.com/magefile/mage v1.15.0
1819
github.com/prometheus/client_golang v1.22.0
@@ -26,6 +27,7 @@ require (
2627
google.golang.org/api v0.239.0
2728
gopkg.in/yaml.v2 v2.4.0
2829
honnef.co/go/tools v0.6.1
30+
modernc.org/sqlite v1.38.0
2931
)
3032

3133
require (
@@ -46,6 +48,7 @@ require (
4648
github.com/cespare/xxhash/v2 v2.3.0 // indirect
4749
github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect
4850
github.com/davecgh/go-spew v1.1.1 // indirect
51+
github.com/dustin/go-humanize v1.0.1 // indirect
4952
github.com/elastic/go-sysinfo v1.15.1 // indirect
5053
github.com/elastic/go-windows v1.0.2 // indirect
5154
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
@@ -60,14 +63,17 @@ require (
6063
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
6164
github.com/gorilla/handlers v1.5.2 // indirect
6265
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
66+
github.com/mattn/go-isatty v0.0.20 // indirect
6367
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
68+
github.com/ncruces/go-strftime v0.1.9 // indirect
6469
github.com/pkg/errors v0.9.1 // indirect
6570
github.com/pkg/xattr v0.4.10 // indirect
6671
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
6772
github.com/pmezard/go-difflib v1.0.0 // indirect
6873
github.com/prometheus/client_model v0.6.1 // indirect
6974
github.com/prometheus/common v0.63.0 // indirect
7075
github.com/prometheus/procfs v0.16.0 // indirect
76+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
7177
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
7278
github.com/zeebo/errs v1.4.0 // indirect
7379
go.elastic.co/apm/module/apmhttp/v2 v2.7.1 // indirect
@@ -84,6 +90,7 @@ require (
8490
go.opentelemetry.io/otel/trace v1.36.0 // indirect
8591
go.uber.org/multierr v1.11.0 // indirect
8692
golang.org/x/crypto v0.39.0 // indirect
93+
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
8794
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
8895
golang.org/x/mod v0.25.0 // indirect
8996
golang.org/x/net v0.41.0 // indirect
@@ -99,4 +106,7 @@ require (
99106
google.golang.org/protobuf v1.36.6 // indirect
100107
gopkg.in/yaml.v3 v3.0.1 // indirect
101108
howett.net/plist v1.0.1 // indirect
109+
modernc.org/libc v1.65.10 // indirect
110+
modernc.org/mathutil v1.7.1 // indirect
111+
modernc.org/memory v1.11.0 // indirect
102112
)

‎go.sum‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
117117
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
118118
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
119119
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
120+
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
121+
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
120122
github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
121123
github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
122124
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
@@ -138,6 +140,8 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1
138140
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
139141
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
140142
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
143+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
144+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
141145
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
142146
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
143147
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
@@ -167,6 +171,8 @@ github.com/minio/minio-go/v7 v7.0.86 h1:DcgQ0AUjLJzRH6y/HrxiZ8CXarA70PAIufXHodP4
167171
github.com/minio/minio-go/v7 v7.0.86/go.mod h1:VbfO4hYwUu3Of9WqGLBZ8vl3Hxnxo4ngxK4hzQDf4x4=
168172
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
169173
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
174+
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
175+
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
170176
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
171177
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
172178
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -185,6 +191,8 @@ github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA
185191
github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18=
186192
github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM=
187193
github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg=
194+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
195+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
188196
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
189197
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
190198
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
@@ -249,6 +257,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
249257
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
250258
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
251259
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
260+
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
261+
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
252262
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
253263
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
254264
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -277,6 +287,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
277287
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
278288
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
279289
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
290+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
280291
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
281292
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
282293
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -343,3 +354,27 @@ honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI=
343354
honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4=
344355
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
345356
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
357+
modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s=
358+
modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
359+
modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
360+
modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
361+
modernc.org/fileutil v1.3.3 h1:3qaU+7f7xxTUmvU1pJTZiDLAIoJVdUSSauJNHg9yXoA=
362+
modernc.org/fileutil v1.3.3/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
363+
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
364+
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
365+
modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc=
366+
modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po=
367+
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
368+
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
369+
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
370+
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
371+
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
372+
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
373+
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
374+
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
375+
modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI=
376+
modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE=
377+
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
378+
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
379+
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
380+
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

‎indexer.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
type Indexer interface {
1717
Init(context.Context) error
1818
Get(context.Context, *packages.GetOptions) (packages.Packages, error)
19+
Close(context.Context) error
1920
}
2021

2122
type CombinedIndexer []Indexer
@@ -51,6 +52,17 @@ func (c CombinedIndexer) Get(ctx context.Context, opts *packages.GetOptions) (pa
5152
return packages, nil
5253
}
5354

55+
func (c CombinedIndexer) Close(ctx context.Context) error {
56+
for _, indexer := range c {
57+
err := indexer.Close(ctx)
58+
if err != nil {
59+
return err
60+
}
61+
}
62+
63+
return nil
64+
}
65+
5466
func latestPackagesVersion(source packages.Packages) (result packages.Packages) {
5567
sort.Sort(byNameVersion(source))
5668

‎internal/database/memoryrepository.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License 2.0;
3+
// you may not use this file except in compliance with the Elastic License 2.0.
4+
5+
package database
6+
7+
import (
8+
"context"
9+
"database/sql"
10+
"fmt"
11+
12+
_ "modernc.org/sqlite" // Import the SQLite driver
13+
)
14+
15+
func NewMemorySQLDB(path string) (*SQLiteRepository, error) {
16+
db, err := sql.Open("sqlite", ":memory:")
17+
if err != nil {
18+
return nil, fmt.Errorf("failed to open database: %w", err)
19+
}
20+
21+
if err := db.Ping(); err != nil {
22+
return nil, fmt.Errorf("failed to connect to database: %w", err)
23+
}
24+
25+
dbRepo, err := newSQLiteRepository(db)
26+
if err != nil {
27+
return nil, fmt.Errorf("failed to create SQLite repository: %w", err)
28+
}
29+
30+
if err := dbRepo.Initialize(context.Background()); err != nil {
31+
return nil, fmt.Errorf("failed to create database: %w", err)
32+
}
33+
dbRepo.path = "memory-" + path
34+
return dbRepo, nil
35+
}

0 commit comments

Comments
 (0)