[upd] Update pdqsort (fixes potential overflow)

Author: reindexer-bot

.github/workflows/test.yml

@@ -6,39 +6,35 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        compiler: ["Visual Studio 17 2022"]
+        compiler: ["Visual Studio 17 2022", "MinGW Makefiles"]
       fail-fast: false
+    env:
+      EXTRA_FLAGS: "${{ matrix.compiler == 'Visual Studio 17 2022' && '-Ax64' || '' }}"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" .. -Ax64
+          cmake -G "${{matrix.compiler}}" -DBUILD_ANN_INDEXES=builtin .. $EXTRA_FLAGS
           cmake --build . --config Release
           cmake --build . --config Release --target face
           cmake --build . --config Release --target swagger
           cpack
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: windows-latest
-          path: build/*.exe
-          if-no-files-found: error
 
   build-windows-2019:
     runs-on: windows-2019
     strategy:
       matrix:
-        compiler: ["Visual Studio 16 2019", "MinGW Makefiles"]
+        compiler: ["Visual Studio 16 2019"]
       fail-fast: false
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Reindexer
         run: |
           mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" ..
+          cmake -G "${{matrix.compiler}}" -DBUILD_ANN_INDEXES=none ..
           cmake --build . --config Release
           cmake --build . --config Release --target face
           cmake --build . --config Release --target swagger
@@ -82,7 +78,7 @@ jobs:
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
             if [[ $SANITIZER == 'ASAN' ]]; then
-              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_SHARDING=On ..
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_ASAN_MEMORY_HEAVY=On ..
             else
               cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
             fi
@@ -124,9 +120,6 @@ jobs:
             sanitizer: ASAN
             test: 'GO'
 #         - os: ubuntu-latest
-#           sanitizer: TSAN
-#           cc: gcc-12
-#           cxx: g++-12
 #           test: 'C++'
           - os: ubuntu-latest
             sanitizer: TSAN
@@ -183,7 +176,12 @@ jobs:
             if [[ -z "$SANITIZER" ]]; then
               go test -timeout 15m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
             else
-              go test -timeout 35m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
+              export TSAN_OPTIONS="halt_on_error=1 suppressions=$PWD/test/tsan/builtin.tsan.suppressions"
+              mkdir gotests
+              # Using preocompiled tests to get readable Thread sanitizer backtraces from builtin C++ library
+              go test ./test/... -c -o gotests -tags tiny_vectors
+              ./gotests/test.test -test.timeout 35m -test.bench . -test.benchmem -test.benchtime 100ms -seedcount 50000
+              ./gotests/ft.test -test.bench . -test.benchmem -test.benchtime 100ms -seedcount 50000
             fi
           else
             cd build
@@ -219,9 +217,9 @@ jobs:
         run: |
           cd build
           if [[ $OS == ubuntu* ]]; then
-            sudo dpkg -i reindexer-4-dev*.deb
+            sudo dpkg -i reindexer-dev*.deb
             sudo apt-get install -f
-            sudo dpkg -i reindexer-4-server*.deb
+            sudo dpkg -i reindexer-server*.deb
             sudo apt-get install -f
           else
             for f in reindexer-*.tar.gz; do tar -xvzf "$f"; done

.gitignore

@@ -19,8 +19,8 @@ __pycache__
 *.so
 .pytest_cache/
 qa_tests/logs/
-qa_tests/logs/
 qa_tests/venv/
+qa_tests/.venv/
 
 #ignore directory with compile commands
 .cache

CMakeLists.txt

@@ -1,10 +1,11 @@
-cmake_minimum_required(VERSION 3.10..3.13)
+cmake_minimum_required(VERSION 3.18)
 
 project(reindexer)
 
 enable_testing()
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_PCH_WARN_INVALID OFF)
 set(REINDEXER_SOURCE_PATH ${PROJECT_SOURCE_DIR}/cpp_src)
 
 add_subdirectory(cpp_src)

appveyor.yml

@@ -11,9 +11,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	otelattr "go.opentelemetry.io/otel/attribute"
 
-	"github.com/restream/reindexer/v4/bindings"
-	"github.com/restream/reindexer/v4/bindings/builtinserver/config"
-	"github.com/restream/reindexer/v4/cjson"
+	"github.com/restream/reindexer/v5/bindings"
+	"github.com/restream/reindexer/v5/bindings/builtinserver/config"
+	"github.com/restream/reindexer/v5/cjson"
 )
 
 const (
@@ -32,6 +32,10 @@ func (db *reindexerImpl) modifyItem(ctx context.Context, namespace string, ns *r
 		}
 	}
 
+	if item == nil {
+		return 0, fmt.Errorf("rq: nil value in item modify call for '%s' namespace", namespace)
+	}
+
 	for tryCount := 0; tryCount < 2; tryCount++ {
 		ser := cjson.NewPoolSerializer()
 		defer ser.Close()

bindings.go

@@ -18,8 +18,8 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/restream/reindexer/v4/bindings"
-	"github.com/restream/reindexer/v4/cjson"
+	"github.com/restream/reindexer/v5/bindings"
+	"github.com/restream/reindexer/v5/cjson"
 )
 
 const defCgoLimit = 2000
@@ -250,12 +250,13 @@ func (binding *Builtin) Init(u []url.URL, eh bindings.EventsHandler, options ...
 	caps := *bindings.DefaultBindingCapabilities().
 		WithResultsWithShardIDs(true).
 		WithQrIdleTimeouts(true).
-		WithIncarnationTags(true)
+		WithIncarnationTags(true).
+		WithFloatRank(true)
 	ccaps := C.BindingCapabilities{
 		caps: C.int64_t(caps.Value),
 	}
 
-	return err2go(C.reindexer_connect_v4(binding.rx, str2c(u[0].Host+u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
+	return err2go(C.reindexer_connect(binding.rx, str2c(u[0].Host+u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
 }
 
 func (binding *Builtin) StartWatchOnCtx(ctx context.Context) (CCtxWrapper, error) {
@@ -677,6 +678,7 @@ func (binding *Builtin) Finalize() error {
 	if binding.eventsHandler != nil && binding.rx != 0 {
 		binding.eventsHandler.Unsubscribe(binding.rx)
 	}
+	bufFree.free_buffers_sync()
 	C.destroy_reindexer(binding.rx)
 	binding.rx = 0
 	if binding.cgoLimiterStat != nil {
@@ -716,6 +718,10 @@ func (binding *Builtin) Unsubscribe(ctx context.Context) error {
 	return binding.eventsHandler.Unsubscribe(binding.rx)
 }
 
+func (binding *Builtin) DBMSVersion() (string, error) {
+	return C.GoString(C.reindexer_version()), nil
+}
+
 func newBufFreeBatcher() (bf *bufFreeBatcher) {
 	bf = &bufFreeBatcher{
 		bufs:   make([]*RawCBuffer, 0, 100),
@@ -732,35 +738,46 @@ type bufFreeBatcher struct {
 	cbufs  []C.reindexer_resbuffer
 	lock   sync.Mutex
 	kickCh chan struct{}
-}
 
-func (bf *bufFreeBatcher) loop() {
-	for {
-		<-bf.kickCh
+	rxTerminationLock sync.Mutex
+}
 
-		bf.lock.Lock()
-		if len(bf.bufs) == 0 {
-			bf.lock.Unlock()
-			continue
-		}
-		bf.bufs, bf.bufs2 = bf.bufs2, bf.bufs
+func (bf *bufFreeBatcher) free_buffers_impl() {
+	bf.rxTerminationLock.Lock()
+	defer bf.rxTerminationLock.Unlock()
+	bf.lock.Lock()
+	if len(bf.bufs) == 0 {
 		bf.lock.Unlock()
+		return
+	}
+	bf.bufs, bf.bufs2 = bf.bufs2, bf.bufs
+	bf.lock.Unlock()
 
-		for _, buf := range bf.bufs2 {
-			bf.cbufs = append(bf.cbufs, buf.cbuf)
-		}
+	for _, buf := range bf.bufs2 {
+		bf.cbufs = append(bf.cbufs, buf.cbuf)
+	}
 
-		C.reindexer_free_buffers(&bf.cbufs[0], C.int(len(bf.cbufs)))
+	C.reindexer_free_buffers(&bf.cbufs[0], C.int(len(bf.cbufs)))
 
-		for _, buf := range bf.bufs2 {
-			buf.cbuf.results_ptr = 0
-			bf.toPool(buf)
-		}
-		bf.cbufs = bf.cbufs[:0]
-		bf.bufs2 = bf.bufs2[:0]
+	for _, buf := range bf.bufs2 {
+		buf.cbuf.results_ptr = 0
+		bf.toPool(buf)
+	}
+	bf.cbufs = bf.cbufs[:0]
+	bf.bufs2 = bf.bufs2[:0]
+}
+
+func (bf *bufFreeBatcher) loop() {
+	for {
+		<-bf.kickCh
+		bf.free_buffers_impl()
 	}
 }
 
+func (bf *bufFreeBatcher) free_buffers_sync() {
+	bf.free_buffers_impl()
+}
+
 func (bf *bufFreeBatcher) add(buf *RawCBuffer) {
 	if buf.cbuf.results_ptr != 0 {
 		bf.toFree(buf)

bindings/builtin/builtin.go

@@ -4,7 +4,7 @@
 package builtin
 
 // #cgo pkg-config: libreindexer
-// #cgo CXXFLAGS: -std=c++17 -g -O2 -Wall -Wpedantic -Wextra
+// #cgo CXXFLAGS: -std=c++20 -g -O2 -Wall -Wpedantic -Wextra
 // #cgo CFLAGS: -std=c99 -g -O2 -Wall -Wpedantic -Wno-unused-variable
 // #cgo LDFLAGS:  -g
 import "C"

bindings/builtin/builtin_posix.go

@@ -3,7 +3,7 @@
 
 package builtin
 
-// #cgo CXXFLAGS: -std=c++17 -g -O2 -Wall -Wpedantic -Wextra -I../../cpp_src
+// #cgo CXXFLAGS: -std=c++20 -g -O2 -Wall -Wpedantic -Wextra -I../../cpp_src
 // #cgo CFLAGS: -std=c99 -g -O2 -Wall -Wpedantic -Wno-unused-variable -I../../cpp_src
-// #cgo LDFLAGS: -L${SRCDIR}/../../build/cpp_src/ -lreindexer -lleveldb -lsnappy -g -lshlwapi -ldbghelp -lws2_32
+// #cgo LDFLAGS: -L${SRCDIR}/../../build/cpp_src/ -lreindexer -lleveldb -lsnappy -g -lgomp -lshlwapi -ldbghelp -lws2_32
 import "C"

bindings/builtin/builtin_windows.go

@@ -12,7 +12,7 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/restream/reindexer/v4/bindings"
+	"github.com/restream/reindexer/v5/bindings"
 )
 
 type CGOEventsHandler struct {

bindings/builtin/cgoeventshandler.go

@@ -3,7 +3,7 @@
 
 package builtin
 
-// #cgo CXXFLAGS: -std=c++17 -g -O2 -Wall -Wpedantic -Wextra @cgo_cxx_flags@
+// #cgo CXXFLAGS: -std=c++20 -g -O2 -Wall -Wpedantic -Wextra @cgo_cxx_flags@
 // #cgo CFLAGS: -std=c99 -g -O2 -Wall -Wpedantic -Wno-unused-variable @cgo_c_flags@
 // #cgo LDFLAGS: @cgo_ld_flags@ -g
 import "C"