Merge branch 'feature/optimal-sort-order-improvment' into 'develop'

Improve sorting heuristics #2357

See merge request itv-backend/reindexer!2414

Author: reindexer-bot

.clang-format

@@ -154,6 +154,9 @@ LambdaBodyIndentation: Signature
 LineEnding:      DeriveLF
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
+Macros:
+  - RX_GET_WITHOUT_MUTEX_ANALYSIS=[]()
+  - RX_MUTEX_ACCESS=private
 MaxEmptyLinesToKeep: 1
 NamespaceIndentation: None
 ObjCBinPackProtocolList: Auto
@@ -234,6 +237,7 @@ StatementMacros:
   - Q_UNUSED
   - QT_REQUIRE_VERSION
 TabWidth:        4
+TypenameMacros: []
 UseTab:          Always
 VerilogBreakBetweenInstancePorts: true
 WhitespaceSensitiveMacros:

.github/workflows/install_gbench.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-git clone --branch v1.6.2 https://github.com/google/benchmark.git
-git clone --branch release-1.12.1 https://github.com/google/googletest.git benchmark/googletest
+git clone --branch v1.9.5 https://github.com/google/benchmark.git
+git clone --branch v1.17.0 https://github.com/google/googletest.git benchmark/googletest
 cd benchmark
 cmake -E make_directory "build"
 cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_TESTING=OFF ../

.github/workflows/install_grpc.sh

@@ -5,7 +5,7 @@ cd grpc
 git submodule update --init
 mkdir -p cmake/build
 cd cmake/build
-cmake -DCMAKE_BUILD_TYPE=Release DgRPC_INSTALL=ON ../..
+cmake -DCMAKE_BUILD_TYPE=Release -DgRPC_INSTALL=ON  -DgRPC_BUILD_TESTS=OFF ../..
 make -j4
 sudo make install
 sudo ldconfig

.github/workflows/install_gtest.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone --branch release-1.12.1 https://github.com/google/googletest.git
+git clone --branch v1.17.0 https://github.com/google/googletest.git
 cd googletest
 mkdir build
 cd build

.github/workflows/test.yml

@@ -6,39 +6,22 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        compiler: ["Visual Studio 17 2022"]
+        compiler: ['Visual Studio 17 2022', 'MinGW Makefiles']
       fail-fast: false
+    env:
+      EXTRA_FLAGS:
+        "${{ matrix.compiler == 'Visual Studio 17 2022' && '-Ax64' || '' }}"
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Build Reindexer
-        run: |
-          mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" .. -Ax64
-          cmake --build . --config Release
-          cmake --build . --config Release --target face
-          cmake --build . --config Release --target swagger
-          cpack
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/checkout@v6
+      - name: Install NSIS
+        uses: repolevedavaj/install-nsis@v1.1.0
         with:
-          name: windows-latest
-          path: build/*.exe
-          if-no-files-found: error
-
-  build-windows-2019:
-    runs-on: windows-2019
-    strategy:
-      matrix:
-        compiler: ["Visual Studio 16 2019", "MinGW Makefiles"]
-      fail-fast: false
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+          nsis-version: '3.11'
       - name: Build Reindexer
         run: |
           mkdir build && cd build
-          cmake -G "${{matrix.compiler}}" ..
+          cmake -G "${{matrix.compiler}}" -DBUILD_ANN_INDEXES=builtin .. $EXTRA_FLAGS
           cmake --build . --config Release
           cmake --build . --config Release --target face
           cmake --build . --config Release --target swagger
@@ -47,14 +30,12 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-15]
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
           - os: ubuntu-latest
             sanitizer: TSAN
-            cc: gcc-12
-            cxx: g++-12
       fail-fast: false
     runs-on: ${{matrix.os}}
     env:
@@ -64,7 +45,7 @@ jobs:
       SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Prepare Environment
         run: |
           ./.github/workflows/install_gtest.sh
@@ -82,7 +63,7 @@ jobs:
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
             if [[ $SANITIZER == 'ASAN' ]]; then
-              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_SHARDING=On ..
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_ASAN_MEMORY_HEAVY=On ..
             else
               cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
             fi
@@ -94,7 +75,7 @@ jobs:
           make -j4
           STRIP=/bin/true cpack
       - name: 'C++ tests'
-        if: ${{ matrix.os == 'macos-13' }}
+        if: ${{ matrix.os == 'macos-15' }}
         run: |
           echo "Running C++ directly in this job due to Action's problem with artifacts transition for macos-11 and macos-12 runners"
           cd build
@@ -105,7 +86,7 @@ jobs:
           rm -rf ./build/_CPack_Packages ./build/cpp_src/server/reindexer_server_resources
           tar -cvf artifacts.tar build/ bindings/builtin/builtin_posix.go bindings/builtinserver/builtinserver_posix.go dependencies.sh
       - name: Archive Artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: ${{matrix.os}}${{matrix.sanitizer}}
           path: artifacts.tar
@@ -114,7 +95,7 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-15]
         test: ['C++', 'GO']
         include:
           - os: ubuntu-latest
@@ -123,14 +104,12 @@ jobs:
           - os: ubuntu-latest
             sanitizer: ASAN
             test: 'GO'
-#         - os: ubuntu-latest
-#           sanitizer: TSAN
-#           cc: gcc-12
-#           cxx: g++-12
-#           test: 'C++'
+          #  os: ubuntu-latest
+          #  test: 'C++'
           - os: ubuntu-latest
             sanitizer: TSAN
             test: 'GO'
+
       fail-fast: false
     runs-on: ${{matrix.os}}
     needs: build
@@ -142,24 +121,24 @@ jobs:
       SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        uses: actions/checkout@v4
+        if: ${{ matrix.os != 'macos-15' || matrix.test == 'GO' }}
+        uses: actions/checkout@v6
       - name: Install Go
-        uses: actions/setup-go@v5
-        if: ${{ matrix.os == 'macos-13' && matrix.test == 'GO' }}
+        uses: actions/setup-go@v6
+        if: ${{ matrix.os == 'macos-15' && matrix.test == 'GO' }}
         with:
-          go-version: '1.22.x'
+          go-version: '1.26.x'
           check-latest: true
       - name: Download ${{matrix.os}}${{matrix.sanitizer}} Artifacts
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
-        uses: actions/download-artifact@v4
+        if: ${{ matrix.os != 'macos-15' || matrix.test == 'GO' }}
+        uses: actions/download-artifact@v8
         with:
           name: ${{matrix.os}}${{matrix.sanitizer}}
       - name: 'Untar Artifacts'
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-15' || matrix.test == 'GO' }}
         run: tar -xvf artifacts.tar
       - name: Prepare Environment
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-15' || matrix.test == 'GO' }}
         env:
           OS: ${{matrix.os}}
         run: |
@@ -174,7 +153,7 @@ jobs:
             ./.github/workflows/install_gtest_parallel.sh
           fi
       - name: Tests
-        if: ${{ matrix.os != 'macos-13' || matrix.test == 'GO' }}
+        if: ${{ matrix.os != 'macos-15' || matrix.test == 'GO' }}
         run: |
           if [[ $TEST == 'GO' ]]; then
             if [[ $SANITIZER == 'ASAN' ]]; then
@@ -183,7 +162,12 @@ jobs:
             if [[ -z "$SANITIZER" ]]; then
               go test -timeout 15m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
             else
-              go test -timeout 35m ./test/... -bench . -benchmem -benchtime 100ms -seedcount 50000
+              export TSAN_OPTIONS="halt_on_error=1 suppressions=$PWD/test/tsan/builtin.tsan.suppressions"
+              mkdir gotests
+              # Using preocompiled tests to get readable Thread sanitizer backtraces from builtin C++ library
+              go test ./test/... -c -o gotests -tags tiny_vectors
+              ./gotests/test.test -test.timeout 35m -test.bench . -test.benchmem -test.benchtime 100ms -seedcount 50000
+              ./gotests/ft.test -test.bench . -test.benchmem -test.benchtime 100ms -seedcount 50000
             fi
           else
             cd build
@@ -202,7 +186,7 @@ jobs:
       OS: ${{matrix.os}}
     steps:
       - name: Download ${{matrix.os}} Artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: ${{matrix.os}}
       - name: 'Untar Artifacts'
@@ -219,9 +203,9 @@ jobs:
         run: |
           cd build
           if [[ $OS == ubuntu* ]]; then
-            sudo dpkg -i reindexer-4-dev*.deb
+            sudo dpkg -i reindexer-dev*.deb
             sudo apt-get install -f
-            sudo dpkg -i reindexer-4-server*.deb
+            sudo dpkg -i reindexer-server*.deb
             sudo apt-get install -f
           else
             for f in reindexer-*.tar.gz; do tar -xvzf "$f"; done
@@ -236,7 +220,7 @@ jobs:
             cp ./usr/local/etc/reindexer.conf.pkg /usr/local/etc/reindexer.conf.pkg
           fi
       - name: Clone PyReindexer
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           repository: restream/reindexer-py
       - name: Install PyReindexer
@@ -247,4 +231,3 @@ jobs:
         run: |
           cd pyreindexer
           ../.github/workflows/test.sh
-

.gitignore

@@ -1,5 +1,6 @@
 .idea/
 .vscode/ipch
+.cursor/
 reindexer.cbp
 reindexer.iml
 .DS_Store
@@ -19,8 +20,8 @@ __pycache__
 *.so
 .pytest_cache/
 qa_tests/logs/
-qa_tests/logs/
 qa_tests/venv/
+qa_tests/.venv/
 
 #ignore directory with compile commands
 .cache

CMakeLists.txt

@@ -1,10 +1,23 @@
-cmake_minimum_required(VERSION 3.10..3.13)
+cmake_minimum_required(VERSION 3.18)
 
 project(reindexer)
 
 enable_testing()
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_PCH_WARN_INVALID OFF)
 set(REINDEXER_SOURCE_PATH ${PROJECT_SOURCE_DIR}/cpp_src)
+# Do not scan for C++ modules to save some overhead
+set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
 
 add_subdirectory(cpp_src)
+
+# uninstall target
+if(NOT TARGET uninstall)
+  configure_file(
+    "${PROJECT_SOURCE_DIR}/cpp_src/cmake/cmake_uninstall.cmake.in"
+    "${PROJECT_BINARY_DIR}/cmake_uninstall.cmake"
+    IMMEDIATE @ONLY)
+
+  add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${PROJECT_BINARY_DIR}/cmake_uninstall.cmake)
+endif()