build.yaml

name: Tests

on:
  push:
    branches:
      - main
      - 'integration/**'
  pull_request:
    branches:
      - main
      - 'integration/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  get_dependencies:
    name: "Dependencies"
    uses: ./.github/workflows/dependencies.yaml
    with:
      cxx_standard: cpp23

  get_dependencies_cpp03:
    name: "Dependencies [C++03]"
    uses: ./.github/workflows/dependencies.yaml
    with:
      cxx_standard: cpp03

  build_ubuntu:
    name: "Build [ubuntu]"
    needs: get_dependencies
    uses: ./.github/workflows/build-ubuntu.yaml
    with:
      ref: ${{ github.sha }}
      target: "bmqbrkr bmqtool bmqstoragetool all.it"
      save_build_as_artifacts: true
      run_unit_tests: false
      cxx_standard: cpp23

  build_ubuntu_cpp03:
    name: "Build [ubuntu, C++03]"
    needs: get_dependencies_cpp03
    uses: ./.github/workflows/build-ubuntu.yaml
    with:
      ref: ${{ github.sha }}
      target: "bmqbrkr bmqtool bmq.t mqb.t"
      save_build_as_artifacts: false
      run_unit_tests: false
      cxx_standard: cpp03

  unit_tests_cxx:
    name: "UT [c++]"
    needs: get_dependencies
    uses: ./.github/workflows/build-ubuntu.yaml
    with:
      ref: ${{ github.sha }}
      target: "all.t"
      save_build_as_artifacts: false
      run_unit_tests: true
      cxx_standard: cpp23

  unit_tests_python:
    name: UT [python]
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v6

      - name: Run Python Unit Tests
        env:
          PYTHONPATH: ${{ github.workspace }}/src/python
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements-test.txt
          pip install "xsdata[cli]==25.7"
          cd ${{ github.workspace }}
          src/python/bin/schemagen
          src/python/bin/tweakgen
          pytest src/python

      - name: Print generated schemas
        if: failure()
        run: |
          cd ${{ github.workspace }}
          cat src/python/blazingmq/schemas/mqbcfg.py
          cat src/python/blazingmq/schemas/mqbconf.py
          cat src/python/blazingmq/dev/it/tweaks/generated.py

  integration_tests_ubuntu:
    name: IT [${{ matrix.cluster }}/${{ matrix.mode }}/${{ matrix.consistency }}]
    strategy:
      matrix:
        mode: ["legacy_mode", "fsm_mode"]
        cluster: ["single", "multi"]
        consistency: ["eventual_consistency", "strong_consistency", "not (eventual_consistency or strong_consistency)"]
      fail-fast: false
    runs-on: ubuntu-latest
    needs: build_ubuntu
    env:
        RUN_UID: ${{ matrix.cluster }}_${{ matrix.mode }}_${{ matrix.consistency }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          name: ${{needs.build_ubuntu.outputs.artifact_key }}

        # Solaris: make sure all ITs scripts are compatible with python3.8
      - uses: actions/setup-python@v6
        with:
          python-version: '3.8'

        # Set execution permissions for all .tsk
        # which were lost when the artifact was created
      - name: Chmod all .tsk
        run: find ./build -type f -name "*.tsk" -exec chmod +x {} +

      - name: Setup core_pattern
        run: |
          # Default Ubuntu core_pattern: '|/usr/lib/systemd/systemd-coredump %P %u %g %s %t 9223372036854775808 %h'
          # To enable core dumps copying to the failure-logs folder (implemented in src/python/blazingmq/dev/it/cluster.py)
          # the core_pattern must be an absolute path and must contain %p (i.e. process id)

          sudo mkdir /cores
          sudo chmod 777 /cores
          echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern

      - name: Run Integration Tests
        run: |
          # Allow core dumps
          ulimit -c unlimited

          # Pin tproxy of 27 Feb 2026:
          go install github.com/kevwan/tproxy@d6d45cbbd1b3c5dad6e5043b74120f9ea962826b
          export GOPATH=$HOME/go
          export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin

          pip install -r ${{ github.workspace }}/src/python/requirements-test.txt

          ${{ github.workspace }}/src/integration-tests/run-tests \
            "${{ matrix.mode }} and ${{ matrix.cluster }} and ${{ matrix.consistency }}" \
            --log-level ERROR                   \
            --log-file-level=info               \
            --bmq-tolerate-dirty-shutdown       \
            --bmq-log-dir=failure-logs          \
            --bmq-log-level=INFO                \
            --junitxml=integration-tests.xml    \
            --tb long                           \
            --durations=0                       \
            -n logical -v

      - name: Print core information
        if: failure()
        run: |
          sudo apt-get update && sudo apt-get install -qy gdb

          python3 ${{ github.workspace }}/.github/workflows/ext/print_cores.py \
          --cores-dir=${{ github.workspace }}/src/integration-tests/failure-logs \
          --build-dir=${{ github.workspace }}/build/blazingmq

      - name: Upload failure-logs as artifacts
        if: failure()
        uses: actions/upload-artifact@v5
        with:
          name: ${{ github.run_id }}_failure_logs_${{ env.RUN_UID }}
          path: ${{ github.workspace }}/src/integration-tests/failure-logs
          retention-days: 5

      - name: Upload broker executable as artifacts to debug the core
        if: failure()
        uses: actions/upload-artifact@v5
        with:
          name: ${{ github.run_id }}_bmqbrkr
          overwrite: true # broker binary is the same across all matrix runs
          path: ${{ github.workspace }}/build/blazingmq/src/applications/bmqbrkr/bmqbrkr.tsk
          retention-days: 5

  fuzz_tests_ubuntu:
    name: Fuzz test [${{ matrix.request }}]
    strategy:
      matrix:
        request: [
          "authentication",
          "negotiation_bypass_authentication",
          "negotiation",
          "open_queue",
          "configure_stream",
          "configure_queue_stream",
          "put",
          "confirm",
          "close_queue",
          "disconnect"
          ]
      fail-fast: false
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          name: ${{needs.build_ubuntu.outputs.artifact_key }}

        # Set execution permissions for all .tsk
        # which were lost when the artifact was created
      - name: Chmod all .tsk
        run: |
          find ./build -type f -name "*.tsk" -exec chmod +x {} +
          chmod +x ./build/blazingmq/src/applications/bmqbrkr/run

      - name: Setup core_pattern
        run: |
          sudo mkdir /cores
          sudo chmod 777 /cores
          echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern

      - name: Run Fuzz Test
        env:
          BLAZINGMQ_BUILD_DIR: ${{ github.workspace }}/build/blazingmq
          PYTHONPATH: ${{ github.workspace }}/src/python
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements-test.txt
          pytest "${{ github.workspace }}/src/fuzz-tests/test_fuzz.py::test_fuzz_request[${{ matrix.request }}]" --tb=short > ${{ matrix.request}}_fuzz.log

        # Do not print everything, GitHub Actions is not able to handle 100MB+ of logs.
        # Typically the error found with fuzzer is related to the latest requests and broker state.
        # If this is not enough, consider running the test on your local machine and get full logs.
      - name: Print Fuzz Log tail
        if: failure()
        run: |
          tail -n 2000 ${{ matrix.request }}_fuzz.log

      - name: Print core information
        if: failure()
        run: |
          sudo apt-get update && sudo apt-get install -qy gdb

          python3 ${{ github.workspace }}/.github/workflows/ext/print_cores.py \
          --cores-dir=/cores \
          --build-dir=${{ github.workspace }}/build/blazingmq

      - name: Upload fuzz-logs as artifacts
        if: failure()
        uses: actions/upload-artifact@v5
        with:
          name: ${{ matrix.request }}_fuzz.log
          path: ${{ github.workspace }}/${{ matrix.request }}_fuzz.log
          retention-days: 5

  sanitize_ubuntu:
    name: Sanitize
    uses: ./.github/workflows/sanitize.yaml
    needs: integration_tests_ubuntu
    if: ${{ !cancelled() }}

  integration_tests_storagetool_ubuntu:
    name: IT [Storage tool]
    runs-on: ubuntu-latest
    needs: build_ubuntu
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          name: ${{needs.build_ubuntu.outputs.artifact_key }}

        # Set execution permissions for all .tsk
        # which were lost when the artifact was created
      - name: Chmod all .tsk
        run: find ./build -type f -name "*.tsk" -exec chmod +x {} +

      - name: Run Storage Tool integration tests
        run: |
          pip install -r ${{ github.workspace }}/src/python/requirements-test.txt
          cd ${{ github.workspace }}/src/applications/bmqstoragetool/integration-tests
          ./run-tests

  build_macosx:
    name: Build [macosx_${{ matrix.arch }}]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        include:
          - os: macos-14
            arch: arm64
    steps:
      - uses: actions/checkout@v6

      - name: Set up dependencies
        run: |
          brew install \
            bison \
            flex \
            google-benchmark \
            googletest \
            python@3.10 \
            zlib

      - name: Build BlazingMQ
        env:
          CXXFLAGS: "-Werror -Wno-poison-system-directories"
        run: |
          echo "::add-matcher::.github/workflows/problem-matchers/cpp.json"
          bin/build-darwin.sh
          echo "::remove-matcher owner=cpp"

  build_and_test_bmqprometheus_plugin:
    name: "Build Prometheus plugin [ubuntu]"
    runs-on: ubuntu-latest
    needs: get_dependencies
    steps:
      - uses: actions/checkout@v6

      - uses: actions/cache/restore@v4
        with:
          path: |
            deps
            /opt/bb
          key: ${{ needs.get_dependencies.outputs.cache_key }}

      - name: Set up plugins dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy build-essential \
            gdb \
            curl \
            python3.10 \
            cmake \
            ninja-build \
            pkg-config \
            bison \
            libfl-dev \
            libbenchmark-dev \
            libgmock-dev \
            libgtest-dev \
            libz-dev \
            autoconf \
            libtool
      - name: Install cached non packaged dependencies
        working-directory: deps
        run: ../docker/build_deps.sh

      - name: Create dependency fetcher working directory
        run: mkdir -p deps

      - name: Fetch & Build non packaged plugins dependencies
        working-directory: deps
        run:  ${{ github.workspace }}/src/plugins/bmqprometheus/build_prometheus_deps.sh

      - name: Build plugins
        env:
          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig
        run: |
          cmake --preset gh-prometheus-ubuntu-latest-cpp23 -S .
          echo "::add-matcher::.github/workflows/problem-matchers/cpp-prometheus.json"
          cmake --build --preset gh-prometheus-ubuntu-latest-cpp23
          echo "::remove-matcher owner=cpp-prometheus"

      - name: Create prometheus dirs
        run:  mkdir -p prometheus_dir/pull prometheus_dir/push

      - name: Download Prometheus
        run:  curl -SL "https://github.com/prometheus/prometheus/releases/download/v2.45.1/prometheus-2.45.1.linux-amd64.tar.gz" | tar -xzC prometheus_dir/

      - name: Run Prometheus for "pull" mode
        run:  ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/.github/workflows/ext/prometheus_localhost_pull.yaml --web.listen-address=0.0.0.0:9090 --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/pull &

      - name: Run Prometheus for "push" mode
        run:  ./prometheus_dir/prometheus-2.45.1.linux-amd64/prometheus --config.file=${{ github.workspace }}/.github/workflows/ext/prometheus_localhost_push.yaml --web.listen-address=0.0.0.0:9092 --storage.tsdb.path=${{ github.workspace }}/prometheus_dir/push &

      - name: Run Pushgateway
        run: |
          # 30 Oct 2025, v1.11.2:
          docker pull prom/pushgateway@sha256:c835998819105d84d484918908df132807c78c05f3e0e649df0c0479c6780d98
          docker run -d -p 9091:9091 prom/pushgateway

      - name: Prometheus plugin IT[pull]
        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m pull --no-docker --prometheus-host localhost:9090

      - name: Prometheus plugin IT[push]
        run: ${{ github.workspace }}/src/plugins/bmqprometheus/tests/bmqprometheus_prometheusstatconsumer_test.py -p ${{ github.workspace }}/build/blazingmq -m push --no-docker --prometheus-host localhost:9092

  documentation:
    name: "Documentation"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Set up dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -qy doxygen

      - name: Set up output directory
        run: |
          mkdir -p docs/docs/apidocs

      - name: Build docs
        run: |
          echo "::add-matcher::.github/workflows/problem-matchers/doxygen.json"
          doxygen Doxyfile
          echo "::remove-matcher owner=doxygen"

  shellcheck:
    name: Shellcheck
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Run ShellCheck
        uses: ludeeus/action-shellcheck@00b27aa7cb85167568cb48a3838b75f4265f2bca  # 20 Jun 2024

  docker_build_ubuntu:
    name: "Docker [ubuntu]"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6

      - name: Docker Single-Node Workflow
        run: docker compose -f docker/single-node/docker-compose.yaml up --build -d

      - name: Docker Cluster Workflow
        run: docker compose -f docker/cluster/docker-compose.yaml up --build -d

      - name: Inspect Docker images
        run: docker image ls