Skip to content

Commit 6c3c0b8

Browse files
darwin67darwin67
committed
ci: reduce heavy PR check time
1 parent 16025a8 commit 6c3c0b8

8 files changed

Lines changed: 163 additions & 34 deletions

File tree

‎.github/scripts/ci-changes.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
set -euo pipefail
5+
6+
if [[ "${GITHUB_EVENT_NAME:-}" == "workflow_dispatch" ]]; then
7+
cpp=true
8+
rust=true
9+
else
10+
base_ref="${GITHUB_BASE_REF:-main}"
11+
if ! git rev-parse --verify --quiet "origin/$base_ref" >/dev/null; then
12+
git fetch --no-tags --depth=1 origin "$base_ref"
13+
fi
14+
15+
cpp=false
16+
rust=false
17+
while IFS= read -r file; do
18+
case "$file" in
19+
*.c|*.cc|*.cpp|*.h|*.hh|CMakeLists.txt|*/CMakeLists.txt|cmake/*|configure.py|.gitmodules|flake.nix|flake.lock|Makefile|.github/workflows/pr-builds.yml|.github/workflows/build-ssdb.yaml|.github/scripts/ci-changes.sh)
20+
cpp=true
21+
;;
22+
esac
23+
24+
case "$file" in
25+
rust/*|Cargo.toml|Cargo.lock|flake.nix|flake.lock|Makefile|.github/workflows/pr-builds.yml|.github/scripts/ci-changes.sh)
26+
rust=true
27+
;;
28+
esac
29+
done < <(git diff --name-only --diff-filter=ACMR "origin/$base_ref"...HEAD)
30+
fi
31+
32+
{
33+
echo "cpp=$cpp"
34+
echo "rust=$rust"
35+
} >> "$GITHUB_OUTPUT"

‎.github/scripts/clang-tidy-changed.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
set -euo pipefail
5+
6+
base_ref="${GITHUB_BASE_REF:-main}"
7+
build_dir="${BUILD_DIR:-build}"
8+
checks="${CLANG_TIDY_CHECKS:--*,bugprone-use-after-move}"
9+
10+
if ! git rev-parse --verify --quiet "origin/$base_ref" >/dev/null; then
11+
git fetch --no-tags --depth=1 origin "$base_ref"
12+
fi
13+
14+
mapfile -t sources < <(
15+
git diff --name-only --diff-filter=ACMR "origin/$base_ref"...HEAD -- \
16+
'*.c' '*.cc' '*.cpp' \
17+
| sort
18+
)
19+
20+
if [[ "${#sources[@]}" -eq 0 ]]; then
21+
echo "No changed C/C++ translation units; skipping clang-tidy."
22+
exit 0
23+
fi
24+
25+
printf 'Running clang-tidy for %d changed translation unit(s):\n' "${#sources[@]}"
26+
printf ' %s\n' "${sources[@]}"
27+
28+
for source in "${sources[@]}"; do
29+
if [[ -f "$source" ]]; then
30+
.github/scripts/clang-tidy-nix.sh --checks="$checks" -p "$build_dir" "$source"
31+
fi
32+
done

‎.github/workflows/build-ssdb.yaml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ on:
1212
type: boolean
1313
default: true
1414
run_smoke_tests:
15-
description: 'run a focused inherited test.py smoke suite after building'
15+
description: 'run a focused inherited test binary after building'
1616
type: boolean
1717
default: false
1818
outputs:
@@ -24,7 +24,7 @@ jobs:
2424
build:
2525
runs-on: depot-ubuntu-24.04-8
2626
env:
27-
CCACHE_MAXSIZE: 2G
27+
CCACHE_MAXSIZE: 5G
2828
CCACHE_BASEDIR: ${{ github.workspace }}
2929
CCACHE_NOHASHDIR: true
3030
NODISTCC: true
@@ -35,6 +35,7 @@ jobs:
3535
with:
3636
submodules: recursive
3737
- uses: DeterminateSystems/nix-installer-action@v3
38+
- uses: DeterminateSystems/magic-nix-cache-action@main
3839
- name: Restore ccache
3940
if: ${{ inputs.enable_ccache }}
4041
uses: actions/cache@v4
@@ -71,7 +72,17 @@ jobs:
7172
- name: Run inherited smoke tests
7273
if: ${{ inputs.run_smoke_tests }}
7374
run: |
74-
nix develop .#cpp -c ./test.py --mode ${{ inputs.build_mode }} --jobs 1 --timeout 900 --tmpdir testlog boost/UUID_test
75+
nix develop .#cpp -c build/${{ inputs.build_mode }}/test/boost/UUID_test \
76+
--report_level=no \
77+
--catch_system_errors=no \
78+
--color_output=false \
79+
-- \
80+
--overprovisioned \
81+
--unsafe-bypass-fsync 1 \
82+
--kernel-page-cache 1 \
83+
--blocked-reactor-notify-ms 2000000 \
84+
--collectd 0 \
85+
--max-networking-io-control-blocks=100
7586
- name: Upload inherited test logs
7687
if: ${{ always() && inputs.run_smoke_tests }}
7788
uses: actions/upload-artifact@v4

‎.github/workflows/clang-tidy.yaml‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ on:
1616
- 'configure.py'
1717
- '.gitmodules'
1818
- '.github/workflows/clang-tidy.yaml'
19+
- '.github/scripts/clang-tidy-*.sh'
1920
- 'flake.nix'
2021
- 'flake.lock'
2122
workflow_dispatch:
@@ -25,7 +26,8 @@ env:
2526
BUILD_DIR: build
2627
CLANG_TIDY_CHECKS: '-*,bugprone-use-after-move'
2728

28-
permissions: {}
29+
permissions:
30+
contents: read
2931

3032
# cancel the in-progress run upon a repush
3133
concurrency:
@@ -38,15 +40,17 @@ jobs:
3840
runs-on: depot-ubuntu-24.04-8
3941
env:
4042
CCACHE_DIR: ${{ github.workspace }}/.ccache
41-
CCACHE_MAXSIZE: 2G
43+
CCACHE_MAXSIZE: 5G
4244
CCACHE_BASEDIR: ${{ github.workspace }}
4345
CCACHE_NOHASHDIR: true
4446
NODISTCC: true
4547
steps:
4648
- uses: actions/checkout@v4
4749
with:
50+
fetch-depth: 0
4851
submodules: true
4952
- uses: DeterminateSystems/nix-installer-action@v3
53+
- uses: DeterminateSystems/magic-nix-cache-action@main
5054
- name: Restore ccache
5155
uses: actions/cache@v4
5256
with:
@@ -62,29 +66,64 @@ jobs:
6266
nix develop .#cpp -c ccache --max-size="$CCACHE_MAXSIZE"
6367
nix develop .#cpp -c ccache --zero-stats
6468
- name: Generate the building system
69+
env:
70+
GITHUB_BASE_REF: ${{ github.base_ref || 'main' }}
6571
run: |
66-
nix develop .#cpp -c cmake \
67-
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
68-
-DCMAKE_C_COMPILER=clang \
69-
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
70-
-DCMAKE_CXX_FLAGS="-Wno-error=deprecated-builtins" \
71-
-DScylla_USE_LINKER=ld.lld \
72-
-DScylla_SEASTAR_UNUSED_RESULT_ERROR=OFF \
73-
-DCMAKE_CXX_COMPILER=clang++ \
74-
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
75-
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
76-
-DCMAKE_CXX_CLANG_TIDY="bash;$GITHUB_WORKSPACE/.github/scripts/clang-tidy-nix.sh;--checks=$CLANG_TIDY_CHECKS" \
77-
-G Ninja \
78-
-B $BUILD_DIR \
72+
cmake_args=(
73+
-DCMAKE_BUILD_TYPE=$BUILD_TYPE
74+
-DCMAKE_C_COMPILER=clang
75+
-DCMAKE_C_COMPILER_LAUNCHER=ccache
76+
-DCMAKE_CXX_FLAGS="-Wno-error=deprecated-builtins"
77+
-DScylla_USE_LINKER=ld.lld
78+
-DScylla_SEASTAR_UNUSED_RESULT_ERROR=OFF
79+
-DCMAKE_CXX_COMPILER=clang++
80+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
81+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
82+
-G Ninja
83+
-B "$BUILD_DIR"
7984
-S .
85+
)
86+
if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then
87+
cmake_args+=(
88+
-DCMAKE_CXX_CLANG_TIDY="bash;$GITHUB_WORKSPACE/.github/scripts/clang-tidy-nix.sh;--checks=$CLANG_TIDY_CHECKS"
89+
)
90+
fi
91+
nix develop .#cpp -c cmake "${cmake_args[@]}"
92+
- name: Build generated headers
93+
if: github.event_name == 'pull_request'
94+
run: |
95+
nix develop .#cpp -c bash -c '
96+
swagger_targets=""
97+
for f in api/api-doc/*.json; do
98+
if test "${f#*.}" = json; then
99+
name=$(basename "$f" .json)
100+
if test $name != swagger20_header; then
101+
swagger_targets+=" scylla_swagger_gen_$name"
102+
fi
103+
fi
104+
done
105+
cmake \
106+
--build build \
107+
--target seastar_http_request_parser \
108+
--target idl-sources \
109+
--target $swagger_targets
110+
'
80111
# see https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md
81112
- run: |
82113
echo "::add-matcher::.github/clang-matcher.json"
114+
- name: Run clang-tidy on changed sources
115+
if: github.event_name == 'pull_request'
116+
env:
117+
GITHUB_BASE_REF: ${{ github.base_ref || 'main' }}
118+
run: |
119+
nix develop .#cpp -c .github/scripts/clang-tidy-changed.sh
83120
- name: Build with clang-tidy enabled
121+
if: github.event_name == 'workflow_dispatch'
84122
run: |
85123
nix develop .#cpp -c cmake --build $BUILD_DIR --target scylla
86124
- run: |
87125
echo "::remove-matcher owner=clang::"
126+
if: always()
88127
- name: Show ccache stats
89128
if: ${{ always() }}
90129
run: |

‎.github/workflows/iwyu.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,15 @@ jobs:
3838
runs-on: depot-ubuntu-24.04-8
3939
env:
4040
CCACHE_DIR: ${{ github.workspace }}/.ccache
41-
CCACHE_MAXSIZE: 2G
41+
CCACHE_MAXSIZE: 5G
4242
CCACHE_BASEDIR: ${{ github.workspace }}
4343
CCACHE_NOHASHDIR: true
4444
steps:
4545
- uses: actions/checkout@v4
4646
with:
4747
submodules: true
4848
- uses: DeterminateSystems/nix-installer-action@v3
49+
- uses: DeterminateSystems/magic-nix-cache-action@main
4950
- name: Restore ccache
5051
uses: actions/cache@v4
5152
with:

‎.github/workflows/pr-builds.yml‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,40 @@ concurrency:
1616
cancel-in-progress: true
1717

1818
jobs:
19-
cxx-configure:
20-
name: C++ configure smoke
19+
changes:
20+
name: Detect changed areas
2121
runs-on: depot-ubuntu-24.04
22-
timeout-minutes: 30
22+
outputs:
23+
cpp: ${{ steps.filter.outputs.cpp }}
24+
rust: ${{ steps.filter.outputs.rust }}
2325
steps:
2426
- uses: actions/checkout@v4
2527
with:
26-
submodules: recursive
27-
- uses: DeterminateSystems/nix-installer-action@v3
28-
- name: Configure inherited C++ build
29-
run: make cpp-configure
28+
fetch-depth: 0
29+
- id: filter
30+
env:
31+
GITHUB_BASE_REF: ${{ github.base_ref || 'main' }}
32+
run: .github/scripts/ci-changes.sh
3033

3134
ssdb-build:
3235
name: Build ssdb
36+
needs: changes
37+
if: needs.changes.outputs.cpp == 'true'
3338
uses: ./.github/workflows/build-ssdb.yaml
3439
with:
3540
build_mode: dev
3641
run_smoke_tests: true
3742

3843
rust:
3944
name: Rust checks and tests
45+
needs: changes
46+
if: needs.changes.outputs.rust == 'true'
4047
runs-on: depot-ubuntu-24.04
4148
timeout-minutes: 45
4249
steps:
4350
- uses: actions/checkout@v4
4451
- uses: DeterminateSystems/nix-installer-action@v3
52+
- uses: DeterminateSystems/magic-nix-cache-action@main
4553
- name: Check Rust formatting
4654
run: make rust-fmt-check
4755
- name: Check Rust workspace

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,10 @@ cpp-configure: ## Configure the inherited C++ build inside the Linux C++ shell.
5959
$(NIX) develop .#cpp -c ./configure.py --mode dev --with scylla --disable-dpdk
6060

6161
.PHONY: cpp-test-smoke
62-
cpp-test-smoke: ## Build and run the focused inherited C++ test.py smoke suite.
62+
cpp-test-smoke: ## Build and run the focused inherited C++ smoke test binary.
6363
$(NIX) develop .#cpp -c ./configure.py --mode $(CPP_TEST_MODE) --with scylla $(addprefix --with ,$(CPP_TEST_SMOKE_ARTIFACTS)) --disable-dpdk --no-seastar-unused-result-error
6464
$(NIX) develop .#cpp -c ninja build/$(CPP_TEST_MODE)/scylla $(CPP_TEST_SMOKE_TARGETS)
65-
$(NIX) develop .#cpp -c ./test.py --mode $(CPP_TEST_MODE) --jobs 1 --timeout 900 --tmpdir testlog $(CPP_TEST_SMOKE)
65+
$(NIX) develop .#cpp -c build/$(CPP_TEST_MODE)/test/boost/UUID_test --report_level=no --catch_system_errors=no --color_output=false -- --overprovisioned --unsafe-bypass-fsync 1 --kernel-page-cache 1 --blocked-reactor-notify-ms 2000000 --collectd 0 --max-networking-io-control-blocks=100
6666

6767
.PHONY: inventory
6868
inventory: ## Regenerate relicensing inventory files.

‎docs/ci/workflow-audit.md‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,17 @@ changes, and which should be removed.
2020

2121
| Workflow | File | Category | Current role | Audit note |
2222
|-----------------------------------|------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------|
23-
| ssdb Build and Test | `.github/workflows/pr-builds.yml` | Core PR CI | Installs Nix, validates the flake, runs the C++ configure smoke through the C++ shell, runs the reusable inherited binary build with a focused `test.py` smoke test, and runs Rust checks/tests through Makefile/Nix targets. | Keep as the fork's primary PR signal. Expand inherited test coverage and Rust replacement modules as they become testable. |
23+
| ssdb Build and Test | `.github/workflows/pr-builds.yml` | Core PR CI | Detects changed areas, runs the reusable inherited binary build with a focused inherited C++ test-binary smoke test for C++ changes, and runs Rust checks/tests through Makefile/Nix targets for Rust changes. | Keep as the fork's primary PR signal. Expand inherited test coverage and Rust replacement modules as they become testable. |
2424
| ssdb PR | `.github/workflows/ssdb-pr.yaml` | Core PR CI | Runs relicensing Phase 1 checks and prints relicensing status. | Keep until provenance checks are merged into broader CI. |
2525
| PR Conventional Commit Validation | `.github/workflows/commits.yml` | Core PR CI | Validates conventional PR titles without adding labels. | Keep for release-note hygiene. |
2626
| codespell | `.github/workflows/codespell.yaml` | Core PR CI | Warns on spelling issues. | Keep, but consider making it fail once inherited false positives are cleaned up. |
27-
| Build ssdb | `.github/workflows/build-ssdb.yaml` | Reusable CI | Installs Nix, enters the C++ shell, builds the inherited database executable for a requested mode, and can optionally run focused inherited `test.py` smoke tests. | Keep as reusable CI. Internal target paths still use `scylla` until the build tree is renamed. |
27+
| Build ssdb | `.github/workflows/build-ssdb.yaml` | Reusable CI | Installs Nix, enters the C++ shell, builds the inherited database executable for a requested mode, and can optionally run a focused inherited C++ test-binary smoke test. | Keep as reusable CI. Internal target paths still use `scylla` until the build tree is renamed. |
2828

2929
## Modify
3030

3131
| Workflow | File | Category | Current role | Recommended change |
3232
|-------------------------------|---------------------------------------------|-----------------------|-------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
33-
| clang-tidy | `.github/workflows/clang-tidy.yaml` | Core PR CI | Runs clang-tidy on C++ changes through the Nix C++ shell. | Keep path-filtered. Continue rebranding user-visible names only; CMake variables such as `Scylla_USE_LINKER` remain inherited build internals for now. |
33+
| clang-tidy | `.github/workflows/clang-tidy.yaml` | Core PR CI | Runs clang-tidy on changed C/C++ translation units for PRs and keeps the full clang-tidy build available by manual dispatch. | Keep path-filtered. Continue rebranding user-visible names only; CMake variables such as `Scylla_USE_LINKER` remain inherited build internals for now. |
3434
| iwyu | `.github/workflows/iwyu.yaml` | Core PR CI | Runs include-cleaner on C++ paths through the Nix C++ shell. | Keep path-filtered. It now validates submodule, flake, and workflow changes like clang-tidy. |
3535
| Check Reproducible Build | `.github/workflows/reproducible-build.yaml` | Heavy or scheduled CI | Builds twice through the reusable Nix-backed build and compares checksums. | Keep manual/scheduled. Consider reducing frequency until full binary rename and release process are settled. |
3636
| clang-nightly | `.github/workflows/clang-nightly.yaml` | Heavy or scheduled CI | Builds with a nightly Clang snapshot. | Keep manual/scheduled if compiler-forward compatibility matters. Rename workflow once the inherited C++ build has ssdb naming. |
@@ -56,10 +56,13 @@ changes, and which should be removed.
5656

5757
- Full inherited C++ builds and C++ source-analysis builds use
5858
`depot-ubuntu-24.04-8` so build parallelism has enough CPU.
59-
- Normal PR build, configure, Rust, and relicensing jobs install Nix before
59+
- Normal PR build, Rust, and relicensing jobs install Nix before
6060
running repository tooling so CI follows the same flake-backed shells used by
6161
local development.
62-
- Heavy non-reproducibility builds use `ccache` with GitHub Actions cache.
63-
Reproducible builds disable compiler caching to preserve a clean comparison.
62+
- Nix-backed PR jobs use Magic Nix Cache to reduce repeated shell materialization
63+
cost.
64+
- Heavy non-reproducibility builds use `ccache` with GitHub Actions cache and a
65+
5 GiB budget. Reproducible builds disable compiler caching to preserve a clean
66+
comparison.
6467
- Lightweight validation, relicensing, Rust, Nix, submodule, and reusable
6568
metadata jobs use `depot-ubuntu-24.04`.

0 commit comments

Comments
 (0)