Skip to content

Commit de9d6ad

Browse files
FrooodleFrooodle
authored
Add CI coverage summaries and aggregate JaCoCo report (Stirling-Tools#6451)
1 parent 2c0ebc2 commit de9d6ad

15 files changed

Lines changed: 1583 additions & 23 deletions

‎.github/workflows/backend-build.yml‎

Lines changed: 66 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
name: Backend build, format check, and coverage
22

33
# Reusable workflow called from build.yml. Runs the backend build matrix
4-
# (JDK 25 × spring-security on/off), Spotless formatting check, JUnit, and
4+
# (JDK 25 × every flavor), Spotless formatting check, JUnit, and
55
# posts Jacoco coverage to PRs.
6+
#
7+
# Flavor axis (maps to STIRLING_FLAVOR in settings.gradle):
8+
# core - DISABLE_ADDITIONAL_FEATURES=true, no proprietary, no saas
9+
# proprietary - default build, no saas
10+
# saas - proprietary + the saas subproject (build + JUnit only,
11+
# never any runtime/integration testing)
612
on:
713
workflow_call:
814

@@ -25,7 +31,7 @@ jobs:
2531
fail-fast: false
2632
matrix:
2733
jdk-version: [25]
28-
spring-security: [true, false]
34+
flavor: [core, proprietary, saas]
2935
steps:
3036
- name: Harden Runner
3137
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
@@ -58,7 +64,10 @@ jobs:
5864
- name: Install Task
5965
uses: go-task/setup-task@3be4020d41929789a01026e0e427a4321ce0ad44 # v2.0.0
6066
- name: Check Java formatting (Spotless)
61-
if: matrix.jdk-version == 25 && matrix.spring-security == false
67+
# Runs once per matrix combination - pick the cheapest leg
68+
# (core - no proprietary, no saas) so we don't wait for the
69+
# heavier flavors just to fail formatting.
70+
if: matrix.jdk-version == 25 && matrix.flavor == 'core'
6271
id: spotless-check
6372
run: task backend:format:check
6473
continue-on-error: true
@@ -143,17 +152,24 @@ jobs:
143152
});
144153
}
145154
146-
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
155+
- name: Build with Gradle (flavor=${{ matrix.flavor }})
156+
# STIRLING_FLAVOR is read by settings.gradle and expands into the
157+
# right combination of DISABLE_ADDITIONAL_FEATURES + ENABLE_SAAS
158+
# so we don't have to set them by hand. The saas flavor pulls in
159+
# the app/saas subproject (unit tests only - no runtime tests).
147160
run: task backend:build:ci
148161
env:
149162
MAVEN_USER: ${{ secrets.MAVEN_USER }}
150163
MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }}
151164
MAVEN_PUBLIC_URL: ${{ secrets.MAVEN_PUBLIC_URL }}
152-
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }}
165+
STIRLING_FLAVOR: ${{ matrix.flavor }}
153166

154167
- name: Check Test Reports Exist
155168
if: always()
156169
run: |
170+
# Common + core + proprietary always build (proprietary is
171+
# excluded only at runtime, not from the gradle subproject
172+
# graph). Saas builds add a fourth report dir.
157173
declare -a dirs=(
158174
"app/core/build/reports/tests/"
159175
"app/core/build/test-results/"
@@ -162,6 +178,9 @@ jobs:
162178
"app/proprietary/build/reports/tests/"
163179
"app/proprietary/build/test-results/"
164180
)
181+
if [ "${{ matrix.flavor }}" = "saas" ]; then
182+
dirs+=("app/saas/build/reports/tests/" "app/saas/build/test-results/")
183+
fi
165184
for dir in "${dirs[@]}"; do
166185
if [ ! -d "$dir" ]; then
167186
echo "Missing $dir"
@@ -173,7 +192,7 @@ jobs:
173192
if: always()
174193
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
175194
with:
176-
name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }}
195+
name: test-reports-jdk-${{ matrix.jdk-version }}-flavor-${{ matrix.flavor }}
177196
path: |
178197
app/**/build/reports/jacoco/test
179198
app/**/build/reports/tests/
@@ -183,7 +202,47 @@ jobs:
183202
retention-days: 3
184203
if-no-files-found: warn
185204

186-
- name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }}
205+
- name: Install defusedxml for coverage summary
206+
# coverage-summary.py parses JaCoCo XML through defusedxml to
207+
# silence security scanners that pattern-match on the stdlib
208+
# xml.etree.ElementTree.parse call.
209+
if: always() && matrix.flavor == 'saas'
210+
run: python -m pip install --quiet defusedxml
211+
212+
- name: JaCoCo coverage step summary
213+
# Only the saas leg posts the JUnit summary - it's a strict
214+
# superset of the core + proprietary legs (same .exec files plus
215+
# the saas subproject). Posting from all three would mean three
216+
# near-identical tables crowding out the aggregate report.
217+
if: always() && matrix.flavor == 'saas'
218+
run: |
219+
python scripts/coverage-summary.py \
220+
--title "Backend JUnit coverage (JDK ${{ matrix.jdk-version }})" \
221+
--jacoco "common=app/common/build/reports/jacoco/test/jacocoTestReport.xml" \
222+
--jacoco "core=app/core/build/reports/jacoco/test/jacocoTestReport.xml" \
223+
--jacoco "proprietary=app/proprietary/build/reports/jacoco/test/jacocoTestReport.xml" \
224+
--jacoco "saas=app/saas/build/reports/jacoco/test/jacocoTestReport.xml" \
225+
--github-step-summary
226+
227+
- name: Upload raw JUnit .exec for aggregate merge
228+
# Same dedup rationale as the summary step: upload from the saas
229+
# leg only (the most complete set, includes app/saas/.../test.exec)
230+
# so the aggregate workflow merges the union rather than three
231+
# overlapping subsets.
232+
#
233+
# Separate artifact from the HTML reports so the aggregate
234+
# workflow can grab just the .exec files with a name pattern
235+
# (`jacoco-exec-*`) instead of unpacking the whole test-reports
236+
# tarball.
237+
if: always() && matrix.flavor == 'saas'
238+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
239+
with:
240+
name: jacoco-exec-junit-jdk-${{ matrix.jdk-version }}
241+
path: app/*/build/jacoco/*.exec
242+
retention-days: 7
243+
if-no-files-found: warn
244+
245+
- name: Add coverage to PR (flavor=${{ matrix.flavor }}, JDK=${{ matrix.jdk-version }})
187246
# The action only supports the pull_request event (it posts a PR comment),
188247
# so skip it for merge_group runs and workflow_dispatch.
189248
if: github.event_name == 'pull_request'

‎.github/workflows/build.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,24 @@ jobs:
185185
uses: ./.github/workflows/dependency-review.yml
186186
secrets: inherit
187187

188+
# Coverage aggregate: merges the JUnit + e2e:live + cucumber .exec
189+
# artifacts produced by the jobs above into one report, plus pulls
190+
# in vitest + Playwright frontend coverage for the per-area matrix.
191+
# `if: always()` so a producer failing partway still gets credit
192+
# for whatever did record. Advisory only - intentionally NOT in
193+
# all-checks-passed, so a flaky aggregate run never blocks merging.
194+
coverage-aggregate:
195+
if: always()
196+
needs:
197+
- build
198+
- playwright-e2e-live
199+
- docker-compose-tests
200+
- frontend-validation
201+
permissions:
202+
contents: read
203+
uses: ./.github/workflows/coverage-aggregate.yml
204+
secrets: inherit
205+
188206
# Single status check that branch protection should mark as required.
189207
# Succeeds when every upstream job is either `success` or `skipped` (path-
190208
# gated jobs that didn't apply this run). Any `failure` or `cancelled`

‎.github/workflows/coverage-aggregate.yml‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
name: Aggregate backend coverage
2+
3+
# Reusable workflow called from build.yml after every backend coverage
4+
# producer (JUnit, e2e:live, cucumber) has run. Downloads each job's raw
5+
# .exec, merges them into one JaCoCo report, and posts a combined step
6+
# summary alongside the per-source ones.
7+
#
8+
# Kept separate from the per-source jobs so:
9+
# - the per-source jobs stay fast and independent (no cross-job waits)
10+
# - this job can `if: always()` and still produce something useful when
11+
# one of the producers fails partway through
12+
# - frontend producers can be added later without touching the
13+
# producers themselves
14+
on:
15+
workflow_call:
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
pick:
22+
uses: ./.github/workflows/_runner-pick.yml
23+
24+
aggregate:
25+
needs: pick
26+
runs-on: ${{ needs.pick.outputs.is_fork == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-4' }}
27+
timeout-minutes: 15
28+
steps:
29+
- name: Harden Runner
30+
uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
31+
with:
32+
egress-policy: audit
33+
- name: Checkout repository
34+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
35+
36+
- name: Set up JDK 25
37+
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
38+
with:
39+
java-version: "25"
40+
distribution: "temurin"
41+
42+
- name: Cache Gradle dependency artifacts
43+
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
44+
with:
45+
path: |
46+
~/.gradle/wrapper
47+
~/.gradle/caches/modules-2/files-2.1
48+
~/.gradle/caches/modules-2/metadata-2.*
49+
key: gradle-deps-${{ runner.os }}-jdk-25-${{ hashFiles('**/gradle/wrapper/gradle-wrapper.properties', '**/*.gradle', '**/*.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'gradle/libs.versions.toml') }}
50+
51+
- name: Setup Gradle
52+
uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
53+
with:
54+
gradle-version: 9.3.1
55+
cache-disabled: true
56+
57+
- name: Set up Python
58+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
59+
with:
60+
python-version: "3.12"
61+
62+
- name: Install defusedxml for coverage scripts
63+
# Both coverage-summary.py and coverage-matrix.py parse JaCoCo
64+
# XML through defusedxml - see the script headers for context.
65+
run: python -m pip install --quiet defusedxml
66+
67+
# Pattern matches every artifact this PR's producers might upload:
68+
# jacoco-exec-junit-jdk-25 (uploaded only by the saas
69+
# leg of backend-build, which
70+
# is a strict superset of the
71+
# core + proprietary legs)
72+
# jacoco-exec-e2e-live
73+
# jacoco-exec-cucumber
74+
# Each lands as a sibling dir under coverage-execs/, with the .exec
75+
# files preserving their original relative paths.
76+
- name: Download all .exec artifacts
77+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
78+
with:
79+
pattern: jacoco-exec-*
80+
path: coverage-execs/
81+
merge-multiple: false
82+
continue-on-error: true
83+
84+
- name: Inventory .exec files
85+
id: inventory
86+
# Splits the downloaded artifacts into two buckets:
87+
# * e2e-only = cucumber + Playwright live (user-flow coverage)
88+
# * all = the above plus JUnit (everything we test)
89+
#
90+
# Bucketing is by artifact-name prefix: download-artifact preserves
91+
# the artifact name as the top-level dir, so JUnit's `.exec`s live
92+
# under coverage-execs/jacoco-exec-junit-*/... while the others
93+
# are under coverage-execs/jacoco-exec-{e2e-live,cucumber}/...
94+
#
95+
# If nothing was uploaded (e.g. all producers crashed before
96+
# writing) we exit gracefully so this advisory job never fails CI.
97+
run: |
98+
mapfile -t all_execs < <(find coverage-execs -name '*.exec' -type f | sort)
99+
mapfile -t e2e_execs < <(find coverage-execs -name '*.exec' -type f -not -path '*/jacoco-exec-junit-*' | sort)
100+
if [ "${#all_execs[@]}" -eq 0 ]; then
101+
echo "::warning::No .exec artifacts found - skipping aggregate report"
102+
echo "found_all=false" >> "$GITHUB_OUTPUT"
103+
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
104+
exit 0
105+
fi
106+
printf 'All %d .exec files:\n' "${#all_execs[@]}"
107+
printf ' %s\n' "${all_execs[@]}"
108+
IFS=','; all_joined="${all_execs[*]}"
109+
echo "files_all=$all_joined" >> "$GITHUB_OUTPUT"
110+
echo "found_all=true" >> "$GITHUB_OUTPUT"
111+
if [ "${#e2e_execs[@]}" -eq 0 ]; then
112+
echo "::notice::No e2e/cucumber .exec files - e2e-only report will be skipped"
113+
echo "found_e2e=false" >> "$GITHUB_OUTPUT"
114+
else
115+
printf 'E2E-only %d .exec files:\n' "${#e2e_execs[@]}"
116+
printf ' %s\n' "${e2e_execs[@]}"
117+
unset IFS
118+
IFS=','; e2e_joined="${e2e_execs[*]}"
119+
echo "files_e2e=$e2e_joined" >> "$GITHUB_OUTPUT"
120+
echo "found_e2e=true" >> "$GITHUB_OUTPUT"
121+
fi
122+
123+
- name: Compile classes for JaCoCo class lookup
124+
# jacocoReportFromExec only needs the compiled .class files
125+
# under each subproject's build/classes/java/main/. `classes`
126+
# (compileJava + processResources) is enough; we skipped the
127+
# heavier `assemble` to avoid building bootJar / fat jars that
128+
# add 60+ seconds per run for no gain to the report.
129+
if: steps.inventory.outputs.found_all == 'true'
130+
run: ./gradlew classes -PnoSpotless
131+
132+
- name: Generate e2e-only JaCoCo report
133+
# "Real user-flow" coverage: only counts code reached by an actual
134+
# HTTP request from cucumber or live Playwright. Useful for
135+
# questions like "how much of our backend does a user actually
136+
# hit?". Skipped when neither producer uploaded a .exec.
137+
if: steps.inventory.outputs.found_e2e == 'true'
138+
run: |
139+
./gradlew jacocoReportFromExec \
140+
-PexecFile="${{ steps.inventory.outputs.files_e2e }}" \
141+
-PreportDir=build/reports/jacoco/aggregate-e2e \
142+
-PnoSpotless
143+
144+
- name: Generate combined JaCoCo report (everything)
145+
if: steps.inventory.outputs.found_all == 'true'
146+
run: |
147+
./gradlew jacocoReportFromExec \
148+
-PexecFile="${{ steps.inventory.outputs.files_all }}" \
149+
-PreportDir=build/reports/jacoco/aggregate-all \
150+
-PnoSpotless
151+
152+
- name: E2E-only step summary
153+
# Rendered first so it gets prime real estate in the Summary
154+
# tab - this is the number most readers actually want
155+
# ("how much of the backend do real user flows cover?").
156+
if: steps.inventory.outputs.found_e2e == 'true'
157+
run: |
158+
python scripts/coverage-summary.py \
159+
--title "Real user-flow backend coverage (e2e:live + cucumber)" \
160+
--jacoco "merged=build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml" \
161+
--github-step-summary
162+
163+
- name: ALL-sources step summary
164+
# Separate call (not a multi-input one) because the helper's
165+
# rightmost "Aggregate" column would sum the two reports - which
166+
# is meaningless when one is a strict superset of the other.
167+
if: steps.inventory.outputs.found_all == 'true'
168+
run: |
169+
python scripts/coverage-summary.py \
170+
--title "Combined backend coverage (JUnit + e2e:live + cucumber)" \
171+
--jacoco "merged=build/reports/jacoco/aggregate-all/jacocoTestReport.xml" \
172+
--github-step-summary
173+
174+
- name: Upload combined aggregate report
175+
if: steps.inventory.outputs.found_all == 'true'
176+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
177+
with:
178+
name: jacoco-aggregate-all-${{ github.run_id }}
179+
path: build/reports/jacoco/aggregate-all/
180+
retention-days: 14
181+
182+
- name: Upload e2e-only aggregate report
183+
if: steps.inventory.outputs.found_e2e == 'true'
184+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
185+
with:
186+
name: jacoco-aggregate-e2e-${{ github.run_id }}
187+
path: build/reports/jacoco/aggregate-e2e/
188+
retention-days: 14
189+
190+
# --------------------------------------------------------------
191+
# Per-area matrix: rolls backend + frontend coverage into one
192+
# table indexed by core/proprietary/saas/desktop. Pulls the
193+
# frontend artifacts now (after the JaCoCo step has done its
194+
# work) so the per-source backend summaries still render first
195+
# even if the matrix step fails.
196+
# --------------------------------------------------------------
197+
- name: Download vitest coverage artifact
198+
# frontend-validation uploads as `frontend-coverage`. Tolerate
199+
# absence so a backend-only PR still produces the matrix with
200+
# just backend rows populated.
201+
if: always()
202+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
203+
with:
204+
name: frontend-coverage
205+
path: matrix-inputs/vitest/
206+
continue-on-error: true
207+
208+
- name: Download Playwright frontend coverage artifact
209+
# e2e-live uploads as `playwright-frontend-coverage-<run_id>`.
210+
# Same tolerance as vitest - matrix script handles missing inputs.
211+
if: always()
212+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6.0.0
213+
with:
214+
name: playwright-frontend-coverage-${{ github.run_id }}
215+
path: matrix-inputs/playwright/
216+
continue-on-error: true
217+
218+
- name: Coverage matrix step summary
219+
if: always()
220+
# Matrix references the two aggregate JaCoCo XMLs (already
221+
# generated above) plus whichever frontend artifacts landed.
222+
# Every input is optional; missing ones render as "-".
223+
run: |
224+
python scripts/coverage-matrix.py \
225+
${{ steps.inventory.outputs.found_all == 'true' && '--jacoco-all build/reports/jacoco/aggregate-all/jacocoTestReport.xml' || '' }} \
226+
${{ steps.inventory.outputs.found_e2e == 'true' && '--jacoco-e2e build/reports/jacoco/aggregate-e2e/jacocoTestReport.xml' || '' }} \
227+
--vitest matrix-inputs/vitest/coverage-summary.json \
228+
--playwright-frontend matrix-inputs/playwright/coverage-pw-summary/coverage-summary.json \
229+
--title "Coverage matrix (per-area, e2e vs all)" \
230+
--github-step-summary

0 commit comments

Comments
 (0)