Skip to content

Commit a8b123b

Browse files
Catalin-Stratulat-EricssonCatalin-Stratulat-Ericsson
authored
Adding compatibility matrix (porch-server vs porchctl versions) (kptdev#515)
* adding a compatibility matrix runner for porch-server versions vs porchctl versions * changing to last 4 versions and adding a few more comments in workflow * removed on_pull_request trigger for job
1 parent 92cbcdc commit a8b123b

1 file changed

Lines changed: 328 additions & 0 deletions

File tree

‎.github/workflows/porch-compat-matrix.yaml‎

Lines changed: 328 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,328 @@
1+
# Copyright 2026 The Nephio Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: CLI Compatibility Matrix
16+
17+
on:
18+
workflow_dispatch:
19+
schedule:
20+
- cron: "0 8 * * 1" # Every Monday at 8AM UTC
21+
22+
concurrency:
23+
group: ${{ github.workflow }}-${{ github.ref }}
24+
cancel-in-progress: true
25+
26+
env:
27+
KIND_CONTEXT_NAME: porch-test
28+
29+
jobs:
30+
# ---------------------------------------------------------------------------
31+
# 1. Resolve the last 4 published releases and build the server matrix
32+
# ---------------------------------------------------------------------------
33+
resolve-versions:
34+
name: Resolve Versions
35+
runs-on: ubuntu-latest
36+
outputs:
37+
releases: ${{ steps.releases.outputs.versions }}
38+
server_matrix: ${{ steps.matrix.outputs.server_matrix }}
39+
steps:
40+
- name: Get last 4 releases
41+
id: releases
42+
env:
43+
GH_TOKEN: ${{ github.token }}
44+
UPSTREAM_REPO: nephio-project/porch
45+
run: |
46+
# Always query upstream repo for releases (works correctly from forks)
47+
VERSIONS=$(gh api repos/${UPSTREAM_REPO}/releases \
48+
--jq '[.[] | select(.draft == false)] | sort_by(.published_at) | reverse | .[0:4] | [.[].tag_name]')
49+
echo "versions=${VERSIONS}" >> "$GITHUB_OUTPUT"
50+
echo "Resolved releases: ${VERSIONS}"
51+
52+
- name: Build server matrix
53+
id: matrix
54+
run: |
55+
RELEASES='${{ steps.releases.outputs.versions }}'
56+
SERVER_MATRIX=$(echo "$RELEASES" | jq -c '{include: ([{server_version: "latest"}] + [.[] | {server_version: .}])}')
57+
echo "server_matrix=${SERVER_MATRIX}" >> "$GITHUB_OUTPUT"
58+
echo "Server matrix:"
59+
echo "$SERVER_MATRIX" | jq .
60+
61+
# ---------------------------------------------------------------------------
62+
# 2. One self-contained runner per porch server version
63+
# Each runner: checks out the version source, builds & deploys from it,
64+
# then tests all CLI versions sequentially.
65+
# ---------------------------------------------------------------------------
66+
compat-test:
67+
name: "Server ${{ matrix.server_version }}"
68+
runs-on: ubuntu-latest
69+
timeout-minutes: 60
70+
needs: resolve-versions
71+
strategy:
72+
fail-fast: false
73+
matrix: ${{ fromJson(needs.resolve-versions.outputs.server_matrix) }}
74+
env:
75+
RELEASES: ${{ needs.resolve-versions.outputs.releases }}
76+
steps:
77+
# --- Step 1: Checkout the server version's source code ---
78+
79+
- name: Checkout upstream latest (1.5 branch)
80+
if: matrix.server_version == 'latest'
81+
uses: actions/checkout@v4
82+
with:
83+
repository: nephio-project/porch
84+
ref: '1.5' # NOTE this should revert back to main when we merge back
85+
86+
- name: Checkout upstream tag ${{ matrix.server_version }}
87+
if: matrix.server_version != 'latest'
88+
uses: actions/checkout@v4
89+
with:
90+
repository: nephio-project/porch
91+
ref: ${{ matrix.server_version }}
92+
93+
# --- Step 2: Setup tooling ---
94+
95+
- name: Set up Go
96+
uses: actions/setup-go@v5
97+
with:
98+
go-version-file: go.mod
99+
cache: true
100+
101+
# --- Install's kpt based on porch go.mod native version ---
102+
103+
- name: Install kpt
104+
run: |
105+
KPT_VERSION=$(go list -m -f '{{.Version}}' github.com/kptdev/kpt 2>/dev/null)
106+
if [ -z "${KPT_VERSION}" ]; then
107+
echo "::error::Failed to resolve kpt version from go.mod. Cannot proceed with tests."
108+
exit 1
109+
fi
110+
echo "Installing kpt ${KPT_VERSION}"
111+
curl -fsSL "https://github.com/kptdev/kpt/releases/download/${KPT_VERSION}/kpt_linux_amd64-${KPT_VERSION#v}.tar.gz" | tar -xz -C /usr/local/bin/
112+
# Save kpt version for result reporting
113+
mkdir -p .build
114+
echo "${KPT_VERSION}" > .build/kpt-version.txt
115+
116+
# --- Install's Kind without deploying a cluster ---
117+
118+
- name: Install Kind
119+
uses: helm/kind-action@v1
120+
with:
121+
version: v0.30.0
122+
install_only: true
123+
124+
- name: Set up Git config
125+
run: |
126+
git config --global user.name "Porch Compat"
127+
git config --global user.email "porch-compat@porch.dev"
128+
129+
# --- Step 3: Setup dev env (kind + gitea + MetalLB + porchctl) ---
130+
131+
- name: Setup dev env
132+
run: |
133+
echo "=============================================="
134+
echo "[SETUP] Setting up dev env for server ${{ matrix.server_version }}"
135+
echo "=============================================="
136+
./scripts/setup-dev-env.sh
137+
138+
# --- Step 4: Build and deploy porch with DB cache ---
139+
140+
- name: Build and deploy porch (DB cache, no git server)
141+
run: |
142+
echo "=============================================="
143+
echo "[DEPLOY] Building and deploying porch server ${{ matrix.server_version }} with DB cache"
144+
echo "=============================================="
145+
make run-in-kind-db-cache-no-git
146+
147+
# --- Step 5: Download the CLI binaries ---
148+
149+
- name: Download released CLI versions
150+
run: |
151+
UPSTREAM_REPO="nephio-project/porch"
152+
for VERSION in $(echo "$RELEASES" | jq -r '.[]'); do
153+
VERSION_NUM="${VERSION#v}"
154+
CLI_URL="https://github.com/${UPSTREAM_REPO}/releases/download/${VERSION}/porchctl_${VERSION_NUM}_linux_amd64.tar.gz"
155+
echo "Downloading porchctl ${VERSION} from ${CLI_URL}"
156+
mkdir -p ".build/cli/${VERSION}"
157+
curl -fsSL "${CLI_URL}" | tar -xz -C ".build/cli/${VERSION}/"
158+
chmod +x ".build/cli/${VERSION}/porchctl"
159+
done
160+
161+
# Also save the locally built porchctl as "latest"
162+
mkdir -p .build/cli/latest
163+
cp .build/porchctl .build/cli/latest/porchctl
164+
165+
echo "Available CLI versions:"
166+
for d in .build/cli/*/; do
167+
VERSION=$(basename "$d")
168+
echo " ${VERSION}: $("${d}porchctl" version 2>/dev/null || echo 'version check skipped')"
169+
done
170+
171+
# --- Step 6: Run CLI tests with each porchctl version sequentially ---
172+
173+
- name: Run CLI tests with all porchctl versions
174+
id: run-tests
175+
run: |
176+
set +e
177+
mkdir -p .build/compat-results
178+
179+
CLI_VERSIONS="latest $(echo "$RELEASES" | jq -r '.[]')"
180+
181+
for CLI_VERSION in $CLI_VERSIONS; do
182+
echo ""
183+
echo "=============================================="
184+
echo "[TEST] CLI ${CLI_VERSION} against server ${{ matrix.server_version }}"
185+
echo "=============================================="
186+
187+
# Swap the porchctl binary
188+
cp "${GITHUB_WORKSPACE}/.build/cli/${CLI_VERSION}/porchctl" "${GITHUB_WORKSPACE}/.build/porchctl"
189+
echo "[TEST] Using porchctl from ${CLI_VERSION}:"
190+
.build/porchctl version 2>/dev/null || echo " version check skipped"
191+
192+
LOG_FILE=".build/compat-results/test-${CLI_VERSION}.log"
193+
194+
E2E=1 go test -v -timeout 20m ./test/e2e/cli 2>&1 | tee "${LOG_FILE}"
195+
TEST_EXIT=${PIPESTATUS[0]}
196+
197+
# Parse results
198+
RESULT="pass"
199+
FAILED_TESTS=""
200+
if [ $TEST_EXIT -ne 0 ]; then
201+
RESULT="fail"
202+
FAILED_TESTS=$(grep -E '^\s*--- FAIL:' "${LOG_FILE}" | sed 's/.*--- FAIL: //' | sed 's/ (.*//' | tr '\n' ',' | sed 's/,$//')
203+
fi
204+
205+
# Write result JSON
206+
cat > ".build/compat-results/result-${CLI_VERSION}.json" <<EOJSON
207+
{
208+
"server_version": "${{ matrix.server_version }}",
209+
"cli_version": "${CLI_VERSION}",
210+
"kpt_version": "$(cat .build/kpt-version.txt 2>/dev/null || echo 'unknown')",
211+
"result": "${RESULT}",
212+
"failed_tests": "${FAILED_TESTS}"
213+
}
214+
EOJSON
215+
216+
echo "Result: ${RESULT}"
217+
echo "Failed tests: ${FAILED_TESTS}"
218+
done
219+
220+
# --- Step 7: Upload results + server logs ---
221+
222+
- name: Upload test results
223+
if: always()
224+
uses: actions/upload-artifact@v4
225+
with:
226+
name: "compat-results-${{ matrix.server_version }}"
227+
path: .build/compat-results/
228+
retention-days: 5
229+
230+
- name: Export porch server logs
231+
if: always()
232+
run: |
233+
name=$(kubectl -n porch-system get pod -l app=porch-server -o custom-columns=NAME:.metadata.name --no-headers=true 2>/dev/null || echo "")
234+
if [ -n "$name" ]; then
235+
kubectl -n porch-system logs "$name" > server.log 2>/dev/null || true
236+
fi
237+
- name: Archive server logs
238+
if: always()
239+
uses: actions/upload-artifact@v4
240+
with:
241+
name: "server-log-${{ matrix.server_version }}"
242+
path: server.log
243+
compression-level: 0
244+
retention-days: 2
245+
if-no-files-found: ignore
246+
247+
# ---------------------------------------------------------------------------
248+
# 3. Aggregate results into a compatibility matrix summary
249+
# ---------------------------------------------------------------------------
250+
compat-summary:
251+
name: Compatibility Matrix Summary
252+
if: always()
253+
needs: [resolve-versions, compat-test]
254+
runs-on: ubuntu-latest
255+
steps:
256+
- name: Download all result artifacts
257+
uses: actions/download-artifact@v4
258+
with:
259+
pattern: compat-results-*
260+
path: results/
261+
262+
- name: Generate compatibility matrix
263+
run: |
264+
RELEASES='${{ needs.resolve-versions.outputs.releases }}'
265+
266+
# Collect all result JSONs into a single array
267+
RESULTS=$(find results/ -name 'result-*.json' -exec cat {} \; | jq -s '.')
268+
269+
# Build the markdown table
270+
CLI_VERSIONS=$(echo "$RELEASES" | jq -r '["latest"] + . | .[]')
271+
SERVER_VERSIONS=$(echo "$RELEASES" | jq -r '["latest"] + . | .[]')
272+
273+
# Header
274+
HEADER="| Server ↓ \ CLI → |"
275+
SEP="|---|"
276+
for cv in $CLI_VERSIONS; do
277+
HEADER="${HEADER} ${cv} |"
278+
SEP="${SEP}---|"
279+
done
280+
281+
{
282+
echo "## CLI Compatibility Matrix (DB Cache)"
283+
echo ""
284+
echo "$HEADER"
285+
echo "$SEP"
286+
287+
for sv in $SERVER_VERSIONS; do
288+
# Extract kpt version for this server from any of its results
289+
KPT_VER=$(echo "$RESULTS" | jq -r --arg sv "$sv" '
290+
map(select(.server_version == $sv)) | .[0].kpt_version // "unknown"
291+
')
292+
ROW="| ${sv} (kpt ${KPT_VER}) |"
293+
for cv in $CLI_VERSIONS; do
294+
CELL=$(echo "$RESULTS" | jq -r --arg sv "$sv" --arg cv "$cv" '
295+
map(select(.server_version == $sv and .cli_version == $cv)) |
296+
if length == 0 then "⚪ N/A"
297+
elif .[0].result == "pass" then "✅ All tests passed"
298+
else
299+
.[0].failed_tests as $ft |
300+
if ($ft | length) > 0 then
301+
"❌ " + ($ft | split(",") | map(select(. != "TestPorch")) | map(split("/") | last) | .[0:3] | join(", ")) +
302+
(if ($ft | split(",") | map(select(. != "TestPorch")) | length) > 3 then " +" + (($ft | split(",") | map(select(. != "TestPorch")) | length) - 3 | tostring) + " more" else "" end)
303+
else "❌"
304+
end
305+
end
306+
')
307+
ROW="${ROW} ${CELL} |"
308+
done
309+
echo "$ROW"
310+
done
311+
312+
echo ""
313+
echo "- ✅ = All CLI tests passed"
314+
echo "- ❌ = One or more tests failed (failed test names shown)"
315+
echo "- ⚪ = Test did not run"
316+
echo ""
317+
echo "**Note:** Tests are run using the native kpt version for each porch server release (shown in parentheses)."
318+
echo "This version is resolved from the go.mod file of each server version. Results may differ if a different kpt version is used."
319+
} | tee -a "$GITHUB_STEP_SUMMARY"
320+
321+
- name: Check for failures
322+
run: |
323+
# Fail the workflow if any test failed
324+
FAILED=$(find results/ -name 'result-*.json' -exec grep -l '"result": "fail"' {} \;)
325+
if [ -n "$FAILED" ]; then
326+
echo "::error::Some compatibility tests failed. See the matrix summary above."
327+
exit 1
328+
fi

0 commit comments

Comments
 (0)