Skip to content

Commit c900891

Browse files
Catalin-Stratulat-EricssonCatalin-Stratulat-Ericsson
authored
Fixed CLI Compatibility Matrix runs (#1128)
* fixing compat matrix - 3 versions - only schemantic versions - each server vs ctl in its own section for cleaner dev debug Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * changing to every PR run just for testing. will remove on next commit if successful Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * changing make target to match e2e cli tests Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * removing on every pull request run which was just used for test run Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * addressing copilot comments Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * removed duplication with the use of script call as per comment suggestion Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * changing to on every pull request just to test Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * checking out script from latest before runs since it didnt exist in those versions or current latest Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> * removing on every pull request run for test Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com> --------- Signed-off-by: Catalin-Stratulat-Ericsson <catalin.stratulat@ericsson.com>
1 parent 29467b4 commit c900891

2 files changed

Lines changed: 85 additions & 51 deletions

File tree

‎.github/workflows/porch-compat-matrix.yaml‎

Lines changed: 37 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ env:
2828

2929
jobs:
3030
# ---------------------------------------------------------------------------
31-
# 1. Resolve the last 4 published releases and build the server matrix
31+
# 1. Resolve the last 2 published releases + latest and build the server matrix
3232
# ---------------------------------------------------------------------------
3333
resolve-versions:
3434
name: Resolve Versions
@@ -45,7 +45,7 @@ jobs:
4545
run: |
4646
# Always query upstream repo for releases (works correctly from forks)
4747
VERSIONS=$(gh api repos/${UPSTREAM_REPO}/releases \
48-
--jq '[.[] | select(.draft == false)] | sort_by(.published_at) | reverse | .[0:4] | [.[].tag_name]')
48+
--jq '[.[] | select(.draft == false) | select(.tag_name | test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(.published_at) | reverse | .[0:2] | [.[].tag_name]')
4949
echo "versions=${VERSIONS}" >> "$GITHUB_OUTPUT"
5050
echo "Resolved releases: ${VERSIONS}"
5151
@@ -76,12 +76,12 @@ jobs:
7676
steps:
7777
# --- Step 1: Checkout the server version's source code ---
7878

79-
- name: Checkout upstream latest (1.5 branch)
79+
- name: Checkout upstream latest (main branch)
8080
if: matrix.server_version == 'latest'
8181
uses: actions/checkout@v4
8282
with:
8383
repository: kptdev/porch
84-
ref: '1.5' # NOTE this should revert back to main when we merge back
84+
ref: main
8585

8686
- name: Checkout upstream tag ${{ matrix.server_version }}
8787
if: matrix.server_version != 'latest'
@@ -90,6 +90,18 @@ jobs:
9090
repository: kptdev/porch
9191
ref: ${{ matrix.server_version }}
9292

93+
- name: Overlay scripts from main
94+
if: matrix.server_version != 'latest'
95+
uses: actions/checkout@v4
96+
with:
97+
repository: kptdev/porch
98+
ref: main
99+
sparse-checkout: scripts
100+
path: .scripts-main
101+
- name: Copy scripts from main
102+
if: matrix.server_version != 'latest'
103+
run: cp -r .scripts-main/scripts/. scripts/
104+
93105
# --- Step 2: Setup tooling ---
94106

95107
- name: Set up Go
@@ -137,12 +149,12 @@ jobs:
137149
138150
# --- Step 4: Build and deploy porch with DB cache ---
139151

140-
- name: Build and deploy porch (DB cache, no git server)
152+
- name: Build and deploy porch (DB cache, push drafts)
141153
run: |
142154
echo "=============================================="
143155
echo "[DEPLOY] Building and deploying porch server ${{ matrix.server_version }} with DB cache"
144156
echo "=============================================="
145-
make run-in-kind-db-cache-no-git
157+
make run-in-kind-db-cache-push-drafts
146158
147159
# --- Step 5: Download the CLI binaries ---
148160

@@ -168,54 +180,28 @@ jobs:
168180
echo " ${VERSION}: $("${d}porchctl" version 2>/dev/null || echo 'version check skipped')"
169181
done
170182
171-
# --- Step 6: Run CLI tests with each porchctl version sequentially ---
183+
# --- Step 6: Prepare results directory ---
172184

173-
- name: Run CLI tests with all porchctl versions
174-
id: run-tests
175-
run: |
176-
set +e
177-
mkdir -p .build/compat-results
185+
- name: Prepare compat results directory
186+
run: mkdir -p .build/compat-results
178187

179-
CLI_VERSIONS="latest $(echo "$RELEASES" | jq -r '.[]')"
188+
# --- Step 6.1: Run CLI tests - porchctl latest ---
180189

181-
for CLI_VERSION in $CLI_VERSIONS; do
182-
echo ""
183-
echo "=============================================="
184-
echo "[TEST] CLI ${CLI_VERSION} against server ${{ matrix.server_version }}"
185-
echo "=============================================="
186-
187-
# Swap the porchctl binary
188-
cp "${GITHUB_WORKSPACE}/.build/cli/${CLI_VERSION}/porchctl" "${GITHUB_WORKSPACE}/.build/porchctl"
189-
echo "[TEST] Using porchctl from ${CLI_VERSION}:"
190-
.build/porchctl version 2>/dev/null || echo " version check skipped"
191-
192-
LOG_FILE=".build/compat-results/test-${CLI_VERSION}.log"
193-
194-
E2E=1 go test -v -timeout 20m ./test/e2e/cli 2>&1 | tee "${LOG_FILE}"
195-
TEST_EXIT=${PIPESTATUS[0]}
196-
197-
# Parse results
198-
RESULT="pass"
199-
FAILED_TESTS=""
200-
if [ $TEST_EXIT -ne 0 ]; then
201-
RESULT="fail"
202-
FAILED_TESTS=$(grep -E '^\s*--- FAIL:' "${LOG_FILE}" | sed 's/.*--- FAIL: //' | sed 's/ (.*//' | tr '\n' ',' | sed 's/,$//')
203-
fi
204-
205-
# Write result JSON
206-
cat > ".build/compat-results/result-${CLI_VERSION}.json" <<EOJSON
207-
{
208-
"server_version": "${{ matrix.server_version }}",
209-
"cli_version": "${CLI_VERSION}",
210-
"kpt_version": "$(cat .build/kpt-version.txt 2>/dev/null || echo 'unknown')",
211-
"result": "${RESULT}",
212-
"failed_tests": "${FAILED_TESTS}"
213-
}
214-
EOJSON
215-
216-
echo "Result: ${RESULT}"
217-
echo "Failed tests: ${FAILED_TESTS}"
218-
done
190+
- name: "porch-server ${{ matrix.server_version }} vs porchctl latest CLI Test"
191+
id: test-cli-latest
192+
run: ./scripts/run-compat-cli-test.sh latest "${{ matrix.server_version }}"
193+
194+
# --- Step 6.2: Run CLI tests - porchctl n-1 ---
195+
196+
- name: "porch-server ${{ matrix.server_version }} vs porchctl ${{ fromJson(needs.resolve-versions.outputs.releases)[0] }} CLI Test"
197+
id: test-cli-n1
198+
run: ./scripts/run-compat-cli-test.sh "$(echo "$RELEASES" | jq -r '.[0]')" "${{ matrix.server_version }}"
199+
200+
# --- Step 6.3: Run CLI tests - porchctl n-2 ---
201+
202+
- name: "porch-server ${{ matrix.server_version }} vs porchctl ${{ fromJson(needs.resolve-versions.outputs.releases)[1] }} CLI Test"
203+
id: test-cli-n2
204+
run: ./scripts/run-compat-cli-test.sh "$(echo "$RELEASES" | jq -r '.[1]')" "${{ matrix.server_version }}"
219205

220206
# --- Step 7: Upload results + server logs ---
221207

‎scripts/run-compat-cli-test.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2026 The kpt Authors
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o pipefail
17+
18+
CLI_VERSION="${1:?Usage: $0 <cli_version> <server_version>}"
19+
SERVER_VERSION="${2:?Usage: $0 <cli_version> <server_version>}"
20+
21+
cp "${GITHUB_WORKSPACE}/.build/cli/${CLI_VERSION}/porchctl" "${GITHUB_WORKSPACE}/.build/porchctl"
22+
echo "Using porchctl from ${CLI_VERSION}:"
23+
.build/porchctl version 2>/dev/null || echo "version check skipped"
24+
25+
LOG_FILE=".build/compat-results/test-${CLI_VERSION}.log"
26+
set +e
27+
E2E=1 go test -v -timeout 20m ./test/e2e/cli 2>&1 | tee "${LOG_FILE}"
28+
TEST_EXIT=${PIPESTATUS[0]}
29+
set -e
30+
31+
RESULT="pass"
32+
FAILED_TESTS=""
33+
if [ "${TEST_EXIT}" -ne 0 ]; then
34+
RESULT="fail"
35+
FAILED_TESTS=$(grep -E '^[[:space:]]*--- FAIL:' "${LOG_FILE}" | sed 's/.*--- FAIL: //' | sed 's/ (.*//' | tr '\n' ',' | sed 's/,$//')
36+
fi
37+
38+
cat > ".build/compat-results/result-${CLI_VERSION}.json" <<EOJSON
39+
{
40+
"server_version": "${SERVER_VERSION}",
41+
"cli_version": "${CLI_VERSION}",
42+
"kpt_version": "$(cat .build/kpt-version.txt 2>/dev/null || echo 'unknown')",
43+
"result": "${RESULT}",
44+
"failed_tests": "${FAILED_TESTS}"
45+
}
46+
EOJSON
47+
48+
echo "Result: ${RESULT} | Failed tests: ${FAILED_TESTS}"

0 commit comments

Comments
 (0)