[CI] Add stage-feature tests to ci-demo CI

INFINIOPS-296
What: Add job_matrix_stages_k8.yaml, run multiple matrix
files via CI_K8_FILES, assert stage ordering from the
Jenkins console log, and document the stage property.
How: Extend local_gha_ci.sh to iterate multiple confs and
compare declared stage order against Starting stage logs;
wire the stages matrix into the GHA workflow.
Why: Provide real CI coverage that stages run sequentially
in the declared order (build, test, publish).

Signed-off-by: Benny Itkin <bitkin@nvidia.com>

Author: BennyItkin

.ci/examples/job_matrix_stages.yaml

@@ -8,9 +8,9 @@ runs_on_dockers:
   - {name: 'ubuntu2204', url: 'ubuntu:22.04', arch: 'x86_64'}
   - {name: 'rhel9', url: 'redhat/ubi9', arch: 'x86_64'}
 
-# Steps with the same 'stage' value are grouped and run in parallel
-# under a single Jenkins stage block. Steps without 'stage' run
-# sequentially as usual.
+# Steps are assigned to Jenkins stages based on the 'stage' field.
+# All steps with the same 'stage' value are grouped together and executed sequentially within that stage block.
+# Steps without a 'stage' field are assigned to a default stage and executed in order.
 
 steps:
   - name: Build lib

.ci/job_matrix_stages_k8.yaml

@@ -0,0 +1,69 @@
+---
+job: ci-k8-stages
+
+kubernetes:
+  cloud: swx-k8s
+  namespace: default
+  serviceAccount: jenkins
+  hostNetwork: false
+  privileged: false
+  limits: "{memory: 2Gi, cpu: 1000m}"
+  requests: "{memory: 512Mi, cpu: 250m}"
+  caps_add: "[]"
+  tolerations: "[]"
+  imagePullSecrets: "[]"
+
+arch_table:
+  x86_64:
+    nodeSelector: "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
+    jnlpImage: "jenkins/inbound-agent:latest"
+  aarch64:
+    nodeSelector: "kubernetes.io/os=linux,kubernetes.io/arch=arm64"
+    jnlpImage: "jenkins/inbound-agent:latest"
+
+empty_volumes:
+  - {mountPath: /tmp/workspace, memory: false}
+
+runs_on_dockers:
+  - {name: "ubuntu-x86_64", url: "ubuntu:22.04", arch: "x86_64"}
+  - {name: "ubuntu-aarch64", url: "ubuntu:22.04", arch: "aarch64"}
+
+matrix:
+  axes:
+    arch:
+      - x86_64
+      - aarch64
+  include:
+    - {arch: '${TARGET_ARCH}'}
+
+# Steps are grouped into Jenkins stages via the 'stage' field.
+# Stages run sequentially in the order they first appear (build -> test -> publish),
+# and steps within a stage run together in that stage block.
+steps:
+  - name: Build lib
+    stage: build
+    run: echo "building lib for ${arch}"
+
+  - name: Build tools
+    stage: build
+    run: echo "building tools for ${arch}"
+
+  - name: Unit tests
+    stage: test
+    run: echo "running unit tests for ${arch}"
+
+  - name: Integration tests
+    stage: test
+    run: echo "running integration tests for ${arch}"
+
+  - name: Publish
+    stage: publish
+    run: echo "publishing artifacts for ${arch}"
+
+pipeline_start:
+  run: echo "pipeline_start for ${job}"
+
+pipeline_stop:
+  run: echo "pipeline_stop status=${pipeline_status}"
+
+failFast: false

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
           BRANCH: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}
           REPO_URL: "https://github.com/${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }}.git"
           TARGET_ARCHES: ${{ env.TARGET_ARCH }}
-          CI_K8_FILE: .ci/job_matrix_gha_k8.yaml
+          CI_K8_FILES: ".ci/job_matrix_gha_k8.yaml .ci/job_matrix_stages_k8.yaml"
           KEEP_JENKINS: "false"
           KEEP_K8S: "false"
         run: |

README.md

@@ -134,6 +134,12 @@ TARGET_ARCHES=aarch64 KEEP_JENKINS=false KEEP_K8S=false make -C .ci local-gha-ci
 CI_K8_FILE=.ci/job_matrix_debug.yaml make -C .ci local-gha-ci
 ```
 
+Run several matrix files in a single Jenkins/k3s setup:
+
+```bash
+CI_K8_FILES=".ci/job_matrix_gha_k8.yaml .ci/job_matrix_stages_k8.yaml" make -C .ci local-gha-ci
+```
+
 ## Matrix YAML Essentials
 
 A matrix config must include:

USERGUIDE.md

@@ -153,6 +153,7 @@ All are optional in schema.
 | `credentialsId` | `str/list(str)` |
 | `args` | `list/map/str` |
 | `env` | `map` |
+| `stage` | `str` |
 
 ## Minimal Valid Example
 
@@ -194,6 +195,44 @@ steps:
     run: uname -m
 ```
 
+## Stages
+
+Steps can be grouped into named stages with the optional `stage` key in `step_conf`.
+This lets you define sequential phases (for example build, then test, then deploy)
+while running the work inside each phase in parallel.
+
+Behavior:
+
+- Stage names are collected from the `stage` field of each step, in the order they
+  first appear. Steps without a `stage` fall into a stage named `default`.
+- Stages execute sequentially in that first-seen order; every task in a stage
+  finishes before the next stage starts (each is rendered as a Jenkins stage block).
+- Within a stage, tasks for all matching images/agents run in parallel (subject to
+  `batchSize` and `failFast`). Multiple steps of the same stage on a single image run
+  in their defined order unless the step sets `parallel: true`.
+- Stage grouping only takes effect when more than one distinct stage is present. If
+  every step resolves to the same stage (for example when none define `stage`), all
+  tasks run together in parallel, as before.
+
+```yaml
+steps:
+  - name: Build lib
+    run: echo build lib
+    stage: build
+  - name: Build tools
+    run: echo build tools
+    stage: build
+  - name: Unit tests
+    run: echo unit
+    stage: test
+  - name: Publish
+    run: echo publish
+    stage: publish
+```
+
+Here `build` runs first (both build steps in parallel across images), then `test`,
+then `publish`. See `.ci/examples/job_matrix_stages.yaml` for a complete example.
+
 ## Validation
 
 Validate a file with:

scripts/local_gha_ci.sh

@@ -22,6 +22,8 @@ JENKINS_K8S_DNS_NAME=${JENKINS_K8S_DNS_NAME:-${JENKINS_NETWORK_ALIAS}.default.sv
 KEEP_JENKINS=${KEEP_JENKINS:-true}
 
 CI_K8_FILE=${CI_K8_FILE:-.ci/job_matrix_gha_k8.yaml}
+# Accept one or more config files (space/comma separated). Falls back to CI_K8_FILE.
+CI_K8_FILES=${CI_K8_FILES:-${CI_K8_FILE}}
 TARGET_ARCHES=${TARGET_ARCHES:-${TARGET_ARCH:-}}
 SKIP_REGEX=${SKIP_REGEX:-}
 AGENT_EXECUTORS=${AGENT_EXECUTORS:-8}
@@ -119,12 +121,22 @@ fi
 
 mkdir -p "${LOG_DIR}"
 
-echo "[1/9] Using static workflow config ${CI_K8_FILE}"
-if [[ ! -f "${CI_K8_FILE}" ]]; then
-  echo "ERROR: Config file not found: ${CI_K8_FILE}" >&2
+conf_files=()
+while IFS= read -r conf; do
+  [[ -n "${conf}" ]] && conf_files+=("${conf}")
+done < <(echo "${CI_K8_FILES}" | tr ', ' '\n' | awk 'NF')
+if [[ "${#conf_files[@]}" -eq 0 ]]; then
+  echo "ERROR: No config files specified (set CI_K8_FILES or CI_K8_FILE)" >&2
   exit 1
 fi
-conf_files=("${CI_K8_FILE}")
+
+echo "[1/9] Using static workflow config(s): ${conf_files[*]}"
+for conf in "${conf_files[@]}"; do
+  if [[ ! -f "${conf}" ]]; then
+    echo "ERROR: Config file not found: ${conf}" >&2
+    exit 1
+  fi
+done
 
 target_arch_list=()
 while IFS= read -r arch; do
@@ -297,6 +309,55 @@ save_jenkins_artifacts() {
   docker logs "${JENKINS_NAME}" > "${LOG_DIR}/${prefix}.jenkins-container.log" 2>&1 || true
 }
 
+# Extract declared stage names from a matrix YAML in first-appearance order (deduped).
+extract_declared_stages() {
+  local file="$1"
+  awk '
+    /^[[:space:]]*stage:[[:space:]]*/ {
+      line=$0
+      sub(/^[[:space:]]*stage:[[:space:]]*/, "", line)
+      sub(/[[:space:]]*(#.*)?$/, "", line)
+      gsub(/["'\'']/, "", line)
+      if (line != "" && !(line in seen)) { seen[line]=1; order[++n]=line }
+    }
+    END { for (i=1;i<=n;i++) print order[i] }
+  ' "${file}"
+}
+
+# Extract stage names in the order they were entered from a Jenkins console log (deduped).
+extract_console_stages() {
+  local logf="$1"
+  grep -a 'Starting stage:' "${logf}" 2>/dev/null \
+    | sed -E 's/.*Starting stage:[[:space:]]*//' \
+    | sed -E 's/[^A-Za-z0-9_.-].*$//' \
+    | awk 'NF && !seen[$0]++'
+}
+
+# Assert stages ran sequentially in declared order. Self-activating: no-op unless
+# the config declares >= 2 stages. Returns non-zero on mismatch.
+assert_stage_order() {
+  local conf_file="$1" console_log="$2" label="$3"
+  local expected actual nstages
+  expected=$(extract_declared_stages "${conf_file}")
+  nstages=$(printf '%s\n' "${expected}" | awk 'NF' | wc -l | tr -d ' ')
+  if [[ "${nstages}" -lt 2 ]]; then
+    return 0
+  fi
+  if [[ ! -f "${console_log}" ]]; then
+    echo "STAGE-ORDER FAIL ${label}: console log not found (${console_log})"
+    return 1
+  fi
+  actual=$(extract_console_stages "${console_log}")
+  if [[ "${expected}" == "${actual}" ]]; then
+    echo "STAGE-ORDER PASS ${label}: $(echo ${expected} | tr '\n' ' ')"
+    return 0
+  fi
+  echo "STAGE-ORDER FAIL ${label}"
+  echo "  expected: $(echo ${expected} | tr '\n' ' ')"
+  echo "  actual:   $(echo ${actual} | tr '\n' ' ')"
+  return 1
+}
+
 echo "Waiting for Jenkins CLI readiness"
 cli_ready=false
 for i in $(seq 1 90); do
@@ -575,6 +636,9 @@ for conf in "${conf_files[@]}"; do
       fail_count=$((fail_count + 1))
     else
       echo "PASS ${conf_rel} TARGET_ARCH=${target_arch}"
+      if ! assert_stage_order "${conf}" "${LOG_DIR}/${output_prefix}.jenkins-console.log" "${conf_base} TARGET_ARCH=${target_arch}"; then
+        fail_count=$((fail_count + 1))
+      fi
     fi
   done
 done