Merge pull request #195 from shawn-hurley/cross-org-ci

✨ Adding cross org CI with comment on pr

Author: shawn-hurley

.github/workflows/build-nightly-images.yaml

@@ -31,6 +31,10 @@ jobs:
   build-images:
     name: Build Images ${{ matrix.image.image }}-${{ matrix.os.arch }}
     runs-on: ${{ matrix.os.runner }}
+    env:
+      GONOSUMCHECK: "*"
+      GOFLAGS: "-mod=mod"
+      GOPROXY: direct
     strategy:
       fail-fast: false
       matrix: 
@@ -42,28 +46,52 @@ jobs:
         uses: actions/checkout@v5
         with:
           repository: ${{ matrix.image.repo }}
-          ref: ${{ (inputs.tested_repo == '' || matrix.image.repo == inputs.tested_repo) && inputs.branch || github.base_ref || github.ref_name }}
+          # This will:
+          # Use an override ref if set for the matrix
+          # Use the branch that is passed in as input
+          # If a pr, will use the base_ref
+          # If not a pr will use the ref 
+          ref: ${{ matrix.image.ref || (matrix.image.repo == inputs.tested_repo && github.ref) || github.base_ref || github.ref_name }}
 
       - name: Setup golang
         uses: actions/setup-go@v6
         with:
           go-version: '1.25' 
 
+      - name: Apply go mod replaces
+        id: go_mod_replaces
+        shell: bash
+        if: matrix.image.go_mod_replaces != '' && matrix.image.go_mod_replaces[0] != null
+        env:
+          GO_MOD_REPLACES_JSON: ${{ toJSON(matrix.image.go_mod_replaces) }}
+        run: |
+          GO_MOD_DIR="${{ matrix.image.go_mod_directory || '.' }}"
+          cd "$GO_MOD_DIR"
+          echo "Applying go mod replaces in directory: $(pwd)"
+
+          REPLACE_ARGS=()
+          while IFS= read -r replace; do
+            echo "  -replace $replace"
+            REPLACE_ARGS+=("-replace" "$replace")
+          done < <(echo "$GO_MOD_REPLACES_JSON" | jq -r '.[]')
+          go mod edit "${REPLACE_ARGS[@]}"
+
       - name: Update go mod to the branch
         id: go_mod
         shell: bash
         if: matrix.image.go_mod_update != '' && matrix.image.go_mod_update[0] != null
+        env:
+          GO_MOD_UPDATE_JSON: ${{ toJSON(matrix.image.go_mod_update) }}
         run: |
           # Change to the specified directory if provided, otherwise use current directory
           GO_MOD_DIR="${{ matrix.image.go_mod_directory || '.' }}"
           cd "$GO_MOD_DIR"
           echo "Running go mod commands in directory: $(pwd)"
 
-          for mod_update in ${{ join(matrix.image.go_mod_update, ' ') }}
-          do
+          while IFS= read -r mod_update; do
             echo "updating go mod $mod_update"
-            go get $mod_update
-          done
+            go get "$mod_update"
+          done < <(echo "$GO_MOD_UPDATE_JSON" | jq -r '.[]')
 
           # Run go mod tidy to clean up
           go mod tidy

.github/workflows/e2e-image-build.yaml

@@ -65,13 +65,48 @@ jobs:
         with:
           python-version: '3.12' 
 
+      - name: Get ci comments
+        id: get_ci_comments
+        if: github.event.pull_request.number
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+        run: |
+          OVERRIDES=$(gh pr view "$PR_NUMBER" -R "$REPO" --json comments \
+            | jq -c '[.comments[] | select(.body | startswith("/ci test-with")) | .body | ltrimstr("/ci test-with ")]')
+          echo "overrides=$OVERRIDES" >> $GITHUB_OUTPUT
+
       - name: Prepare matrix configs
         id: prepare
         shell: bash
+        env:
+          OVERRIDES: ${{ steps.get_ci_comments.outputs.overrides }}
+          TESTED_REPO_FORK: ${{ github.event.pull_request.head.repo.full_name }}
+          TESTED_REPO_BRANCH: ${{ github.event.pull_request.head.ref }}
+          GH_TOKEN: ${{ github.token }}
         run: |
-          
+
           pip install -r scripts/requirements.txt
-          python scripts/parse_matrix_config.py .github/workflows/nightly-matrix-config.yaml -t ${{ inputs.tag || github.run_id }} -b ${{ github.base_ref || github.ref_name }} -r ${{ inputs.repo || github.repository }} out
+
+          cmd=(
+            python scripts/parse_matrix_config.py
+            .github/workflows/nightly-matrix-config.yaml
+            -t "${{ inputs.tag || github.run_id }}"
+            -b "${{ github.base_ref || github.ref_name }}"
+            -r "${{ inputs.repo || github.repository }}"
+          )
+
+          if [ -n "$OVERRIDES" ] && [ "$OVERRIDES" != "[]" ]; then
+            echo "Applying CI comment overrides: $OVERRIDES"
+            cmd+=(-o "$OVERRIDES")
+            if [ -n "$TESTED_REPO_FORK" ] && [ -n "$TESTED_REPO_BRANCH" ]; then
+              cmd+=(--tested-repo-head "${TESTED_REPO_FORK}:${TESTED_REPO_BRANCH}")
+            fi
+          fi
+
+          "${cmd[@]}" out
 
           # The output may be one ore more of these, we should handle the case where these are empty.
           matrix_0=$(cat out/level_0.json)

README.md

@@ -19,6 +19,51 @@ Component | CI (after merge) | Nightly (cron)
 **Kantra CLI** | | [![Nightly CLI test for main](https://github.com/konveyor-ecosystem/kantra-cli-tests/actions/workflows/nightly-main-latest.yaml/badge.svg)](https://github.com/konveyor-ecosystem/kantra-cli-tests/actions/workflows/nightly-main-latest.yaml)
 
 
+## NOTE About Future
+
+Today all of the workflows below are deprecated, and we will be moving away from them.
+
+* [global-ci.yml](.github/workflows/global-ci.yml)
+* [nightly-release-0.8.yaml](.github/workflows/nightly-release-0.8.yaml)
+* [ci-repo.yaml](.github/workflows/ci-repo.yaml)
+* [global-ci-bundle.yml](.github/workflows/global-ci-bundle.yml)
+* [nightly-main.yaml](.github/workflows/nightly-main.yaml)
+* [nightly-release-0.7.yaml](.github/workflows/nightly-release-0.7.yaml)
+* [validate-shared-tests.yml](.github/workflows/validate-shared-tests.yml)
+
+These workflows are the new workflows and related files:
+
+* [nightly-matrix-config.yaml](.github/workflows/nightly-matrix-config.yaml)
+> This is the source of truth for how repositories are tied together, what images need to be based on others.
+> Also contains the information for how to build the image for the given repository.
+
+* [nightly-koncur](.github/workflows/nightly-koncur.yaml) and [nightly-koncur-0.9.yaml](.github/workflows/nightly-koncur-0.9.yaml)
+> These are used for the nightly tests, running as a cron.
+> This is responsible for using the [script](scripts/parse_matrix_config.py) to determine the dependency levels for image builds.
+> It will then run for all valid levels with the `build-nightly-images.yaml` workflow.
+> Once all the images are built, it will run koncur for hub and kantra.
+> Note: the release-0.9 version hardcodes the branch and reuses `nightly-koncur.yaml`.
+
+* [build-nightly-images.yaml](.github/workflows/build-nightly-images.yaml)
+> This is used for building the images for a given level of image builds. It creates a matrix of the build images, that will:
+> * Check out the repo
+> * Update Go module dependencies and apply any cross-repo replacements
+> * Build the image using the image build action.
+
+
+These are the new e2e workflows, to potentially be re-used by repos in the organization.
+
+* [e2e-image-build.yaml](.github/workflows/e2e-image-build.yaml)
+> This is the core re-usable way to correctly build all the images in the system that need to be rebuilt based on a PR.
+> Uses the nightly-matrix-config and the build-nightly-images.yaml to build the correct subset of images.
+
+* [e2e-hub-koncur](.github/workflows/e2e-hub-koncur.yaml)
+> Uses the e2e-image-build to build the correct images and then runs the hub koncur action.
+
+* [e2e-kantra-koncur](.github/workflows/e2e-kantra-koncur.yaml)
+> Uses the e2e-image-build to build the correct images and then runs the kantra koncur action for Linux and Windows.
+> Note: macOS is currently too slow to run.
+
 ## Using the global-ci github workflow
 
 This repository is home to the [global-ci github workflow](https://github.com/konveyor/ci/tree/main/.github/workflows/global-ci.yml).