poi-vcf.yml

name: photon-os-installer CI on VCF

on:
  push:
  workflow_dispatch:

env:
  POI_IMAGE_NAME_BASE: photon/installer
  GITHUB_BRANCH: master
  POI_REGISTRY: poi-registry:5000

jobs:
    lint-checks:
        runs-on: self-hosted
        steps:
          - name: Checkout code
            uses: actions/checkout@v3

          - name: Lint checks
            run: |
              VENV_DIR="$(mktemp -d /var/tmp/venv.XXXXXX)"
              python3 -m venv "$VENV_DIR"
              "$VENV_DIR/bin/pip" install isort black flake8 shellcheck-py --index-url https://packages.vcfd.broadcom.net/artifactory/api/pypi/pypi/simple/
              export PATH="$VENV_DIR/bin:$PATH"
              bash lint-checks.sh
              STATUS=$?
              rm -r "$VENV_DIR"
              exit $STATUS

    dialog-tests:
        runs-on: self-hosted
        steps:
          - name: Checkout code
            uses: actions/checkout@v3

          - name: Run interactive installer dialog tests
            run: |
              VENV_DIR="$(mktemp -d /var/tmp/venv.XXXXXX)"
              # --without-pip + a --python-targeted install avoids relying on
              # ensurepip's bundled wheels, which aren't available for every
              # python3 in this runner fleet.
              python3 -m venv --without-pip "$VENV_DIR"
              # requirements.txt is needed too: the dialog code under test
              # (photon_installer/commandutils.py, installer.py) imports
              # requests/yaml/OpenSSL/jc transitively, even though the tests
              # themselves never touch the network/disk/packages.
              python3 -m pip --python "$VENV_DIR/bin/python3" install -r requirements.txt pytest pexpect --index-url https://packages.vcfd.broadcom.net/artifactory/api/pypi/pypi/simple/
              "$VENV_DIR/bin/pytest" -x tests/test_iso_dialogs.py
              STATUS=$?
              rm -r "$VENV_DIR"
              exit $STATUS

    build-container:
        runs-on: [ self-hosted, "docker:root" ]
        steps:
            - name: Checkout code
              uses: actions/checkout@v3

            - name: Build Docker image
              run: |
                POI_IMAGE_NAME=${POI_IMAGE_NAME_BASE}:${GITHUB_SHA::7}
                cd docker
                docker build -t $POI_IMAGE_NAME --build-context poi-helper=$(realpath $(pwd)/..) .
                docker tag ${POI_IMAGE_NAME} ${POI_REGISTRY}/${POI_IMAGE_NAME}
                docker push ${POI_REGISTRY}/${POI_IMAGE_NAME}
                docker system prune -f

    cayman_poi:
        runs-on: self-hosted
        steps:
            - name: Checkout Cayman POI
              uses: actions/checkout@master
              with:
                  repository: vcf/cayman-poi
                  ref: vmware-master
                  path: ./cayman-poi
                  submodules: "true"
                  fetch-depth: 0
                  ssh-key: ${{ secrets.POI_CICD_SSH_KEY }}
                  ssh-strict: "false"

            - name: create branch and push
              run: |
                  cd ./cayman-poi
                  BRANCH_NAME="test/poi-submodule/${GITHUB_SHA::7}"

                  if git show-ref --verify --quiet "refs/heads/$BRANCH_NAME"; then
                      echo "Branch $BRANCH_NAME already exists locally, switching to it"
                      git checkout "$BRANCH_NAME"
                  elif git show-ref --verify --quiet "refs/remotes/origin/$BRANCH_NAME"; then
                      echo "Branch $BRANCH_NAME exists remotely, checking it out"
                      git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
                  else
                      echo "Branch $BRANCH_NAME does not exist, creating it"
                      git checkout -b "$BRANCH_NAME"
                  fi

                  pushd poi/src

                  # sometimes the sha is not available yet
                  git fetch

                  git checkout ${GITHUB_SHA::7}
                  popd
                  git add poi/src
                  git config --global user.email "poi-cd@broadcom.com"
                  git config --global user.name "POI CI/CD"

                  # Only commit if there are changes
                  if git diff --cached --quiet; then
                      echo "No changes to commit, skipping commit step"
                  else
                      git commit -m "update poi/src to ${GITHUB_SHA::7} for testing branch ${{ github.ref_name }}"
                  fi

                  git push origin "$BRANCH_NAME"

            - name: Wait for and monitor cayman-poi workflow
              env:
                  GITHUB_TOKEN: ${{ secrets.PAT_POI_CICD_RO }}
              run: |
                  BRANCH_NAME="test/poi-submodule/${GITHUB_SHA::7}"
                  echo "Monitoring branch: $BRANCH_NAME"

                  # Function to get workflow runs for the branch
                  get_workflow_runs() {
                      curl -s \
                          -H "Accept: application/vnd.github+json" \
                          -H "Authorization: Bearer $GITHUB_TOKEN" \
                          -H "X-GitHub-Api-Version: 2022-11-28" \
                          "https://github-vcf.devops.broadcom.net/api/v3/repos/vcf/cayman-poi/actions/runs?branch=$BRANCH_NAME" \
                          | jq -r '.workflow_runs[] | select(.head_branch == "'$BRANCH_NAME'") | "\(.id) \(.status) \(.conclusion // "null") \(.created_at)"'
                  }

                  # Wait for workflow to start (up to 5 minutes)
                  echo "Waiting for workflow to start on branch $BRANCH_NAME..."
                  timeout=300
                  interval=30
                  elapsed=0

                  while [ $elapsed -lt $timeout ]; do
                      runs=$(get_workflow_runs)
                      if [ -n "$runs" ]; then
                          echo "Workflow detected:"
                          echo "$runs"
                          break
                      fi
                      echo "No workflow found yet, waiting ${interval}s..."
                      sleep $interval
                      elapsed=$((elapsed + interval))
                  done

                  if [ $elapsed -ge $timeout ]; then
                      echo "ERROR: No workflow was triggered within $timeout seconds"
                      exit 1
                  fi

                  # Get the workflow run ID (first field from the output)
                  run_id=$(echo "$runs" | head -1 | cut -d' ' -f1)
                  echo "Monitoring workflow run ID: $run_id"

                  # Monitor workflow status (up to 1 hour)
                  timeout=3600
                  interval=60
                  elapsed=0

                  while [ $elapsed -lt $timeout ]; do
                      run_info=$(curl -s \
                          -H "Accept: application/vnd.github+json" \
                          -H "Authorization: Bearer $GITHUB_TOKEN" \
                          -H "X-GitHub-Api-Version: 2022-11-28" \
                          "https://github-vcf.devops.broadcom.net/api/v3/repos/vcf/cayman-poi/actions/runs/$run_id")

                      status=$(echo "$run_info" | jq -r '.status')
                      conclusion=$(echo "$run_info" | jq -r '.conclusion // "null"')
                      url=$(echo "$run_info" | jq -r '.html_url')

                      echo "Workflow status: $status, conclusion: $conclusion"

                      if [ "$status" = "completed" ]; then
                          echo "Workflow completed with conclusion: $conclusion"
                          echo "Workflow URL: $url"

                          if [ "$conclusion" = "success" ]; then
                              echo "✅ Cayman POI workflow succeeded!"
                              exit 0
                          elif [ "$conclusion" = "null" ]; then
                              echo "❌ Cayman POI workflow completed but conclusion is null"
                              echo "Check the workflow at: $url"
                              exit 1
                          else
                              echo "❌ Cayman POI workflow failed with conclusion: $conclusion"
                              echo "Check the workflow at: $url"
                              exit 1
                          fi
                      fi

                      sleep $interval
                      elapsed=$((elapsed + interval))
                  done

                  echo "ERROR: Workflow did not complete within $timeout seconds"
                  echo "Workflow URL: $url"
                  exit 1

    ova-poi-harness:
        runs-on: [ self-hosted, "docker:root" ]
        needs: build-container

        strategy:
            fail-fast: false
            matrix:
                include:
                    - name: minimal
                      ova_config: minimal_ova.yaml
                      release: "5.0"
                    - name: minimal_lvm
                      ova_config: minimal_ova.yaml
                      release: "5.0"
                    - name: minimal_secure
                      ova_config: minimal_ova.yaml
                      release: "5.0"
                    - name: minimal
                      ova_config: minimal_ova.yaml
                      release: "4.0"
                    - name: gitlab-runner
                      ova_config: gitlab-runner_ova.yaml
                      release: "5.0"

        steps:
            - name: Checkout code
              uses: actions/checkout@master

            - name: Checkout POI Harness
              uses: actions/checkout@master
              with:
                  repository: vcf/photon-os-installer-harness
                  ref: main
                  path: ./poi-harness
                  ssh-key: ${{ secrets.POI_CICD_SSH_KEY }}
                  ssh-strict: "false"

            - name: build OVA
              env:
                  NAME: ${{ matrix.name }}
                  OVA_CONFIG: ${{ matrix.ova_config }}
                  PHOTON_RELEASE: ${{ matrix.release }}
              run: |
                  POI_IMAGE_NAME=${POI_IMAGE_NAME_BASE}:${GITHUB_SHA::7}
                  KS_FILE=${NAME}_ks.yaml
                  PHOTON_RELEASE="${PHOTON_RELEASE:-5.0}"
                  VM_NAME_BASE=ova-poi-harness-${GITHUB_SHA::7}
                  VM_NAME=${VM_NAME_BASE}-${NAME}-${PHOTON_RELEASE}
                  # used by pytest
                  export VC_IP=${{ vars.VC_IP }}
                  export VC_USER=${{ vars.VC_USER }}
                  export VC_PASSWORD=${{ secrets.VC_PASSWORD }}

                  docker pull ${POI_REGISTRY}/${POI_IMAGE_NAME}
                  docker tag ${POI_REGISTRY}/${POI_IMAGE_NAME} ${POI_IMAGE_NAME}

                  pushd examples/ova
                  docker run --rm --privileged -v/dev:/dev -v$(pwd):/workdir ${POI_IMAGE_NAME} create-image --repo-paths= -c ${KS_FILE} -v ${PHOTON_RELEASE}
                  docker run --rm -v$(pwd):/workdir -w/workdir ${POI_IMAGE_NAME} create-ova --ovf --mf --vmdk --installer-config ${KS_FILE} --ova-config ${OVA_CONFIG} --compression-level 6
                  sudo chown -R $(id -u -n):$(id -g -n) .
                  popd

                  (cd poi-harness && echo "poi-harness sha is $(git rev-parse --short HEAD)")
                  pytest ./poi-harness/ci/pytest/ -rs --deploy --ovf examples/ova/*.ovf --name ${VM_NAME} --ks_config examples/ova/${KS_FILE} --ova_config examples/ova/${OVA_CONFIG}

    iso-poi-harness:
        runs-on: [ self-hosted, "docker:root" ]
        needs: build-container
        strategy:
            fail-fast: false
            matrix:
                name: [minimal]
                boot_mode: [efi, bios]
        steps:
            - name: Checkout code
              uses: actions/checkout@master

            - name: Checkout POI Harness
              uses: actions/checkout@master
              with:
                  repository: vcf/photon-os-installer-harness
                  ref: main
                  path: ./poi-harness
                  ssh-key: ${{ secrets.POI_CICD_SSH_KEY }}
                  ssh-strict: false

            - name: build ISO
              env:
                  NAME: ${{ matrix.name }}
                  BOOT_MODE: ${{ matrix.boot_mode }}
              run: |
                  POI_IMAGE_NAME=${POI_IMAGE_NAME_BASE}:${GITHUB_SHA::7}
                  VM_NAME_BASE=poi-iso-boot-${GITHUB_SHA::7}
                  ISO_OVF=photon5-iso-boot-${GITHUB_SHA::7}-${BOOT_MODE}.ovf
                  KS_FILE=${NAME}_ks.yaml
                  VM_NAME=${VM_NAME_BASE}-${NAME}-${BOOT_MODE}
                  # used by pytest
                  export VC_IP=${{ vars.VC_IP }}
                  export VC_USER=${{ vars.VC_USER }}
                  export VC_PASSWORD=${{ secrets.VC_PASSWORD }}

                  docker pull ${POI_REGISTRY}/${POI_IMAGE_NAME}
                  docker tag ${POI_REGISTRY}/${POI_IMAGE_NAME} ${POI_IMAGE_NAME}

                  pushd examples/iso
                  # create ISO
                  docker run --rm --privileged -v/dev:/dev -v$(pwd):/workdir ${POI_IMAGE_NAME} photon-iso-builder --config iso.yaml
                  # create OVF that includes that ISO, and a blank hard disk
                  # base64 encode the ks file, must have no new lines (-w0)
                  KSDATA64=$(base64 -w0 < ${KS_FILE})
                  docker run --rm -v$(pwd):/workdir ${POI_IMAGE_NAME} ova-compose -i iso_ova.yaml -o ${ISO_OVF} -m --param ksdata64=${KSDATA64} --param boot_mode=${BOOT_MODE}
                  sudo chown -R $(id -u -n):$(id -g -n) .
                  popd

                  (cd poi-harness && echo "poi-harness sha is $(git rev-parse --short HEAD)")
                  pytest ./poi-harness/ci/pytest/ -rs --deploy --ovf examples/iso/${ISO_OVF} --name ${VM_NAME} --ks_config examples/iso/${KS_FILE} --ova_config examples/iso/iso_ova.yaml --param ksdata64=${KSDATA64} --param boot_mode=${BOOT_MODE}

    update-cayman-poi-master:
        runs-on: self-hosted
        needs:
            - cayman_poi
            - ova-poi-harness
            - iso-poi-harness
        if: github.ref_name == 'master'
        steps:
            - name: Checkout Cayman POI
              uses: actions/checkout@v3
              with:
                  repository: vcf/cayman-poi
                  ref: vmware-master
                  path: ./cayman-poi
                  submodules: "true"
                  fetch-depth: 0
                  ssh-key: ${{ secrets.POI_CICD_SSH_KEY }}
                  ssh-strict: "false"

            - name: update poi/src and push to vmware-master
              run: |
                  cd ./cayman-poi
                  git config --global user.email "poi-cd@broadcom.com"
                  git config --global user.name "POI CI/CD"

                  pushd poi/src

                  # sometimes the sha is not available yet
                  git fetch

                  git checkout ${GITHUB_SHA::7}
                  popd

                  git add poi/src

                  # Only commit if there are changes
                  if git diff --cached --quiet; then
                      echo "No changes to commit, skipping commit step"
                  else
                      git commit -m "update poi/src to ${GITHUB_SHA::7}"
                      git push origin vmware-master
                  fi

    github-public:
        runs-on: self-hosted
        needs:
            - cayman_poi
            - ova-poi-harness
            - iso-poi-harness
#        if: github.ref_name == 'master'
        # disable until we figured out the authentication problem
        if: false
        steps:
           - name: Checkout code
             uses: actions/checkout@master
             with:
                 fetch-depth: 0
           - name: push to public GitHub
             run: |
                 # token needs "repo, workflow" permissions
                 git remote add github https://gerrit-photon:${{ secrets.GERRIT_PHOTON_GITHUB_TOKEN }}@github.com/vmware/photon-os-installer.git || true
                 git push github