Pipeline

.one-pipeline.yaml

@@ -0,0 +1,214 @@
+version: '1'
+
+setup:
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+    echo "setup"
+
+test:
+  dind: true
+  abort_on_failure: true
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+    # Download and configure golang
+    wget --header "Accept: application/octet-stream"  "https://golang.org/dl/go1.16.linux-amd64.tar.gz" 
+    rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
+    export PATH=$PATH:/usr/local/go/bin
+    apt-get update
+    apt-get -y install build-essential
+    # Run unit tests
+    #export DOCKER_USERNAME=$(get_env docker-username)
+    #export DOCKER_PASSWORD=$(get_env docker-password)
+    make unit-test
+    #make docker-login
+    #make build-image
+
+static-scan:
+  dind: true
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+    read -r SONAR_HOST_URL <<< "$(get_env sonarqube | jq -r '.parameters.dashboard_url' | sed 's:/*$::')"
+    read -r SONAR_USER <<< "$(get_env sonarqube | jq -r '.parameters.user_login')"
+    SONARQUBE_INSTANCE_ID=$(get_env sonarqube | jq -r '.instance_id')
+    read -r SONAR_PASS <<< "$(jq -r --arg sonar_instance "$SONARQUBE_INSTANCE_ID" '[.services[] | select(."service_id"=="sonarqube")][] | select(."instance_id"==$sonar_instance) | .parameters.user_password' /toolchain/toolchain.json)"  
+    touch "$WORKSPACE"/open-liberty-operator/sonar-project.properties
+    cat << EOF > "$WORKSPACE"/open-liberty-operator/sonar-project.properties
+    sonar.projectKey=open-liberty-operator
+    sonar.host.url=$SONAR_HOST_URL
+    sonar.sources=.
+    sonar.login=$SONAR_USER
+    sonar.password=$SONAR_PASS
+    sonar.c.file.suffixes=-
+    sonar.cpp.file.suffixes=-
+    sonar.objc.file.suffixes=-
+    EOF
+    chmod -x "$WORKSPACE"/open-liberty-operator/sonar-project.properties
+    #echo "$SONAR_PASS" >> /tmp/sonarqube-token
+    "${ONE_PIPELINE_PATH}"/internal/sonarqube/sonarqube_run
+    set_env ibmcloud-api-orig $(get_env ibmcloud-api)
+    set_env ibmcloud-api cloud.ibm.com
+
+containerize:
+  dind: true
+  abort_on_failure: true
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+
+    if [[ "$PIPELINE_DEBUG" == 1 ]]; then
+      trap env EXIT
+      env
+      set -x
+    fi
+    set_env ibmcloud-api $(get_env ibmcloud-api-orig)
+    # Download and configure golang
+    wget --header "Accept: application/octet-stream"  "https://golang.org/dl/go1.16.linux-amd64.tar.gz"
+    rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
+    export PATH=$PATH:/usr/local/go/bin
+    apt-get update
+    apt-get -qq -y install build-essential software-properties-common uidmap
+    # Download and install skopeo
+    if ! command -v skopeo &> /dev/null; then
+      if [ ! -f "/apt/sources.list.d/devel:kubic:libcontainers:stable.list" ]; then
+        sudo sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
+        wget --no-check-certificate -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_18.04/Release.key -O- | sudo apt-key add -
+        sudo apt-get update -qq
+      fi
+      sudo apt-get -y install skopeo
+    else
+      skopeo --version
+    fi
+    # Build images
+    export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
+    export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key)
+    PIPELINE_REGISTRY=$(get_env pipeline-registry)
+    PIPELINE_OPERATOR_IMAGE=$(get_env pipeline-operator-image)
+    echo "TRIGGER_TYPE=$TRIGGER_TYPE"
+    export RELEASE_TARGET=$(get_env branch)
+    export PR_NUMBER=1234
+    # Build amd64 image
+    make build-pipeline-releases
+    # Build ppc64le and s390x images
+    # ./scripts/pipeline/launch-travis.sh -t $(get_env travis-token) -r "https://github.com/OpenLiberty/open-liberty-operator" -b $(get_env branch) -l
+    # Build manifest
+    make build-pipeline-manifest
+    # Build bundle image
+    ./scripts/pipeline/launch-catalog-build.sh -t $(get_env travis-token) -r "https://github.com/OpenLiberty/open-liberty-operator" -b $(get_env branch) -l
+    # Save artifacts
+    # declare -a tags=("daily-amd64" "daily-ppc64le" "daily-s390x")
+    declare -a tags=("daily-amd64")
+    for i in "${tags[@]}"
+    do
+      IMAGE=$PIPELINE_REGISTRY/$PIPELINE_OPERATOR_IMAGE:$i
+      DIGEST="$(skopeo inspect docker://$IMAGE | grep Digest | grep -o 'sha[^\"]*')"
+      ARCH=$(echo $i | cut -d'-' -f 2)
+      echo "Saving artifact $i name=$IMAGE digest=$DIGEST"
+      save_artifact $i type=image name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"
+    done
+    declare -a catalogs=("catalog-daily")
+    for i in "${catalogs[@]}"
+    do
+      IMAGE=$PIPELINE_REGISTRY/$PIPELINE_OPERATOR_IMAGE:$i
+      DIGEST="$(skopeo inspect docker://$IMAGE | grep Digest | grep -o 'sha[^\"]*')"
+      ARCH=$(echo $i | cut -d'-' -f 2)
+      echo "Saving artifact $i name=$IMAGE digest=$DIGEST"
+      save_artifact $i type=image name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"
+    done
+
+deploy:
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+
+    if [[ "$PIPELINE_DEBUG" == 1 ]]; then
+      trap env EXIT
+      env
+      set -x
+    fi
+    echo "deploy"
+
+dynamic-scan: 
+  abort_on_failure: false
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+    echo "dynamic-scan"
+    #export APP_URL=$(cat ../app-url)
+    # feature preivew this until evidence locker v2 usage is full feature ready 
+    # can be triggered, and owasp will run for preview purposes 
+    #source scripts/zap/trigger_api_scan.sh
+
+acceptance-test:
+  dind: true  
+  abort_on_failure: true
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+  script: |
+    #!/usr/bin/env bash
+    echo "acceptance-test"
+    # Download and configure golang
+    wget --header "Accept: application/octet-stream"  "https://golang.org/dl/go1.16.linux-amd64.tar.gz"
+    rm -rf /usr/local/go && tar -C /usr/local -xf go1.16.linux-amd64.tar.gz
+    export PATH=$PATH:/usr/local/go/bin
+    apt-get update
+    apt-get -y install build-essential
+    export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
+    export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key)
+    export DOCKER_USERNAME=$(get_env docker-username)
+    export DOCKER_PASSWORD=$(get_env docker-password)
+    export CLUSTER_URL=$(get_env test-cluster-url)
+    export CLUSTER_TOKEN=$(get_env test-cluster-token)
+    export TRAVIS_BUILD_NUMBER=$BUILD_NUMBER
+    make setup
+    make test-pipeline-e2e
+
+scan-artifact:
+  abort_on_failure: false
+  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.6@sha256:7f588468622a981f89cf5e1212aaf75fface9da6169b5345ca52ab63d8215907
+  script: |
+    #!/usr/bin/env bash
+    echo "twistlock-scan"
+    ./scripts/pipeline/twistlock-scan.sh
+    echo "VA scan"
+    . scripts/pipeline/va_scan
+    if which list_artifacts >/dev/null; then
+      list_artifacts | while IFS= read -r artifact; do
+        image="$(load_artifact "$artifact" "name")"
+        type="$(load_artifact "$artifact" "type")"
+        digest="$(load_artifact "$artifact" "digest")"
+        name="$(echo "$artifact" | awk '{print $1}')"
+
+        if [[ "$type" == "image" ]]; then
+          if [[ "$image" == *"icr.io"* ]]; then
+            start_va_scan "$name" "$image" "$digest"
+          fi
+        fi
+      done
+    fi
+
+release:
+  abort_on_failure: false
+  image: wcp-compliance-automation-team-docker-local.artifactory.swg-devops.com/ibm-compliance-automation:1.9.1@sha256:3f3e344a1efb160d83c48cf2ee878a39cbad058c8640c423472e0546316232fd
+  script: |
+    #!/usr/bin/env bash
+    echo "release"
+    set_env ibmcloud-api-key $(get_env ibmcloud-api-key-prod)
+    echo -n "$(get_env ibmcloud-api-key-prod)" > /config/ibmcloud-api-key
+    set_env ibmcloud-api "cloud.ibm.com"
+
+#owasp-zap-api:
+#  dind: true
+#  abort_on_failure: false
+#  image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
+#  script: |
+#    !/usr/bin/env bash
+#    if [[ "$PIPELINE_DEBUG" == 1 ]]; then
+#      trap env EXIT
+#      env
+#      set -x
+#    fi
+
+#    source scripts/zap/setup_api_scan.sh
+#    source scripts/zap/run_api_scan.sh

.travis.yml

@@ -14,6 +14,7 @@ services:
 
 stages:
   - name: unit-test
+    if: branch = master AND fork = false
   # Builds are split into 'e2e-test' and 'build' to allow e2e tests to run first. If e2e fails, don't bother
   # building and pushing the images for the other architectures.
   - name: e2e-test

Makefile

@@ -8,6 +8,9 @@ OPERATOR_SDK_RELEASE_VERSION ?= v1.6.4
 VERSION ?= 0.8.0
 
 OPERATOR_IMAGE ?= openliberty/operator
+PIPELINE_REGISTRY ?= cp.stg.icr.io
+PIPELINE_REGISTRY_NAMESPACE ?= cp
+PIPELINE_OPERATOR_IMAGE ?= ${PIPELINE_REGISTRY_NAMESPACE}/olo-operator
 
 # Type of release. Can be "daily", "releases", or a release tag.
 RELEASE_TARGET := $(or ${RELEASE_TARGET}, ${TRAVIS_TAG}, daily)
@@ -231,19 +234,41 @@ unit-test: ## Run unit tests
 docker-login:
 	docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}"
 
+build-pipeline-multiarch-image: ## Build operator image
+	./scripts/build-releases.sh -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_OPERATOR_IMAGE}"
+
+push-pipeline-multiarch-image: ## Push operator image
+	./scripts/build-releases.sh --push -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_OPERATOR_IMAGE}"
+
 build-manifest: setup-manifest
 	./scripts/build-manifest.sh --image "${PUBLISH_REGISTRY}/${OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
 
+build-pipeline-manifest: setup-manifest
+	./scripts/build-manifest.sh -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_REGISTRY}/${PIPELINE_OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
+
 test-e2e:
 	./scripts/e2e-release.sh --registry-name default-route --registry-namespace openshift-image-registry \
                      --test-tag "${TRAVIS_BUILD_NUMBER}" --target "${RELEASE_TARGET}"
 
+test-pipeline-e2e:
+	./scripts/pipeline/fyre-e2e.sh -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" \
+                     --cluster-url "${CLUSTER_URL}" --cluster-token "${CLUSTER_TOKEN}" \
+                     --registry-name "${PIPELINE_REGISTRY}" --registry-namespace "${PIPELINE_REGISTRY_NAMESPACE}" \
+                     --registry-user "${PIPELINE_USERNAME}" --registry-password "${PIPELINE_PASSWORD}" \
+                     --test-tag "${TRAVIS_BUILD_NUMBER}" --release "${RELEASE_TARGET}"
+
 build-releases:
 	./scripts/build-releases.sh --image "${PUBLISH_REGISTRY}/${OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
 
+build-pipeline-releases:
+	./scripts/build-releases.sh -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_REGISTRY}/${PIPELINE_OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
+
 bundle-releases:
 	./scripts/bundle-releases.sh --image "${PUBLISH_REGISTRY}/${OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
 
+bundle-pipeline-releases:
+	./scripts/bundle-releases.sh -u "${PIPELINE_USERNAME}" -p "${PIPELINE_PASSWORD}" --registry "${PIPELINE_REGISTRY}" --image "${PIPELINE_REGISTRY}/${PIPELINE_OPERATOR_IMAGE}" --target "${RELEASE_TARGET}"
+
 install-podman:
 	./scripts/installers/install-podman.sh
 
@@ -261,3 +286,6 @@ build-catalog:
 
 push-catalog: docker-login
 	podman push --format=docker "${CATALOG_IMG}"
+
+push-pipeline-catalog:
+	podman push --format=docker "${CATALOG_IMG}"

scripts/build-manifest.sh

@@ -36,7 +36,11 @@ main() {
     exit 1
   fi
 
-  echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin
+  if [[ -z "${REGISTRY}" ]]; then
+    echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin
+  else
+    echo "${DOCKER_PASSWORD}" | docker login "${REGISTRY}" -u "${DOCKER_USERNAME}" --password-stdin
+  fi
 
   # Build manifest for target release(s)
   if [[ "${TARGET}" != "releases" ]]; then
@@ -98,6 +102,10 @@ parse_args() {
       shift
       readonly DOCKER_PASSWORD="${1}"
       ;;
+    --registry)
+      shift
+      readonly REGISTRY="${1}"
+      ;;
     --image)
       shift
       readonly IMAGE="${1}"

scripts/build-release.sh

@@ -55,10 +55,20 @@ main() {
     readonly release_tag="${RELEASE}"
   fi
 
-  readonly full_image="${IMAGE}:${release_tag}-${arch}"
+  if [[ -z "${PR_NUMBER}" ]]; then
+    readonly full_image="${IMAGE}:${release_tag}-${arch}"
+  else
+    readonly full_image="${IMAGE}:${PR_NUMBER}-${release_tag}"
+  fi
+
+  echo "full_image=$full_image"
 
   ## login to docker
-  echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin
+  if [[ -z "${REGISTRY}" ]]; then 
+    echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin
+  else
+    echo "${DOCKER_PASSWORD}" | docker login "${REGISTRY}" -u "${DOCKER_USERNAME}" --password-stdin
+  fi   
 
   ## build or push latest main branch
   echo "****** Building release: ${RELEASE}"
@@ -99,6 +109,10 @@ parse_args() {
       shift
       readonly DOCKER_PASSWORD="${1}"
       ;;
+    --registry)
+      shift
+      readonly REGISTRY="${1}"
+      ;;
     --image)
       shift
       readonly IMAGE="${1}"