feat(ci): decouple cpc build tools (#671)

* feat(ci): decouple with cpc_build_tools

* chore(deps): bump skopeo version to 1.20.0

* chore: bump skopeo runner to 24.04

Author: zhijie-yang

.github/actions/upload-rock/action.yml

@@ -48,7 +48,7 @@ runs:
       run: |
         docker run --rm \
           -v $PWD:/workdir -w /workdir \
-          "quay.io/skopeo/stable:v1.15.1" \
+          "quay.io/skopeo/stable:v1.20.0" \
           inspect oci-archive:${{ inputs.artifact_name }} > metadata.json
 
         digest="$(cat metadata.json | jq -r .Digest)"
@@ -60,7 +60,7 @@ runs:
         for tag in ${{ inputs.tags }}; do
           docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
           -v $PWD:/workdir -w /workdir \
-          "quay.io/skopeo/stable:v1.15.1" \
+          "quay.io/skopeo/stable:v1.20.0" \
           copy \
           --dest-username "${{ inputs.username }}" \
           --dest-password "${{ inputs.password }}" \

.github/workflows/Image.yaml

@@ -194,7 +194,7 @@ jobs:
     secrets: inherit
 
   prepare-upload:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [prepare-build, build-rock, test-rock]
     name: Prepare upload
     if: ${{ inputs.upload || (github.ref_name == 'main' && github.event_name == 'push') }}
@@ -276,7 +276,7 @@ jobs:
           key: ${{ steps.prepare-matrix.outputs.revision-data-cache-key }}
 
   upload:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [prepare-build, prepare-upload]
     name: Upload
     strategy:
@@ -301,7 +301,7 @@ jobs:
           ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
           CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
           # CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
-          SKOPEO_BRANCH: "v1.9.1"
+          SKOPEO_BRANCH: "v1.20.0"
           SKOPEO_URL: "https://github.com/containers/skopeo"
         run: |
           ./src/image/requirements.sh

.github/workflows/Release.yaml

@@ -74,7 +74,7 @@ jobs:
 
   
   do-releases:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     name: Release
     needs: [validate-push-release-request]
     outputs:
@@ -109,7 +109,7 @@ jobs:
           ROCKS_DEV_LP_USERNAME: ${{ secrets.ROCKS_DEV_LP_USERNAME }}
           CPC_BUILD_TOOLS_REPO: git.launchpad.net/~cloudware/cloudware/+git/cpc_build_tools
           # CPC_BUILD_TOOLS_REPO_REF: 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
-          SKOPEO_BRANCH: 'v1.9.1'
+          SKOPEO_BRANCH: 'v1.20.0'
           SKOPEO_URL: 'https://github.com/containers/skopeo'
         run: | 
           ./src/image/requirements.sh

.github/workflows/Test-Rock.yaml

@@ -59,7 +59,7 @@ env:
   VULNERABILITY_REPORT_SUFFIX: ".vulnerability-report.json"  # TODO: inherit string from caller
   TEST_IMAGE_NAME: "test-img"
   TEST_IMAGE_TAG: "test"
-  SKOPEO_IMAGE: "quay.io/skopeo/stable:v1.15.1"
+  SKOPEO_IMAGE: "quay.io/skopeo/stable:v1.20.0"
   UMOCI_VERSION: "v0.4.7"
   UMOCI_BINARY: "umoci.amd64"
   DIVE_IMAGE: "wagoodman/dive:v0.12"

.github/workflows/Vulnerability-Scan.yaml

@@ -29,7 +29,7 @@ on:
 
 env:
   VULNERABILITY_REPORT_SUFFIX: '.vulnerability-report.json' # TODO: inherit string from caller
-  SKOPEO_IMAGE: 'quay.io/skopeo/stable:v1.15.1'
+  SKOPEO_IMAGE: 'quay.io/skopeo/stable:v1.20.0'
 
 jobs:
   configure-scan:

src/image/requirements.sh

@@ -1,41 +1,13 @@
 #!/bin/bash -e
 
-# LP configurations
-mkdir -p $HOME/.ssh
-chmod 700 $HOME/.ssh
-
-ssh-keyscan -H git.launchpad.net | tee $HOME/.ssh/known_hosts
-
-echo "${ROCKS_DEV_LP_SSH_PRIVATE}" >$HOME/.ssh/id_rsa
-
-if [[ "$RUNNER_DEBUG" == "1" ]]; then
-  set -x
-fi
-chmod 600 $HOME/.ssh/id_rsa
-
-## To avoid installing Snaps, just take the needed Python script
-## for later tagging and publishing rocks
-git init /tmp/cpc-build-tools
-pushd /tmp/cpc-build-tools
-git remote add origin git+ssh://${ROCKS_DEV_LP_USERNAME}@${CPC_BUILD_TOOLS_REPO}
-# using main instead of ${CPC_BUILD_TOOLS_REPO_REF} because of an unexpected
-# new issue with Launchpad: 
-# error: Server does not allow request for unadvertised object 9b716ed8a8ba728d036b54b1bb17a8f49dbda434
-git fetch --depth 1 origin devel # ${CPC_BUILD_TOOLS_REPO_REF}
-git checkout FETCH_HEAD
-
-sudo mv /tmp/cpc-build-tools/* /usr/local/bin/
-sudo chmod +x /usr/local/bin/oci_registry_upload.py
-ln -s oci_registry_upload.py /usr/local/bin/cpc-build-tools.oci-registry-upload
-popd
-##
-
 ## Install Skopeo
 git clone -b ${SKOPEO_BRANCH} --depth 1 ${SKOPEO_URL} /tmp/skopeo
 pushd /tmp/skopeo
 
 docker run -v $PWD:/src -w /src -e DISABLE_DOCS=1 \
-    golang:1.18 sh -c 'apt update; apt install -y libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config; make'
+    golang:1.25 sh -c \
+    'apt update; apt install -y libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config; \
+    git config --global --add safe.directory /src; make'
 
 sudo mv bin/skopeo /usr/local/bin/
 sudo chmod +x /usr/local/bin/skopeo

src/image/tag_and_publish.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -ex
 
-source $(dirname $0)/../shared/logs.sh
+source "$(dirname "$0")/../shared/logs.sh"
 
 # SUITE="{{ suite }}"
 # RELEASE="{{ release }}"
@@ -21,14 +21,14 @@ source_img="${1}"
 image_name="${2}"
 shift 2
 # The tag names are handled by the CI
-tag_names=($@)
+tag_names=("$@")
 
 publish_with_auth_token()
 {
     local token=$1 name=$2
     shift 2
     log_info "Publishing to Docker registry repository $name ..."
-    REGISTRY_AUTH=$token cpc-build-tools.oci-registry-upload \
+    REGISTRY_AUTH=$token "$(dirname "$0")/../uploads/oci_registry_upload.py" \
         "${source_img}" "$name" "$@"
     log_info "Publishing to Docker registry repository $name finished"
 }
@@ -95,7 +95,7 @@ acr_repo_name="${ACR_NAMESPACE}/${image_name}"
 ecr_repo_name="${ECR_NAMESPACE}/${image_name}"
 # ecr_lts_repo_name="${ECR_LTS_NAMESPACE}/${image_name}"
 
-log_info "Publishing ${image_name} to registries with tags: ${tag_names[@]}"
+log_info "Publishing ${image_name} to registries with tags: ${tag_names[*]}"
 
 trace_suspend
 if [ ! -z $GHCR_REPO ]; then

src/tests/get_released_revisions.py

@@ -5,7 +5,7 @@
 form and return a list of image names in their canonical format, i.e.:
     ghcr.io/canonical/oci-factory/<img-name>:<canonical-track>_<revision>
     ...
-    
+
 TODO: this script could eventually be adjusted and converted to a Temporal
 Activity that runs from within a scheduled workflow.
 """
@@ -20,7 +20,7 @@
 
 from ..shared.logs import get_logger
 
-SKOPEO_IMAGE = os.getenv("SKOPEO_IMAGE", "quay.io/skopeo/stable:v1.15.1")
+SKOPEO_IMAGE = os.getenv("SKOPEO_IMAGE", "quay.io/skopeo/stable:v1.20.0")
 REGISTRY = "ghcr.io/canonical/oci-factory"
 
 logger = get_logger(stream=sys.stdout, level="INFO")

src/uploads/oci_registry_upload.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import os.path
+import sys
+import tempfile
+from pathlib import Path
+from subprocess import check_call
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from shared.logs import get_logger
+
+logger = get_logger()
+
+
+def parse_args():
+    argp = argparse.ArgumentParser(description=__doc__)
+    argp.add_argument("source_uri")
+    argp.add_argument("target_name")
+    argp.add_argument("target_tags", nargs="+")
+    argp.add_argument("--registry-auth")
+    args = argp.parse_args()
+    if not args.registry_auth:
+        args.registry_auth = os.getenv("REGISTRY_AUTH")
+    return args
+
+
+def main():
+    args = parse_args()
+    base_cmd = ["skopeo", "copy", "--preserve-digests", args.source_uri]
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        if args.registry_auth:
+            auth_config = {"auths": {args.target_name: {"auth": args.registry_auth}}}
+            auth_file = os.path.join(tmp_dir, "auth.json")
+            with open(auth_file, "w") as f:
+                os.fchmod(f.fileno(), 0o600)
+                json.dump(auth_config, f)
+            base_cmd += ["--authfile", auth_file]
+
+        target_uri = "docker://" + args.target_name + ":" + args.target_tags[0]
+        cmd = base_cmd + ["--multi-arch", "all", target_uri]
+        logger.info(" ".join(cmd))
+        check_call(cmd)
+
+        for tag in args.target_tags[1:]:
+            target_uri = "docker://" + args.target_name + ":" + tag
+            cmd = base_cmd + ["--multi-arch", "index-only", target_uri]
+            logger.info(" ".join(cmd))
+            check_call(cmd)
+
+
+if __name__ == "__main__":
+    main()