Skip to content

Commit aac0df4

Browse files
yangchiuyangchiu
committed
ci: extend mirror-csi-images pipeline to suse storage images and use ecr as private registry for appco testing
(1) extend mirror-csi-images pipeline to support mirroring suse storage images (2) use ecr as private registry for appco testing instead of creating private registry on demand (3) remove longhorn_manifest.sh from appco pipeline since it always installs via helm chart Signed-off-by: Yang Chiu <yang.chiu@suse.com>
1 parent aa8a832 commit aac0df4

11 files changed

Lines changed: 69 additions & 837 deletions

File tree

‎mirror_csi_images/Dockerfile.setup‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,15 @@ ENV WORKSPACE /src/longhorn-tests
44

55
WORKDIR $WORKSPACE
66

7-
RUN apk add --no-cache skopeo docker jq bash grep
7+
COPY SUSE_Trust_Root.crt /usr/local/share/ca-certificates/SUSE_Trust_Root.crt
8+
RUN apk update && \
9+
apk add --no-cache ca-certificates yq curl bash openssl && \
10+
update-ca-certificates && \
11+
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
12+
chmod 700 get_helm.sh && \
13+
./get_helm.sh
14+
15+
RUN apk add --no-cache skopeo docker jq bash grep aws-cli
816

917
COPY --from=docker/buildx-bin:v0.30 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
1018

‎mirror_csi_images/Jenkinsfile‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,32 @@ node {
1010

1111
checkout scm
1212

13-
withCredentials([usernamePassword(credentialsId: 'DOCKER_CREDS', passwordVariable: 'DOCKER_PASSWORD', usernameVariable: 'DOCKER_USERNAME')]) {
13+
withCredentials([
14+
usernamePassword(credentialsId: 'DOCKER_CREDS', passwordVariable: 'DOCKER_PASSWORD', usernameVariable: 'DOCKER_USERNAME'),
15+
usernamePassword(credentialsId: 'AWS_CREDS', passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'),
16+
string(credentialsId: 'AWS_DEFAULT_REGION', variable: 'AWS_DEFAULT_REGION'),
17+
string(credentialsId: 'AWS_PRIVATE_ECR', variable: 'AWS_PRIVATE_ECR'),
18+
file(credentialsId: 'SUSE_INTERNAL_CA', variable: 'CA_CERT_PATH'),
19+
usernamePassword(credentialsId: 'APPCO_CREDS', passwordVariable: 'APPCO_PASSWORD', usernameVariable: 'APPCO_USERNAME'),
20+
]) {
1421
stage('build') {
1522

23+
sh "cp ${CA_CERT_PATH} ./SUSE_Trust_Root.crt"
1624
sh "mirror_csi_images/scripts/build.sh"
1725

1826
sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
1927
--env DOCKER_USERNAME="${DOCKER_USERNAME}" \
2028
--env DOCKER_PASSWORD="${DOCKER_PASSWORD}" \
29+
--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY} \
30+
--env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY} \
31+
--env AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \
32+
--env AWS_PRIVATE_ECR=${AWS_PRIVATE_ECR} \
2133
--env LONGHORN_IMAGES_FILE_URL=${LONGHORN_IMAGES_FILE_URL} \
2234
--env CSI_IMAGES=${CSI_IMAGES} \
35+
--env APPCO_USERNAME=${APPCO_USERNAME} \
36+
--env APPCO_PASSWORD=${APPCO_PASSWORD} \
37+
--dns=10.113.53.53 \
38+
--dns=10.113.53.54 \
2339
${imageName}
2440
"""
2541
}

‎mirror_csi_images/scripts/image-mirror.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ export DOCKER_CLI_EXPERIMENTAL="enabled"
55

66
ARCH_LIST="amd64 arm64 arm s390x"
77

8+
AWS_PAGER=""
9+
810
function copy_if_changed {
911
SOURCE_REF="${1}"
1012
DEST_REF="${2}"
@@ -63,6 +65,10 @@ function mirror_image {
6365

6466
trap 'echo -e "===\nFailed copying image for ${DEST_SPEC}\n==="' ERR
6567

68+
if [[ "${DEST_SPEC}" =~ "${AWS_PRIVATE_ECR}" ]]; then
69+
aws ecr describe-repositories --repository-names "${SOURCE_SPEC}" --region "${AWS_DEFAULT_REGION}" >/dev/null 2>&1 || aws ecr create-repository --repository-name "${SOURCE_SPEC}" --region "${AWS_DEFAULT_REGION}"
70+
fi
71+
6672
# ensure that source specifies an explicit registry and repository
6773
IFS=/ read -a SOURCE <<< ${SOURCE_SPEC}
6874
if grep -vqE '[.:]|localhost' <<< ${SOURCE[0]}; then

‎mirror_csi_images/scripts/publish.sh‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,21 @@ touch "${INFILE}"
66
if [[ -n "${LONGHORN_IMAGES_FILE_URL}" ]]; then
77

88
LONGHORN_IMAGES_FILE=${PWD}/longhorn-images.txt
9-
wget "${LONGHORN_IMAGES_FILE_URL}" -O "${LONGHORN_IMAGES_FILE}"
9+
if [[ "${LONGHORN_IMAGES_FILE_URL}" =~ "oci://dp.apps.rancher.io/charts/suse-storage" ]]; then
10+
set +x
11+
helm registry login dp.apps.rancher.io --username "${APPCO_USERNAME}" --password "${APPCO_PASSWORD}"
12+
set -x
13+
helm pull "${LONGHORN_IMAGES_FILE_URL}" --untar --untardir "${PWD}"
14+
yq '.images[].image' ./suse-storage/images-lock.yaml > "${LONGHORN_IMAGES_FILE}"
15+
else
16+
wget "${LONGHORN_IMAGES_FILE_URL}" -O "${LONGHORN_IMAGES_FILE}"
17+
fi
1018

1119
while read -r LINE; do
12-
if [[ "${LINE}" =~ csi-|livenessprobe ]]; then
20+
if [[ "${LINE}" =~ "registry.suse.de" ]] || [[ "${LINE}" =~ "dp.apps.rancher.io" ]]; then
21+
IFS=: read -ra IMAGE_TAG_PAIR <<< "${LINE}"
22+
echo "${IMAGE_TAG_PAIR[0]}" "${AWS_PRIVATE_ECR}/${IMAGE_TAG_PAIR[0]}" "${IMAGE_TAG_PAIR[1]}" >> "${INFILE}"
23+
elif [[ "${LINE}" =~ csi-|livenessprobe ]]; then
1324
CSI_IMAGE=$(echo "${LINE}" | sed -e "s/longhornio\///g")
1425
IFS=: read -ra IMAGE_TAG_PAIR <<< "${CSI_IMAGE}"
1526
echo "registry.k8s.io/sig-storage/${IMAGE_TAG_PAIR[0]}" "longhornio/${IMAGE_TAG_PAIR[0]}" "${IMAGE_TAG_PAIR[1]}" >> "${INFILE}"
@@ -27,7 +38,9 @@ else
2738
IFS=, read -ra CSI_IMAGES_ARR <<< "${CSI_IMAGES}"
2839
for CSI_IMAGE in "${CSI_IMAGES_ARR[@]}"; do
2940
IFS=: read -ra IMAGE_TAG_PAIR <<< "$CSI_IMAGE"
30-
if [[ "${CSI_IMAGE}" =~ csi-|livenessprobe ]]; then
41+
if [[ "${CSI_IMAGE}" =~ "registry.suse.de" ]] || [[ "${CSI_IMAGE}" =~ "dp.apps.rancher.io" ]]; then
42+
echo "${IMAGE_TAG_PAIR[0]}" "${AWS_PRIVATE_ECR}/${IMAGE_TAG_PAIR[0]}" "${IMAGE_TAG_PAIR[1]}" >> "${INFILE}"
43+
elif [[ "${CSI_IMAGE}" =~ csi-|livenessprobe ]]; then
3144
echo "registry.k8s.io/sig-storage/${IMAGE_TAG_PAIR[0]}" "longhornio/${IMAGE_TAG_PAIR[0]}" "${IMAGE_TAG_PAIR[1]}" >> "${INFILE}"
3245
elif [[ "${CSI_IMAGE}" =~ "support-bundle-kit" ]]; then
3346
echo "rancher/${IMAGE_TAG_PAIR[0]}" "longhornio/${IMAGE_TAG_PAIR[0]}" "${IMAGE_TAG_PAIR[1]}" >> "${INFILE}"
@@ -38,5 +51,7 @@ else
3851
fi
3952

4053
docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}"
54+
echo "${APPCO_PASSWORD}" | docker login dp.apps.rancher.io --username "${APPCO_USERNAME}" --password-stdin
55+
aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${AWS_PRIVATE_ECR}"
4156

4257
"${PWD}/mirror_csi_images/scripts/image-mirror.sh" "${INFILE}"

‎pipelines/appco/Dockerfile.setup‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ WORKDIR $WORKSPACE
1515
# SUSE_Trust_Root.crt stored on Jenkins
1616
COPY SUSE_Trust_Root.crt /usr/local/share/ca-certificates/SUSE_Trust_Root.crt
1717
RUN apk update && \
18-
apk add --no-cache ca-certificates && \
18+
apk add --no-cache ca-certificates aws-cli && \
1919
update-ca-certificates
2020

2121
RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \

‎pipelines/appco/Jenkinsfile‎

Lines changed: 8 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -38,49 +38,6 @@ def RESOURCE_OWNER = "longhorn-infra"
3838
def TEST_TYPE = params.TEST_TYPE ? params.TEST_TYPE : "pytest"
3939
def BLOCK_DEVICE_SIZE = params.TEST_TYPE == "robot" ? 80 : 40
4040

41-
def envArgs
42-
def vars = [
43-
'CUSTOM_LONGHORN_ENGINE_IMAGE',
44-
'CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE',
45-
'CUSTOM_LONGHORN_MANAGER_IMAGE',
46-
'CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE',
47-
'CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE',
48-
'CUSTOM_LONGHORN_UI_IMAGE',
49-
'CUSTOM_LONGHORN_SUPPORT_BUNDLE_IMAGE',
50-
'CUSTOM_LONGHORN_CSI_ATTACHER_IMAGE',
51-
'CUSTOM_LONGHORN_CSI_PROVISIONER_IMAGE',
52-
'CUSTOM_LONGHORN_CSI_NODE_DRIVER_REGISTRAR_IMAGE',
53-
'CUSTOM_LONGHORN_CSI_RESIZER_IMAGE',
54-
'CUSTOM_LONGHORN_CSI_SNAPSHOTTER_IMAGE',
55-
'CUSTOM_LONGHORN_CSI_LIVENESSPROBE_IMAGE',
56-
'STABLE_MANAGER_IMAGE',
57-
'STABLE_ENGINE_IMAGE',
58-
'STABLE_INSTANCE_MANAGER_IMAGE',
59-
'STABLE_SHARE_MANAGER_IMAGE',
60-
'STABLE_BACKING_IMAGE_MANAGER_IMAGE',
61-
'STABLE_UI_IMAGE',
62-
'STABLE_SUPPORT_BUNDLE_IMAGE',
63-
'STABLE_CSI_ATTACHER_IMAGE',
64-
'STABLE_CSI_PROVISIONER_IMAGE',
65-
'STABLE_CSI_NODE_REGISTRAR_IMAGE',
66-
'STABLE_CSI_RESIZER_IMAGE',
67-
'STABLE_CSI_SNAPSHOTTER_IMAGE',
68-
'STABLE_CSI_LIVENESSPROBE_IMAGE',
69-
'TRANSIENT_MANAGER_IMAGE',
70-
'TRANSIENT_ENGINE_IMAGE',
71-
'TRANSIENT_INSTANCE_MANAGER_IMAGE',
72-
'TRANSIENT_SHARE_MANAGER_IMAGE',
73-
'TRANSIENT_BACKING_IMAGE_MANAGER_IMAGE',
74-
'TRANSIENT_UI_IMAGE',
75-
'TRANSIENT_SUPPORT_BUNDLE_IMAGE',
76-
'TRANSIENT_CSI_ATTACHER_IMAGE',
77-
'TRANSIENT_CSI_PROVISIONER_IMAGE',
78-
'TRANSIENT_CSI_NODE_REGISTRAR_IMAGE',
79-
'TRANSIENT_CSI_RESIZER_IMAGE',
80-
'TRANSIENT_CSI_SNAPSHOTTER_IMAGE',
81-
'TRANSIENT_CSI_LIVENESSPROBE_IMAGE'
82-
]
83-
8441
node {
8542

8643
withCredentials([
@@ -92,6 +49,7 @@ node {
9249
string(credentialsId: 'LAB_URL', variable: 'LAB_URL'),
9350
file(credentialsId: 'SUSE_INTERNAL_CA', variable: 'CA_CERT_PATH'), // Add the secret file credential here
9451
string(credentialsId: 'QASE_TOKEN', variable: 'QASE_TOKEN'),
52+
string(credentialsId: 'AWS_PRIVATE_ECR', variable: 'AWS_PRIVATE_ECR'),
9553
]) {
9654

9755
if (params.SEND_SLACK_NOTIFICATION) {
@@ -101,121 +59,6 @@ node {
10159
checkout scm
10260

10361
try {
104-
stage ('fetch test images list') {
105-
sh "cp ${CA_CERT_PATH} ./SUSE_Trust_Root.crt"
106-
sh "pipelines/appco/scripts/build.sh"
107-
sh """ docker run -itd --name fetch-test-images-${JOB_BASE_NAME}-${BUILD_NUMBER} \
108-
-v /var/run/docker.sock:/var/run/docker.sock \
109-
--env TF_VAR_docker_hub_username="${REGISTRY_USERNAME}" \
110-
--env TF_VAR_docker_hub_password="${REGISTRY_PASSWORD}" \
111-
--env APPCO_USERNAME=${APPCO_USERNAME} \
112-
--env APPCO_PASSWORD=${APPCO_PASSWORD} \
113-
--dns=10.113.53.53 \
114-
--dns=10.113.53.54 \
115-
${imageName}
116-
"""
117-
118-
sh """ docker exec \
119-
--env LONGHORN_CHART_URI=${LONGHORN_CHART_URI} \
120-
--env LONGHORN_VERSION=${LONGHORN_VERSION} \
121-
--env APPCO_LONGHORN_COMPONENT_IMAGE_PATH=${APPCO_LONGHORN_COMPONENT_IMAGE_PATH} \
122-
--env LONGHORN_COMPONENT_TAG=${LONGHORN_COMPONENT_TAG} \
123-
--env CSI_ATTACHER_TAG=${CSI_ATTACHER_TAG} \
124-
--env CSI_PROVISIONER_TAG=${CSI_PROVISIONER_TAG} \
125-
--env CSI_RESIZER_TAG=${CSI_RESIZER_TAG} \
126-
--env CSI_SNAPSHOTTER_TAG=${CSI_SNAPSHOTTER_TAG} \
127-
--env CSI_LIVENESSPROBE_TAG=${CSI_LIVENESSPROBE_TAG} \
128-
--env CSI_REGISTRAR_TAG=${CSI_REGISTRAR_TAG} \
129-
--env SUPPORT_BUNDLE_TAG=${SUPPORT_BUNDLE_TAG} \
130-
--env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \
131-
--env LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} \
132-
--env LONGHORN_STABLE_VERSION_CHART_URI=${LONGHORN_STABLE_VERSION_CHART_URI} \
133-
--env LONGHORN_TRANSIENT_VERSION=${LONGHORN_TRANSIENT_VERSION} \
134-
--env LONGHORN_TRANSIENT_VERSION_CHART_URI=${LONGHORN_TRANSIENT_VERSION_CHART_URI} \
135-
fetch-test-images-${JOB_BASE_NAME}-${BUILD_NUMBER} \
136-
bash -c './pipelines/appco/scripts/fetch_image_list.sh && source /tmp/longhorn_env_vars.sh && env'
137-
"""
138-
139-
// Pull selected variables from inside the container back into Jenkins environment
140-
vars.each { varName ->
141-
def value = sh(
142-
script: "docker exec fetch-test-images-${JOB_BASE_NAME}-${BUILD_NUMBER} bash -c 'source /tmp/longhorn_env_vars.sh && echo \$$varName'",
143-
returnStdout: true
144-
).trim()
145-
env."${varName}" = value
146-
echo "${varName} = ${value}"
147-
}
148-
149-
envArgs = vars.collect { key ->
150-
def v = env."${key}" ?: ''
151-
"--env ${key}=${v}"
152-
}.join(" \\\n ")
153-
154-
155-
sh "docker stop fetch-test-images-${JOB_BASE_NAME}-${BUILD_NUMBER}"
156-
sh "docker rm -v fetch-test-images-${JOB_BASE_NAME}-${BUILD_NUMBER}"
157-
sh "docker rmi ${imageName}"
158-
159-
return
160-
}
161-
162-
if (params.AIR_GAP_INSTALLATION) {
163-
stage('private registry build') {
164-
sh "airgap/scripts/build.sh"
165-
sh """ docker run -itd --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \
166-
-v /var/run/docker.sock:/var/run/docker.sock \
167-
--env TF_VAR_docker_hub_username="${REGISTRY_USERNAME}" \
168-
--env TF_VAR_docker_hub_password="${REGISTRY_PASSWORD}" \
169-
--env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \
170-
--env TF_VAR_aws_secret_key=${AWS_SECRET_KEY} \
171-
--env TF_VAR_arch=${ARCH}\
172-
--env TF_VAR_registry_aws_instance_type=${registry_instance_type}\
173-
--env TF_VAR_appco_test="true"\
174-
--dns=10.113.53.53 \
175-
--dns=10.113.53.54 \
176-
airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}
177-
"""
178-
}
179-
180-
stage ('private registry setup') {
181-
sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/terraform-setup.sh"
182-
REGISTRY_URL = sh (
183-
script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_url",
184-
returnStdout: true
185-
)
186-
println REGISTRY_URL
187-
REGISTRY_USERNAME = sh (
188-
script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_username",
189-
returnStdout: true
190-
)
191-
REGISTRY_PASSWORD = sh (
192-
script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_password",
193-
returnStdout: true
194-
)
195-
// Pull appco images and push to private registry
196-
sh """ docker exec \
197-
--env REGISTRY_URL=${REGISTRY_URL} \
198-
--env REGISTRY_USERNAME="${REGISTRY_USERNAME}" \
199-
--env REGISTRY_PASSWORD="${REGISTRY_PASSWORD}" \
200-
--env APPCO_USERNAME=${APPCO_USERNAME} \
201-
--env APPCO_PASSWORD=${APPCO_PASSWORD} \
202-
--env LONGHORN_CHART_URI=${LONGHORN_CHART_URI} \
203-
--env LONGHORN_VERSION=${LONGHORN_VERSION} \
204-
--env APPCO_LONGHORN_COMPONENT_IMAGE_PATH=${APPCO_LONGHORN_COMPONENT_IMAGE_PATH} \
205-
--env LONGHORN_COMPONENT_TAG=${LONGHORN_COMPONENT_TAG} \
206-
--env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \
207-
--env LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} \
208-
--env LONGHORN_STABLE_VERSION_CHART_URI=${LONGHORN_STABLE_VERSION_CHART_URI} \
209-
--env LONGHORN_TRANSIENT_VERSION=${LONGHORN_TRANSIENT_VERSION} \
210-
--env LONGHORN_TRANSIENT_VERSION_CHART_URI=${LONGHORN_TRANSIENT_VERSION_CHART_URI} \
211-
${envArgs} \
212-
airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \
213-
bash -c './pipelines/appco/scripts/prepare_airgap_appco_images.sh'
214-
"""
215-
216-
return
217-
}
218-
}
21962

22063
stage('build') {
22164
if(JOB_BASE_NAME == "longhorn-tests-regression") {
@@ -231,6 +74,11 @@ node {
23174
RESOURCE_OWNER = BUILD_TRIGGER_BY.replace("\nStarted by user ", "")
23275
}
23376

77+
if (params.AIR_GAP_INSTALLATION) {
78+
REGISTRY_URL = AWS_PRIVATE_ECR
79+
}
80+
81+
sh "cp ${CA_CERT_PATH} ./SUSE_Trust_Root.crt"
23482
sh "pipelines/appco/scripts/build.sh"
23583
sh """ docker run -itd --cap-add=NET_ADMIN \
23684
--name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
@@ -250,8 +98,8 @@ node {
25098
--env JOB_NAME=${JOB_BASE_NAME}-${BUILD_NUMBER} \
25199
--env AIR_GAP_INSTALLATION=${AIR_GAP_INSTALLATION} \
252100
--env REGISTRY_URL=${REGISTRY_URL} \
253-
--env REGISTRY_USERNAME="${REGISTRY_USERNAME}" \
254-
--env REGISTRY_PASSWORD="${REGISTRY_PASSWORD}" \
101+
--env AWS_PRIVATE_ECR=${AWS_PRIVATE_ECR} \
102+
--env CUSTOM_LONGHORN_ENGINE_IMAGE=${APPCO_LONGHORN_COMPONENT_IMAGE_PATH}/longhorn-engine:${LONGHORN_COMPONENT_TAG} \
255103
--env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \
256104
--env CUSTOM_TEST_OPTIONS="${CUSTOM_TEST_OPTIONS}" \
257105
--env DISTRO=${DISTRO} \
@@ -300,7 +148,6 @@ node {
300148
--env TEST_TYPE=${TEST_TYPE} \
301149
--env OUT_OF_CLUSTER=${OUT_OF_CLUSTER} \
302150
--env IMAGE_NAME=${imageName} \
303-
${envArgs} \
304151
${imageName}
305152
"""
306153

0 commit comments

Comments
 (0)