.gitlab-ci.yml

---
default:
  # we use this tag to select the runner to be used for a given job
  tags:
    - crab3

variables:
  # having descriptions, options and value simplifies manually triggering the pipeline
  # Ref: https://docs.gitlab.com/ci/pipelines/#prefill-variables-in-manual-pipelines
  VERBOSE:
    description: "Will pass $Verbose env. var. to various scripts 0=~silent, 1=medium, 2=long, 3=fullblown"
    options:
      - "0"
      - "1"
      - "2"
      - "3"
    value: "0"
  # DEBUG _TEST do not uncomment !! if not set, is ignored, if set (any value) ST will use only one task to allow quick debug
  ENV_NAME:
    description: "the target for deployment and test: preprod or testX, if empty will be taken from tag name"
    options:
      - ""
      - "preprod"
      - "test1"
      - "test2"
      - "test11"
      - "test12"
      - "test14"
    value: ""
  IMAGE_TAG:
    description: "tag for the image to be built/deployed"
    value: "${CI_COMMIT_REF_SLUG}"  # to distinct it from commit tag and final image tag
  RELEASE_IMAGE_TAG: "${CI_COMMIT_TAG}-stable" # final tag name, e.g., v3.240904-stable
  # The `DOCKER_TLS_CERTDIR` variables is needed to run Docker-in-Docker, `DOCKER_BUILDKIT` is to make sure the docker build use the new BuildKit.
  # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#docker-in-docker-with-tls-enabled-in-the-docker-executor
  # Creating a docker context is required to be able to cache to the registry using Buildkit.
  # https://docs.docker.com/build/cache/backends/
  DOCKER_TLS_CERTDIR: "/certs"
  DOCKER_BUILDKIT: 1
  STATUS_TRACKING:
    description: "Whether to execute StatusTracking tests"
    options:
      - "true"
      - "false"
    value: "true"
  CLIENT_CONFIGURATION_VALIDATION:
    description: "Whether to execute CCV tests"
    options:
      - "true"
      - "false"
    value: "true"
  CLIENT_VALIDATION:
    description: "Whether to execute CV tests"
    options:
      - "true"
      - "false"
    value: "true"
  SKIP_BUILD:
    description: "Skip build step"
    options:
      - "true"
      - "false"
    value: "false"
  SKIP_DEPLOY:
    description: "Skip deploy step"
    options:
      - "true"
      - "false"
    value: "false"
  SKIP_SUBMIT:
    description: "Skip submit step"
    options:
      - "true"
      - "false"
    value: "false"
  SKIP_CLEANUP:
    description: "Skip Rucio rules cleanup step"
    options:
      - "true"
      - "false"
    value: "false"

# This key define variables which are later used in `!reference` tag in `rules`.
# Ref https://docs.gitlab.com/ee/ci/jobs/index.html#hide-jobs
# and https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
.default_rules:
  # $CI_COMMIT_TAG is also used by the script in get_env job used inside prepare_env stage,
  # make sure to keep consistent semantic in the two places
  release:
    - if: $CI_COMMIT_TAG =~ /.*v3\.[0-9]{6}.*/  # match tag: *v3.240101 (i.e. also pypi-test*v201010 )
  default:
    # make sure that rules default: and release: are mutually exclusive
    - if: ($CI_COMMIT_TAG =~ /^pypi-.*/) && ($CI_COMMIT_TAG !~ /.*v3\.[0-9]{6}.*/)   # match tag: pypi-(prod|preprod|test*)-1714418922
  skip_build:
    - if: '$SKIP_BUILD == "true"'
      when: never                             # helper var
  skip_deploy:
    - if: '$SKIP_DEPLOY == "true"'
      when: never 
  skip_submit:
    - if: '$SKIP_SUBMIT == "true"'
      when: never
  skip_check:
    - if: '$STATUS_TRACKING == "false" && $CLIENT_CONFIGURATION_VALIDATION == "false" && $CLIENT_VALIDATION == "false"'
      when: never 

stages:
  - prepare_env
  - prepare_release
  - build_docker
  - deploy
  - submit_testsuite
  - sleep
  - run_testsuite_CV
  - run_testsuite_CCV
  - run_testsuite_ST
  - cleanup
  - tagging_release

.cmssw_matrix_release: &cmssw_matrix_release
  - CMSSW_release: CMSSW_15_0_18
    SCRAM_ARCH: el8_amd64_gcc12
  - CMSSW_release: CMSSW_13_0_2
    SCRAM_ARCH: el8_amd64_gcc11
  - CMSSW_release: CMSSW_12_6_4
    SCRAM_ARCH: el8_amd64_gcc10
  - CMSSW_release: CMSSW_11_3_4
    SCRAM_ARCH: slc7_amd64_gcc900
  - CMSSW_release: CMSSW_10_6_26
    SCRAM_ARCH: slc7_amd64_gcc700
  - CMSSW_release: CMSSW_9_4_21
    SCRAM_ARCH: slc7_amd64_gcc630
  - CMSSW_release: CMSSW_8_0_36
    SCRAM_ARCH: slc7_amd64_gcc530

get_env:
  # NB rules are evaluated like python `any()`. If there is no rule the job runs all of the times.
  rules:
    # !reference in following lines tells GITLAB to replace with the value of the indicated variables.
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: prepare_env
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - printenv  # debug check ci env
    - cicd/gitlab/parseEnv.sh $CI_COMMIT_TAG  # create .env
  artifacts:
    paths:
      - .env
    expire_in: 1 week

set_version_name:
  rules:
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: prepare_release
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - |
        echo -e "\n__version__ = \"${CI_COMMIT_TAG}\" #Automatically added during build process\n" >> src/python/TaskWorker/__init__.py;
    - |
        echo -e "\n__version__ = \"${CI_COMMIT_TAG}\" #Automatically added during build process\n" >> src/python/CRABInterface/__init__.py;
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
        - src/python/CRABInterface/__init__.py
      policy: push

build_rest_image:
  rules:
    - !reference [.default_rules, skip_build]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  before_script:
    - docker info
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push -f "${CI_PROJECT_DIR}/cicd/crabserver_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabserver:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabserver:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG}" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/CRABInterface/__init__.py
      policy: pull

build_tw_image:
  rules:
    - !reference [.default_rules, skip_build]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push -f "${CI_PROJECT_DIR}/cicd/crabtaskworker_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtaskworker:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtaskworker:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG}" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_monit_image:
  rules:
    - !reference [.default_rules, skip_build]
    - !reference [.default_rules, release]
  stage: build_docker
  needs: ["build_tw_image", "get_env"]
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/monit_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtwmonit:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtwmonit:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtwmonit:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabtwmonit:v3.latest" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_crabtwfilebeat_image:
  rules:
    - !reference [.default_rules, skip_build]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/filebeat/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtwfilebeat:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtwfilebeat:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtwfilebeat:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabtwmonit:v3.latest" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_spark_image:
  rules:
    - !reference [.default_rules, skip_build]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/monit_spark/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabspark:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabspark:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabspark:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabspark:v3.latest" .


deploy_server:
  rules:
    - !reference [.default_rules, skip_deploy]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: deploy
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    # load KUBECONTEXT
    - source .env
    - echo $KUBECONFIG_FILE
    - yq -i -y ".\"current-context\" = \"$KUBECONTEXT\"" $KUBECONFIG_FILE
    - export KUBECONFIG=$KUBECONFIG_FILE
    - export DESIRED_IMAGE="registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG}"
    - |
      if [[ "$KUBECONTEXT" =~ $ARGOCD_ALLOWED_ENVS_REGEX ]]; then
        export DEPLOY_ENV="${KUBECONTEXT#cmsweb-}"  # e.g. testX,preprod
        export NEW_IMAGE_TAG="${IMAGE_TAG}"
        ./cicd/scripts/updateCRABServerImageTag.sh
        ./cicd/scripts/waitForArgoCDToDeployed.sh

      else
        kubectl -n crab set image deployment/crabserver crabserver="$DESIRED_IMAGE"
      fi
    - kubectl -n crab rollout status deployment/crabserver --watch --timeout=5m

.deploy_tw_template:
  rules:
    - !reference [.default_rules, skip_deploy]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: deploy
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - source .env
    - export Environment # from .env
    - export Service="${SERVICE}"
    - export Image="${IMAGE_TAG}"
    - export SSH_KEY=$(cicd/gitlab/credFile.sh $CRAB_TW_SSH_KEY ssh)
    - bash -x cicd/gitlab/deployTW.sh

deploy_taskworker:
  extends: .deploy_tw_template
  variables:
    SERVICE: TaskWorker

deploy_publisher_schedd:
  extends: .deploy_tw_template
  variables:
    SERVICE: Publisher_schedd

deploy_publisher_rucio:
  extends: .deploy_tw_template
  variables:
    SERVICE: Publisher_rucio

task_submission:
  variables:
    CMSSW_release: CMSSW_15_0_18
  rules:
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, skip_check]
    - !reference [.default_rules, default]
  stage: submit_testsuite
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Task_Submission_Status_Tracking=$STATUS_TRACKING
    - export Client_Configuration_Validation=$CLIENT_CONFIGURATION_VALIDATION
    - export Client_Validation_Suite=$CLIENT_VALIDATION
    - export Verbose=$VERBOSE
    - bash cicd/gitlab/submitTestTasks.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: push

task_submission_release:
  rules:
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, skip_check]
    - !reference [.default_rules, release]
  stage: submit_testsuite
  parallel:
    matrix:
      *cmssw_matrix_release
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Task_Submission_Status_Tracking=$STATUS_TRACKING
    - export Client_Configuration_Validation=$CLIENT_CONFIGURATION_VALIDATION
    - export Client_Validation_Suite=$CLIENT_VALIDATION
    - export Verbose=$VERBOSE
    - bash cicd/gitlab/submitTestTasks.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: push

sleep_before_validation:
  variables:
    CMSSW_release: CMSSW_15_0_18
  rules:
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, default]
  stage: sleep
  tags:
    - crab3-shell
  script:
    - echo "Sleeping for 5 minutes after task submission"
    - sleep 300
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: pull

sleep_before_validation_release:
  rules:
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, release]
  stage: sleep
  parallel:
    matrix:
      *cmssw_matrix_release
  tags:
    - crab3-shell
  script:
    - echo "Sleeping for 20 minutes after task submission"
    - sleep 1200
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: pull

client_validation_suite:
  variables:
    CMSSW_release: CMSSW_15_0_18
  rules:
    - if: '$CLIENT_VALIDATION == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, default]
  stage: run_testsuite_CV
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Client_Validation_Suite=true
    - export Verbose=$VERBOSE
    - ls workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
    # CV test uses one task, wait up to 2h checking every 5 minutes for that to complete
    # (5 min is the status cache refresh time in the AP, half an hour is needed to detect
    # successful transfer after submission to FTS
    - export RETRY_SLEEP_SECONDS=300
    - export RETRY_MAX=24
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/CV_config.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
      policy: pull

client_validation_suite_release:
  rules:
    - if: '$CLIENT_VALIDATION == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, release]
  stage: run_testsuite_CV
  parallel:
    matrix:
      *cmssw_matrix_release
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Client_Validation_Suite=true
    - export Verbose=$VERBOSE
    - ls workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
    - export RETRY_SLEEP_SECONDS=300
    - export RETRY_MAX=24
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/CV_config.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CV
      policy: pull
  allow_failure: true

client_configuration_validation:
  variables:
    CMSSW_release: CMSSW_15_0_18
  rules:
    - if: '$CLIENT_CONFIGURATION_VALIDATION == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, default]
  stage: run_testsuite_CCV
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Client_Configuration_Validation=true
    - export Verbose=$VERBOSE
    - ls workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
    - export RETRY_SLEEP_SECONDS=300
    - export RETRY_MAX=24
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/CCV_config.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
      policy: pull

client_configuration_validation_release:
  rules:
    - if: '$CLIENT_CONFIGURATION_VALIDATION == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, release]
  stage: run_testsuite_CCV
  parallel:
    matrix:
      *cmssw_matrix_release
  resource_group: crab3  # Only one job runs at a time
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export Client_Configuration_Validation=true
    - export Verbose=$VERBOSE
    - ls workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
    - export RETRY_SLEEP_SECONDS=300
    - export RETRY_MAX=24
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/CCV_config.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_CCV
      policy: pull
  allow_failure: true

check_status_tracking:
  variables:
    CMSSW_release: CMSSW_15_0_18
  rules:
    - if: '$STATUS_TRACKING == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, default]
  stage: run_testsuite_ST
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export REST_Instance  # from .env
    - export CRABClient_version  # from .env
    - export ROOT_DIR
    - echo $ROOT_DIR
    - pwd
    - ls -ltrR
    - export CMSSW_release=$CMSSW_release
    - export SCRAM_ARCH=el8_amd64_gcc11
    - export Check_Publication_Status=Yes
    - export RETRY_SLEEP_SECONDS=1800
    - export Verbose=$VERBOSE
    - export RETRY_SLEEP_SECONDS=900
    - export RETRY_MAX=16
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/executeStatusTracking.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: pull

check_status_tracking_release:
  rules:
    - if: '$STATUS_TRACKING == "false"'
      when: never
    - !reference [.default_rules, skip_submit]
    - !reference [.default_rules, release]
  stage: run_testsuite_ST
  parallel:
    matrix:
      *cmssw_matrix_release
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export REST_Instance  # from .env
    - export CRABClient_version  # from .env
    - export ROOT_DIR
    - export CMSSW_release=$CMSSW_release
    - export SCRAM_ARCH=$SCRAM_ARCH
    - export Check_Publication_Status=Yes
    - export Verbose=$VERBOSE
    - export RETRY_SLEEP_SECONDS=900
    - export RETRY_MAX=16
    - bash cicd/gitlab/retry.sh bash cicd/gitlab/executeStatusTracking.sh
  cache:
    - key: "Key_${CI_PIPELINE_ID}_${CMSSW_release}"
      paths:
        - workdir_${CI_PIPELINE_ID}_${CMSSW_release}/submitted_tasks_ST
      policy: pull
  allow_failure: true

cleanup_rucio_rules:
  rules:
    - if: '$SKIP_CLEANUP == "true"'
      when: never
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: cleanup
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export DRY_RUN=false
    - export PYTHONPATH=$PYTHONPATH:$(pwd)/src/python
    - bash cicd/rucio/cleanup.sh

# if test is pass, retag with `*-stable`
release_stable:
  rules:
    - !reference [.default_rules, release]
  stage: tagging_release
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  variables:
    GIT_STRATEGY: none
  script:
    - crane auth login -u ${CMSCRAB_REGISTRY_USER} -p ${CMSCRAB_REGISTRY_PASSWORD} ${CMSCRAB_REGISTRY_URL}
    # rest
    - crane cp registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG} registry.cern.ch/cmscrab/crabserver:${RELEASE_IMAGE_TAG}
    # tw
    - crane cp registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtaskworker:${RELEASE_IMAGE_TAG}
    # monit
    - crane cp registry.cern.ch/cmscrab/crabtwmonit:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtwmonit:${RELEASE_IMAGE_TAG}
    # filebeat
    - crane cp registry.cern.ch/cmscrab/crabtwfilebeat:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtwfilebeat:${RELEASE_IMAGE_TAG}
    # spark
    - crane cp registry.cern.ch/cmscrab/crabspark:${IMAGE_TAG} registry.cern.ch/cmscrab/crabspark:${RELEASE_IMAGE_TAG}