Suffix the depot path with a colon (#61)

This ensure depot-shipped package images can be re-used.

Also fixes CI as much as possible without tagging a release.

Author: maleadt

.buildkite/pipeline.yml

@@ -33,15 +33,15 @@ steps:
 
       # Launch Linux jobs
       AGENT_OS="linux"
-      upload_pipelines x86_64  "1 1.6.0 1.8 nightly"
+      upload_pipelines x86_64  "1 1.10 nightly"
       upload_pipelines i686    "1 nightly"
       upload_pipelines aarch64 "1 nightly"
 
-      # Waiting for the armv7l build pipeline to be fixed so we can get newer builds
-      upload_pipelines arm  "1.7"
+      # PowerPC was last supported in Julia 1.10
+      upload_pipelines powerpc64le "1.10"
 
-      # We don't have a 1.8.X build for ppc64le, so install 1.7 and nightly
-      upload_pipelines powerpc64le "1.7 nightly"
+      # ARMv7 was last supported in Julia 1.7
+      upload_pipelines arm  "1.7"
 
       # Launch macOS jobs
       AGENT_OS="macos"

hooks/pre-command

@@ -30,30 +30,35 @@ fi
 # packages, artifacts and compiled `.ji` files
 if [[ "${BUILDKITE_PLUGIN_JULIA_ISOLATED_DEPOT:-true}" == "true" ]]; then
     # Export `JULIA_DEPOT_PATH` so that future julia invocations use this depot
-    export JULIA_DEPOT_PATH="${CACHE_DIR}/depots/${BUILDKITE_PIPELINE_ID}"
+    export DEPOT_DIR="${CACHE_DIR}/depots/${BUILDKITE_PIPELINE_ID}"
+    if [[ "$OSTYPE" == "msys" ]]; then
+        export JULIA_DEPOT_PATH="${DEPOT_DIR};"
+    else
+        export JULIA_DEPOT_PATH="${DEPOT_DIR}:"
+    fi
     PERSIST_DEPOT_DIRS="$(echo ${BUILDKITE_PLUGIN_JULIA_PERSIST_DEPOT_DIRS:-packages,artifacts,compiled,logs,datadeps,scratchspaces} | tr ',' ' ' )"
 
     # Helper function to join a list of arguments by a particular character
     function join_by { local IFS="$1"; shift; echo "$*"; }
 
     # If the depot we're isolating ourselves inside of actually exists, we clean out
     # any directories not explicitly saved via `PERSIST_DEPOT_DIRS`
-    if [[ -d "${JULIA_DEPOT_PATH}" ]]; then
-        echo "--- :broom: Cleaning out depot at ${JULIA_DEPOT_PATH}, persisting: '${PERSIST_DEPOT_DIRS}'"
-        pushd "${JULIA_DEPOT_PATH}" >/dev/null
+    if [[ -d "${DEPOT_DIR}" ]]; then
+        echo "--- :broom: Cleaning out depot at ${DEPOT_DIR}, persisting: '${PERSIST_DEPOT_DIRS}'"
+        pushd "${DEPOT_DIR}" >/dev/null
         rm -rf -- !($(join_by "|" ${PERSIST_DEPOT_DIRS}))
         popd >/dev/null
 
         # Clean out any stale pidfiles that may have come from a cancelled run,
         # as it's not possible for another job to be running concurrently.
-        if [[ -d "${JULIA_DEPOT_PATH}"/compiled ]]; then
-          find "${JULIA_DEPOT_PATH}"/compiled -name '*.pidfile' -delete
+        if [[ -d "${DEPOT_DIR}"/compiled ]]; then
+          find "${DEPOT_DIR}"/compiled -name '*.pidfile' -delete
         fi
     fi
 
     # Mark the depot as in-use by touching it
-    if [[ -d "${JULIA_DEPOT_PATH}" ]]; then
-        touch "${JULIA_DEPOT_PATH}"
+    if [[ -d "${DEPOT_DIR}" ]]; then
+        touch "${DEPOT_DIR}"
     fi
 else
     if [[ -v BUILDKITE_PLUGIN_JULIA_PERSIST_DEPOT_DIRS ]]; then
@@ -320,3 +325,5 @@ else
     done
     export PATH="${MSYS_PATH}"
 fi
+
+export BUILDKITE_PLUGIN_JULIA_SUCCESS="true"

hooks/pre-exit

@@ -2,15 +2,24 @@
 set -euo pipefail
 shopt -s extglob
 
+
+## Set some global configuration options, but allow users to override them
+
 ## If we're running in debug mode, be REALLY VERBOSE:
 if [[ "${BUILDKITE_PLUGIN_JULIA_DEBUG_PLUGIN:-false}" == "true" ]]; then
+    PS4="> "
     set -x
 fi
 
 CACHE_DIR=${BUILDKITE_PLUGIN_JULIA_CACHE_DIR:-${HOME}/.cache/julia-buildkite-plugin}
 
 echo "--- :broom: Performing clean-up"
 
+if [[ "${BUILDKITE_PLUGIN_JULIA_SUCCESS:-false}" != "true" ]]; then
+    echo "Skipping clean-up since the plugin did not successfully run."
+    exit 0
+fi
+
 
 ### Step 1: Remove any code coverage files.
 
@@ -23,21 +32,23 @@ fi
 
 # Some packages check coverage files in, so only remove writable ones
 if [[ "${BUILDKITE_PLUGIN_JULIA_ISOLATED_DEPOT:-true}" == "true" && \
-      -d "${JULIA_DEPOT_PATH}"/packages ]]; then
-    find "${JULIA_DEPOT_PATH}"/packages -name "*.jl.*.cov" -type f -perm -200 -delete
+      -d "${DEPOT_DIR}"/packages ]]; then
+    find "${DEPOT_DIR}"/packages -name "*.jl.*.cov" -type f -perm -200 -delete
 fi
 
 
 ### Step 2: Reduce the size of our depot by removing old precompilation files.
 
+echo "Removing old precompilation files"
+
 # Julia does not track when a precompilation file was last used, so we simply
 # remove files in the `compiled` directory until we are below the limit.
 
 # default limit: 1 GiB
 COMPILED_LIMIT="${BUILDKITE_PLUGIN_JULIA_COMPILED_SIZE_LIMIT:-1073741824}"
 
 if [[ "${BUILDKITE_PLUGIN_JULIA_ISOLATED_DEPOT:-true}" == "true" ]]; then
-    COMPILED_DIR="${JULIA_DEPOT_PATH}/compiled"
+    COMPILED_DIR="${DEPOT_DIR}/compiled"
     if [[ -d "${COMPILED_DIR}" ]]; then
         COMPILED_SIZE_HUMAN=$(du -h -s "${COMPILED_DIR}" | cut -f 1)
         # `-k` gives consistently the number of kilobytes on macOS and Linux,
@@ -87,6 +98,8 @@ fi
 
 ### Step 3: Reduce the size of our depot by removing old artifacts.
 
+echo "Removing old artifacts"
+
 # This is normally done by `Pkg.gc`, however, since manifests are ephemeral
 # (and logs directories may not even persist) that doesn't work well.
 # So instead we remove directories based on their creation time.
@@ -95,7 +108,7 @@ fi
 ARTIFACTS_LIMIT="${BUILDKITE_PLUGIN_JULIA_ARTIFACTS_SIZE_LIMIT:-10737418240}"
 
 if [[ "${BUILDKITE_PLUGIN_JULIA_ISOLATED_DEPOT:-true}" == "true" ]]; then
-    ARTIFACTS_DIR="${JULIA_DEPOT_PATH}/artifacts"
+    ARTIFACTS_DIR="${DEPOT_DIR}/artifacts"
     if [[ -d "${ARTIFACTS_DIR}" ]]; then
         ARTIFACTS_SIZE_HUMAN=$(du -h -s "${ARTIFACTS_DIR}" | cut -f 1)
         # `-k` gives consistently the number of kilobytes on macOS and Linux,
@@ -151,22 +164,27 @@ DEPOT_LIMIT_DEFAULT=$(($COMPILED_LIMIT+$ARTIFACTS_LIMIT+10737418240))
 DEPOT_LIMIT="${BUILDKITE_PLUGIN_JULIA_DEPOT_SIZE_LIMIT:-${DEPOT_LIMIT_DEFAULT}}"
 
 if [[ "${BUILDKITE_PLUGIN_JULIA_ISOLATED_DEPOT:-true}" == "true" ]]; then
-    DEPOT_SIZE_HUMAN=$(du -h -s "${JULIA_DEPOT_PATH}" | cut -f 1)
-    # `-k` gives consistently the number of kilobytes on both macOS and Linux,
-    # without it BSD `du` would give the number of multiples of 512 bytes.
-    DEPOT_SIZE=$(($(du -k -s "${JULIA_DEPOT_PATH}" | cut -f 1) * 1024))
-    echo "The depot size is: ${DEPOT_SIZE_HUMAN}"
-
-    if [[ ${DEPOT_SIZE} -gt ${DEPOT_LIMIT} ]]; then
-        echo "This is greater than the limit (${DEPOT_SIZE} > ${DEPOT_LIMIT} bytes), so we will clear the entire depot"
-        rm -rf "${JULIA_DEPOT_PATH}"
-        mkdir -p "${JULIA_DEPOT_PATH}"
+    echo "Checking overall depot size at ${DEPOT_DIR}"
+    if [[ -d "${DEPOT_DIR}" ]]; then
+        DEPOT_SIZE_HUMAN=$(du -h -s "${DEPOT_DIR}" | cut -f 1)
+        # `-k` gives consistently the number of kilobytes on both macOS and Linux,
+        # without it BSD `du` would give the number of multiples of 512 bytes.
+        DEPOT_SIZE=$(($(du -k -s "${DEPOT_DIR}" | cut -f 1) * 1024))
+        echo "The depot size is: ${DEPOT_SIZE_HUMAN}"
+
+        if [[ ${DEPOT_SIZE} -gt ${DEPOT_LIMIT} ]]; then
+            echo "This is greater than the limit (${DEPOT_SIZE} > ${DEPOT_LIMIT} bytes), so we will clear the entire depot"
+            rm -rf "${DEPOT_DIR}"
+            mkdir -p "${DEPOT_DIR}"
+        fi
     fi
 fi
 
 
 ### Step 5: Remove old depots.
 
+echo "Removing old depots"
+
 # We mark a depot as in-use by `touch`ing it at the start of the pipeline,
 # so we can remove any depots that are older than a certain age.