Restore pipefail in Tests workflow to fail on non-zero exit codes (#161)

Author: Wiktor Maj

.github/workflows/tests.yml

@@ -22,8 +22,13 @@ jobs:
             /usr/local/share/boost \
             /usr/local/graalvm \
             /usr/local/share/chromium \
-            /usr/local/share/powershell
-          sudo docker image prune --all --force
+            /usr/local/share/powershell \
+            /usr/share/swift \
+            /usr/local/share/vcpkg \
+            /opt/hostedtoolcache
+          sudo apt-get clean
+          sudo docker system prune --all --volumes --force
+          df -h
 
       - uses: "actions/checkout@v4"
       - uses: "actions/setup-node@v3"
@@ -32,6 +37,7 @@ jobs:
       - uses: "hashicorp/setup-terraform@v3"
         with:
           terraform_version: "1.0.10"
+          terraform_wrapper: false
       - uses: "actions/setup-python@v3"
         with:
           python-version: "3.10.12"
@@ -49,6 +55,7 @@ jobs:
 
       - name: "Build and pre-load service images into test containers"
         run: |
+          set -o pipefail
           build_and_load() {
             local image_name="$1"
             local build_context="$2"
@@ -71,6 +78,7 @@ jobs:
 
       - name: "Run Ansible testsuite"
         run: |
+          set -o pipefail
           ansible-playbook --inventory inventory.yml deploy.yml \
             2>&1 | tee -a /tmp/superset_cluster_testsuite.log
         working-directory: "${{ env.ANSIBLE_WORKING_DIRECTORY }}"

CHANGELOG.md

@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Made `create_directory()` idempotent to support deployment re-runs. (#35)
 * Added default inventory directive to `ansible.cfg`. (#42)
 * Fixed Publish workflow `write_package` permission denied by adding OCI source labels for GHCR repository linking. (#99)
+* Restore `pipefail` in Tests workflow to propagate non-zero exit codes through pipes. (#160)
 
 ## 1.0 - 2024-10-13
 

services/mysql-server/Dockerfile

@@ -39,6 +39,14 @@ RUN \
   mkdir \
     "/etc/mysql/ssl" \
   && \
+  mkdir \
+    --parents \
+      "/var/log/mysql" \
+  && \
+  chown \
+    mysql:mysql \
+      "/var/log/mysql" \
+  && \
   apt-get \
     clean \
   && \

services/superset/Dockerfile

@@ -19,7 +19,7 @@ RUN \
     install \
       --no-cache-dir \
       "redis==4.5.4" \
-      "mysql-connector-python==9.0.0"
+      "mysql-connector-python==8.4.0"
 
 USER root
 

services/superset/superset_config.py

@@ -43,12 +43,17 @@ class CeleryConfig:  # pylint: disable=too-few-public-methods
 
 CELERY_CONFIG = CeleryConfig  # pylint: disable=invalid-name
 RESULTS_BACKEND = flask_caching.backends.rediscache.RedisCache(host="redis", port=6379, key_prefix="superset_results")
+
+
+def SQL_QUERY_MUTATOR(sql, **kwargs):  # pylint: disable=invalid-name,unused-argument
+    return sql
+
+
 SQL_MAX_ROW = 10000
 DEFAULT_SQLLAB_LIMIT = 1000
 SQLLAB_TIMEOUT = 300
 SQLLAB_VALIDATION_TIMEOUT = 60
 SQLLAB_DEFAULT_DBID = 1
-SQL_QUERY_MUTATOR = None
 
 FILTER_STATE_CACHE_CONFIG = {
     "CACHE_TYPE": "RedisCache",

src/container.py

@@ -103,9 +103,12 @@ def __init__(self, container: str | None) -> None:
     @staticmethod
     def pull_or_build_image(client: docker.client.DockerClient, image: str, build_context: str) -> None:
         try:
-            client.images.pull(image)
-        except (docker.errors.DockerException, requests.exceptions.RequestException):
-            client.images.build(path=build_context, tag=image)
+            client.images.get(image)
+        except docker.errors.ImageNotFound:
+            try:
+                client.images.pull(image)
+            except (docker.errors.DockerException, requests.exceptions.RequestException):
+                client.images.build(path=build_context, tag=image)
 
     def run_command_on_the_container(
             self,
@@ -440,9 +443,10 @@ def run(self) -> None:
                 ContainerConnection.pull_or_build_image(
                     self.client, image, "/opt/superset-cluster/superset"
                 )
+                image_id = self.client.images.get(image).id
                 self.client.services.create(
                     name="superset",
-                    image=image,
+                    image=image_id,
                     networks=["superset-network"],
                     secrets=[
                         docker.types.SecretReference(

src/crypto.py

@@ -65,7 +65,7 @@ def generate_mysql_root_password() -> str:
 
     @staticmethod
     def generate_mysql_superset_password() -> str:
-        charset = string.ascii_letters + string.digits + string.punctuation
+        charset = string.ascii_letters + string.digits
         return "".join(secrets.choice(charset) for _ in range(24))
 
     @staticmethod

src/remote.py

@@ -125,7 +125,9 @@ def run_python_container_command(self, command: str) -> dict:
         ) as memfile:
             source = memfile.read() + command
         self.upload_file(content=source, remote_file_path=f'/opt/{nonce}.py')
-        _, stdout, stderr = self.ssh_client.exec_command(f"python3 /opt/{nonce}.py")
+        _, stdout, stderr = self.ssh_client.exec_command(
+            f"PYTHONPATH=/home/superset/.local/lib/python3.10/site-packages python3 /opt/{nonce}.py"
+        )
         result = {
             "output": stdout.read().decode(),
             "error": stderr.read().decode()

tests/testsuite/roles/testing/tasks/system.yml

@@ -16,7 +16,7 @@
   async: 2100
   poll: 60
   register: "run_command"
-  changed_when: "run_command.rc != 0"
+  changed_when: false
 
 - name: "Collect container health from each node"
   community.docker.docker_container_exec: