Add --cleanup CLI flag with deploy/cleanup actions

szachovy · Copilot · szachovy · commit 6b1d5da9e964 · 2026-04-08T16:08:28.000+02:00
Replace blanket '|| true' error suppression in teardown with proper conditional logic: if/then guards for swarm leave, directory removal, and VIP cleanup; xargs -r for container, volume, and network removal. Exit codes are checked via _run_ssh_command() and real failures raise RuntimeError. Deploy action (default) always tears down existing state first, making it idempotent and safe for both fresh installs and redeployments. Add redeploy and cleanup test stages to the test suite: - redeploy.yml: restarts stopped nodes from disaster tests, runs a fresh deploy, and verifies the cluster is fully functional again. - cleanup.yml: runs --cleanup and verifies no containers, volumes, deployment directories, .pyc files, or VIP addresses remain. Closes #49 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * CI workflow to build and push service images to GitHub Container Registry on master merge. (#97)
 * Pull-first with local build fallback for container images during deployment. (#97)
 
+### Added
+
+* `--cleanup` CLI flag and `deploy`/`cleanup` actions with proper teardown. (#49)
+* Redeploy and cleanup test stages with post-disaster recovery verification. (#49)
+
 ### Changed
 
 * Completed [ARCHITECTURE.md](./docs/ARCHITECTURE.md) (#93)
diff --git a/src/initialize.py b/src/initialize.py
@@ -131,7 +131,10 @@ def credentials(self) -> None:
             node.certificate = self.cert_manager.generate_certificate(f'Superset-Cluster-{node.node}',
                                                                       node.csr,
                                                                       self.ca_key)
-            node.create_directory('/opt/superset-cluster')
+            try:
+                node.create_directory('/opt/superset-cluster')
+            except IOError:
+                pass
         for node in self.mgmt_nodes:
             node.superset_key = self.cert_manager.generate_private_key()
             node.superset_csr = self.cert_manager.generate_csr(self.virtual_ip_address, node.superset_key)
@@ -280,20 +283,92 @@ def start_superset(self) -> None:
                           mysql_superset_password=self.mysql_superset_password)
             )
 
+    def _close_connections(self) -> None:
+        for node in list(itertools.chain(self.mysql_nodes, self.mgmt_nodes)):
+            node.ssh_client.close()
+            node.sftp_client.close()
+
+    def _run_ssh_command(self, node: remote.RemoteConnection, command: str) -> None:
+        _, stdout, stderr = node.ssh_client.exec_command(command)
+        exit_status = stdout.channel.recv_exit_status()
+        if exit_status != 0:
+            raise RuntimeError(
+                f"Command failed on {node.node} (exit {exit_status}): "
+                f"{command}\n{stderr.read().decode().strip()}"
+            )
+
+    def _prepare_nodes(self) -> None:
+        for node in itertools.chain(self.mysql_nodes, self.mgmt_nodes):
+            try:
+                node.sftp_client.mkdir('/opt/superset-cluster')
+            except IOError:
+                pass
+
+    def teardown_node(self, node: remote.RemoteConnection, is_mgmt: bool = False) -> None:
+        commands = [
+            "if docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null"
+            " | grep -qvx inactive; then docker swarm leave --force; fi",
+            "docker ps -aq | xargs -r docker rm -f",
+            "docker volume ls -q | xargs -r docker volume rm",
+            "docker network ls --filter name=superset-network -q"
+            " | xargs -r docker network rm",
+            "docker network ls --filter name=docker_gwbridge -q"
+            " | xargs -r docker network rm",
+            "if [ -d /opt/superset-cluster ]; then"
+            " rm -rf /opt/superset-cluster; fi",
+            "find /opt -maxdepth 1 -name '*.pyc' -delete",
+        ]
+        if is_mgmt:
+            commands.extend([
+                "if ip addr show dev {iface} 2>/dev/null"
+                " | grep -q '{vip}/'; then"
+                " ip addr del {vip}/{mask} dev {iface}; fi".format(
+                    iface=self.virtual_network_interface,
+                    vip=self.virtual_ip_address,
+                    mask=self.virtual_network_mask),
+                "if ip route show | grep -q '{vip} '; then"
+                " ip route del {vip}; fi".format(
+                    vip=self.virtual_ip_address),
+            ])
+        for cmd in commands:
+            self._run_ssh_command(node, cmd)
+
+    def teardown_cluster(self) -> None:
+        for node in self.mgmt_nodes:
+            self.teardown_node(node, is_mgmt=True)
+        for node in self.mysql_nodes:
+            self.teardown_node(node, is_mgmt=False)
+
     def start_cluster(self) -> None:
+        self.start_mysql_servers()
+        self.start_mysql_mgmt(node=self.mgmt_nodes[0], state="MASTER", priority=100)
+        self.start_mysql_mgmt(node=self.mgmt_nodes[1], state="BACKUP", priority=90)
+        self.start_superset()
+
+    def deploy(self) -> None:
+        try:
+            self.teardown_cluster()
+            self._prepare_nodes()
+            self.start_cluster()
+        finally:
+            self._close_connections()
+
+    def cleanup(self) -> None:
         try:
-            self.start_mysql_servers()
-            self.start_mysql_mgmt(node=self.mgmt_nodes[0], state="MASTER", priority=100)
-            self.start_mysql_mgmt(node=self.mgmt_nodes[1], state="BACKUP", priority=90)
-            self.start_superset()
+            self.teardown_cluster()
         finally:
-            for node in list(itertools.chain(self.mysql_nodes, self.mgmt_nodes)):
-                node.ssh_client.close()
-                node.sftp_client.close()
+            self._close_connections()
 
 
 if __name__ == "__main__":
-    if len(sys.argv) != 6:
+    if len(sys.argv) < 6:
         print("Invalid form of arguments provided")
         sys.exit(1)
-    Controller().start_cluster()
+
+    action = sys.argv[6] if len(sys.argv) > 6 else "deploy"
+    controller = Controller()
+
+    if action == "cleanup":
+        controller.cleanup()
+    else:
+        controller.deploy()
diff --git a/superset-cluster b/superset-cluster
@@ -20,6 +20,8 @@ display_help() {
     echo "  --virtual-network-mask <mask>           Network mask for the virtual network gateway."
     echo "                                          Example: --virtual-network-mask 24"
     echo
+    echo "  --cleanup                               Remove all cluster resources from the specified nodes."
+    echo
     echo "  -h, --help                              Show this help message and exit."
     echo
     echo "Example:"
@@ -33,6 +35,7 @@ display_help() {
 
 
 parse_arguments() {
+    action="deploy"
     while [[ "$#" -gt 0 ]]; do
         case $1 in
             --mgmt-nodes)
@@ -59,6 +62,9 @@ parse_arguments() {
                 display_help
                 exit 0
                 ;;
+            --cleanup)
+                action="cleanup"
+                ;;
             *)
                 echo "Unknown option: $1"
                 display_help
@@ -77,14 +83,16 @@ if python --version &>/dev/null; then
     '${virtual_network_interface}' \
     '${virtual_network_mask}' \
     '${mgmt_nodes[*]}' \
-    '${mysql_nodes[*]}'"
+    '${mysql_nodes[*]}' \
+    '${action}'"
 elif python3 --version &>/dev/null; then
   eval "python3 ./src/initialize.py \
     '${virtual_ip_address}' \
     '${virtual_network_interface}' \
     '${virtual_network_mask}' \
     '${mgmt_nodes[*]}' \
-    '${mysql_nodes[*]}'"
+    '${mysql_nodes[*]}' \
+    '${action}'"
 else
   echo "Neither python nor python3 is set as the main executable in this environment, check python location with 'which python'."
   exit 1
diff --git a/tests/testsuite/deploy.yml b/tests/testsuite/deploy.yml
@@ -22,3 +22,19 @@
   tasks:
     - name: "Run functional tasks from testing role"
       ansible.builtin.include_role: {name: "testing", tasks_from: "functional"}
+
+- name: "Redeploy testing"
+  connection: "local"
+  hosts: "testing"
+  any_errors_fatal: true
+  tasks:
+    - name: "Run redeploy tasks from testing role"
+      ansible.builtin.include_role: {name: "testing", tasks_from: "redeploy"}
+
+- name: "Cleanup testing"
+  connection: "local"
+  hosts: "testing"
+  any_errors_fatal: true
+  tasks:
+    - name: "Run cleanup tasks from testing role"
+      ansible.builtin.include_role: {name: "testing", tasks_from: "cleanup"}
diff --git a/tests/testsuite/roles/testing/tasks/cleanup.yml b/tests/testsuite/roles/testing/tasks/cleanup.yml
@@ -0,0 +1,80 @@
+---
+- name: "Run cleanup"
+  ansible.builtin.shell: |
+    set -euxo pipefail
+    {
+      ./superset-cluster \
+        --cleanup \
+        --mgmt-nodes {{ node_prefix }}-0,{{ node_prefix }}-1 \
+        --mysql-nodes {{ node_prefix }}-2,{{ node_prefix }}-3,{{ node_prefix }}-4 \
+        --virtual-ip-address {{ virtual_ip_address }} \
+        --virtual-network-interface {{ virtual_network_interface }} \
+        --virtual-network-mask {{ virtual_network_mask }}
+    } >> /tmp/superset_cluster_testsuite.log 2>&1
+  args:
+    executable: "/bin/bash"
+    chdir: "../../"
+  register: "cleanup_command"
+  changed_when: "cleanup_command.rc != 0"
+
+- name: "Verify no running containers on any node"
+  community.docker.docker_container_exec:
+    container: "{{ item }}"
+    command: "/bin/bash -c 'test $(docker ps -q | wc -l) -eq 0'"
+  loop:
+    - "{{ node_prefix }}-0"
+    - "{{ node_prefix }}-1"
+    - "{{ node_prefix }}-2"
+    - "{{ node_prefix }}-3"
+    - "{{ node_prefix }}-4"
+  register: "no_containers"
+
+- name: "Verify no docker volumes on any node"
+  community.docker.docker_container_exec:
+    container: "{{ item }}"
+    command: "/bin/bash -c 'test $(docker volume ls -q | wc -l) -eq 0'"
+  loop:
+    - "{{ node_prefix }}-0"
+    - "{{ node_prefix }}-1"
+    - "{{ node_prefix }}-2"
+    - "{{ node_prefix }}-3"
+    - "{{ node_prefix }}-4"
+  register: "no_volumes"
+
+- name: "Verify deployment directory removed from all nodes"
+  community.docker.docker_container_exec:
+    container: "{{ item }}"
+    command: "/bin/bash -c 'test ! -d /opt/superset-cluster'"
+  loop:
+    - "{{ node_prefix }}-0"
+    - "{{ node_prefix }}-1"
+    - "{{ node_prefix }}-2"
+    - "{{ node_prefix }}-3"
+    - "{{ node_prefix }}-4"
+  register: "no_deploy_dir"
+
+- name: "Verify no .pyc files in /opt on any node"
+  community.docker.docker_container_exec:
+    container: "{{ item }}"
+    command: >-
+      /bin/bash -c
+      'test $(find /opt -maxdepth 1 -name "*.pyc" | wc -l) -eq 0'
+  loop:
+    - "{{ node_prefix }}-0"
+    - "{{ node_prefix }}-1"
+    - "{{ node_prefix }}-2"
+    - "{{ node_prefix }}-3"
+    - "{{ node_prefix }}-4"
+  register: "no_pyc_files"
+
+- name: "Verify VIP removed from management nodes"
+  community.docker.docker_container_exec:
+    container: "{{ item }}"
+    command: >-
+      /bin/bash -c
+      '! ip addr show dev {{ virtual_network_interface }}
+      | grep -q "{{ virtual_ip_address }}/"'
+  loop:
+    - "{{ node_prefix }}-0"
+    - "{{ node_prefix }}-1"
+  register: "no_vip"
diff --git a/tests/testsuite/roles/testing/tasks/redeploy.yml b/tests/testsuite/roles/testing/tasks/redeploy.yml
