Merge branch 'master' into fix/75-parallel-replication

Author: Wiktor Maj

CHANGELOG.md

@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+* Upload `.py` source instead of `.pyc` bytecode to decouple host Python version. (#38, #41)
 * Fixed `run_mysql_server()` not instantiating `MySQLServer` class. (#94)
 * Disabled MD060 markdownlint rule to fix table column style false positives in documentation. (#94)
 * Made `create_directory()` idempotent to support deployment re-runs. (#35)

README.md

@@ -32,7 +32,7 @@ Follow [ARCHITECTURE.md](docs/ARCHITECTURE.md) for more.
 
 The following software needs to be installed on the user's host:
 
-* `python v3.10.12` with the following third party packages:
+* `python >= 3.10` with the following third party packages:
   * `paramiko v3.5.0`
 
 The following software needs to be installed on the external nodes:

docs/ARCHITECTURE.md

@@ -114,9 +114,9 @@ The controller runs on the user's workstation and communicates with cluster node
 [Paramiko](https://www.paramiko.org/). The `RemoteConnection` class provides:
 
 * **SSH/SFTP connections**: connects as the `superset` user, either directly or through `~/.ssh/config`.
-* **Python bytecode execution**: `container.py` source is compiled to `.pyc`, uploaded to the remote node,
-  and executed via `python3 /opt/<nonce>.pyc`. This allows running Docker API commands on remote nodes
-  without installing additional management software.
+* **Python source execution**: `container.py` source is uploaded to the remote node along with the
+  command to execute, and run via `python3 /opt/<nonce>.py`. This allows running Docker API commands on
+  remote nodes without installing additional management software.
 * **Directory and file uploads**: recursive SFTP-based uploads of service directories, certificates,
   and passwords.
 

src/initialize.py

@@ -48,12 +48,17 @@
 
 # pylint: disable=consider-using-f-string
 # pylint: disable=attribute-defined-outside-init
+# pylint: disable=wrong-import-position
+
+import sys
+
+if sys.version_info < (3, 10):
+    sys.exit(f"Python >= 3.10 required (found {sys.version_info.major}.{sys.version_info.minor})")
 
 import base64
 import functools
 import ipaddress
 import itertools
-import sys
 import re
 import socket
 

src/remote.py

@@ -39,7 +39,6 @@
 import functools
 import io
 import logging
-import marshal
 import os
 import pathlib
 import random
@@ -124,17 +123,14 @@ def run_python_container_command(self, command: str) -> dict:
             mode="r",
             encoding="utf-8"
         ) as memfile:
-            code_object = compile(memfile.read() + command, filename=nonce, mode="exec")
-            pyc_file = io.BytesIO()
-            pyc_file.write(b'o\r\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
-            marshal.dump(code_object, pyc_file)
-            self.upload_file(content=pyc_file.getvalue(), remote_file_path=f'/opt/{nonce}.pyc')
-        _, stdout, stderr = self.ssh_client.exec_command(f"python3 /opt/{nonce}.pyc")
+            source = memfile.read() + command
+        self.upload_file(content=source, remote_file_path=f'/opt/{nonce}.py')
+        _, stdout, stderr = self.ssh_client.exec_command(f"python3 /opt/{nonce}.py")
         result = {
             "output": stdout.read().decode(),
             "error": stderr.read().decode()
         }
-        self.ssh_client.exec_command(f"rm -f /opt/{nonce}.pyc")
+        self.ssh_client.exec_command(f"rm -f /opt/{nonce}.py")
         return result
 
     def upload_directory(self, local_directory_path: str, remote_directory_path: str) -> None: