Merge branch 'master' into fix/79-superset-metastore-cache

Author: Wiktor Maj

.github/workflows/publish.yml

@@ -15,6 +15,7 @@ jobs:
       contents: "read"
       packages: "write"
     strategy:
+      fail-fast: false
       matrix:
         include:
           - image: "ghcr.io/szachovy/superset-cluster-mysql-server"

CHANGELOG.md

@@ -22,11 +22,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Completed [ARCHITECTURE.md](./docs/ARCHITECTURE.md) (#93)
 * Migrated CI from self-hosted to GitHub-hosted runners with Docker-in-Docker test infrastructure. (#94)
 * Test workflow always builds service images locally for reproducibility. (#97)
+* Enabled parallel replication applier threads on secondary MySQL nodes. (#75)
+* Clean up temporary `.pyc` files on remote nodes after execution. (#51)
+* Set `innodb_flush_method=O_DIRECT` to eliminate double caching in containers. (#74)
+* Reduced InnoDB change buffer size for read-heavy BI workload. (#72)
+* Set InnoDB buffer pool size to 1G for improved query performance. (#71)
+* Relaxed Terraform version constraint to accept any 1.x release. (#40)
+
+### Removed
+
+* Removed personal contact section from README. (#159)
 
 ### Fixed
 
+* Upload `.py` source instead of `.pyc` bytecode to decouple host Python version. (#38, #41)
 * Fixed `run_mysql_server()` not instantiating `MySQLServer` class. (#94)
 * Disabled MD060 markdownlint rule to fix table column style false positives in documentation. (#94)
+* Made `create_directory()` idempotent to support deployment re-runs. (#35)
+* Added default inventory directive to `ansible.cfg`. (#42)
+* Fixed Publish workflow `write_package` permission denied by adding OCI source labels for GHCR repository linking. (#99)
 
 ## 1.0 - 2024-10-13
 

README.md

@@ -32,7 +32,7 @@ Follow [ARCHITECTURE.md](docs/ARCHITECTURE.md) for more.
 
 The following software needs to be installed on the user's host:
 
-* `python v3.10.12` with the following third party packages:
+* `python >= 3.10` with the following third party packages:
   * `paramiko v3.5.0`
 
 The following software needs to be installed on the external nodes:
@@ -104,10 +104,6 @@ If you notice anything missing, spot a bug, or have an enhancement proposal,
 feel free to open an issue with the appropriate label.
 Pull requests are welcome. Please ensure that the tests are updated as necessary.
 
-## Personal contact information
-
-In case of any inquiries, please write an email to: _wjmaj98@gmail.com_
-
 ## Additional resources
 
 * [What is Apache Superset?](https://superset.apache.org/docs/intro)

docs/ARCHITECTURE.md

@@ -114,9 +114,9 @@ The controller runs on the user's workstation and communicates with cluster node
 [Paramiko](https://www.paramiko.org/). The `RemoteConnection` class provides:
 
 * **SSH/SFTP connections**: connects as the `superset` user, either directly or through `~/.ssh/config`.
-* **Python bytecode execution**: `container.py` source is compiled to `.pyc`, uploaded to the remote node,
-  and executed via `python3 /opt/<nonce>.pyc`. This allows running Docker API commands on remote nodes
-  without installing additional management software.
+* **Python source execution**: `container.py` source is uploaded to the remote node along with the
+  command to execute, and run via `python3 /opt/<nonce>.py`. This allows running Docker API commands on
+  remote nodes without installing additional management software.
 * **Directory and file uploads**: recursive SFTP-based uploads of service directories, certificates,
   and passwords.
 

services/mysql-mgmt/Dockerfile

@@ -1,7 +1,8 @@
 ARG BUILDPLATFORM="amd64"
 FROM --platform=${BUILDPLATFORM} ubuntu:22.04
 
-LABEL version="1.0"
+LABEL version="1.0" \
+      org.opencontainers.image.source="https://github.com/szachovy/superset-cluster"
 
 ARG MYSQL_SHELL_VERSION="mysql-shell-8.3.0-linux-glibc2.28-x86-64bit" \
     MYSQL_SHELL_MD5_CHECKSUM="c71280416014b9340b34b90160f42e23" \

services/mysql-server/Dockerfile

@@ -1,7 +1,8 @@
 ARG BUILDPLATFORM="amd64"
 FROM --platform=${BUILDPLATFORM} mysql:8.0-debian
 
-LABEL version="1.0"
+LABEL version="1.0" \
+      org.opencontainers.image.source="https://github.com/szachovy/superset-cluster"
 
 ENV LANG="C.UTF-8" \
     LC_ALL="C.UTF-8"

services/mysql-server/mysql_config.cnf.tpl

@@ -14,3 +14,10 @@ ssl-key = "/etc/mysql/ssl/mysql_server_key.pem"
 require_secure_transport = "ON"
 max_connections = "50"
 max_connect_errors = "50"
+replica_parallel_workers = "4"
+replica_parallel_type = "LOGICAL_CLOCK"
+replica_preserve_commit_order = "ON"
+innodb_flush_method = "O_DIRECT"
+innodb_change_buffer_max_size = "10"
+innodb_buffer_pool_size = "1G"
+innodb_buffer_pool_instances = "1"

services/superset/Dockerfile

@@ -1,7 +1,8 @@
 ARG BUILDPLATFORM="amd64"
 FROM --platform=${BUILDPLATFORM} apache/superset:4.0.2
 
-LABEL version="1.0"
+LABEL version="1.0" \
+      org.opencontainers.image.source="https://github.com/szachovy/superset-cluster"
 
 COPY --chown=superset:superset "." "/app/"
 

src/initialize.py

@@ -48,12 +48,17 @@
 
 # pylint: disable=consider-using-f-string
 # pylint: disable=attribute-defined-outside-init
+# pylint: disable=wrong-import-position
+
+import sys
+
+if sys.version_info < (3, 10):
+    sys.exit(f"Python >= 3.10 required (found {sys.version_info.major}.{sys.version_info.minor})")
 
 import base64
 import functools
 import ipaddress
 import itertools
-import sys
 import re
 import socket
 

src/remote.py

@@ -39,7 +39,6 @@
 import functools
 import io
 import logging
-import marshal
 import os
 import pathlib
 import random
@@ -124,16 +123,15 @@ def run_python_container_command(self, command: str) -> dict:
             mode="r",
             encoding="utf-8"
         ) as memfile:
-            code_object = compile(memfile.read() + command, filename=nonce, mode="exec")
-            pyc_file = io.BytesIO()
-            pyc_file.write(b'o\r\r\n\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
-            marshal.dump(code_object, pyc_file)
-            self.upload_file(content=pyc_file.getvalue(), remote_file_path=f'/opt/{nonce}.pyc')
-        _, stdout, stderr = self.ssh_client.exec_command(f"python3 /opt/{nonce}.pyc")
-        return {
+            source = memfile.read() + command
+        self.upload_file(content=source, remote_file_path=f'/opt/{nonce}.py')
+        _, stdout, stderr = self.ssh_client.exec_command(f"python3 /opt/{nonce}.py")
+        result = {
             "output": stdout.read().decode(),
             "error": stderr.read().decode()
         }
+        self.ssh_client.exec_command(f"rm -f /opt/{nonce}.py")
+        return result
 
     def upload_directory(self, local_directory_path: str, remote_directory_path: str) -> None:
         stack = [(local_directory_path, remote_directory_path)]
@@ -152,7 +150,7 @@ def upload_directory(self, local_directory_path: str, remote_directory_path: str
                     self.sftp_client.put(local_item_path, remote_item_path)
 
     def create_directory(self, remote_directory_path: str) -> None:
-        self.sftp_client.mkdir(remote_directory_path)
+        self.ssh_client.exec_command(f"mkdir -p {remote_directory_path}")
 
     def upload_file(self, content: str | bytes, remote_file_path: str) -> None:
         if isinstance(content, str):