@@ -118,13 +118,67 @@ def __init__(self) -> None:
118118 ] # type: ignore[assignment]
119119 self .cert_manager = crypto .OpenSSL ()
120120
121+ def _recover_existing_credentials (self ) -> bool :
122+ for node in self .mysql_nodes :
123+ try :
124+ _ , stdout , _ = node .ssh_client .exec_command (
125+ "cat /opt/superset-cluster/mysql-server/mysql_root_password"
126+ )
127+ password = stdout .read ().decode ().strip ()
128+ if not password :
129+ continue
130+ _ , stdout , _ = node .ssh_client .exec_command (
131+ "cat /opt/superset-cluster/mysql-server/superset_cluster_ca_key.pem"
132+ )
133+ ca_key_pem = stdout .read ().decode ().strip ()
134+ _ , stdout , _ = node .ssh_client .exec_command (
135+ "cat /opt/superset-cluster/mysql-server/superset_cluster_ca_certificate.pem"
136+ )
137+ ca_bundle = stdout .read ().decode ().strip ()
138+ if not (ca_key_pem and ca_bundle ):
139+ continue
140+ certs = ca_bundle .split ('-----END CERTIFICATE-----' )
141+ ca_cert_pem = [c for c in certs if '-----BEGIN CERTIFICATE-----' in c ][- 1 ]
142+ ca_cert_pem = ca_cert_pem .strip () + '\n -----END CERTIFICATE-----\n '
143+ self .ca_key = self .cert_manager .serialization (ca_key_pem )
144+ self .ca_certificate = self .cert_manager .serialization (ca_cert_pem )
145+ self .mysql_root_password = password
146+ break
147+ except (OSError , IndexError , ValueError ):
148+ continue
149+ else :
150+ return False
151+ for node in self .mgmt_nodes :
152+ try :
153+ _ , stdout , _ = node .ssh_client .exec_command (
154+ "cat /opt/superset-cluster/mysql-mgmt/mysql_superset_password"
155+ )
156+ superset_pw = stdout .read ().decode ().strip ()
157+ if not superset_pw :
158+ continue
159+ _ , stdout , _ = node .ssh_client .exec_command (
160+ "docker exec $(docker ps --filter name=superset"
161+ " --format '{{.ID}}' | head -1)"
162+ " cat /run/secrets/superset_secret_key 2>/dev/null"
163+ )
164+ secret_key = stdout .read ().decode ().strip ()
165+ if not secret_key :
166+ continue
167+ self .mysql_superset_password = superset_pw
168+ self .superset_secret_key = secret_key
169+ return True
170+ except OSError :
171+ continue
172+ return False
173+
121174 @decorators .Overlay .run_selected_methods_once
122175 def credentials (self ) -> None :
123- self .ca_key = self .cert_manager .generate_private_key ()
124- self .ca_certificate = self .cert_manager .generate_certificate ('Superset-Cluster' , self .ca_key )
125- self .mysql_root_password = self .cert_manager .generate_mysql_root_password ()
126- self .mysql_superset_password = self .cert_manager .generate_mysql_superset_password ()
127- self .superset_secret_key = self .cert_manager .generate_superset_secret_key ()
176+ if not self ._recover_existing_credentials ():
177+ self .ca_key = self .cert_manager .generate_private_key ()
178+ self .ca_certificate = self .cert_manager .generate_certificate ('Superset-Cluster' , self .ca_key )
179+ self .mysql_root_password = self .cert_manager .generate_mysql_root_password ()
180+ self .mysql_superset_password = self .cert_manager .generate_mysql_superset_password ()
181+ self .superset_secret_key = self .cert_manager .generate_superset_secret_key ()
128182 for node in list (itertools .chain (self .mysql_nodes , self .mgmt_nodes )):
129183 node .key = self .cert_manager .generate_private_key ()
130184 node .csr = self .cert_manager .generate_csr (f'Superset-Cluster-{ node .node } , node .key )
@@ -164,6 +218,13 @@ def get_mylogin_cnf(self, node: remote.RemoteConnection) -> bytes:
164218
165219 def start_mysql_servers (self ) -> None :
166220 for node in self .mysql_nodes :
221+ _ , stdout , _ = node .ssh_client .exec_command (
222+ "docker inspect --format='{{.State.Health.Status}}'"
223+ " mysql 2>/dev/null"
224+ )
225+ if stdout .read ().decode ().strip () == "healthy" :
226+ continue
227+ node .ssh_client .exec_command ("docker rm -f mysql 2>/dev/null" )
167228 node .upload_directory (
168229 local_directory_path = "./services/mysql-server" ,
169230 remote_directory_path = "/opt/superset-cluster/mysql-server"
@@ -195,6 +256,16 @@ def start_mysql_servers(self) -> None:
195256 )
196257
197258 def start_mysql_mgmt (self , node : remote .RemoteConnection , state : str , priority : int ) -> None :
259+ _ , stdout , _ = node .ssh_client .exec_command (
260+ "docker inspect --format='{{.State.Health.Status}}'"
261+ " mysql-mgmt 2>/dev/null"
262+ )
263+ if stdout .read ().decode ().strip () == "healthy" :
264+ return
265+ node .ssh_client .exec_command (
266+ "docker rm -f mysql-mgmt mysql-mgmt-initcontainer 2>/dev/null;"
267+ " docker volume rm mysql-mgmt_default_generated 2>/dev/null"
268+ )
198269 node .upload_directory (
199270 local_directory_path = "./services/mysql-mgmt" ,
200271 remote_directory_path = "/opt/superset-cluster/mysql-mgmt"
@@ -251,6 +322,25 @@ def start_mysql_mgmt(self, node: remote.RemoteConnection, state: str, priority:
251322
252323 def start_superset (self ) -> None :
253324 for node in self .mgmt_nodes :
325+ _ , redis_out , _ = node .ssh_client .exec_command (
326+ "docker inspect --format='{{.State.Health.Status}}'"
327+ " redis 2>/dev/null"
328+ )
329+ _ , svc_out , _ = node .ssh_client .exec_command (
330+ "docker service ps superset"
331+ " --format='{{.CurrentState}}'"
332+ " --filter desired-state=running 2>/dev/null"
333+ )
334+ redis_healthy = redis_out .read ().decode ().strip () == "healthy"
335+ superset_running = "Running" in svc_out .read ().decode ()
336+ if redis_healthy and superset_running :
337+ continue
338+ node .ssh_client .exec_command (
339+ "docker service rm superset 2>/dev/null;"
340+ " docker rm -f redis 2>/dev/null;"
341+ " docker swarm leave --force 2>/dev/null;"
342+ " docker network rm superset-network docker_gwbridge 2>/dev/null"
343+ )
254344 node .upload_directory (
255345 local_directory_path = "./services/superset" ,
256346 remote_directory_path = '/opt/superset-cluster/superset'
@@ -283,27 +373,6 @@ def start_superset(self) -> None:
283373 mysql_superset_password = self .mysql_superset_password )
284374 )
285375
286- def _close_connections (self ) -> None :
287- for node in list (itertools .chain (self .mysql_nodes , self .mgmt_nodes )):
288- node .ssh_client .close ()
289- node .sftp_client .close ()
290-
291- def _run_ssh_command (self , node : remote .RemoteConnection , command : str ) -> None :
292- _ , stdout , stderr = node .ssh_client .exec_command (command )
293- exit_status = stdout .channel .recv_exit_status ()
294- if exit_status != 0 :
295- raise RuntimeError (
296- f"Command failed on { node .node } { exit_status }
297- f"{ command } \n { stderr .read ().decode ().strip ()}
298- )
299-
300- def _prepare_nodes (self ) -> None :
301- for node in itertools .chain (self .mysql_nodes , self .mgmt_nodes ):
302- try :
303- node .sftp_client .mkdir ('/opt/superset-cluster' )
304- except IOError :
305- pass
306-
307376 def teardown_node (self , node : remote .RemoteConnection , is_mgmt : bool = False ) -> None :
308377 commands = [
309378 "if docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null"
@@ -316,7 +385,6 @@ def teardown_node(self, node: remote.RemoteConnection, is_mgmt: bool = False) ->
316385 " | xargs -r docker network rm" ,
317386 "if [ -d /opt/superset-cluster ]; then"
318387 " rm -rf /opt/superset-cluster; fi" ,
319- "find /opt -maxdepth 1 -name '*.pyc' -delete" ,
320388 ]
321389 if is_mgmt :
322390 commands .extend ([
@@ -330,34 +398,41 @@ def teardown_node(self, node: remote.RemoteConnection, is_mgmt: bool = False) ->
330398 " ip route del {vip}; fi" .format (
331399 vip = self .virtual_ip_address ),
332400 ])
333- for cmd in commands :
334- self ._run_ssh_command (node , cmd )
335-
336- def teardown_cluster (self ) -> None :
337- for node in self .mgmt_nodes :
338- self .teardown_node (node , is_mgmt = True )
339- for node in self .mysql_nodes :
340- self .teardown_node (node , is_mgmt = False )
401+ _ , stdout , stderr = node .ssh_client .exec_command (" && " .join (commands ))
402+ exit_status = stdout .channel .recv_exit_status ()
403+ if exit_status != 0 :
404+ raise RuntimeError (
405+ f"Teardown failed on { node .node } { exit_status }
406+ f"{ stderr .read ().decode ().strip ()}
407+ )
341408
342409 def start_cluster (self ) -> None :
343- self .start_mysql_servers ()
344- self .start_mysql_mgmt (node = self .mgmt_nodes [0 ], state = "MASTER" , priority = 100 )
345- self .start_mysql_mgmt (node = self .mgmt_nodes [1 ], state = "BACKUP" , priority = 90 )
346- self .start_superset ()
347-
348- def deploy (self ) -> None :
349410 try :
350- self .teardown_cluster ()
351- self ._prepare_nodes ()
352- self .start_cluster ()
411+ self .start_mysql_servers ()
412+ self .start_mysql_mgmt (node = self .mgmt_nodes [0 ], state = "MASTER" , priority = 100 )
413+ self .start_mysql_mgmt (node = self .mgmt_nodes [1 ], state = "BACKUP" , priority = 90 )
414+ self .start_superset ()
353415 finally :
354- self ._close_connections ()
416+ for node in list (itertools .chain (self .mysql_nodes , self .mgmt_nodes )):
417+ node .ssh_client .close ()
418+ node .sftp_client .close ()
355419
356420 def cleanup (self ) -> None :
357421 try :
358- self .teardown_cluster ()
422+ for node in self .mgmt_nodes :
423+ try :
424+ self .teardown_node (node , is_mgmt = True )
425+ except (OSError , RuntimeError ):
426+ pass
427+ for node in self .mysql_nodes :
428+ try :
429+ self .teardown_node (node , is_mgmt = False )
430+ except (OSError , RuntimeError ):
431+ pass
359432 finally :
360- self ._close_connections ()
433+ for node in list (itertools .chain (self .mysql_nodes , self .mgmt_nodes )):
434+ node .ssh_client .close ()
435+ node .sftp_client .close ()
361436
362437
363438if __name__ == "__main__" :
@@ -371,4 +446,4 @@ def cleanup(self) -> None:
371446 if action == "cleanup" :
372447 controller .cleanup ()
373448 else :
374- controller .deploy ()
449+ controller .start_cluster ()
0 commit comments