fix: support scaling up/down etcd

builtin/core/defaults/inventory/localhost.yaml

@@ -14,6 +14,7 @@ spec:
     #     port: 22
     #     user: root
     #     password: 123456
+    #   internal_ipv4: 1.1.1.1
   groups:
     # all kubernetes nodes.
     k8s_cluster:

builtin/core/playbooks/add_nodes.yaml

@@ -36,8 +36,48 @@
     - etcd
   gather_facts: true
   roles:
-    - role: etcd
+    - role: etcd/prepare
       when: .etcd.deployment_type | eq "external"
+    - role: etcd/backup
+      when: .etcd.deployment_type | eq "external"
+- hosts:
+    - etcd
+  serial: 1
+  roles:
+    - role: etcd/scaling_up/learner
+      when:
+        - .etcd.deployment_type | eq "external"  
+        - .installed_etcd | empty | not      
+        - .need_installed_etcd | default list | has .inventory_hostname
+    - role: etcd/install
+      when: 
+        - .etcd.deployment_type | eq "external"
+        - .installed_etcd | empty | not      
+        - .need_installed_etcd | default list | has .inventory_hostname
+    - role: etcd/scaling_up/promote
+      when:
+        - .etcd.deployment_type | eq "external"
+        - .installed_etcd | empty | not      
+        - .need_installed_etcd | default list | has .inventory_hostname
+- hosts:
+    - etcd
+  gather_facts: true
+  roles:
+    - role: etcd/postprocess
+      when:
+        - .etcd.deployment_type | eq "external"
+        - .installed_etcd | empty | not      
+        - .need_installed_etcd | default list | has .inventory_hostname
+
+- hosts:
+    - kube_control_plane
+  serial: 1
+  roles:
+    - role: kubernetes/sync-etcd-config
+      when:
+        - .need_installed_etcd | empty | not
+        - .etcd.deployment_type | eq "external"
+        - .kubernetes_install_ActiveState.stdout | eq "active"
 
 - hosts:
     - k8s_cluster
@@ -47,7 +87,6 @@
     - role: kubernetes/pre-kubernetes
       when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)
     - role: kubernetes/init-kubernetes
-      when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)
     - role: kubernetes/join-kubernetes
       when:
         - or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)

builtin/core/playbooks/create_cluster.yaml

@@ -35,8 +35,11 @@
 - hosts:
     - etcd
   roles:
-    - role: etcd
+    - role: etcd/prepare
       when: .etcd.deployment_type | eq "external"
+    - role: etcd/install
+      when: .etcd.deployment_type | eq "external"
+
 
 # Install the private image registry
 - hosts:

builtin/core/playbooks/delete_cluster.yaml

@@ -32,8 +32,8 @@
 - hosts:
     - etcd
   roles:
-    - role: uninstall/etcd
-      when: 
+    - role: etcd/scaling_down
+      when:
         - .delete.etcd
         - .etcd.deployment_type | eq "external"
 

builtin/core/playbooks/delete_nodes.yaml

@@ -10,14 +10,39 @@
   gather_facts: true
   roles:
     - defaults
-    - precheck
 
 - hosts:
-    - kube_control_plane
+    - etcd
+  roles:
+    - role: etcd/prepare
+      when: 
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
+- hosts:
+    - etcd
+  serial: 1 
+  roles:
+    - role: etcd/scaling_down
+      when: 
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
+        - .need_uninstall_etcd | has .inventory_hostname
+- hosts:
+    - etcd
   gather_facts: true
-  tasks:
+  roles:
+    - role: etcd/postprocess
+      when: 
+        - .delete.etcd
+        - .etcd.deployment_type | eq "external"
+        - .need_uninstall_etcd | empty | not
+        - .need_uninstall_etcd | has .inventory_hostname | not 
+
+- hosts:
+    - kube_control_plane
+  serial: 1
+  pre_tasks:
     - name: DeleteNode | Ensure at least one control plane node remains in the cluster
-      run_once: true
       command: |
         {{- $cpNodes := list -}}
         {{- range .groups.kube_control_plane -}}
@@ -29,6 +54,13 @@
         echo "At least one control plane node must be retained in the cluster." >&2
         exit 1
         {{- end }}
+  roles:
+    - role: kubernetes/sync-etcd-config
+      when:
+        - .need_installed_etcd | empty | not
+        - .etcd.deployment_type | eq "external"
+        - .kubernetes_install_ActiveState.stdout | eq "active"
+        - .delete_nodes | default list | has .inventory_hostname | not
 
 - hosts: 
     - k8s_cluster
@@ -71,7 +103,7 @@
 - hosts:
     - etcd
   roles:
-    - role: uninstall/etcd
+    - role: etcd
       when: 
         - .delete.etcd
         - .etcd.deployment_type | eq "external"

builtin/core/roles/certs/init/tasks/main.yaml

@@ -42,6 +42,8 @@
           {{ .binary_dir }}/pki/front-proxy.crt
 
 - name: Cert | Generate the etcd certificate file
+  loop: "{{ .groups.etcd | toJson }}"  
+  when: .item | empty | not
   gen_cert:
     root_key: >-
       {{ .binary_dir }}/pki/root.key
@@ -50,24 +52,43 @@
     cn: etcd
     sans: >-
       {{- $ips := list -}}
-      {{- range .groups.etcd | default list -}}
-        {{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}}
-        {{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}}
-        {{- if $internalIPv4 | empty | not -}}
-          {{- $ips = append $ips $internalIPv4 -}}
-        {{- end -}}
-        {{- if $internalIPv6 | empty | not -}}
-          {{- $ips = append $ips $internalIPv6 -}}
-        {{- end -}}
+      {{- $hostname := index .hostvars .item "hostname" | default "" -}}
+      {{- if .native.set_hostname -}}
+        {{- $hostname = .item -}}
+      {{- end -}}
+      {{- if $hostname | empty | not -}}
+        {{- $ips = append $ips $hostname -}}
+      {{- end -}}
+      {{- $internalIPv4 := index .hostvars .item "internal_ipv4" | default "" -}}
+      {{- if $internalIPv4 | empty | not -}}
+        {{- $ips = append $ips $internalIPv4 -}}
+      {{- end -}}
+      {{- $internalIPv6 := index .hostvars .item "internal_ipv6" | default "" -}}
+      {{- if $internalIPv6 | empty | not -}}
+        {{- $ips = append $ips $internalIPv6 -}}
       {{- end -}}
       {{ $ips | toJson }}
     date: "{{ .certs.etcd.date }}"
     policy: "{{ .certs.etcd.gen_cert_policy }}"
     out_key: >-
-      {{ .binary_dir }}/pki/etcd.key
+      {{ .binary_dir }}/pki/etcd-{{ .item }}.key
     out_cert: >-
-      {{ .binary_dir }}/pki/etcd.crt
+      {{ .binary_dir }}/pki/etcd-{{ .item }}.crt
+
+- name: Cert | Generate the etcd client certificate file
   when: .groups.etcd | default list | empty | not
+  gen_cert:
+    root_key: >-
+      {{ .binary_dir }}/pki/root.key
+    root_cert: >-
+      {{ .binary_dir }}/pki/root.crt
+    cn: etcd
+    date: "{{ .certs.etcd.date }}"
+    policy: "{{ .certs.etcd.gen_cert_policy }}"
+    out_key: >-
+      {{ .binary_dir }}/pki/etcd-client.key
+    out_cert: >-
+      {{ .binary_dir }}/pki/etcd-client.crt
 
 - name: Cert | Generate the image registry certificate file
   tags: ["image_registry"]

builtin/core/roles/certs/renew/etcd/tasks/main.yaml

@@ -2,19 +2,19 @@
 - name: ETCD | Copy CA certificate to remote host
   copy:
     src: >-
-      {{ ..etcd.ca_file }}
+      {{ .etcd.ca_file }}
     dest: /etc/ssl/etcd/ssl/ca.crt
 
 - name: ETCD | Copy server certificate to remote host
   copy:
     src: >-
-      {{ .etcd.cert_file }}
+      {{ tpl .etcd.server_cert_file .inventory_hostname }}
     dest: /etc/ssl/etcd/ssl/server.crt
 
 - name: ETCD | Copy server private key to remote host
   copy:
     src: >-
-      {{ .etcd.key_file }}
+      {{ tpl .etcd.server_key_file .inventory_hostname }}
     dest: /etc/ssl/etcd/ssl/server.key
 
 - name: ETCD | Restart etcd service to apply new certificates

builtin/core/roles/certs/renew/kubernetes/tasks/etcd.yaml

@@ -9,13 +9,13 @@
 - name: ETCD | Copy client certificate to remote host
   copy:
     src: >-
-      {{ .etcd.cert_file }}
+      {{ .etcd.client_cert_file }}
     dest: /etc/kubernetes/pki/etcd/client.crt
     mode: 0755
 
 - name: ETCD | Copy client key to remote host
   copy:
     src: >-
-      {{ .etcd.key_file }}
+      {{ .etcd.client_key_file }}
     dest: /etc/kubernetes/pki/etcd/client.key
     mode: 0755

builtin/core/roles/defaults/defaults/main/02-certs.yaml

@@ -8,6 +8,8 @@
 # CA (self-signed or provided)
 # |- etcd.cert
 # |- etcd.key
+# |- etcd-client.cert
+# |- etcd-client.key
 # |
 # |- image_registry.cert
 # |- image_registry.key

builtin/core/roles/defaults/defaults/main/04-etcd.yaml

@@ -9,7 +9,8 @@ etcd:
       {{ .image_registry.dockerio_registry }}
     repository: kubesphere/etcd
     tag: "{{ .etcd.etcd_version }}"
-  #    endpoints: ["https://127.1.1.1:2379"]
+  port: 2379
+  peer_port: 2380
   # Environment variables for etcd service
   env:
     election_timeout: 5000
@@ -37,7 +38,11 @@ etcd:
   traffic_priority: false
   ca_file: >-
     {{ .binary_dir }}/pki/root.crt
-  cert_file: >-
-    {{ .binary_dir }}/pki/etcd.crt
-  key_file: >-
-    {{ .binary_dir }}/pki/etcd.key
+  server_cert_file: >-
+    {{ .binary_dir }}/pki/etcd-{{ "{{ . }}" }}.crt
+  server_key_file: >-
+    {{ .binary_dir }}/pki/etcd-{{ "{{ . }}" }}.key
+  client_cert_file: >-
+    {{ .binary_dir }}/pki/etcd-client.crt
+  client_key_file: >-
+    {{ .binary_dir }}/pki/etcd-client.key

builtin/core/roles/etcd/backup/tasks/main.yaml

@@ -0,0 +1,29 @@
+- name: Backup | Get leader node name
+  run_once: true
+  delegate_to: "{{ .installed_etcd }}"
+  command: |
+    unset ETCDCTL_ENDPOINTS ETCDCTL_KEY ETCDCTL_CERT ETCDCTL_CACERT
+
+    # Get leader ID from endpoint status (text format: endpoint, ID, version, size, leader, term, index)
+    LEADER_ID=$(ETCDCTL_API=3 etcdctl \
+      --endpoints=https://localhost:{{ .etcd.port }} \
+      --cacert=/etc/ssl/etcd/ssl/ca.crt \
+      --cert=/etc/ssl/etcd/ssl/server.crt \
+      --key=/etc/ssl/etcd/ssl/server.key \
+      endpoint status | awk -F',' '{gsub(/^[ \t]+|[ \t]+$/, "", $5); print $5}')
+
+    # Convert decimal ID to hex (member list shows hex ID)
+    LEADER_ID_HEX=$(printf "%x" "$LEADER_ID")
+
+    # Get leader name from member list (format: ID,name,peerURL,clientURL,learners)
+    ETCDCTL_API=3 etcdctl \
+      --endpoints=https://localhost:{{ .etcd.port }} \
+      --cacert=/etc/ssl/etcd/ssl/ca.crt \
+      --cert=/etc/ssl/etcd/ssl/server.crt \
+      --key=/etc/ssl/etcd/ssl/server.key \
+      member list | grep "^${LEADER_ID_HEX}" | awk -F',' '{gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2}'
+  register: etcd_backup_leader_name
+
+- name: Backup | Backup on leader node
+  when: .inventory_hostname | eq .etcd_backup_leader_name.stdout
+  command: BACKUP_DIR="{{ .etcd.backup.backup_dir }}/install/etcd-v{{ index .etcd_install_version "stdout" "etcd Version" }}-$(date +%Y-%m-%d-%H-%M-%S)" /usr/local/bin/kube-scripts/backup_etcd.sh

builtin/core/roles/etcd/files/etcd.service → builtin/core/roles/etcd/install/files/etcd.service

@@ -13,6 +13,7 @@ NotifyAccess=all
 RestartSec=10s
 LimitNOFILE=40000
 Restart=always
+TimeoutStartSec=10min
 
 [Install]
 WantedBy=multi-user.target

builtin/core/roles/etcd/tasks/install.yaml → builtin/core/roles/etcd/install/tasks/install.yaml

@@ -12,11 +12,6 @@
       loop:
        - "{{ .etcd.env.data_dir }}"
 
-- name: Install | Generate etcd environment configuration file
-  template:
-    src: etcd.env
-    dest: /etc/etcd.env
-
 - name: Install | Deploy etcd systemd service file
   copy:
     src: etcd.service
@@ -31,10 +26,10 @@
 - name: Install | Configure network traffic priority for etcd
   command: |
     tc qdisc add dev eth0 root handle 1: prio bands 3
-    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1
-    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1
-    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1
-    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport {{ .etcd.peer_port }} 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport {{ .etcd.peer_port }} 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport {{ .etcd.port }} 0xffff flowid 1:1
+    tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport {{ .etcd.port }} 0xffff flowid 1:1
   when: .etcd.traffic_priority
 
 - name: Install | Start and enable etcd systemd service

builtin/core/roles/etcd/install/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+- name: Install | Set when
+  when: 
+    - .etcd_install_LoadState.stdout | eq "not-found"
+    - .need_installed_etcd | has .inventory_hostname
+  block:
+    - name: Install | Render /etc/etcd.env configuration file
+      template:
+        src: etcd.env
+        dest: /etc/etcd.env
+
+    - include_tasks: install.yaml
+
+    - include_tasks: backup_service.yaml