Skip to content

Commit 2ba49db

Browse files
redscholarredscholar
committed
fix: scaling down etcd
Signed-off-by: redscholar <blacktiledhouse@gmail.com>
1 parent 50221a1 commit 2ba49db

File tree

20 files changed

+340
-105
lines changed

20 files changed

+340
-105
lines changed

‎builtin/core/playbooks/add_nodes.yaml‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,90 @@
3636
- etcd
3737
gather_facts: true
3838
roles:
39-
- etcd
39+
- role: etcd
40+
when:
41+
- .etcd.deployment_type | eq "external"
42+
43+
- hosts:
44+
- kube_control_plane
45+
tasks:
46+
- name: AddNodes | Check if should update apiserver certificates
47+
run_once: true
48+
add_hostvars:
49+
hosts: kube_control_plane
50+
vars:
51+
need_installed_etcd: >-
52+
{{- $needInstalled := list -}}
53+
{{- range .groups.etcd -}}
54+
{{- if and ((index $.hostvars . "etcd_install_LoadState" "stdout") | eq "not-found") ($.delete_nodes | default list | has . | not) -}}
55+
{{- $needInstalled = append $needInstalled . -}}
56+
{{- end -}}
57+
{{- end -}}
58+
{{ $needInstalled | toJson }}
59+
- name: AddNodes | Update apiserver etcd certificates
60+
when:
61+
- .need_installed_etcd | fromJson | empty | not
62+
- .etcd.deployment_type | eq "external"
63+
block:
64+
- name: AddNodes | Copy etcd CA certificate to control plane node
65+
copy:
66+
src: >-
67+
{{ .etcd.ca_file }}
68+
dest: /etc/kubernetes/pki/etcd/ca.crt
69+
- name: AddNodes | Copy etcd client certificate to control plane node
70+
copy:
71+
src: >-
72+
{{ .etcd.cert_file }}
73+
dest: /etc/kubernetes/pki/etcd/client.crt
74+
- name: AddNodes | Copy etcd client key to control plane node
75+
copy:
76+
src: >-
77+
{{ .etcd.key_file }}
78+
dest: /etc/kubernetes/pki/etcd/client.key
79+
- name: AddNodes | update ks-apiserver
80+
command: |
81+
{{- $endpoints := list -}}
82+
{{- range .groups.etcd | default list -}}
83+
{{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
84+
{{- end -}}
85+
ETCD_ENDPOINTS="{{ join "," $endpoints }}"
86+
87+
if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
88+
kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
89+
fi
90+
91+
awk -v ep="$ETCD_ENDPOINTS" '
92+
BEGIN {
93+
n = split(ep, arr, ",")
94+
for (i = 1; i <= n; i++) {
95+
print " - " arr[i]
96+
}
97+
}
98+
' > /etc/kubernetes/kubeadm_new_endpoints.yaml
99+
# delete old endpoint
100+
sed -i '/^[[:space:]]*endpoints:/{
101+
:loop
102+
N
103+
s/\n[[:space:]]\+-.*//; t loop
104+
s/\n[[:space:]]*\n/\n/g
105+
P
106+
D
107+
}' /etc/kubernetes/kubeadm-config.yaml
108+
# insert new endpoint
109+
sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml
110+
rm /etc/kubernetes/kubeadm_new_endpoints.yaml
111+
# update kubeadm-config
112+
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
113+
kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
114+
{{- else }}
115+
kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
116+
{{- end }}
117+
# regenerate kube-apiserver
118+
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
119+
kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
120+
{{- else }}
121+
kubeadm init phase control-plane apiserver
122+
{{- end }}
40123

41124
- hosts:
42125
- k8s_cluster

‎builtin/core/playbooks/create_cluster.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
- hosts:
3636
- etcd
3737
roles:
38-
- etcd
38+
- role: etcd
39+
when: .etcd.deployment_type | eq "external"
3940

4041
# Install the private image registry
4142
- hosts:

‎builtin/core/playbooks/delete_cluster.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@
3131
- hosts:
3232
- etcd
3333
roles:
34-
- role: uninstall/etcd
35-
when: .delete.etcd
34+
- role: etcd/scaling_down
35+
when:
36+
- .delete.etcd
37+
- .etcd.deployment_type | eq "external"
3638

3739
- hosts:
3840
- image_registry

‎builtin/core/playbooks/delete_nodes.yaml‎

Lines changed: 93 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,17 @@
1111
- defaults
1212
- precheck
1313

14+
- hosts:
15+
- etcd
16+
roles:
17+
- role: etcd
18+
when:
19+
- .delete.etcd
20+
- .etcd.deployment_type | eq "external"
21+
1422
- hosts:
1523
- kube_control_plane
16-
gather_facts: true
17-
tasks:
24+
pre_tasks:
1825
- name: DeleteNode | Ensure at least one control plane node remains in the cluster
1926
run_once: true
2027
command: |
@@ -28,6 +35,90 @@
2835
echo "At least one control plane node must be retained in the cluster." >&2
2936
exit 1
3037
{{- end }}
38+
tasks:
39+
- name: DeleteNode | Update etcd certificate for kube_control_plane
40+
when:
41+
- .delete.etcd
42+
- .etcd.deployment_type | eq "external"
43+
block:
44+
- name: DeleteNode | Check if should update apiserver certificates
45+
run_once: true
46+
add_hostvars:
47+
hosts: kube_control_plane
48+
vars:
49+
need_uninstall_etcd: >-
50+
{{- $needUnInstalled := list -}}
51+
{{- range .groups.etcd -}}
52+
{{- if $.delete_nodes | default list | has . -}}
53+
{{- $needUnInstalled = append $needUnInstalled . -}}
54+
{{- end -}}
55+
{{- end -}}
56+
{{ $needUnInstalled | toJson }}
57+
- name: DeleteNode | Update apiserver etcd certificates
58+
when:
59+
- .need_uninstall_etcd | fromJson | empty | not
60+
block:
61+
- name: DeleteNode | Copy etcd CA certificate to control plane node
62+
copy:
63+
src: >-
64+
{{ .etcd.ca_file }}
65+
dest: /etc/kubernetes/pki/etcd/ca.crt
66+
- name: DeleteNode | Copy etcd client certificate to control plane node
67+
copy:
68+
src: >-
69+
{{ .etcd.cert_file }}
70+
dest: /etc/kubernetes/pki/etcd/client.crt
71+
- name: DeleteNode | Copy etcd client key to control plane node
72+
copy:
73+
src: >-
74+
{{ .etcd.key_file }}
75+
dest: /etc/kubernetes/pki/etcd/client.key
76+
- name: DeleteNode | update ks-apiserver
77+
command: |
78+
{{- $endpoints := list -}}
79+
{{- range .groups.etcd | default list -}}
80+
{{- if $.need_uninstall_etcd | fromJson | has . | not -}}
81+
{{- $endpoints = append $endpoints (printf "https://%s:2379" (index $.hostvars . "internal_ipv4")) -}}
82+
{{- end -}}
83+
{{- end -}}
84+
ETCD_ENDPOINTS="{{ join "," $endpoints }}"
85+
86+
if ! grep -q 'ClusterConfiguration' /etc/kubernetes/kubeadm-config.yaml 2>/dev/null; then
87+
kubectl get cm kubeadm-config -n kube-system -o=jsonpath='{.data.ClusterConfiguration}' > /etc/kubernetes/kubeadm-config.yaml
88+
fi
89+
90+
awk -v ep="$ETCD_ENDPOINTS" '
91+
BEGIN {
92+
n = split(ep, arr, ",")
93+
for (i = 1; i <= n; i++) {
94+
print " - " arr[i]
95+
}
96+
}
97+
' > /etc/kubernetes/kubeadm_new_endpoints.yaml
98+
# delete old endpoint
99+
sed -i '/^[[:space:]]*endpoints:/{
100+
:loop
101+
N
102+
s/\n[[:space:]]\+-.*//; t loop
103+
s/\n[[:space:]]*\n/\n/g
104+
P
105+
D
106+
}' /etc/kubernetes/kubeadm-config.yaml
107+
# insert new endpoint
108+
sed -i "/^[[:space:]]*endpoints:/r /etc/kubernetes/kubeadm_new_endpoints.yaml" /etc/kubernetes/kubeadm-config.yaml
109+
rm /etc/kubernetes/kubeadm_new_endpoints.yaml
110+
# update kubeadm-config
111+
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
112+
kubeadm config upload from-file --config /etc/kubernetes/kubeadm-config.yaml
113+
{{- else }}
114+
kubeadm init phase upload-config kubeadm --config /etc/kubernetes/kubeadm-config.yaml
115+
{{- end }}
116+
# regenerate kube-apiserver
117+
{{- if .kubernetes.kube_version | semverCompare "<v1.27.0" }}
118+
kubeadm init phase control-plane apiserver --config /etc/kubernetes/kubeadm-config.yaml
119+
{{- else }}
120+
kubeadm init phase control-plane apiserver
121+
{{- end }}
31122
32123
- hosts:
33124
- k8s_cluster
@@ -67,14 +158,6 @@
67158
- .delete.dns
68159
- .delete_nodes | default list | has .inventory_hostname
69160

70-
- hosts:
71-
- etcd
72-
roles:
73-
- role: uninstall/etcd
74-
when:
75-
- .delete.etcd
76-
- .delete_nodes | default list | has .inventory_hostname
77-
78161
- hosts:
79162
- image_registry
80163
roles:

‎builtin/core/roles/defaults/vars/v1.31.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,4 +89,3 @@ image_manifests:
8989
- docker.io/openebs/linux-utils:4.1.0
9090
- docker.io/openebs/provisioner-localpv:4.1.0
9191
- quay.io/tigera/operator:v1.34.5
92-

builtin/core/roles/etcd/files/backup.service renamed to builtin/core/roles/etcd/install/files/backup.service

File renamed without changes.

builtin/core/roles/etcd/files/etcd.service renamed to builtin/core/roles/etcd/install/files/etcd.service

File renamed without changes.

builtin/core/roles/etcd/tasks/backup_service.yaml renamed to builtin/core/roles/etcd/install/tasks/backup_service.yaml

File renamed without changes.

builtin/core/roles/etcd/tasks/install.yaml renamed to builtin/core/roles/etcd/install/tasks/install.yaml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,6 @@
1212
loop:
1313
- "{{ .etcd.env.data_dir }}"
1414

15-
- name: Install | Generate etcd environment configuration file
16-
template:
17-
src: etcd.env
18-
dest: /etc/etcd.env
19-
2015
- name: Install | Deploy etcd systemd service file
2116
copy:
2217
src: etcd.service

‎builtin/core/roles/etcd/install/tasks/main.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
- include_tasks: install.yaml
3+
4+
- include_tasks: backup_service.yaml

0 commit comments

Comments
 (0)