Skip to content

Commit c17e4fa

Browse files
oliverbaehleroliverbaehler
authored
feat: add leader-election tuning (#2008)
* feat: add leader-election tuning Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add leader-election tuning Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add tracing for admission Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: add tracing for admission Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * add tenant --------- Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
1 parent f7c5523 commit c17e4fa

51 files changed

Lines changed: 5194 additions & 41 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,8 @@ e2e-install: helm-controller-version ko-build-all dev-install-gw-api-crds
486486
--set 'manager.options.workers=4' \
487487
--set 'manager.options.clientConnectionQPS=2000' \
488488
--set 'manager.options.clientConnectionBurst=1000' \
489+
--set 'manager.options.leaderElection.leaseDuration=60s' \
490+
--set 'manager.options.leaderElection.renewDeadline=40s' \
489491
--set 'manager.rbac.minimal=true' \
490492
--set 'webhooks.hooks.nodes.enabled=true' \
491493
--set "webhooks.exclusive=true"\
@@ -516,6 +518,8 @@ e2e-install-openshift: helm-controller-version ko-build-all
516518
--set 'manager.resources=null'\
517519
--set "manager.image.tag=$(VERSION)" \
518520
--set 'manager.livenessProbe.failureThreshold=10' \
521+
--set 'manager.options.leaderElection.leaseDuration=60s' \
522+
--set 'manager.options.leaderElection.renewDeadline=40s' \
519523
--set 'webhooks.hooks.nodes.enabled=true' \
520524
--set "webhooks.exclusive=true"\
521525
--set "manager.options.logLevel=debug"\

‎charts/capsule/README.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ The following Values have changed key or Value:
112112
| manager.daemonsetStrategy | object | `{"type":"RollingUpdate"}` | [Daemonset Strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#creating-a-daemonset-with-rollingupdate-update-strategy) |
113113
| manager.deploymentStrategy | object | `{"type":"RollingUpdate"}` | [Deployment Strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) |
114114
| manager.env | list | `[]` | Additional Environment Variables |
115-
| manager.extraArgs | list | `["--enable-leader-election=true"]` | A list of extra arguments for the capsule controller |
115+
| manager.extraArgs | list | `[]` | A list of extra arguments for the capsule controller |
116116
| manager.hostNetwork | bool | `false` | Specifies if the container should be started in hostNetwork mode. Required for use in some managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working |
117117
| manager.hostPID | bool | `false` | Specifies if the container should be started in hostPID mode. |
118118
| manager.hostUsers | bool | `true` | Don't use Host Users (User Namespaces) |
@@ -137,6 +137,10 @@ The following Values have changed key or Value:
137137
| manager.options.ignoreUserWithGroups | list | `[]` | Define groups which when found in the request of a user will be ignored by the Capsule this might be useful if you have one group where all the users are in, but you want to separate administrators from normal users with additional groups. |
138138
| manager.options.impersonation | object | `{}` | Impersonation |
139139
| manager.options.labels | object | `{}` | Additional labels to add to the CapsuleConfiguration resource |
140+
| manager.options.leaderElection.enabled | bool | `true` | Enable Leader Election for capsule controller |
141+
| manager.options.leaderElection.leaseDuration | string | `""` | Leader election lease duration. Empty uses controller-runtime's default. |
142+
| manager.options.leaderElection.renewDeadline | string | `""` | Leader election renew deadline. Empty uses controller-runtime's default. |
143+
| manager.options.leaderElection.retryPeriod | string | `""` | Leader election retry period. Empty uses controller-runtime's default. |
140144
| manager.options.logLevel | string | `"info"` | Set the log verbosity of the capsule with a value from 1 to 5 |
141145
| manager.options.nodeMetadata | object | `{"forbiddenAnnotations":{"denied":[],"deniedRegex":""},"forbiddenLabels":{"denied":[],"deniedRegex":""}}` | Allows to set the forbidden metadata for the worker nodes that could be patched by a Tenant |
142146
| manager.options.protectedNamespaceRegex | string | `""` | If specified, disallows creation of namespaces matching the passed regexp |
@@ -145,6 +149,20 @@ The following Values have changed key or Value:
145149
| manager.options.rbac.deleter | string | `"capsule-namespace-deleter"` | Name for the ClusterRole required to grant Namespace Deletion permissions. |
146150
| manager.options.rbac.promotionClusterRoles | list | `["capsule-namespace-provisioner","capsule-namespace-deleter"]` | The ClusterRoles applied for ServiceAccounts which had owner Promotion |
147151
| manager.options.rbac.provisioner | string | `"capsule-namespace-provisioner"` | Name for the ClusterRole required to grant Namespace Provision permissions. |
152+
| manager.options.tracing.basicAuth.existingSecret.name | string | `""` | Existing Secret containing OTLP basic auth credentials. |
153+
| manager.options.tracing.basicAuth.existingSecret.passwordKey | string | `"password"` | Secret key containing the basic auth password. |
154+
| manager.options.tracing.basicAuth.existingSecret.usernameKey | string | `"username"` | Secret key containing the basic auth username. |
155+
| manager.options.tracing.basicAuth.password | string | `""` | Basic auth password for the OTLP gRPC trace exporter. Prefer existingSecret for production. |
156+
| manager.options.tracing.basicAuth.username | string | `""` | Basic auth username for the OTLP gRPC trace exporter. Prefer existingSecret for production. |
157+
| manager.options.tracing.compression | string | `""` | Compression for OTLP gRPC trace exports. Supported value: gzip. Empty disables compression. |
158+
| manager.options.tracing.enabled | bool | `false` | Enable OpenTelemetry tracing for admission webhook requests. |
159+
| manager.options.tracing.endpoint | string | `""` | OTLP gRPC endpoint for exporting traces. If empty, OpenTelemetry environment variables are used. |
160+
| manager.options.tracing.headers | object | `{}` | OTLP gRPC metadata headers to send with each trace export request. |
161+
| manager.options.tracing.insecure | bool | `true` | Disable transport security for the OTLP gRPC trace exporter. |
162+
| manager.options.tracing.sampleRatio | float | `1` | Trace sampling ratio for admission webhook requests. Must be between 0 and 1. |
163+
| manager.options.tracing.timeout | string | `""` | Timeout for exporting a batch of spans. Empty uses OpenTelemetry's default. |
164+
| manager.options.tracing.tls.insecureSkipVerify | bool | `false` | Skip OTLP gRPC trace exporter TLS certificate verification. Not recommended for production. |
165+
| manager.options.tracing.tls.serverName | string | `""` | TLS server name override for the OTLP gRPC trace exporter. |
148166
| manager.options.userNames | list | `[]` | DEPRECATED: use users properties. Names of the users considered as Capsule users. |
149167
| manager.options.users | list | `[{"kind":"Group","name":"projectcapsule.dev"}]` | Define entities which are considered part of the Capsule construct. Users not mentioned here will be ignored by Capsule |
150168
| manager.options.workers | int | `1` | Workers (MaxConcurrentReconciles) is the maximum number of concurrent Reconciles which can be run (ALPHA). |

‎charts/capsule/templates/_pod.tpl‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,37 @@ spec:
7070
{{- with .Values.manager.options.cacheSyncTimeout }}
7171
- --cache-sync-timeout={{ . }}
7272
{{- end }}
73+
- --enable-leader-election={{ .Values.manager.options.leaderElection.enabled }}
74+
{{- with .Values.manager.options.leaderElection.leaseDuration }}
75+
- --leader-election-lease-duration={{ . }}
76+
{{- end }}
77+
{{- with .Values.manager.options.leaderElection.renewDeadline }}
78+
- --leader-election-renew-deadline={{ . }}
79+
{{- end }}
80+
{{- with .Values.manager.options.leaderElection.retryPeriod }}
81+
- --leader-election-retry-period={{ . }}
82+
{{- end }}
83+
{{- if .Values.manager.options.tracing.enabled }}
84+
- --enable-tracing=true
85+
{{- with .Values.manager.options.tracing.endpoint }}
86+
- --tracing-otlp-endpoint={{ . }}
87+
{{- end }}
88+
- --tracing-otlp-insecure={{ .Values.manager.options.tracing.insecure }}
89+
- --tracing-sample-ratio={{ .Values.manager.options.tracing.sampleRatio }}
90+
{{- range $key, $value := .Values.manager.options.tracing.headers }}
91+
- --tracing-otlp-header={{ $key }}={{ $value }}
92+
{{- end }}
93+
{{- with .Values.manager.options.tracing.timeout }}
94+
- --tracing-otlp-timeout={{ . }}
95+
{{- end }}
96+
{{- with .Values.manager.options.tracing.compression }}
97+
- --tracing-otlp-compression={{ . }}
98+
{{- end }}
99+
{{- with .Values.manager.options.tracing.tls.serverName }}
100+
- --tracing-otlp-tls-server-name={{ . }}
101+
{{- end }}
102+
- --tracing-otlp-tls-insecure-skip-verify={{ .Values.manager.options.tracing.tls.insecureSkipVerify }}
103+
{{- end }}
73104
{{- with .Values.manager.extraArgs }}
74105
{{- toYaml . | nindent 8 }}
75106
{{- end }}
@@ -84,8 +115,32 @@ spec:
84115
valueFrom:
85116
fieldRef:
86117
fieldPath: spec.serviceAccountName
118+
{{- if .Values.manager.options.tracing.enabled }}
119+
{{- if or .Values.manager.options.tracing.basicAuth.username .Values.manager.options.tracing.basicAuth.existingSecret.name }}
120+
- name: CAPSULE_TRACING_OTLP_BASIC_AUTH_USERNAME
121+
{{- if .Values.manager.options.tracing.basicAuth.existingSecret.name }}
122+
valueFrom:
123+
secretKeyRef:
124+
name: {{ .Values.manager.options.tracing.basicAuth.existingSecret.name }}
125+
key: {{ .Values.manager.options.tracing.basicAuth.existingSecret.usernameKey }}
126+
{{- else }}
127+
value: {{ .Values.manager.options.tracing.basicAuth.username | quote }}
128+
{{- end }}
129+
{{- end }}
130+
{{- if or .Values.manager.options.tracing.basicAuth.password .Values.manager.options.tracing.basicAuth.existingSecret.name }}
131+
- name: CAPSULE_TRACING_OTLP_BASIC_AUTH_PASSWORD
132+
{{- if .Values.manager.options.tracing.basicAuth.existingSecret.name }}
133+
valueFrom:
134+
secretKeyRef:
135+
name: {{ .Values.manager.options.tracing.basicAuth.existingSecret.name }}
136+
key: {{ .Values.manager.options.tracing.basicAuth.existingSecret.passwordKey }}
137+
{{- else }}
138+
value: {{ .Values.manager.options.tracing.basicAuth.password | quote }}
139+
{{- end }}
140+
{{- end }}
141+
{{- end }}
87142
{{- with .Values.manager.env }}
88-
{{- toYaml . | nindent 6 }}
143+
{{- toYaml . | nindent 6 }}
89144
{{- end }}
90145
ports:
91146
{{- if not (.Values.manager.hostNetwork) }}

‎charts/capsule/values.schema.json‎

Lines changed: 98 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -357,10 +357,7 @@
357357
},
358358
"extraArgs": {
359359
"description": "A list of extra arguments for the capsule controller",
360-
"type": "array",
361-
"items": {
362-
"type": "string"
363-
}
360+
"type": "array"
364361
},
365362
"hostNetwork": {
366363
"description": "Specifies if the container should be started in hostNetwork mode. Required for use in some managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working",
@@ -480,6 +477,27 @@
480477
"description": "Additional labels to add to the CapsuleConfiguration resource",
481478
"type": "object"
482479
},
480+
"leaderElection": {
481+
"type": "object",
482+
"properties": {
483+
"enabled": {
484+
"description": "Enable Leader Election for capsule controller",
485+
"type": "boolean"
486+
},
487+
"leaseDuration": {
488+
"description": "Leader election lease duration. Empty uses controller-runtime's default.",
489+
"type": "string"
490+
},
491+
"renewDeadline": {
492+
"description": "Leader election renew deadline. Empty uses controller-runtime's default.",
493+
"type": "string"
494+
},
495+
"retryPeriod": {
496+
"description": "Leader election retry period. Empty uses controller-runtime's default.",
497+
"type": "string"
498+
}
499+
}
500+
},
483501
"logLevel": {
484502
"description": "Set the log verbosity of the capsule with a value from 1 to 5",
485503
"type": "string"
@@ -544,6 +562,82 @@
544562
}
545563
}
546564
},
565+
"tracing": {
566+
"type": "object",
567+
"properties": {
568+
"basicAuth": {
569+
"type": "object",
570+
"properties": {
571+
"existingSecret": {
572+
"type": "object",
573+
"properties": {
574+
"name": {
575+
"description": "Existing Secret containing OTLP basic auth credentials.",
576+
"type": "string"
577+
},
578+
"passwordKey": {
579+
"description": "Secret key containing the basic auth password.",
580+
"type": "string"
581+
},
582+
"usernameKey": {
583+
"description": "Secret key containing the basic auth username.",
584+
"type": "string"
585+
}
586+
}
587+
},
588+
"password": {
589+
"description": "Basic auth password for the OTLP gRPC trace exporter. Prefer existingSecret for production.",
590+
"type": "string"
591+
},
592+
"username": {
593+
"description": "Basic auth username for the OTLP gRPC trace exporter. Prefer existingSecret for production.",
594+
"type": "string"
595+
}
596+
}
597+
},
598+
"compression": {
599+
"description": "Compression for OTLP gRPC trace exports. Supported value: gzip. Empty disables compression.",
600+
"type": "string"
601+
},
602+
"enabled": {
603+
"description": "Enable OpenTelemetry tracing for admission webhook requests.",
604+
"type": "boolean"
605+
},
606+
"endpoint": {
607+
"description": "OTLP gRPC endpoint for exporting traces. If empty, OpenTelemetry environment variables are used.",
608+
"type": "string"
609+
},
610+
"headers": {
611+
"description": "OTLP gRPC metadata headers to send with each trace export request.",
612+
"type": "object"
613+
},
614+
"insecure": {
615+
"description": "Disable transport security for the OTLP gRPC trace exporter.",
616+
"type": "boolean"
617+
},
618+
"sampleRatio": {
619+
"description": "Trace sampling ratio for admission webhook requests. Must be between 0 and 1.",
620+
"type": "number"
621+
},
622+
"timeout": {
623+
"description": "Timeout for exporting a batch of spans. Empty uses OpenTelemetry's default.",
624+
"type": "string"
625+
},
626+
"tls": {
627+
"type": "object",
628+
"properties": {
629+
"insecureSkipVerify": {
630+
"description": "Skip OTLP gRPC trace exporter TLS certificate verification. Not recommended for production.",
631+
"type": "boolean"
632+
},
633+
"serverName": {
634+
"description": "TLS server name override for the OTLP gRPC trace exporter.",
635+
"type": "string"
636+
}
637+
}
638+
}
639+
}
640+
},
547641
"userNames": {
548642
"description": "DEPRECATED: use users properties. Names of the users considered as Capsule users.",
549643
"type": "array"

0 commit comments

Comments
 (0)