Skip to content

Commit 45778ac

Browse files
melancholictheorymelancholictheory
committed
feat(api): pod/container securityContext for Valkey workloads
ValkeyCluster pods could not be admitted in namespaces enforcing the restricted Pod Security Standard: the StatefulSet pod template set only fsGroup/runAsUser at pod level and no container securityContext, so config-init, valkey and the metrics exporter were rejected. Add spec.podSecurityContext and spec.containerSecurityContext (both API versions; conversion is a lossless round-trip). When unset, the operator applies restricted-PSA-compatible defaults that preserve the historical fsGroup/runAsUser 1000: pod: runAsNonRoot, seccompProfile RuntimeDefault, fsGroup/runAsUser 1000 container: allowPrivilegeEscalation false, capabilities drop ALL, runAsNonRoot, seccompProfile RuntimeDefault Applied to every container of the main StatefulSet (config-init, valkey, exporter) and the Sentinel StatefulSet; the per-shard layout inherits the main pod template. User-set contexts replace the defaults verbatim. The operator's Jobs (cluster bootstrap/reshard, backup/restore) are a follow-up: the valkey-image Jobs are straightforward, but the aws-cli backup/restore image runs as root and needs explicit handling. Refs #9.
1 parent d3c87a2 commit 45778ac

9 files changed

Lines changed: 1897 additions & 13 deletions

File tree

‎api/v1alpha1/valkeycluster_types.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,19 @@ type ValkeyClusterSpec struct {
165165
// +optional
166166
TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
167167

168+
// PodSecurityContext sets the pod-level security context for Valkey pods.
169+
// If nil the operator applies a restricted-PSA-compatible default
170+
// (fsGroup/runAsUser 1000, runAsNonRoot, seccompProfile RuntimeDefault).
171+
// +optional
172+
PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
173+
174+
// ContainerSecurityContext is applied to every operator-managed Valkey
175+
// container (config-init, valkey, and the metrics exporter). If nil the
176+
// operator applies a restricted-PSA-compatible default (allowPrivilegeEscalation
177+
// false, capabilities drop ALL, runAsNonRoot, seccompProfile RuntimeDefault).
178+
// +optional
179+
ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"`
180+
168181
// PodDisruptionBudget controls voluntary disruptions for the Valkey StatefulSet.
169182
// Defaults to MaxUnavailable=1 when nil.
170183
// +optional

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1beta1/valkeycluster_types.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,19 @@ type ValkeyClusterSpec struct {
168168
// +optional
169169
TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
170170

171+
// PodSecurityContext sets the pod-level security context for Valkey pods.
172+
// If nil the operator applies a restricted-PSA-compatible default
173+
// (fsGroup/runAsUser 1000, runAsNonRoot, seccompProfile RuntimeDefault).
174+
// +optional
175+
PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"`
176+
177+
// ContainerSecurityContext is applied to every operator-managed Valkey
178+
// container (config-init, valkey, and the metrics exporter). If nil the
179+
// operator applies a restricted-PSA-compatible default (allowPrivilegeEscalation
180+
// false, capabilities drop ALL, runAsNonRoot, seccompProfile RuntimeDefault).
181+
// +optional
182+
ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"`
183+
171184
// PodDisruptionBudget controls voluntary disruptions for the Valkey StatefulSet.
172185
// Defaults to MaxUnavailable=1 when nil.
173186
// +optional

‎api/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/valkey-operator/crds/cache.wellcake.io_valkeyclusters.yaml‎

Lines changed: 858 additions & 0 deletions
Large diffs are not rendered by default.

‎config/crd/bases/cache.wellcake.io_valkeyclusters.yaml‎

Lines changed: 858 additions & 0 deletions
Large diffs are not rendered by default.

‎internal/controller/resources.go‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,50 @@ func updateStrategyFor(proactive bool) appsv1.StatefulSetUpdateStrategy {
711711
return appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType}
712712
}
713713

714+
// valkeyRunAsID is the uid/gid the Valkey image runs as; kept as the historical
715+
// default so existing clusters don't change identity when the restricted-PSA
716+
// defaults are applied.
717+
const valkeyRunAsID int64 = 1000
718+
719+
// podSecurityContext returns the pod-level security context for Valkey pods: the
720+
// user's spec.podSecurityContext if set, else a restricted-PSA-compatible default
721+
// that preserves the historical fsGroup/runAsUser 1000.
722+
func podSecurityContext(vc *cachev1beta1.ValkeyCluster) *corev1.PodSecurityContext {
723+
if vc.Spec.PodSecurityContext != nil {
724+
return vc.Spec.PodSecurityContext
725+
}
726+
return &corev1.PodSecurityContext{
727+
FSGroup: ptr.To(valkeyRunAsID),
728+
RunAsUser: ptr.To(valkeyRunAsID),
729+
RunAsNonRoot: ptr.To(true),
730+
SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault},
731+
}
732+
}
733+
734+
// containerSecurityContext returns the container-level security context applied to
735+
// every operator-managed Valkey container: the user's
736+
// spec.containerSecurityContext if set, else a restricted-PSA-compatible default.
737+
func containerSecurityContext(vc *cachev1beta1.ValkeyCluster) *corev1.SecurityContext {
738+
if vc.Spec.ContainerSecurityContext != nil {
739+
return vc.Spec.ContainerSecurityContext
740+
}
741+
return &corev1.SecurityContext{
742+
AllowPrivilegeEscalation: ptr.To(false),
743+
RunAsNonRoot: ptr.To(true),
744+
Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}},
745+
SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault},
746+
}
747+
}
748+
749+
// applyContainerSecurityContext sets sc on every container in the given slices.
750+
func applyContainerSecurityContext(sc *corev1.SecurityContext, groups ...[]corev1.Container) {
751+
for _, g := range groups {
752+
for i := range g {
753+
g[i].SecurityContext = sc
754+
}
755+
}
756+
}
757+
714758
func buildStatefulSet(vc *cachev1beta1.ValkeyCluster, configHash string, proactive bool) *appsv1.StatefulSet {
715759
labels := labelsFor(vc)
716760
replicas := statefulSetReplicas(vc)
@@ -827,6 +871,7 @@ func buildStatefulSet(vc *cachev1beta1.ValkeyCluster, configHash string, proacti
827871
if metricsEnabled(vc) {
828872
containers = append(containers, buildExporter(vc))
829873
}
874+
applyContainerSecurityContext(containerSecurityContext(vc), initContainers, containers)
830875

831876
volumes := []corev1.Volume{
832877
{
@@ -932,10 +977,7 @@ func buildStatefulSet(vc *cachev1beta1.ValkeyCluster, configHash string, proacti
932977
Tolerations: vc.Spec.Tolerations,
933978
Affinity: affinity,
934979
TopologySpreadConstraints: tsc,
935-
SecurityContext: &corev1.PodSecurityContext{
936-
FSGroup: ptr.To[int64](1000),
937-
RunAsUser: ptr.To[int64](1000),
938-
},
980+
SecurityContext: podSecurityContext(vc),
939981
},
940982
},
941983
VolumeClaimTemplates: pvcs,

‎internal/controller/resources_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1318,3 +1318,85 @@ func TestBuildScaleDownJobIsDataSafe(t *testing.T) {
13181318
t.Errorf("scale-down must refuse del-node while the node still owns slots; script:\n%s", s)
13191319
}
13201320
}
1321+
1322+
func assertContainerRestricted(t *testing.T, who string, sc *corev1.SecurityContext) {
1323+
t.Helper()
1324+
if sc == nil {
1325+
t.Errorf("%s: nil container securityContext", who)
1326+
return
1327+
}
1328+
if sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation {
1329+
t.Errorf("%s: allowPrivilegeEscalation must be false", who)
1330+
}
1331+
if sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot {
1332+
t.Errorf("%s: runAsNonRoot must be true", who)
1333+
}
1334+
if sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault {
1335+
t.Errorf("%s: seccompProfile must be RuntimeDefault", who)
1336+
}
1337+
dropsAll := false
1338+
if sc.Capabilities != nil {
1339+
for _, c := range sc.Capabilities.Drop {
1340+
if c == "ALL" {
1341+
dropsAll = true
1342+
}
1343+
}
1344+
}
1345+
if !dropsAll {
1346+
t.Errorf("%s: capabilities must drop ALL", who)
1347+
}
1348+
}
1349+
1350+
func assertPodTemplateRestricted(t *testing.T, spec corev1.PodSpec) {
1351+
t.Helper()
1352+
ps := spec.SecurityContext
1353+
if ps == nil {
1354+
t.Fatal("nil pod securityContext")
1355+
}
1356+
if ps.RunAsNonRoot == nil || !*ps.RunAsNonRoot {
1357+
t.Error("pod runAsNonRoot must be true")
1358+
}
1359+
if ps.SeccompProfile == nil || ps.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault {
1360+
t.Error("pod seccompProfile must be RuntimeDefault")
1361+
}
1362+
if ps.RunAsUser == nil || *ps.RunAsUser != 1000 || ps.FSGroup == nil || *ps.FSGroup != 1000 {
1363+
t.Error("pod must keep fsGroup/runAsUser 1000")
1364+
}
1365+
if len(spec.InitContainers) == 0 || len(spec.Containers) == 0 {
1366+
t.Fatal("expected init + main containers")
1367+
}
1368+
for _, c := range spec.InitContainers {
1369+
assertContainerRestricted(t, "init/"+c.Name, c.SecurityContext)
1370+
}
1371+
for _, c := range spec.Containers {
1372+
assertContainerRestricted(t, c.Name, c.SecurityContext)
1373+
}
1374+
}
1375+
1376+
func TestPodAndContainerSecurityContextRestrictedDefaults(t *testing.T) {
1377+
// Main StatefulSet, metrics on so the exporter sidecar is covered too.
1378+
vc := minimalCR()
1379+
vc.Spec.Metrics = &cachev1beta1.MetricsSpec{Enabled: true}
1380+
sts := buildStatefulSet(vc, "h", false)
1381+
if got := len(sts.Spec.Template.Spec.Containers); got < 2 {
1382+
t.Fatalf("expected valkey + exporter containers, got %d", got)
1383+
}
1384+
assertPodTemplateRestricted(t, sts.Spec.Template.Spec)
1385+
1386+
// Sentinel StatefulSet.
1387+
assertPodTemplateRestricted(t, buildSentinelStatefulSet(sentinelCR(), false).Spec.Template.Spec)
1388+
1389+
// User-provided contexts replace the defaults verbatim.
1390+
custom := minimalCR()
1391+
custom.Spec.PodSecurityContext = &corev1.PodSecurityContext{RunAsUser: ptr.To[int64](2000)}
1392+
custom.Spec.ContainerSecurityContext = &corev1.SecurityContext{Privileged: ptr.To(true)}
1393+
cs := buildStatefulSet(custom, "h", false)
1394+
if ps := cs.Spec.Template.Spec.SecurityContext; ps == nil || ps.RunAsUser == nil || *ps.RunAsUser != 2000 {
1395+
t.Errorf("user podSecurityContext not honored: %+v", ps)
1396+
}
1397+
for _, c := range cs.Spec.Template.Spec.Containers {
1398+
if c.SecurityContext == nil || c.SecurityContext.Privileged == nil || !*c.SecurityContext.Privileged {
1399+
t.Errorf("%s: user containerSecurityContext not honored", c.Name)
1400+
}
1401+
}
1402+
}

‎internal/controller/sentinel.go‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"k8s.io/apimachinery/pkg/api/resource"
1616
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1717
"k8s.io/apimachinery/pkg/util/intstr"
18-
"k8s.io/utils/ptr"
1918
ctrl "sigs.k8s.io/controller-runtime"
2019
"sigs.k8s.io/controller-runtime/pkg/client"
2120
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -332,6 +331,7 @@ func buildSentinelStatefulSet(vc *cachev1beta1.ValkeyCluster, proactive bool) *a
332331
{Name: configVolumeName, MountPath: "/etc/sentinel", ReadOnly: true},
333332
{Name: dataVolumeName, MountPath: dataMountPath},
334333
},
334+
SecurityContext: containerSecurityContext(vc),
335335
}},
336336
Containers: []corev1.Container{{
337337
Name: componentSentinel,
@@ -343,15 +343,13 @@ func buildSentinelStatefulSet(vc *cachev1beta1.ValkeyCluster, proactive bool) *a
343343
ContainerPort: sentinelPort,
344344
Protocol: corev1.ProtocolTCP,
345345
}},
346-
VolumeMounts: volumeMounts,
347-
ReadinessProbe: tcpProbe(sentinelPort, 5, 5),
348-
LivenessProbe: tcpProbe(sentinelPort, 15, 20),
346+
VolumeMounts: volumeMounts,
347+
ReadinessProbe: tcpProbe(sentinelPort, 5, 5),
348+
LivenessProbe: tcpProbe(sentinelPort, 15, 20),
349+
SecurityContext: containerSecurityContext(vc),
349350
}},
350-
Volumes: volumes,
351-
SecurityContext: &corev1.PodSecurityContext{
352-
FSGroup: ptr.To[int64](1000),
353-
RunAsUser: ptr.To[int64](1000),
354-
},
351+
Volumes: volumes,
352+
SecurityContext: podSecurityContext(vc),
355353
Affinity: defaultAntiAffinity(vc),
356354
TopologySpreadConstraints: defaultTopologySpread(vc),
357355
},

0 commit comments

Comments
 (0)