|
| 1 | +/* |
| 2 | +Copyright 2026 The Wellcake Authors. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package controller |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "time" |
| 22 | + |
| 23 | + . "github.com/onsi/ginkgo/v2" |
| 24 | + . "github.com/onsi/gomega" |
| 25 | + |
| 26 | + coordinationv1 "k8s.io/api/coordination/v1" |
| 27 | + "k8s.io/client-go/kubernetes/scheme" |
| 28 | + "k8s.io/utils/ptr" |
| 29 | + ctrl "sigs.k8s.io/controller-runtime" |
| 30 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 31 | + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" |
| 32 | +) |
| 33 | + |
| 34 | +// Op1: prove HA handover between leader-elected operator replicas. Two managers |
| 35 | +// contend for the same lease against the envtest API server; the standby must |
| 36 | +// not steal an actively-renewed lease, and must take over promptly once the |
| 37 | +// active leader steps down. This exercises the real coordination.k8s.io Lease |
| 38 | +// path the deployed operator uses (`--leader-elect`), not a mock. |
| 39 | +var _ = Describe("Leader election handover (Op1)", func() { |
| 40 | + It("promotes a standby manager when the active leader steps down", func() { |
| 41 | + const leaseID = "op1-handover.wellcake.io" |
| 42 | + const leaseNS = "default" |
| 43 | + |
| 44 | + newMgr := func() ctrl.Manager { |
| 45 | + m, err := ctrl.NewManager(cfg, ctrl.Options{ |
| 46 | + Scheme: scheme.Scheme, |
| 47 | + LeaderElection: true, |
| 48 | + LeaderElectionID: leaseID, |
| 49 | + LeaderElectionNamespace: leaseNS, |
| 50 | + LeaderElectionReleaseOnCancel: true, |
| 51 | + // Short timings so the test converges in seconds; the invariant |
| 52 | + // (single holder, clean handover) is independent of the values. |
| 53 | + LeaseDuration: ptr.To(6 * time.Second), |
| 54 | + RenewDeadline: ptr.To(4 * time.Second), |
| 55 | + RetryPeriod: ptr.To(1 * time.Second), |
| 56 | + Metrics: metricsserver.Options{BindAddress: "0"}, |
| 57 | + HealthProbeBindAddress: "0", |
| 58 | + }) |
| 59 | + Expect(err).NotTo(HaveOccurred()) |
| 60 | + return m |
| 61 | + } |
| 62 | + |
| 63 | + // holder returns the current lease holder identity ("" if none). |
| 64 | + holder := func() string { |
| 65 | + var lease coordinationv1.Lease |
| 66 | + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: leaseNS, Name: leaseID}, &lease); err != nil { |
| 67 | + return "" |
| 68 | + } |
| 69 | + if lease.Spec.HolderIdentity == nil { |
| 70 | + return "" |
| 71 | + } |
| 72 | + return *lease.Spec.HolderIdentity |
| 73 | + } |
| 74 | + |
| 75 | + By("starting leader A") |
| 76 | + mgrA := newMgr() |
| 77 | + ctxA, cancelA := context.WithCancel(ctx) |
| 78 | + go func() { |
| 79 | + defer GinkgoRecover() |
| 80 | + _ = mgrA.Start(ctxA) |
| 81 | + }() |
| 82 | + |
| 83 | + Eventually(holder, 30*time.Second, 250*time.Millisecond).ShouldNot(BeEmpty()) |
| 84 | + leaderA := holder() |
| 85 | + By("leader A holds the lease: " + leaderA) |
| 86 | + |
| 87 | + By("starting standby B — it must not steal an actively-renewed lease") |
| 88 | + mgrB := newMgr() |
| 89 | + ctxB, cancelB := context.WithCancel(ctx) |
| 90 | + defer cancelB() |
| 91 | + go func() { |
| 92 | + defer GinkgoRecover() |
| 93 | + _ = mgrB.Start(ctxB) |
| 94 | + }() |
| 95 | + Consistently(holder, 5*time.Second, 1*time.Second).Should(Equal(leaderA)) |
| 96 | + |
| 97 | + By("leader A steps down (graceful cancel releases the lease)") |
| 98 | + cancelA() |
| 99 | + |
| 100 | + By("standby B takes over with a different, non-empty identity") |
| 101 | + Eventually(holder, 30*time.Second, 250*time.Millisecond). |
| 102 | + ShouldNot(Or(BeEmpty(), Equal(leaderA))) |
| 103 | + |
| 104 | + cancelB() |
| 105 | + }) |
| 106 | +}) |
0 commit comments