Skip to content

Commit 5af85a0

Browse files
weyfonkthardeck
weyfonk
authored and
thardeck
committed
Support graceful agent shutdown (#3918)
The Fleet agent must be able to properly handle `SIGTERM`, which amounts to exiting with error code 0 (instead of 1) when its context is canceled. New end-to-end test cases for SIGTERM handling should uncover possible future regressions such as the one fixed by the previous commit. * Deploy the Fleet agent into namespace `cattle-fleet-local-system` That configuration is consistent with what `dev/setup-fleet` does, which is itself aligned on Rancher behaviour.
1 parent 9933345 commit 5af85a0

4 files changed

Lines changed: 171 additions & 4 deletions

File tree

‎.github/scripts/deploy-fleet.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ eventually helm upgrade --install fleet charts/fleet \
6161
--set agentImage.repository="$agentRepo" \
6262
--set agentImage.tag="$agentTag" \
6363
--set agentImage.imagePullPolicy=IfNotPresent \
64+
--set bootstrap.agentNamespace=cattle-fleet-local-system \
6465
--set apiServerCA="$ca" \
6566
--set apiServerURL="$server" \
6667
$shards_settings \
@@ -71,7 +72,7 @@ eventually helm upgrade --install fleet charts/fleet \
7172
# wait for controller and agent rollout
7273
kubectl -n cattle-fleet-system rollout status deploy/fleet-controller
7374
{ grep -E -q -m 1 "fleet-agent-local.*1/1"; kill $!; } < <(kubectl get bundles -n fleet-local -w)
74-
kubectl -n cattle-fleet-system rollout status deployment/fleet-agent
75+
kubectl -n cattle-fleet-local-system rollout status deployment/fleet-agent
7576

7677
# label local cluster
7778
kubectl patch clusters.fleet.cattle.io -n fleet-local local --type=json -p '[{"op": "add", "path": "/metadata/labels/management.cattle.io~1cluster-display-name", "value": "local" }]'

‎e2e/installation/verify_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ var _ = Describe("Fleet Installation", func() {
1919
k kubectl.Command
2020
version = "dev"
2121
// this is the default for fleet standalone
22-
localAgentNamespace = "cattle-fleet-system"
22+
localAgentNamespace = "cattle-fleet-local-system"
2323
agentNamespace = "cattle-fleet-system"
2424
)
2525

‎e2e/single-cluster/signals_test.go‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package singlecluster_test
2+
3+
import (
4+
"encoding/json"
5+
6+
. "github.com/onsi/ginkgo/v2"
7+
. "github.com/onsi/gomega"
8+
9+
"github.com/rancher/fleet/e2e/testenv/kubectl"
10+
)
11+
12+
type containerLastTerminatedState struct {
13+
ExitCode int `json:"exitCode"`
14+
Reason string `json:"reason"`
15+
}
16+
17+
var _ = Describe("Shuts down gracefully", func() {
18+
var (
19+
k kubectl.Command
20+
ns string
21+
labels string
22+
)
23+
24+
When("the agent receives SIGTERM", func() {
25+
BeforeEach(func() {
26+
ns = "cattle-fleet-local-system"
27+
k = env.Kubectl.Namespace(ns)
28+
})
29+
30+
JustBeforeEach(func() {
31+
out, err := k.Run("exec", "deploy/fleet-agent", "--", "kill", "1")
32+
Expect(err).ToNot(HaveOccurred(), out)
33+
})
34+
35+
It("exits gracefully", func() {
36+
Eventually(func(g Gomega) {
37+
out, err := k.Get("pod", "-l", "app=fleet-agent", "-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}")
38+
g.Expect(err).ToNot(HaveOccurred())
39+
40+
var state containerLastTerminatedState
41+
err = json.Unmarshal([]byte(out), &state)
42+
g.Expect(err).ToNot(HaveOccurred())
43+
44+
g.Expect(state.ExitCode).To(Equal(0))
45+
g.Expect(state.Reason).To(Equal("Completed"))
46+
}).Should(Succeed())
47+
})
48+
})
49+
50+
// Note: when dealing with sharded deployments, running `kubectl exec <deployment_name>` does not resolve to the
51+
// right deployment, as if name resolution were prefix-based. This causes checks against pods' container states
52+
// to fail, because changes would have been applied to a different deployment.
53+
// Therefore, we explicitly compute pod names based on labels.
54+
When("the fleet-controller deployment receives SIGTERM", func() {
55+
BeforeEach(func() {
56+
ns = "cattle-fleet-system"
57+
k = env.Kubectl.Namespace(ns)
58+
labels = "app=fleet-controller,fleet.cattle.io/shard-default=true"
59+
})
60+
61+
JustBeforeEach(func() {
62+
pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
63+
Expect(err).ToNot(HaveOccurred(), pod)
64+
65+
out, err := k.Run("exec", pod, "--", "kill", "1")
66+
Expect(err).ToNot(HaveOccurred(), out)
67+
})
68+
69+
It("exits gracefully", func() {
70+
Eventually(func(g Gomega) {
71+
out, err := k.Get(
72+
"pod",
73+
"-l", labels,
74+
"-o", `jsonpath={.items[0].status.containerStatuses[?(@.name=="fleet-controller")].lastState.terminated}`,
75+
)
76+
g.Expect(err).ToNot(HaveOccurred())
77+
78+
var state containerLastTerminatedState
79+
err = json.Unmarshal([]byte(out), &state)
80+
g.Expect(err).ToNot(HaveOccurred(), out)
81+
82+
g.Expect(state.ExitCode).To(Equal(0))
83+
g.Expect(state.Reason).To(Equal("Completed"))
84+
}).Should(Succeed())
85+
})
86+
})
87+
88+
When("the gitjob deployment receives SIGTERM", func() {
89+
BeforeEach(func() {
90+
ns = "cattle-fleet-system"
91+
k = env.Kubectl.Namespace(ns)
92+
labels = "app=gitjob,fleet.cattle.io/shard-default=true"
93+
})
94+
95+
JustBeforeEach(func() {
96+
pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
97+
Expect(err).ToNot(HaveOccurred(), pod)
98+
99+
out, err := k.Run("exec", pod, "--", "kill", "1")
100+
Expect(err).ToNot(HaveOccurred(), out)
101+
})
102+
103+
It("exits gracefully", func() {
104+
Eventually(func(g Gomega) {
105+
out, err := k.Get(
106+
"pod",
107+
"-l", labels,
108+
"-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}",
109+
)
110+
g.Expect(err).ToNot(HaveOccurred())
111+
112+
var state containerLastTerminatedState
113+
err = json.Unmarshal([]byte(out), &state)
114+
g.Expect(err).ToNot(HaveOccurred())
115+
116+
g.Expect(state.ExitCode).To(Equal(0))
117+
g.Expect(state.Reason).To(Equal("Completed"))
118+
}).Should(Succeed())
119+
})
120+
})
121+
122+
When("the helmops deployment receives SIGTERM", func() {
123+
BeforeEach(func() {
124+
ns = "cattle-fleet-system"
125+
k = env.Kubectl.Namespace(ns)
126+
labels = "app=helmops,fleet.cattle.io/shard-default=true"
127+
})
128+
129+
JustBeforeEach(func() {
130+
pod, err := k.Get("pod", "-l", labels, "-o", "jsonpath={.items[0].metadata.name}")
131+
Expect(err).ToNot(HaveOccurred(), pod)
132+
133+
out, err := k.Run("exec", pod, "--", "kill", "1")
134+
Expect(err).ToNot(HaveOccurred(), out)
135+
})
136+
137+
It("exits gracefully", func() {
138+
Eventually(func(g Gomega) {
139+
out, err := k.Get(
140+
"pod",
141+
"-l", labels,
142+
"-o", "jsonpath={.items[0].status.containerStatuses[0].lastState.terminated}",
143+
)
144+
g.Expect(err).ToNot(HaveOccurred())
145+
146+
var state containerLastTerminatedState
147+
err = json.Unmarshal([]byte(out), &state)
148+
g.Expect(err).ToNot(HaveOccurred())
149+
150+
g.Expect(state.ExitCode).To(Equal(0))
151+
g.Expect(state.Reason).To(Equal("Completed"))
152+
}).Should(Succeed())
153+
})
154+
})
155+
})

‎internal/cmd/agent/root.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,19 @@ func (a *FleetAgent) Run(cmd *cobra.Command, args []string) error {
161161
}
162162
},
163163
OnStoppedLeading: func() {
164-
setupLog.Info("stopped leading")
165-
os.Exit(1)
164+
select {
165+
case <-ctx.Done():
166+
// Request to terminate.
167+
// This must be handled gracefully to prevent Kubernetes from recording the
168+
// container as exiting in error when termination was requested.
169+
// This situation matches SIGTERM being sent, which happens when a node is shut
170+
// down.
171+
setupLog.Info("termination requested, exiting")
172+
os.Exit(0)
173+
default:
174+
setupLog.Info("stopped leading")
175+
os.Exit(1)
176+
}
166177
},
167178
OnNewLeader: func(identity string) {
168179
if identity == identifier {

0 commit comments

Comments
 (0)