Skip to content

Commit 388293c

Browse files
lukasfranklukasfrank
committed
Gate machine scheduling on MachinePool Ready condition
Signed-off-by: Lukas Frank <lukas.frank@sap.com>
1 parent bb80028 commit 388293c

2 files changed

Lines changed: 81 additions & 1 deletion

File tree

‎internal/controllers/compute/machine_scheduler.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,17 @@ func (s *MachineScheduler) matchesLabels(_ context.Context, pool *scheduler.Cont
8282
return machinePoolSelector.Matches(nodeLabels)
8383
}
8484

85+
func (s *MachineScheduler) poolReady(_ context.Context, pool *scheduler.ContainerInfo) bool {
86+
cond := computev1alpha1.FindMachinePoolCondition(pool.Node().Status.Conditions, computev1alpha1.MachinePoolReady)
87+
88+
return cond != nil && cond.Status == corev1.ConditionTrue
89+
}
90+
8591
func (s *MachineScheduler) tolerateTaints(_ context.Context, pool *scheduler.ContainerInfo, machine *computev1alpha1.Machine) bool {
8692
return v1alpha1.TolerateTaints(machine.Spec.Tolerations, pool.Node().Spec.Taints)
8793
}
8894

8995
func (s *MachineScheduler) fitsPool(_ context.Context, pool *scheduler.ContainerInfo, machine *computev1alpha1.Machine) bool {
90-
9196
return pool.MaxAllocatable(machine.Spec.MachineClassRef.Name) > 0
9297
}
9398

@@ -102,6 +107,10 @@ func (s *MachineScheduler) reconcileExists(ctx context.Context, log logr.Logger,
102107

103108
var filteredNodes []*scheduler.ContainerInfo
104109
for _, node := range nodes {
110+
if !s.poolReady(ctx, node) {
111+
log.Info("node filtered", "reason", "pool not ready")
112+
continue
113+
}
105114
if !s.tolerateTaints(ctx, node, machine) {
106115
log.Info("node filtered", "reason", "taints do not match")
107116
continue

‎internal/controllers/compute/machine_scheduler_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ import (
2121
corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1"
2222
)
2323

24+
func markMachinePoolReady(machinePool *computev1alpha1.MachinePool) {
25+
machinePool.Status.Conditions = computev1alpha1.SetMachinePoolCondition(machinePool.Status.Conditions,
26+
computev1alpha1.MachinePoolCondition{
27+
Type: computev1alpha1.MachinePoolReady,
28+
Status: corev1.ConditionTrue,
29+
})
30+
}
31+
2432
var _ = Describe("MachineScheduler", func() {
2533
ns := SetupNamespace(&k8sClient)
2634
machineClass := SetupMachineClass()
@@ -40,6 +48,7 @@ var _ = Describe("MachineScheduler", func() {
4048
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
4149
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
4250
}
51+
markMachinePoolReady(machinePool)
4352
})).Should(Succeed())
4453

4554
By("creating a machine w/ the requested machine class")
@@ -94,6 +103,7 @@ var _ = Describe("MachineScheduler", func() {
94103
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
95104
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
96105
}
106+
markMachinePoolReady(machinePool)
97107
})).Should(Succeed())
98108

99109
By("waiting for the machine to be scheduled onto the machine pool")
@@ -118,6 +128,7 @@ var _ = Describe("MachineScheduler", func() {
118128
machinePoolNoMatchingLabels.Status.Allocatable = corev1alpha1.ResourceList{
119129
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
120130
}
131+
markMachinePoolReady(machinePoolNoMatchingLabels)
121132
})).Should(Succeed())
122133

123134
By("creating a machine pool w/ matching labels")
@@ -137,6 +148,7 @@ var _ = Describe("MachineScheduler", func() {
137148
machinePoolMatchingLabels.Status.Allocatable = corev1alpha1.ResourceList{
138149
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
139150
}
151+
markMachinePoolReady(machinePoolMatchingLabels)
140152
})).Should(Succeed())
141153

142154
By("creating a machine w/ the requested machine class")
@@ -191,6 +203,7 @@ var _ = Describe("MachineScheduler", func() {
191203
taintedMachinePool.Status.Allocatable = corev1alpha1.ResourceList{
192204
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
193205
}
206+
markMachinePoolReady(taintedMachinePool)
194207
})).Should(Succeed())
195208

196209
By("creating a machine")
@@ -236,6 +249,11 @@ var _ = Describe("MachineScheduler", func() {
236249
})
237250
Expect(k8sClient.Patch(ctx, machine, client.MergeFrom(machineBase))).To(Succeed(), "failed to patch the machine's spec")
238251

252+
By("re-asserting the machine pool is ready")
253+
Eventually(UpdateStatus(taintedMachinePool, func() {
254+
markMachinePoolReady(taintedMachinePool)
255+
})).Should(Succeed())
256+
239257
By("observing the machine is scheduled onto the machine pool")
240258
Eventually(Object(machine)).Should(SatisfyAll(
241259
HaveField("Spec.MachinePoolRef", Equal(&corev1.LocalObjectReference{Name: taintedMachinePool.Name})),
@@ -257,6 +275,7 @@ var _ = Describe("MachineScheduler", func() {
257275
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
258276
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
259277
}
278+
markMachinePoolReady(machinePool)
260279
})).Should(Succeed())
261280

262281
By("creating a second machine pool")
@@ -289,6 +308,7 @@ var _ = Describe("MachineScheduler", func() {
289308
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("5"),
290309
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, secondMachineClass.Name): resource.MustParse("100"),
291310
}
311+
markMachinePoolReady(secondMachinePool)
292312
})).Should(Succeed())
293313

294314
By("creating a machine")
@@ -326,6 +346,7 @@ var _ = Describe("MachineScheduler", func() {
326346
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
327347
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("50"),
328348
}
349+
markMachinePoolReady(machinePool)
329350
})).Should(Succeed())
330351

331352
By("creating a second machine pool")
@@ -344,6 +365,7 @@ var _ = Describe("MachineScheduler", func() {
344365
secondMachinePool.Status.Allocatable = corev1alpha1.ResourceList{
345366
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("50"),
346367
}
368+
markMachinePoolReady(secondMachinePool)
347369
})).Should(Succeed())
348370

349371
By("creating machines")
@@ -394,6 +416,7 @@ var _ = Describe("MachineScheduler", func() {
394416
By("patching the machine pool status to contain a machine class")
395417
Eventually(UpdateStatus(machinePool, func() {
396418
machinePool.Status.AvailableMachineClasses = []corev1.LocalObjectReference{{Name: machineClass.Name}}
419+
markMachinePoolReady(machinePool)
397420
})).Should(Succeed())
398421

399422
By("creating a machine")
@@ -420,6 +443,7 @@ var _ = Describe("MachineScheduler", func() {
420443
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
421444
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
422445
}
446+
markMachinePoolReady(machinePool)
423447
})).Should(Succeed())
424448

425449
By("checking that the machine is scheduled onto the machine pool")
@@ -443,6 +467,7 @@ var _ = Describe("MachineScheduler", func() {
443467
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
444468
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("2"),
445469
}
470+
markMachinePoolReady(machinePool)
446471
})).Should(Succeed())
447472

448473
By("creating the first machine")
@@ -512,6 +537,7 @@ var _ = Describe("MachineScheduler", func() {
512537
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
513538
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("5"),
514539
}
540+
markMachinePoolReady(machinePool)
515541
})).Should(Succeed())
516542

517543
By("creating a second machine pool")
@@ -545,6 +571,7 @@ var _ = Describe("MachineScheduler", func() {
545571
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("5"),
546572
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, secondMachineClass.Name): resource.MustParse("5"),
547573
}
574+
markMachinePoolReady(secondMachinePool)
548575
})).Should(Succeed())
549576

550577
By("creating a machine")
@@ -566,4 +593,48 @@ var _ = Describe("MachineScheduler", func() {
566593
HaveField("Spec.MachinePoolRef.Name", Equal(secondMachinePool.Name)),
567594
))
568595
})
596+
597+
It("should not schedule machines onto a machine pool that is not ready", func(ctx SpecContext) {
598+
By("creating a machine pool")
599+
machinePool := &computev1alpha1.MachinePool{
600+
ObjectMeta: metav1.ObjectMeta{
601+
GenerateName: "test-pool-",
602+
},
603+
}
604+
Expect(k8sClient.Create(ctx, machinePool)).To(Succeed(), "failed to create machine pool")
605+
606+
By("patching the machine pool status to have capacity but no ready condition")
607+
Eventually(UpdateStatus(machinePool, func() {
608+
machinePool.Status.AvailableMachineClasses = []corev1.LocalObjectReference{{Name: machineClass.Name}}
609+
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
610+
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("10"),
611+
}
612+
})).Should(Succeed())
613+
614+
By("creating a machine w/ the requested machine class")
615+
machine := &computev1alpha1.Machine{
616+
ObjectMeta: metav1.ObjectMeta{
617+
Namespace: ns.Name,
618+
GenerateName: "test-machine-",
619+
},
620+
Spec: computev1alpha1.MachineSpec{
621+
MachineClassRef: corev1.LocalObjectReference{Name: machineClass.Name},
622+
},
623+
}
624+
Expect(k8sClient.Create(ctx, machine)).To(Succeed(), "failed to create machine")
625+
626+
By("observing the machine isn't scheduled onto the not-ready machine pool")
627+
Consistently(Object(machine)).Should(HaveField("Spec.MachinePoolRef", BeNil()))
628+
629+
By("marking the machine pool ready")
630+
Eventually(UpdateStatus(machinePool, func() {
631+
markMachinePoolReady(machinePool)
632+
})).Should(Succeed())
633+
634+
By("waiting for the machine to be scheduled once the pool is ready")
635+
Eventually(Object(machine)).Should(SatisfyAll(
636+
HaveField("Spec.MachinePoolRef", Equal(&corev1.LocalObjectReference{Name: machinePool.Name})),
637+
HaveField("Status.State", Equal(computev1alpha1.MachineStatePending)),
638+
))
639+
})
569640
})

0 commit comments

Comments
 (0)