Skip to content

Commit 9036bfe

Browse files
ushabelgurushabelgur
authored
Change MachineClass capabilities to support arbitrary resources (#1311)
1 parent eabbe21 commit 9036bfe

10 files changed

Lines changed: 249 additions & 155 deletions

File tree

‎broker/machinebroker/server/status.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,15 +80,18 @@ func (s *Server) filterIronCoreMachineClasses(
8080
}
8181

8282
func (s *Server) convertIronCoreMachineClassStatus(machineClass *computev1alpha1.MachineClass, quantity *resource.Quantity) *iri.MachineClassStatus {
83-
cpu := machineClass.Capabilities.CPU()
84-
memory := machineClass.Capabilities.Memory()
83+
resources := map[string]int64{}
84+
resourceList := machineClass.Capabilities
85+
86+
for resource, quantity := range resourceList {
87+
resources[string(resource)] = quantity.Value()
88+
}
8589

8690
return &iri.MachineClassStatus{
8791
MachineClass: &iri.MachineClass{
8892
Name: machineClass.Name,
8993
Capabilities: &iri.MachineClassCapabilities{
90-
CpuMillis: cpu.MilliValue(),
91-
MemoryBytes: memory.Value(),
94+
Resources: resources,
9295
},
9396
},
9497
Quantity: quantity.Value(),

‎internal/controllers/compute/machine_scheduler_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,4 +434,75 @@ var _ = Describe("MachineScheduler", func() {
434434
HaveField("Spec.MachinePoolRef", Equal(&corev1.LocalObjectReference{Name: machinePool.Name})),
435435
))
436436
})
437+
438+
It("should schedule machine on pool with correctly allocatable resources", func(ctx SpecContext) {
439+
By("creating a machine pool")
440+
machinePool := &computev1alpha1.MachinePool{
441+
ObjectMeta: metav1.ObjectMeta{
442+
GenerateName: "test-pool-",
443+
},
444+
}
445+
Expect(k8sClient.Create(ctx, machinePool)).To(Succeed(), "failed to create machine pool")
446+
447+
By("patching the machine pool status to contain a machine class")
448+
Eventually(UpdateStatus(machinePool, func() {
449+
machinePool.Status.AvailableMachineClasses = []corev1.LocalObjectReference{{Name: machineClass.Name}}
450+
machinePool.Status.Allocatable = corev1alpha1.ResourceList{
451+
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("5"),
452+
}
453+
})).Should(Succeed())
454+
455+
By("creating a second machine pool")
456+
secondMachinePool := &computev1alpha1.MachinePool{
457+
ObjectMeta: metav1.ObjectMeta{
458+
GenerateName: "second-test-pool-",
459+
},
460+
}
461+
Expect(k8sClient.Create(ctx, secondMachinePool)).To(Succeed(), "failed to create the second machine pool")
462+
463+
By("creating a second machine class")
464+
secondMachineClass := &computev1alpha1.MachineClass{
465+
ObjectMeta: metav1.ObjectMeta{
466+
GenerateName: "second-machine-class-",
467+
},
468+
Capabilities: corev1alpha1.ResourceList{
469+
corev1alpha1.ResourceCPU: resource.MustParse("1"),
470+
corev1alpha1.ResourceMemory: resource.MustParse("1Gi"),
471+
"type-a.vendor.com/gpu": resource.MustParse("1"),
472+
},
473+
}
474+
Expect(k8sClient.Create(ctx, secondMachineClass)).To(Succeed(), "failed to create second machine class")
475+
476+
By("patching the second machine pool status to contain a both machine classes")
477+
Eventually(UpdateStatus(secondMachinePool, func() {
478+
secondMachinePool.Status.AvailableMachineClasses = []corev1.LocalObjectReference{
479+
{Name: machineClass.Name},
480+
{Name: secondMachineClass.Name},
481+
}
482+
secondMachinePool.Status.Allocatable = corev1alpha1.ResourceList{
483+
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, machineClass.Name): resource.MustParse("5"),
484+
corev1alpha1.ClassCountFor(corev1alpha1.ClassTypeMachineClass, secondMachineClass.Name): resource.MustParse("5"),
485+
}
486+
})).Should(Succeed())
487+
488+
By("creating a machine")
489+
machine := &computev1alpha1.Machine{
490+
ObjectMeta: metav1.ObjectMeta{
491+
Namespace: ns.Name,
492+
GenerateName: "test-machine-",
493+
},
494+
Spec: computev1alpha1.MachineSpec{
495+
Image: "my-image",
496+
MachineClassRef: corev1.LocalObjectReference{
497+
Name: secondMachineClass.Name,
498+
},
499+
},
500+
}
501+
Expect(k8sClient.Create(ctx, machine)).To(Succeed(), "failed to create the machine")
502+
503+
By("checking that the machine is scheduled onto the machine pool")
504+
Eventually(Object(machine)).Should(SatisfyAll(
505+
HaveField("Spec.MachinePoolRef.Name", Equal(secondMachinePool.Name)),
506+
))
507+
})
437508
})

‎iri/apis/machine/v1alpha1/api.pb.go‎

Lines changed: 86 additions & 91 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎iri/apis/machine/v1alpha1/api.proto‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ message EventFilter {
4646
}
4747

4848
message MachineClassCapabilities {
49-
int64 cpu_millis = 1;
50-
int64 memory_bytes = 2;
49+
map<string, int64> resources = 1;
5150
}
5251

5352
message Machine {

‎irictl-machine/tableconverters/status.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package tableconverters
55

66
import (
7+
corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1"
78
iri "github.com/ironcore-dev/ironcore/iri/apis/machine/v1alpha1"
89
"github.com/ironcore-dev/ironcore/irictl/api"
910
"github.com/ironcore-dev/ironcore/irictl/tableconverter"
@@ -23,8 +24,8 @@ var (
2324
Rows: tableconverter.SingleRowFrom(func(status *iri.MachineClassStatus) (api.Row, error) {
2425
return api.Row{
2526
status.MachineClass.Name,
26-
resource.NewMilliQuantity(status.MachineClass.Capabilities.CpuMillis, resource.DecimalSI).String(),
27-
resource.NewQuantity(status.MachineClass.Capabilities.MemoryBytes, resource.DecimalSI).String(),
27+
resource.NewMilliQuantity(status.MachineClass.Capabilities.Resources[string(corev1alpha1.ResourceCPU)], resource.DecimalSI).String(),
28+
resource.NewQuantity(status.MachineClass.Capabilities.Resources[string(corev1alpha1.ResourceMemory)], resource.DecimalSI).String(),
2829
resource.NewQuantity(status.Quantity, resource.DecimalSI).String(),
2930
}, nil
3031
}),

‎poollet/machinepoollet/controllers/controllers_suite_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,10 @@ func SetupTest() (*corev1.Namespace, *computev1alpha1.MachinePool, *computev1alp
189189
MachineClass: &iri.MachineClass{
190190
Name: mc.Name,
191191
Capabilities: &iri.MachineClassCapabilities{
192-
CpuMillis: mc.Capabilities.CPU().MilliValue(),
193-
MemoryBytes: mc.Capabilities.Memory().Value(),
192+
Resources: map[string]int64{
193+
string(corev1alpha1.ResourceCPU): mc.Capabilities.CPU().Value(),
194+
string(corev1alpha1.ResourceMemory): mc.Capabilities.Memory().Value(),
195+
},
194196
},
195197
},
196198
},

‎poollet/machinepoollet/controllers/machine_controller.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -717,12 +717,15 @@ func (r *MachineReconciler) prepareIRIMachineClass(ctx context.Context, machine
717717
}
718718

719719
func getIRIMachineClassCapabilities(machineClass *computev1alpha1.MachineClass) *iri.MachineClassCapabilities {
720-
cpu := machineClass.Capabilities.CPU()
721-
memory := machineClass.Capabilities.Memory()
720+
resources := map[string]int64{}
721+
resourceList := machineClass.Capabilities
722+
723+
for resource, quantity := range resourceList {
724+
resources[string(resource)] = quantity.Value()
725+
}
722726

723727
return &iri.MachineClassCapabilities{
724-
CpuMillis: cpu.MilliValue(),
725-
MemoryBytes: memory.Value(),
728+
Resources: resources,
726729
}
727730
}
728731

‎poollet/machinepoollet/controllers/machinepool_controller_test.go‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,10 @@ var _ = Describe("MachinePoolController", func() {
4343
MachineClass: &iri.MachineClass{
4444
Name: machineClass.Name,
4545
Capabilities: &iri.MachineClassCapabilities{
46-
CpuMillis: machineClass.Capabilities.CPU().MilliValue(),
47-
MemoryBytes: machineClass.Capabilities.Memory().Value(),
46+
Resources: map[string]int64{
47+
string(corev1alpha1.ResourceCPU): machineClass.Capabilities.CPU().Value(),
48+
string(corev1alpha1.ResourceMemory): machineClass.Capabilities.Memory().Value(),
49+
},
4850
},
4951
},
5052
Quantity: machineClassCapacity,
@@ -55,8 +57,10 @@ var _ = Describe("MachinePoolController", func() {
5557
MachineClass: &iri.MachineClass{
5658
Name: machineClass2.Name,
5759
Capabilities: &iri.MachineClassCapabilities{
58-
CpuMillis: machineClass2.Capabilities.CPU().MilliValue(),
59-
MemoryBytes: machineClass2.Capabilities.Memory().Value(),
60+
Resources: map[string]int64{
61+
string(corev1alpha1.ResourceCPU): machineClass2.Capabilities.CPU().Value(),
62+
string(corev1alpha1.ResourceMemory): machineClass2.Capabilities.Memory().Value(),
63+
},
6064
},
6165
},
6266
Quantity: machineClass2Capacity,
@@ -140,6 +144,7 @@ var _ = Describe("MachinePoolController", func() {
140144
Capabilities: corev1alpha1.ResourceList{
141145
corev1alpha1.ResourceCPU: resource.MustParse("2"),
142146
corev1alpha1.ResourceMemory: resource.MustParse("2Gi"),
147+
"type-a.vendor.com/gpu": resource.MustParse("1"),
143148
},
144149
}
145150
Expect(k8sClient.Create(ctx, machineClass)).To(Succeed(), "failed to create test machine class")
@@ -150,8 +155,11 @@ var _ = Describe("MachinePoolController", func() {
150155
MachineClass: &iri.MachineClass{
151156
Name: machineClass.Name,
152157
Capabilities: &iri.MachineClassCapabilities{
153-
CpuMillis: machineClass.Capabilities.CPU().MilliValue(),
154-
MemoryBytes: machineClass.Capabilities.Memory().Value(),
158+
Resources: map[string]int64{
159+
string(corev1alpha1.ResourceCPU): machineClass.Capabilities.CPU().Value(),
160+
string(corev1alpha1.ResourceMemory): machineClass.Capabilities.Memory().Value(),
161+
"type-a.vendor.com/gpu": 1,
162+
},
155163
},
156164
},
157165
},
@@ -189,8 +197,11 @@ var _ = Describe("MachinePoolController", func() {
189197
MachineClass: &iri.MachineClass{
190198
Name: machineClass.Name,
191199
Capabilities: &iri.MachineClassCapabilities{
192-
CpuMillis: machineClass.Capabilities.CPU().MilliValue(),
193-
MemoryBytes: machineClass.Capabilities.Memory().Value(),
200+
Resources: map[string]int64{
201+
string(corev1alpha1.ResourceCPU): machineClass.Capabilities.CPU().Value(),
202+
string(corev1alpha1.ResourceMemory): machineClass.Capabilities.Memory().Value(),
203+
"type-a.vendor.com/gpu": 1,
204+
},
194205
},
195206
},
196207
},
@@ -200,8 +211,10 @@ var _ = Describe("MachinePoolController", func() {
200211
MachineClass: &iri.MachineClass{
201212
Name: machineClass2.Name,
202213
Capabilities: &iri.MachineClassCapabilities{
203-
CpuMillis: machineClass2.Capabilities.CPU().MilliValue(),
204-
MemoryBytes: machineClass2.Capabilities.Memory().Value(),
214+
Resources: map[string]int64{
215+
string(corev1alpha1.ResourceCPU): machineClass2.Capabilities.CPU().Value(),
216+
string(corev1alpha1.ResourceMemory): machineClass2.Capabilities.Memory().Value(),
217+
},
205218
},
206219
},
207220
},

‎poollet/machinepoollet/mcm/generic.go‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package mcm
66
import (
77
"context"
88
"fmt"
9+
"reflect"
910
"sync"
1011
"time"
1112

@@ -20,16 +21,21 @@ import (
2021
ctrl "sigs.k8s.io/controller-runtime"
2122
)
2223

23-
type capabilities struct {
24-
cpuMillis int64
25-
memoryBytes int64
24+
type capabilities map[string]int64
25+
26+
type machineClassQuantity struct {
27+
*iri.MachineClass
28+
quantity int64
2629
}
2730

28-
func getCapabilities(iriCaps *iri.MachineClassCapabilities) capabilities {
29-
return capabilities{
30-
cpuMillis: iriCaps.CpuMillis,
31-
memoryBytes: iriCaps.MemoryBytes,
31+
func getMachineClassByCapabilities(machineClassByCapabilities map[machineClassQuantity]capabilities, capabilities capabilities) []machineClassQuantity {
32+
matchingMachineClassStatus := []machineClassQuantity{}
33+
for machineClassStatus, machineClassCapabilities := range machineClassByCapabilities {
34+
if reflect.DeepEqual(machineClassCapabilities, capabilities) {
35+
matchingMachineClassStatus = append(matchingMachineClassStatus, machineClassStatus)
36+
}
3237
}
38+
return matchingMachineClassStatus
3339
}
3440

3541
type Generic struct {
@@ -40,8 +46,8 @@ type Generic struct {
4046

4147
listener sets.Set[*listener]
4248

43-
machineClassByName map[string]*iri.MachineClassStatus
44-
machineClassByCapabilities map[capabilities][]*iri.MachineClassStatus
49+
machineClassByName map[string]*iri.MachineClassStatus
50+
machineClassCapabilities map[machineClassQuantity]capabilities
4551

4652
machineRuntime machine.RuntimeService
4753

@@ -97,19 +103,19 @@ func (g *Generic) relist(ctx context.Context, log logr.Logger) error {
97103
oldMachineClassByName := maps.Clone(g.machineClassByName)
98104

99105
maps.Clear(g.machineClassByName)
100-
maps.Clear(g.machineClassByCapabilities)
106+
maps.Clear(g.machineClassCapabilities)
101107

102108
var notify bool
103109
for _, machineClassStatus := range res.MachineClassStatus {
104110
machineClass := machineClassStatus.GetMachineClass()
105111
notify = notify || shouldNotify(oldMachineClassByName, machineClassStatus)
106112

107-
caps := capabilities{
108-
cpuMillis: machineClass.Capabilities.CpuMillis,
109-
memoryBytes: machineClass.Capabilities.MemoryBytes,
110-
}
111113
g.machineClassByName[machineClass.Name] = machineClassStatus
112-
g.machineClassByCapabilities[caps] = append(g.machineClassByCapabilities[caps], machineClassStatus)
114+
machineClassWithQuantity := machineClassQuantity{
115+
MachineClass: machineClass,
116+
quantity: machineClassStatus.Quantity,
117+
}
118+
g.machineClassCapabilities[machineClassWithQuantity] = machineClass.Capabilities.Resources
113119
}
114120

115121
if notify {
@@ -121,12 +127,12 @@ func (g *Generic) relist(ctx context.Context, log logr.Logger) error {
121127

122128
for _, machineClassStatus := range res.MachineClassStatus {
123129
machineClass := machineClassStatus.GetMachineClass()
124-
caps := capabilities{
125-
cpuMillis: machineClass.Capabilities.CpuMillis,
126-
memoryBytes: machineClass.Capabilities.MemoryBytes,
127-
}
128130
g.machineClassByName[machineClass.Name] = machineClassStatus
129-
g.machineClassByCapabilities[caps] = append(g.machineClassByCapabilities[caps], machineClassStatus)
131+
machineClassWithQuantity := machineClassQuantity{
132+
MachineClass: machineClass,
133+
quantity: machineClassStatus.Quantity,
134+
}
135+
g.machineClassCapabilities[machineClassWithQuantity] = machineClass.Capabilities.Resources
130136
}
131137

132138
if !g.sync {
@@ -151,23 +157,21 @@ func (g *Generic) GetMachineClassFor(ctx context.Context, name string, caps *iri
151157
g.mu.RLock()
152158
defer g.mu.RUnlock()
153159

154-
expected := getCapabilities(caps)
155-
if byName, ok := g.machineClassByName[name]; ok && getCapabilities(byName.MachineClass.Capabilities) == expected {
160+
expected := caps.Resources
161+
if byName, ok := g.machineClassByName[name]; ok && reflect.DeepEqual(byName.MachineClass.Capabilities.Resources, expected) {
156162
return byName.MachineClass, byName.Quantity, nil
157163
}
158164

159-
if byCaps, ok := g.machineClassByCapabilities[expected]; ok {
160-
switch len(byCaps) {
161-
case 0:
162-
return nil, 0, ErrNoMatchingMachineClass
163-
case 1:
164-
return byCaps[0].MachineClass, byCaps[0].Quantity, nil
165-
default:
166-
return nil, 0, ErrAmbiguousMatchingMachineClass
167-
}
165+
byCaps := getMachineClassByCapabilities(g.machineClassCapabilities, expected)
166+
switch len(byCaps) {
167+
case 0:
168+
return nil, 0, ErrNoMatchingMachineClass
169+
case 1:
170+
return byCaps[0].MachineClass, byCaps[0].quantity, nil
171+
default:
172+
return nil, 0, ErrAmbiguousMatchingMachineClass
168173
}
169174

170-
return nil, 0, ErrNoMatchingMachineClass
171175
}
172176

173177
func (g *Generic) WaitForSync(ctx context.Context) error {
@@ -192,12 +196,12 @@ func setGenericOptionsDefaults(o *GenericOptions) {
192196
func NewGeneric(runtime machine.RuntimeService, opts GenericOptions) MachineClassMapper {
193197
setGenericOptionsDefaults(&opts)
194198
return &Generic{
195-
synced: make(chan struct{}),
196-
machineClassByName: map[string]*iri.MachineClassStatus{},
197-
machineClassByCapabilities: map[capabilities][]*iri.MachineClassStatus{},
198-
listener: sets.New[*listener](),
199-
machineRuntime: runtime,
200-
relistPeriod: opts.RelistPeriod,
199+
synced: make(chan struct{}),
200+
machineClassByName: map[string]*iri.MachineClassStatus{},
201+
machineClassCapabilities: map[machineClassQuantity]capabilities{},
202+
listener: sets.New[*listener](),
203+
machineRuntime: runtime,
204+
relistPeriod: opts.RelistPeriod,
201205
}
202206
}
203207

0 commit comments

Comments
 (0)