Implement Nvidia GPU passthrough

.gitignore

@@ -8,6 +8,9 @@
 bin
 go.work*
 
+# macOS
+.DS_Store
+
 # Test binary, build with `go test -c`
 *.test
 

api/common_types.go

@@ -17,3 +17,8 @@ const (
 const (
 	MachineManager = "libvirt-provider"
 )
+
+const (
+	// NvidiaGPUPlugin is the resource name for NVIDIA GPU devices to be claimed
+	NvidiaGPUPlugin = "nvidia.com/gpu"
+)

api/machine.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	apiutils "github.com/ironcore-dev/provider-utils/apiutils/api"
+	"github.com/ironcore-dev/provider-utils/claimutils/pci"
 	"k8s.io/utils/ptr"
 )
 
@@ -27,6 +28,7 @@ type MachineSpec struct {
 
 	Volumes           []*VolumeSpec           `json:"volumes"`
 	NetworkInterfaces []*NetworkInterfaceSpec `json:"networkInterfaces"`
+	Gpu               []pci.Address           `json:"gpu"`
 
 	ShutdownAt time.Time `json:"shutdownAt,omitempty"`
 

cmd/libvirt-provider/app/app.go

@@ -16,6 +16,8 @@ import (
 	"sync"
 	"time"
 
+	claim "github.com/ironcore-dev/provider-utils/claimutils/claim"
+
 	"github.com/go-logr/logr"
 	"github.com/ironcore-dev/ironcore-image/oci/remote"
 	ocistore "github.com/ironcore-dev/ironcore-image/oci/store"
@@ -37,6 +39,9 @@ import (
 	"github.com/ironcore-dev/libvirt-provider/internal/raw"
 	"github.com/ironcore-dev/libvirt-provider/internal/server"
 	"github.com/ironcore-dev/libvirt-provider/internal/strategy"
+	"github.com/ironcore-dev/libvirt-provider/internal/utils"
+	"github.com/ironcore-dev/provider-utils/claimutils/gpu"
+	"github.com/ironcore-dev/provider-utils/claimutils/pci"
 	"github.com/ironcore-dev/provider-utils/eventutils/event"
 	"github.com/ironcore-dev/provider-utils/eventutils/recorder"
 	ocihostutils "github.com/ironcore-dev/provider-utils/ociutils/host"
@@ -312,6 +317,27 @@ func Run(ctx context.Context, opts Options) error {
 
 	eventStore := recorder.NewEventStore(log, opts.MachineEventStore)
 
+	pciReader, err := pci.NewReader(log, pci.VendorNvidia, pci.Class3DController)
+	if err != nil {
+		setupLog.Error(err, "failed to initialize PCI reader")
+		return err
+	}
+
+	claimedPCIAddrs, err := utils.GetClaimedPCIAddressesFromMachineStore(ctx, machineStore)
+	if err != nil {
+		setupLog.Error(err, "failed to get claimed PCI addresses from machine store")
+		return err
+	}
+	setupLog.Info("Recovered claimed PCI addresses from machine store", "addresses", fmt.Sprintf("%v", claimedPCIAddrs))
+
+	resClaimer, err := claim.NewResourceClaimer(
+		log, gpu.NewGPUClaimPlugin(log, api.NvidiaGPUPlugin, pciReader, claimedPCIAddrs),
+	)
+	if err != nil {
+		setupLog.Error(err, "failed to initialize resource claimer")
+		return err
+	}
+
 	machineReconciler, err := controllers.NewMachineReconciler(
 		log.WithName("machine-reconciler"),
 		providerHost,
@@ -324,6 +350,7 @@ func Run(ctx context.Context, opts Options) error {
 			Raw:                            rawInst,
 			VolumePluginManager:            volumePlugins,
 			NetworkInterfacePlugin:         nicPlugin,
+			ResourceClaimer:                resClaimer,
 			ResyncIntervalGarbageCollector: opts.ResyncIntervalGarbageCollector,
 			EnableHugepages:                opts.EnableHugepages,
 			GCVMGracefulShutdownTimeout:    opts.GCVMGracefulShutdownTimeout,
@@ -356,6 +383,7 @@ func Run(ctx context.Context, opts Options) error {
 		MachineClasses:  machineClasses,
 		VolumePlugins:   volumePlugins,
 		NetworkPlugins:  nicPlugin,
+		ResourceClaimer: resClaimer,
 		EnableHugepages: opts.EnableHugepages,
 		GuestAgent:      opts.GuestAgent.GetAPIGuestAgent(),
 	})
@@ -369,6 +397,18 @@ func Run(ctx context.Context, opts Options) error {
 		Log:     log.WithName("health-check"),
 	}
 
+	setupLog.Info("Starting resource claimer")
+	go func() {
+		if err := resClaimer.Start(ctx); err != nil && ctx.Err() == nil {
+			setupLog.Error(err, "failed to start resource claimer")
+		}
+	}()
+
+	if err = resClaimer.WaitUntilStarted(ctx); err != nil {
+		setupLog.Error(err, "failed to wait until resource claimer started")
+		return err
+	}
+
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {

config/development/machineclasses.json

@@ -1,23 +1,30 @@
 [
   {
-    "name": "extra-huge",
+    "name": "gpu",
     "capabilities": {
-      "cpu_millis": 2000,
-      "memory_bytes": 16106127361
+      "resources": {
+        "cpu": 4,
+        "memory": 8589934592,
+        "nvidia.com/gpu": 1
+      }
     }
   },
   {
     "name": "t3-small",
     "capabilities": {
-      "cpu_millis": 2000,
-      "memory_bytes": 2147483648
+      "resources": {
+        "cpu": 2,
+        "memory": 2147483648
+      }
     }
   },
   {
     "name": "x3-xlarge",
     "capabilities": {
-      "cpu_millis": 4000,
-      "memory_bytes": 8589934592
+      "resources": {
+        "cpu": 4,
+        "memory": 8589934592
+      }
     }
   }
 ]

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/ironcore-dev/ironcore v0.2.5-0.20251024123813-eecafba97af9
 	github.com/ironcore-dev/ironcore-image v0.3.0
 	github.com/ironcore-dev/ironcore-net v0.2.4
-	github.com/ironcore-dev/provider-utils v0.0.0-20251010114402-394db8015df4
+	github.com/ironcore-dev/provider-utils v0.0.0-20260127115859-8d108ef8e4b4
 	github.com/moby/term v0.5.2
 	github.com/onsi/ginkgo/v2 v2.28.1
 	github.com/onsi/gomega v1.39.1
@@ -92,7 +92,7 @@ require (
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/tklauser/go-sysconf v0.3.16 // indirect
 	github.com/tklauser/numcpus v0.11.0 // indirect

go.sum

@@ -169,8 +169,10 @@ github.com/ironcore-dev/ironcore-image v0.3.0 h1:K4nTfvYk/3iPMJCXDeKlqFHoMcPpo0J
 github.com/ironcore-dev/ironcore-image v0.3.0/go.mod h1:DDCwFPMQ+KqKcM4NSctdIx7ArAF+bOF/EbwNd5b3hlg=
 github.com/ironcore-dev/ironcore-net v0.2.4 h1:r2SqxwStws86KBvxJcK/myRxb946qAZVpPKd1rJzfsQ=
 github.com/ironcore-dev/ironcore-net v0.2.4/go.mod h1:dSn6aWF+bxCIeWJzPjDl5dVTH3Rd7wx1LuTQpYUU4Lg=
-github.com/ironcore-dev/provider-utils v0.0.0-20251010114402-394db8015df4 h1:sRQ/gcL5z3CqcqI+9If8XmgVk9pK4WebZOAMNFYXREQ=
-github.com/ironcore-dev/provider-utils v0.0.0-20251010114402-394db8015df4/go.mod h1:qxuFuJy31Lfu3go5wsGpFGAZwCw1++1EKGJmHG2HbLw=
+github.com/ironcore-dev/provider-utils v0.0.0-20260126095734-e10d4f178c5f h1:C2Qh9xHu0HK/Vgf3ni3FBbhsh9AL0lTGvH9bzITQRG8=
+github.com/ironcore-dev/provider-utils v0.0.0-20260126095734-e10d4f178c5f/go.mod h1:Se6VYgCzzhVjqUGXeHY3NxuNhLrUkq8qNKpgfayGpHk=
+github.com/ironcore-dev/provider-utils v0.0.0-20260127115859-8d108ef8e4b4 h1:d8bABd4w70/iX1Fff4qaGyGWCNk1XJRyJUQOEFKCJa4=
+github.com/ironcore-dev/provider-utils v0.0.0-20260127115859-8d108ef8e4b4/go.mod h1:Se6VYgCzzhVjqUGXeHY3NxuNhLrUkq8qNKpgfayGpHk=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
@@ -262,8 +264,8 @@ github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
+github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho=
 github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U=
 github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc=

internal/console/server_test.go

@@ -4,12 +4,15 @@
 package console
 
 import (
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"github.com/ironcore-dev/ironcore/api/core/v1alpha1"
 	"github.com/ironcore-dev/libvirt-provider/api"
 	libvirtserver "github.com/ironcore-dev/libvirt-provider/internal/server"
+	claim "github.com/ironcore-dev/provider-utils/claimutils/claim"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -31,8 +34,9 @@ var _ = Describe("HTTP Handler", func() {
 	BeforeEach(func() {
 		var err error
 		server, err = libvirtserver.New(libvirtserver.Options{
-			BaseURL:    baseURL,
-			GuestAgent: api.GuestAgentNone,
+			BaseURL:         baseURL,
+			GuestAgent:      api.GuestAgentNone,
+			ResourceClaimer: &NOOPResourceClaimer{},
 		})
 		Expect(err).ShouldNot(HaveOccurred())
 		router = NewHandler(server, HandlerOptions{})
@@ -56,3 +60,25 @@ var _ = Describe("HTTP Handler", func() {
 		})
 	})
 })
+
+type NOOPResourceClaimer struct{}
+
+// Claim implements [claim.Claimer].
+func (n *NOOPResourceClaimer) Claim(ctx context.Context, resources v1alpha1.ResourceList) (claim.Claims, error) {
+	return nil, nil
+}
+
+// Release implements [claim.Claimer].
+func (n *NOOPResourceClaimer) Release(ctx context.Context, claims claim.Claims) error {
+	return nil
+}
+
+// Start implements [claim.Claimer].
+func (n *NOOPResourceClaimer) Start(ctx context.Context) error {
+	return nil
+}
+
+// WaitUntilStarted implements [claim.Claimer].
+func (n *NOOPResourceClaimer) WaitUntilStarted(ctx context.Context) error {
+	return nil
+}

internal/controllers/controllers_suite_test.go

@@ -12,6 +12,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ironcore-dev/provider-utils/claimutils/claim"
+	"github.com/ironcore-dev/provider-utils/claimutils/gpu"
+
 	"github.com/digitalocean/go-libvirt"
 	"github.com/google/uuid"
 	"github.com/ironcore-dev/ironcore-image/oci/remote"
@@ -29,6 +32,7 @@ import (
 	"github.com/ironcore-dev/libvirt-provider/internal/raw"
 	"github.com/ironcore-dev/libvirt-provider/internal/strategy"
 	apiutils "github.com/ironcore-dev/provider-utils/apiutils/api"
+	"github.com/ironcore-dev/provider-utils/claimutils/pci"
 	"github.com/ironcore-dev/provider-utils/eventutils/event"
 	"github.com/ironcore-dev/provider-utils/eventutils/recorder"
 	ocihostutils "github.com/ironcore-dev/provider-utils/ociutils/host"
@@ -68,6 +72,7 @@ var (
 	tempDir           string
 	controllerCtx     context.Context
 	controllerCancel  context.CancelFunc
+	resClaimer        claim.Claimer
 )
 
 func TestControllers(t *testing.T) {
@@ -180,6 +185,12 @@ var _ = BeforeSuite(func() {
 	}
 	Expect(networkPlugin.Init(providerHost)).To(Succeed())
 
+	By("setting up resource claimer")
+	resClaimer, err = claim.NewResourceClaimer(
+		log, gpu.NewGPUClaimPlugin(log, api.NvidiaGPUPlugin, NewTestingPCIReader([]pci.Address{}), []pci.Address{}),
+	)
+	Expect(err).ToNot(HaveOccurred())
+
 	By("creating machine controller")
 	machineController, err = controllers.NewMachineReconciler(
 		log.WithName("machine-controller"),
@@ -193,6 +204,7 @@ var _ = BeforeSuite(func() {
 			Raw:                            rawInst,
 			VolumePluginManager:            volumePlugins,
 			NetworkInterfacePlugin:         networkPlugin,
+			ResourceClaimer:                resClaimer,
 			ResyncIntervalGarbageCollector: resyncGarbageCollectorInterval,
 			EnableHugepages:                false,
 			GCVMGracefulShutdownTimeout:    gracefulShutdownTimeout,
@@ -289,3 +301,17 @@ func cleanupMachine(machineID string) func(SpecContext) {
 		}).WithPolling(time.Second).Should(BeTrue())
 	}
 }
+
+type TestingPCIReader struct {
+	pciAddrs []pci.Address
+}
+
+func (t TestingPCIReader) Read() ([]pci.Address, error) {
+	return t.pciAddrs, nil
+}
+
+func NewTestingPCIReader(addrs []pci.Address) *TestingPCIReader {
+	return &TestingPCIReader{
+		pciAddrs: addrs,
+	}
+}

internal/controllers/machine_controller.go

@@ -27,6 +27,7 @@ import (
 	providervolume "github.com/ironcore-dev/libvirt-provider/internal/plugins/volume"
 	"github.com/ironcore-dev/libvirt-provider/internal/raw"
 	"github.com/ironcore-dev/libvirt-provider/internal/utils"
+	"github.com/ironcore-dev/provider-utils/claimutils/claim"
 	"github.com/ironcore-dev/provider-utils/eventutils/event"
 	"github.com/ironcore-dev/provider-utils/eventutils/recorder"
 	ociutils "github.com/ironcore-dev/provider-utils/ociutils/oci"
@@ -69,6 +70,7 @@ type MachineReconcilerOptions struct {
 	Raw                            raw.Raw
 	VolumePluginManager            *providervolume.PluginManager
 	NetworkInterfacePlugin         providernetworkinterface.Plugin
+	ResourceClaimer                claim.Claimer
 	VolumeEvents                   event.Source[*api.Machine]
 	ResyncIntervalGarbageCollector time.Duration
 	EnableHugepages                bool
@@ -96,6 +98,10 @@ func NewMachineReconciler(
 		return nil, fmt.Errorf("must specify machine events")
 	}
 
+	if opts.ResourceClaimer == nil {
+		return nil, fmt.Errorf("must specify resource claimer")
+	}
+
 	return &MachineReconciler{
 		log:                            log,
 		queue:                          workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()),
@@ -109,6 +115,7 @@ func NewMachineReconciler(
 		raw:                            opts.Raw,
 		volumePluginManager:            opts.VolumePluginManager,
 		networkInterfacePlugin:         opts.NetworkInterfacePlugin,
+		resourceClaimer:                opts.ResourceClaimer,
 		resyncIntervalGarbageCollector: opts.ResyncIntervalGarbageCollector,
 		enableHugepages:                opts.EnableHugepages,
 		gcVMGracefulShutdownTimeout:    opts.GCVMGracefulShutdownTimeout,
@@ -130,6 +137,7 @@ type MachineReconciler struct {
 
 	volumePluginManager    *providervolume.PluginManager
 	networkInterfacePlugin providernetworkinterface.Plugin
+	resourceClaimer        claim.Claimer
 
 	machines      store.Store[*api.Machine]
 	machineEvents event.Source[*api.Machine]
@@ -195,7 +203,7 @@ func (r *MachineReconciler) Start(ctx context.Context) error {
 		r.queue.ShutDown()
 	}()
 
-	for i := 0; i < workerSize; i++ {
+	for range workerSize {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -292,6 +300,11 @@ func (r *MachineReconciler) processMachineDeletion(ctx context.Context, log logr
 	}
 	log.V(1).Info("Removed network interfaces")
 
+	if err := r.releaseResourceClaims(ctx, log, machine.Spec.Gpu); err != nil {
+		return fmt.Errorf("failed to release resource claims: %w", err)
+	}
+	log.V(1).Info("Released resource claims")
+
 	if err := os.RemoveAll(r.host.MachineDir(machine.ID)); err != nil {
 		return fmt.Errorf("failed to remove machine directory: %w", err)
 	}
@@ -681,6 +694,7 @@ func (r *MachineReconciler) domainFor(
 					},
 				},
 			},
+			Hostdevs: ClaimedGPUsToHostDevs(machine),
 		},
 	}