Support Vetu virtualization on Linux in addition to Tart on macOS

Author: edigaryev

.cirrus.yml

@@ -5,10 +5,10 @@ task:
       name: dev-mini
     resources:
       tart-vms: 2
-  pull_script:
-    - tart pull ghcr.io/cirruslabs/macos-sonoma-base:latest
+  pre_pull_script:
+    - tart pull ghcr.io/cirruslabs/macos-tahoe-base:latest
   test_script:
-    - go test -ldflags="-B gobuildid" -v -count=1 ./...
+    - go test -v -count=1 ./...
   always:
     cleanup_script:
       - tart list

.github/workflows/main.yml

@@ -0,0 +1,22 @@
+on:
+  push:
+
+jobs:
+  test:
+    name: Test (Linux)
+    runs-on: ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: stable
+      - name: Install Vetu
+        run: |
+          sudo apt-get update && sudo apt-get -y install apt-transport-https ca-certificates
+          echo "deb [trusted=yes] https://apt.fury.io/cirruslabs/ /" | sudo tee /etc/apt/sources.list.d/cirruslabs.list
+          sudo apt-get update && sudo apt-get -y install vetu
+      - name: Pre-pull default Vetu image for use in tests
+        run: |
+          vetu pull ghcr.io/cirruslabs/ubuntu-runner-amd64:latest
+      - name: Run tests
+        run: go test -v -count=1 ./...

api/openapi.yaml

@@ -558,6 +558,32 @@ components:
       title: Virtual Machine Specification
       type: object
       properties:
+        os:
+          type: string
+          description: |
+            Operating system used by a VM.
+
+            Set to `linux` to work around the Apple's limitation of 2 macOS VMs per host.
+
+            This field cannot be changed after the VM is created.
+          default: darwin
+          enum: [ darwin, linux ]
+        arch:
+          type: string
+          description: |
+            Hardware architecture to use for a VM.
+
+            This field cannot be changed after the VM is created.
+          default: arm64
+          enum: [ arm64, amd64 ]
+        runtime:
+          type: string
+          description: |
+            Runtime to use for a VM.
+
+            This field cannot be changed after the VM is created.
+          default: tart
+          enum: [ tart, vetu ]
         image:
           type: string
           description: VM image for this VM
@@ -709,20 +735,20 @@ components:
             
             When set to `stopped` or `suspended`, the VM does not consume any `resources`
             and can serve as a source for creating new Orchard VMs on the same worker. See
-            `tartName` for more details.
+            `localName` for more details.
             
             Note that you can only transition into `stopped` or `suspended` only once
             at the moment.
           default: running
           enum: [ running, stopped, suspended ]
-        tartName:
+        localName:
           type: string
           description: |
-            Name of the Tart VM backing this VM resource.
+            Name of the local VM backing this VM resource.
             
-            `tartName` is specific to a worker, whereas `name` is cluster-wide.
+            `localName` is specific to a worker, whereas `name` is cluster-wide.
             
-            `tartName` is useful in combination with `powerState` for creating stopped or suspended VMs
+            `localName` is useful in combination with `powerState` for creating stopped or suspended VMs
             that can be used to start or resume new VMs on the same worker.
             
             However, with great power comes great responsibility. You need to make sure:
@@ -735,6 +761,12 @@ components:
             new MAC addresses automatically, which will stop them from booting,
             since the suspend‑resume machinery expects the same MAC address
           readOnly: true
+        tartName:
+          type: string
+          description: |
+            Deprecated alias for `localName`.
+          readOnly: true
+          deprecated: true
     VMState:
       title: Virtual Machine State
       type: object

go.mod

@@ -6,7 +6,6 @@ go 1.25.1
 replace github.com/gin-gonic/gin v1.11.0 => github.com/gin-gonic/gin v1.10.0
 
 require (
-	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/avast/retry-go/v4 v4.7.0
 	github.com/cirruslabs/chacha v0.16.3
 	github.com/coder/websocket v1.8.14

go.sum

@@ -21,8 +21,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
-github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
-github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio=
 github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q=
 github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=

internal/controller/api_vms.go

@@ -49,7 +49,9 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 	vm.RestartCount = 0
 	vm.UID = uuid.New().String()
 	vm.PowerState = v1.PowerStateRunning
-	vm.TartName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+	vm.LocalName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+	//nolint:staticcheck // yes, this is deprecated, but we still maintain it for backward compatibility
+	vm.TartName = vm.LocalName
 	vm.Generation = 0
 	vm.ObservedGeneration = 0
 	vm.Conditions = []v1.Condition{
@@ -66,12 +68,17 @@ func (controller *Controller) createVM(ctx *gin.Context) responder.Responder {
 		vm.NetSoftnet = true
 	}
 
-	// Provide resource defaults
-	if vm.Resources == nil {
-		vm.Resources = make(v1.Resources)
-	}
-	if _, ok := vm.Resources[v1.ResourceTartVMs]; !ok {
-		vm.Resources[v1.ResourceTartVMs] = 1
+	// Apple limits the number of macOS VMs to 2,
+	// so we need to provide a resource default
+	// (if not otherwise overridden by the user)
+	// to avoid a case when more than 2 VMs run
+	if vm.OS == v1.OSDarwin && vm.Runtime == v1.RuntimeTart {
+		if vm.Resources == nil {
+			vm.Resources = make(v1.Resources)
+		}
+		if _, ok := vm.Resources[v1.ResourceTartVMs]; !ok {
+			vm.Resources[v1.ResourceTartVMs] = 1
+		}
 	}
 
 	// Validate image pull policy and provide a default value if it's missing
@@ -148,6 +155,12 @@ func (controller *Controller) updateVMSpec(ctx *gin.Context) responder.Responder
 				NewErrorResponse("cannot update VM in a terminal state"))
 		}
 
+		// Platform sanity checks
+		if dbVM.OS != userVM.OS || dbVM.Arch != userVM.Arch || dbVM.Runtime != userVM.Runtime {
+			return responder.JSON(http.StatusPreconditionFailed, NewErrorResponse("\"os\", \"arch\" "+
+				"and \"runtime\" fields cannot be modified"))
+		}
+
 		// Softnet-specific logic: automatically enable Softnet when NetSoftnetAllow or NetSoftnetBlock are set
 		// and propagate deprecated and non-deprecated boolean fields into each other
 		if userVM.NetSoftnetDeprecated || userVM.NetSoftnet || len(userVM.NetSoftnetAllow) != 0 || len(userVM.NetSoftnetBlock) != 0 {

internal/controller/api_workers.go

@@ -97,6 +97,8 @@ func (controller *Controller) createWorker(ctx *gin.Context) responder.Responder
 				"from a different machine ID, delete this worker first to be able to re-create it"))
 		}
 
+		dbWorker.Arch = worker.Arch
+		dbWorker.Runtime = worker.Runtime
 		dbWorker.LastSeen = worker.LastSeen
 		dbWorker.Resources = worker.Resources
 		dbWorker.Labels = worker.Labels

internal/controller/scheduler/scheduler.go

@@ -273,6 +273,7 @@ NextVM:
 
 			if worker.Offline(scheduler.workerOfflineTimeout) ||
 				worker.SchedulingPaused ||
+				!compatibleArchAndRuntime(unscheduledVM, worker) ||
 				!resourcesRemaining.CanFit(unscheduledVM.Resources) ||
 				!worker.Labels.Contains(unscheduledVM.Labels) {
 				continue NextWorker
@@ -328,6 +329,10 @@ NextVM:
 					return ErrWorkerSchedulingSkipped
 				}
 
+				if !compatibleArchAndRuntime(unscheduledVM, *currentWorker) {
+					return ErrWorkerSchedulingSkipped
+				}
+
 				if currentWorker.MachineID != worker.MachineID ||
 					!currentWorker.Resources.Equal(worker.Resources) {
 					// Worker has changed
@@ -433,6 +438,19 @@ func ProcessVMs(vms []v1.VM) ([]v1.VM, WorkerInfos) {
 	return unscheduledVMs, workerToResources
 }
 
+func compatibleArchAndRuntime(vm v1.VM, worker v1.Worker) bool {
+	// Backward compatibility
+	if vm.Arch == "" {
+		vm.Arch = v1.ArchitectureARM64
+	}
+
+	if vm.Runtime == "" {
+		vm.Runtime = v1.RuntimeTart
+	}
+
+	return vm.Arch == worker.Arch && vm.Runtime == worker.Runtime
+}
+
 func (scheduler *Scheduler) healthCheckingLoopIteration() (int, error) {
 	// Stats for the caller
 	var numVMs int
@@ -515,7 +533,9 @@ func (scheduler *Scheduler) healthCheckVM(txn storepkg.Transaction, vm v1.VM) er
 		vm.ScheduledAt = time.Time{}
 		vm.StartedAt = time.Time{}
 		vm.PowerState = v1.PowerStateRunning
-		vm.TartName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+		vm.LocalName = ondiskname.New(vm.Name, vm.UID, vm.RestartCount).String()
+		//nolint:staticcheck // yes, this is deprecated, but we still maintain it for backward compatibility
+		vm.TartName = vm.LocalName
 		vm.Conditions = []v1.Condition{
 			{
 				Type:  v1.ConditionTypeScheduled,

internal/imageconstant/imageconstant.go

@@ -1,3 +1,6 @@
 package imageconstant
 
-const DefaultMacosImage = "ghcr.io/cirruslabs/macos-tahoe-base:latest"
+const (
+	DefaultMacosImage = "ghcr.io/cirruslabs/macos-tahoe-base:latest"
+	DefaultLinuxImage = "ghcr.io/cirruslabs/ubuntu-runner-amd64:latest"
+)

internal/tests/integration_test.go

@@ -14,8 +14,10 @@ import (
 	"github.com/cirruslabs/orchard/internal/controller"
 	"github.com/cirruslabs/orchard/internal/imageconstant"
 	"github.com/cirruslabs/orchard/internal/tests/devcontroller"
+	"github.com/cirruslabs/orchard/internal/tests/platformdependent"
 	"github.com/cirruslabs/orchard/internal/tests/wait"
 	"github.com/cirruslabs/orchard/internal/worker/ondiskname"
+	"github.com/cirruslabs/orchard/internal/worker/vmmanager"
 	"github.com/cirruslabs/orchard/internal/worker/vmmanager/tart"
 	v1 "github.com/cirruslabs/orchard/pkg/resource/v1"
 	"github.com/google/uuid"
@@ -34,20 +36,14 @@ func TestSingleVM(t *testing.T) {
 		t.Fatal(err)
 	}
 	assert.Equal(t, 1, len(workers))
-	err = devClient.VMs().Create(context.Background(), &v1.VM{
-		Meta: v1.Meta{
-			Name: "test-vm",
-		},
-		Image:    imageconstant.DefaultMacosImage,
-		CPU:      4,
-		Memory:   8 * 1024,
-		Headless: true,
-		Status:   v1.VMStatusPending,
-		StartupScript: &v1.VMScript{
-			ScriptContent: "echo \"Hello, $FOO!\"\nfor i in $(seq 1 1000); do echo \"$i\"; done",
-			Env:           map[string]string{"FOO": "Bar"},
-		},
-	})
+
+	vm := platformdependent.VM("test-vm")
+	vm.StartupScript = &v1.VMScript{
+		ScriptContent: "echo \"Hello, $FOO!\"\nfor i in $(seq 1 1000); do echo \"$i\"; done",
+		Env:           map[string]string{"FOO": "Bar"},
+	}
+
+	err = devClient.VMs().Create(context.Background(), vm)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,7 +78,7 @@ func TestSingleVM(t *testing.T) {
 	assert.Contains(t, strings.Join(logLines, "\n"), strings.Join(expectedLogs, "\n"))
 
 	// Ensure that the VM exists on disk before deleting it
-	require.True(t, hasVMByPredicate(t, func(info tart.VMInfo) bool {
+	require.True(t, hasVMByPredicate(t, func(info vmmanager.VMInfo) bool {
 		return strings.Contains(info.Name, runningVM.UID)
 	}, nil))
 
@@ -93,7 +89,7 @@ func TestSingleVM(t *testing.T) {
 	assert.True(t, wait.Wait(2*time.Minute, func() bool {
 		t.Logf("Waiting for the VM to be garbage collected...")
 
-		return !hasVMByPredicate(t, func(info tart.VMInfo) bool {
+		return !hasVMByPredicate(t, func(info vmmanager.VMInfo) bool {
 			return strings.Contains(info.Name, runningVM.UID)
 		}, nil)
 	}), "VM was not garbage collected in a timely manner")
@@ -107,19 +103,13 @@ func TestFailedStartupScript(t *testing.T) {
 		t.Fatal(err)
 	}
 	assert.Equal(t, 1, len(workers))
-	err = devClient.VMs().Create(context.Background(), &v1.VM{
-		Meta: v1.Meta{
-			Name: "test-vm",
-		},
-		Image:    imageconstant.DefaultMacosImage,
-		CPU:      4,
-		Memory:   8 * 1024,
-		Headless: true,
-		Status:   v1.VMStatusPending,
-		StartupScript: &v1.VMScript{
-			ScriptContent: "exit 123",
-		},
-	})
+
+	vm := platformdependent.VM("test-vm")
+	vm.StartupScript = &v1.VMScript{
+		ScriptContent: "exit 123",
+	}
+
+	err = devClient.VMs().Create(context.Background(), vm)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -145,15 +135,7 @@ func TestPortForwarding(t *testing.T) {
 	devClient, _, _ := devcontroller.StartIntegrationTestEnvironment(t)
 
 	// Create a generic macOS VM
-	err := devClient.VMs().Create(ctx, &v1.VM{
-		Meta: v1.Meta{
-			Name: "test-vm",
-		},
-		Image:    imageconstant.DefaultMacosImage,
-		CPU:      4,
-		Memory:   8 * 1024,
-		Headless: true,
-	})
+	err := devClient.VMs().Create(ctx, platformdependent.VM("test-vm"))
 	require.NoError(t, err)
 
 	// Establish port forwarding to VMs SSH port
@@ -468,12 +450,12 @@ func TestHostDirsInvalidPolicy(t *testing.T) {
 }
 
 func hasVM(t *testing.T, name string, logger *zap.Logger) bool {
-	return hasVMByPredicate(t, func(vmInfo tart.VMInfo) bool {
+	return hasVMByPredicate(t, func(vmInfo vmmanager.VMInfo) bool {
 		return vmInfo.Name == name
 	}, logger)
 }
 
-func hasVMByPredicate(t *testing.T, predicate func(tart.VMInfo) bool, logger *zap.Logger) bool {
+func hasVMByPredicate(t *testing.T, predicate func(vmmanager.VMInfo) bool, logger *zap.Logger) bool {
 	if logger == nil {
 		logger = zap.Must(zap.NewDevelopment())
 	}