Mirror console logs to file

Currently, the only way to observe boot logs is by attaching to the
serial console manually. This often misses very early boot logs which
cannot be recovered.

This commit adds a log config to the serial console to ensure its output
is mirrored into a file so that it can be inspected even when the early
boot phase was missed.

Resolves: #737
Signed-off-by: Maximilian Moehl <maximilian@moehl.eu>

Author: maxmoehl

docs/concepts/console.md

@@ -1 +1,22 @@
-# Console
+# Console
+
+Every machine is equipped with a serial console which allows interactive access to the guest, e.g. via the
+`Exec` endpoint of the IRI `MachineRuntime` service.
+
+## Console log
+
+The console output of a machine is additionally mirrored to a log file on the hypervisor. This is an always-on
+feature and uses the libvirt [`<log>`](https://libvirt.org/formatdomain.html#element-log) sub-element of the
+serial chardev, so the interactive console stays usable while all traffic is persisted.
+
+The log file is stored per machine at:
+
+```text
+<libvirt-provider-dir>/machines/<machine-uid>/logs/console.log
+```
+
+It is written with `append="on"`, i.e. the content is preserved across domain restarts. This allows operators
+to inspect early boot logs (firmware, bootloader, kernel) of a machine, which is especially useful when a
+machine fails to boot.
+
+The console log is removed together with the machine directory when the machine is deleted.

internal/controllers/machine_controller.go

@@ -689,6 +689,13 @@ func (r *MachineReconciler) domainFor(
 					Target: &libvirtxml.DomainSerialTarget{
 						Type: serialTargetType,
 					},
+					// Always mirror the serial console output to a host-side log file,
+					// so early boot logs can be inspected for troubleshooting. The
+					// interactive pty console stays usable alongside the log.
+					Log: &libvirtxml.DomainChardevLog{
+						File:   r.host.MachineConsoleLogFile(machine.ID),
+						Append: "on",
+					},
 				},
 			},
 			Consoles: []libvirtxml.DomainConsole{

internal/controllers/machine_controller_test.go

@@ -11,6 +11,7 @@ import (
 	libvirtutils "github.com/ironcore-dev/libvirt-provider/internal/libvirt/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"libvirt.org/go/libvirtxml"
 )
 
 var _ = Describe("MachineController", func() {
@@ -56,6 +57,17 @@ var _ = Describe("MachineController", func() {
 				g.Expect(m.Status.State).To(Equal(api.MachineStateRunning))
 			}).Should(Succeed())
 
+			By("ensuring the console output is mirrored to a log file")
+			domainXML := &libvirtxml.Domain{}
+			Expect(domainXML.Unmarshal(domainXMLData)).To(Succeed())
+			Expect(domainXML.Devices).NotTo(BeNil())
+			Expect(domainXML.Devices.Serials).To(HaveLen(1))
+			consoleLog := domainXML.Devices.Serials[0].Log
+			Expect(consoleLog).NotTo(BeNil())
+			Expect(consoleLog.File).To(Equal(providerHost.MachineConsoleLogFile(machine.ID)))
+			Expect(consoleLog.Append).To(Equal("on"))
+			Expect(providerHost.MachineLogsDir(machine.ID)).To(BeADirectory())
+			Eventually(providerHost.MachineConsoleLogFile(machine.ID)).Should(BeAnExistingFile())
 		})
 
 		It("should handle machine with boot image and network interface", func(ctx SpecContext) {

internal/host/host.go

@@ -25,6 +25,8 @@ const (
 	DefaultMachineRootFSFile           = "rootfs"
 	DefaultMachinePluginsDir           = "plugins"
 	DefaultMachineNetworkInterfacesDir = "networkinterfaces"
+	DefaultMachineLogsDir              = "logs"
+	DefaultMachineConsoleLogFile       = "console.log"
 )
 
 type Paths interface {
@@ -51,6 +53,9 @@ type Paths interface {
 	MachineNetworkInterfacesDir(machineUID string) string
 	MachineNetworkInterfaceDir(machineUID string, networkInterfaceName string) string
 
+	MachineLogsDir(machineUID string) string
+	MachineConsoleLogFile(machineUID string) string
+
 	MachineIgnitionsDir(machineUID string) string
 	MachineIgnitionFile(machineUID string) string
 }
@@ -127,6 +132,14 @@ func (p *paths) MachineNetworkInterfaceDir(machineUID string, networkInterfaceNa
 	return filepath.Join(p.MachineNetworkInterfacesDir(machineUID), networkInterfaceName)
 }
 
+func (p *paths) MachineLogsDir(machineUID string) string {
+	return filepath.Join(p.MachineDir(machineUID), DefaultMachineLogsDir)
+}
+
+func (p *paths) MachineConsoleLogFile(machineUID string) string {
+	return filepath.Join(p.MachineLogsDir(machineUID), DefaultMachineConsoleLogFile)
+}
+
 func (p *paths) MachineIgnitionsDir(machineUID string) string {
 	return filepath.Join(p.MachineDir(machineUID), DefaultMachineIgnitionsDir)
 }
@@ -225,5 +238,8 @@ func MakeMachineDirs(paths Paths, machineUID string) error {
 	if err := os.MkdirAll(paths.MachineNetworkInterfacesDir(machineUID), os.ModePerm); err != nil {
 		return fmt.Errorf("error creating machine network interfaces directory: %w", err)
 	}
+	if err := os.MkdirAll(paths.MachineLogsDir(machineUID), os.ModePerm); err != nil {
+		return fmt.Errorf("error creating machine logs directory: %w", err)
+	}
 	return nil
 }