fix: write the uploaded etcd snapshot atomically

The etcd snapshot upload truncated the snapshot file in place, and removed it when the upload failed. The etcd recovery reads that same file, and it runs after the bootstrap API has already returned success. This way, an upload repeated during a running recovery cut the snapshot short and left a partial etcd data directory behind, and every bootstrap after that got rejected.

Write the upload to a temporary file next to the snapshot and move it into place only once it is complete. A running recovery keeps reading the snapshot it started with, and a failed upload removes only its own temporary file.

Signed-off-by: Utku Ozdemir <utku.ozdemir@siderolabs.com>

Author: utkuozdemir

internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go

@@ -2026,7 +2026,10 @@ func (s *Server) EtcdRecover(srv machine.MachineService_EtcdRecoverServer) error
 		return err
 	}
 
-	snapshot, err := os.OpenFile(constants.EtcdRecoverySnapshotPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o700)
+	// The snapshot is written to a temporary file and moved into place only once it is complete,
+	// so that the recovery running in the etcd service keeps reading the snapshot it started with:
+	// a repeated upload never truncates or removes the file under an in-progress recovery.
+	snapshot, err := os.CreateTemp(filepath.Dir(constants.EtcdRecoverySnapshotPath), filepath.Base(constants.EtcdRecoverySnapshotPath)+".*")
 	if err != nil {
 		return fmt.Errorf("error creating etcd recovery snapshot: %w", err)
 	}
@@ -2067,6 +2070,10 @@ func (s *Server) EtcdRecover(srv machine.MachineService_EtcdRecoverServer) error
 		return fmt.Errorf("error closing snapshot: %w", err)
 	}
 
+	if err = os.Rename(snapshot.Name(), constants.EtcdRecoverySnapshotPath); err != nil {
+		return fmt.Errorf("error moving snapshot into place: %w", err)
+	}
+
 	successfulUpload = true
 
 	return srv.SendAndClose(&machine.EtcdRecoverResponse{