🐛 fix(fireglab): resolve gitlab-runner execution issues in VMs

- Fix RunOnce() to use run-single with CLI args (--url, --token,
  --executor, --builds-dir, --cache-dir) instead of --config flag
  since run-single doesn't read from config.toml
- Remove existing config.toml before registration to prevent
  duplicate [[runners]] entries when service restarts
- Store metadata during Register() for use in RunOnce()
- Change config path from /etc/gitlab-runner to /home/runner/.gitlab-runner
  to avoid permission issues when running as runner user
- Add --work-dir and --config runtime arguments to systemd service

Author: thpham

fireglab/runner/runner.go

@@ -45,6 +45,8 @@ type Runner struct {
 	stdout           io.Writer
 	stderr           io.Writer
 	log              *logrus.Logger
+	// metadata stores registration info for run-single mode
+	metadata *mmds.Metadata
 }
 
 // Option is a functional option for configuring the Runner.
@@ -156,6 +158,13 @@ func (r *Runner) Register(ctx context.Context, metadata *mmds.Metadata) error {
 		return fmt.Errorf("failed to create config directory: %w", err)
 	}
 
+	// Remove any existing config to prevent duplicate [[runners]] entries on restart
+	// gitlab-runner register appends to existing config, which causes issues if
+	// the service restarts after a failed run attempt
+	if err := os.Remove(r.configPath); err != nil && !os.IsNotExist(err) {
+		r.log.Warnf("Failed to remove existing config file: %v", err)
+	}
+
 	// Build registration command
 	// With the new runner authentication tokens (glrt-*), we use --token directly
 	// The runner was already created via the API, so we just need to register locally
@@ -194,18 +203,21 @@ func (r *Runner) Register(ctx context.Context, metadata *mmds.Metadata) error {
 		return fmt.Errorf("registration failed: %w", err)
 	}
 
+	// Store metadata for run-single mode
+	r.metadata = metadata
+
 	r.log.Info("Runner registered successfully")
 	return nil
 }
 
 // Run starts the gitlab-runner daemon and blocks until it exits.
 // This should be called after Register.
-// The daemon runs with --max-builds 1 to exit after completing one job (ephemeral mode).
+// The runner will continuously poll for jobs until stopped or the context is cancelled.
 func (r *Runner) Run(ctx context.Context) error {
-	r.log.Info("Starting gitlab-runner daemon (ephemeral mode)")
+	r.log.Info("Starting gitlab-runner daemon (continuous mode)")
 
-	// Use run-single or run with max-builds=1 for ephemeral behavior
-	// run-single executes exactly one build and then exits
+	// Use 'run' command which reads from config.toml
+	// The runner will poll for jobs until the context is cancelled
 	args := []string{
 		"run",
 		"--config", r.configPath,
@@ -256,16 +268,33 @@ func (r *Runner) Run(ctx context.Context) error {
 }
 
 // RunOnce starts gitlab-runner to run exactly one job and then exit.
-// This uses the run-single command which is designed for ephemeral runners.
+// Uses run-single command which processes one job and exits - true ephemeral behavior.
+// This requires metadata to be stored during Register().
 func (r *Runner) RunOnce(ctx context.Context) error {
-	r.log.Info("Starting gitlab-runner in single-job mode")
+	r.log.Info("Starting gitlab-runner in single-job mode (run-single)")
 
-	// run-single is specifically designed for ephemeral runners
-	// It picks up one job, executes it, and exits
+	if r.metadata == nil {
+		return fmt.Errorf("metadata not available - Register() must be called before RunOnce()")
+	}
+
+	// Build directories
+	buildsDir := filepath.Join(r.workDir, "builds")
+	cacheDir := filepath.Join(r.workDir, "cache")
+
+	// Use 'run-single' command which executes exactly one job and exits
+	// This provides true ephemeral behavior without relying on external VM termination
 	args := []string{
 		"run-single",
-		"--config", r.configPath,
-		"--wait-timeout", "0", // Wait indefinitely for a job
+		"--url", r.metadata.GitLabInstanceURL,
+		"--token", r.metadata.RunnerToken,
+		"--executor", r.executor,
+		"--builds-dir", buildsDir,
+		"--cache-dir", cacheDir,
+	}
+
+	// Add runner name if available
+	if r.metadata.RunnerName != "" {
+		args = append(args, "--name", r.metadata.RunnerName)
 	}
 
 	cmd := exec.CommandContext(ctx, r.gitlabRunnerPath, args...)

images/docker/ubuntu-24.04/Dockerfile

@@ -280,8 +280,8 @@ RUN case "$TARGETARCH" in amd64|x86_64|i386) export RUNNER_ARCH="amd64";; arm64)
     && chmod +x gitlab-runner                                                                                                                     \
     && ln -s /opt/gitlab-runner/gitlab-runner /usr/local/bin/gitlab-runner                                                                       \
     && chown -R runner:docker /opt/gitlab-runner                                                                                                 \
-    && mkdir -p /etc/gitlab-runner                                                                                                               \
-    && chown -R runner:docker /etc/gitlab-runner
+    && mkdir -p /home/runner/.gitlab-runner                                                                                                      \
+    && chown -R runner:docker /home/runner/.gitlab-runner
 
 # Install GitLab CLI (glab)
 RUN curl -fsSL https://raw.githubusercontent.com/upciti/wakemeops/main/assets/install_repository | bash \

images/docker/ubuntu-24.04/overlay/gitlab/etc/systemd/system/fireglab.service

@@ -18,9 +18,15 @@ User=root
 # The runner command:
 # 1. Fetches metadata from MMDS (glrt-* token, gitlab URL, tags)
 # 2. Registers gitlab-runner with GitLab using the token
-# 3. Starts gitlab-runner daemon in single-job mode
+# 3. Starts gitlab-runner daemon in single-job mode (run-single)
 # 4. Exits when job completes (triggering VM reboot via SuccessAction)
-ExecStart=/usr/bin/fireglab runner --log-level=info --single-job
+#
+# Runtime paths configured for runner user context:
+# - work-dir: Runner's home directory for builds/cache
+# - config: User-specific config to avoid /etc permission issues
+ExecStart=/usr/bin/fireglab runner --log-level=info --single-job \
+    --work-dir=/home/runner \
+    --config=/home/runner/.gitlab-runner/config.toml
 Restart=on-failure
 RestartSec=5s