✨ feat(fireteact): add graceful shutdown and registry cache support

- Add graceful shutdown with Gitea runner unregistration on SIGTERM/SIGINT
- Clean up stale .sock/.log files from previous runs on startup
- Fix Gitea API response parsing for runners endpoint (wrapped response + label objects)
- Add cloud-init parity with fireactions: Zot registry mirrors, Docker daemon, BuildKit config
- Fix dnsmasq per-subnet DHCP options using tag: directive
- Add debug logging for Gitea API troubleshooting

Author: thpham

fireteact/internal/firecracker/manager.go

@@ -95,9 +95,87 @@ func NewManager(cfg *config.Config, log *logrus.Logger) (*Manager, error) {
 		return nil, fmt.Errorf("failed to create pool directory: %w", err)
 	}
 
+	// Clean up stale resources from previous runs
+	m.cleanupStaleResources()
+
 	return m, nil
 }
 
+// cleanupStaleResources removes orphaned socket and log files from previous runs.
+// This is called on startup to clean up after unclean shutdowns.
+func (m *Manager) cleanupStaleResources() {
+	// Scan all pool directories
+	entries, err := os.ReadDir(DefaultPoolDir)
+	if err != nil {
+		m.log.Warnf("Failed to read pool directory %s: %v", DefaultPoolDir, err)
+		return
+	}
+
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		poolDir := filepath.Join(DefaultPoolDir, entry.Name())
+		m.cleanupPoolDirectory(poolDir)
+	}
+}
+
+// cleanupPoolDirectory removes stale socket and log files from a pool directory.
+func (m *Manager) cleanupPoolDirectory(poolDir string) {
+	files, err := os.ReadDir(poolDir)
+	if err != nil {
+		m.log.Warnf("Failed to read pool directory %s: %v", poolDir, err)
+		return
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+
+		// Only check socket files
+		if filepath.Ext(file.Name()) != ".sock" {
+			continue
+		}
+
+		socketPath := filepath.Join(poolDir, file.Name())
+
+		// Try to connect to the socket to check if firecracker is still running
+		if m.isSocketActive(socketPath) {
+			m.log.Debugf("Socket %s is still active, skipping cleanup", socketPath)
+			continue
+		}
+
+		// Socket is stale, remove it and the corresponding log file
+		m.log.Infof("Removing stale socket: %s", socketPath)
+		if err := os.Remove(socketPath); err != nil {
+			m.log.Warnf("Failed to remove stale socket %s: %v", socketPath, err)
+		}
+
+		// Also remove corresponding log file
+		logPath := socketPath[:len(socketPath)-5] + ".log" // Replace .sock with .log
+		if _, err := os.Stat(logPath); err == nil {
+			m.log.Infof("Removing stale log: %s", logPath)
+			if err := os.Remove(logPath); err != nil {
+				m.log.Warnf("Failed to remove stale log %s: %v", logPath, err)
+			}
+		}
+	}
+}
+
+// isSocketActive checks if a socket file has an active firecracker process.
+func (m *Manager) isSocketActive(socketPath string) bool {
+	// Try to connect to the socket with a short timeout
+	conn, err := net.DialTimeout("unix", socketPath, 100*time.Millisecond)
+	if err != nil {
+		// Connection failed - socket is stale
+		return false
+	}
+	conn.Close()
+	return true
+}
+
 // Close closes the manager and releases resources.
 func (m *Manager) Close() error {
 	if m.containerd != nil {

fireteact/internal/gitea/client.go

@@ -30,15 +30,23 @@ type RegistrationToken struct {
 	Token string `json:"token"`
 }
 
+// RunnerLabel represents a label attached to a runner in Gitea.
+type RunnerLabel struct {
+	ID   int64  `json:"id"`
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
 // Runner represents a registered runner in Gitea.
 type Runner struct {
-	ID          int64    `json:"id"`
-	Name        string   `json:"name"`
-	Status      string   `json:"status"`
-	Busy        bool     `json:"busy"`
-	Labels      []string `json:"labels"`
-	Version     string   `json:"version"`
-	LastContact string   `json:"last_contact,omitempty"`
+	ID          int64         `json:"id"`
+	Name        string        `json:"name"`
+	Status      string        `json:"status"`
+	Busy        bool          `json:"busy"`
+	Ephemeral   bool          `json:"ephemeral"`
+	Labels      []RunnerLabel `json:"labels"`
+	Version     string        `json:"version,omitempty"`
+	LastContact string        `json:"last_contact,omitempty"`
 }
 
 // Job represents a Gitea Actions job.
@@ -153,6 +161,12 @@ func (c *Client) DeleteRunner(ctx context.Context, runnerID int64) error {
 	return nil
 }
 
+// runnersResponse wraps the Gitea API response which contains runners and pagination info.
+type runnersResponse struct {
+	Runners    []Runner `json:"runners"`
+	TotalCount int      `json:"total_count"`
+}
+
 // ListRunners returns all runners registered with Gitea.
 func (c *Client) ListRunners(ctx context.Context) ([]Runner, error) {
 	endpoint := c.getRunnersListEndpoint()
@@ -182,19 +196,82 @@ func (c *Client) ListRunners(ctx context.Context) ([]Runner, error) {
 		return nil, fmt.Errorf("failed to list runners: status %d, body: %s", resp.StatusCode, string(body))
 	}
 
+	// Log raw response for debugging
+	c.log.WithField("response_body", string(body)).Debug("Raw Gitea runners response")
+
+	// Try parsing as direct array first (older Gitea versions)
 	var runners []Runner
-	if err := json.Unmarshal(body, &runners); err != nil {
-		return nil, fmt.Errorf("failed to parse runners response: %w", err)
+	if err := json.Unmarshal(body, &runners); err == nil {
+		c.log.WithField("runner_count", len(runners)).Debug("Parsed as direct array")
+		return runners, nil
 	}
 
-	return runners, nil
+	// Try parsing as wrapped response (newer Gitea versions with pagination)
+	var wrappedResponse runnersResponse
+	if err := json.Unmarshal(body, &wrappedResponse); err != nil {
+		c.log.WithField("response_body", string(body)).Error("Failed to parse runners response")
+		return nil, fmt.Errorf("failed to parse runners response: %w (body: %.200s)", err, string(body))
+	}
+
+	c.log.WithField("runner_count", len(wrappedResponse.Runners)).Debug("Parsed as wrapped response")
+	return wrappedResponse.Runners, nil
 }
 
 // GetInstanceURL returns the Gitea instance URL.
 func (c *Client) GetInstanceURL() string {
 	return c.instanceURL
 }
 
+// DeleteRunnerByName finds and deletes a runner by its name.
+// This is useful during graceful shutdown when we only have the runner name.
+// Returns nil if the runner is not found (already deleted/never registered).
+func (c *Client) DeleteRunnerByName(ctx context.Context, name string) error {
+	runners, err := c.ListRunners(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list runners: %w", err)
+	}
+
+	c.log.WithFields(logrus.Fields{
+		"target_name":  name,
+		"runner_count": len(runners),
+	}).Info("Searching for runner to delete")
+
+	for _, runner := range runners {
+		c.log.WithFields(logrus.Fields{
+			"gitea_name": runner.Name,
+			"target":     name,
+			"runner_id":  runner.ID,
+		}).Debug("Comparing runner names")
+
+		if runner.Name == name {
+			c.log.WithFields(logrus.Fields{
+				"runner_name": name,
+				"runner_id":   runner.ID,
+			}).Info("Found runner, deleting from Gitea")
+			if err := c.DeleteRunner(ctx, runner.ID); err != nil {
+				return err
+			}
+			c.log.WithFields(logrus.Fields{
+				"runner_name": name,
+				"runner_id":   runner.ID,
+			}).Info("Successfully deleted runner from Gitea")
+			return nil
+		}
+	}
+
+	// Runner not found - log the names we did find for debugging
+	var foundNames []string
+	for _, r := range runners {
+		foundNames = append(foundNames, r.Name)
+	}
+	c.log.WithFields(logrus.Fields{
+		"target_name":  name,
+		"found_names":  foundNames,
+		"runner_count": len(runners),
+	}).Warn("Runner not found in Gitea runner list")
+	return nil
+}
+
 // GetPendingJobs retrieves pending jobs that match the given labels.
 // Note: This is a placeholder - the actual implementation depends on Gitea's API.
 func (c *Client) GetPendingJobs(ctx context.Context, labels []string) ([]Job, error) {

fireteact/internal/pool/pool.go

@@ -134,6 +134,7 @@ func (p *Pool) Start(ctx context.Context) error {
 }
 
 // Stop gracefully stops the pool and all runners.
+// This includes unregistering runners from Gitea and destroying VMs.
 func (p *Pool) Stop() error {
 	p.cancel()
 	if p.scaleTicker != nil {
@@ -145,7 +146,22 @@ func (p *Pool) Stop() error {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
+	// Create a context with timeout for shutdown operations
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
 	for id, runner := range p.runners {
+		// First, try to unregister the runner from Gitea
+		// This prevents the runner from showing as offline in Gitea UI
+		if runner.Name != "" {
+			p.log.Infof("Unregistering runner %s from Gitea", runner.Name)
+			if err := p.gitea.DeleteRunnerByName(shutdownCtx, runner.Name); err != nil {
+				p.log.Warnf("Failed to unregister runner %s from Gitea: %v", runner.Name, err)
+				// Continue with VM destruction even if unregistration fails
+			}
+		}
+
+		// Then destroy the VM
 		if runner.VMID != "" {
 			p.log.Infof("Stopping runner %s (VM: %s)", id, runner.VMID)
 			if err := p.vmManager.DestroyVM(runner.VMID); err != nil {

modules/fireactions/registry-cache.nix

@@ -507,10 +507,12 @@ in
         cache-size = 1000;
         log-queries = false;
         # Use list for dhcp-range so it can merge with fireteact's range
-        dhcp-range = [ "${dhcpStart},${dhcpEnd},${netmask},12h" ];
+        # Use set: tag to scope this range, allowing per-subnet dhcp-options
+        dhcp-range = [ "set:fireactions,${dhcpStart},${dhcpEnd},${netmask},12h" ];
+        # Use tag: to scope options to fireactions subnet only
         dhcp-option = [
-          "3,${gateway}"
-          "6,${gateway}"
+          "tag:fireactions,3,${gateway}" # Gateway for fireactions subnet
+          "tag:fireactions,6,${gateway}" # DNS server for fireactions subnet
         ];
         dhcp-rapid-commit = true;
       };