Skip to content

Commit 07604d1

Browse files
committed
🐛 fix(runners): prevent SIGTERM on docker restart during cloud-init
Remove Requires=docker.service from systemd service files to prevent systemd from sending SIGTERM to runner services when docker restarts. Root cause: cloud-init runcmd phase restarts docker for CA cert updates, which triggers systemd to kill services with Requires=docker.service. Changes: - Remove Requires=docker.service from fireteact, fireglab services - Add After=docker.service to fireactions for consistent ordering - Add SilenceUsage/SilenceErrors to Cobra for cleaner error output - Add graceful signal handling to distinguish shutdown from real errors - Add GetRunnerByName() helper to Gitea client
1 parent dc79791 commit 07604d1

6 files changed

Lines changed: 32 additions & 3 deletions

File tree

fireteact/commands/root.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ fireactions for GitHub Actions.
2626
When run without a subcommand, fireteact starts the orchestrator server.
2727
Use 'fireteact runner' inside a VM to start the runner agent.`,
2828
Version: fmt.Sprintf("%s (commit: %s, built: %s)", Version, Commit, Date),
29+
// Don't print usage on errors - we handle errors with proper logging
30+
SilenceUsage: true,
31+
// Don't print errors twice - we log them ourselves
32+
SilenceErrors: true,
2933
}
3034

3135
// Execute adds all child commands to the root command and sets flags appropriately.

fireteact/commands/runner.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,13 +129,23 @@ func runRunner(cmd *cobra.Command, args []string) error {
129129
// Register with Gitea
130130
log.Info("Registering runner with Gitea...")
131131
if err := r.Register(ctx, metadata); err != nil {
132+
// Check if this was a signal-induced shutdown (not a real error)
133+
if ctx.Err() != nil {
134+
log.Info("Registration interrupted by shutdown signal")
135+
return nil // Clean exit, not an error
136+
}
132137
log.Errorf("Failed to register runner: %v", err)
133138
return err
134139
}
135140

136141
// Run the runner daemon
137142
log.Info("Starting act_runner daemon...")
138143
if err := r.Run(ctx); err != nil {
144+
// Check if this was a signal-induced shutdown (not a real error)
145+
if ctx.Err() != nil {
146+
log.Info("Runner stopped by shutdown signal")
147+
return nil // Clean exit, not an error
148+
}
139149
log.Errorf("Runner error: %v", err)
140150
return err
141151
}

fireteact/internal/gitea/client.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,23 @@ func (c *Client) DeleteRunnerByName(ctx context.Context, name string) error {
272272
return nil
273273
}
274274

275+
// GetRunnerByName finds a runner by its name and returns its full status.
276+
// Returns nil if the runner is not found.
277+
func (c *Client) GetRunnerByName(ctx context.Context, name string) (*Runner, error) {
278+
runners, err := c.ListRunners(ctx)
279+
if err != nil {
280+
return nil, fmt.Errorf("failed to list runners: %w", err)
281+
}
282+
283+
for _, runner := range runners {
284+
if runner.Name == name {
285+
return &runner, nil
286+
}
287+
}
288+
289+
return nil, nil // Not found
290+
}
291+
275292
// GetPendingJobs retrieves pending jobs that match the given labels.
276293
// Note: This is a placeholder - the actual implementation depends on Gitea's API.
277294
func (c *Client) GetPendingJobs(ctx context.Context, labels []string) ([]Job, error) {

images/docker/ubuntu-24.04/overlay/gitea/etc/systemd/system/fireteact.service

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ Documentation=https://github.com/thpham/nixos-fireactions
99
# 3. Fireteact starts with CA bundle containing our cert
1010
# After= is ignored if service doesn't exist (e.g., no cloud-init installed)
1111
After=network.target cloud-config.service docker.service
12-
Requires=docker.service
1312
SuccessAction=reboot
1413

1514
[Service]

images/docker/ubuntu-24.04/overlay/github/etc/systemd/system/fireactions.service

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Documentation=https://github.com/hostinger/fireactions
88
# (workaround for Ubuntu 24.04 bug where ca_certs doesn't update bundle)
99
# 3. Fireactions starts with CA bundle containing our cert
1010
# After= is ignored if service doesn't exist (e.g., no cloud-init installed)
11-
After=network.target cloud-config.service
11+
After=network.target cloud-config.service docker.service
1212
SuccessAction=reboot
1313

1414
[Service]

images/docker/ubuntu-24.04/overlay/gitlab/etc/systemd/system/fireglab.service

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ Documentation=https://github.com/thpham/nixos-fireactions
99
# 3. Fireglab starts with CA bundle containing our cert
1010
# After= is ignored if service doesn't exist (e.g., no cloud-init installed)
1111
After=network.target cloud-config.service docker.service
12-
Requires=docker.service
1312
SuccessAction=reboot
1413

1514
[Service]

0 commit comments

Comments
 (0)