🐛 fix(runners): improve YAML output and DNS configuration

- Switch from pyyaml to ruamel-yaml for proper literal block scalar (|)
  output in generated configs (user-data, app_private_key fields)
- Add OOM protection to runner manager services (OOMScoreAdjust=-900)
- Enable systemd-resolved in base image for DNS caching via stub resolver
- Remove redundant DNS override from cloud-init runcmd - let DHCP/
  systemd-networkd handle DNS configuration (dnsmasq option 6)

DNS flow now: app → 127.0.0.53 (systemd-resolved) → gateway (dnsmasq)

Author: thpham

images/docker/ubuntu-24.04/Dockerfile

@@ -158,6 +158,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Copy common overlay files (network, docker service drop-in)
 COPY overlay/common/etc /etc
 
+# Enable systemd-resolved for DNS caching (without systemctl during build)
+# DNS servers are obtained via DHCP when using systemd-networkd (UseDNS=yes)
+RUN mkdir -p /etc/systemd/system/multi-user.target.wants \
+    && ln -sf /lib/systemd/system/systemd-resolved.service \
+              /etc/systemd/system/multi-user.target.wants/systemd-resolved.service
+
+# Configure resolv.conf symlink at runtime (cannot modify during build - Docker bind-mount)
+# This oneshot service runs before systemd-resolved and sets up the stub resolver symlink
+RUN printf '[Unit]\n\
+Description=Setup resolv.conf symlink for systemd-resolved\n\
+DefaultDependencies=no\n\
+Before=systemd-resolved.service\n\
+Before=network.target\n\
+\n\
+[Service]\n\
+Type=oneshot\n\
+RemainAfterExit=yes\n\
+ExecStart=/bin/sh -c "rm -f /etc/resolv.conf && ln -s /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf"\n\
+\n\
+[Install]\n\
+WantedBy=sysinit.target\n' > /etc/systemd/system/resolv-conf-symlink.service \
+    && mkdir -p /etc/systemd/system/sysinit.target.wants \
+    && ln -sf /etc/systemd/system/resolv-conf-symlink.service \
+              /etc/systemd/system/sysinit.target.wants/resolv-conf-symlink.service
+
+# Harden systemd-resolved for Firecracker VMs (disable unused features)
+RUN mkdir -p /etc/systemd/resolved.conf.d \
+    && printf '[Resolve]\nLLMNR=no\nMulticastDNS=no\nDNSSEC=no\nDNSOverTLS=no\n' \
+      > /etc/systemd/resolved.conf.d/firecracker.conf
+
 # Enable cloud-init services (runs before Docker starts)
 RUN systemctl enable cloud-init-local.service \
     cloud-init.service \

modules/fireactions/inject-secrets.py

@@ -9,21 +9,19 @@
 - Squid SSL bump CA certificate (only when needed)
 - Debug SSH key injection
 """
-import yaml
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
 import os
 import json
 
-# Custom representer for multi-line strings (block scalar style)
-def str_representer(dumper, data):
-    if "\n" in data:
-        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
-    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
-
-yaml.add_representer(str, str_representer)
+# Initialize ruamel.yaml with round-trip mode for proper formatting
+yaml = YAML()
+yaml.preserve_quotes = True
+yaml.default_flow_style = False
 
 # Read the base config
 with open("/etc/fireactions/config.yaml", "r") as f:
-    config = yaml.safe_load(f)
+    config = yaml.load(f)
 
 # Inject GitHub secrets from files
 if "github" in config:
@@ -35,7 +33,7 @@ def str_representer(dumper, data):
     private_key_file = os.environ.get("PRIVATE_KEY_FILE", "")
     if private_key_file:
         with open(private_key_file, "r") as f:
-            config["github"]["app_private_key"] = f.read()
+            config["github"]["app_private_key"] = LiteralScalarString(f.read())
 
 # Inject metadata into all pools
 if "pools" in config:
@@ -265,12 +263,9 @@ def str_representer(dumper, data):
             "    fi",
         ])
 
-    # DNS configuration (always, if gateway is set)
-    if gateway:
-        user_data_lines.extend([
-            "  # Set DNS to use host gateway (centralized DNS via dnsmasq)",
-            f"  - echo 'nameserver {gateway}' > /etc/resolv.conf",
-        ])
+    # Note: DNS is automatically configured via DHCP (dnsmasq provides option 6)
+    # systemd-networkd accepts DNS from DHCP (UseDNS=yes in 10-eth0.network)
+    # systemd-resolved caches and forwards to the gateway
 
     # Hostname from MMDS
     user_data_lines.extend([
@@ -282,7 +277,7 @@ def str_representer(dumper, data):
         "    fi",
     ])
 
-    user_data = '\n'.join(user_data_lines) + '\n'
+    user_data = LiteralScalarString('\n'.join(user_data_lines) + '\n')
 
     # Inject user-data into all pools (firecracker/metadata already initialized above)
     for pool in config["pools"]:
@@ -293,6 +288,6 @@ def str_representer(dumper, data):
 
 # Write the final config
 with open("/run/fireactions/config.yaml", "w") as f:
-    yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+    yaml.dump(config, f)
 
 print("Config with secrets written to /run/fireactions/config.yaml")

modules/fireactions/services.nix

@@ -371,7 +371,7 @@ in
             ) registryCacheCfg._internal.caCertPath
           }"
 
-          ${pkgs.python3.withPackages (ps: [ ps.pyyaml ])}/bin/python3 ${./inject-secrets.py}
+          ${pkgs.python3.withPackages (ps: [ ps.ruamel-yaml ])}/bin/python3 ${./inject-secrets.py}
 
           # Set proper permissions
           chown ${cfg.user}:${cfg.group} /run/fireactions/config.yaml
@@ -446,6 +446,10 @@ in
           # Working directory
           WorkingDirectory = cfg.dataDir;
 
+          # OOM protection - critical infrastructure service
+          OOMScoreAdjust = -900;
+          OOMPolicy = "continue";
+
           #
           # Security hardening (built-in, always enabled)
           #

modules/fireglab/inject-secrets.py

@@ -11,7 +11,8 @@
 - Squid SSL bump CA certificate (only when needed)
 - Debug SSH key injection
 """
-import yaml
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
 import os
 import json
 
@@ -25,18 +26,14 @@ def read_secret_file(env_var):
     return None
 
 
-# Custom representer for multi-line strings (block scalar style)
-def str_representer(dumper, data):
-    if "\n" in data:
-        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
-    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
-
-
-yaml.add_representer(str, str_representer)
+# Initialize ruamel.yaml with round-trip mode for proper formatting
+yaml = YAML()
+yaml.preserve_quotes = True
+yaml.default_flow_style = False
 
 # Read the base config
 with open("/etc/fireglab/config.yaml", "r") as f:
-    config = yaml.safe_load(f)
+    config = yaml.load(f)
 
 # Inject GitLab secrets from files
 if "gitlab" in config:
@@ -299,13 +296,9 @@ def str_representer(dumper, data):
                 "    fi",
             ])
 
-    # DNS configuration
-    # Use fireglab gateway for DNS (10.202.0.1) which forwards to main dnsmasq
-    if fireglab_gateway:
-        user_data_lines.extend([
-            "  # Set DNS to use fireglab gateway (centralized DNS via dnsmasq)",
-            f"  - echo 'nameserver {fireglab_gateway}' > /etc/resolv.conf",
-        ])
+    # Note: DNS is automatically configured via DHCP (dnsmasq provides option 6)
+    # systemd-networkd accepts DNS from DHCP (UseDNS=yes in 10-eth0.network)
+    # systemd-resolved caches and forwards to the gateway
 
     # Hostname from MMDS
     user_data_lines.extend([
@@ -317,7 +310,7 @@ def str_representer(dumper, data):
         "    fi",
     ])
 
-    user_data = '\n'.join(user_data_lines) + '\n'
+    user_data = LiteralScalarString('\n'.join(user_data_lines) + '\n')
 
     # Inject user-data into all pools
     for pool in config["pools"]:
@@ -329,6 +322,6 @@ def str_representer(dumper, data):
 # Write the modified config
 os.makedirs("/run/fireglab", exist_ok=True)
 with open("/run/fireglab/config.yaml", "w") as f:
-    yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+    yaml.dump(config, f)
 
 print("fireglab config generated at /run/fireglab/config.yaml")

modules/fireglab/services.nix

@@ -353,7 +353,7 @@ in
             ) registryCacheCfg._internal.caCertPath
           }"
 
-          ${pkgs.python3.withPackages (ps: [ ps.pyyaml ])}/bin/python3 ${./inject-secrets.py}
+          ${pkgs.python3.withPackages (ps: [ ps.ruamel-yaml ])}/bin/python3 ${./inject-secrets.py}
 
           chown ${cfg.user}:${cfg.group} /run/fireglab/config.yaml
           chmod 0640 /run/fireglab/config.yaml
@@ -427,6 +427,10 @@ in
           Restart = "always";
           RestartSec = "10s";
 
+          # OOM protection - critical infrastructure service
+          OOMScoreAdjust = -900;
+          OOMPolicy = "continue";
+
           # Security hardening
           NoNewPrivileges = false;
           ProtectSystem = "strict";

modules/fireteact/inject-secrets.py

@@ -11,7 +11,8 @@
 - Squid SSL bump CA certificate (only when needed)
 - Debug SSH key injection
 """
-import yaml
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
 import os
 import json
 
@@ -25,18 +26,14 @@ def read_secret_file(env_var):
     return None
 
 
-# Custom representer for multi-line strings (block scalar style)
-def str_representer(dumper, data):
-    if "\n" in data:
-        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
-    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
-
-
-yaml.add_representer(str, str_representer)
+# Initialize ruamel.yaml with round-trip mode for proper formatting
+yaml = YAML()
+yaml.preserve_quotes = True
+yaml.default_flow_style = False
 
 # Read the base config
 with open("/etc/fireteact/config.yaml", "r") as f:
-    config = yaml.safe_load(f)
+    config = yaml.load(f)
 
 # Inject Gitea secrets from files
 if "gitea" in config:
@@ -293,13 +290,9 @@ def str_representer(dumper, data):
                 "    fi",
             ])
 
-    # DNS configuration
-    # Use fireteact gateway for DNS (10.201.0.1) which forwards to main dnsmasq
-    if fireteact_gateway:
-        user_data_lines.extend([
-            "  # Set DNS to use fireteact gateway (centralized DNS via dnsmasq)",
-            f"  - echo 'nameserver {fireteact_gateway}' > /etc/resolv.conf",
-        ])
+    # Note: DNS is automatically configured via DHCP (dnsmasq provides option 6)
+    # systemd-networkd accepts DNS from DHCP (UseDNS=yes in 10-eth0.network)
+    # systemd-resolved caches and forwards to the gateway
 
     # Hostname from MMDS
     user_data_lines.extend([
@@ -311,7 +304,7 @@ def str_representer(dumper, data):
         "    fi",
     ])
 
-    user_data = '\n'.join(user_data_lines) + '\n'
+    user_data = LiteralScalarString('\n'.join(user_data_lines) + '\n')
 
     # Inject user-data into all pools
     for pool in config["pools"]:
@@ -323,6 +316,6 @@ def str_representer(dumper, data):
 # Write the modified config
 os.makedirs("/run/fireteact", exist_ok=True)
 with open("/run/fireteact/config.yaml", "w") as f:
-    yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+    yaml.dump(config, f)
 
 print("fireteact config generated at /run/fireteact/config.yaml")

modules/fireteact/services.nix

@@ -353,7 +353,7 @@ in
             ) registryCacheCfg._internal.caCertPath
           }"
 
-          ${pkgs.python3.withPackages (ps: [ ps.pyyaml ])}/bin/python3 ${./inject-secrets.py}
+          ${pkgs.python3.withPackages (ps: [ ps.ruamel-yaml ])}/bin/python3 ${./inject-secrets.py}
 
           chown ${cfg.user}:${cfg.group} /run/fireteact/config.yaml
           chmod 0640 /run/fireteact/config.yaml
@@ -427,6 +427,10 @@ in
           Restart = "always";
           RestartSec = "10s";
 
+          # OOM protection - critical infrastructure service
+          OOMScoreAdjust = -900;
+          OOMPolicy = "continue";
+
           # Security hardening
           NoNewPrivileges = false;
           ProtectSystem = "strict";