fix(server): re-establish the sandbox image before each run (#1478)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: FelixTJDietrich

.changeset/sandbox-image-self-heal.md

@@ -0,0 +1,5 @@
+---
+"hephaestus": patch
+---
+
+Practice reviews recover on their own when the host reclaims the agent image. The image is only referenced while a review runs, so a host that prunes unused images removes it between reviews — and because it was fetched once at startup, every later review failed to start its container and retried into the same failure. The image is now re-established before each run.

server/src/main/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/DockerSandboxConfiguration.java

@@ -6,6 +6,7 @@
 import com.github.dockerjava.httpclient5.ApacheDockerHttpClient;
 import de.tum.cit.aet.hephaestus.agent.job.AgentJobRepository;
 import de.tum.cit.aet.hephaestus.agent.proxy.MentorProxyCredentialRegistry;
+import de.tum.cit.aet.hephaestus.agent.runtime.AgentImageProperties;
 import de.tum.cit.aet.hephaestus.agent.sandbox.InteractiveSandboxProperties;
 import de.tum.cit.aet.hephaestus.agent.sandbox.SandboxProperties;
 import de.tum.cit.aet.hephaestus.agent.sandbox.docker.interactive.DockerInteractiveSandboxAdapter;
@@ -132,13 +133,31 @@ public ExecutorService dockerWaitExecutor(SandboxProperties properties) {
         );
     }
 
+    @Bean
+    public SandboxImageGuard sandboxImageGuard(
+        DockerClientOperations ops,
+        AgentImageProperties agentImageProperties,
+        MeterRegistry meterRegistry
+    ) {
+        return image ->
+            ImagePullBootstrapperSupport.applyPolicy(
+                image,
+                agentImageProperties.pullPolicy(),
+                ops,
+                "sandbox.image.pull",
+                meterRegistry,
+                log
+            );
+    }
+
     @Bean
     public SandboxContainerManager sandboxContainerManager(
         DockerClientOperations ops,
+        SandboxImageGuard imageGuard,
         SandboxProperties properties,
         ExecutorService dockerWaitExecutor
     ) {
-        return new SandboxContainerManager(ops, properties, dockerWaitExecutor);
+        return new SandboxContainerManager(ops, imageGuard, properties, dockerWaitExecutor);
     }
 
     @Bean

server/src/main/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/SandboxContainerManager.java

@@ -26,6 +26,7 @@ public class SandboxContainerManager {
     private static final int SIGKILL_EXIT_CODE = 137;
 
     private final DockerContainerOperations containerOps;
+    private final SandboxImageGuard imageGuard;
     private final SandboxProperties properties;
 
     /**
@@ -39,10 +40,12 @@ public class SandboxContainerManager {
 
     public SandboxContainerManager(
         DockerContainerOperations containerOps,
+        SandboxImageGuard imageGuard,
         SandboxProperties properties,
         ExecutorService dockerWaitExecutor
     ) {
         this.containerOps = containerOps;
+        this.imageGuard = imageGuard;
         this.properties = properties;
         this.dockerWaitExecutor = dockerWaitExecutor;
     }
@@ -53,6 +56,7 @@ public SandboxContainerManager(
      * @return the container ID
      */
     public String createContainer(DockerOperations.ContainerSpec spec) {
+        imageGuard.ensurePresent(spec.image());
         return containerOps.createContainer(spec);
     }
 

server/src/main/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/SandboxImageGuard.java

@@ -0,0 +1,13 @@
+package de.tum.cit.aet.hephaestus.agent.sandbox.docker;
+
+/**
+ * Establishes that a sandbox image is on the daemon before a container is created from it.
+ *
+ * <p>The image is referenced only while a job runs, so a host that prunes unused images reclaims it
+ * between jobs. Fetching it once at startup leaves the job's retry to fail identically, because
+ * nothing re-pulls.
+ */
+@FunctionalInterface
+public interface SandboxImageGuard {
+    void ensurePresent(String image);
+}

server/src/test/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/DockerSandboxLiveTest.java

@@ -80,7 +80,7 @@ void setUp() {
 
         dockerOps = new DockerClientOperations(dockerClient);
         dockerWaitExecutor = Executors.newCachedThreadPool();
-        containerManager = new SandboxContainerManager(dockerOps, properties, dockerWaitExecutor);
+        containerManager = new SandboxContainerManager(dockerOps, image -> {}, properties, dockerWaitExecutor);
         networkManager = new SandboxNetworkManager(dockerOps, properties);
         workspaceManager = new SandboxWorkspaceManager(dockerOps);
         ContainerSecurityPolicy securityPolicy = new ContainerSecurityPolicy(properties, null);

server/src/test/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/RepositoryTreeStagingLiveTest.java

@@ -100,7 +100,7 @@ void setUp() {
         );
         DockerClientOperations dockerOps = new DockerClientOperations(dockerClient);
         dockerWaitExecutor = Executors.newCachedThreadPool();
-        containerManager = new SandboxContainerManager(dockerOps, properties, dockerWaitExecutor);
+        containerManager = new SandboxContainerManager(dockerOps, image -> {}, properties, dockerWaitExecutor);
         networkManager = new SandboxNetworkManager(dockerOps, properties);
         sandboxAdapter = new DockerSandboxAdapter(
             networkManager,

server/src/test/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/SandboxContainerManagerTest.java

@@ -12,6 +12,7 @@
 import de.tum.cit.aet.hephaestus.agent.sandbox.spi.SandboxException;
 import de.tum.cit.aet.hephaestus.testconfig.BaseUnitTest;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
@@ -28,13 +29,14 @@ class SandboxContainerManagerTest extends BaseUnitTest {
     private DockerContainerOperations containerOps;
 
     private SandboxContainerManager manager;
+    private SandboxProperties properties;
     private ExecutorService executor;
 
     private static final String CONTAINER_ID = "abc123";
 
     @BeforeEach
     void setUp() {
-        SandboxProperties properties = new SandboxProperties(
+        properties = new SandboxProperties(
             "unix:///var/run/docker.sock",
             false,
             null,
@@ -49,14 +51,45 @@ void setUp() {
             null
         );
         executor = Executors.newSingleThreadExecutor();
-        manager = new SandboxContainerManager(containerOps, properties, executor);
+        manager = new SandboxContainerManager(containerOps, image -> {}, properties, executor);
     }
 
     @AfterEach
     void tearDown() {
         executor.shutdownNow();
     }
 
+    @Nested
+    class CreateContainer {
+
+        @Test
+        void shouldEnsureImageIsPresentWhenCreatingContainer() {
+            List<String> guarded = new ArrayList<>();
+            SandboxContainerManager guardedManager = new SandboxContainerManager(
+                containerOps,
+                guarded::add,
+                properties,
+                executor
+            );
+            DockerOperations.ContainerSpec spec = new DockerOperations.ContainerSpec(
+                "ghcr.io/example/agent:latest",
+                List.of("true"),
+                Map.of(),
+                null,
+                null,
+                null,
+                Map.of(),
+                null,
+                List.of()
+            );
+            when(containerOps.createContainer(spec)).thenReturn(CONTAINER_ID);
+
+            guardedManager.createContainer(spec);
+
+            assertThat(guarded).containsExactly("ghcr.io/example/agent:latest");
+        }
+    }
+
     @Nested
     class WaitForCompletion {
 

server/src/test/java/de/tum/cit/aet/hephaestus/agent/sandbox/docker/interactive/DockerInteractiveSandboxLiveTest.java

@@ -126,7 +126,7 @@ void setUp() throws Exception {
 
         dockerOps = new DockerClientOperations(dockerClient);
         dockerWaitExecutor = Executors.newCachedThreadPool();
-        containerManager = new SandboxContainerManager(dockerOps, sandboxProperties, dockerWaitExecutor);
+        containerManager = new SandboxContainerManager(dockerOps, image -> {}, sandboxProperties, dockerWaitExecutor);
         networkManager = new SandboxNetworkManager(dockerOps, sandboxProperties);
         workspaceManager = new SandboxWorkspaceManager(dockerOps);
         securityPolicy = new ContainerSecurityPolicy(sandboxProperties, null);