pwnshop: per-test timeout (#84)

Author: zardus

AGENTS.md

@@ -10,9 +10,9 @@ This is the pwn.college challenge monorepo containing cybersecurity CTF challeng
 
 ### Challenge CLI
 
-All workflows run through the `./pwnshop` CLI (implemented with Click/Rich in `tools/pwnshop/src/pwnshop/commands` and backed by shared helpers in `tools/pwnshop/src/pwnshop/lib`). The older `./build` script has been retired; never call it or duplicate its behavior.
+All workflows run through the `./pwnshop` CLI (implemented in `tools/pwnshop/src/pwnshop/commands` and backed by shared helpers in `tools/pwnshop/src/pwnshop/lib`).
 
-Each subcommand accepts either a direct path or a challenge slug (e.g., `web-security/path-traversal-1`). Slugs must contain the module and challenge, and the CLI searches `./challenges` for that exact path.
+Each subcommand accepts either a direct path or a challenge slug (e.g., `challenge/web-security/path-traversal-1`). Slugs must contain the base path to all challenges ("challenge"), then the module and challenge.
 
 Primary commands:
 

tools/pwnshop/src/pwnshop/commands/run.py

@@ -9,7 +9,7 @@
 logger = logging.getLogger(__name__)
 
 
-@click.command("run")
+@click.command("run", context_settings=dict(allow_interspersed_args=False))
 @click.argument(
     "challenge_path",
     type=click.Path(path_type=pathlib.Path, exists=True, dir_okay=True, file_okay=False, resolve_path=True),

tools/pwnshop/src/pwnshop/commands/test.py

@@ -24,6 +24,13 @@
 @click.option("--modified-since", metavar="REF", help="Only include challenges changed versus REF.")
 @click.option("--jobs", "-j", metavar="N", type=click.IntRange(1, None), help="Parallel challenges (default: cores).")
 @click.option("--require-tests", is_flag=True, help="Fail if any challenge has no tests.")
+@click.option(
+    "--test-timeout",
+    metavar="N",
+    type=click.IntRange(1, None),
+    default=None,
+    help="Timeout in seconds for each individual test.",
+)
 @click.argument(
     "targets",
     nargs=-1,
@@ -36,7 +43,7 @@
         resolve_path=False,
     ),
 )
-def test_command(targets, modified_since, jobs, require_tests):
+def test_command(targets, modified_since, jobs, require_tests, test_timeout):
     """Test one or more challenges."""
     if not (challenge_paths := lib.resolve_targets(targets, modified_since=modified_since)):
         if modified_since:
@@ -63,17 +70,23 @@ def test_challenge(challenge_path):
                 test_name = test.relative_to(rendered)
                 logger.debug("running test %s in %s", test_name, challenge_path)
                 with lib.run_challenge(image_id, volumes=[test]) as (container, _):
-                    run = subprocess.run(
-                        ["docker", "exec", "--user=1000:1000", container, f"{test}"],
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.STDOUT,
-                        text=True,
-                    )
+                    try:
+                        run = subprocess.run(
+                            ["docker", "exec", "--user=1000:1000", container, f"{test}"],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
+                            text=True,
+                            timeout=test_timeout,
+                        )
+                    except subprocess.TimeoutExpired as e:
+                        logger.warning("test %s timed out after %ds in %s", test_name, test_timeout, challenge_path)
+                        results.append((test_name, False, f"TIMEOUT after {test_timeout}s\n{e.stdout or ''}"))
+                        continue
                 passed = run.returncode == 0
                 logger.debug("test %s %s (rc=%d)", test_name, "PASSED" if passed else "FAILED", run.returncode)
                 results.append((test_name, passed, run.stdout or ""))
             return {"path": challenge_path, "tests": results}
-        except (FileNotFoundError, RuntimeError) as error:
+        except (FileNotFoundError, RuntimeError, subprocess.CalledProcessError) as error:
             logger.error("test setup failed for %s: %s", challenge_path, error)
             return {"path": challenge_path, "error": str(error)}
         finally:

tools/pwnshop/src/pwnshop/lib/__init__.py

@@ -208,7 +208,7 @@ def run_challenge(
         subprocess.run(
             ["docker", "kill", container],
             stdout=subprocess.DEVNULL,
-            check=True,
+            stderr=subprocess.DEVNULL,
         )