chore: Adding FS sandboxed unit tests in CI (#6754)

Author: yhakbar

.github/scripts/ci/no-network-exec.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run a command without the two side effects a unit test run should not have:
+#
+# - network reach beyond loopback
+# - writes outside the directories the run owns
+#
+# Written for `go test -exec`:
+#
+#   go test -exec "$PWD/.github/scripts/ci/sandbox-exec.sh" ./...
+#
+# Loopback stays reachable so tests can stand up httptest servers, and writes
+# stay open in the temp dir, the Go caches and Terragrunt's user cache.
+#
+# How much of that holds depends on the platform:
+#
+# - macOS confines both through one seatbelt profile.
+# - Linux confines the network alone, through a network namespace.
+#
+# The --check flag confirms the sandbox is working properly.
+
+CURDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROFILE="$CURDIR/sandbox.sb"
+
+run_sandboxed() {
+	case "$(uname -s)" in
+	Darwin)
+		# sandbox-exec(1) has been deprecated since macOS 10.8, but it is still
+		# the only thing on macOS that applies a profile to an arbitrary command,
+		# and this never ships anywhere but a contributor's machine. Restrictions
+		# reach spawned processes because a child inherits its parent's sandbox,
+		# per sandbox(7).
+		exec /usr/bin/sandbox-exec \
+			-D "TMP=$(cd "${TMPDIR:-/tmp}" && pwd -P)" \
+			-D "GOCACHE=$(go env GOCACHE)" \
+			-D "GOMODCACHE=$(go env GOMODCACHE)" \
+			-D "TGCACHE=$HOME/Library/Caches/terragrunt" \
+			-f "$PROFILE" \
+			"$@"
+		;;
+	Linux)
+		# A fresh network namespace starts with `lo` down, so bring it up before
+		# handing over to the real command. See network_namespaces(7) for what
+		# the namespace covers and user_namespaces(7) for what --map-root-user
+		# buys an unprivileged caller.
+		exec unshare --map-root-user --net -- \
+			sh -c 'ip link set lo up && exec "$@"' sh "$@"
+		;;
+	*)
+		echo "sandbox-exec.sh: no sandbox mechanism for $(uname -s)" >&2
+		exit 1
+		;;
+	esac
+}
+
+check_egress() {
+	local script="$1"
+
+	if ! command -v curl >/dev/null 2>&1; then
+		echo "sandbox-exec.sh: --check needs curl" >&2
+		exit 1
+	fi
+
+	if "$script" curl --silent --show-error --max-time 15 --output /dev/null https://example.com; then
+		echo "sandbox-exec.sh: check failed, https://example.com is still reachable inside the sandbox" >&2
+		exit 1
+	fi
+
+	echo "sandbox-exec.sh: egress is blocked"
+}
+
+check_writes() {
+	local script="$1"
+
+	local allowed
+	allowed="$(mktemp)"
+
+	if ! "$script" touch "$allowed"; then
+		rm -f "$allowed"
+		echo "sandbox-exec.sh: check failed, the temp dir is not writable inside the sandbox" >&2
+		exit 1
+	fi
+
+	rm -f "$allowed"
+
+	local refused="$CURDIR/.sandbox-check"
+
+	if "$script" touch "$refused" 2>/dev/null; then
+		rm -f "$refused"
+		echo "sandbox-exec.sh: check failed, the source tree is still writable inside the sandbox" >&2
+		exit 1
+	fi
+
+	echo "sandbox-exec.sh: writes are confined to the temp dir and the caches"
+}
+
+run_check() {
+	local script="${BASH_SOURCE[0]}"
+
+	if ! "$script" true; then
+		echo "sandbox-exec.sh: check failed, the sandbox cannot run a command at all" >&2
+		exit 1
+	fi
+
+	check_egress "$script"
+
+	if [[ "$(uname -s)" != "Darwin" ]]; then
+		echo "sandbox-exec.sh: writes are NOT confined on $(uname -s)"
+		return
+	fi
+
+	check_writes "$script"
+}
+
+if [[ "${1:-}" == "--check" ]]; then
+	run_check
+	exit 0
+fi
+
+run_sandboxed "$@"

.github/scripts/ci/no-network.sb

@@ -0,0 +1,40 @@
+;; macOS seatbelt profile that blocks the two side effects a unit test run
+;; should not have. It binds the process and everything it spawns.
+;;
+;; Apple documents sandbox-exec(1) and sandbox(7) but not the profile language
+;; below. The working reference for the syntax is the set of profiles Apple
+;; ships in /System/Library/Sandbox/Profiles.
+;;
+;; Network: nothing beyond loopback. Loopback stays open so tests can stand up
+;; httptest servers. Unix domain sockets stay open too, because the platform
+;; reaches system daemons (mDNSResponder, notifyd) through them. Name resolution
+;; still succeeds as a result. Connecting to what it resolves does not.
+;;
+;; Writes: nothing outside the directories a run owns, which are its temp dir,
+;; the Go build and module caches, and Terragrunt's user cache. The checked-out
+;; source tree is not among them. A test that writes into testdata or a fixture
+;; fails here instead of leaving the change behind for the next run.
+;;
+;; Reads are untouched.
+(version 1)
+(allow default)
+
+(deny network*)
+(allow network-bind (local ip "localhost:*"))
+(allow network-inbound (local ip "localhost:*"))
+(allow network-outbound (remote ip "localhost:*"))
+(allow network-outbound (remote unix-socket))
+
+(deny file-write*)
+(allow file-write* (subpath (param "TMP")))
+(allow file-write* (subpath (param "GOCACHE")))
+(allow file-write* (subpath (param "GOMODCACHE")))
+(allow file-write* (subpath (param "TGCACHE")))
+
+;; Terminal and null devices, which the test binary and anything it spawns write
+;; through for ordinary output.
+(allow file-write-data (literal "/dev/null"))
+(allow file-write-data (literal "/dev/dtracehelper"))
+(allow file-write* (regex #"^/dev/tty"))
+(allow file-write* (regex #"^/dev/pty"))
+(allow file-write* (regex #"^/dev/ptmx"))

.github/scripts/ci/sandbox-exec.sh

@@ -8,6 +8,9 @@ on:
 # either exercises an unvirtualized code path or is an integration test hiding in
 # the untagged suite. Both are worth catching here rather than as a flake on a day
 # the registry is slow.
+#
+# Only the network half applies here. The wrapper confines writes through a
+# seatbelt profile, which runs only on macOS, so this job does not check them.
 jobs:
   test:
     name: Sandboxed Test
@@ -64,10 +67,10 @@ jobs:
         run: go build ./... && go mod download
         shell: bash
 
-      - name: Verify the sandbox blocks egress
-        run: ./.github/scripts/ci/no-network-exec.sh --check
+      - name: Verify the sandbox is in force
+        run: ./.github/scripts/ci/sandbox-exec.sh --check
         shell: bash
 
       - name: Run Tests
-        run: go test -exec "$PWD/.github/scripts/ci/no-network-exec.sh" -timeout 45m ./...
+        run: go test -exec "$PWD/.github/scripts/ci/sandbox-exec.sh" -timeout 45m ./...
         shell: bash

.github/scripts/ci/sandbox.sb

@@ -335,25 +335,38 @@ The convention we use for race tests is to prefix them with `WithRacing`. The Te
 
 #### Sandboxed tests
 
-Terragrunt routes its side effects through a virtualized environment, so tests can drive filesystem, subprocess and HTTP behavior without touching the real thing. A test that still reaches the network either exercises a code path that bypasses that environment or is an integration test sitting in the untagged suite. Either way, it makes the unit suite slower and flakier. Every run pays for the round trip, and a slow registry or a rate-limited git host turns into a red build that has nothing to do with the change under review.
+Terragrunt routes its side effects through a virtualized environment, so tests can drive filesystem, subprocess and HTTP behavior without touching the real thing.
 
-To check for that, run the suite with every socket except loopback closed:
+A test can break that in two ways, and neither shows up as a test failure:
+
+1. A test that reaches the network either exercises a code path that bypasses the virtualized environment or is an integration test sitting in the untagged suite.
+1. A test that writes outside its own temp directory leaves state behind for the next run.
+
+To check for both, run the suite through the sandbox wrapper:
 
 ```bash
-go test -exec "$PWD/.github/scripts/ci/no-network-exec.sh" ./...
+go test -exec "$PWD/.github/scripts/ci/sandbox-exec.sh" ./...
 ```
 
-Before reading anything into a passing run, confirm the sandbox took effect:
+<Aside type="tip">
+
+Before reading anything into a passing run, confirm the sandbox took effect. The output names what this platform confines:
 
 ```bash
-./.github/scripts/ci/no-network-exec.sh --check
+./.github/scripts/ci/sandbox-exec.sh --check
 ```
 
-Loopback stays reachable, so tests that stand up an `httptest` server keep working. Compilation happens outside the sandbox, so module downloads are unaffected. How the sandbox is built differs by platform:
+</Aside>
+
+Loopback stays reachable, so tests that stand up an `httptest` server keep working. Writes stay open in the temp directory, the Go caches and Terragrunt's user cache. Compilation happens outside the sandbox, so module downloads are unaffected. Reads are untouched, so none of this says anything about a test picking up `~/.gitconfig` or `~/.terraformrc`.
+
+How the sandbox is built, and how much it covers, differ by platform:
 
 <Tabs syncKey="operating-systems">
   <TabItem label="Linux">
-    Each test binary runs in its own network namespace, created with `unshare`.
+    Each test binary runs in its own network namespace, created with `unshare`. The `network_namespaces(7)` and `user_namespaces(7)` man pages cover what that isolates. Writes are not confined. Doing that needs bubblewrap or a Landlock wrapper, neither of which is wired up, so `--check` reports the network half alone.
+
+    This is the platform Continuous Integration runs on, so CI checks the network and nothing else.
 
     Ubuntu 24.04 and later confine unprivileged user namespaces with AppArmor, which stops `unshare` from creating that namespace. If the check reports that the sandbox cannot run a command at all, lift the restriction:
 
@@ -362,9 +375,9 @@ Loopback stays reachable, so tests that stand up an `httptest` server keep worki
     ```
   </TabItem>
   <TabItem label="macOS">
-    Each test binary runs under the seatbelt profile at `.github/scripts/ci/no-network.sb`, applied with `sandbox-exec`.
+    Each test binary runs under the seatbelt profile at `.github/scripts/ci/sandbox.sb`, applied with `sandbox-exec`. That profile confines the network and writes together.
 
-    Nothing to set up. `sandbox-exec` ships with macOS.
+    Nothing to set up. `sandbox-exec` ships with macOS. Apple documents `sandbox-exec(1)` and `sandbox(7)` but not the profile language, so the profiles under `/System/Library/Sandbox/Profiles` are the working reference for changing `sandbox.sb`.
   </TabItem>
 </Tabs>
 

.github/workflows/sandboxed-test.yml

@@ -58,12 +58,16 @@ func TestFindModules(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.repoPath, func(t *testing.T) {
 			t.Parallel()
-			// Unfortunately, we are unable to commit the `.git` directory. We have to temporarily rename it while running the tests.
-			os.Rename(filepath.Join(tc.repoPath, "gitdir"), filepath.Join(tc.repoPath, ".git"))
-			defer os.Rename(
-				filepath.Join(tc.repoPath, ".git"),
-				filepath.Join(tc.repoPath, "gitdir"),
-			)
+
+			// The fixture carries its git metadata as `gitdir`, because a real
+			// `.git` directory cannot be committed. Copying first keeps that
+			// rename out of the checked-out tree.
+			repoDir := filepath.Join(t.TempDir(), "repo")
+			require.NoError(t, os.CopyFS(repoDir, os.DirFS(tc.repoPath)))
+			require.NoError(t, os.Rename(
+				filepath.Join(repoDir, "gitdir"),
+				filepath.Join(repoDir, ".git"),
+			))
 
 			ctx := t.Context()
 
@@ -74,7 +78,7 @@ func TestFindModules(t *testing.T) {
 				ctx,
 				logger.CreateLogger(),
 				v,
-				&module.RepoOpts{CloneURL: tc.repoPath},
+				&module.RepoOpts{CloneURL: repoDir},
 			)
 			require.NoError(t, err)
 

docs/src/content/docs/05-community/01-contributing.mdx

@@ -1320,6 +1320,10 @@ func RunTerragruntValidateInputs(
 ) {
 	t.Helper()
 
+	// Terragrunt writes a .terragrunt-cache into whatever directory it runs in,
+	// so this runs against a copy rather than the fixture in the checked-out tree.
+	moduleDir = filepath.Join(CopyEnvironment(t, moduleDir), moduleDir)
+
 	maybeNested := filepath.Join(moduleDir, "module")
 	if vfs.Exists(vfs.NewOSFS(), maybeNested) {
 		// Nested module test case with included file, so run terragrunt from the nested module.