chore: Adding exec sandboxed logic to seatbelt (#6760)

Author: yhakbar

.github/scripts/ci/sandbox-exec.sh

@@ -1,21 +1,23 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Run a command without the two side effects a unit test run should not have:
+# Run a command without the side effects a unit test run should not have:
 #
 # - network reach beyond loopback
 # - writes outside the directories the run owns
+# - execution of a real OpenTofu or Terraform binary
 #
 # Written for `go test -exec`:
 #
 #   go test -exec "$PWD/.github/scripts/ci/sandbox-exec.sh" ./...
 #
 # Loopback stays reachable so tests can stand up httptest servers, and writes
-# stay open in the temp dir, the Go caches and Terragrunt's user cache.
+# stay open in the temp dir, the Go caches and Terragrunt's user cache. A test
+# that needs a real toolchain belongs behind the `tf` build tag.
 #
 # How much of that holds depends on the platform:
 #
-# - macOS confines both through one seatbelt profile.
+# - macOS confines all of them through one seatbelt profile.
 # - Linux confines the network alone, through a network namespace.
 #
 # The --check flag confirms the sandbox is working properly.
@@ -95,6 +97,25 @@ check_writes() {
 	echo "sandbox-exec.sh: writes are confined to the temp dir and the caches"
 }
 
+check_tf_exec() {
+	local script="$1"
+
+	local binary
+	binary="$(command -v tofu || command -v terraform || true)"
+
+	if [[ -z "$binary" ]]; then
+		echo "sandbox-exec.sh: no tofu or terraform on PATH, skipping the execution check"
+		return
+	fi
+
+	if "$script" "$binary" -version >/dev/null 2>&1; then
+		echo "sandbox-exec.sh: check failed, $binary still runs inside the sandbox" >&2
+		exit 1
+	fi
+
+	echo "sandbox-exec.sh: toolchain execution is blocked"
+}
+
 run_check() {
 	local script="${BASH_SOURCE[0]}"
 
@@ -106,11 +127,12 @@ run_check() {
 	check_egress "$script"
 
 	if [[ "$(uname -s)" != "Darwin" ]]; then
-		echo "sandbox-exec.sh: writes are NOT confined on $(uname -s)"
+		echo "sandbox-exec.sh: writes and execution are NOT confined on $(uname -s)"
 		return
 	fi
 
 	check_writes "$script"
+	check_tf_exec "$script"
 }
 
 if [[ "${1:-}" == "--check" ]]; then

.github/scripts/ci/sandbox.sb

@@ -1,5 +1,5 @@
-;; macOS seatbelt profile that blocks the two side effects a unit test run
-;; should not have. It binds the process and everything it spawns.
+;; macOS seatbelt profile that blocks the side effects a unit test run should
+;; not have. It binds the process and everything it spawns.
 ;;
 ;; Apple documents sandbox-exec(1) and sandbox(7) but not the profile language
 ;; below. The working reference for the syntax is the set of profiles Apple
@@ -15,6 +15,11 @@
 ;; source tree is not among them. A test that writes into testdata or a fixture
 ;; fails here instead of leaving the change behind for the next run.
 ;;
+;; Execution: no real OpenTofu or Terraform binary. The `tf` build tag marks
+;; the tests that need one, and the integration jobs run those against a real
+;; toolchain. A test that needs to stand in for the binary substitutes
+;; vexec.NewMemExec instead of putting a mock on PATH.
+;;
 ;; Reads are untouched.
 (version 1)
 (allow default)
@@ -31,6 +36,8 @@
 (allow file-write* (subpath (param "GOMODCACHE")))
 (allow file-write* (subpath (param "TGCACHE")))
 
+(deny process-exec (regex #"/(tofu|terraform)$"))
+
 ;; Terminal and null devices, which the test binary and anything it spawns write
 ;; through for ordinary output.
 (allow file-write-data (literal "/dev/null"))

.github/workflows/sandboxed-test.yml

@@ -9,8 +9,9 @@ on:
 # the untagged suite. Both are worth catching here rather than as a flake on a day
 # the registry is slow.
 #
-# Only the network half applies here. The wrapper confines writes through a
-# seatbelt profile, which runs only on macOS, so this job does not check them.
+# Only the network half applies here. The wrapper confines writes and toolchain
+# execution through a seatbelt profile, which runs only on macOS, so this job
+# does not check either.
 jobs:
   test:
     name: Sandboxed Test

docs/src/content/docs/05-community/01-contributing.mdx

@@ -337,12 +337,13 @@ The convention we use for race tests is to prefix them with `WithRacing`. The Te
 
 Terragrunt routes its side effects through a virtualized environment, so tests can drive filesystem, subprocess and HTTP behavior without touching the real thing.
 
-A test can break that in two ways, and neither shows up as a test failure:
+A test can break that in ways that never show up as a test failure:
 
 1. A test that reaches the network either exercises a code path that bypasses the virtualized environment or is an integration test sitting in the untagged suite.
 1. A test that writes outside its own temp directory leaves state behind for the next run.
+1. A test that runs a real OpenTofu or Terraform binary belongs behind the `tf` build tag, where the integration jobs give it a real toolchain. A test that only needs to stand in for the binary substitutes `vexec.NewMemExec`.
 
-To check for both, run the suite through the sandbox wrapper:
+To check for all of them, run the suite through the sandbox wrapper:
 
 ```bash
 go test -exec "$PWD/.github/scripts/ci/sandbox-exec.sh" ./...
@@ -358,13 +359,13 @@ Before reading anything into a passing run, confirm the sandbox took effect. The
 
 </Aside>
 
-Loopback stays reachable, so tests that stand up an `httptest` server keep working. Writes stay open in the temp directory, the Go caches and Terragrunt's user cache. Compilation happens outside the sandbox, so module downloads are unaffected. Reads are untouched, so none of this says anything about a test picking up `~/.gitconfig` or `~/.terraformrc`.
+Loopback stays reachable, so tests that stand up an `httptest` server keep working. Writes stay open in the temp directory, the Go caches and Terragrunt's user cache. A test that needs to stand in for the toolchain substitutes `vexec.NewMemExec` rather than putting a mock binary on `PATH`. Compilation happens outside the sandbox, so module downloads are unaffected. Reads are untouched, so none of this says anything about a test picking up `~/.gitconfig` or `~/.terraformrc`.
 
 How the sandbox is built, and how much it covers, differ by platform:
 
 <Tabs syncKey="operating-systems">
   <TabItem label="Linux">
-    Each test binary runs in its own network namespace, created with `unshare`. The `network_namespaces(7)` and `user_namespaces(7)` man pages cover what that isolates. Writes are not confined. Doing that needs bubblewrap or a Landlock wrapper, neither of which is wired up, so `--check` reports the network half alone.
+    Each test binary runs in its own network namespace, created with `unshare`. The `network_namespaces(7)` and `user_namespaces(7)` man pages cover what that isolates. Writes and execution are not confined. Doing either needs bubblewrap or a Landlock wrapper, neither of which is wired up, so `--check` reports the network half alone.
 
     This is the platform Continuous Integration runs on, so CI checks the network and nothing else.
 
@@ -375,7 +376,7 @@ How the sandbox is built, and how much it covers, differ by platform:
     ```
   </TabItem>
   <TabItem label="macOS">
-    Each test binary runs under the seatbelt profile at `.github/scripts/ci/sandbox.sb`, applied with `sandbox-exec`. That profile confines the network and writes together.
+    Each test binary runs under the seatbelt profile at `.github/scripts/ci/sandbox.sb`, applied with `sandbox-exec`. That profile confines the network, writes and toolchain execution together.
 
     Nothing to set up. `sandbox-exec` ships with macOS. Apple documents `sandbox-exec(1)` and `sandbox(7)` but not the profile language, so the profiles under `/System/Library/Sandbox/Profiles` are the working reference for changing `sandbox.sb`.
   </TabItem>

test/integration_dependency_outputs_tf_test.go

@@ -0,0 +1,61 @@
+//go:build tf
+
+package test_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/gruntwork-io/terragrunt/test/helpers"
+)
+
+func TestTFDependencyOutputSkipDependencyOutputsFlag(t *testing.T) {
+	t.Parallel()
+
+	helpers.CleanupTerraformFolder(t, testFixtureGetOutput)
+
+	// The subtests all drive the same unit, so each one needs its own copy of the
+	// fixture. Sharing a working directory makes them race for its state lock.
+	noOutputPath := func(t *testing.T) string {
+		t.Helper()
+
+		tmpEnvPath := helpers.CopyEnvironment(t, testFixtureGetOutput)
+
+		return filepath.Join(tmpEnvPath, testFixtureGetOutput, "integration", "skip-dependency-outputs")
+	}
+
+	t.Run("plan without flag fails", func(t *testing.T) {
+		t.Parallel()
+		_, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt plan --non-interactive --working-dir "+noOutputPath(t))
+		require.ErrorContains(t, err, "resolving dependency \"app1\" outputs")
+	})
+
+	t.Run("flag rejected without experiment", func(t *testing.T) {
+		t.Parallel()
+
+		if helpers.IsExperimentMode(t) {
+			t.Skip("Skipping: TG_EXPERIMENT_MODE forces the optional-dependency-outputs experiment on, so its disabled-state error can't be verified")
+		}
+
+		_, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt init --no-dependency-outputs --non-interactive --working-dir "+noOutputPath(t))
+		require.ErrorContains(t, err, "--no-dependency-outputs requires the 'optional-dependency-outputs' experiment")
+	})
+
+	for _, cmd := range []string{"init", "validate", "plan"} {
+		t.Run(cmd+" succeeds with flag", func(t *testing.T) {
+			t.Parallel()
+			_, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt "+cmd+" --experiment optional-dependency-outputs --no-dependency-outputs --non-interactive --working-dir "+noOutputPath(t))
+			require.NoError(t, err)
+		})
+	}
+
+	for _, cmd := range []string{"init", "validate", "plan"} {
+		t.Run("run --all "+cmd+" succeeds with flag", func(t *testing.T) {
+			t.Parallel()
+			_, _, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run --all --experiment optional-dependency-outputs "+cmd+" --no-dependency-outputs --non-interactive --working-dir "+noOutputPath(t))
+			require.NoError(t, err)
+		})
+	}
+}

test/integration_hooks_test.go

@@ -1,13 +1,10 @@
 package test_test
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"testing"
 
-	"github.com/gruntwork-io/terragrunt/internal/cli/commands/info/print"
 	"github.com/gruntwork-io/terragrunt/test/helpers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -69,36 +66,3 @@ func assertNoHookOutputFiles(t *testing.T, unitPaths ...string) {
 		assert.NoFileExists(t, filepath.Join(unitPath, "error.out"))
 	}
 }
-
-func TestTerragruntInfo(t *testing.T) {
-	t.Parallel()
-
-	helpers.CleanupTerraformFolder(t, testFixtureHooksInitOnceWithSourceNoBackendSuppressHookStdout)
-	tmpEnvPath := helpers.CopyEnvironment(t, "fixtures/hooks/init-once")
-	rootPath := filepath.Join(
-		tmpEnvPath,
-		testFixtureHooksInitOnceWithSourceNoBackendSuppressHookStdout,
-	)
-
-	showStdout := bytes.Buffer{}
-	showStderr := bytes.Buffer{}
-
-	err := helpers.RunTerragruntCommand(
-		t,
-		"terragrunt info print --non-interactive --working-dir "+rootPath,
-		&showStdout,
-		&showStderr,
-	)
-	require.NoError(t, err)
-
-	helpers.LogBufferContentsLineByLine(t, showStdout, "show stdout")
-
-	var dat print.InfoOutput
-
-	errUnmarshal := json.Unmarshal(showStdout.Bytes(), &dat)
-	require.NoError(t, errUnmarshal)
-
-	assert.Equal(t, fmt.Sprintf("%s/%s", rootPath, helpers.TerragruntCache), dat.DownloadDir)
-	assert.Equal(t, wrappedBinary(t.Context()), dat.TerraformBinary)
-	assert.Empty(t, dat.IAMRole)
-}

test/integration_hooks_tf_test.go

@@ -4,12 +4,14 @@ package test_test
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/gruntwork-io/terragrunt/internal/cli/commands/info/print"
 	"github.com/gruntwork-io/terragrunt/test/helpers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -649,3 +651,36 @@ func TestTFTerragruntHookExitCodeError(t *testing.T) {
 	assert.Contains(t, output, `exited with non-zero exit code 2`)
 	assert.Contains(t, output, "lint warning: something is wrong")
 }
+
+func TestTFTerragruntInfo(t *testing.T) {
+	t.Parallel()
+
+	helpers.CleanupTerraformFolder(t, testFixtureHooksInitOnceWithSourceNoBackendSuppressHookStdout)
+	tmpEnvPath := helpers.CopyEnvironment(t, "fixtures/hooks/init-once")
+	rootPath := filepath.Join(
+		tmpEnvPath,
+		testFixtureHooksInitOnceWithSourceNoBackendSuppressHookStdout,
+	)
+
+	showStdout := bytes.Buffer{}
+	showStderr := bytes.Buffer{}
+
+	err := helpers.RunTerragruntCommand(
+		t,
+		"terragrunt info print --non-interactive --working-dir "+rootPath,
+		&showStdout,
+		&showStderr,
+	)
+	require.NoError(t, err)
+
+	helpers.LogBufferContentsLineByLine(t, showStdout, "show stdout")
+
+	var dat print.InfoOutput
+
+	errUnmarshal := json.Unmarshal(showStdout.Bytes(), &dat)
+	require.NoError(t, errUnmarshal)
+
+	assert.Equal(t, fmt.Sprintf("%s/%s", rootPath, helpers.TerragruntCache), dat.DownloadDir)
+	assert.Equal(t, wrappedBinary(t.Context()), dat.TerraformBinary)
+	assert.Empty(t, dat.IAMRole)
+}