chore: Completing venv abstraction

🤞🏽 Should complete the venv abstraction 🤞🏽.

Closes some gaps related to GOOS and GOARCH usage. Also closes direct homedir access.

Closes more gaps from direct filesystem and exec usage.

Also adds some optimization for listing files in a directory with a
particular suffix... Then I added one more optimization in
`getTerraformInputNamesFromAutomaticVarFiles`.

Addressing `EOF` in input stream.

Dropping `iofs` aliases.

Using TLS server for engine download tests.

As a consequence, we can do a bit more parallelization in tests.

Author: yhakbar

.golangci.yml

@@ -116,6 +116,15 @@ linters:
         - pattern: ^os\.(UserHomeDir|UserCacheDir|UserConfigDir)$
           pkg: ^os$
           msg: use the venv's Platform handles instead of reading the invoking user's directories
+        - pattern: ^os\.(Getpid|Executable|Hostname)$
+          pkg: ^os$
+          msg: use the venv's Platform handles instead of reading the running process
+        - pattern: ^os\.Args$
+          pkg: ^os$
+          msg: take the arguments the CLI was handed rather than reading the process arguments
+        - pattern: ^runtime\.(GOOS|GOARCH)$
+          pkg: ^runtime$
+          msg: take the platform from the venv's Platform.GOOS and Platform.GOARCH instead of the compiled-in constants
         - pattern: ^os\.(Stdin|Stdout|Stderr)$
           pkg: ^os$
           msg: read and write the venv's Reader and Writers, and ask its Terminal whether a stream is a tty
@@ -128,11 +137,6 @@ linters:
         - pattern: ^http\.Default(Client|Transport)$
           pkg: ^net/http$
           msg: send through the venv's vhttp.Client instead of the shared default client
-        # A composite literal builds a client without calling anything the rules
-        # above match, so it is the remaining way to reach the network past the
-        # venv. The two production sites that survive it are the GCS client
-        # constructors, whose nolint directives cover this rule as well; both
-        # wrap a transport derived from the venv's own client.
         - pattern: ^http\.Client$
           pkg: ^net/http$
           msg: take the client from the venv's vhttp.Client rather than building one
@@ -147,9 +151,6 @@ linters:
           msg: resolve links on the venv's vfs.FS via vfs.EvalSymlinks, or vfs.ResolveForCompare when the result is to be compared against another path
         - pattern: ^(vfs\.NewOSFS|vexec\.NewOSExec|vhttp\.NewOSClient|vsops\.NewOSDecrypter)$
           msg: take the handle from the venv already in scope rather than building a fresh OS-backed one
-        # Cloud SDKs build their own transport when handed no client, so these
-        # entry points leak past the venv without any net/http call of our own
-        # for the rules above to catch.
         - pattern: ^config\.LoadDefaultConfig$
           pkg: ^github.com/aws/aws-sdk-go-v2/config$
           msg: build AWS config via awshelper.NewAWSConfigBuilder so requests ride the venv's HTTP client
@@ -221,33 +222,39 @@ linters:
       - linters:
           - forbidigo
         path: ^(internal/view/tui/tty\.go|internal/tf/run_cmd\.go)$
-
-      # Everything below are the call sites that predate the forbidigo
-      # rules above. They reach the real OS directly and still need porting to
-      # venv. Delete entries as they are ported.
-
-      - linters:
-          - forbidigo
-        path: ^(internal/cli/commands/browse/tui/model\.go|internal/cli/commands/catalog/cli\.go|internal/view/tui/form/form\.go|internal/cli/commands/catalog/tui/tags_layout\.go|internal/cli/commands/commands\.go|internal/cli/commands/hcl/validate/validate\.go|internal/cli/commands/help/cli\.go|internal/cli/commands/render/render\.go|internal/cli/commands/scaffold/cli\.go|internal/cli/commands/scaffold/scaffold\.go)$
-      - linters:
-          - forbidigo
-        path: ^(internal/tf/cache/handlers/filesystem_mirror_provider\.go|internal/tf/cache/helpers/http\.go|internal/tf/cliconfig/credentials\.go|internal/tf/getproviders/constraints\.go|internal/tf/getproviders/hash\.go|internal/tf/getproviders/lock\.go|internal/tf/getproviders/package_authentication\.go|internal/tf/source\.go|internal/tf/tf\.go)$
+      # main is where the process arguments enter the program and become the
+      # arguments every layer below is handed, so it is the one place that has
+      # nothing to read them from.
       - linters:
           - forbidigo
-        path: ^(internal/runner/common/unit_runner\.go|internal/runner/run/debug\.go|internal/runner/run/run\.go|internal/runner/runnerpool/runner\.go)$
-      - linters:
-          - forbidigo
-        path: ^(pkg/config/catalog\.go|pkg/config/config_helpers\.go|pkg/config/config_partial\.go|pkg/config/config\.go|pkg/config/dependency\.go|pkg/config/stack\.go)$
-      # DefaultWrappedPath is a package-level var, so it probes for `tofu` at
-      # init time, before any venv exists.
-      #
-      # TODO: Make the lookup lazy with a once so we can inject venv into this.
+        text: use of `os.Args`
+        path: ^main\.go$
+
+      # The vendored urfave/cli help renderer reads CLI_TEMPLATE_ERROR_DEBUG
+      # from the process with os.Getenv, so the only way to turn its template
+      # diagnostics on is to put the variable where it will look.
       - linters:
           - forbidigo
-        path: ^pkg/options/options\.go$
+        text: use of `os.Setenv`
+        path: ^internal/cli/commands/help/cli\.go$
+
+      # Pre-existing style debt in two paths, held back so it can be cleaned
+      # up on its own. The venv rules are deliberately absent from this list,
+      # so they still apply to both paths.
       - linters:
-          - forbidigo
-        path: ^(internal/discovery/phase_worktree\.go|internal/engine/engine\.go|internal/engine/verification\.go|internal/getter/tfrhelpers\.go|internal/git/git\.go|internal/remotestate/remote_state\.go|internal/remotestate/terraform_state_file\.go|internal/report/writer\.go|internal/services/catalog/module/doc\.go|internal/services/catalog/module/module\.go|internal/shell/run_cmd\.go|internal/stacks/clean/clean\.go|internal/tfimpl/tfimpl\.go|internal/util/lockfile\.go|internal/worktrees/worktrees\.go)$
+          - errcheck
+          - govet
+          - mnd
+          - paralleltest
+          - perfsprint
+          - staticcheck
+          - testifylint
+          - thelper
+          - tparallel
+          - unused
+          - wsl_v5
+        path: ^(internal/github/|test/integration_docs_test\.go$)
+
       # We end up with duplicated content in this package to save us from duplicating code in other packages.
       - linters:
           - dupl
@@ -266,10 +273,10 @@ linters:
           - lll
         path-except: "^(internal/awshelper/|internal/cas/|internal/cli/commands/(backend/(delete|migrate)|browse|catalog/tui/command|dag/graph|exec|find|help|list|scaffold|stack)/|internal/cloner/|internal/codegen/|internal/configbridge/|internal/discovery/|internal/engine/|internal/errorconfig/|internal/errors/|internal/experiment/|internal/filter/|internal/gcphelper/|internal/getter/|internal/git/|internal/os/exec/|internal/prepare/|internal/queue/|internal/report/|internal/retry/|internal/runner/(common|graph|run/creds|runall|runcfg)/|internal/stacks/(generate|output)/|internal/telemetry/|internal/tf/cache/(controllers|middleware)/|internal/tflint/|internal/tips/|internal/vfs/|internal/worktrees/|pkg/log/(format/(options|placeholders)|writer)/|pkg/options/)"
     paths:
-      - docs
-      - _ci
-      - .github
-      - .circleci
+      - ^docs/
+      - ^_ci/
+      - ^\.github/
+      - ^\.circleci/
       - third_party$
       - builtin$
       - examples$
@@ -285,10 +292,10 @@ formatters:
   exclusions:
     generated: lax
     paths:
-      - docs
-      - _ci
-      - .github
-      - .circleci
+      - ^docs/
+      - ^_ci/
+      - ^\.github/
+      - ^\.circleci/
       - third_party$
       - builtin$
       - examples$

docs/src/data/changelog/v1.1.4/prompt-answer-without-newline.mdx

@@ -0,0 +1,8 @@
+---
+version: "v1.1.4"
+category: "bug-fixes"
+---
+
+#### Prompts accept a piped answer that has no trailing newline
+
+Piping an answer to a confirmation prompt, as in `printf yes | terragrunt run --all destroy`, failed with an `EOF` error because Terragrunt discarded a final answer that ended without a newline. The answer is now read as typed, and only a prompt that gets no input at all reports `EOF`.

internal/cas/cas.go

@@ -13,7 +13,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"errors"
 
@@ -804,6 +803,8 @@ func (c *CAS) ensureBlob(
 	hash string,
 	gitPerm os.FileMode,
 ) (err error) {
+	v.RequireGOOS()
+
 	needsWrite, lock, err := c.blobStore.EnsureWithWait(v, hash)
 	if err != nil {
 		return err
@@ -839,7 +840,7 @@ func (c *CAS) ensureBlob(
 		return err
 	}
 
-	if runtime.GOOS == "windows" {
+	if v.Platform.GOOS == WindowsOS {
 		if err = tmpHandle.Sync(); err != nil {
 			return err
 		}

internal/cas/content.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"runtime"
 
 	"github.com/gruntwork-io/terragrunt/internal/telemetry"
 	"github.com/gruntwork-io/terragrunt/internal/venv"
@@ -361,6 +360,8 @@ func (c *Content) Read(v *venv.Venv, hash string) ([]byte, error) {
 // writeContentToFile writes data to a temporary file, sets appropriate
 // permissions, and performs an atomic rename.
 func (c *Content) writeContentToFile(l log.Logger, v *venv.Venv, hash string, data []byte) error {
+	v.RequireGOOS()
+
 	path := c.getPath(hash)
 	tempPath := path + ".tmp"
 
@@ -411,7 +412,7 @@ func (c *Content) writeContentToFile(l log.Logger, v *venv.Venv, hash string, da
 		return fmt.Errorf("chmod temp %s: %w", tempPath, err)
 	}
 
-	if runtime.GOOS == WindowsOS {
+	if v.Platform.GOOS == WindowsOS {
 		if _, err := v.FS.Stat(path); err == nil {
 			if err := v.FS.Chmod(path, RegularFilePerms); err != nil {
 				l.Warnf("failed to make destination file writable %s: %v", path, err)
@@ -427,7 +428,7 @@ func (c *Content) writeContentToFile(l log.Logger, v *venv.Venv, hash string, da
 		return fmt.Errorf("finalize %s: %w", path, err)
 	}
 
-	if runtime.GOOS == WindowsOS {
+	if v.Platform.GOOS == WindowsOS {
 		if err := v.FS.Chmod(path, StoredFilePerms); err != nil {
 			return fmt.Errorf("chmod %s: %w", path, err)
 		}

internal/cli/app_test.go

@@ -15,6 +15,7 @@ import (
 	awsproviderpatch "github.com/gruntwork-io/terragrunt/internal/cli/commands/aws-provider-patch"
 	"github.com/gruntwork-io/terragrunt/internal/cli/commands/hcl"
 	hclformat "github.com/gruntwork-io/terragrunt/internal/cli/commands/hcl/format"
+	"github.com/gruntwork-io/terragrunt/internal/vexec"
 
 	"github.com/gruntwork-io/terragrunt/internal/cli/commands/run"
 	"github.com/gruntwork-io/terragrunt/internal/cli/flags"
@@ -43,10 +44,8 @@ func TestParseTerragruntOptionsFromArgs(t *testing.T) {
 		t.Skip("Skipping test on Windows")
 	}
 
-	workingDir, err := os.Getwd()
-	if err != nil {
-		t.Fatal(err)
-	}
+	workingDir, err := venvtest.New().Platform.Getwd()
+	require.NoError(t, err)
 
 	testCases := []struct {
 		expectedErr     error
@@ -450,7 +449,7 @@ func TestParseTerragruntOptionsFromArgs(t *testing.T) {
 		t.Run(fmt.Sprintf("testCase-%d", i), func(t *testing.T) {
 			t.Parallel()
 
-			opts := options.NewTerragruntOptions()
+			opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 			l := log.New(
 				log.WithOutput(os.Stderr),
@@ -725,7 +724,7 @@ func TestFilterTerragruntArgs(t *testing.T) {
 		t.Run(fmt.Sprintf("testCase-%d", i), func(t *testing.T) {
 			t.Parallel()
 
-			opts := options.NewTerragruntOptions()
+			opts := options.NewTerragruntOptions(vexec.NewOSExec())
 			l := log.New(
 				log.WithOutput(os.Stderr),
 				log.WithLevel(defaultLogLevel),
@@ -802,7 +801,7 @@ func TestParseMultiStringArg(t *testing.T) {
 		t.Run(fmt.Sprintf("testCase-%d", i), func(t *testing.T) {
 			t.Parallel()
 
-			opts := options.NewTerragruntOptions()
+			opts := options.NewTerragruntOptions(vexec.NewOSExec())
 			l := log.New(
 				log.WithOutput(os.Stderr),
 				log.WithLevel(defaultLogLevel),
@@ -879,7 +878,7 @@ func TestParseMutliStringKeyValueArg(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		opts := options.NewTerragruntOptions()
+		opts := options.NewTerragruntOptions(vexec.NewOSExec())
 		opts.AwsProviderPatchOverrides = tc.defaultValue
 		l := log.New(
 			log.WithOutput(os.Stderr),
@@ -918,7 +917,7 @@ func TestTerragruntVersion(t *testing.T) {
 
 	for _, tc := range testCases {
 		output := &bytes.Buffer{}
-		opts := options.NewTerragruntOptions()
+		opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 		testV := venvtest.New()
 
@@ -941,7 +940,7 @@ func TestTerragruntHelp(t *testing.T) {
 
 	terragruntPrefix := flags.Prefix{flags.TerragruntPrefix}
 
-	opts := options.NewTerragruntOptions()
+	opts := options.NewTerragruntOptions(vexec.NewOSExec())
 	app := cli.NewApp(logger.CreateLogger(), opts, venvtest.New())
 
 	testCases := []struct {
@@ -980,7 +979,7 @@ func TestTerragruntHelp(t *testing.T) {
 			t.Parallel()
 
 			output := &bytes.Buffer{}
-			opts := options.NewTerragruntOptions()
+			opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 			testV := venvtest.New()
 
@@ -1006,7 +1005,7 @@ func TestTerraformHelp_wrongHelpFlag(t *testing.T) {
 
 	output := &bytes.Buffer{}
 
-	opts := options.NewTerragruntOptions()
+	opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 	testV := venvtest.New()
 
@@ -1077,7 +1076,9 @@ func (err argMissingValueError) Error() string {
 	return "flag needs an argument: -" + string(err)
 }
 
-func TestAutocomplete(t *testing.T) { //nolint:paralleltest
+func TestAutocomplete(t *testing.T) {
+	t.Parallel()
+
 	testCases := []struct {
 		compLine          string
 		expectedCompletes []string
@@ -1102,7 +1103,7 @@ func TestAutocomplete(t *testing.T) { //nolint:paralleltest
 
 	for _, tc := range testCases {
 		output := &bytes.Buffer{}
-		opts := options.NewTerragruntOptions()
+		opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 		// Autocomplete reads COMP_LINE from the venv's environment, so the
 		// completion request is handed over rather than exported to the process.

internal/cli/app_tf_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/gruntwork-io/terragrunt/internal/cli"
+	"github.com/gruntwork-io/terragrunt/internal/vexec"
 
 	"github.com/gruntwork-io/terragrunt/internal/tf"
 	"github.com/gruntwork-io/terragrunt/internal/venv"
@@ -41,7 +42,7 @@ func TestTFTerraformHelp(t *testing.T) {
 
 	for _, tc := range testCases {
 		output := &bytes.Buffer{}
-		opts := options.NewTerragruntOptions()
+		opts := options.NewTerragruntOptions(vexec.NewOSExec())
 
 		testV := venv.OSVenv()
 

internal/cli/commands/backend/bootstrap/bootstrap.go

@@ -43,7 +43,7 @@ func runBootstrap(
 			return err
 		}
 
-		return remoteState.Bootstrap(ctx, l, v, configbridge.RemoteStateOptsFromOpts(opts))
+		return remoteState.Bootstrap(ctx, l, v, configbridge.RemoteStateOptsFromOpts(v.Env, opts))
 	})
 }
 

internal/cli/commands/backend/delete/delete.go

@@ -44,7 +44,7 @@ func runDelete(
 			ctx,
 			l,
 			v,
-			configbridge.RemoteStateOptsFromOpts(opts),
+			configbridge.RemoteStateOptsFromOpts(v.Env, opts),
 		)
 		if err != nil && !errors.As(err, new(backend.BucketDoesNotExistError)) {
 			return err
@@ -63,7 +63,7 @@ func runDelete(
 		return fmt.Errorf("flag -%s is not supported yet", BucketFlagName)
 	}
 
-	return remoteState.Delete(ctx, l, v, configbridge.RemoteStateOptsFromOpts(opts))
+	return remoteState.Delete(ctx, l, v, configbridge.RemoteStateOptsFromOpts(v.Env, opts))
 }
 
 func runAll(

internal/cli/commands/backend/migrate/migrate.go

@@ -113,7 +113,7 @@ func Run(
 			ctx,
 			l,
 			srcV,
-			configbridge.RemoteStateOptsFromOpts(srcOpts),
+			configbridge.RemoteStateOptsFromOpts(v.Env, srcOpts),
 		)
 		if err != nil && !errors.As(err, new(backend.BucketDoesNotExistError)) {
 			return err
@@ -130,8 +130,8 @@ func Run(
 	return srcRemoteState.Migrate(
 		ctx, l,
 		srcV, dstV,
-		configbridge.RemoteStateOptsFromOpts(srcOpts),
-		configbridge.RemoteStateOptsFromOpts(dstOpts),
+		configbridge.RemoteStateOptsFromOpts(v.Env, srcOpts),
+		configbridge.RemoteStateOptsFromOpts(v.Env, dstOpts),
 		dstRemoteState,
 	)
 }

internal/cli/commands/browse/browse.go

@@ -73,7 +73,9 @@ func Run(ctx context.Context, l log.Logger, v *venv.Venv, opts *Options) error {
 		color = tui.ColorEnabled
 	}
 
-	err = tui.Run(ctx, l, v.FS, root, color, resultCh, warnCh)
+	v.RequireUserHomeDir()
+
+	err = tui.Run(ctx, l, v.FS, v.Platform.UserHomeDir, root, color, resultCh, warnCh)
 
 	cancel()
 	<-done