chore: Integrate vexec into engine (#5957)

Author: yhakbar

internal/engine/engine.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -24,6 +23,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/internal/github"
 	"github.com/gruntwork-io/terragrunt/internal/os/signal"
+	"github.com/gruntwork-io/terragrunt/internal/vexec"
 
 	"github.com/hashicorp/go-hclog"
 
@@ -90,10 +90,13 @@ type engineInstance struct {
 	execOptions  *ExecutionOptions
 }
 
-// Run executes the given command with the experimental engine.
+// Run executes the given command with the experimental engine. The provided
+// vexec.Exec is used to spawn the engine plugin subprocess and must be
+// OS-backed.
 func Run(
 	ctx context.Context,
 	l log.Logger,
+	e vexec.Exec,
 	execOptions *ExecutionOptions,
 ) (*util.CmdOutput, error) {
 	engineClients, err := engineClientsFromContext(ctx)
@@ -110,7 +113,7 @@ func Run(
 			return nil, errors.New(err)
 		}
 
-		terragruntEngine, client, createEngineErr := createEngine(ctx, l, execOptions)
+		terragruntEngine, client, createEngineErr := createEngine(ctx, l, e, execOptions)
 		if createEngineErr != nil {
 			return nil, errors.New(createEngineErr)
 		}
@@ -542,6 +545,7 @@ func logEngineMessage(l log.Logger, logLevel proto.LogLevel, content string) {
 func createEngine(
 	ctx context.Context,
 	l log.Logger,
+	e vexec.Exec,
 	execOptions *ExecutionOptions,
 ) (*proto.EngineClient, *plugin.Client, error) {
 	if execOptions.EngineConfig == nil {
@@ -597,23 +601,27 @@ func createEngine(
 	// We use without cancel here to ensure that the plugin isn't killed when the main context is cancelled,
 	// like it is in the RunCommandWithOutput function. This ensures that we don't cancel the shutdown
 	// when the command is cancelled.
-	cmd := exec.CommandContext(
-		context.WithoutCancel(ctx),
-		localEnginePath,
-	)
-	cmd.Cancel = func() error {
-		if cmd.Process == nil {
-			return nil
+	cmd := e.Command(context.WithoutCancel(ctx), localEnginePath)
+	cmd.SetEnv([]string{fmt.Sprintf("%s=%s", engineLogLevelEnv, engineLogLevel)})
+	cmd.SetCancel(func() error {
+		sig := signal.SignalFromContext(ctx)
+		if sig == nil {
+			sig = os.Kill
 		}
 
-		if sig := signal.SignalFromContext(ctx); sig != nil {
-			return cmd.Process.Signal(sig)
+		if err := cmd.Signal(sig); err != nil && !errors.Is(err, vexec.ErrProcessNotStarted) {
+			return err
 		}
 
-		return cmd.Process.Signal(os.Kill)
+		return nil
+	})
+
+	// hashicorp/go-plugin's ClientConfig requires a concrete *exec.Cmd.
+	osCmder, ok := cmd.(vexec.OSCmder)
+	if !ok {
+		return nil, nil, errors.Errorf("engine plugin spawn: %w", vexec.ErrNotOSBacked)
 	}
-	// pass log level to engine
-	cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", engineLogLevelEnv, engineLogLevel))
+
 	client := plugin.NewClient(&plugin.ClientConfig{
 		Logger: logger,
 		HandshakeConfig: plugin.HandshakeConfig{
@@ -624,7 +632,7 @@ func createEngine(
 		Plugins: map[string]plugin.Plugin{
 			"plugin": &engine.TerragruntGRPCEngine{},
 		},
-		Cmd: cmd,
+		Cmd: osCmder.OSCmd(),
 		GRPCDialOptions: []grpc.DialOption{
 			grpc.WithTransportCredentials(insecure.NewCredentials()),
 		},

internal/engine/engine_test.go

@@ -1,11 +1,16 @@
 package engine_test
 
 import (
+	"context"
 	"io"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/gruntwork-io/terragrunt/internal/engine"
+	"github.com/gruntwork-io/terragrunt/internal/vexec"
 	"github.com/gruntwork-io/terragrunt/internal/writer"
+	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -48,3 +53,34 @@ func TestReadEngineOutput(t *testing.T) {
 	err := engine.ReadEngineOutput(runOptions, false, outputFn)
 	assert.NoError(t, err)
 }
+
+func TestRun_NonOSBackedExecReturnsSentinel(t *testing.T) {
+	t.Parallel()
+
+	sourceFile := filepath.Join(t.TempDir(), "fake-engine")
+	require.NoError(t, os.WriteFile(sourceFile, []byte("not-a-real-engine"), 0o600))
+
+	ctx := engine.WithEngineValues(context.Background())
+
+	memExec := vexec.NewMemExec(func(_ context.Context, _ vexec.Invocation) vexec.Result {
+		return vexec.Result{}
+	})
+
+	opts := &engine.ExecutionOptions{
+		Writers: writer.Writers{Writer: io.Discard, ErrWriter: io.Discard},
+		EngineOptions: &engine.EngineOptions{
+			SkipChecksumCheck: true,
+			LogLevel:          "warn",
+		},
+		EngineConfig: &engine.EngineConfig{
+			Source:  sourceFile,
+			Version: "v0.0.0",
+			Type:    "test",
+		},
+		WorkingDir: t.TempDir(),
+	}
+
+	_, err := engine.Run(ctx, log.New(), memExec, opts)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, vexec.ErrNotOSBacked)
+}

internal/shell/run_cmd.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/engine"
 	"github.com/gruntwork-io/terragrunt/internal/experiment"
 	"github.com/gruntwork-io/terragrunt/internal/os/exec"
+	"github.com/gruntwork-io/terragrunt/internal/vexec"
 	"github.com/gruntwork-io/terragrunt/internal/writer"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 
@@ -219,7 +220,7 @@ func RunCommandWithOutput(
 			if runOpts.EngineConfig != nil && runOpts.Experiments.Evaluate(experiment.IacEngine) && !runOpts.NoEngine() {
 				l.Debugf("Using engine to run command: %s %s", command, strings.Join(args, " "))
 
-				cmdOutput, err := engine.Run(ctx, l, &engine.ExecutionOptions{
+				cmdOutput, err := engine.Run(ctx, l, vexec.NewOSExec(), &engine.ExecutionOptions{
 					Writers: writer.Writers{
 						Writer:                 writer.NewWrappedWriter(cmdStdout, runOpts.Writers.Writer),
 						ErrWriter:              writer.NewWrappedWriter(cmdStderr, runOpts.Writers.ErrWriter),

internal/vexec/vexec.go

@@ -31,6 +31,8 @@ var (
 	ErrStderrAlreadySet = errors.New("vexec: Stderr already set")
 	// ErrProcessNotStarted is returned from Signal before Start.
 	ErrProcessNotStarted = errors.New("vexec: process not started")
+	// ErrNotOSBacked reports that a Cmd does not satisfy OSCmder.
+	ErrNotOSBacked = errors.New("vexec: Cmd is not OS-backed")
 )
 
 // Exec is the process-execution interface used throughout the codebase.
@@ -83,6 +85,13 @@ type ExitCoder interface {
 	ExitCode() int
 }
 
+// OSCmder exposes the underlying *exec.Cmd of an OS-backed Cmd. It is
+// intended as an escape hatch for callers that must pass the concrete type
+// to a library that does not accept the Cmd interface.
+type OSCmder interface {
+	OSCmd() *exec.Cmd
+}
+
 // ExitCode extracts an exit code from err. It returns 0 if err is nil, or -1
 // if err does not carry an exit code.
 func ExitCode(err error) int {
@@ -140,6 +149,8 @@ func (c *osCmd) SetDir(dir string)     { c.cmd.Dir = dir }
 
 func (c *osCmd) SetCancel(fn func() error) { c.cmd.Cancel = fn }
 
+func (c *osCmd) OSCmd() *exec.Cmd { return c.cmd }
+
 func (c *osCmd) Signal(sig os.Signal) error {
 	if c.cmd.Process == nil {
 		return ErrProcessNotStarted

internal/vexec/vexec_test.go

@@ -48,6 +48,47 @@ func TestMemExec_HandlerReceivesInvocation(t *testing.T) {
 	assert.Equal(t, []byte("input"), out)
 }
 
+func TestOSCmder_OSBackendSatisfies(t *testing.T) {
+	t.Parallel()
+
+	cmd := vexec.NewOSExec().Command(t.Context(), "some-binary", "arg1")
+
+	oc, ok := cmd.(vexec.OSCmder)
+	require.True(t, ok, "OS-backed Cmd must satisfy OSCmder")
+
+	got := oc.OSCmd()
+	require.NotNil(t, got)
+	assert.Equal(t, []string{"some-binary", "arg1"}, got.Args)
+}
+
+func TestOSCmder_PropagatesSetters(t *testing.T) {
+	t.Parallel()
+
+	cmd := vexec.NewOSExec().Command(t.Context(), "some-binary", "arg1")
+
+	stdin := strings.NewReader("input")
+	cmd.SetStdin(stdin)
+	cmd.SetEnv([]string{"FOO=bar"})
+	cmd.SetDir("/work")
+
+	got := cmd.(vexec.OSCmder).OSCmd()
+	assert.Equal(t, []string{"some-binary", "arg1"}, got.Args)
+	assert.Equal(t, []string{"FOO=bar"}, got.Env)
+	assert.Equal(t, "/work", got.Dir)
+	assert.Same(t, stdin, got.Stdin)
+}
+
+func TestOSCmder_MemBackendDoesNot(t *testing.T) {
+	t.Parallel()
+
+	e := vexec.NewMemExec(func(_ context.Context, _ vexec.Invocation) vexec.Result {
+		return vexec.Result{}
+	})
+
+	_, ok := e.Command(t.Context(), "echo").(vexec.OSCmder)
+	assert.False(t, ok, "MemExec Cmd must not satisfy OSCmder")
+}
+
 func TestMemExec_CombinedOutput(t *testing.T) {
 	t.Parallel()