SOPS decode change porting to v0.99 (#5549)

Author: denis256

internal/runner/graph/graph.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gruntwork-io/terragrunt/internal/runner"
 	"github.com/gruntwork-io/terragrunt/internal/runner/common"
+	"github.com/gruntwork-io/terragrunt/internal/runner/run/creds"
 	"github.com/gruntwork-io/terragrunt/internal/runner/runall"
 
 	"github.com/gruntwork-io/terragrunt/internal/os/stdout"
@@ -21,6 +22,16 @@ import (
 )
 
 func Run(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) error {
+	// Get credentials BEFORE config parsing — sops_decrypt_file() and
+	// get_aws_account_id() in locals need auth-provider credentials
+	// available in opts.Env during HCL evaluation.
+	// *Getter discarded: graph.Run only needs creds in opts.Env for initial config parse.
+	// Per-unit creds are re-fetched in runnerpool task (intentional: each unit may have
+	// different opts after clone).
+	if _, err := creds.ObtainCredsForParsing(ctx, l, opts); err != nil {
+		return err
+	}
+
 	cfg, err := config.ReadTerragruntConfig(ctx, l, opts, config.DefaultParserOptions(l, opts))
 	if err != nil {
 		return err

internal/runner/run/creds/getter.go

@@ -6,6 +6,7 @@ import (
 	"maps"
 
 	"github.com/gruntwork-io/terragrunt/internal/runner/run/creds/providers"
+	"github.com/gruntwork-io/terragrunt/internal/runner/run/creds/providers/externalcmd"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/gruntwork-io/terragrunt/pkg/options"
 )
@@ -45,3 +46,16 @@ func (getter *Getter) ObtainAndUpdateEnvIfNecessary(ctx context.Context, l log.L
 
 	return nil
 }
+
+// ObtainCredsForParsing creates a new Getter, obtains external-command
+// credentials, and populates opts.Env before HCL parsing.
+// Use when sops_decrypt_file() or get_aws_account_id() may appear in locals.
+// See https://github.com/gruntwork-io/terragrunt/issues/5515
+func ObtainCredsForParsing(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) (*Getter, error) {
+	g := NewGetter()
+	if err := g.ObtainAndUpdateEnvIfNecessary(ctx, l, opts, externalcmd.NewProvider(l, opts)); err != nil {
+		return nil, err
+	}
+
+	return g, nil
+}

internal/runner/runnerpool/runner.go

@@ -22,7 +22,6 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/report"
 	"github.com/gruntwork-io/terragrunt/internal/runner/common"
 	"github.com/gruntwork-io/terragrunt/internal/runner/run/creds"
-	"github.com/gruntwork-io/terragrunt/internal/runner/run/creds/providers/externalcmd"
 	"github.com/gruntwork-io/terragrunt/internal/telemetry"
 	"github.com/gruntwork-io/terragrunt/pkg/config"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
@@ -491,6 +490,15 @@ func (r *Runner) Run(ctx context.Context, l log.Logger, opts *options.Terragrunt
 				unitLogger = l
 			}
 
+			// Get credentials BEFORE config parsing — sops_decrypt_file() and
+			// get_aws_account_id() in locals need auth-provider credentials
+			// available in opts.Env during HCL evaluation.
+			// See https://github.com/gruntwork-io/terragrunt/issues/5515
+			credsGetter, err := creds.ObtainCredsForParsing(childCtx, unitLogger, u.Execution.TerragruntOptions)
+			if err != nil {
+				return err
+			}
+
 			cfg, err := config.ReadTerragruntConfig(
 				childCtx,
 				unitLogger,
@@ -503,16 +511,6 @@ func (r *Runner) Run(ctx context.Context, l log.Logger, opts *options.Terragrunt
 
 			runCfg := cfg.ToRunConfig(unitLogger)
 
-			credsGetter := creds.NewGetter()
-			if err = credsGetter.ObtainAndUpdateEnvIfNecessary(
-				childCtx,
-				unitLogger,
-				u.Execution.TerragruntOptions,
-				externalcmd.NewProvider(unitLogger, u.Execution.TerragruntOptions),
-			); err != nil {
-				return err
-			}
-
 			err = unitRunner.Run(
 				childCtx,
 				u.Execution.TerragruntOptions,

pkg/config/config_helpers.go

@@ -1258,9 +1258,27 @@ func sopsDecryptFile(ctx context.Context, pctx *ParsingContext, l log.Logger, pa
 	var result string
 
 	err := telemetry.TelemeterFromContext(ctx).Collect(ctx, "hcl_fn_sops_decrypt_file", attrs, func(childCtx context.Context) error {
+		if len(params) != 1 {
+			return errors.New(WrongNumberOfParamsError{Func: "sops_decrypt_file", Expected: "1", Actual: len(params)})
+		}
+
+		format, err := getSopsFileFormat(sourceFile)
+		if err != nil {
+			return errors.New(err)
+		}
+
+		path := sourceFile
+
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(pctx.TerragruntOptions.WorkingDir, path)
+			path = filepath.Clean(path)
+		}
+
+		trackFileRead(pctx.FilesRead, path)
+
 		var innerErr error
 
-		result, innerErr = sopsDecryptFileImpl(childCtx, pctx, l, params)
+		result, innerErr = sopsDecryptFileImpl(childCtx, pctx, l, path, format, decrypt.File)
 
 		return innerErr
 	})
@@ -1269,60 +1287,59 @@ func sopsDecryptFile(ctx context.Context, pctx *ParsingContext, l log.Logger, pa
 }
 
 // sopsDecryptFileImpl contains the actual implementation of sopsDecryptFile
-func sopsDecryptFileImpl(ctx context.Context, pctx *ParsingContext, _ log.Logger, params []string) (string, error) {
-	numParams := len(params)
-
-	var sourceFile string
-
-	if numParams > 0 {
-		sourceFile = params[0]
-	}
+func sopsDecryptFileImpl(ctx context.Context, pctx *ParsingContext, l log.Logger, path string, format string, decryptFn func(string, string) ([]byte, error)) (string, error) {
+	// Fast path: check cache before acquiring lock.
+	// Cache has its own sync.RWMutex, safe for concurrent reads.
+	if val, ok := sopsCache.Get(ctx, path); ok {
+		l.Debugf("sops decrypt: cache hit for %s (len=%d)", path, len(val))
 
-	if numParams != 1 {
-		return "", errors.New(WrongNumberOfParamsError{Func: "sops_decrypt_file", Expected: "1", Actual: numParams})
+		return val, nil
 	}
 
-	format, err := getSopsFileFormat(sourceFile)
-	if err != nil {
-		return "", errors.New(err)
-	}
+	// Cache miss: acquire lock for env mutation + decrypt.
+	// The lock serializes os.Setenv/os.Unsetenv to prevent race conditions
+	// when multiple units decrypt concurrently with different auth credentials.
+	// See https://github.com/gruntwork-io/terragrunt/issues/5515
+	l.Debugf("sops decrypt: cache miss, acquiring lock for %s (format=%s)", path, format)
 
-	path := sourceFile
+	locks.EnvLock.Lock()
+	defer locks.EnvLock.Unlock()
 
-	if !filepath.IsAbs(path) {
-		path = filepath.Join(pctx.TerragruntOptions.WorkingDir, path)
-		path = filepath.Clean(path)
-	}
+	// Set env vars from opts.Env that are missing from process env.
+	// Auth-provider credentials (e.g., AWS_SESSION_TOKEN) may not exist
+	// in process env yet — SOPS needs them for KMS auth.
+	// Existing process env vars are preserved to avoid overriding real
+	// credentials with empty auth-provider values.
+	env := pctx.TerragruntOptions.Env
 
-	// Track that this file was read during parsing
-	trackFileRead(pctx.FilesRead, path)
+	var setKeys []string
 
-	// Set environment variables from the TerragruntOptions.Env map.
-	// This is especially useful for integrations with things like the `auth-provider` flag,
-	// which can set environment variables that are used for decryption.
-	//
-	// Due to the fact that sops doesn't expose a way of explicitly setting authentication configurations
-	// for decryption, we have to rely on environment variables to pass these configurations.
-	// This can cause a race condition, so we have to be careful to avoid having anything else
-	// running concurrently that might interfere with the environment variables.
-	env := pctx.TerragruntOptions.Env
-	if len(env) > 0 {
-		locks.EnvLock.Lock()
-		defer locks.EnvLock.Unlock()
-
-		for k, v := range env {
-			if os.Getenv(k) == "" {
-				os.Setenv(k, v)      //nolint:errcheck
-				defer os.Unsetenv(k) //nolint:errcheck
-			}
+	for k, v := range env {
+		if _, exists := os.LookupEnv(k); exists {
+			continue
 		}
+
+		os.Setenv(k, v) //nolint:errcheck
+
+		setKeys = append(setKeys, k)
 	}
 
+	defer func() {
+		for _, k := range setKeys {
+			os.Unsetenv(k) //nolint:errcheck
+		}
+	}()
+
+	// Double-check: another goroutine may have populated cache while we waited for the lock.
 	if val, ok := sopsCache.Get(ctx, path); ok {
+		l.Debugf("sops decrypt: cache hit after lock for %s (len=%d)", path, len(val))
+
 		return val, nil
 	}
 
-	rawData, err := decrypt.File(path, format)
+	l.Debugf("sops decrypt: decrypting %s", path)
+
+	rawData, err := decryptFn(path, format)
 	if err != nil {
 		return "", errors.New(extractSopsErrors(err))
 	}
@@ -1334,7 +1351,7 @@ func sopsDecryptFileImpl(ctx context.Context, pctx *ParsingContext, _ log.Logger
 		return value, nil
 	}
 
-	return "", errors.New(InvalidSopsFormatError{SourceFilePath: sourceFile})
+	return "", errors.New(InvalidSopsFormatError{SourceFilePath: path})
 }
 
 // Mapping of SOPS format to string