fix: Propagate original credential configs for requests to modules

Author: yhakbar

internal/providercache/nested_modules_credentials_test.go

@@ -0,0 +1,161 @@
+package providercache_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/gruntwork-io/terragrunt/internal/providercache"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cache"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cache/handlers"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cache/services"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cliconfig"
+	"github.com/gruntwork-io/terragrunt/test/helpers"
+	"github.com/gruntwork-io/terragrunt/test/helpers/logger"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+// fakeDiscoverer pretends the upstream registry advertised the given modules.v1
+// path during well-known discovery. Used to bypass real network discovery in tests.
+type fakeDiscoverer struct {
+	modulesV1 string
+}
+
+func (d *fakeDiscoverer) DiscoveryURL(_ context.Context, _ string) (*handlers.RegistryURLs, error) {
+	return &handlers.RegistryURLs{
+		ProvidersV1: "/v1/providers",
+		ModulesV1:   d.modulesV1,
+	}, nil
+}
+
+// TestNestedModuleCredentials reproduces issue #5970: when TG_PROVIDER_CACHE is on,
+// the cache server was forwarding nested module-registry requests with its own
+// x-api-key bearer token instead of the user's real upstream credentials, causing
+// 403s. The cache server must strip its own auth header and re-inject the user's
+// configured credentials when proxying modules.v1 requests upstream.
+func TestNestedModuleCredentials(t *testing.T) {
+	t.Parallel()
+
+	const realUserToken = "real-user-token"
+
+	var (
+		upstreamHits   atomic.Int32
+		upstreamAuth   atomic.Value
+		upstreamReject atomic.Int32
+	)
+
+	const versionsBody = `{"modules":[{"versions":[{"version":"0.1.0"},{"version":"0.2.0"}]}]}`
+
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		auth := r.Header.Get("Authorization")
+		upstreamAuth.Store(auth)
+		upstreamHits.Add(1)
+
+		if auth != "Bearer "+realUserToken {
+			upstreamReject.Add(1)
+			w.WriteHeader(http.StatusForbidden)
+
+			return
+		}
+
+		switch r.URL.Path {
+		case "/v1/modules/private/lambda/aws/versions":
+			w.Header().Set("Content-Type", "application/json")
+
+			if _, err := io.WriteString(w, versionsBody); err != nil {
+				t.Errorf("upstream write failed: %v", err)
+			}
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(upstream.Close)
+
+	registryName := strings.TrimPrefix(upstream.URL, "http://")
+
+	// Build a credentials source that has the user's real token for the upstream host.
+	cliCfg := &cliconfig.Config{
+		Credentials: []cliconfig.ConfigCredentials{
+			{Name: "127.0.0.1", Token: realUserToken},
+		},
+	}
+	credsSource := cliCfg.CredentialsSource()
+
+	// The fake discoverer returns the upstream's full URL as modules.v1, so the
+	// proxy targets the httptest server (HTTP, not HTTPS) without DNS lookups.
+	discoverer := &fakeDiscoverer{modulesV1: upstream.URL + "/v1/modules/"}
+
+	cacheToken := fmt.Sprintf("%s:%s", providercache.APIKeyAuth, uuid.New().String())
+
+	providerCacheDir := helpers.TmpDirWOSymlinks(t)
+	pluginCacheDir := helpers.TmpDirWOSymlinks(t)
+
+	l := logger.CreateLogger()
+	providerService := services.NewProviderService(providerCacheDir, pluginCacheDir, nil, l)
+	proxyProviderHandler := handlers.NewProxyProviderHandler(l, credsSource)
+	proxyModuleHandler := handlers.NewProxyModuleHandler(l, credsSource, discoverer)
+
+	server := cache.NewServer(
+		cache.WithToken(cacheToken),
+		cache.WithProviderService(providerService),
+		cache.WithProxyProviderHandler(proxyProviderHandler),
+		cache.WithProxyModuleHandler(proxyModuleHandler),
+		cache.WithCacheProviderHTTPStatusCode(providercache.CacheProviderHTTPStatusCode),
+		cache.WithLogger(l),
+	)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+
+	ln, err := server.Listen(ctx)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		if err := ln.Close(); err != nil && !errors.Is(err, net.ErrClosed) {
+			t.Errorf("listener close failed: %v", err)
+		}
+	})
+
+	g, gctx := errgroup.WithContext(ctx)
+	g.Go(func() error { return server.Run(gctx, ln) })
+
+	// Build the same URL OpenTofu/Terraform would hit via the host block:
+	// <cache server>/v1/modules/<cache_request_id>/<registry>/<module path>
+	moduleURL := server.ModuleController.URL()
+	moduleURL.Path += "/" + uuid.New().String() + "/" + registryName + "/private/lambda/aws/versions"
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, moduleURL.String(), nil)
+	require.NoError(t, err)
+	// OpenTofu sends the host block's TF_TOKEN_<host> value, which Terragrunt has
+	// rewritten to the cache server's API key. The cache server must NOT forward
+	// this token upstream; it must look up the user's real token instead.
+	req.Header.Set("Authorization", "Bearer "+cacheToken)
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, resp.Body.Close())
+	require.NoError(t, err)
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "expected upstream success; body=%s", string(body))
+	assert.JSONEq(t, versionsBody, string(body))
+
+	assert.Equal(t, int32(1), upstreamHits.Load(), "upstream registry should have been hit exactly once")
+	assert.Equal(t, int32(0), upstreamReject.Load(), "upstream registry should not have rejected the request")
+	assert.Equal(t, "Bearer "+realUserToken, upstreamAuth.Load(),
+		"cache server must forward the user's real upstream credentials, not its own API key")
+
+	cancel()
+	require.NoError(t, g.Wait())
+}

internal/providercache/providercache.go

@@ -167,13 +167,16 @@ func (pc *ProviderCache) Init(l log.Logger, pcOpts *pcoptions.ProviderCacheOptio
 	// This avoids .well-known/terraform.json lookups for registries that don't support it.
 	populateCustomHostDiscoveryCache(cliCfg.Hosts, providerHandlers)
 
+	proxyModuleHandler := handlers.NewProxyModuleHandler(l, cliCfg.CredentialsSource(), providerHandlers)
+
 	cacheServer := cache.NewServer(
 		cache.WithHostname(pcOpts.Hostname),
 		cache.WithPort(pcOpts.Port),
 		cache.WithToken(pcOpts.Token),
 		cache.WithProviderService(providerService),
 		cache.WithProviderHandlers(providerHandlers...),
 		cache.WithProxyProviderHandler(proxyProviderHandler),
+		cache.WithProxyModuleHandler(proxyModuleHandler),
 		cache.WithCacheProviderHTTPStatusCode(CacheProviderHTTPStatusCode),
 		cache.WithLogger(l),
 	)
@@ -413,8 +416,9 @@ func (pc *ProviderCache) createLocalCLIConfig(ctx context.Context, implementatio
 	return pc.saveCLIConfig(cfg, filename)
 }
 
-// configureRegistryHosts sets up host redirects for each registry, routing provider
-// requests through the cache server. Returns the list of provider installation includes.
+// configureRegistryHosts sets up host redirects for each registry, routing both
+// provider and module requests through the cache server. Returns the list of
+// provider installation includes.
 func (pc *ProviderCache) configureRegistryHosts(
 	ctx context.Context,
 	cfg *cliconfig.Config,
@@ -426,7 +430,7 @@ func (pc *ProviderCache) configureRegistryHosts(
 	for _, registryName := range registryNames {
 		includes = append(includes, registryName+"/*/*")
 
-		modulesURL, err := pc.resolveModulesURL(ctx, registryName)
+		hasModules, err := pc.registrySupportsModules(ctx, registryName)
 		if err != nil {
 			return nil, err
 		}
@@ -435,8 +439,11 @@ func (pc *ProviderCache) configureRegistryHosts(
 			serviceProvidersV1: fmt.Sprintf("%s/%s/%s/", pc.ProviderController.URL(), cacheRequestID, registryName),
 		}
 
-		if modulesURL != "" {
-			hostServices[serviceModulesV1] = modulesURL
+		if hasModules {
+			// Route module requests through the cache server so it can swap the
+			// cache server's API key (which TF_TOKEN_<host> is forced to) back out
+			// for the user's real upstream credentials before forwarding upstream.
+			hostServices[serviceModulesV1] = fmt.Sprintf("%s/%s/%s/", pc.ModuleController.URL(), cacheRequestID, registryName)
 		}
 
 		cfg.AddHost(registryName, hostServices)
@@ -445,21 +452,22 @@ func (pc *ProviderCache) configureRegistryHosts(
 	return includes, nil
 }
 
-// resolveModulesURL returns the modules URL for a registry. For custom hosts, it uses
-// the service URL from the host block directly. For standard registries, it performs discovery.
-func (pc *ProviderCache) resolveModulesURL(ctx context.Context, registryName string) (string, error) {
+// registrySupportsModules reports whether the registry advertises a modules.v1
+// endpoint. For custom hosts it consults the host block; for standard registries
+// it performs discovery (using the populated discovery cache where available).
+func (pc *ProviderCache) registrySupportsModules(ctx context.Context, registryName string) (bool, error) {
 	for _, host := range pc.cliCfg.Hosts {
 		if host.Name == registryName {
-			return host.Services[serviceModulesV1], nil
+			return host.Services[serviceModulesV1] != "", nil
 		}
 	}
 
 	apiURLs, err := pc.DiscoveryURL(ctx, registryName)
 	if err != nil {
-		return "", err
+		return false, err
 	}
 
-	return ResolveModulesURL(registryName, apiURLs.ModulesV1), nil
+	return apiURLs.ModulesV1 != "", nil
 }
 
 // saveCLIConfig writes the CLI config to disk, creating the directory if needed.
@@ -722,14 +730,3 @@ func FilterRegistriesByImplementation(registryNames []string, implementation tfi
 	// User explicitly set registry names, return as-is
 	return registryNames
 }
-
-// ResolveModulesURL resolves the modules.v1 URL from registry discovery.
-// If the URL is already absolute (contains "://"), it is returned as-is.
-// Otherwise, it is treated as a relative path and combined with the registry name.
-func ResolveModulesURL(registryName, modulesV1 string) string {
-	if strings.Contains(modulesV1, "://") {
-		return modulesV1
-	}
-
-	return fmt.Sprintf("https://%s%s", registryName, modulesV1)
-}

internal/providercache/resolve_modules_url_test.go

@@ -65,6 +65,13 @@ func WithProxyProviderHandler(handler *handlers.ProxyProviderHandler) Option {
 	}
 }
 
+func WithProxyModuleHandler(handler *handlers.ProxyModuleHandler) Option {
+	return func(cfg Config) Config {
+		cfg.proxyModuleHandler = handler
+		return cfg
+	}
+}
+
 func WithCacheProviderHTTPStatusCode(statusCode int) Option {
 	return func(cfg Config) Config {
 		cfg.cacheProviderHTTPStatusCode = statusCode
@@ -83,6 +90,7 @@ type Config struct {
 	logger                      log.Logger
 	providerService             *services.ProviderService
 	proxyProviderHandler        *handlers.ProxyProviderHandler
+	proxyModuleHandler          *handlers.ProxyModuleHandler
 	hostname                    string
 	token                       string
 	providerHandlers            handlers.ProviderHandlers

internal/tf/cache/config.go

@@ -0,0 +1,49 @@
+package controllers
+
+import (
+	"github.com/gruntwork-io/terragrunt/internal/tf/cache/handlers"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cache/router"
+	"github.com/gruntwork-io/terragrunt/pkg/log"
+	"github.com/labstack/echo/v4"
+)
+
+const (
+	moduleName = "modules.v1"
+	modulePath = "/modules"
+)
+
+// ModuleController exposes the modules.v1 registry protocol on the Terragrunt
+// cache server. It accepts requests authenticated with the cache server's API key
+// and forwards them to the upstream registry with the user's real credentials,
+// fixing 403s on nested module lookups when the user-set TF_TOKEN_<host> is
+// overridden by the cache server's token.
+type ModuleController struct {
+	*router.Router
+
+	AuthMiddleware     echo.MiddlewareFunc
+	ProxyModuleHandler *handlers.ProxyModuleHandler
+	Logger             log.Logger
+}
+
+// Endpoints implements controllers.Endpointer.
+func (c *ModuleController) Endpoints() map[string]any {
+	return map[string]any{moduleName: c.URL().Path}
+}
+
+// Register implements router.Controller.
+func (c *ModuleController) Register(r *router.Router) {
+	c.Router = r.Group(modulePath)
+
+	if c.AuthMiddleware != nil {
+		c.Use(c.AuthMiddleware)
+	}
+
+	c.GET("/:cache_request_id/:registry_name/*", c.proxyAction)
+}
+
+func (c *ModuleController) proxyAction(ctx echo.Context) error {
+	registryName := ctx.Param("registry_name")
+	rest := ctx.Param("*")
+
+	return c.ProxyModuleHandler.Proxy(ctx, registryName, rest)
+}