fix: support custom host blocks in Provider Cache Server (#5917)

* fix: support custom host blocks in Provider Cache Server

The Provider Cache Server now correctly handles providers from custom
registries configured via host blocks in .terraformrc / TF_CLI_CONFIG_FILE.

Changes:
- Add custom host names to RegistryNames so handlers are created for them
- Pre-populate discovery URL cache from host block services to avoid
  .well-known/terraform.json lookups for registries that don't support it
- Handle absolute URLs in providers.v1 service (e.g. from host blocks)
- Merge host services in AddHost instead of replacing to preserve endpoints
- Extract service discovery key literals into constants
- Set auth tokens for custom registries in providerCacheEnvironment

Fixes #5916

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

---------

Co-authored-by: elkh510 <elkh510@users.noreply.github.com>

Author: elkh510

internal/providercache/filter_registries_test.go

@@ -0,0 +1,103 @@
+package providercache_test
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terragrunt/internal/providercache"
+	"github.com/gruntwork-io/terragrunt/internal/tf/cliconfig"
+	"github.com/gruntwork-io/terragrunt/internal/tfimpl"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFilterRegistriesByImplementation(t *testing.T) {
+	t.Parallel()
+
+	defaultRegistries := []string{"registry.terraform.io", "registry.opentofu.org"}
+
+	tests := []struct {
+		name           string
+		registryNames  []string
+		implementation tfimpl.Type
+		expected       []string
+	}{
+		{
+			name:           "defaults + OpenTofu returns only opentofu registry",
+			registryNames:  defaultRegistries,
+			implementation: tfimpl.OpenTofu,
+			expected:       []string{"registry.opentofu.org"},
+		},
+		{
+			name:           "defaults + Terraform returns only terraform registry",
+			registryNames:  defaultRegistries,
+			implementation: tfimpl.Terraform,
+			expected:       []string{"registry.terraform.io"},
+		},
+		{
+			name:           "defaults + Unknown returns both",
+			registryNames:  defaultRegistries,
+			implementation: tfimpl.Unknown,
+			expected:       defaultRegistries,
+		},
+		{
+			name:           "user-replaced list returned as-is for OpenTofu",
+			registryNames:  []string{"registry.terraform.io"},
+			implementation: tfimpl.OpenTofu,
+			expected:       []string{"registry.terraform.io"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := providercache.FilterRegistriesByImplementation(tt.registryNames, tt.implementation)
+
+			assert.Equal(t, tt.expected, got)
+		})
+	}
+}
+
+func TestFilterRegistriesByImplementationWithCustomHosts(t *testing.T) {
+	t.Parallel()
+
+	withCustom := []string{"registry.terraform.io", "registry.opentofu.org", "nexus.corp"}
+
+	tests := []struct {
+		name           string
+		implementation tfimpl.Type
+		expected       []string
+	}{
+		{
+			name:           "custom host + OpenTofu: only opentofu + custom",
+			implementation: tfimpl.OpenTofu,
+			expected:       []string{"registry.opentofu.org", "nexus.corp"},
+		},
+		{
+			name:           "custom host + Terraform: only terraform + custom",
+			implementation: tfimpl.Terraform,
+			expected:       []string{"registry.terraform.io", "nexus.corp"},
+		},
+		{
+			name:           "custom host + Unknown: all three",
+			implementation: tfimpl.Unknown,
+			expected:       withCustom,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// Simulate what Init does: standard registries already in opts, custom host added separately.
+			// FilterRegistriesByImplementation must NOT receive custom hosts mixed in — it receives
+			// pc.opts.RegistryNames which stays clean; custom hosts come from cliCfg.Hosts.
+			baseRegistries := []string{"registry.terraform.io", "registry.opentofu.org"}
+			customHosts := []cliconfig.ConfigHost{{Name: "nexus.corp"}}
+
+			filtered := providercache.FilterRegistriesByImplementation(baseRegistries, tt.implementation)
+			got := providercache.AppendCustomHostRegistries(customHosts, filtered)
+
+			assert.Equal(t, tt.expected, got)
+		})
+	}
+}

internal/providercache/providercache.go

@@ -48,6 +48,10 @@ const (
 	// Retry configuration for registry operations during cache warm-up
 	registryRetryMaxAttempts   = 3
 	registryRetrySleepInterval = 5 * time.Second
+
+	// Terraform service discovery keys used in host blocks and registry URLs.
+	serviceProvidersV1 = "providers.v1"
+	serviceModulesV1   = "modules.v1"
 )
 
 var (
@@ -149,11 +153,20 @@ func (pc *ProviderCache) Init(l log.Logger, pcOpts *pcoptions.ProviderCacheOptio
 	providerService := services.NewProviderService(pcOpts.Dir, userProviderDir, cliCfg.CredentialsSource(), l, services.WithFS(pc.FS()))
 	proxyProviderHandler := handlers.NewProxyProviderHandler(l, cliCfg.CredentialsSource())
 
-	providerHandlers, err := handlers.NewProviderHandlers(cliCfg, l, pcOpts.RegistryNames)
+	// Custom hosts need handlers, but must not pollute pcOpts.RegistryNames — FilterRegistriesByImplementation
+	// relies on that slice containing only the standard registries to detect impl-based filtering.
+	// See: https://github.com/gruntwork-io/terragrunt/issues/5916
+	registryNamesForHandlers := AppendCustomHostRegistries(cliCfg.Hosts, pcOpts.RegistryNames)
+
+	providerHandlers, err := handlers.NewProviderHandlers(cliCfg, l, registryNamesForHandlers)
 	if err != nil {
 		return errors.Errorf("creating provider handlers failed: %w", err)
 	}
 
+	// Pre-populate discovery cache for custom hosts using service URLs from user config.
+	// This avoids .well-known/terraform.json lookups for registries that don't support it.
+	populateCustomHostDiscoveryCache(cliCfg.Hosts, providerHandlers)
+
 	cacheServer := cache.NewServer(
 		cache.WithHostname(pcOpts.Hostname),
 		cache.WithPort(pcOpts.Port),
@@ -377,27 +390,12 @@ func (pc *ProviderCache) createLocalCLIConfig(ctx context.Context, implementatio
 	cfg := pc.cliCfg.Clone()
 	cfg.PluginCacheDir = ""
 
-	// Filter registries based on OpenTofu or Terraform implementation to avoid contacting unnecessary registries
-	filteredRegistryNames := filterRegistriesByImplementation(
-		pc.opts.RegistryNames,
-		implementation,
-	)
-
-	var providerInstallationIncludes = make([]string, 0, len(filteredRegistryNames))
+	filteredRegistryNames := FilterRegistriesByImplementation(pc.opts.RegistryNames, implementation)
+	filteredRegistryNames = AppendCustomHostRegistries(pc.cliCfg.Hosts, filteredRegistryNames)
 
-	for _, registryName := range filteredRegistryNames {
-		providerInstallationIncludes = append(providerInstallationIncludes, registryName+"/*/*")
-
-		apiURLs, err := pc.DiscoveryURL(ctx, registryName)
-		if err != nil {
-			return err
-		}
-
-		cfg.AddHost(registryName, map[string]string{
-			"providers.v1": fmt.Sprintf("%s/%s/%s/", pc.ProviderController.URL(), cacheRequestID, registryName),
-			// Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
-			"modules.v1": ResolveModulesURL(registryName, apiURLs.ModulesV1),
-		})
+	providerInstallationIncludes, err := pc.configureRegistryHosts(ctx, cfg, filteredRegistryNames, cacheRequestID)
+	if err != nil {
+		return err
 	}
 
 	if cacheRequestID == "" {
@@ -412,7 +410,60 @@ func (pc *ProviderCache) createLocalCLIConfig(ctx context.Context, implementatio
 		cliconfig.NewProviderInstallationDirect(nil, nil),
 	)
 
-	// Use VFS for directory operations
+	return pc.saveCLIConfig(cfg, filename)
+}
+
+// configureRegistryHosts sets up host redirects for each registry, routing provider
+// requests through the cache server. Returns the list of provider installation includes.
+func (pc *ProviderCache) configureRegistryHosts(
+	ctx context.Context,
+	cfg *cliconfig.Config,
+	registryNames []string,
+	cacheRequestID string,
+) ([]string, error) {
+	includes := make([]string, 0, len(registryNames))
+
+	for _, registryName := range registryNames {
+		includes = append(includes, registryName+"/*/*")
+
+		modulesURL, err := pc.resolveModulesURL(ctx, registryName)
+		if err != nil {
+			return nil, err
+		}
+
+		hostServices := map[string]string{
+			serviceProvidersV1: fmt.Sprintf("%s/%s/%s/", pc.ProviderController.URL(), cacheRequestID, registryName),
+		}
+
+		if modulesURL != "" {
+			hostServices[serviceModulesV1] = modulesURL
+		}
+
+		cfg.AddHost(registryName, hostServices)
+	}
+
+	return includes, nil
+}
+
+// resolveModulesURL returns the modules URL for a registry. For custom hosts, it uses
+// the service URL from the host block directly. For standard registries, it performs discovery.
+func (pc *ProviderCache) resolveModulesURL(ctx context.Context, registryName string) (string, error) {
+	for _, host := range pc.cliCfg.Hosts {
+		if host.Name == registryName {
+			return host.Services[serviceModulesV1], nil
+		}
+	}
+
+	apiURLs, err := pc.DiscoveryURL(ctx, registryName)
+	if err != nil {
+		return "", err
+	}
+
+	return ResolveModulesURL(registryName, apiURLs.ModulesV1), nil
+}
+
+// saveCLIConfig writes the CLI config to disk, creating the directory if needed.
+func (pc *ProviderCache) saveCLIConfig(cfg *cliconfig.Config, filename string) error {
 	fs := pc.FS()
 	cfgDir := filepath.Dir(filename)
 
@@ -521,11 +572,14 @@ func (pc *ProviderCache) providerCacheEnvironment(env map[string]string, impleme
 	maps.Copy(envs, env)
 
 	// Filter registries based on OpenTofu or Terraform implementation to avoid setting env vars for unnecessary registries
-	filteredRegistryNames := filterRegistriesByImplementation(
+	filteredRegistryNames := FilterRegistriesByImplementation(
 		pc.opts.RegistryNames,
 		implementation,
 	)
 
+	// Include custom host blocks so auth tokens are set for them too.
+	filteredRegistryNames = AppendCustomHostRegistries(pc.cliCfg.Hosts, filteredRegistryNames)
+
 	for _, registryName := range filteredRegistryNames {
 		envName := fmt.Sprintf(tf.EnvNameTFTokenFmt, strings.ReplaceAll(registryName, ".", "_"))
 
@@ -588,15 +642,49 @@ func convertToMultipleCommandsByPlatforms(args []string) [][]string {
 	return commandsArgs
 }
 
-// filterRegistriesByImplementation filters registry names based on the Terraform implementation being used.
+// AppendCustomHostRegistries adds custom host names from user config to the registry list
+// if they are not already present. This ensures the cache server handles them.
+// See: https://github.com/gruntwork-io/terragrunt/issues/5916
+func AppendCustomHostRegistries(hosts []cliconfig.ConfigHost, registryNames []string) []string {
+	toAdd := make([]string, 0, len(hosts))
+
+	for _, host := range hosts {
+		if !slices.Contains(registryNames, host.Name) {
+			toAdd = append(toAdd, host.Name)
+		}
+	}
+
+	return slices.Concat(registryNames, toAdd)
+}
+
+// populateCustomHostDiscoveryCache pre-populates the discovery URL cache for custom hosts
+// using service URLs from user config, avoiding .well-known/terraform.json lookups.
+func populateCustomHostDiscoveryCache(hosts []cliconfig.ConfigHost, providerHandlers handlers.ProviderHandlers) {
+	for _, host := range hosts {
+		providersURL, hasProviders := host.Services[serviceProvidersV1]
+		if !hasProviders {
+			continue
+		}
+
+		urls := &handlers.RegistryURLs{ProvidersV1: providersURL}
+
+		if v, ok := host.Services[serviceModulesV1]; ok {
+			urls.ModulesV1 = v
+		}
+
+		providerHandlers.SetDiscoveryURLCache(host.Name, urls)
+	}
+}
+
+// FilterRegistriesByImplementation filters registry names based on the Terraform implementation being used.
 // If the registry names match the default registries (both registry.terraform.io and registry.opentofu.org),
 // it filters them based on the implementation:
 //   - OpenTofuImpl: returns only registry.opentofu.org
 //   - TerraformImpl: returns only registry.terraform.io
 //   - UnknownImpl: returns both (backward compatibility)
 //
 // If the user has explicitly set registry names (don't match defaults), returns them as-is.
-func filterRegistriesByImplementation(registryNames []string, implementation tfimpl.Type) []string {
+func FilterRegistriesByImplementation(registryNames []string, implementation tfimpl.Type) []string {
 	// Default registries in the same order as defined in options/options.go
 	defaultRegistries := []string{
 		"registry.terraform.io",

internal/tf/cache/handlers/common_provider.go

@@ -12,10 +12,10 @@ import (
 type CommonProviderHandler struct {
 	logger log.Logger
 
-	// registryURLCache stores discovered registry URLs
+	// discoveryURLCache stores discovered registry URLs
 	// We use [xsync.MapOf](https://github.com/puzpuzpuz/xsync?tab=readme-ov-file#map)
 	// instead of standard `sync.Map` since it's faster and has generic types.
-	registryURLCache *xsync.MapOf[string, *RegistryURLs]
+	discoveryURLCache *xsync.MapOf[string, *RegistryURLs]
 
 	// includeProviders and excludeProviders are sets of provider matching patterns that together define which providers are eligible to be potentially installed from the corresponding Source.
 	includeProviders models.Providers
@@ -35,10 +35,10 @@ func NewCommonProviderHandler(logger log.Logger, includes, excludes *[]string) *
 	}
 
 	return &CommonProviderHandler{
-		logger:           logger,
-		includeProviders: includeProviders,
-		excludeProviders: excludeProviders,
-		registryURLCache: xsync.NewMapOf[string, *RegistryURLs](),
+		logger:            logger,
+		includeProviders:  includeProviders,
+		excludeProviders:  excludeProviders,
+		discoveryURLCache: xsync.NewMapOf[string, *RegistryURLs](),
 	}
 }
 
@@ -54,9 +54,14 @@ func (handler *CommonProviderHandler) CanHandleProvider(provider *models.Provide
 	}
 }
 
+// SetDiscoveryURLCache pre-populates the discovery cache for a given registry.
+func (handler *CommonProviderHandler) SetDiscoveryURLCache(registryName string, urls *RegistryURLs) {
+	handler.discoveryURLCache.Store(registryName, urls)
+}
+
 // DiscoveryURL implements ProviderHandler.DiscoveryURL.
 func (handler *CommonProviderHandler) DiscoveryURL(ctx context.Context, registryName string) (*RegistryURLs, error) {
-	if urls, ok := handler.registryURLCache.Load(registryName); ok {
+	if urls, ok := handler.discoveryURLCache.Load(registryName); ok {
 		return urls, nil
 	}
 
@@ -72,7 +77,7 @@ func (handler *CommonProviderHandler) DiscoveryURL(ctx context.Context, registry
 		handler.logger.Debugf("Discovered %q registry URLs: %s", registryName, urls)
 	}
 
-	handler.registryURLCache.Store(registryName, urls)
+	handler.discoveryURLCache.Store(registryName, urls)
 
 	return urls, nil
 }