chore: Wiring expansion into the unit and stack parse

Author: yhakbar

pkg/config/expansion_test.go

@@ -38,7 +38,7 @@ unit "app" {
   }
 
   source = "./modules/app"
-  path   = "app"
+  path   = "app/${each.key}"
 }
 `
 
@@ -55,7 +55,7 @@ stack "team" {
   }
 
   source = "./stacks/team"
-  path   = "team"
+  path   = "team/${count.index}"
 }
 `
 

pkg/config/stack.go

@@ -48,10 +48,21 @@ const (
 
 // StackConfigFile represents the structure of terragrunt.stack.hcl stack file.
 type StackConfigFile struct {
-	Locals   *terragruntLocal    `hcl:"locals,block"`
-	Includes []*StackIncludeFile `hcl:"include,block"`
-	Stacks   []*Stack            `hcl:"stack,block"`
-	Units    []*Unit             `hcl:"unit,block"`
+	Locals      *terragruntLocal    `hcl:"locals,block"`
+	Includes    []*StackIncludeFile `hcl:"include,block"`
+	StackBlocks []componentHeader   `hcl:"stack,block"`
+	UnitBlocks  []componentHeader   `hcl:"unit,block"`
+	Stacks      []*Stack
+	Units       []*Unit
+}
+
+// componentHeader keeps `unit` and `stack` in the stack file's schema without evaluating the
+// block body. Source, path and values are attributes, so a whole-file decode resolves them
+// before expansion can bind each.*/count.index, and a block referencing either fails on an
+// undefined variable.
+type componentHeader struct {
+	Remain hcl.Body `hcl:",remain"`
+	Name   string   `hcl:",label"`
 }
 
 // StackIncludeFile represents an include block in a stack file.
@@ -1162,6 +1173,11 @@ func ParseStackConfig(
 		return nil, decodeErr
 	}
 
+	config.Units, config.Stacks, err = decodeComponents(file, evalParsingContext)
+	if err != nil {
+		return nil, err
+	}
+
 	// Process include blocks and merge any generated stack-level autoinclude file.
 	stackDir := filepath.Dir(file.ConfigPath)
 
@@ -1213,15 +1229,6 @@ func ParseStackConfig(
 	return stackConfig, nil
 }
 
-// stackComponentHeaders captures only the label and path of each unit/stack block
-// so component paths can be resolved before the full decode evaluates values.
-// source, values, and every other attribute are left in the block body, unevaluated.
-type stackComponentHeaders struct {
-	Remain hcl.Body                `hcl:",remain"`
-	Stacks []*stackComponentHeader `hcl:"stack,block"`
-	Units  []*stackComponentHeader `hcl:"unit,block"`
-}
-
 // stackComponentHeader is the path-only shape of a unit or stack block.
 type stackComponentHeader struct {
 	Remain  hcl.Body `hcl:",remain"`
@@ -1249,14 +1256,14 @@ func injectStackComponentRefs(
 	stackDir string,
 	parserOpts []hclparse.Option,
 ) error {
-	headers := &stackComponentHeaders{}
-	if err := file.Decode(headers, evalCtx); err != nil {
+	baseUnits, baseStacks, err := decodeComponentHeaders(file, evalCtx)
+	if err != nil {
 		return err
 	}
 
 	// Publish the base refs first so a sibling autoinclude block whose path references unit.<name>.path /
 	// stack.<name>.path can resolve against the base components, matching how the full decode resolves them.
-	setStackComponentRefVars(evalCtx, stackDir, headers.Units, headers.Stacks)
+	setStackComponentRefVars(evalCtx, stackDir, baseUnits, baseStacks)
 
 	autoUnits, autoStacks, err := stackAutoIncludeComponentHeaders(
 		fsys,
@@ -1269,8 +1276,8 @@ func injectStackComponentRefs(
 	}
 
 	// Republish so an overridden component's path reflects the override, not the base path it replaced.
-	units := util.MergeNamed(headers.Units, autoUnits, componentHeaderName)
-	stacks := util.MergeNamed(headers.Stacks, autoStacks, componentHeaderName)
+	units := util.MergeNamed(baseUnits, autoUnits, componentHeaderName)
+	stacks := util.MergeNamed(baseStacks, autoStacks, componentHeaderName)
 	setStackComponentRefVars(evalCtx, stackDir, units, stacks)
 
 	return nil
@@ -1336,16 +1343,69 @@ func stackAutoIncludeComponentHeaders(
 		return nil, nil, fmt.Errorf("failed to read stack autoinclude %q: %w", autoIncludePath, err)
 	}
 
-	headers := &stackComponentHeaders{}
-	if decodeErr := incFile.Decode(headers, evalCtx); decodeErr != nil {
+	autoUnits, autoStacks, decodeErr := decodeComponentHeaders(incFile, evalCtx)
+	if decodeErr != nil {
 		return nil, nil, fmt.Errorf(
 			"failed to decode stack autoinclude headers %q: %w",
 			autoIncludePath,
 			decodeErr,
 		)
 	}
 
-	return headers.Units, headers.Stacks, nil
+	return autoUnits, autoStacks, nil
+}
+
+// decodeComponentHeaders reads the label and path of each unit and stack block. Blocks are
+// expanded so that a path referencing each.*/count.index still decodes, and only unexpanded
+// components come back: a component reference names a whole block, which an expanded one has
+// no single path to answer for.
+func decodeComponentHeaders(
+	file *hclparse.File,
+	evalCtx *hcl.EvalContext,
+) ([]*stackComponentHeader, []*stackComponentHeader, error) {
+	units, err := expandComponentHeaders(file, MetadataUnit, evalCtx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	stacks, err := expandComponentHeaders(file, MetadataStack, evalCtx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return units, stacks, nil
+}
+
+func expandComponentHeaders(
+	file *hclparse.File,
+	blockType string,
+	evalCtx *hcl.EvalContext,
+) ([]*stackComponentHeader, error) {
+	instances, err := file.ExpandBlocks(blockType, &stackComponentHeader{}, evalCtx)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := make([]*stackComponentHeader, 0, len(instances))
+
+	for _, instance := range instances {
+		if instance.Expanded() {
+			continue
+		}
+
+		header, ok := instance.Value.(*stackComponentHeader)
+		if !ok {
+			panic(fmt.Sprintf(
+				"ExpandBlocks returned %T for a %s block, but it decodes into the type it is given",
+				instance.Value,
+				blockType,
+			))
+		}
+
+		headers = append(headers, header)
+	}
+
+	return headers, nil
 }
 
 // componentHeaderName returns a header's block name, or an empty string for a nil entry so MergeNamed leaves it untouched.
@@ -1462,6 +1522,93 @@ func pruneOverriddenStackAutoIncludes(
 	return nil
 }
 
+// componentAddress identifies one instance of a unit or stack block. Every instance of an
+// expanded block carries its label, so the label alone would fold a whole set into one entry.
+func componentAddress(name string, expansion *hclparse.ExpansionBlock) string {
+	if expansion == nil || !expansion.Expanded() {
+		return name
+	}
+
+	return name + "[" + expansion.Key() + "]"
+}
+
+// decodeComponents decodes a stack file's unit and stack blocks, returning one value per
+// iteration element. A block that declares no expansion yields a single value.
+func decodeComponents(
+	file *hclparse.File,
+	evalCtx *hcl.EvalContext,
+) ([]*Unit, []*Stack, error) {
+	units, err := decodeUnitBlocks(file, evalCtx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	stacks, err := decodeStackBlocks(file, evalCtx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return units, stacks, nil
+}
+
+// decodeUnitBlocks decodes a stack file's unit blocks, returning one Unit per iteration
+// element. A block that declares no expansion yields a single Unit.
+func decodeUnitBlocks(file *hclparse.File, evalContext *hcl.EvalContext) ([]*Unit, error) {
+	instances, err := file.ExpandBlocks(MetadataUnit, &Unit{}, evalContext)
+	if err != nil {
+		return nil, err
+	}
+
+	units := make([]*Unit, 0, len(instances))
+
+	for _, instance := range instances {
+		unit, ok := instance.Value.(*Unit)
+		if !ok {
+			panic(fmt.Sprintf(
+				"ExpandBlocks returned %T for a unit block, but it decodes into the type it is given",
+				instance.Value,
+			))
+		}
+
+		if unit.Expansion != nil {
+			unit.Expansion.InstanceKey = instance.InstanceKey
+		}
+
+		units = append(units, unit)
+	}
+
+	return units, nil
+}
+
+// decodeStackBlocks decodes a stack file's stack blocks, returning one Stack per iteration
+// element. A block that declares no expansion yields a single Stack.
+func decodeStackBlocks(file *hclparse.File, evalContext *hcl.EvalContext) ([]*Stack, error) {
+	instances, err := file.ExpandBlocks(MetadataStack, &Stack{}, evalContext)
+	if err != nil {
+		return nil, err
+	}
+
+	stacks := make([]*Stack, 0, len(instances))
+
+	for _, instance := range instances {
+		stack, ok := instance.Value.(*Stack)
+		if !ok {
+			panic(fmt.Sprintf(
+				"ExpandBlocks returned %T for a stack block, but it decodes into the type it is given",
+				instance.Value,
+			))
+		}
+
+		if stack.Expansion != nil {
+			stack.Expansion.InstanceKey = instance.InstanceKey
+		}
+
+		stacks = append(stacks, stack)
+	}
+
+	return stacks, nil
+}
+
 // processStackConfigIncludes resolves include blocks during stack file parsing.
 // It reads each included file, parses it with the same eval context, and merges
 // its units and stacks into the main config so generation sees all components,
@@ -1494,6 +1641,11 @@ func processStackConfigIncludes(
 			return fmt.Errorf("failed to decode include %q: %w", inc.Name, decodeErr)
 		}
 
+		included.Units, included.Stacks, err = decodeComponents(incFile, evalCtx)
+		if err != nil {
+			return fmt.Errorf("failed to decode include %q: %w", inc.Name, err)
+		}
+
 		if included.Locals != nil {
 			return fmt.Errorf("included stack file %q must not define locals", inc.Name)
 		}
@@ -1510,22 +1662,24 @@ func processStackConfigIncludes(
 	seen := make(map[string]struct{}, len(config.Units))
 
 	for _, u := range config.Units {
-		if _, exists := seen[u.Name]; exists {
+		address := componentAddress(u.Name, u.Expansion)
+		if _, exists := seen[address]; exists {
 			return inthclparse.DuplicateUnitNameError{Name: u.Name}
 		}
 
-		seen[u.Name] = struct{}{}
+		seen[address] = struct{}{}
 	}
 
 	// Validate no duplicate stack names after merge.
 	seen = make(map[string]struct{}, len(config.Stacks))
 
 	for _, s := range config.Stacks {
-		if _, exists := seen[s.Name]; exists {
+		address := componentAddress(s.Name, s.Expansion)
+		if _, exists := seen[address]; exists {
 			return inthclparse.DuplicateStackNameError{Name: s.Name}
 		}
 
-		seen[s.Name] = struct{}{}
+		seen[address] = struct{}{}
 	}
 
 	return nil
@@ -1588,6 +1742,11 @@ func mergeStackAutoIncludeFile(
 		return fmt.Errorf("failed to decode stack autoinclude %q: %w", autoIncludePath, decodeErr)
 	}
 
+	included.Units, included.Stacks, err = decodeComponents(incFile, evalCtx)
+	if err != nil {
+		return fmt.Errorf("failed to decode stack autoinclude %q: %w", autoIncludePath, err)
+	}
+
 	if included.Locals != nil {
 		return fmt.Errorf("stack autoinclude %q must not define locals", autoIncludePath)
 	}
@@ -1621,11 +1780,12 @@ func validateUniqueComponentNames(units []*Unit, stacks []*Stack) error {
 			continue
 		}
 
-		if _, dup := seenUnits[u.Name]; dup {
+		address := componentAddress(u.Name, u.Expansion)
+		if _, dup := seenUnits[address]; dup {
 			return inthclparse.DuplicateUnitNameError{Name: u.Name}
 		}
 
-		seenUnits[u.Name] = struct{}{}
+		seenUnits[address] = struct{}{}
 	}
 
 	seenStacks := make(map[string]struct{}, len(stacks))
@@ -1635,11 +1795,12 @@ func validateUniqueComponentNames(units []*Unit, stacks []*Stack) error {
 			continue
 		}
 
-		if _, dup := seenStacks[s.Name]; dup {
+		address := componentAddress(s.Name, s.Expansion)
+		if _, dup := seenStacks[address]; dup {
 			return inthclparse.DuplicateStackNameError{Name: s.Name}
 		}
 
-		seenStacks[s.Name] = struct{}{}
+		seenStacks[address] = struct{}{}
 	}
 
 	return nil