Skip to content

Commit 79a8d8b

Browse files
denis256denis256
committed
Discovery includes
1 parent 0112c2b commit 79a8d8b

4 files changed

Lines changed: 293 additions & 5 deletions

File tree

‎internal/discovery/discovery.go‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,7 @@ func (d *Discovery) isInHiddenDirectory(path string) bool {
320320
if strings.HasPrefix(part, ".") {
321321
// Check if this hidden directory should be included
322322
shouldInclude := false
323+
323324
for _, includeDir := range d.includeHiddenDirs {
324325
if part == includeDir {
325326
shouldInclude = true
@@ -352,6 +353,7 @@ func (d *Discovery) matchesIncludePatterns(path string) bool {
352353
// If we can't get a relative path, use the absolute path
353354
relPath = path
354355
}
356+
355357
relPathSlash := filepath.ToSlash(relPath)
356358

357359
for _, pattern := range d.includeDirs {
@@ -367,6 +369,20 @@ func (d *Discovery) matchesIncludePatterns(path string) bool {
367369
if strings.HasPrefix(relPathSlash, patternSlash+"/") {
368370
return true
369371
}
372+
373+
// Check if the pattern is a glob that matches the directory name
374+
// For example, "app*" should match "app" and "app/frontend"
375+
if strings.Contains(patternSlash, "*") {
376+
// Split the path into components and check each level
377+
pathParts := strings.Split(relPathSlash, "/")
378+
for i := 0; i < len(pathParts); i++ {
379+
// Check if any part of the path matches the pattern
380+
testPath := strings.Join(pathParts[:i+1], "/")
381+
if matched, err := filepath.Match(patternSlash, testPath); err == nil && matched {
382+
return true
383+
}
384+
}
385+
}
370386
}
371387

372388
return false
@@ -382,6 +398,35 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
382398
filenames = DefaultConfigFilenames
383399
}
384400

401+
// Expand include directory patterns ahead of time so we can:
402+
// 1) Allow explicit directories even when they don't match relative include patterns
403+
// 2) Process directories outside of the working directory (external)
404+
expandedIncludeDirs := map[string]struct{}{}
405+
406+
if d.excludeByDefault && len(d.includeDirs) > 0 {
407+
for _, pattern := range d.includeDirs {
408+
absPattern := pattern
409+
if !filepath.IsAbs(pattern) {
410+
absPattern = filepath.Join(d.workingDir, pattern)
411+
}
412+
413+
// Treat include entries as explicit paths only (no glob expansion)
414+
candidate := absPattern
415+
416+
info, err := os.Stat(candidate)
417+
if err != nil {
418+
continue
419+
}
420+
421+
dir := candidate
422+
if !info.IsDir() {
423+
dir = filepath.Dir(candidate)
424+
}
425+
426+
expandedIncludeDirs[filepath.Clean(dir)] = struct{}{}
427+
}
428+
}
429+
385430
processFn := func(path string, info os.FileInfo, err error) error {
386431
if err != nil {
387432
return errors.New(err)
@@ -400,6 +445,16 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
400445
if base == fname {
401446
configDir := filepath.Dir(path)
402447

448+
// Apply include pattern filtering during initial discovery with support for
449+
// explicitly expanded include directories (even outside working dir)
450+
if d.excludeByDefault {
451+
if !d.matchesIncludePatterns(configDir) {
452+
if _, ok := expandedIncludeDirs[configDir]; !ok {
453+
return nil
454+
}
455+
}
456+
}
457+
403458
cfgType := ConfigTypeUnit
404459
if fname == config.DefaultStackFile {
405460
cfgType = ConfigTypeStack
@@ -431,6 +486,34 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
431486
return cfgs, errors.New(err)
432487
}
433488

489+
// Explicitly process expanded include directories that weren't discovered by the walk
490+
// (e.g., external directories outside the working directory)
491+
if len(expandedIncludeDirs) > 0 {
492+
// Build a quick lookup to avoid duplicates
493+
existing := make(map[string]struct{}, len(cfgs))
494+
for _, c := range cfgs {
495+
existing[c.Path] = struct{}{}
496+
}
497+
498+
for dir := range expandedIncludeDirs {
499+
if _, seen := existing[dir]; seen {
500+
continue
501+
}
502+
503+
for _, fname := range filenames {
504+
candidate := filepath.Join(dir, fname)
505+
506+
info, err := os.Stat(candidate)
507+
if err != nil || info.IsDir() {
508+
continue
509+
}
510+
511+
// Let processFn apply the remaining validations and append
512+
_ = processFn(candidate, info, nil)
513+
}
514+
}
515+
}
516+
434517
errs := []error{}
435518

436519
// We do an initial parse loop if we know we need to parse configurations,
@@ -478,6 +561,7 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
478561
if len(d.includeDirs) > 0 {
479562
dependencyDiscovery = dependencyDiscovery.WithIncludeDirs(d.includeDirs)
480563
}
564+
481565
if d.strictInclude {
482566
dependencyDiscovery = dependencyDiscovery.WithStrictInclude()
483567
}
@@ -530,10 +614,10 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
530614
type DependencyDiscovery struct {
531615
discoveryContext *DiscoveryContext
532616
cfgs DiscoveredConfigs
617+
includeDirs []string
533618
depthRemaining int
534619
discoverExternal bool
535620
suppressParseErrors bool
536-
includeDirs []string
537621
strictInclude bool
538622
}
539623

‎internal/discovery/discovery_test.go‎

Lines changed: 191 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -628,8 +628,8 @@ func TestDiscoveryMatchesIncludePatterns(t *testing.T) {
628628
tests := []struct {
629629
name string
630630
includeDirs []string
631-
strictInclude bool
632631
wantUnits []string
632+
strictInclude bool
633633
}{
634634
{
635635
name: "no include patterns - should match everything",
@@ -737,8 +737,8 @@ dependency "vpc" {
737737
tests := []struct {
738738
name string
739739
includeDirs []string
740-
strictInclude bool
741740
wantUnits []string
741+
strictInclude bool
742742
}{
743743
{
744744
name: "no include patterns - should match everything",
@@ -824,8 +824,8 @@ func TestDiscoveryWithStrictIncludeMode(t *testing.T) {
824824
tests := []struct {
825825
name string
826826
includeDirs []string
827-
strictInclude bool
828827
wantUnits []string
828+
strictInclude bool
829829
}{
830830
{
831831
name: "strict include mode - only app directories",
@@ -873,3 +873,191 @@ func TestDiscoveryWithStrictIncludeMode(t *testing.T) {
873873
})
874874
}
875875
}
876+
877+
func TestDiscoveryWithExcludeByDefault(t *testing.T) {
878+
t.Parallel()
879+
880+
// Create a temporary directory for testing
881+
tmpDir := t.TempDir()
882+
883+
// Create test directory structure
884+
testDirs := []string{
885+
"app",
886+
"app/frontend",
887+
"infra",
888+
"infra/vpc",
889+
"shared",
890+
}
891+
892+
for _, dir := range testDirs {
893+
err := os.MkdirAll(filepath.Join(tmpDir, dir), 0755)
894+
require.NoError(t, err)
895+
}
896+
897+
// Create test files
898+
testFiles := map[string]string{
899+
"app/terragrunt.hcl": "",
900+
"app/frontend/terragrunt.hcl": "",
901+
"infra/terragrunt.hcl": "",
902+
"infra/vpc/terragrunt.hcl": "",
903+
"shared/terragrunt.hcl": "",
904+
}
905+
906+
for path, content := range testFiles {
907+
err := os.WriteFile(filepath.Join(tmpDir, path), []byte(content), 0644)
908+
require.NoError(t, err)
909+
}
910+
911+
tests := []struct {
912+
name string
913+
includeDirs []string
914+
wantUnits []string
915+
excludeByDefault bool
916+
}{
917+
{
918+
name: "exclude by default with app pattern",
919+
includeDirs: []string{"app"},
920+
excludeByDefault: true,
921+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend")},
922+
},
923+
{
924+
name: "exclude by default with infra pattern",
925+
includeDirs: []string{"infra"},
926+
excludeByDefault: true,
927+
wantUnits: []string{filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc")},
928+
},
929+
{
930+
name: "exclude by default with multiple patterns",
931+
includeDirs: []string{"app", "infra"},
932+
excludeByDefault: true,
933+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend"), filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc")},
934+
},
935+
{
936+
name: "exclude by default with glob pattern",
937+
includeDirs: []string{"app*"},
938+
excludeByDefault: true,
939+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend")},
940+
},
941+
{
942+
name: "exclude by default with no patterns - should include all",
943+
includeDirs: []string{},
944+
excludeByDefault: true,
945+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend"), filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc"), filepath.Join(tmpDir, "shared")},
946+
},
947+
}
948+
949+
for _, tt := range tests {
950+
t.Run(tt.name, func(t *testing.T) {
951+
t.Parallel()
952+
953+
d := discovery.NewDiscovery(tmpDir).WithIncludeDirs(tt.includeDirs)
954+
if tt.excludeByDefault {
955+
d = d.WithExcludeByDefault()
956+
}
957+
958+
opts, err := options.NewTerragruntOptionsForTest(tmpDir)
959+
require.NoError(t, err)
960+
961+
configs, err := d.Discover(t.Context(), logger.CreateLogger(), opts)
962+
require.NoError(t, err)
963+
964+
units := configs.Filter(discovery.ConfigTypeUnit).Paths()
965+
assert.ElementsMatch(t, tt.wantUnits, units)
966+
})
967+
}
968+
}
969+
970+
func TestDiscoveryWithStrictIncludeAndExcludeByDefault(t *testing.T) {
971+
t.Parallel()
972+
973+
// Create a temporary directory for testing
974+
tmpDir := t.TempDir()
975+
976+
// Create test directory structure
977+
testDirs := []string{
978+
"app",
979+
"app/frontend",
980+
"infra",
981+
"infra/vpc",
982+
"shared",
983+
}
984+
985+
for _, dir := range testDirs {
986+
err := os.MkdirAll(filepath.Join(tmpDir, dir), 0755)
987+
require.NoError(t, err)
988+
}
989+
990+
// Create test files
991+
testFiles := map[string]string{
992+
"app/terragrunt.hcl": "",
993+
"app/frontend/terragrunt.hcl": "",
994+
"infra/terragrunt.hcl": "",
995+
"infra/vpc/terragrunt.hcl": "",
996+
"shared/terragrunt.hcl": "",
997+
}
998+
999+
for path, content := range testFiles {
1000+
err := os.WriteFile(filepath.Join(tmpDir, path), []byte(content), 0644)
1001+
require.NoError(t, err)
1002+
}
1003+
1004+
tests := []struct {
1005+
name string
1006+
includeDirs []string
1007+
wantUnits []string
1008+
strictInclude bool
1009+
excludeByDefault bool
1010+
}{
1011+
{
1012+
name: "strict include with exclude by default - app only",
1013+
includeDirs: []string{"app"},
1014+
strictInclude: true,
1015+
excludeByDefault: true,
1016+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend")},
1017+
},
1018+
{
1019+
name: "strict include with exclude by default - infra only",
1020+
includeDirs: []string{"infra"},
1021+
strictInclude: true,
1022+
excludeByDefault: true,
1023+
wantUnits: []string{filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc")},
1024+
},
1025+
{
1026+
name: "strict include with exclude by default - multiple patterns",
1027+
includeDirs: []string{"app", "infra"},
1028+
strictInclude: true,
1029+
excludeByDefault: true,
1030+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend"), filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc")},
1031+
},
1032+
{
1033+
name: "strict include without exclude by default - should include all",
1034+
includeDirs: []string{"app"},
1035+
strictInclude: true,
1036+
excludeByDefault: false,
1037+
wantUnits: []string{filepath.Join(tmpDir, "app"), filepath.Join(tmpDir, "app", "frontend"), filepath.Join(tmpDir, "infra"), filepath.Join(tmpDir, "infra", "vpc"), filepath.Join(tmpDir, "shared")},
1038+
},
1039+
}
1040+
1041+
for _, tt := range tests {
1042+
t.Run(tt.name, func(t *testing.T) {
1043+
t.Parallel()
1044+
1045+
d := discovery.NewDiscovery(tmpDir).WithIncludeDirs(tt.includeDirs)
1046+
if tt.strictInclude {
1047+
d = d.WithStrictInclude()
1048+
}
1049+
if tt.excludeByDefault {
1050+
d = d.WithExcludeByDefault()
1051+
}
1052+
1053+
opts, err := options.NewTerragruntOptionsForTest(tmpDir)
1054+
require.NoError(t, err)
1055+
1056+
configs, err := d.Discover(t.Context(), logger.CreateLogger(), opts)
1057+
require.NoError(t, err)
1058+
1059+
units := configs.Filter(discovery.ConfigTypeUnit).Paths()
1060+
assert.ElementsMatch(t, tt.wantUnits, units)
1061+
})
1062+
}
1063+
}

‎internal/runner/runnerpool/builder.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,23 @@ func Build(ctx context.Context, l log.Logger, terragruntOptions *options.Terragr
2525
WithIncludeHiddenDirs([]string{config.StackDir}).
2626
WithDiscoveryContext(&discovery.DiscoveryContext{Cmd: terragruntOptions.TerraformCommand})
2727

28+
// Apply include directory features based on terragrunt options
29+
if len(terragruntOptions.UnitsReading) > 0 {
30+
d = d.WithIncludeDirs(terragruntOptions.UnitsReading)
31+
}
32+
33+
if len(terragruntOptions.ModulesThatInclude) > 0 {
34+
d = d.WithIncludeDirs(terragruntOptions.ModulesThatInclude)
35+
}
36+
37+
if terragruntOptions.StrictInclude {
38+
d = d.WithStrictInclude()
39+
}
40+
41+
if terragruntOptions.ExcludeByDefault {
42+
d = d.WithExcludeByDefault()
43+
}
44+
2845
// Wrap discovery with telemetry
2946
var discovered discovery.DiscoveredConfigs
3047

‎internal/runner/runnerpool/runner.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ type Runner struct {
3838

3939
// NewRunnerPoolStack creates a new stack from discovered units.
4040
func NewRunnerPoolStack(ctx context.Context, l log.Logger, terragruntOptions *options.TerragruntOptions, discovered discovery.DiscoveredConfigs, opts ...common.Option) (common.StackRunner, error) {
41-
4241
if len(discovered) == 0 {
4342
return nil, errors.New(common.ErrNoUnitsFound)
4443
}

0 commit comments

Comments
 (0)