fix: mirror plan files from the working directory under --out-dir for Git filters

Author: yhakbar

docs/src/data/changelog/v1.1.4/out-dir-git-filter-plan-key.mdx

@@ -0,0 +1,10 @@
+---
+version: "v1.1.4"
+category: "bug-fixes"
+---
+
+#### `--out-dir` writes plans to the same place with and without a Git filter
+
+`run --all` with [`--out-dir`](/reference/cli/commands/run#out-dir) mirrors each unit under the output directory. When a Git-based filter selected the unit and `--working-dir` was a subdirectory of the repository, the mirrored path started at the root of the repository rather than at the working directory, so a unit at `live/unit` was written to `<out-dir>/live/unit` with a filter and to `<out-dir>/unit` without one.
+
+Splitting a run in two, planning with a filter and applying the same units later without one, then failed: the apply looked for each plan under a path the plan had not written, and tofu reported that it could not load the file as a plan. Both ways of selecting a unit now mirror it from `--working-dir`, including units generated by a stack. A unit that a Git filter selects from outside `--working-dir` is still mirrored from the root of the repository, which keeps it within the output directory.

internal/component/component.go

@@ -69,8 +69,9 @@ const (
 // because it can help us determine how the
 // Component should be run or enqueued later.
 type DiscoveryContext struct {
-	WorkingDir string
-	Ref        string
+	WorkingDir    string
+	OutputKeyBase string
+	Ref           string
 
 	origin Origin
 

internal/component/component_test.go

@@ -1,6 +1,7 @@
 package component_test
 
 import (
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -272,3 +273,72 @@ func TestUnitGuardConfigParseWithRacing(t *testing.T) {
 	assert.Equal(t, int32(1), parses.Load(), "config should be parsed exactly once")
 	assert.NotNil(t, unit.Config(), "config should be populated after parsing")
 }
+
+// TestUnitOutputFile checks that the same unit is mirrored under the output folder at the
+// same path whether it was found on the filesystem or in a Git worktree, and that a unit
+// found outside the working directory is still mirrored inside the output folder.
+func TestUnitOutputFile(t *testing.T) {
+	t.Parallel()
+
+	base := t.TempDir()
+	workingDir := filepath.Join(base, "repo", "live")
+	worktreeRoot := filepath.Join(base, "worktree")
+	outputFolder := filepath.Join(base, "plans")
+
+	testCases := []struct {
+		discoveryContext *component.DiscoveryContext
+		name             string
+		unitPath         string
+		expected         string
+	}{
+		{
+			name:     "filesystem discovery",
+			unitPath: filepath.Join(workingDir, "unit2"),
+			discoveryContext: &component.DiscoveryContext{
+				WorkingDir: workingDir,
+			},
+			expected: filepath.Join(outputFolder, "unit2", "tfplan.tfplan"),
+		},
+		{
+			name:     "worktree discovery",
+			unitPath: filepath.Join(worktreeRoot, "live", "unit2"),
+			discoveryContext: &component.DiscoveryContext{
+				WorkingDir:    worktreeRoot,
+				OutputKeyBase: filepath.Join(worktreeRoot, "live"),
+			},
+			expected: filepath.Join(outputFolder, "unit2", "tfplan.tfplan"),
+		},
+		{
+			name:     "worktree discovery outside the working directory",
+			unitPath: filepath.Join(worktreeRoot, "other", "unit3"),
+			discoveryContext: &component.DiscoveryContext{
+				WorkingDir:    worktreeRoot,
+				OutputKeyBase: filepath.Join(worktreeRoot, "live"),
+			},
+			expected: filepath.Join(outputFolder, "other", "unit3", "tfplan.tfplan"),
+		},
+		{
+			name:     "worktree discovery without a mapped working directory",
+			unitPath: filepath.Join(worktreeRoot, "live", "unit2"),
+			discoveryContext: &component.DiscoveryContext{
+				WorkingDir: worktreeRoot,
+			},
+			expected: filepath.Join(outputFolder, "live", "unit2", "tfplan.tfplan"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			unit := component.NewUnit(tc.unitPath).WithDiscoveryContext(tc.discoveryContext)
+
+			assert.Equal(t, tc.expected, unit.OutputFile(workingDir, outputFolder))
+			assert.Equal(
+				t,
+				filepath.Join(filepath.Dir(tc.expected), "tfplan.json"),
+				unit.OutputJSONFile(workingDir, outputFolder),
+			)
+		})
+	}
+}

internal/component/unit.go

@@ -340,10 +340,7 @@ func (u *Unit) planFilePath(rootWorkingDir, outputFolder, fileName string) strin
 		return ""
 	}
 
-	// Use discoveryContext.WorkingDir as base (always populated).
-	// This is critical for git-based filters where units are discovered in temporary worktrees.
-	// Using rootWorkingDir would cause relative paths to escape the outputFolder.
-	relPath, err := filepath.Rel(u.discoveryContext.WorkingDir, u.path)
+	relPath, err := filepath.Rel(u.outputKeyBase(), u.path)
 	if err != nil {
 		relPath = u.path
 	}
@@ -358,3 +355,21 @@ func (u *Unit) planFilePath(rootWorkingDir, outputFolder, fileName string) strin
 
 	return filepath.Join(dir, fileName)
 }
+
+// outputKeyBase returns the directory that plan output paths are computed relative to.
+func (u *Unit) outputKeyBase() string {
+	// A unit found by a Git filter lives in a temporary worktree, and its discovery context
+	// records the worktree root, so mirroring from there keys the plan by the unit's path from
+	// the root of the repository while a filesystem walk keys the same unit by its path from
+	// the working directory. OutputKeyBase is the working directory's counterpart inside the
+	// worktree, which makes the two agree.
+	//
+	// A unit outside the working directory, which only a Git filter reaches, keys off the
+	// worktree root instead: that always contains it, so the mirrored path stays inside the
+	// output folder.
+	if base := u.discoveryContext.OutputKeyBase; base != "" && util.HasPathPrefix(u.path, base) {
+		return base
+	}
+
+	return u.discoveryContext.WorkingDir
+}

internal/discovery/phase_worktree.go

@@ -223,6 +223,7 @@ func (p *WorktreePhase) discoverInWorktree(
 	discoveryContext := discovery.discoveryContext.Copy()
 	discoveryContext.Ref = wt.Ref
 	discoveryContext.WorkingDir = wt.Path
+	discoveryContext.OutputKeyBase = discovery.worktrees.WorkingDir(ctx, wt.Path)
 	discoveryContext.SuggestOrigin(component.OriginWorktreeDiscovery)
 
 	if discoveryContext.Args != nil {
@@ -446,61 +447,39 @@ func (p *WorktreePhase) walkChangedStack(
 
 	parentFilters := discovery.filters.ExcludingGitFilters()
 
-	discoveryGroup.Go(func() error {
-		fromDiscovery := NewDiscovery(fromStack.Path()).
-			WithDiscoveryContext(fromDiscoveryContext).
-			WithFilters(parentFilters).
-			WithNumWorkers(p.numWorkers)
-
-		var fromDiscoveryErr error
-
-		fromComponents, fromDiscoveryErr = fromDiscovery.Discover(discoveryCtx, l, v, input.Opts)
-		if fromDiscoveryErr != nil {
-			mu.Lock()
-
-			errs = append(errs, fromDiscoveryErr)
-
-			mu.Unlock()
-
-			return nil
-		}
-
-		for _, c := range fromComponents {
-			dc := c.DiscoveryContext().CopyWithNewOrigin(component.OriginWorktreeDiscovery)
-			dc.WorkingDir = fromStack.DiscoveryContext().WorkingDir
-			c.SetDiscoveryContext(dc)
-		}
-
-		return nil
-	})
-
-	discoveryGroup.Go(func() error {
-		toDiscovery := NewDiscovery(toStack.Path()).
-			WithDiscoveryContext(toDiscoveryContext).
-			WithFilters(parentFilters).
-			WithNumWorkers(p.numWorkers)
+	for _, side := range []struct {
+		stack            *component.Stack
+		discoveryContext *component.DiscoveryContext
+		components       *component.Components
+	}{
+		{stack: fromStack, discoveryContext: fromDiscoveryContext, components: &fromComponents},
+		{stack: toStack, discoveryContext: toDiscoveryContext, components: &toComponents},
+	} {
+		discoveryGroup.Go(func() error {
+			components, err := p.walkStackSide(
+				discoveryCtx,
+				l,
+				v,
+				input,
+				side.stack,
+				side.discoveryContext,
+				parentFilters,
+			)
+			if err != nil {
+				mu.Lock()
 
-		var toDiscoveryErr error
+				errs = append(errs, err)
 
-		toComponents, toDiscoveryErr = toDiscovery.Discover(discoveryCtx, l, v, input.Opts)
-		if toDiscoveryErr != nil {
-			mu.Lock()
+				mu.Unlock()
 
-			errs = append(errs, toDiscoveryErr)
+				return nil
+			}
 
-			mu.Unlock()
+			*side.components = components
 
 			return nil
-		}
-
-		for _, c := range toComponents {
-			dc := c.DiscoveryContext().CopyWithNewOrigin(component.OriginWorktreeDiscovery)
-			dc.WorkingDir = toStack.DiscoveryContext().WorkingDir
-			c.SetDiscoveryContext(dc)
-		}
-
-		return nil
-	})
+		})
+	}
 
 	if err = discoveryGroup.Wait(); err != nil {
 		return nil, err
@@ -571,6 +550,38 @@ func (p *WorktreePhase) walkChangedStack(
 	return finalComponents, nil
 }
 
+// walkStackSide walks one side of a changed stack and records the components it finds as
+// worktree discoveries rooted at that side's worktree.
+func (p *WorktreePhase) walkStackSide(
+	ctx context.Context,
+	l log.Logger,
+	v *venv.Venv,
+	input *PhaseInput,
+	stack *component.Stack,
+	discoveryContext *component.DiscoveryContext,
+	filters filter.Filters,
+) (component.Components, error) {
+	discovery := input.Discovery
+
+	components, err := NewDiscovery(stack.Path()).
+		WithDiscoveryContext(discoveryContext).
+		WithFilters(filters).
+		WithNumWorkers(p.numWorkers).
+		Discover(ctx, l, v, input.Opts)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, c := range components {
+		dc := c.DiscoveryContext().CopyWithNewOrigin(component.OriginWorktreeDiscovery)
+		dc.WorkingDir = stack.DiscoveryContext().WorkingDir
+		dc.OutputKeyBase = discovery.worktrees.WorkingDir(ctx, dc.WorkingDir)
+		c.SetDiscoveryContext(dc)
+	}
+
+	return components, nil
+}
+
 // ComponentPair represents a pair of matched components from different worktrees.
 type ComponentPair struct {
 	FromComponent component.Component