chore: Adding LockContext

Author: yhakbar

docs/src/content/docs/03-features/07-caching/04-cas.mdx

@@ -206,7 +206,7 @@ For cold clones, where the content is not already in the CAS:
 5. Content is stored in the CAS, partitioned by hash prefix
 6. The tree structure is read from the CAS and hard links are created to the target directory
 
-If the central Git store cannot be used (for example, the lock cannot be acquired or the fetch fails), Terragrunt logs a warning and falls back to a clone in a temporary directory.
+Concurrent units that target the same remote URL share one fetch instead of cloning in parallel, so the objects are typically transferred once and reused. If the shared fetch hangs or fails, Terragrunt logs a warning and falls back to a clone in a temporary directory.
 
 #### Warm Clones
 

docs/src/data/changelog/v1.0.4/cas-central-git-store.mdx

@@ -7,7 +7,7 @@ category: "experiments-updated"
 
 CAS now keeps one bare Git repository per remote URL inside its store, under `~/.cache/terragrunt/cas/store/git/` on Linux by default. See [Storage](/features/caching/cas#storage) for where this lives on macOS and Windows. On a cache miss, Terragrunt fetches just the requested ref into that repository instead of running a fresh shallow clone into a temporary directory. Repeated misses against the same remote reuse the existing pack files, so fetching a second ref from the same repository transfers only the new objects.
 
-Concurrent Terragrunt invocations are coordinated by a per-URL lock so pack-file writes do not interleave. If the central store cannot be used for any reason, Terragrunt logs a warning and falls back to the previous temporary-clone path so cloning still succeeds.
+Concurrent Terragrunt runs against the same remote URL share one fetch instead of cloning in parallel; later runs reuse what the first one transferred. If the shared fetch hangs or fails, Terragrunt logs a warning and falls back to a temporary clone so cloning still succeeds.
 
 You can reclaim space at any time by deleting the `git/` subdirectory:
 

internal/cas/gitstore.go

@@ -6,19 +6,32 @@ import (
 	"encoding/hex"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/gruntwork-io/terragrunt/internal/errors"
 	"github.com/gruntwork-io/terragrunt/internal/git"
 	"github.com/gruntwork-io/terragrunt/internal/vfs"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 )
 
-const gitStoreURLHashLen = 16
+const (
+	gitStoreURLHashLen = 16
+	// gitStoreLockTimeout bounds how long EnsureRef waits for the per-URL
+	// lock before giving up and letting the caller fall back to a temporary
+	// clone. Generous enough to outlast a typical fetch, short enough that a
+	// hung holder does not stall every concurrent unit indefinitely.
+	gitStoreLockTimeout = 5 * time.Minute
+)
 
 // GitStore keeps one bare git repository per remote URL on disk so CAS cache
 // misses can issue an incremental git fetch instead of a full shallow clone.
 // Each per-URL repository is gated by an exclusive flock because pack-file
 // writes are not safe to interleave with concurrent reads of the same repo.
+// EnsureRef waits up to gitStoreLockTimeout for the lock; on context
+// cancellation or timeout the caller can fall back to a temporary clone
+// rather than block indefinitely on a hung holder. After acquiring the
+// lock, EnsureRef re-checks for the requested object so a unit that simply
+// waited out a peer's fetch can proceed without re-doing the work.
 // The flock is held from EnsureRef return until the caller releases it.
 type GitStore struct {
 	runner   *git.GitRunner
@@ -62,7 +75,10 @@ func (s *GitStore) EnsureRef(
 		return "", nil, fmt.Errorf("create git store entry %s: %w", dir, errors.Join(ErrGitStorePath, err))
 	}
 
-	unlocker, err := vfs.Lock(fs, lockPath)
+	lockCtx, cancel := context.WithTimeout(ctx, gitStoreLockTimeout)
+	defer cancel()
+
+	unlocker, err := vfs.LockContext(lockCtx, fs, lockPath)
 	if err != nil {
 		return "", nil, fmt.Errorf("lock git store for %s: %w", url, errors.Join(ErrGitStoreLock, err))
 	}

internal/cas/gitstore_test.go

@@ -1,10 +1,13 @@
 package cas_test
 
 import (
+	"context"
+	"errors"
 	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gruntwork-io/terragrunt/internal/cas"
 	"github.com/gruntwork-io/terragrunt/internal/git"
@@ -108,6 +111,69 @@ func TestGitStoreEnsureRefConcurrentSameURLWithRacing(t *testing.T) {
 	}
 }
 
+func TestGitStoreEnsureRef_LockHeldRespectsContextCancellation(t *testing.T) {
+	t.Parallel()
+
+	url := startTestServer(t)
+	hash := resolveHead(t, url)
+
+	store, fs, root := newTestGitStore(t)
+	require.NotEmpty(t, root)
+
+	l := logger.CreateLogger()
+
+	// First caller takes the per-URL lock and holds it.
+	repoPath, unlock, err := store.EnsureRef(t.Context(), l, fs, url, "main", hash, 0)
+	require.NoError(t, err)
+	require.NotEmpty(t, repoPath)
+	t.Cleanup(func() { _ = unlock.Unlock() })
+
+	// Second caller arrives with a short deadline. With the lock held it
+	// must return a context error rather than block.
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+
+	_, _, err = store.EnsureRef(ctx, l, fs, url, "main", hash, 0)
+	require.Error(t, err)
+	assert.Less(t, time.Since(start), 5*time.Second, "EnsureRef should not block past the context deadline")
+	assert.True(
+		t,
+		errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled),
+		"expected context error, got %v", err,
+	)
+}
+
+func TestGitStoreEnsureRefLockReleaseAllowsWaiterToProceedWithRacing(t *testing.T) {
+	t.Parallel()
+
+	url := startTestServer(t)
+	hash := resolveHead(t, url)
+
+	store, fs, root := newTestGitStore(t)
+	require.NotEmpty(t, root)
+
+	l := logger.CreateLogger()
+
+	_, unlock, err := store.EnsureRef(t.Context(), l, fs, url, "main", hash, 0)
+	require.NoError(t, err)
+
+	// Release the holder after a short delay so the waiter sees the lock open.
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+
+		_ = unlock.Unlock()
+	}()
+
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+
+	_, unlock2, err := store.EnsureRef(ctx, l, fs, url, "main", hash, 0)
+	require.NoError(t, err)
+	require.NoError(t, unlock2.Unlock())
+}
+
 func TestGitStoreEnsureRef_FetchFailureSurfacesError(t *testing.T) {
 	t.Parallel()
 

internal/vfs/vfs.go

@@ -5,6 +5,7 @@ package vfs
 import (
 	"archive/zip"
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -15,6 +16,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/charlievieth/fastwalk"
 	"github.com/gofrs/flock"
@@ -48,6 +50,16 @@ type Locker interface {
 	TryLock(name string) (Unlocker, bool, error)
 }
 
+// ContextLocker is an optional interface for filesystems whose locks can be
+// acquired with a context. Implementations should poll/retry until the lock
+// is acquired or ctx is canceled. On ctx cancellation, the returned error
+// wraps ctx.Err(). Implementations also impose a hard upper bound on the
+// total wait (see [maxLockWait]), so a never-canceled ctx will still return
+// after that bound elapses.
+type ContextLocker interface {
+	LockContext(ctx context.Context, name string) (Unlocker, error)
+}
+
 // ErrNoHardLink is returned when a filesystem does not support hard links.
 var ErrNoHardLink = errors.New("hard link not supported")
 
@@ -159,6 +171,23 @@ func TryLock(fs FS, name string) (Unlocker, bool, error) {
 	return locker.TryLock(name)
 }
 
+// LockContext acquires a lock for the given name, blocking until it is
+// available or ctx is canceled. Filesystems that implement ContextLocker
+// use their native context-aware path; otherwise the call falls back to a
+// blocking Lock without context support.
+//
+// ContextLocker implementations cap the total wait at [maxLockWait], so the
+// call returns after that bound elapses even when ctx is never canceled.
+// Callers that want a shorter deadline should pass a ctx with their own
+// timeout.
+func LockContext(ctx context.Context, fs FS, name string) (Unlocker, error) {
+	if cl, ok := fs.(ContextLocker); ok {
+		return cl.LockContext(ctx, name)
+	}
+
+	return Lock(fs, name)
+}
+
 // WalkDirParallelOption configures a [WalkDirParallel] call.
 type WalkDirParallelOption func(*walkDirParallelConfig)
 
@@ -288,6 +317,43 @@ func (fs *osFS) TryLock(name string) (Unlocker, bool, error) {
 	return l, true, nil
 }
 
+const (
+	// osFlockRetryDelay is how often osFS.LockContext polls for the flock
+	// when the lock is held by another process. gofrs/flock uses a syscall
+	// per attempt, so the tick is coarse enough to limit churn while still
+	// reacting quickly when the holder releases.
+	osFlockRetryDelay = 50 * time.Millisecond
+
+	// memMapLockRetryDelay is how often memMapFS.LockContext retries its
+	// in-process sync.Mutex TryLock. The retry is essentially free, so the
+	// tick is tighter than [osFlockRetryDelay] to keep tests snappy.
+	memMapLockRetryDelay = 10 * time.Millisecond
+
+	// maxLockWait is the hard upper bound on any LockContext call regardless
+	// of the caller's context. It guarantees the retry loop terminates even
+	// if a caller passes a never-canceled context and the lock is permanently
+	// held by another process.
+	maxLockWait = 30 * time.Minute
+)
+
+func (fs *osFS) LockContext(ctx context.Context, name string) (Unlocker, error) {
+	ctx, cancel := context.WithTimeout(ctx, maxLockWait)
+	defer cancel()
+
+	l := flock.New(name)
+
+	acquired, err := l.TryLockContext(ctx, osFlockRetryDelay)
+	if err != nil {
+		return nil, err
+	}
+
+	if !acquired {
+		return nil, ctx.Err()
+	}
+
+	return l, nil
+}
+
 // memMapFS wraps afero.MemMapFs with in-memory symlink support.
 type memMapFS struct {
 	afero.Fs
@@ -362,6 +428,25 @@ func (fs *memMapFS) TryLock(name string) (Unlocker, bool, error) {
 	return l, true, nil
 }
 
+func (fs *memMapFS) LockContext(ctx context.Context, name string) (Unlocker, error) {
+	ctx, cancel := context.WithTimeout(ctx, maxLockWait)
+	defer cancel()
+
+	l := fs.getOrCreateLock(name)
+
+	for {
+		if l.mu.TryLock() {
+			return l, nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(memMapLockRetryDelay):
+		}
+	}
+}
+
 func (fs *memMapFS) getOrCreateLock(name string) *memLock {
 	fs.locksMu.Lock()
 	defer fs.locksMu.Unlock()