feat: Adding support for commit refs in CAS

Author: yhakbar

internal/cas/cas.go

@@ -159,35 +159,29 @@ func (c *CAS) SynthStore() *Store { return c.synthStore }
 //
 // TODO: Make options optional
 func (c *CAS) Clone(ctx context.Context, l log.Logger, opts *CloneOptions, url string) error {
-	// Ensure the store paths exist
-	if err := c.fs.MkdirAll(c.blobStore.Path(), DefaultDirPerms); err != nil {
-		return fmt.Errorf("failed to create blob store path: %w", err)
-	}
-
-	if err := c.fs.MkdirAll(c.treeStore.Path(), DefaultDirPerms); err != nil {
-		return fmt.Errorf("failed to create tree store path: %w", err)
+	if err := c.ensureCloneStores(); err != nil {
+		return err
 	}
 
 	return telemetry.TelemeterFromContext(ctx).Collect(ctx, "cas_clone", map[string]any{
 		"url":    url,
 		"branch": opts.Branch,
 	}, func(childCtx context.Context) error {
-		hash, err := c.resolveReference(childCtx, url, opts.Branch)
+		ref, err := c.resolveReference(childCtx, url, opts.Branch)
 		if err != nil {
 			return err
 		}
 
 		targetDir := c.prepareTargetDirectory(opts.Dir, url)
 
-		if c.treeStore.NeedsWrite(hash) {
-			if err := c.populateTreeFromGit(childCtx, l, opts, url, hash); err != nil {
-				return err
-			}
+		canonicalHash, err := c.populateTreeFromRef(childCtx, l, opts, ref)
+		if err != nil {
+			return err
 		}
 
 		treeContent := NewContent(c.treeStore)
 
-		treeData, err := treeContent.Read(hash)
+		treeData, err := treeContent.Read(canonicalHash)
 		if err != nil {
 			return err
 		}
@@ -201,46 +195,172 @@ func (c *CAS) Clone(ctx context.Context, l log.Logger, opts *CloneOptions, url s
 	})
 }
 
-// populateTreeFromGit fetches the commit at hash and stores its tree and
-// reachable blobs in the CAS. It tries the central git store first. Any
-// error from the central store is logged as a warning, and the function
-// then falls back to a bare clone in a temporary directory.
-func (c *CAS) populateTreeFromGit(ctx context.Context, l log.Logger, opts *CloneOptions, url, hash string) error {
+// ensureCloneStores creates the blob and tree store directories that
+// [CAS.Clone] writes to. Defensive: [New] already creates them, but a
+// long-lived [CAS] instance could see them removed between calls.
+func (c *CAS) ensureCloneStores() error {
+	for _, s := range []*Store{c.blobStore, c.treeStore} {
+		if err := c.fs.MkdirAll(s.Path(), DefaultDirPerms); err != nil {
+			return fmt.Errorf("create CAS store path %s: %w", s.Path(), err)
+		}
+	}
+
+	return nil
+}
+
+// populateTreeFromRef dispatches by ref kind, short-circuiting on a
+// cached [symbolicRef] and otherwise calling the kind-specific
+// populate. Returns the canonical commit hash.
+func (c *CAS) populateTreeFromRef(
+	ctx context.Context,
+	l log.Logger,
+	opts *CloneOptions,
+	ref resolvedRef,
+) (string, error) {
+	switch ref := ref.(type) {
+	case *symbolicRef:
+		if !c.treeStore.NeedsWrite(ref.Hash) {
+			return ref.Hash, nil
+		}
+
+		if err := c.populateTreeFromSymbolicRef(ctx, l, opts, ref); err != nil {
+			return "", err
+		}
+
+		return ref.Hash, nil
+
+	case *commitRef:
+		return c.populateTreeFromCommitRef(ctx, l, opts, ref)
+
+	default:
+		return "", fmt.Errorf("unsupported resolved ref type %T", ref)
+	}
+}
+
+// populateTreeFromSymbolicRef stores the tree and reachable blobs
+// for ref.Hash in the CAS. Tries the central [GitStore] first; on
+// any error from it, logs a warning and falls back to a bare clone
+// in a temporary directory.
+func (c *CAS) populateTreeFromSymbolicRef(
+	ctx context.Context,
+	l log.Logger,
+	opts *CloneOptions,
+	ref *symbolicRef,
+) error {
 	depth := resolveCloneDepth(opts.Depth, c.cloneDepth)
 
-	repoPath, unlocker, err := c.gitStore.EnsureRef(ctx, l, c.fs, url, opts.Branch, hash, depth)
+	repo, err := c.gitStore.EnsureRef(ctx, l, c.fs, ref.URL, ref.Branch, ref.Hash, depth)
+	if err == nil {
+		defer repo.Release(l)
+
+		runner := c.git.WithWorkDir(repo.Path)
+
+		return c.storeRootTreeFrom(ctx, l, runner, ref.Hash, opts)
+	}
+
+	l.Warnf("central git store unavailable for %s, falling back to temporary clone: %v", ref.URL, err)
+
+	tempDir, cleanup, err := c.makeFallbackCloneDir(l)
+	if err != nil {
+		return err
+	}
+
+	defer cleanup()
+
+	runner := c.git.WithWorkDir(tempDir)
+
+	if err := runner.Clone(ctx, ref.URL, true, depth, ref.Branch); err != nil {
+		return err
+	}
+
+	return c.storeRootTreeFrom(ctx, l, runner, ref.Hash, opts)
+}
+
+// populateTreeFromCommitRef resolves ref via [GitStore.EnsureCommit]
+// (full-depth fetch on a cache miss) and stores its tree in the CAS.
+// Returns the canonical commit hash. Falls back to a temporary bare
+// clone if the central [GitStore] is unavailable.
+func (c *CAS) populateTreeFromCommitRef(
+	ctx context.Context,
+	l log.Logger,
+	opts *CloneOptions,
+	ref *commitRef,
+) (string, error) {
+	repo, err := c.gitStore.EnsureCommit(ctx, l, c.fs, ref.URL, ref.RawRef)
 	if err == nil {
-		defer func() {
-			if unlockErr := unlocker.Unlock(); unlockErr != nil {
-				l.Warnf("git store: failed to release lock for %s: %v", url, unlockErr)
+		defer repo.Release(l)
+
+		if !c.treeStore.NeedsWrite(repo.Hash) {
+			return repo.Hash, nil
+		}
+
+		runner := c.git.WithWorkDir(repo.Path)
+
+		if err := c.storeRootTreeFrom(ctx, l, runner, repo.Hash, opts); err != nil {
+			return "", err
+		}
+
+		return repo.Hash, nil
+	}
+
+	if errors.Is(err, git.ErrNoMatchingReference) {
+		return "", err
+	}
+
+	l.Warnf("central git store unavailable for %s, falling back to temporary clone: %v", ref.URL, err)
+
+	tempDir, cleanup, err := c.makeFallbackCloneDir(l)
+	if err != nil {
+		return "", err
+	}
+
+	defer cleanup()
+
+	runner := c.git.WithWorkDir(tempDir)
+
+	if err := runner.Clone(ctx, ref.URL, true, 0, ""); err != nil {
+		return "", err
+	}
+
+	canonicalHash, err := runner.RevParseCommit(ctx, ref.RawRef)
+	if err != nil {
+		if errors.Is(err, git.ErrUnknownRevision) {
+			return "", &git.WrappedError{
+				Op:      "git_clone_resolve",
+				Context: fmt.Sprintf("%q in %s", ref.RawRef, ref.URL),
+				Err:     git.ErrNoMatchingReference,
 			}
-		}()
+		}
 
-		runner := c.git.WithWorkDir(repoPath)
+		return "", err
+	}
+
+	if !c.treeStore.NeedsWrite(canonicalHash) {
+		return canonicalHash, nil
+	}
 
-		return c.storeRootTreeFrom(ctx, l, runner, hash, opts)
+	if err := c.storeRootTreeFrom(ctx, l, runner, canonicalHash, opts); err != nil {
+		return "", err
 	}
 
-	l.Warnf("central git store unavailable for %s, falling back to temporary clone: %v", url, err)
+	return canonicalHash, nil
+}
 
+// makeFallbackCloneDir creates a temporary directory for a bare clone
+// fallback and returns a cleanup function that removes it.
+func (c *CAS) makeFallbackCloneDir(l log.Logger) (string, func(), error) {
 	tempDir, err := vfs.MkdirTemp(c.fs, "", "terragrunt-cas-fallback-*")
 	if err != nil {
-		return fmt.Errorf("create fallback clone dir: %w", errors.Join(ErrFallbackCloneDir, err))
+		return "", nil, fmt.Errorf("create fallback clone dir: %w", errors.Join(ErrFallbackCloneDir, err))
 	}
 
-	defer func() {
+	cleanup := func() {
 		if rmErr := c.fs.RemoveAll(tempDir); rmErr != nil {
 			l.Warnf("cleanup error: %v", rmErr)
 		}
-	}()
-
-	runner := c.git.WithWorkDir(tempDir)
-
-	if err := runner.Clone(ctx, url, true, depth, opts.Branch); err != nil {
-		return err
 	}
 
-	return c.storeRootTreeFrom(ctx, l, runner, hash, opts)
+	return tempDir, cleanup, nil
 }
 
 func resolveCloneDepth(optDepth, casDepth int) int {
@@ -269,21 +389,59 @@ func (c *CAS) prepareTargetDirectory(dir, url string) string {
 	return filepath.Clean(targetDir)
 }
 
-func (c *CAS) resolveReference(ctx context.Context, url, branch string) (string, error) {
+// resolvedRef is what [CAS.resolveReference] returns: a [symbolicRef]
+// when ls-remote resolved the input to a branch, tag, or HEAD; a
+// [commitRef] when it did not. Sealed by package visibility.
+type resolvedRef interface {
+	// CommitHash returns the canonical commit hash for [symbolicRef]
+	// and the user-supplied SHA for [commitRef]. Abbreviated SHAs
+	// passed in as commit refs remain abbreviated.
+	CommitHash() string
+}
+
+// symbolicRef carries an ls-remote-resolved branch, tag, or HEAD.
+type symbolicRef struct {
+	URL    string
+	Branch string // ref name, used for the per-ref fetch
+	Hash   string // canonical commit hash
+}
+
+// CommitHash returns the canonical commit hash ls-remote resolved.
+func (r *symbolicRef) CommitHash() string { return r.Hash }
+
+// commitRef carries a commit-form ref ls-remote did not recognize.
+// Resolution against the central git store happens later via
+// rev-parse, with a full-depth fetch on a cache miss.
+type commitRef struct {
+	URL    string
+	RawRef string // user-supplied SHA, full or abbreviated
+}
+
+// CommitHash returns the user-supplied SHA before central-store
+// canonicalization.
+func (r *commitRef) CommitHash() string { return r.RawRef }
+
+// resolveReference resolves branch into a [resolvedRef]. ls-remote is
+// the authoritative source for symbolic refs (branches, tags, HEAD);
+// a hit returns a [*symbolicRef] carrying the canonical hash. A miss
+// returns a [*commitRef] so the caller resolves the input locally
+// against the central git store, where commit-form SHAs (full or
+// abbreviated) are looked up via rev-parse and a full-history fetch.
+func (c *CAS) resolveReference(ctx context.Context, url, branch string) (resolvedRef, error) {
 	results, err := c.git.LsRemote(ctx, url, branch)
 	if err != nil {
-		return "", err
+		if errors.Is(err, git.ErrNoMatchingReference) {
+			return &commitRef{URL: url, RawRef: branch}, nil
+		}
+
+		return nil, err
 	}
 
 	if len(results) == 0 {
-		return "", &WrappedError{
-			Op:      "clone",
-			Context: "no matching reference",
-			Err:     ErrNoMatchingReference,
-		}
+		return &commitRef{URL: url, RawRef: branch}, nil
 	}
 
-	return results[0].Hash, nil
+	return &symbolicRef{URL: url, Branch: branch, Hash: results[0].Hash}, nil
 }
 
 // storeRootTreeFrom reads the recursive tree at hash from the supplied