fix(orchestrator): measure ext4 free space from block groups (#3282)

The ensure-free-disk phase previously trusted debugfs's top-level "Free
blocks" value, which comes from the ext4 superblock. After an unclean
shutdown, Linux can leave this aggregate stale, and e2fsck -E
journal_only returns before rebuilding it from the block-group
descriptors. This could cause builds to make an incorrect resize
decision and leave less free space than requested.

Flush the NBD device after replaying the journal, then calculate free
space by summing debugfs's per-group counters and subtracting reserved
blocks. Require one counter for every group so incomplete or unexpected
output fails instead of silently producing an incorrect result. Pin
LC_ALL=C because the parser relies on debugfs's English field names.

Add Linux unit coverage for stale global counters, multi-group sums,
singular block output, incomplete group output, and global-only output.

Author: gm-e2b

packages/orchestrator/pkg/template/build/core/filesystem/ext4.go

@@ -26,6 +26,12 @@ import (
 
 var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/core/filesystem")
 
+var (
+	groupHeaderPattern     = regexp.MustCompile(`(?m)^[ \t]*Group[ \t]+\d+:`)
+	groupFreeBlocksPattern = regexp.MustCompile(`(?m)^[ \t]+(\d+)[ \t]+free blocks?,`)
+	reservedBlocksPattern  = regexp.MustCompile(`Reserved block count:\s+(\d+)`)
+)
+
 const (
 	// creates an inode for every bytes-per-inode byte of space on the disk
 	inodesRatio = int64(4096)
@@ -196,6 +202,8 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6
 	defer statSpan.End()
 
 	cmd := exec.CommandContext(ctx, "debugfs", "-R", "stats", rootfsPath)
+	// The parser below relies on debugfs's English field names.
+	cmd.Env = append(os.Environ(), "LC_ALL=C")
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	err := cmd.Run()
@@ -206,7 +214,6 @@ func GetFreeSpace(ctx context.Context, rootfsPath string, blockSize int64) (int6
 		return 0, fmt.Errorf("error statting ext4: %w", err)
 	}
 
-	// Extract block size and free blocks
 	freeBlocks, err := parseFreeBlocks(output)
 	if err != nil {
 		return 0, fmt.Errorf("could not parse free blocks: %w", err)
@@ -376,25 +383,38 @@ func LogMetadata(ctx context.Context, rootfsPath string, extraFields ...zap.Fiel
 	logger.L().With(extraFields...).Debug(ctx, "tune2fs -l output", zap.String("path", rootfsPath), zap.String("output", string(output)), zap.Error(err))
 }
 
-// parseFreeBlocks extracts the "Free blocks:" value from debugfs output
+// parseFreeBlocks returns the sum of the free-block counters stored in the
+// block-group descriptors. e2fsck documents that the global count can be stale
+// after an unclean unmount and normally rebuilds it from the group counts, but
+// its journal_only path skips that repair:
+// https://github.com/tytso/e2fsprogs/blob/v1.47.0/e2fsck/unix.c#L370-L441
+// debugfs prints each group from ext2fs_bg_free_blocks_count(), so those are the
+// counters we must sum after replaying only the journal:
+// https://github.com/tytso/e2fsprogs/blob/v1.47.0/debugfs/debugfs.c#L486-L501
 func parseFreeBlocks(debugfsOutput string) (int64, error) {
-	re := regexp.MustCompile(`Free blocks:\s+(\d+)`)
-	matches := re.FindStringSubmatch(debugfsOutput)
-	if len(matches) < 2 {
-		return 0, errors.New("could not find free blocks in debugfs output")
+	groups := groupHeaderPattern.FindAllStringIndex(debugfsOutput, -1)
+	matches := groupFreeBlocksPattern.FindAllStringSubmatch(debugfsOutput, -1)
+	// Require one counter per group so truncated or unexpected debugfs output
+	// cannot silently undercount free space and trigger an incorrect resize.
+	if len(groups) == 0 || len(matches) != len(groups) {
+		return 0, fmt.Errorf("could not parse free blocks for every block group: found %d groups and %d counters", len(groups), len(matches))
 	}
-	freeBlocks, err := strconv.ParseInt(matches[1], 10, 64)
-	if err != nil {
-		return 0, fmt.Errorf("could not parse free blocks: %w", err)
+
+	var freeBlocks int64
+	for _, match := range matches {
+		groupFreeBlocks, err := strconv.ParseInt(match[1], 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("could not parse block-group free blocks: %w", err)
+		}
+		freeBlocks += groupFreeBlocks
 	}
 
 	return freeBlocks, nil
 }
 
-// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output
+// parseReservedBlocks extracts the "Reserved block count:" value from debugfs output.
 func parseReservedBlocks(debugfsOutput string) (int64, error) {
-	re := regexp.MustCompile(`Reserved block count:\s+(\d+)`)
-	matches := re.FindStringSubmatch(debugfsOutput)
+	matches := reservedBlocksPattern.FindStringSubmatch(debugfsOutput)
 	if len(matches) < 2 {
 		return 0, errors.New("could not find reserved blocks in debugfs output")
 	}

packages/orchestrator/pkg/template/build/core/filesystem/ext4_test.go

@@ -19,18 +19,33 @@ func TestParseFreeBlocks(t *testing.T) {
 		wantErr  bool
 	}{
 		{
-			name:     "standard debugfs output",
-			input:    "Block count:              131072\nFree blocks:              120000\nFirst block:              0\n",
-			expected: 120000,
+			name: "sums group counters instead of stale superblock",
+			input: `Free blocks:              50000
+ Group  0: block bitmap at 1, inode bitmap at 2, inode table at 3
+           28629 free blocks, 100 free inodes, 1 used directory
+ Group  1: block bitmap at 4, inode bitmap at 5, inode table at 6
+           28639 free blocks, 100 free inodes, 1 used directory
+`,
+			expected: 57268,
+		},
+		{
+			name: "supports singular block counter",
+			input: ` Group  0: block bitmap at 1, inode bitmap at 2, inode table at 3
+           1 free block, 1 free inode, 1 used directory
+`,
+			expected: 1,
 		},
 		{
-			name:     "large block count",
-			input:    "Free blocks:              999999999\n",
-			expected: 999999999,
+			name: "rejects incomplete group output",
+			input: ` Group  0: block bitmap at 1, inode bitmap at 2, inode table at 3
+           10 free blocks, 100 free inodes, 1 used directory
+ Group  1: block bitmap at 4, inode bitmap at 5, inode table at 6
+`,
+			wantErr: true,
 		},
 		{
-			name:    "missing free blocks",
-			input:   "Block count:              131072\n",
+			name:    "rejects global counter without block groups",
+			input:   "Free blocks:              50000\n",
 			wantErr: true,
 		},
 	}

packages/orchestrator/pkg/template/build/phases/ensurefreedisk/grow.go

@@ -126,6 +126,11 @@ func (b *EnsureFreeDiskBuilder) measureFree(
 	if _, err := filesystem.ReplayJournal(ctx, device.path); err != nil {
 		return 0, fmt.Errorf("replay source journal: %w", err)
 	}
+	// Make the recovered block-group metadata visible from the backend before
+	// debugfs reopens the device and reads its free-space counters.
+	if err := device.mnt.Flush(ctx); err != nil {
+		return 0, fmt.Errorf("flush recovered source journal: %w", err)
+	}
 	free, err = filesystem.GetFreeSpace(ctx, device.path, blockSize)
 	if err != nil {
 		return 0, fmt.Errorf("measure source free space: %w", err)