feat(caching): add negative entry caching to stat cache

cfg/config.go

@@ -682,6 +682,8 @@ type MetadataCacheConfig struct {
 
 	EnableMetadataPrefetch bool `yaml:"enable-metadata-prefetch"`
 
+	EnableNonexistentEntryCaching bool `yaml:"enable-nonexistent-entry-caching"`
+
 	EnableNonexistentTypeCache bool `yaml:"enable-nonexistent-type-cache"`
 
 	ExperimentalMetadataPrefetchOnMount string `yaml:"experimental-metadata-prefetch-on-mount"`
@@ -999,6 +1001,8 @@ func BuildFlagSet(flagSet *pflag.FlagSet) error {
 		return err
 	}
 
+	flagSet.BoolP("enable-nonexistent-entry-caching", "", false, "Cache paths confirmed to not exist (404s) in the unified stat cache.")
+
 	flagSet.BoolP("enable-nonexistent-type-cache", "", false, "Once set, if an inode is not found in GCS, a type cache entry with type NonexistentType will be created. This also means new file/dir created might not be seen. For example, if this flag is set, and metadata-cache-ttl-secs is set, then if we create the same file/node in the meantime using the same mount, since we are not refreshing the cache, it will still return nil. This flag has been deprecated in favour of a single unified flag metadata-cache-negative-ttl-secs.")
 
 	flagSet.BoolP("enable-rapid-appends", "", true, "Enables support for appends to unfinalized object using streaming writes")
@@ -1608,6 +1612,10 @@ func BindFlags(v *viper.Viper, flagSet *pflag.FlagSet) error {
 		return err
 	}
 
+	if err := v.BindPFlag("metadata-cache.enable-nonexistent-entry-caching", flagSet.Lookup("enable-nonexistent-entry-caching")); err != nil {
+		return err
+	}
+
 	if err := v.BindPFlag("metadata-cache.enable-nonexistent-type-cache", flagSet.Lookup("enable-nonexistent-type-cache")); err != nil {
 		return err
 	}

cfg/params.yaml

@@ -949,6 +949,12 @@ params:
     deprecated: false
     hide-flag: false
 
+  - config-path: "metadata-cache.enable-nonexistent-entry-caching"
+    flag-name: "enable-nonexistent-entry-caching"
+    type: "bool"
+    usage: "Cache paths confirmed to not exist (404s) in the unified stat cache."
+    default: false
+
   - config-path: "metadata-cache.enable-nonexistent-type-cache"
     flag-name: "enable-nonexistent-type-cache"
     type: "bool"

cmd/mount.go

@@ -99,6 +99,7 @@ be interacting with the file system.`)
 		StatCacheMaxSizeMB:                 uint64(newConfig.MetadataCache.StatCacheMaxSizeMb),
 		StatCacheTTL:                       time.Duration(newConfig.MetadataCache.TtlSecs) * time.Second,
 		NegativeStatCacheTTL:               time.Duration(newConfig.MetadataCache.NegativeTtlSecs) * time.Second,
+		EnableNonexistentEntryCaching:      newConfig.MetadataCache.EnableNonexistentEntryCaching,
 		EnableMonitoring:                   cfg.IsMetricsEnabled(&newConfig.Metrics),
 		LogSeverity:                        newConfig.Logging.Severity,
 		AppendThreshold:                    1 << 21, // 2 MiB, a total guess.

docs/semantics.md

@@ -209,7 +209,11 @@ The behavior of stat cache is controlled by the following flags/config parameter
    If this config variable is missing, then the value of ```--stat-cache-ttl``` is used.
    * ```--stat-cache-ttl``` commandline flag, which can be set to a value like ```10s``` or ```1.5h```. The default is one minute. This has been deprecated (starting v2.0) and is currently only available for backward compatibility. If ```metadata-cache: ttl-secs``` is set, ```--stat-cache-ttl``` is ignored.
 
-   Positive and negative stat results will be cached for the specified amount of time.
+   Positive stat results (existing files) will be cached for this specified amount of time.
+
+3. **Negative Stat-cache (Non-existent Entry Caching)**: Controls caching of non-existent paths (`ErrNotExist` / 404 results) to optimize workloads that aggressively poll missing files (e.g., JupyterLab).
+   * `metadata-cache: enable-nonexistent-entry-caching` in the config-file (boolean). Must be set to `true` to enable negative caching. **Disabled by default.**
+   * `metadata-cache: negative-ttl-secs` in the config-file (integer). Sets the TTL in seconds for negative entries when `enable-nonexistent-entry-caching` is `true`. Default is 5 seconds. Use -1 for infinite TTL or 0 to bypass.
 
 Warnings: 
 - Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations:

internal/fs/caching_test.go

@@ -42,6 +42,7 @@ import (
 
 const ttl = 10 * time.Minute
 const negativeCacheTTL = 1 * time.Minute
+const enableNonexistentEntryCaching = true
 
 var (
 	uncachedBucket gcs.Bucket
@@ -69,6 +70,7 @@ func (t *cachingTestCommon) SetUpTestSuite() {
 		negativeCacheTTL,
 		IsTypeCacheDeprecated,
 		isImplicitDir,
+		enableNonexistentEntryCaching,
 	)
 
 	// Enable directory type caching.
@@ -479,6 +481,7 @@ func (t *MultiBucketMountCachingTest) SetUpTestSuite() {
 			negativeCacheTTL,
 			IsTypeCacheDeprecated,
 			isImplicitDir,
+			enableNonexistentEntryCaching,
 		)
 	}
 

internal/fs/gcs_metrics_test.go

@@ -20,11 +20,16 @@ import (
 	"os"
 	"testing"
 
+	"time"
+
 	"github.com/googlecloudplatform/gcsfuse/v3/cfg"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/monitor"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil"
@@ -463,3 +468,83 @@ func TestGCSMetrics_RetryCount(t *testing.T) {
 		attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")),
 		1)
 }
+
+type fakeBucketManagerForShortCircuit struct {
+	bucket gcs.Bucket
+}
+
+func (bm *fakeBucketManagerForShortCircuit) SetUpBucket(ctx context.Context, name string, _ bool, _ metrics.MetricHandle) (gcsx.SyncerBucket, error) {
+	return gcsx.NewSyncerBucket(0, 120, 10, ".gcsfuse_tmp/", gcsx.NewContentTypeBucket(bm.bucket)), nil
+}
+func (bm *fakeBucketManagerForShortCircuit) ShutDown() {}
+
+// TestGCSMetrics_RequestCount_NegativeCachingShortCircuit validates that when negative entry caching is enabled,
+// repeated lookups for non-existent files short-circuit in memory and do not emit redundant backend GCS requests.
+func TestGCSMetrics_RequestCount_NegativeCachingShortCircuit(t *testing.T) {
+	ctx := context.Background()
+	origProvider := otel.GetMeterProvider()
+	t.Cleanup(func() { otel.SetMeterProvider(origProvider) })
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	otel.SetMeterProvider(provider)
+
+	mh, err := metrics.NewOTelMetrics(ctx, 1, 100)
+	require.NoError(t, err, "metrics.NewOTelMetrics")
+
+	bucketName := "test-bucket"
+	rawBucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Hierarchical: false})
+
+	// Enforce production wrapping order: Caching layer MUST wrap Monitoring layer.
+	monitoringInnerBucket := monitor.NewMonitoringBucket(rawBucket, mh)
+	lruCache := lru.NewCache(1024 * 1024)
+	statCacheView := metadata.NewStatCacheBucketView(lruCache, "")
+	cachedOuterBucket := caching.NewFastStatBucket(
+		time.Minute,
+		statCacheView,
+		timeutil.RealClock(),
+		monitoringInnerBucket,
+		time.Minute,
+		true, // isTypeCacheDeprecated
+		true, // implicitDir
+		true, // enableNonexistentEntryCaching
+	)
+
+	serverCfg := &fs.ServerConfig{
+		NewConfig: &cfg.Config{
+			EnableNewReader: true,
+		},
+		MetricHandle: mh,
+		TraceHandle:  tracing.NewNoopTracer(),
+		CacheClock:   &timeutil.SimulatedClock{},
+		BucketName:   bucketName,
+		BucketManager: &fakeBucketManagerForShortCircuit{
+			bucket: cachedOuterBucket,
+		},
+	}
+
+	server, err := fs.NewFileSystem(ctx, serverCfg)
+	require.NoError(t, err, "NewFileSystem")
+
+	lookupOp := &fuseops.LookUpInodeOp{
+		Parent: fuseops.RootInodeID,
+		Name:   "missing_file",
+	}
+
+	// 1. First Lookup (Cache Miss) -> Emits backend probes through monitoring layer
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// Probing a missing file checks dir then file -> 2 actual backend network calls.
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		2)
+
+	// 2. Second Lookup (Negative Cache Hit) -> Intercepted by outer cache layer
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// Verify backend request count remains exactly 2 (zero new network requests emitted)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		2)
+}

internal/fs/hns_bucket_test.go

@@ -172,7 +172,8 @@ func (t *HNSCachedBucketMountTest) SetupSuite() {
 		uncachedHNSBucket,
 		negativeCacheTTL,
 		IsTypeCacheDeprecated,
-		isImplicitDir)
+		isImplicitDir,
+		enableNonexistentEntryCaching)
 
 	// Enable directory type caching.
 	t.serverCfg.DirTypeCacheTTL = ttl

internal/fs/inode/dir.go

@@ -686,6 +686,7 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error)
 		// 1. Try Directory FIRST (since it's the preferred return type)
 		var dirResult *Core
 		var err error
+		var dirCacheMiss bool
 		if d.Bucket().BucketType().Hierarchical {
 			dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), true)
 		} else {
@@ -696,19 +697,27 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error)
 		if dirResult != nil {
 			return dirResult, nil
 		}
-		// If we hit a real error (not a cache miss), exit early.
-		if err != nil && !errors.As(err, &cacheMissErr) {
+		if errors.As(err, &cacheMissErr) {
+			dirCacheMiss = true
+		} else if err != nil {
 			return nil, err
 		}
 
 		// 2. Try File ONLY if directory wasn't found
+		var fileCacheMiss bool
 		fileResult, err := findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), true)
-		if err != nil && !errors.As(err, &cacheMissErr) {
+		if fileResult != nil {
+			return fileResult, nil
+		}
+		if errors.As(err, &cacheMissErr) {
+			fileCacheMiss = true
+		} else if err != nil {
 			return nil, err
 		}
 
-		if fileResult != nil {
-			return fileResult, nil
+		// 3. Short-circuit on confirmed negative hits for BOTH dir and file
+		if !dirCacheMiss && !fileCacheMiss {
+			return nil, nil // Definitive ENOENT
 		}
 	}
 

internal/gcsx/bucket_manager.go

@@ -45,9 +45,10 @@ type BucketConfig struct {
 	// Config for TTL of entries for existing file in stat cache
 	StatCacheTTL time.Duration
 	// Config for TTL of entries for non-existing file in stat cache
-	NegativeStatCacheTTL time.Duration
-	EnableMonitoring     bool
-	LogSeverity          cfg.LogSeverity
+	NegativeStatCacheTTL          time.Duration
+	EnableNonexistentEntryCaching bool
+	EnableMonitoring              bool
+	LogSeverity                   cfg.LogSeverity
 
 	// Files backed by on object of length at least AppendThreshold that have
 	// only been appended to (i.e. none of the object's contents have been
@@ -244,7 +245,8 @@ func (bm *bucketManager) SetUpBucket(
 			b,
 			bm.config.NegativeStatCacheTTL,
 			bm.config.IsTypeCacheDeprecated,
-			bm.config.ImplicitDir)
+			bm.config.ImplicitDir,
+			bm.config.EnableNonexistentEntryCaching)
 	}
 
 	// Enable content type awareness

internal/storage/caching/fast_stat_bucket.go

@@ -51,15 +51,17 @@ func NewFastStatBucket(
 	negativeCacheTTL time.Duration,
 	isTypeCacheDeprecated bool,
 	implicitDir bool,
+	enableNonexistentEntryCaching bool,
 ) (b gcs.Bucket) {
 	fsb := &fastStatBucket{
-		cache:                 cache,
-		clock:                 clock,
-		wrapped:               wrapped,
-		primaryCacheTTL:       primaryCacheTTL,
-		negativeCacheTTL:      negativeCacheTTL,
-		isTypeCacheDeprecated: isTypeCacheDeprecated,
-		implicitDir:           implicitDir,
+		cache:                         cache,
+		clock:                         clock,
+		wrapped:                       wrapped,
+		primaryCacheTTL:               primaryCacheTTL,
+		negativeCacheTTL:              negativeCacheTTL,
+		isTypeCacheDeprecated:         isTypeCacheDeprecated,
+		implicitDir:                   implicitDir,
+		enableNonexistentEntryCaching: enableNonexistentEntryCaching,
 	}
 
 	b = fsb
@@ -92,6 +94,8 @@ type fastStatBucket struct {
 	isTypeCacheDeprecated bool
 
 	implicitDir bool
+
+	enableNonexistentEntryCaching bool
 }
 
 ////////////////////////////////////////////////////////////////////////
@@ -124,6 +128,11 @@ func (b *fastStatBucket) insertListing(ctx context.Context, listing *gcs.Listing
 		return
 	}
 
+	if len(listing.MinObjects) == 0 && len(listing.CollapsedRuns) == 0 && b.enableNonexistentEntryCaching && strings.HasSuffix(dirName, "/") {
+		b.cache.AddNegativeEntry(dirName, b.clock.Now().Add(b.negativeCacheTTL))
+		return
+	}
+
 	expiration := b.clock.Now().Add(b.primaryCacheTTL)
 
 	// 1. Parent Directory Inference (Implicit Check)
@@ -244,6 +253,11 @@ func (b *fastStatBucket) addNegativeEntry(name string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	if !b.enableNonexistentEntryCaching {
+		b.cache.Erase(name)
+		return
+	}
+
 	expiration := b.clock.Now().Add(b.negativeCacheTTL)
 	b.cache.AddNegativeEntry(name, expiration)
 }
@@ -253,6 +267,11 @@ func (b *fastStatBucket) addNegativeEntryForFolder(name string) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
+	if !b.enableNonexistentEntryCaching {
+		b.cache.Erase(name)
+		return
+	}
+
 	expiration := b.clock.Now().Add(b.negativeCacheTTL)
 	b.cache.AddNegativeEntryForFolder(name, expiration)
 }
@@ -271,6 +290,9 @@ func (b *fastStatBucket) lookUp(name string) (hit bool, m *gcs.MinObject) {
 	defer b.mu.Unlock()
 
 	hit, m = b.cache.LookUp(name, b.clock.Now())
+	if hit && m == nil && !b.enableNonexistentEntryCaching {
+		return false, nil
+	}
 	return
 }
 
@@ -279,6 +301,9 @@ func (b *fastStatBucket) lookUpFolder(name string) (bool, *gcs.Folder) {
 	defer b.mu.Unlock()
 
 	hit, f := b.cache.LookUpFolder(name, b.clock.Now())
+	if hit && f == nil && !b.enableNonexistentEntryCaching {
+		return false, nil
+	}
 	return hit, f
 }