feat(caching): implement negative stat cache to optimize polling of missing files

docs/semantics.md

@@ -209,7 +209,12 @@ The behavior of stat cache is controlled by the following flags/config parameter
    If this config variable is missing, then the value of ```--stat-cache-ttl``` is used.
    * ```--stat-cache-ttl``` commandline flag, which can be set to a value like ```10s``` or ```1.5h```. The default is one minute. This has been deprecated (starting v2.0) and is currently only available for backward compatibility. If ```metadata-cache: ttl-secs``` is set, ```--stat-cache-ttl``` is ignored.
 
-   Positive and negative stat results will be cached for the specified amount of time.
+   Positive stat results (existing files) will be cached for this specified amount of time.
+
+3. **Negative Stat-cache (Non-existent Entry Caching)**: Controls caching of non-existent paths (`ErrNotExist` / 404 results) to optimize workloads that aggressively poll missing files (e.g., JupyterLab).
+   * `metadata-cache: negative-ttl-secs` in the config-file (integer). Sets the TTL in seconds for negative entries. Default is 5 seconds. Use -1 for infinite TTL. Setting this to 0 disables negative caching.
+      > [!WARNING]
+      > Using an infinite TTL (`-1`) for negative caching is risky if objects are created externally (by other clients or processes), as those new files/directories will never be visible under the mount until the cache is manually cleared or GCSFuse is restarted.
 
 Warnings: 
 - Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations:

internal/fs/gcs_metrics_test.go

@@ -20,11 +20,16 @@ import (
 	"os"
 	"testing"
 
+	"time"
+
 	"github.com/googlecloudplatform/gcsfuse/v3/cfg"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/monitor"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil"
@@ -463,3 +468,82 @@ func TestGCSMetrics_RetryCount(t *testing.T) {
 		attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")),
 		1)
 }
+
+type fakeBucketManagerForShortCircuit struct {
+	bucket gcs.Bucket
+}
+
+func (bm *fakeBucketManagerForShortCircuit) SetUpBucket(ctx context.Context, name string, _ bool, _ metrics.MetricHandle) (gcsx.SyncerBucket, error) {
+	return gcsx.NewSyncerBucket(0, 120, 10, ".gcsfuse_tmp/", gcsx.NewContentTypeBucket(bm.bucket)), nil
+}
+func (bm *fakeBucketManagerForShortCircuit) ShutDown() {}
+
+// TestGCSMetrics_RequestCount_NegativeCachingShortCircuit validates that when negative entry caching is enabled,
+// repeated lookups for non-existent files short-circuit in memory and do not emit redundant backend GCS requests.
+func TestGCSMetrics_RequestCount_NegativeCachingShortCircuit(t *testing.T) {
+	ctx := context.Background()
+	origProvider := otel.GetMeterProvider()
+	t.Cleanup(func() { otel.SetMeterProvider(origProvider) })
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	otel.SetMeterProvider(provider)
+
+	mh, err := metrics.NewOTelMetrics(ctx, 1, 100)
+	require.NoError(t, err, "metrics.NewOTelMetrics")
+
+	bucketName := "test-bucket"
+	rawBucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Hierarchical: false})
+
+	// Enforce production wrapping order: Caching layer MUST wrap Monitoring layer.
+	monitoringInnerBucket := monitor.NewMonitoringBucket(rawBucket, mh)
+	lruCache := lru.NewCache(1024 * 1024)
+	statCacheView := metadata.NewStatCacheBucketView(lruCache, "")
+	cachedOuterBucket := caching.NewFastStatBucket(
+		time.Minute,
+		statCacheView,
+		timeutil.RealClock(),
+		monitoringInnerBucket,
+		time.Minute,
+		true, // isTypeCacheDeprecated
+		true, // implicitDir
+	)
+
+	serverCfg := &fs.ServerConfig{
+		NewConfig: &cfg.Config{
+			EnableNewReader: true,
+		},
+		MetricHandle: mh,
+		TraceHandle:  tracing.NewNoopTracer(),
+		CacheClock:   &timeutil.SimulatedClock{},
+		BucketName:   bucketName,
+		BucketManager: &fakeBucketManagerForShortCircuit{
+			bucket: cachedOuterBucket,
+		},
+	}
+
+	server, err := fs.NewFileSystem(ctx, serverCfg)
+	require.NoError(t, err, "NewFileSystem")
+
+	lookupOp := &fuseops.LookUpInodeOp{
+		Parent: fuseops.RootInodeID,
+		Name:   "missing_file",
+	}
+
+	// 1. First Lookup (Cache Miss) -> Emits backend probes through monitoring layer
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// Probing a missing file checks dir then file -> 2 actual backend network calls.
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		2)
+
+	// 2. Second Lookup (Negative Cache Hit) -> Intercepted by outer cache layer
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// Verify backend request count remains exactly 2 (zero new network requests emitted)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		2)
+}

internal/storage/caching/fast_stat_bucket.go

@@ -135,6 +135,9 @@ func (b *fastStatBucket) insertListing(ctx context.Context, listing *gcs.Listing
 	if b.implicitDir && dirHasContents && !isDirInListing {
 		b.cache.InsertImplicitDir(dirName, expiration)
 	}
+	if b.implicitDir && !dirHasContents && dirName != "" && b.negativeCacheTTL > 0 {
+		b.cache.AddNegativeEntry(dirName, b.clock.Now().Add(b.negativeCacheTTL))
+	}
 
 	// 2. Cache Explicit Objects
 	for _, o := range listing.MinObjects {

internal/storage/caching/fast_stat_bucket_test.go

@@ -1015,6 +1015,9 @@ func (t *ListObjectsTest_InsertListing) callAndVerify(ctx context.Context, isHNS
 	for _, dir := range expectedImplicitDirs {
 		ExpectCall(t.cache, "InsertImplicitDir")(dir, Any())
 	}
+	if len(listing.MinObjects) == 0 && len(listing.CollapsedRuns) == 0 && prefix != "" {
+		ExpectCall(t.cache, "AddNegativeEntry")(prefix, Any())
+	}
 
 	// Call
 	gotListing, err := t.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{Prefix: prefix})

internal/storage/fake/bucket.go

@@ -30,6 +30,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil"
 	"github.com/jacobsa/syncutil"
@@ -900,6 +901,9 @@ func (b *bucket) ComposeObjects(
 // LOCKS_EXCLUDED(b.mu)
 func (b *bucket) StatObject(ctx context.Context,
 	req *gcs.StatObjectRequest) (m *gcs.MinObject, e *gcs.ExtendedObjectAttributes, err error) {
+	if req.FetchOnlyFromCache {
+		return nil, nil, &caching.CacheMissError{Err: errors.New("fake bucket has no cache")}
+	}
 	// If ExtendedObjectAttributes are requested without fetching from gcs enabled, panic.
 	if !req.ForceFetchFromGcs && req.ReturnExtendedObjectAttributes {
 		panic("invalid StatObjectRequest: ForceFetchFromGcs: false and ReturnExtendedObjectAttributes: true")
@@ -1143,6 +1147,9 @@ func (b *bucket) DeleteFolder(ctx context.Context, folderName string) (err error
 }
 
 func (b *bucket) GetFolder(ctx context.Context, req *gcs.GetFolderRequest) (*gcs.Folder, error) {
+	if req.FetchOnlyFromCache {
+		return nil, &caching.CacheMissError{Err: errors.New("fake bucket has no cache")}
+	}
 	b.mu.Lock()
 	defer b.mu.Unlock()