backingstore: add walk-prefetch-skip-lfs config flag

Summary: Add `backingstore.walk-prefetch-skip-lfs` config flag (default: true) that causes walk file prefetching to skip LFS files. This avoids wasting bandwidth on large LFS blob fetches during background walk prefetches, which are speculative and may never be needed.

Reviewed By: zzl0

Differential Revision: D98202367

fbshipit-source-id: 3715847e71362c13d965513260da879ce36955ad

Author: muirdm

eden/scm/lib/backingstore/src/backingstore.rs

@@ -254,6 +254,7 @@ impl BackingStore {
                     "walk-prefetch-min-interval",
                     || Duration::from_millis(10),
                 )?,
+                skip_lfs: config.get_or("backingstore", "walk-prefetch-skip-lfs", || true)?,
             };
 
             prefetch_manager(

eden/scm/lib/backingstore/src/prefetch.rs

@@ -51,6 +51,7 @@ pub(crate) struct Config {
     pub(crate) max_initial_lag: u64,
     pub(crate) min_ratio: f64,
     pub(crate) min_interval: Duration,
+    pub(crate) skip_lfs: bool,
 }
 
 impl Default for Config {
@@ -59,6 +60,7 @@ impl Default for Config {
             max_initial_lag: 1000,
             min_ratio: 0.1,
             min_interval: Duration::from_millis(10),
+            skip_lfs: true,
         }
     }
 }
@@ -484,7 +486,8 @@ fn prefetch(
             let fctx = FetchContext::new_with_mode_and_cause(
                 FetchMode::AllowRemote | FetchMode::IGNORE_RESULT,
                 FetchCause::EdenWalkPrefetch,
-            );
+            )
+            .with_skip_lfs(config.skip_lfs);
 
             // An important implementation detail for us: the scmstore FileStore spawns a thread
             // when you fetch more than 1_000 keys (i.e. this method will operate asynchronously if
@@ -1247,12 +1250,51 @@ mod test {
         assert!(handle.is_paused());
     }
 
+    #[test]
+    fn test_config_skip_lfs_defaults_true() {
+        let config = Config::default();
+        assert!(config.skip_lfs);
+    }
+
+    #[test]
+    fn test_prefetch_passes_skip_lfs() -> anyhow::Result<()> {
+        let tree_store = Arc::new(TestStore::new());
+        let file_store = Arc::new(TestStore::new());
+
+        let path: RepoPathBuf = "file.txt".to_string().try_into()?;
+        let hgid = file_store.insert_data(Default::default(), &path, b"content".into())?;
+
+        let mut mf = TreeManifest::ephemeral(tree_store);
+        mf.insert(path, FileMetadata::new(hgid, types::FileType::Regular))?;
+
+        let detector = walkdetector::Detector::new();
+        let handle = prefetch(
+            Config::default(),
+            mf,
+            file_store.clone(),
+            detector,
+            PrefetchWork::FileContent("".to_string().try_into()?, 0, None),
+            None,
+        );
+
+        while !handle.is_done() {
+            std::thread::sleep(Duration::from_millis(1));
+        }
+
+        let contexts = file_store.fetch_contexts();
+        assert_eq!(contexts.len(), 1);
+        assert!(contexts[0].skip_lfs());
+
+        Ok(())
+    }
+
     #[test]
     fn test_should_pause_prefetch() -> anyhow::Result<()> {
         let config = Config {
             min_ratio: 0.1,
             max_initial_lag: 20,
             min_interval: Duration::from_millis(1),
+            ..Default::default()
         };
 
         let mut detector = walkdetector::Detector::new();

eden/scm/lib/manifest-tree/src/testutil.rs

@@ -67,6 +67,8 @@ pub struct TestStoreInner {
     entries: HashMap<HgId, Bytes>,
     // Calls to get_content_iter().
     fetched: Vec<Vec<Key>>,
+    // FetchContexts passed to get_content_iter().
+    fetch_contexts: Vec<FetchContext>,
     // Parents recorded via insert_data with InsertOpts.parents.
     parents: HashMap<(RepoPathBuf, HgId), Vec<HgId>>,
     format: SerializationFormat,
@@ -89,6 +91,11 @@ impl TestStore {
         self.inner.read().fetched.clone()
     }
 
+    #[allow(unused)]
+    pub fn fetch_contexts(&self) -> Vec<FetchContext> {
+        self.inner.read().fetch_contexts.clone()
+    }
+
     pub fn key_fetch_count(&self) -> u64 {
         self.inner.read().key_fetch_count.load(Ordering::Relaxed)
     }
@@ -109,14 +116,15 @@ impl TestStore {
 impl KeyStore for TestStore {
     fn get_content_iter(
         &self,
-        _fctx: FetchContext,
+        fctx: FetchContext,
         keys: Vec<Key>,
     ) -> anyhow::Result<BoxIterator<anyhow::Result<(Key, Blob)>>> {
         let mut inner = self.inner.write();
         inner
             .key_fetch_count
             .fetch_add(keys.len() as u64, Ordering::Relaxed);
         inner.fetched.push(keys.clone());
+        inner.fetch_contexts.push(fctx);
         let entries = inner.entries.clone();
         drop(inner);
         let iter = keys