quality pass: bugfixes, hot-path performance, and consolidation

CHANGELOG.md

@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
 
 ## [Unreleased]
 
+### Added
+
+- `omnifs dev` and `omnifs up` now bind providers' required host paths into the runtime container, so providers like the SQLite db provider can reach their backing files.
+
+### Changed
+
+- Faster reads and directory listings, with lower memory use on large directories and objects. Output is unchanged.
+
+### Fixed
+
+- The negative-lookup cache no longer grows without bound on long-running mounts with many missing-path lookups.
+- The arXiv provider no longer crashes when it fails to encode a JSON response.
+
 ## [0.2.1] - 2026-06-08
 
 ### Fixed

crates/omnifs-cache/src/lib.rs

@@ -39,6 +39,14 @@ use dashmap::DashMap;
 /// Shared handle to a host view store.
 pub type Handle = Arc<Store>;
 
+/// One entry for `Store::put_canonical_batch`.
+pub struct CanonicalBatchEntry {
+    pub id: Vec<u8>,
+    pub bytes: Vec<u8>,
+    pub validator: Option<String>,
+    pub view_leaves: Vec<String>,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[repr(u8)]
 pub enum RecordKind {
@@ -164,6 +172,18 @@ const TOMBSTONE_SOFT_CAP: usize = 4096;
 /// Generations of tombstone history retained after GC.
 const TOMBSTONE_RETAIN_GENERATIONS: u64 = 1024;
 
+/// Soft cap on retained negatives per mount. `gc_negatives` fires past this.
+const NEGATIVES_SOFT_CAP: usize = 4096;
+
+/// Wall-clock milliseconds since the Unix epoch. Used only for GC sweep timing
+/// so that `delete_object` does not need a caller-supplied clock argument.
+fn now_millis_for_gc() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
+        .unwrap_or(0)
+}
+
 /// Process-global cache handles. Opened once at startup; shared via `Arc`.
 ///
 /// `Caches::open(dir)` creates a durable `object.redb` and deletes+recreates
@@ -431,6 +451,43 @@ impl Store {
             })
     }
 
+    /// Batch canonical store for effect application. Fenced per-entry; rejected
+    /// entries are skipped and the rest of the batch proceeds. Prior-leaf view
+    /// evictions fire per-object before the batch redb commit.
+    ///
+    /// Ownership is consumed so the caller need not clone; the function drains
+    /// the Vec.
+    pub fn put_canonical_batch(&self, entries: Vec<CanonicalBatchEntry>, op_gen: u64) {
+        let view = &self.caches.view;
+
+        // Per-entry fence check and view eviction; collect accepted entries.
+        let batch: Vec<object::StoreBatchEntry> = entries
+            .into_iter()
+            .filter_map(|entry| {
+                let scoped_id = self.scoped_id(&entry.id);
+                if self.id_tombstoned_after(&scoped_id, op_gen) {
+                    return None;
+                }
+                // Evict prior view leaves before the object is replaced.
+                for scoped_leaf in self.caches.object.leaves_of(&scoped_id) {
+                    view.delete_exact(&scoped_leaf);
+                }
+                let scoped_leaves: Vec<String> =
+                    entry.view_leaves.iter().map(|p| self.scoped(p)).collect();
+                Some(object::StoreBatchEntry {
+                    scoped_id,
+                    canonical: object::Canonical {
+                        bytes: entry.bytes,
+                        validator: entry.validator,
+                    },
+                    new_leaves: scoped_leaves,
+                })
+            })
+            .collect();
+
+        self.caches.object.store_batch(&batch);
+    }
+
     /// Preload index-only store, fenced. Canonical-beats-preload in the object tier.
     pub fn put_index_only(&self, id: &[u8], view_leaves: &[String], op_gen: u64) -> bool {
         let scoped_id = self.scoped_id(id);
@@ -577,6 +634,9 @@ impl Store {
             .object
             .evict_object(&scoped_id, |scoped_leaf| view.delete_exact(scoped_leaf));
         self.gc_tombstones();
+        if self.negatives.len() > NEGATIVES_SOFT_CAP {
+            self.gc_negatives(now_millis_for_gc());
+        }
     }
 
     /// View-only listing invalidation at an exact path.
@@ -610,6 +670,38 @@ impl Store {
             .saturating_sub(TOMBSTONE_RETAIN_GENERATIONS);
         self.tombstones.retain(|_, g| *g >= cutoff);
     }
+
+    /// Prune expired negative entries from `negatives` and keep `neg_by_id`
+    /// consistent. The caller is responsible for checking the soft cap before
+    /// calling (see `delete_object`).
+    fn gc_negatives(&self, now_millis: u64) {
+        // Collect paths of expired entries.
+        let expired: Vec<String> = self
+            .negatives
+            .iter()
+            .filter_map(|entry| {
+                let expired = entry
+                    .value()
+                    .expires_at
+                    .is_some_and(|exp| now_millis >= exp);
+                expired.then(|| entry.key().clone())
+            })
+            .collect();
+
+        for path in &expired {
+            if let Some((_, neg)) = self.negatives.remove(path) {
+                // Drop this path from the reverse index; remove empty sets.
+                if let Some(id) = &neg.id {
+                    if let Some(mut paths_set) = self.neg_by_id.get_mut(id) {
+                        paths_set.remove(path);
+                    }
+                    // Remove the reverse-index entry if its set is now empty.
+                    self.neg_by_id
+                        .remove_if(id, |_, paths_set| paths_set.is_empty());
+                }
+            }
+        }
+    }
 }
 
 pub(crate) fn path_prefix_matches(prefix: &str, path: &str) -> bool {
@@ -935,4 +1027,73 @@ mod tests {
             "/owner/repobaz should remain"
         );
     }
+
+    /// `gc_negatives` prunes expired entries, keeps fresh ones, and leaves
+    /// `neg_by_id` consistent after the sweep.
+    #[test]
+    fn gc_negatives_prunes_expired_keeps_fresh_and_stays_consistent() {
+        let (_dir, _caches, store) = open_store("m");
+        let now = 5_000_u64;
+        let expired_path = "/issues/1/missing";
+        let fresh_path = "/issues/2/missing";
+        let no_ttl_path = "/issues/3/missing";
+
+        let id_expired = b"obj:expired" as &[u8];
+        let id_fresh = b"obj:fresh" as &[u8];
+
+        // Expired: TTL puts deadline in the past relative to `now`.
+        assert!(store.put_negative(&p(expired_path), Some(id_expired), 0, 1_000, 1_000));
+        // Fresh: deadline is in the future relative to `now`.
+        assert!(store.put_negative(&p(fresh_path), Some(id_fresh), 0, 10_000, now));
+        // No TTL (no expiry): must never be swept.
+        assert!(store.put_negative(&p(no_ttl_path), None, 0, 0, now));
+
+        // Force gc_negatives by lowering the negatives count below threshold is
+        // impractical for a unit test, so call the private helper directly.
+        store.gc_negatives(now);
+
+        // The expired negative must be gone.
+        assert!(
+            store.negative_for(&p(expired_path), now).is_none(),
+            "expired negative should have been pruned"
+        );
+        // The fresh negative must survive.
+        assert!(
+            store.negative_for(&p(fresh_path), now).is_some(),
+            "fresh negative should be retained"
+        );
+        // The no-TTL negative must survive.
+        assert!(
+            store.negative_for(&p(no_ttl_path), now).is_some(),
+            "no-TTL negative should be retained"
+        );
+
+        // Reverse index consistency: id_expired must have no entry (or empty set).
+        let scoped_expired = {
+            let mut k = "m".as_bytes().to_vec();
+            k.push(0x1f);
+            k.extend_from_slice(id_expired);
+            k
+        };
+        let scoped_fresh = {
+            let mut k = "m".as_bytes().to_vec();
+            k.push(0x1f);
+            k.extend_from_slice(id_fresh);
+            k
+        };
+        assert!(
+            store
+                .neg_by_id
+                .get(&scoped_expired)
+                .is_none_or(|s| s.is_empty()),
+            "neg_by_id for expired id should be absent or empty"
+        );
+        assert!(
+            store
+                .neg_by_id
+                .get(&scoped_fresh)
+                .is_some_and(|s| !s.is_empty()),
+            "neg_by_id for fresh id should still have entries"
+        );
+    }
 }