feat: multi-size benchmarks (50MB, 200MB, 500MB) (#8)

## Summary

- Run read benchmarks on 3 file sizes (50MB, 200MB, 500MB) instead of
just 50MB
- All files uploaded to a single bucket, single mount per backend
- One comparison table per size in the output
- Write benchmark unchanged (500MB)
- CI posts all tables as PR comment

Author: XciD

.github/workflows/ci.yml

@@ -10,6 +10,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  pull-requests: write
+
 env:
   CARGO_TERM_COLOR: always
 
@@ -101,30 +105,47 @@ jobs:
       - name: Build release binaries
         run: cargo build --release --features nfs
 
+      - name: Install fio
+        run: sudo apt-get install -y fio
+
       - name: Run benchmarks
-        run: cargo test --release --test bench -- --nocapture 2>&1 | tee bench_output.txt
+        timeout-minutes: 10
+        run: cargo test --release --test bench --test fio_bench -- --nocapture 2>&1 | tee bench_output.txt
 
       - name: Post benchmark results
         if: github.event_name == 'pull_request'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Extract the benchmark table from output
-          TABLE=$(sed -n '/^  =====/,/^  =====/p' bench_output.txt)
-          if [ -z "$TABLE" ]; then
+          # Strip ANSI codes, then extract only ==== delimited blocks
+          CLEAN=$(sed 's/\x1b\[[0-9;]*m//g' bench_output.txt)
+          TABLES=$(echo "$CLEAN" | awk '
+            /^=====*$/ { inside=!inside; print; next }
+            inside { print }
+          ')
+          if [ -z "$TABLES" ]; then
             echo "No benchmark table found in output"
             exit 0
           fi
 
-          # Build markdown comment
-          BODY=$(cat <<'HEADER'
+          PR="${{ github.event.pull_request.number }}"
+          MARKER="<!-- bench-results -->"
+          BODY="${MARKER}
           ## Benchmark Results
 
-          ```
-          HEADER
-          )
-          BODY="${BODY}
-          ${TABLE}
+          \`\`\`
+          ${TABLES}
           \`\`\`"
 
-          gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY"
+          # Find existing comment with our marker
+          COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR}/comments" \
+            --jq ".[] | select(.body | startswith(\"${MARKER}\")) | .id" | head -1)
+
+          if [ -n "$COMMENT_ID" ]; then
+            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
+              -X PATCH -f body="$BODY"
+            echo "Updated existing comment $COMMENT_ID"
+          else
+            gh pr comment "$PR" --body "$BODY"
+            echo "Created new comment"
+          fi

tests/bench.rs

@@ -1,33 +1,73 @@
 mod common;
 
-const FILE_SIZE: usize = 50 * 1024 * 1024; // 50 MB
-const WRITE_SIZE: usize = 500 * 1024 * 1024; // 500 MB for write benchmarks
+const BENCH_SIZES: &[(usize, &str)] = &[
+    (50 * 1024 * 1024, "50MB"),
+    (200 * 1024 * 1024, "200MB"),
+    (500 * 1024 * 1024, "500MB"),
+];
 
 #[tokio::test]
 async fn test_bench() {
-    let test_filename = format!("bench_{}.bin", std::process::id());
-    let expected = common::generate_pattern(FILE_SIZE);
-
-    let (token, bucket_id, hub) = match common::setup_bucket_with_file("bench", &test_filename, &expected).await {
+    // Upload files of each size to the same bucket
+    let (token, bucket_id, hub) = match common::setup_bucket("bench").await {
         Some(cfg) => cfg,
         None => return,
     };
 
+    let mut files: Vec<(String, Vec<u8>)> = Vec::new();
+    for &(size, label) in BENCH_SIZES {
+        let filename = format!("bench_{}.bin", label);
+        let data = common::generate_pattern(size);
+        let write_config = common::build_write_config(&hub).await;
+
+        let tmp_dir = std::env::temp_dir().join(format!("hf-mount-bench-setup-{}", label));
+        std::fs::create_dir_all(&tmp_dir).ok();
+        let staging_path = tmp_dir.join(&filename);
+        std::fs::write(&staging_path, &data).expect("write staging file");
+
+        let file_info = common::upload_file(write_config, &staging_path).await;
+        let xet_hash = file_info.hash().to_string();
+        eprintln!("Uploaded {}: xet_hash={}", label, xet_hash);
+
+        let mtime_ms = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis() as u64;
+
+        hub.batch_operations(&[hf_mount::hub_api::BatchOp::AddFile {
+            path: filename.clone(),
+            xet_hash,
+            mtime: mtime_ms,
+            content_type: None,
+        }])
+        .await
+        .expect("batch add failed");
+
+        std::fs::remove_dir_all(&tmp_dir).ok();
+        files.push((filename, data));
+    }
+
     let pid = std::process::id();
 
-    // --- FUSE benchmark (read + write) ---
+    // --- FUSE benchmark (read + write per size) ---
     let fuse_mount = format!("/tmp/hf-bench-fuse-{}", pid);
     let fuse_cache = format!("/tmp/hf-bench-fuse-cache-{}", pid);
 
     let fuse_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
         let child = common::mount_bucket(&bucket_id, &fuse_mount, &fuse_cache, &[]);
-        let r = common::bench::run_read_benchmarks(&fuse_mount, &test_filename, &expected);
 
-        // Write benchmark: 50 MB sequential write
-        let write_result = common::bench::run_write_benchmark(&fuse_mount, WRITE_SIZE);
+        let mut reads = Vec::new();
+        for (filename, expected) in &files {
+            reads.push(common::bench::run_read_benchmarks(&fuse_mount, filename, expected));
+        }
+
+        let mut writes = Vec::new();
+        for &(size, _) in BENCH_SIZES {
+            writes.push(common::bench::run_write_benchmark(&fuse_mount, size));
+        }
 
         common::unmount(&fuse_mount, child, 60);
-        (r, write_result)
+        (reads, writes)
     }));
 
     std::fs::remove_dir_all(&fuse_mount).ok();
@@ -39,161 +79,87 @@ async fn test_bench() {
 
     let nfs_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
         let child = common::mount_bucket_nfs(&bucket_id, &nfs_mount, &nfs_cache, &["--read-only"]);
-        let r = common::bench::run_read_benchmarks(&nfs_mount, &test_filename, &expected);
+        let mut reads = Vec::new();
+        for (filename, expected) in &files {
+            reads.push(common::bench::run_read_benchmarks(&nfs_mount, filename, expected));
+        }
         common::unmount_nfs(&nfs_mount, child, 5);
-        r
+        reads
     }));
 
     std::fs::remove_dir_all(&nfs_mount).ok();
     std::fs::remove_dir_all(&nfs_cache).ok();
 
-    // --- mount-s3 reference benchmark (read only, requires mount-s3 + AWS creds) ---
-    let s3_result = if common::s3_bench_available() {
-        let s3_bucket = common::setup_s3_bench(&expected);
-        let s3_mount = format!("/tmp/hf-bench-s3-{}", pid);
-
-        let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-            let child = common::mount_s3(&s3_bucket, &s3_mount);
-            let r = common::bench::run_read_benchmarks(&s3_mount, "bench_file.bin", &expected);
-            common::cleanup_s3_bench(&s3_mount, child, &s3_bucket);
-            r
-        }));
-
-        match r {
-            Ok(Ok(bench)) => Some(bench),
-            Ok(Err(e)) => {
-                eprintln!("mount-s3 bench error: {}", e);
-                None
-            }
-            Err(e) => std::panic::resume_unwind(e),
-        }
-    } else {
-        eprintln!("Skipping mount-s3 bench (mount-s3 binary or AWS creds not available)");
-        None
-    };
-
-    // --- Raw cleaner benchmark (no FUSE, no kernel) ---
-    let raw_mbps = {
-        let write_config = common::build_write_config(&hub).await;
-        let download_session = data::FileDownloadSession::new(write_config.clone(), None, None)
-            .await
-            .expect("download session");
-        let xet_sessions = hf_mount::xet::XetSessions::new(download_session, Some(write_config));
-        match common::bench::run_raw_cleaner_benchmark(&xet_sessions, WRITE_SIZE).await {
-            Ok(mbps) => Some(mbps),
-            Err(e) => {
-                eprintln!("Raw cleaner bench error: {}", e);
-                None
-            }
-        }
-    };
-
-    // Cleanup bucket
+    // Cleanup
     common::delete_bucket(&common::endpoint(), &token, &bucket_id).await;
 
     // --- Extract results ---
-    let (fuse, write) = match fuse_result {
-        Ok((Ok(r), w)) => (Some(r), w),
-        Ok((Err(e), _)) => {
-            eprintln!("FUSE bench error: {}", e);
-            (None, Err(e))
-        }
+    let (fuse_reads, fuse_writes) = match fuse_result {
+        Ok((reads, writes)) => (reads, writes),
         Err(e) => std::panic::resume_unwind(e),
     };
 
-    let nfs = match nfs_result {
-        Ok(Ok(r)) => Some(r),
-        Ok(Err(e)) => {
-            eprintln!("NFS bench error: {}", e);
-            None
-        }
+    let nfs_reads = match nfs_result {
+        Ok(reads) => reads,
         Err(e) => std::panic::resume_unwind(e),
     };
 
-    // --- Print comparison table ---
-    eprintln!("\n============================================================");
-    eprintln!("  FUSE vs NFS vs mount-s3 Benchmark — 50 MB file");
-    eprintln!("============================================================");
-    eprintln!("  {:30} {:>12} {:>12} {:>12}", "Metric", "FUSE", "NFS", "mount-s3");
-    eprintln!("  {:-<30} {:-<12} {:-<12} {:-<12}", "", "", "", "");
-
-    if let (Some(f), Some(n)) = (&fuse, &nfs) {
-        let s3_read = s3_result.as_ref().map(|s| format!("{:>9.1} MB/s", s.seq_read_mbps));
-        let s3_reread = s3_result.as_ref().map(|s| format!("{:>9.1} MB/s", s.seq_reread_mbps));
-        let s3_range = s3_result.as_ref().map(|s| format!("{:>9.1} ms", s.range_read_ms));
-        let s3_rand = s3_result.as_ref().map(|s| format!("{:>9.1} ms", s.random_avg_ms));
-        let dash = "         —".to_string();
-
-        eprintln!(
-            "  {:30} {:>9.1} MB/s {:>9.1} MB/s {:>12}",
-            "Sequential read",
-            f.seq_read_mbps,
-            n.seq_read_mbps,
-            s3_read.as_deref().unwrap_or(&dash)
-        );
-        eprintln!(
-            "  {:30} {:>9.1} MB/s {:>9.1} MB/s {:>12}",
-            "Sequential re-read",
-            f.seq_reread_mbps,
-            n.seq_reread_mbps,
-            s3_reread.as_deref().unwrap_or(&dash)
-        );
-        eprintln!(
-            "  {:30} {:>9.1} ms   {:>9.1} ms   {:>12}",
-            "Range read (1MB@25MB)",
-            f.range_read_ms,
-            n.range_read_ms,
-            s3_range.as_deref().unwrap_or(&dash)
-        );
-        eprintln!(
-            "  {:30} {:>9.1} ms   {:>9.1} ms   {:>12}",
-            "Random reads (100x4KB avg)",
-            f.random_avg_ms,
-            n.random_avg_ms,
-            s3_rand.as_deref().unwrap_or(&dash)
-        );
-        let f_speedup = if f.neg_cache_second_ms > 0.0 {
-            f.neg_cache_first_ms / f.neg_cache_second_ms
-        } else {
-            f64::INFINITY
-        };
-        let n_speedup = if n.neg_cache_second_ms > 0.0 {
-            n.neg_cache_first_ms / n.neg_cache_second_ms
+    // --- Print tables per size ---
+    for (i, &(_, label)) in BENCH_SIZES.iter().enumerate() {
+        let fuse = fuse_reads[i].as_ref().ok();
+        let nfs = nfs_reads[i].as_ref().ok();
+        let write = fuse_writes[i].as_ref().ok();
+
+        eprintln!("\n============================================================");
+        eprintln!("  Benchmark — {}", label);
+        eprintln!("------------------------------------------------------------");
+        eprintln!("  {:30} {:>12} {:>12}", "Metric", "FUSE", "NFS");
+        eprintln!("  {:-<30} {:-<12} {:-<12}", "", "", "");
+
+        if let (Some(f), Some(n)) = (fuse, nfs) {
+            eprintln!(
+                "  {:30} {:>9.1} MB/s {:>9.1} MB/s",
+                "Sequential read", f.seq_read_mbps, n.seq_read_mbps
+            );
+            eprintln!(
+                "  {:30} {:>9.1} MB/s {:>9.1} MB/s",
+                "Sequential re-read", f.seq_reread_mbps, n.seq_reread_mbps
+            );
+            eprintln!(
+                "  {:30} {:>9.1} ms   {:>9.1} ms",
+                "Range read (1MB@25MB)", f.range_read_ms, n.range_read_ms
+            );
+            eprintln!(
+                "  {:30} {:>9.1} ms   {:>9.1} ms",
+                "Random reads (100x4KB avg)", f.random_avg_ms, n.random_avg_ms
+            );
         } else {
-            f64::INFINITY
-        };
-        let s3_speedup = s3_result.as_ref().map(|s| {
-            if s.neg_cache_second_ms > 0.0 {
-                format!("{:>9.1}x", s.neg_cache_first_ms / s.neg_cache_second_ms)
-            } else {
-                "      inf".to_string()
+            if fuse.is_none() {
+                eprintln!("  FUSE read: FAILED");
             }
-        });
-        eprintln!(
-            "  {:30} {:>9.1}x     {:>9.1}x     {:>12}",
-            "Neg cache speedup",
-            f_speedup,
-            n_speedup,
-            s3_speedup.as_deref().unwrap_or(&dash)
-        );
-    }
+            if nfs.is_none() {
+                eprintln!("  NFS read: FAILED");
+            }
+        }
 
-    if let Ok(ref w) = write {
-        eprintln!("  {:30} {:>9.1} MB/s", "Sequential write (500MB)", w.write_mbps);
-        eprintln!("  {:30} {:>9.3} s", "Close latency (CAS+Hub)", w.close_secs);
-        eprintln!("  {:30} {:>9.1} MB/s", "Write end-to-end", w.total_mbps);
-        eprintln!("  {:30} {:>9.1} MB/s", "Dedup write (same data)", w.dedup_write_mbps);
-        eprintln!("  {:30} {:>9.3} s", "Dedup close latency", w.dedup_close_secs);
-        eprintln!("  {:30} {:>9.1} MB/s", "Dedup end-to-end", w.dedup_total_mbps);
-    } else {
-        eprintln!("  {:30} {:>12}", "Write benchmark", "FAIL");
-    }
-    if let Some(raw) = raw_mbps {
-        eprintln!("  {:30} {:>9.1} MB/s", "Raw add_data() (no FUSE)", raw);
+        if let Some(w) = write {
+            eprintln!("  {:30} {:>9.1} MB/s", "Sequential write (FUSE)", w.write_mbps);
+            eprintln!("  {:30} {:>9.3} s", "Close latency (CAS+Hub)", w.close_secs);
+            eprintln!("  {:30} {:>9.1} MB/s", "Write end-to-end", w.total_mbps);
+            eprintln!("  {:30} {:>9.1} MB/s", "Dedup write", w.dedup_write_mbps);
+            eprintln!("  {:30} {:>9.3} s", "Dedup close latency", w.dedup_close_secs);
+            eprintln!("  {:30} {:>9.1} MB/s", "Dedup end-to-end", w.dedup_total_mbps);
+        } else {
+            eprintln!("  Write: FAILED");
+        }
+        eprintln!("============================================================");
     }
-    eprintln!("============================================================\n");
+    eprintln!();
 
-    assert!(fuse.is_some(), "FUSE benchmark failed");
-    assert!(nfs.is_some(), "NFS benchmark failed");
-    write.expect("Write benchmark failed");
+    // Assertions
+    for (i, &(_, label)) in BENCH_SIZES.iter().enumerate() {
+        assert!(fuse_reads[i].is_ok(), "FUSE read benchmark failed for {}", label);
+        assert!(nfs_reads[i].is_ok(), "NFS read benchmark failed for {}", label);
+        assert!(fuse_writes[i].is_ok(), "Write benchmark failed for {}", label);
+    }
 }