chore: add integration tests and initial ci

.github/workflows/CI.yml

@@ -1,11 +1,17 @@
-name: Rust
+name: Rust CI
 
-on: push
+# Run this workflow on pull requests targeting main and on pushes to main
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
+
   ci:
     needs: [build, test]
     name: CI
@@ -22,7 +28,8 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
       - name: Build
         run: |
@@ -40,4 +47,4 @@ jobs:
 
       - name: Build
         run: |
-          cargo build --verbose
+          cargo build --verbose

.gitignore

@@ -3,3 +3,5 @@ Cargo.lock
 
 vendor/ncbi-vdb/comp/**
 vendor/ncbi-vdb/reconfigure
+
+tests/fixtures/data/

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xsra"
-version = "0.2.23"
+version = "0.2.24"
 edition = "2021"
 license = "MIT"
 authors = ["Noam Teyssier <noam.teyssier@arcinstitute.org>"]
@@ -33,6 +33,8 @@ tokio = { version = "1.44.1", features = ["rt", "rt-multi-thread"] }
 zstd = { version = "0.13.2", features = ["zstdmt"] }
 
 [dev-dependencies]
+assert_cmd = "2.0.17"
+predicates = "3.1.3"
 mockito = "1.7.0"
 tempfile = "3.20.0"
 tokio-test = "0.4.4"

src/lib.rs

@@ -0,0 +1,14 @@
+// Library interface for xsra
+// This allows integration tests to access the internal modules
+
+// Re-export constants needed by modules
+pub const BUFFER_SIZE: usize = 1024 * 1024;
+pub const RECORD_CAPACITY: usize = 1024;
+
+pub mod cli;
+pub mod describe;
+pub mod dump;
+pub mod output;
+pub mod prefetch;
+pub mod recode;
+pub mod utils;

src/prefetch/mod.rs

@@ -1,18 +1,18 @@
-use std::fs::File;
-use std::io::{BufWriter, Write};
-use std::sync::Arc;
-use std::time::Duration;
-
+use crate::cli::{AccessionOptions, MultiInputOptions, Provider};
 use anyhow::{bail, Result};
 use futures::{future::join_all, stream::FuturesUnordered, StreamExt};
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
+use std::{
+    fs::File,
+    io::{BufWriter, Write},
+    sync::Arc,
+    time::Duration,
+};
 use tokio::{sync::Semaphore, time::sleep};
 
 #[cfg(not(test))]
 use reqwest::Client;
 
-use crate::cli::{AccessionOptions, MultiInputOptions, Provider};
-
 /// Semaphore for rate limiting (NCBI limits to 3 requests per second)
 pub const RATE_LIMIT_SEMAPHORE: usize = 3;
 
@@ -583,8 +583,8 @@ mod tests {
     }
 
     // identify_url tests
-    #[test]
-    fn identify_url_succeeds_with_proper_provider() {
+    #[tokio::test]
+    async fn identify_url_succeeds_with_proper_provider() {
         let options = AccessionOptions {
             full_quality: true,
             lite_only: false,
@@ -594,10 +594,8 @@ mod tests {
             gcp_project_id: Some("test-project".to_string()),
         };
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(identify_url("SRR123456", &options));
-
         // Test with SRR123456 which returns GCP URLs in our mock
+        let result = identify_url("SRR123456", &options).await;
         assert!(
             result.is_ok(),
             "Failed to identify GCP URL: {:?}",
@@ -608,8 +606,8 @@ mod tests {
         assert_eq!(url, "gs://test-bucket/sra/SRR123456/SRR123456.sra");
     }
 
-    #[test]
-    fn identify_url_fails_with_unsupported_provider() {
+    #[tokio::test]
+    async fn identify_url_fails_with_unsupported_provider() {
         let options = AccessionOptions {
             full_quality: true,
             lite_only: false,
@@ -619,18 +617,16 @@ mod tests {
             gcp_project_id: None,
         };
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(identify_url("SRR123456", &options));
+        let result = identify_url("SRR123456", &options).await;
         assert!(result.is_err());
     }
 
-    #[test]
-    fn identify_url_fails_with_zero_retries() {
+    #[tokio::test]
+    async fn identify_url_fails_with_zero_retries() {
         let mut options = create_test_accession_options();
         options.retry_limit = 0;
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(identify_url("INVALID", &options));
+        let result = identify_url("INVALID", &options).await;
         assert!(result.is_err());
         assert!(result
             .unwrap_err()
@@ -764,8 +760,8 @@ mod tests {
     }
 
     // prefetch tests
-    #[test]
-    fn prefetch_fails_with_empty_accessions() {
+    #[tokio::test]
+    async fn prefetch_fails_with_empty_accessions() {
         let input = MultiInputOptions {
             accessions: vec![],
             options: AccessionOptions {
@@ -778,17 +774,16 @@ mod tests {
             },
         };
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(prefetch(&input, None));
+        let result = prefetch(&input, None).await;
         assert!(result.is_err());
         assert!(result
             .unwrap_err()
             .to_string()
             .contains("No accessions provided"));
     }
 
-    #[test]
-    fn prefetch_fails_with_unsupported_aws_provider() {
+    #[tokio::test]
+    async fn prefetch_fails_with_unsupported_aws_provider() {
         let input = MultiInputOptions {
             accessions: vec!["SRR123456".to_string()],
             options: AccessionOptions {
@@ -801,8 +796,7 @@ mod tests {
             },
         };
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(prefetch(&input, None));
+        let result = prefetch(&input, None).await;
         assert!(result.is_err());
         let err_msg = result.unwrap_err().to_string();
         assert_eq!(
@@ -811,8 +805,8 @@ mod tests {
         );
     }
 
-    #[test]
-    fn prefetch_fails_with_gcp_provider_missing_project_id() {
+    #[tokio::test]
+    async fn prefetch_fails_with_gcp_provider_missing_project_id() {
         let input = MultiInputOptions {
             accessions: vec!["SRR123456".to_string()],
             options: AccessionOptions {
@@ -825,8 +819,7 @@ mod tests {
             },
         };
 
-        let runtime = tokio::runtime::Runtime::new().unwrap();
-        let result = runtime.block_on(prefetch(&input, None));
+        let result = prefetch(&input, None).await;
         assert!(result.is_err());
         let err_msg = result.unwrap_err().to_string();
         assert!(err_msg.contains("GCP project ID is required for GCP downloads"));

tests/describe.rs

@@ -0,0 +1,66 @@
+use anyhow::Result;
+use xsra::describe::describe_inner;
+
+mod fixtures;
+use fixtures::TestFixtures;
+
+/// Integration tests for describe
+
+#[test]
+fn test_describe_with_valid_sra_fixture() -> Result<()> {
+    let fixtures = TestFixtures::ensure_fixtures()?;
+
+    // Test describe with valid SRA file
+    let stats = describe_inner(&fixtures.small_variable_sra, 0, 100)?;
+
+    // Verify we got meaningful results using available methods
+    let segment_lengths = stats.segment_lengths();
+    assert!(
+        segment_lengths.len() > 0,
+        "No segments found in valid SRA file"
+    );
+    assert!(
+        segment_lengths.iter().any(|&len| len > 0.0),
+        "No meaningful segment lengths found"
+    );
+
+    // Test that we can serialize the stats (exercises the pprint functionality)
+    let mut output = Vec::new();
+    stats.pprint(&mut output)?;
+    assert!(
+        output.len() > 0,
+        "Stats serialization produced empty output"
+    );
+
+    Ok(())
+}
+
+#[test]
+fn test_describe_with_invalid_sra_fixture() -> Result<()> {
+    let fixtures = TestFixtures::ensure_fixtures()?;
+
+    // Test describe with invalid SRA file - should fail
+    let result = describe_inner(&fixtures.invalid_sra, 0, 10);
+
+    assert!(
+        result.is_err(),
+        "Expected describe to fail with invalid SRA file"
+    );
+
+    Ok(())
+}
+
+#[test]
+fn test_describe_with_corrupt_sra_fixture() -> Result<()> {
+    let fixtures = TestFixtures::ensure_fixtures()?;
+
+    // Test describe with corrupt SRA file - should fail
+    let result = describe_inner(&fixtures.corrupt_sra, 0, 10);
+
+    assert!(
+        result.is_err(),
+        "Expected describe to fail with corrupt SRA file"
+    );
+
+    Ok(())
+}