merge: Merge pull request #8 from St4NNi/feat/reworkstorage

feat: Feat/reworkstorage

Author: St4NNi

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "jam-rs"
-version = "0.9.9"
+version = "0.9.10"
 edition = "2024"
 repository = "https://github.com/St4NNi/jam-rs"
 license = "MIT"
@@ -27,23 +27,32 @@ name = "jam_rs"
 path = "src/lib.rs"
 
 [dependencies]
-anyhow = "1.0.100"
+anyhow = "1.0.101"
+dashmap = "6.1.0"
 jamhash = "0.1.2"
-needletail = "0.6.3"
+needletail = { version = "0.6.3", features = [
+    "flate2",
+    "bzip2",
+    "xz2",
+    "zstd",
+] }
 rayon = "1.11.0"
 serde = { version = "1.0.228", features = ["derive"] }
-clap = { version = "4.5.54", features = ["derive"] }
-itertools = "0.14.0"
-heed = "0.22.1-nested-rtxns-6"
-byteorder = "1.5.0"
-integer-encoding = "4.1.0"
-indicatif = { version = "0.18.3", features = ["rayon"] }
+serde_json = "1.0"
+clap = { version = "4.5.57", features = ["derive"] }
 tempfile = "3.24.0"
-crossbeam-channel = "0.5.15"
-serde_json = "1.0.149"
+crossfire = "3.0.4"
+bytemuck = { version = "1.25.0", features = ["derive"] }
+memmap2 = "0.9"
+xorf = { version = "0.12", features = ["serde"] }
+thiserror = "2"
+indicatif = "0.18.3"
+
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+tikv-jemallocator = "0.6"
 
 [dev-dependencies]
-criterion = { version = "0.8.1", features = ["html_reports"] }
+criterion = { version = "0.8.2", features = ["html_reports"] }
 jamhash = "0.1.2"
 murmurhash3 = { version = "0.0.5" }
 fastmurmur3 = "0.2.0"

Cargo.toml

@@ -6,40 +6,33 @@
 
 # jam-rs
 
-Just another minhash (jam) implementation. A high-performance minhashing tool for genomic sequence similarity analysis, specifically optimized for plasmids and other small genomic elements.
+Just another minhash (jam). A high-performance FracMinHash implementation for genomic sequence similarity analysis, optimized for searching plasmids, phages, and other small genomic elements in large datasets.
 
-Implements the FracMinHash algorithm for rapid similarity comparisons with enhanced metadata tracking including GC content and sequence length categories tuned for typical plasmid ranges.
+jam uses a custom hash function ([jamhash](https://github.com/St4NNi/jamhash)) that provides lower collision rates, 2-10x higher speed and better uniformity than murmur3. It also includes a compact memory-mapped database format (`.jam`) for fast random access, and a bias filtering system based on Count-Min Sketches to selectively increase sensitivity for target sequences.
 
 ### Installation
 
-Install the latest release from [crates.io](https://crates.io/):
+From [crates.io](https://crates.io/crates/jam-rs):
 
 ```bash
 cargo install jam-rs
 ```
 
-Or install the development version from git:
+From source:
 
 ```bash
 cargo install --git https://github.com/St4NNi/jam-rs
 ```
 
 ### Key Features
 
-- **Plasmid-optimized**: GC content and length categories specifically tuned for plasmid analysis (30-70% GC, 1kB-500kB lengths)
-- **Fast sketching**: Entropy-filtered k-mers with optimized hash functions to exclude low-complexity regions
-- **Rich metadata**: Enhanced metadata tracking with file index, GC category, and length category for each hash
+- **Custom hash function**: [jamhash](https://github.com/St4NNi/jamhash) provides lower collisions, better uniformity and is faster compared to murmur3
+- **Bias-aware sketching**: Count-Min Sketch based compositional filtering with automatic background extraction
+- **Complexity filtering**: Shannon entropy threshold to exclude low-complexity k-mers
 - **Memory-efficient**: External sorting for processing datasets larger than available RAM
-- **LMDB storage**: Fast random access and compact representation with dual database structure
-- **Parallel execution**: File-level parallelization with configurable thread count
-
-### Scaling Methods
-
-Multiple scaling methods for different use cases:
-- **FracMinHash** (`--fscale`): Restricts hash-space to a fraction of `u64::MAX` / `fscale`
-- **Max hashes** (`--nmax`): Limits maximum number of hashes per sequence (memory control)
-- **Complexity filtering** (`--complexity`): Only hash sequences with Shannon entropy above threshold (default: 0.0)
-- **Singleton mode** (`--singleton`): Creates separate sketch per sequence record
+- **Compact storage**: 256-bucket memory-mapped `.jam` format with binary fuse filters for fast random access
+- **Parallel execution**: File-level parallelization via rayon with configurable thread count
+- **Tuned for speed**: jemalloc allocator, LTO, single codegen unit, `opt-level = 3`
 
 ### Usage
 
@@ -51,21 +44,23 @@ Usage: jam [OPTIONS] <COMMAND>
 
 Commands:
   sketch  Sketch one or more files and write the result to an output file
-  dist    Estimate containment of a (small) sketch against a subset of one or more sketches as database. Requires all sketches to have the same kmer size
-  stats   Display statistics about an LMDB database
+  dist    Estimate containment of a query sequence against a sketch database
+  bias    Build and analyze hash bias tables for filtering
+  stats   Display statistics about a JAM database
   help    Print this message or the help of the given subcommand(s)
 
 Options:
   -t, --threads <THREADS>  Number of threads to use [default: 1]
   -f, --force              Overwrite output files
-  -s, --silent             Silent mode, no (additional) output to stdout Only errors and output files will be printed
+  -s, --silent             Silent mode, no (additional) output to stdout
+  -m, --memory <MEMORY>    Maximum memory usage in GB [default: 2]
   -h, --help               Print help
   -V, --version            Print version
 ```
 
 #### Sketching
 
-Create sketches from FASTA/FASTQ files. Supports single files, multiple files, or directories.
+Create `.jam` databases from FASTA/FASTQ files (plain or gzip/bzip2/xz/zstd compressed). Supports single files, multiple files, or directories.
 
 ```console
 $ jam sketch --help
@@ -77,104 +72,134 @@ Arguments:
   [INPUT]...  Input file(s), directories, or file with list of files to be hashed
 
 Options:
-  -o, --output <OUTPUT>          Output file (.lmdb will be appended if not present)
+  -o, --output <OUTPUT>          Output file (.jam format)
   -k, --kmer-size <KMER_SIZE>    K-mer size, all sketches must have the same size to be compared and below 32 [default: 21]
       --fscale <FSCALE>          Scale the hash space to a minimum fraction of the maximum hash value (FracMinHash)
-      --nmax <NMAX>              Maximum number of k-mers (per record) to be hashed, top cut-off
-      --complexity <COMPLEXITY>  Complexity cut-off, only hash sequences with complexity above this value This is created via shannon entropy [default: 0.0]
-      --singleton                Create a separate sketch for each sequence record Will increase the size of the output file
-  -t, --threads <THREADS>        Number of threads to use [default: 1]
-  -f, --force                    Overwrite output files
+      --complexity <COMPLEXITY>   Complexity cut-off, only hash sequences with complexity above this value [default: 0.0]
+      --singleton                Create a separate sketch for each sequence record
+      --temp-dir <TEMP_DIR>      Custom temporary directory for intermediate files during sorting
+      --bias-table <BIAS_TABLE>  Path to a bias table file (.bias) for compositional filtering
   -h, --help                     Print help
 ```
 
 Examples:
 ```bash
-# Basic plasmid sketching
-jam sketch plasmid.fasta -o sketch.lmdb
+# Sketch a single file
+jam sketch input.fasta -o sketch.jam
 
-# Multiple plasmid files with custom k-mer size
-jam sketch plasmids/ -o plasmid_db.lmdb -k 21 -t 8
+# Sketch a directory with 8 threads and FracMinHash scaling
+jam sketch genomes/ -o db.jam --fscale 1000 -t 8
 
-# Large collections with memory limits and complexity filtering
-jam sketch large_collection/ -o database.lmdb --nmax 10000 --fscale 1000000 --complexity 1.5
+# Filter low-complexity k-mers by Shannon entropy
+jam sketch genomes/ -o db.jam --fscale 1000 --complexity 1.5
 
-# Separate sketch per plasmid sequence
-jam sketch multi_plasmids.fasta -o sketches.lmdb --singleton
+# One sketch per sequence record
+jam sketch multi.fasta -o db.jam --singleton
+
+# Apply bias filtering during sketching
+jam sketch plasmids/ -o filtered.jam --bias-table host_filter.bias
 ```
 
-#### Distance Calculation
+#### Querying
 
-Compare sequences against a sketch database. Supports both raw sequence files and pre-computed sketches.
+Estimate containment of query sequences against a sketch database.
 
 ```console
 $ jam dist --help
-Estimate containment of a (small) sketch against a subset of one or more sketches as database. Requires all sketches to have the same kmer size
+Estimate containment of a query sequence against a sketch database
 
 Usage: jam dist [OPTIONS] --input <INPUT> --database <DATABASE>
 
 Options:
-  -i, --input <INPUT>        Input sketch or raw sequence file
-  -d, --database <DATABASE>  Database sketch (.lmdb file)
+  -i, --input <INPUT>        Input FASTA/FASTQ file to query
+  -d, --database <DATABASE>  Database sketch (.jam file)
   -o, --output <OUTPUT>      Output to file instead of stdout
   -c, --cutoff <CUTOFF>      Cut-off value for similarity/containment [default: 0.0]
-      --singleton            Singleton mode, process each query sequence separately
-  -t, --threads <THREADS>    Number of threads to use [default: 1]
-  -f, --force                Overwrite output files
+      --singleton             Singleton mode, process each query sequence separately
   -h, --help                 Print help
 ```
 
 Examples:
 ```bash
-# Query plasmid against database
-jam dist -i query_plasmid.fasta -d plasmid_db.lmdb -c 0.1 -o results.tsv
-
-# Sketch-to-sketch comparison
-jam dist -i query.lmdb -d plasmid_db.lmdb -c 0.05
+# Query against a database with a containment cutoff
+jam dist -i query.fasta -d db.jam -c 0.1 -o results.tsv
 
-# Process each sequence separately with singleton mode
-jam dist -i multi_query.fasta -d plasmid_db.lmdb --singleton -c 0.1
+# Per-sequence queries
+jam dist -i multi_query.fasta -d db.jam --singleton -c 0.1
 ```
 
-#### Statistics
+Output is tab-separated: `query`, `sample_id`, `hit_count`, `containment`.
 
-Display database statistics including hash counts and distribution analysis.
+#### Bias Table Construction
+
+Bias tables allow compositional filtering to increase sensitivity for target sequences while suppressing background noise. They work by scoring k-mers based on their enrichment in a positive (target) set relative to a negative (background) set.
+
+The underlying data structure is a **Count-Min Sketch (CMS)**, a probabilistic structure that approximates k-mer frequencies using multiple independent hash functions mapped to a fixed-width table. This keeps memory usage constant regardless of the number of distinct k-mers. By default, the CMS uses 1,048,576 columns and 5 hash functions (~5 MB).
+
+**How it works:**
+
+1. K-mer frequencies from both the positive and negative input sets are counted into separate CMS tables.
+2. **Background extraction**: The positive counts are subtracted from the negative counts (floored at zero). This prevents k-mers naturally shared between target and background from being penalized.
+3. A log-ratio weight is computed per CMS cell: `log((pos + alpha) / (adjusted_neg + alpha))`, where `alpha` is a smoothing parameter.
+4. Weights are quantized to `i8` (-127 to +127) for compact storage.
+5. **Threshold calibration**: All 255 possible thresholds are evaluated. The threshold that maximizes fold enrichment (positive retention / negative retention) is selected. If a target fold enrichment is specified, the closest achievable threshold is used instead.
 
 ```console
-$ jam stats --help
-Display statistics about an LMDB database
+$ jam bias create --help
+Create a bias table from positive (target) and negative (background) FASTA files.
+Target signal is always subtracted from background before computing bias weights.
 
-Usage: jam stats [OPTIONS] --input <INPUT>
+Usage: jam bias create [OPTIONS] --positive <POSITIVE> --negative <NEGATIVE> --output <OUTPUT>
 
 Options:
-  -i, --input <INPUT>      Input LMDB database
-  -s, --short              Short summary only
-  -t, --threads <THREADS>  Number of threads to use [default: 1]
-  -f, --force              Overwrite output files
-  -h, --help               Print help
+      --positive <POSITIVE>              Positive (target) FASTA file(s)
+      --negative <NEGATIVE>              Negative (background) FASTA file(s)
+  -o, --output <OUTPUT>                  Output bias table file (.bias)
+  -k, --kmer-size <KMER_SIZE>            K-mer size (must match sketch) [default: 21]
+      --fscale <FSCALE>                  FracMinHash scale (must match sketch) [default: 1000]
+      --cms-width <CMS_WIDTH>            CMS columns, power of 2 recommended [default: 1048576]
+      --cms-depth <CMS_DEPTH>            CMS hash functions [default: 5]
+      --alpha <ALPHA>                    Smoothing parameter for log-ratio [default: 1.0]
+      --fold-enrichment <FOLD_ENRICHMENT>  Target fold enrichment (auto-maximized if not set)
+      --threads <THREADS>                Number of threads
+  -h, --help                             Print help
 ```
 
 Examples:
 ```bash
-# Summary statistics
-jam stats -i plasmid_db.lmdb --short
+# Build a bias table to filter out host sequences
+jam bias create --positive plasmids.fasta --negative host_genome.fasta -o host_filter.bias
 
-# Detailed distributions
-jam stats -i plasmid_db.lmdb
+# With custom fold enrichment target
+jam bias create --positive targets.fasta --negative background.fasta -o filter.bias --fold-enrichment 10.0
+
+# Inspect a bias table
+jam bias stats filter.bias
+jam bias stats filter.bias -o report.json
 ```
 
-### Output Format
+#### Statistics
 
-Distance results are tab-separated with columns:
-```
-query  target  containment_query_in_target  containment_target_in_query  jaccard  shared_hashes  query_hashes  target_hashes
-```
+Display database statistics including hash counts and distribution analysis.
+
+```console
+$ jam stats --help
+Display statistics about a JAM database
 
-Statistics include hash counts, GC content distribution, and sequence length categories optimized for plasmids and small genomic elements.
+Usage: jam stats [OPTIONS] --input <INPUT>
 
-### Algorithm
+Options:
+  -i, --input <INPUT>  Input JAM database (.jam file)
+      --short          Short summary only
+      --full           Include the full entry statistics
+  -h, --help           Print help
+```
 
-JAM uses entropy-filtered k-mers to exclude low-complexity regions, stores rich metadata (file index, GC category, length category) with each hash, and employs external sorting for memory-efficient processing of large datasets. The categorization system is specifically tuned for plasmid analysis, with fine-grained bins in typical plasmid size and GC content ranges.
+Examples:
+```bash
+jam stats -i db.jam --short
+jam stats -i db.jam --full
+```
 
 ### License
 
@@ -186,4 +211,4 @@ If you have any ideas, suggestions, or issues, please don't hesitate to open an
 
 ### Credits
 
-This tool is inspired by [finch-rs](https://github.com/onecodex/finch-rs) and [sourmash](https://github.com/sourmash-bio/sourmash). Check them out if you need a more mature ecosystem with well tested hash functions and more features.
+This tool is inspired by [finch-rs](https://github.com/onecodex/finch-rs) and [sourmash](https://github.com/sourmash-bio/sourmash). Check them out if you need a more mature ecosystem.

README.md

@@ -26,8 +26,7 @@ fn bench_hash_functions(c: &mut Criterion) {
     group.warm_up_time(Duration::from_millis(500));
     group.measurement_time(Duration::from_secs(2));
 
-    // Pre-generate enough values for the benchmark
-    let mut values = (0..100000u64).into_iter().cycle();
+    let mut values = (0..100000u64).cycle();
 
     group.bench_function("xxhash3", |b| {
         b.iter(|| xxhash3(black_box(values.next().unwrap())))