Merge pull request #22 from KontorProtocol/file-id-nonce

add nonce to file id and file metadata

Author: dime-squanderer

benches/bench_main.rs

@@ -96,7 +96,7 @@ fn file_preparation(bencher: Bencher, size_kb: usize) {
     bencher
         .with_inputs(|| generate_test_data(size_kb * 1024, 42))
         .bench_values(|data| {
-            api::prepare_file(black_box(&data), "test.dat").unwrap();
+            api::prepare_file(black_box(&data), "test.dat", b"").unwrap();
         });
 }
 
@@ -118,7 +118,7 @@ mod proving {
 
         for i in 0..num_files {
             let data = generate_test_data(size_kb * 1024, 42 + i as u64);
-            let (prepared, metadata) = api::prepare_file(&data, &format!("f{}", i)).unwrap();
+            let (prepared, metadata) = api::prepare_file(&data, &format!("f{}", i), b"").unwrap();
 
             ledger.add_file(&metadata).unwrap();
 
@@ -181,7 +181,7 @@ mod verification {
         num_challenges: usize,
     ) -> (FileLedger, api::Proof, Vec<Challenge>) {
         let data = generate_test_data(16 * 1024, 42);
-        let (prepared, metadata) = api::prepare_file(&data, "v.dat").unwrap();
+        let (prepared, metadata) = api::prepare_file(&data, "v.dat", b"").unwrap();
         let mut ledger = FileLedger::new();
         ledger.add_file(&metadata).unwrap();
 

src/api/mod.rs

@@ -38,9 +38,10 @@
 //! };
 //! use kontor_crypto::FileLedger;
 //!
-//! // 1. Prepare the file
+//! // 1. Prepare the file (nonce can be user_id, timestamp, or any unique bytes)
 //! let my_data = b"This is a test file for the PoR system.";
-//! let (prepared_file, metadata) = prepare_file(my_data, "test.dat").unwrap();
+//! let nonce = b"user_123"; // User-provided nonce for unique file_id
+//! let (prepared_file, metadata) = prepare_file(my_data, "test.dat", nonce).unwrap();
 //!
 //! // 2. Create ledger and add the file
 //! let mut ledger = FileLedger::new();
@@ -103,17 +104,25 @@ use tracing::debug_span;
 /// # Arguments
 ///
 /// * `data` - The raw data to be processed
-/// * `erasure_config` - Configuration for Reed-Solomon erasure coding
 /// * `filename` - Filename for operator UX and integration
+/// * `nonce` - A user-provided nonce included in the file_id hash. This enables the
+///   same data to have different file_ids (e.g., for different users or uploads).
+///   The nonce is stored in the returned `FileMetadata`.
 ///
 /// # Returns
 ///
 /// Returns a tuple of `(PreparedFile, FileMetadata)` where:
 /// - `PreparedFile` contains the private Merkle tree for the prover
 /// - `FileMetadata` contains the public commitment and reconstruction information
+///
+/// # ID Generation
+///
+/// - `object_id = object_<SHA256(data)>` - content-based, not unique, for file discovery
+/// - `file_id = file_<SHA256(data || nonce)>` - unique per upload, for storage protocol
 pub fn prepare_file(
     data: &[u8],
     filename: &str,
+    nonce: &[u8],
 ) -> Result<(types::PreparedFile, types::FileMetadata)> {
     let _span = debug_span!("prepare_file", data_size = data.len(), filename).entered();
 
@@ -123,10 +132,16 @@ pub fn prepare_file(
         });
     }
 
-    // 1. Calculate file ID
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    let file_id = format!("{:x}", hasher.finalize());
+    // 1a. Calculate object ID: object_<SHA256(data)> - content-based, for file discovery
+    let mut object_hasher = Sha256::new();
+    object_hasher.update(data);
+    let object_id = format!("obj_{:x}", object_hasher.finalize());
+
+    // 1b. Calculate file ID: file_<SHA256(data || nonce)> - unique per upload
+    let mut file_hasher = Sha256::new();
+    file_hasher.update(data);
+    file_hasher.update(nonce);
+    let file_id = format!("file_{:x}", file_hasher.finalize());
 
     // 2. Encode file into 31-byte symbols using multi-codeword RS
     let all_symbols = crate::erasure::encode_file_symbols(data)?;
@@ -142,7 +157,9 @@ pub fn prepare_file(
     // 5. Create metadata (num_data_symbols, num_codewords, total_symbols are derived)
     let metadata = types::FileMetadata {
         root,
+        object_id,
         file_id: file_id.clone(),
+        nonce: nonce.to_vec(),
         padded_len,
         original_size: data.len(),
         filename: filename.to_string(),
@@ -186,7 +203,7 @@ pub fn tree_depth_from_metadata(metadata: &types::FileMetadata) -> usize {
 /// use kontor_crypto::api::{prepare_file, reconstruct_file};
 ///
 /// let data = b"Hello, world!";
-/// let (prepared_file, metadata) = prepare_file(data, "example.dat").unwrap();
+/// let (prepared_file, metadata) = prepare_file(data, "example.dat", b"").unwrap();
 ///
 /// // Simulate having some symbols with some missing (for reconstruction testing)
 /// // In practice, you'd get these from the prepared_file or from storage

src/api/system.rs

@@ -45,9 +45,10 @@ impl<'a> PorSystem<'a> {
         &self,
         data: &[u8],
         filename: &str,
+        nonce: &[u8],
     ) -> Result<(PreparedFile, FileMetadata)> {
         // Use the existing prepare_file function from mod.rs
-        super::prepare_file(data, filename)
+        super::prepare_file(data, filename, nonce)
     }
 
     /// Generate a single compact proof for any set of open Challenges.

src/api/types.rs

@@ -217,8 +217,14 @@ impl Clone for PorParams {
 pub struct FileMetadata {
     /// The Merkle root over all symbols (data + parity)
     pub root: FieldElement,
-    /// SHA256 hash of the original file for identification
+    /// Content-based identifier: `object_<SHA256(data)>`
+    /// Not unique - same data produces same object_id. Used for file discovery.
+    pub object_id: String,
+    /// Unique file identifier: `file_<SHA256(data || nonce)>`
+    /// Unique per upload - used in the storage protocol.
     pub file_id: String,
+    /// The deterministic nonce used to derive the file_id (enables same data to have unique IDs)
+    pub nonce: Vec<u8>,
     /// The total number of leaves in the Merkle tree (padded to power of 2)
     pub padded_len: usize,
     /// Size of original file in bytes (for reconstruction)

src/ledger.rs

@@ -179,7 +179,9 @@ impl FileLedger {
     /// use kontor_crypto::api::prepare_file;
     /// use kontor_crypto::FileLedger;
     ///
-    /// let (prepared, metadata) = prepare_file(b"hello", "test.dat").unwrap();
+    /// // Deterministic nonce ensures unique file_id (e.g., user_id + timestamp)
+    /// let nonce = b"user_123_1704067200";
+    /// let (prepared, metadata) = prepare_file(b"hello", "test.dat", nonce).unwrap();
     /// let mut ledger = FileLedger::new();
     /// ledger.add_file(&metadata).unwrap();
     /// ```

src/lib.rs

@@ -29,8 +29,10 @@
 //! use kontor_crypto::{api::{self, PorSystem}, FileLedger, KontorPoRError};
 //!
 //! // 1. Prepare file with Reed-Solomon erasure coding (fixed 31-byte symbols)
+//! //    Deterministic nonce ensures unique file_id (e.g., user_id + timestamp)
 //! let data = b"Hello, world! This is test data for Nova PoR.";
-//! let (prepared, metadata) = api::prepare_file(data, "test.dat")?;
+//! let nonce = b"user_alice_1704067200";
+//! let (prepared, metadata) = api::prepare_file(data, "test.dat", nonce)?;
 //!
 //! // 2. Create ledger and add the file
 //! let mut ledger = FileLedger::new();
@@ -51,9 +53,9 @@
 //! ```rust,no_run
 //! use kontor_crypto::{api::{self, PorSystem}, FileLedger, KontorPoRError};
 //!
-//! // 1. Prepare multiple files
-//! let (prepared1, metadata1) = api::prepare_file(b"File 1 content", "file1.dat")?;
-//! let (prepared2, metadata2) = api::prepare_file(b"File 2 content", "file2.dat")?;
+//! // 1. Prepare multiple files (deterministic nonces ensure unique file_ids)
+//! let (prepared1, metadata1) = api::prepare_file(b"File 1 content", "file1.dat", b"upload_001")?;
+//! let (prepared2, metadata2) = api::prepare_file(b"File 2 content", "file2.dat", b"upload_002")?;
 //!
 //! // 2. Build ledger
 //! let mut ledger = FileLedger::new();

src/main.rs

@@ -230,7 +230,7 @@ fn setup_network(
         rng.fill_bytes(&mut data);
 
         let (prepared, metadata) =
-            api::prepare_file(&data, &format!("node_file_{}.dat", i)).unwrap();
+            api::prepare_file(&data, &format!("node_file_{}.dat", i), b"").unwrap();
 
         let idx = match category {
             FileSizeCategory::Small => 0,
@@ -267,7 +267,8 @@ fn setup_network(
         let mut data = vec![0u8; size];
         rng.fill_bytes(&mut data);
 
-        let (_, metadata) = api::prepare_file(&data, &format!("network_file_{}.dat", i)).unwrap();
+        let (_, metadata) =
+            api::prepare_file(&data, &format!("network_file_{}.dat", i), b"").unwrap();
         ledger.add_file(&metadata).unwrap();
     }
 

src/params.rs

@@ -64,7 +64,9 @@ fn generate_params_for_shape(
         .map(|i| {
             let metadata = FileMetadata {
                 root: FieldElement::ZERO,
+                object_id: format!("obj_dummy{}", i),
                 file_id: format!("dummy{}", i),
+                nonce: vec![],
                 padded_len: if i == 0 {
                     1 << file_tree_depth // First file at max depth
                 } else {

tests/api_consistency.rs

@@ -3,6 +3,7 @@ use kontor_crypto::{
     api::{self, Challenge, FieldElement, FileMetadata, PorSystem},
     poseidon::{domain_tags, poseidon_hash_tagged},
 };
+use rand::Rng;
 use std::collections::BTreeMap;
 
 mod common;
@@ -22,7 +23,7 @@ fn test_state_evolution_across_files() {
     for i in 0..3 {
         let data = vec![(i * 10) as u8; 50];
         let (prepared, metadata) =
-            api::prepare_file(&data, "test_file.dat").expect("Failed to prepare file");
+            api::prepare_file(&data, "test_file.dat", b"").expect("Failed to prepare file");
 
         files.insert(metadata.file_id.clone(), prepared);
         metadatas.push(metadata);
@@ -92,8 +93,11 @@ fn test_commitments_match_between_api_and_circuit() {
     let data1 = vec![1u8; 100];
     let data2 = vec![2u8; 150];
 
-    let (prep1, meta1) = api::prepare_file(&data1, "test_file.dat").unwrap();
-    let (prep2, meta2) = api::prepare_file(&data2, "test_file.dat").unwrap();
+    // Use random nonces for unique file_ids
+    let nonce1: [u8; 16] = rand::thread_rng().gen();
+    let nonce2: [u8; 16] = rand::thread_rng().gen();
+    let (prep1, meta1) = api::prepare_file(&data1, "test_file.dat", &nonce1).unwrap();
+    let (prep2, meta2) = api::prepare_file(&data2, "test_file.dat", &nonce2).unwrap();
 
     // Create challenges
     let seed = FieldElement::from(999u64);
@@ -148,8 +152,8 @@ fn test_prove_fails_without_required_ledger() {
     let data1 = vec![10u8; 50];
     let data2 = vec![20u8; 60];
 
-    let (prep1, meta1) = api::prepare_file(&data1, "test_file.dat").unwrap();
-    let (prep2, meta2) = api::prepare_file(&data2, "test_file.dat").unwrap();
+    let (prep1, meta1) = api::prepare_file(&data1, "test_file.dat", b"").unwrap();
+    let (prep2, meta2) = api::prepare_file(&data2, "test_file.dat", b"").unwrap();
 
     let challenges = vec![
         Challenge::new_test(meta1.clone(), 1000, 1, FieldElement::from(42u64)),
@@ -182,7 +186,7 @@ fn test_single_file_ignores_ledger() {
 
     let data = vec![30u8; 70];
 
-    let (prepared, metadata) = api::prepare_file(&data, "test_file.dat").unwrap();
+    let (prepared, metadata) = api::prepare_file(&data, "test_file.dat", b"").unwrap();
 
     let challenge = Challenge::new_test(metadata.clone(), 1000, 1, FieldElement::from(123u64));
 
@@ -198,7 +202,9 @@ fn test_single_file_ignores_ledger() {
     // Add another file to make the ledger different
     let other_metadata = FileMetadata {
         root: FieldElement::from(999u64),
+        object_id: "object_other".to_string(),
         file_id: "other_file".to_string(),
+        nonce: vec![],
         padded_len: 8, // depth 3
         original_size: 100,
         filename: "other.dat".to_string(),
@@ -239,7 +245,7 @@ fn test_proof_determinism() {
 
     let data = vec![42u8; 100];
 
-    let (prepared, metadata) = api::prepare_file(&data, "test_file.dat").unwrap();
+    let (prepared, metadata) = api::prepare_file(&data, "test_file.dat", b"").unwrap();
     let challenge = Challenge::new_test(metadata.clone(), 1000, 1, FieldElement::from(777u64));
 
     let mut files = BTreeMap::new();

tests/api_functionality.rs

@@ -17,7 +17,7 @@ fn test_challenge_id_determinism() {
     println!("Testing ChallengeID deterministic derivation");
 
     let data = b"Test data for challenge ID determinism";
-    let (_, metadata) = kontor_crypto::api::prepare_file(data, "test.dat").unwrap();
+    let (_, metadata) = kontor_crypto::api::prepare_file(data, "test.dat", b"").unwrap();
 
     // Same challenge should produce same ID
     let challenge1 = Challenge::new_test(metadata.clone(), 1000, 5, FieldElement::from(42u64));
@@ -67,8 +67,8 @@ fn test_challenge_id_collision_resistance() {
     let data1 = b"First test file for collision resistance testing";
     let data2 = b"Second test file with different content entirely";
 
-    let (_, metadata1) = kontor_crypto::api::prepare_file(data1, "file1.dat").unwrap();
-    let (_, metadata2) = kontor_crypto::api::prepare_file(data2, "file2.dat").unwrap();
+    let (_, metadata1) = kontor_crypto::api::prepare_file(data1, "file1.dat", b"").unwrap();
+    let (_, metadata2) = kontor_crypto::api::prepare_file(data2, "file2.dat", b"").unwrap();
 
     // Same parameters, different files
     let challenge1 = Challenge::new_test(metadata1, 1000, 5, FieldElement::from(42u64));
@@ -85,7 +85,8 @@ fn test_challenge_id_collision_resistance() {
     for i in 0..100 {
         let data = format!("Test file content {}", i);
         let (_, metadata) =
-            kontor_crypto::api::prepare_file(data.as_bytes(), &format!("file{}.dat", i)).unwrap();
+            kontor_crypto::api::prepare_file(data.as_bytes(), &format!("file{}.dat", i), b"")
+                .unwrap();
         let challenge =
             Challenge::new_test(metadata, 1000 + i as u64, 3, FieldElement::from(i as u64));
         let id = challenge.id();
@@ -107,7 +108,7 @@ fn test_proof_serialization_roundtrip() {
 
     let data = b"Test data for proof serialization testing with sufficient length";
     let (prepared, metadata) =
-        kontor_crypto::api::prepare_file(data, "serialize_test.dat").unwrap();
+        kontor_crypto::api::prepare_file(data, "serialize_test.dat", b"").unwrap();
 
     // Create ledger
     let mut ledger = FileLedger::new();
@@ -171,8 +172,10 @@ fn test_batch_seed_validation() {
     let data1 = b"First file for batch seed testing";
     let data2 = b"Second file for batch seed testing";
 
-    let (prepared1, metadata1) = kontor_crypto::api::prepare_file(data1, "batch1.dat").unwrap();
-    let (prepared2, metadata2) = kontor_crypto::api::prepare_file(data2, "batch2.dat").unwrap();
+    let (prepared1, metadata1) =
+        kontor_crypto::api::prepare_file(data1, "batch1.dat", b"").unwrap();
+    let (prepared2, metadata2) =
+        kontor_crypto::api::prepare_file(data2, "batch2.dat", b"").unwrap();
 
     // Create ledger
     let mut ledger = FileLedger::new();
@@ -218,7 +221,8 @@ fn test_porsystem_challenge_id_matching() {
     println!("Testing PorSystem challenge ID matching in verify");
 
     let data = b"Test data for challenge ID matching";
-    let (prepared, metadata) = kontor_crypto::api::prepare_file(data, "id_match_test.dat").unwrap();
+    let (prepared, metadata) =
+        kontor_crypto::api::prepare_file(data, "id_match_test.dat", b"").unwrap();
 
     // Create ledger
     let mut ledger = FileLedger::new();
@@ -260,7 +264,8 @@ fn test_porsystem_file_not_found() {
     println!("Testing PorSystem error handling for missing files");
 
     let data = b"Test data for file not found testing";
-    let (prepared, metadata) = kontor_crypto::api::prepare_file(data, "missing_test.dat").unwrap();
+    let (prepared, metadata) =
+        kontor_crypto::api::prepare_file(data, "missing_test.dat", b"").unwrap();
 
     // Create ledger without adding the file
     let ledger = FileLedger::new();
@@ -294,7 +299,7 @@ fn test_porsystem_prepare_file() {
     let data = b"Test data for PorSystem prepare_file method testing";
 
     // Test successful preparation
-    let result = system.prepare_file(data, "porsystem_test.dat");
+    let result = system.prepare_file(data, "porsystem_test.dat", b"");
     assert!(
         result.is_ok(),
         "PorSystem::prepare_file should succeed with valid inputs"
@@ -322,7 +327,7 @@ fn test_porsystem_prepare_file() {
     );
 
     // Test empty data rejection
-    let empty_result = system.prepare_file(&[], "empty.dat");
+    let empty_result = system.prepare_file(&[], "empty.dat", b"");
     match empty_result {
         Err(kontor_crypto::KontorPoRError::EmptyData { operation }) => {
             assert_eq!(operation, "prepare_file", "Should report correct operation");
@@ -342,12 +347,12 @@ fn test_porsystem_vs_free_function_equivalence() {
 
     // Test prepare_file equivalence
     let (prepared_free, metadata_free) =
-        kontor_crypto::api::prepare_file(data, "free_func.dat").unwrap();
+        kontor_crypto::api::prepare_file(data, "free_func.dat", b"").unwrap();
 
     let ledger = FileLedger::new();
     let system = PorSystem::new(&ledger);
     let (prepared_system, metadata_system) =
-        system.prepare_file(data, "system_method.dat").unwrap();
+        system.prepare_file(data, "system_method.dat", b"").unwrap();
 
     // Results should be equivalent (except for filename)
     assert_eq!(