add macro to cover s3 sync engine generation

Author: Handy-caT

Cargo.toml

@@ -12,7 +12,7 @@ description = "WorkTable is in-memory storage"
 
 [features]
 perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"]
-s3-support = ["dep:rust-s3", "dep:aws-creds", "dep:aws-region", "dep:walkdir"]
+s3-support = ["dep:rust-s3", "dep:aws-creds", "dep:aws-region", "dep:walkdir", "worktable_codegen/s3-support"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

codegen/Cargo.toml

@@ -5,6 +5,9 @@ edition = "2024"
 license = "MIT"
 description = "WorkTable codegeneration crate"
 
+[features]
+s3-support = []
+
 [lib]
 name = "worktable_codegen"
 path = "src/lib.rs"

codegen/src/lib.rs

@@ -2,6 +2,8 @@ mod mem_stat;
 mod name_generator;
 mod persist_index;
 mod persist_table;
+#[cfg(feature = "s3-support")]
+mod s3_persistence;
 mod worktable;
 
 use proc_macro::TokenStream;
@@ -14,6 +16,14 @@ pub fn worktable(input: TokenStream) -> TokenStream {
         .into()
 }
 
+#[cfg(feature = "s3-support")]
+#[proc_macro]
+pub fn s3_sync_persistence(input: TokenStream) -> TokenStream {
+    s3_persistence::expand(input.into())
+        .unwrap_or_else(|e| e.to_compile_error())
+        .into()
+}
+
 #[proc_macro_derive(PersistIndex)]
 pub fn persist_index(input: TokenStream) -> TokenStream {
     persist_index::expand(input.into())

codegen/src/name_generator.rs

@@ -117,4 +117,26 @@ impl WorktableNameGenerator {
             Span::mixed_site(),
         )
     }
+
+    pub fn get_space_secondary_index_ident(&self) -> Ident {
+        Ident::new(
+            format!("{}SpaceSecondaryIndex", self.name).as_str(),
+            Span::mixed_site(),
+        )
+    }
+
+    pub fn get_space_secondary_index_events_ident(&self) -> Ident {
+        Ident::new(
+            format!("{}SpaceSecondaryIndexEvents", self.name).as_str(),
+            Span::mixed_site(),
+        )
+    }
+
+    #[cfg(feature = "s3-support")]
+    pub fn get_s3_sync_persistence_engine_ident(&self) -> Ident {
+        Ident::new(
+            format!("{}SyncPersistenceEngine", self.name).as_str(),
+            Span::mixed_site(),
+        )
+    }
 }

codegen/src/persist_index/generator.rs

@@ -30,20 +30,6 @@ impl WorktableNameGenerator {
             Span::mixed_site(),
         )
     }
-
-    pub fn get_space_secondary_index_events_ident(&self) -> Ident {
-        Ident::new(
-            format!("{}SpaceSecondaryIndexEvents", self.name).as_str(),
-            Span::mixed_site(),
-        )
-    }
-
-    pub fn get_space_secondary_index_ident(&self) -> Ident {
-        Ident::new(
-            format!("{}SpaceSecondaryIndex", self.name).as_str(),
-            Span::mixed_site(),
-        )
-    }
 }
 
 impl Generator {

codegen/src/s3_persistence/mod.rs

@@ -0,0 +1,52 @@
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::parse::{Parse, ParseStream};
+use syn::{Ident, Result};
+
+use crate::name_generator::WorktableNameGenerator;
+
+struct S3PersistenceInput {
+    table_name: Ident,
+}
+
+impl Parse for S3PersistenceInput {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let table_name: Ident = input.parse()?;
+        Ok(S3PersistenceInput { table_name })
+    }
+}
+
+pub fn expand(input: TokenStream) -> Result<TokenStream> {
+    let input: S3PersistenceInput = syn::parse2(input)?;
+
+    let name_str = input.table_name.to_string();
+    let base_name = name_str
+        .strip_suffix("WorkTable")
+        .unwrap_or(&name_str)
+        .to_string();
+
+    let generator = WorktableNameGenerator::from_table_name(base_name);
+
+    let output_ident = generator.get_s3_sync_persistence_engine_ident();
+    let primary_key = generator.get_primary_key_type_ident();
+    let space_secondary_index = generator.get_space_secondary_index_ident();
+    let space_secondary_index_events = generator.get_space_secondary_index_events_ident();
+    let available_indexes = generator.get_available_indexes_ident();
+    let inner_size_const = generator.get_page_inner_size_const_ident();
+    let page_size_const = generator.get_page_size_const_ident();
+
+    Ok(quote! {
+        type #output_ident = S3SyncDiskPersistenceEngine<
+            SpaceData<
+                <<#primary_key as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State,
+                { #inner_size_const },
+                { #page_size_const as u32 },
+            >,
+            SpaceIndex<#primary_key, { #inner_size_const as u32 }>,
+            #space_secondary_index,
+            #primary_key,
+            #space_secondary_index_events,
+            #available_indexes,
+        >;
+    })
+}

src/lib.rs

@@ -19,6 +19,9 @@ pub use table::*;
 pub use data_bucket;
 pub use worktable_codegen::worktable;
 
+#[cfg(feature = "s3-support")]
+pub use worktable_codegen::s3_sync_persistence;
+
 pub mod prelude {
     pub use crate::in_memory::{
         ArchivedRowWrapper, Data, DataPages, Query, RowWrapper, StorableRow,

tests/persistence/s3/mod.rs

@@ -1,5 +1,6 @@
 use crate::remove_dir_if_exists;
 use worktable::prelude::*;
+use worktable::s3_sync_persistence;
 use worktable::worktable;
 
 worktable!(
@@ -11,18 +12,7 @@ worktable!(
     },
 );
 
-type TestS3SyncPersistenceEngine = S3SyncDiskPersistenceEngine<
-    SpaceData<
-        <<TestS3PrimaryKey as TablePrimaryKey>::Generator as PrimaryKeyGeneratorState>::State,
-        { TEST_S_3_INNER_SIZE },
-        { TEST_S_3_PAGE_SIZE as u32 },
-    >,
-    SpaceIndex<TestS3PrimaryKey, { TEST_S_3_INNER_SIZE as u32 }>,
-    TestS3SpaceSecondaryIndex,
-    TestS3PrimaryKey,
-    TestS3SpaceSecondaryIndexEvents,
-    TestS3AvailableIndexes,
->;
+s3_sync_persistence!(TestS3WorkTable);
 
 #[test]
 fn test_s3_engine_compiles() {