Refactored sources directory

Cargo.toml

@@ -10,7 +10,7 @@ license = "Apache-2.0"
 [workspace]
 members = [ "crates/cli",
             "crates/engine",
-            "crates/source",
+            "crates/sources",
             "crates/model",
             "crates/pipeline",
             "crates/server"]

crates/cli/Cargo.toml

@@ -26,7 +26,7 @@ object_store = { workspace = true, features = ["aws", "gcp", "azure", "http"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }
-source = { path = "../source" }
+sources = { path = "../sources" }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 url = { workspace = true }
 

crates/cli/src/main.rs

@@ -16,8 +16,8 @@ use object_store::azure::MicrosoftAzureBuilder;
 use object_store::gcp::GoogleCloudStorageBuilder;
 use object_store::http::HttpBuilder;
 use serde::Deserialize;
-use source::lance::knn_table_function::register_lance_knn_udtf;
-use source::providers::{
+use sources::providers::lance::knn_table_function::register_lance_knn_udtf;
+use sources::providers::{
     iceberg::register_iceberg_table, lance::register_lance_table, mongo::register_mongo_tables,
     mysql::register_mysql_tables, sqlite::register_sqlite_tables,
     sqlx::postgres::register_postgres_tables,
@@ -254,7 +254,7 @@ impl UrlTableFactory for SkardiUrlTableFactory {
 
             if !table_name.is_empty() {
                 let provider =
-                    source::providers::sqlite::create_sqlite_table_provider(db_path, table_name)
+                    sources::providers::sqlite::create_sqlite_table_provider(db_path, table_name)
                         .await
                         .map_err(|e| {
                             datafusion::error::DataFusionError::Execution(e.to_string())

crates/model/Cargo.toml

@@ -20,7 +20,7 @@ tokio = {workspace = true,  features = ["macros", "rt-multi-thread", "sync"]}
 tracing = { workspace = true }
 
 [dev-dependencies]
-source = { path = "../source" }
+sources = { path = "../sources" }
 
 [lints]
 workspace = true

crates/model/examples/ncf_demo.rs

@@ -11,7 +11,7 @@ async fn main() -> Result<()> {
     let mut ctx = SessionContext::new();
 
     // 2. Register the movie_embeddings Lance dataset as a table
-    source::providers::lance::register_lance_table(
+    sources::providers::lance::register_lance_table(
         &mut ctx,
         "movie_embeddings",
         "data/movie_embeddings.lance",

crates/server/Cargo.toml

@@ -23,7 +23,7 @@ serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 skardi-engine = { path = "../engine" }
 pipeline = { path = "../pipeline" }
-source = { path = "../source" }
+sources = { path = "../sources" }
 model = { path = "../model" }
 anyhow = "1.0"
 thiserror = "1.0"

crates/server/src/config.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use thiserror::Error;
 
 use crate::remote_storage::{RemoteStorage, S3Storage};
-pub use source::AccessMode;
+pub use sources::AccessMode;
 
 /// CLI arguments for the Skardi server
 #[derive(Parser, Debug)]
@@ -538,15 +538,15 @@ fn validate_pipeline_sql(
     sql: &str,
     data_sources: &[DataSource],
 ) -> Result<()> {
-    use source::sql_validator::{validate_sql, SqlValidatorConfig};
+    use sources::sql_validator::{validate_sql, SqlValidatorConfig};
 
     // Build validator config from data sources
     let mut validator_config = SqlValidatorConfig::new();
     for ds in data_sources {
         let mode = if ds.access_mode.is_read_write() {
-            source::sql_validator::AccessMode::ReadWrite
+            sources::sql_validator::AccessMode::ReadWrite
         } else {
-            source::sql_validator::AccessMode::ReadOnly
+            sources::sql_validator::AccessMode::ReadOnly
         };
         validator_config = validator_config.with_table(&ds.name, mode);
     }
@@ -742,7 +742,7 @@ async fn register_data_source(
             );
 
             // Register PostgreSQL table using the sqlx-based provider
-            source::providers::sqlx::postgres::register_postgres_tables(
+            sources::providers::sqlx::postgres::register_postgres_tables(
                 session_ctx,
                 &source.name,
                 connection_string,
@@ -782,7 +782,7 @@ async fn register_data_source(
                 source.options
             );
 
-            source::providers::mysql::register_mysql_tables(
+            sources::providers::mysql::register_mysql_tables(
                 session_ctx,
                 &source.name,
                 connection_string,
@@ -815,7 +815,7 @@ async fn register_data_source(
                         error: "Invalid SQLite database path".to_string(),
                     })?;
 
-            source::providers::sqlite::register_sqlite_tables(
+            sources::providers::sqlite::register_sqlite_tables(
                 session_ctx,
                 &source.name,
                 db_path,
@@ -847,7 +847,7 @@ async fn register_data_source(
                         error: "Invalid warehouse path".to_string(),
                     })?;
 
-            source::providers::iceberg::register_iceberg_table(
+            sources::providers::iceberg::register_iceberg_table(
                 session_ctx,
                 &source.name,
                 warehouse_path,
@@ -878,7 +878,7 @@ async fn register_data_source(
                 source.options
             );
 
-            source::providers::mongo::register_mongo_tables(
+            sources::providers::mongo::register_mongo_tables(
                 session_ctx,
                 &source.name,
                 connection_string,
@@ -909,7 +909,7 @@ async fn register_data_source(
                 source.options
             );
 
-            source::redis::datasource::register_redis_tables(
+            sources::providers::redis::datasource::register_redis_tables(
                 session_ctx,
                 &source.name,
                 connection_string,
@@ -934,7 +934,7 @@ async fn register_data_source(
             let dataset_registry = optimizer_registry.map(|reg| reg.lance_datasets());
 
             // Register Lance dataset using the providers module
-            source::providers::lance::register_lance_table(
+            sources::providers::lance::register_lance_table(
                 session_ctx,
                 &source.name,
                 source.path.to_str().unwrap(),

crates/server/src/handlers.rs

@@ -666,7 +666,7 @@ mod tests {
     use datafusion::prelude::SessionContext;
     use pipeline::pipeline::{Pipeline, StandardPipeline};
     use skardi_engine::datafusion::DataFusionEngine;
-    use source::AccessMode;
+    use sources::AccessMode;
     use std::fs;
     use std::path::PathBuf;
     use std::sync::{Arc, RwLock};

crates/server/src/optimizer_registry.rs

@@ -95,7 +95,10 @@ impl OptimizerRegistry {
         tracing::info!("Registering Lance table functions");
 
         // Register lance_knn table function
-        source::lance::knn_table_function::register_lance_knn_udtf(ctx, self.lance_datasets());
+        sources::providers::lance::knn_table_function::register_lance_knn_udtf(
+            ctx,
+            self.lance_datasets(),
+        );
 
         tracing::info!("✓ Registered lance_knn table function");
         Ok(())

crates/source/Cargo.toml → crates/sources/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "source"
+name = "sources"
 version.workspace = true
 edition.workspace = true
 description.workspace = true

crates/source/src/lib.rs → crates/sources/src/lib.rs

@@ -1,7 +1,5 @@
 pub mod access_mode;
-pub mod lance;
 pub mod providers;
-pub mod redis;
 pub mod sql_validator;
 
 // Re-export commonly used types

crates/source/src/lance/mod.rs → crates/sources/src/providers/lance/mod.rs

@@ -1,12 +1,15 @@
 //! Lance integration module
 //!
 //! Provides:
+//! - Table registration for Lance datasets
 //! - Custom execution plan for KNN search (LanceKnnExec)
 //! - Table function for explicit KNN search (lance_knn)
 //! - Integration with DataFusion query engine
 
 pub mod knn_exec;
 pub mod knn_table_function;
+pub mod registration;
 
 pub use knn_exec::LanceKnnExec;
 pub use knn_table_function::{LanceKnnTableFunction, register_lance_knn_udtf};
+pub use registration::register_lance_table;

crates/source/src/providers/lance.rs → crates/sources/src/providers/lance/registration.rs

@@ -16,7 +16,7 @@ use std::sync::{Arc, RwLock};
 /// # Example
 /// ```no_run
 /// use datafusion::prelude::SessionContext;
-/// use source::providers::lance::register_lance_table;
+/// use sources::providers::lance::register_lance_table;
 ///
 /// # async fn example() -> anyhow::Result<()> {
 /// let mut ctx = SessionContext::new();

crates/source/src/providers/mod.rs → crates/sources/src/providers/mod.rs

@@ -2,5 +2,6 @@ pub mod iceberg;
 pub mod lance;
 pub mod mongo;
 pub mod mysql;
+pub mod redis;
 pub mod sqlite;
 pub mod sqlx;

crates/source/src/redis/relation.rs → crates/sources/src/providers/redis/relation.rs

@@ -47,7 +47,7 @@ impl RedisRelation {
     /// # Examples
     ///
     /// ```
-    /// use source::redis::relation::RedisRelation;
+    /// use sources::providers::redis::relation::RedisRelation;
     ///
     /// let pattern = "users:*";
     /// let key = "users:1234";