Update iceberg and lacne version

Cargo.toml

@@ -44,11 +44,12 @@ datafusion-table-providers = { version = "0.10.0", default-features = false, fea
 tokio-rusqlite = "0.7"
 env_logger = "0.11"
 futures = "0.3"
-iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "e844d823c3834e3b9cbb98bdd4c0b4638fdc5b80" }
-iceberg-datafusion = { git = "https://github.com/apache/iceberg-rust", rev = "e844d823c3834e3b9cbb98bdd4c0b4638fdc5b80" }
+iceberg = "0.9.0"
+iceberg-datafusion = "0.9.0"
+iceberg-storage-opendal = "0.9.0"
 insta = { version = "1.41.1", features = ["glob", "filters"] }
-lance = { git = "https://github.com/lance-format/lance", tag = "v4.0.0-beta.7", default-features = false }
-lance-linalg = { git = "https://github.com/lance-format/lance", tag = "v4.0.0-beta.7" }
+lance = { git = "https://github.com/lance-format/lance", tag = "v4.0.0-rc.1", default-features = false }
+lance-linalg = { git = "https://github.com/lance-format/lance", tag = "v4.0.0-rc.1" }
 log = "0.4.27"
 object_store = { version = "0.12.4", default-features = false, features = ["aws", "gcp", "azure", "http"] }
 parking_lot = "0.12"

crates/server/Cargo.toml

@@ -9,7 +9,7 @@ path = "src/main.rs"
 
 [features]
 default = []
-onnx = ["model/onnx"]
+onnx = ["dep:model", "model/onnx"]
 
 [dependencies]
 arrow = { workspace = true }
@@ -28,7 +28,7 @@ serde_yaml = { workspace = true }
 skardi-engine = { path = "../engine" }
 pipeline = { path = "../pipeline" }
 sources = { path = "../sources" }
-model = { path = "../model" }
+model = { path = "../model", optional = true }
 anyhow = "1.0"
 thiserror = "1.0"
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "sync"] }

crates/sources/Cargo.toml

@@ -20,6 +20,7 @@ futures ={ workspace = true }
 hex = "0.4"
 iceberg = { workspace = true }
 iceberg-datafusion = { workspace = true }
+iceberg-storage-opendal = { workspace = true }
 lance = { workspace = true }
 mongodb = "3.1"
 mysql_async = "0.36"

crates/sources/src/providers/iceberg.rs

@@ -7,9 +7,10 @@ use anyhow::{Context, Result};
 use datafusion::prelude::SessionContext;
 use iceberg::NamespaceIdent;
 use iceberg::TableIdent;
-use iceberg::io::FileIO;
+use iceberg::io::{FileIO, FileIOBuilder};
 use iceberg::table::Table;
 use iceberg_datafusion::IcebergStaticTableProvider;
+use iceberg_storage_opendal::OpenDalStorageFactory;
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -150,11 +151,22 @@ async fn build_file_io(warehouse_path: &str, options: &HashMap<String, String>)
         }
     }
 
-    FileIO::from_path(warehouse_path)
-        .with_context(|| format!("Failed to determine FileIO scheme for: {}", warehouse_path))?
-        .with_props(props)
-        .build()
-        .with_context(|| format!("Failed to build FileIO for warehouse: {}", warehouse_path))
+    let factory: Arc<dyn iceberg::io::StorageFactory> =
+        if warehouse_path.starts_with("s3://") || warehouse_path.starts_with("s3a://") {
+            let scheme = if warehouse_path.starts_with("s3a://") {
+                "s3a://"
+            } else {
+                "s3://"
+            };
+            Arc::new(OpenDalStorageFactory::S3 {
+                configured_scheme: scheme.to_string(),
+                customized_credential_load: None,
+            })
+        } else {
+            Arc::new(OpenDalStorageFactory::Fs)
+        };
+
+    Ok(FileIOBuilder::new(factory).with_props(props).build())
 }
 
 async fn find_latest_metadata(file_io: &FileIO, table_path: &str) -> Result<String> {
@@ -418,10 +430,7 @@ mod tests {
         let rt = tokio::runtime::Runtime::new().unwrap();
         rt.block_on(async {
             let temp_dir = TempDir::new().unwrap();
-            let file_io = FileIO::from_path(&format!("file://{}", temp_dir.path().display()))
-                .unwrap()
-                .build()
-                .unwrap();
+            let file_io = FileIO::new_with_fs();
 
             let table_path = format!("file://{}/ns/table", temp_dir.path().display());
             let result = find_latest_metadata(&file_io, &table_path).await;
@@ -447,10 +456,7 @@ mod tests {
             fs::write(metadata_dir.join("v1.metadata.json"), "{}").unwrap();
             fs::write(metadata_dir.join("v2.metadata.json"), "{}").unwrap();
 
-            let file_io = FileIO::from_path(&format!("file://{}", temp_dir.path().display()))
-                .unwrap()
-                .build()
-                .unwrap();
+            let file_io = FileIO::new_with_fs();
 
             let table_path = format!("file://{}", temp_dir.path().display());
             let result = find_latest_metadata(&file_io, &table_path).await;
@@ -471,10 +477,7 @@ mod tests {
             fs::write(metadata_dir.join("v2.metadata.json"), "{}").unwrap();
             fs::write(metadata_dir.join("v3.metadata.json"), "{}").unwrap();
 
-            let file_io = FileIO::from_path(&format!("file://{}", temp_dir.path().display()))
-                .unwrap()
-                .build()
-                .unwrap();
+            let file_io = FileIO::new_with_fs();
 
             let table_path = format!("file://{}", temp_dir.path().display());
             let result = find_latest_metadata(&file_io, &table_path).await;