fix(s3_delta): add default filter

Author: Vixtir

odd_collector_aws/adapters/s3_delta/client.py

@@ -61,7 +61,7 @@ def __init__(self, config: S3DeltaPlugin) -> None:
         self.storage_options: StorageOptions = StorageOptions.from_config(config)
         self.fs = FileSystem(config)
 
-    def load_delta_table(self, delta_table_config: DeltaTableConfig) -> None:
+    def load_delta_table(self, delta_table_config: DeltaTableConfig) -> DeltaTable:
         try:
             return DeltaTable(
                 delta_table_config.path,
@@ -74,14 +74,15 @@ def handle_folder(self, config: DeltaTableConfig) -> Iterable[DTable]:
         logger.debug(f"Getting delta tables from folder {config.path}")
 
         objects = self.fs.get_file_info(remove_protocol(config.path))
+
         allowed = filter(
-            lambda object: not object.is_file and config.allow(object.base_name),
+            lambda obj: not obj.is_file and config.allow(obj.base_name),
             objects,
         )
 
-        for object in allowed:
-            config = config.append_prefix(object.base_name)
-            yield from self.get_table(config)
+        for obj in allowed:
+            new_config = config.append_prefix(obj.base_name)
+            yield from self.get_table(new_config)
 
     def get_table(self, delta_table_config: DeltaTableConfig) -> Iterable[DTable]:
         # sourcery skip: raise-specific-error

odd_collector_aws/domain/plugin.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Literal, Optional
+from typing import List, Literal, Optional
 
 from odd_collector_sdk.domain.filter import Filter
 from odd_collector_sdk.domain.plugin import Plugin
@@ -42,7 +42,7 @@ class DeltaTableConfig(BaseModel):
     scheme: str = Field(default="s3", alias="schema")
     bucket: str
     prefix: str
-    object_filter: Optional[Filter] = None
+    object_filter: Optional[Filter] = Filter()
 
     @property
     def path(self) -> str: