Play around with AWS see if it works.

Author: JoeZiminski

datashuttle/utils/aws.py

@@ -20,7 +20,7 @@ def check_if_aws_bucket_exists(cfg: Configs) -> bool:
 
     bucket_name = get_aws_bucket_name(cfg)
 
-    if bucket_name not in names:
+    if bucket_name != "" and bucket_name not in names:  # TODO: CHECK
         return False
 
     return True
@@ -40,7 +40,11 @@ def raise_if_bucket_absent(cfg: Configs) -> None:
 
 def get_aws_bucket_name(cfg: Configs) -> str:
     """Return the formatted AWS bucket name from the `central_path`."""
-    return cfg["central_path"].as_posix().strip("/").split("/")[0]
+    return (
+        cfg["central_path"].as_posix().strip("/").split("/")[0]
+        if cfg["central_path"] is not None
+        else ""
+    )
 
 
 # -----------------------------------------------------------------------------

datashuttle/utils/rclone.py

@@ -305,8 +305,8 @@ def setup_rclone_config_for_aws(
         "s3 provider AWS "
         f"access_key_id {cfg['aws_access_key_id']} "
         f"secret_access_key {aws_secret_access_key} "
-        f"region {cfg['aws_region']} "
-        f"location_constraint {cfg['aws_region']}",
+        f"region {cfg['aws_region']}",
+        # f"location_constraint {cfg['aws_region']}",
         pipe_std=True,
     )
 
@@ -325,7 +325,10 @@ def check_successful_connection_and_raise_error_on_fail(cfg: Configs) -> None:
     If the command fails, it raises a ConnectionError. The created file is
     deleted thereafter.
     """
-    tempfile_path = (cfg["central_path"] / "temp.txt").as_posix()
+    tempfile_path = (
+        cfg["central_path"] / "temp.txt"
+    ).as_posix()  # TODO: HANDLE NONE!
+
     output = call_rclone(
         f"touch {cfg.get_rclone_config_name()}:{tempfile_path}", pipe_std=True
     )

tests/tests_transfers/aws/aws_test_utils.py

@@ -1,15 +1,21 @@
 import copy
 import os
+import random
+import string
 
 from datashuttle import DataShuttle
 from datashuttle.utils import aws
 
 
 def setup_project_for_aws(project: DataShuttle):
     aws_bucket_name = os.environ["AWS_BUCKET_NAME"]
+
+    characters = string.ascii_letters + string.digits
+    random_string = "".join(random.choices(characters, k=15))
+
     project.update_config_file(
         connection_method="aws",
-        central_path=f"{aws_bucket_name}/main/{project.project_name}",
+        central_path=f"{aws_bucket_name}/main/{random_string}/{project.project_name}",
         aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
         aws_region=os.environ["AWS_REGION"],
     )

tests/tests_transfers/aws/test_aws_transfer.py

@@ -24,7 +24,7 @@ def aws_setup(self, pathtable_and_project):
         yield [pathtable, project]
 
         rclone.call_rclone(
-            f"purge central_{project.project_name}_aws:{project.cfg['central_path']}"
+            f"purge central_{project.project_name}_aws:{project.cfg['central_path'].parent}"
         )
 
     @pytest.mark.parametrize(