Fixes to support tranfer of empty files

integration/config/localstack.ingress.config.ini.tmpl

@@ -9,7 +9,7 @@ resource     = request
 [COGNITO]
 client_id    = __COGNITO_CLIENT_ID__
 username     = __USERNAME__
-password     = not_a_real_password_this_is_only_used_with_localstack_deployments
+password     = Not_a_real_password_this_1s_only_used_with_localstack_deployments!
 region       = localhost
 
 [OTHER]

integration/config/localstack.tfvars.tmpl

@@ -19,8 +19,15 @@ lambda_authorizer_iam_role_arn      = "arn:aws:iam::000000000000:role/pds-nucleu
 nucleus_dum_cognito_initial_users = [
   {
     "username": "__USERNAME__",
-    "password": "not_a_real_password_this_is_only_used_with_localstack_deployments",  # pragma: allowlist secret
-    "group": "PDS_ENG_USERS"
+    "password": "Not_a_real_password_this_1s_only_used_with_localstack_deployments!",  # pragma: allowlist secret
+    "group": "PDS_SBN_USERS"
     "email": "pds-user@jpl.nasa.gov"  # pragma: allowlist secret
   }
 ]
+
+lambda_ingress_service_default_buckets = [
+  {
+    name        = "pds-sbn-staging",
+    description = "Default staging bucket for PDS Small Bodies Node"
+  }
+]

integration/pds/ingress/dum_integration_tests.py

@@ -75,12 +75,15 @@ def instantiate_localstack_tfvars(cls, username):
     @classmethod
     def terraform_localstack(cls):
         """Performs the terraform deployment of DUM on the localstack instance"""
-        subprocess.run(["tflocal", "init"],
-                       cwd=cls.terraform_dir, check=True, capture_output=True)
-        subprocess.run(["tflocal", "plan", "-var-file=localstack.tfvars", "-out=localstack.tfplan", "-no-color"],
-                       cwd=cls.terraform_dir, check=True, capture_output=True)
+        result = subprocess.run(["tflocal", "init"],
+                       cwd=cls.terraform_dir, check=False, capture_output=True)
+        assert(result.returncode == 0)
+        result = subprocess.run(["tflocal", "plan", "-var-file=localstack.tfvars", "-out=localstack.tfplan", "-no-color"],
+                       cwd=cls.terraform_dir, check=False, capture_output=True)
+        assert (result.returncode == 0)
         subprocess.run(["tflocal", "apply", "-auto-approve", "-no-color", "localstack.tfplan"],
-                       cwd=cls.terraform_dir, check=True, capture_output=True)
+                       cwd=cls.terraform_dir, check=False, capture_output=True)
+        assert (result.returncode == 0)
 
         # Get the terraform outputs we need to initialze the DUM client config
         api_gateway_id = subprocess.run(["tflocal", "output", "nucleus_dum_api_id"],
@@ -114,13 +117,18 @@ def setUp(self):
 
         self.test_file_hash = md5.hexdigest()
         self.test_file_last_modified_time = datetime.fromtimestamp(
-            os.path.getmtime(self.test_file.name), tz=timezone.utc).isoformat()
+            int(os.path.getmtime(self.test_file.name)), tz=timezone.utc).isoformat()
+
+        self.empty_test_file = tempfile.NamedTemporaryFile(prefix="dum_empty_", suffix=".dat", delete=False)
 
     def tearDown(self):
-        """Remove the random test file"""
+        """Remove the test files"""
         if exists(self.test_file.name):
             os.unlink(self.test_file.name)
 
+        if exists(self.empty_test_file.name):
+            os.unlink(self.empty_test_file.name)
+
     def test_ingress(self):
         """Ingress integration test between DUM client and Lambda Service (localstack)"""
         # Set up a basic ingress request for a single file
@@ -129,7 +137,7 @@ def test_ingress(self):
                 "-c",
                 join(self.config_dir, "localstack.ingress.config.ini"),
                 "-n",
-                "eng",
+                "sbn",
                 "--prefix",
                 dirname(self.test_file.name),
                 "--num-threads",
@@ -142,9 +150,9 @@ def test_ingress(self):
         pds_ingress_client.main(args)
 
         result = subprocess.run(
-            ["awslocal", "s3api", "head-object", "--bucket", "pds-eng-staging-test", "--key",
-             f"eng/{basename(self.test_file.name)}"],
-            check=True, capture_output=True
+            ["awslocal", "s3api", "head-object", "--bucket", "pds-sbn-staging-localstack", "--key",
+             f"sbn/{basename(self.test_file.name)}"],
+            check=False, capture_output=True
         )
 
         self.assertTrue(result.returncode == 0)
@@ -169,9 +177,9 @@ def test_ingress(self):
         pds_ingress_client.main(args)
 
         result = subprocess.run(
-            ["awslocal", "s3api", "head-object", "--bucket", "pds-eng-staging-test", "--key",
-             f"eng/{basename(self.test_file.name)}"],
-            check=True, capture_output=True
+            ["awslocal", "s3api", "head-object", "--bucket", "pds-sbn-staging-localstack", "--key",
+             f"sbn/{basename(self.test_file.name)}"],
+            check=False, capture_output=True
         )
 
         self.assertTrue(result.returncode == 0)
@@ -186,9 +194,9 @@ def test_ingress(self):
         pds_ingress_client.main(args)
 
         result = subprocess.run(
-            ["awslocal", "s3api", "head-object", "--bucket", "pds-eng-staging-test", "--key",
-             f"eng/{basename(self.test_file.name)}"],
-            check=True, capture_output=True
+            ["awslocal", "s3api", "head-object", "--bucket", "pds-sbn-staging-localstack", "--key",
+             f"sbn/{basename(self.test_file.name)}"],
+            check=False, capture_output=True
         )
 
         self.assertTrue(result.returncode == 0)
@@ -199,3 +207,32 @@ def test_ingress(self):
         # but the modified time relative to local file system should not
         self.assertNotEqual(object_metadata['LastModified'], initial_s3_last_modified_time)
         self.assertEqual(object_metadata["Metadata"]["last_modified"], self.test_file_last_modified_time)
+
+        # Test empty file transfer
+        args = pds_ingress_client.setup_argparser().parse_args(
+            [
+                "-c",
+                join(self.config_dir, "localstack.ingress.config.ini"),
+                "-n",
+                "sbn",
+                "--prefix",
+                dirname(self.test_file.name),
+                "--num-threads",
+                "1",
+                self.empty_test_file.name
+            ]
+        )
+
+        pds_ingress_client.main(args)
+
+        result = subprocess.run(
+            ["awslocal", "s3api", "head-object", "--bucket", "pds-sbn-staging-localstack", "--key",
+             f"sbn/{basename(self.empty_test_file.name)}"],
+            check=False, capture_output=True
+        )
+
+        self.assertTrue(result.returncode == 0)
+
+        object_metadata = json.loads(result.stdout.decode().strip())
+
+        self.assertEqual(int(object_metadata["ContentLength"]), 0)

src/pds/ingress/client/pds_ingress_client.py

@@ -7,6 +7,7 @@
 Client side script used to perform ingress request to the DUM service in AWS.
 """
 import argparse
+import calendar
 import json
 import os
 import sched
@@ -316,14 +317,14 @@ def _prepare_batch_for_ingress(ingress_path_batch, prefix, batch_index, batch_pb
             manifest_entry = MANIFEST[trimmed_path]
             md5_digest = manifest_entry["md5"]
             file_size = manifest_entry["size"]
-            last_modified_time = time.mktime(datetime.fromisoformat(manifest_entry["last_modified"]).timetuple())
+            last_modified_time = calendar.timegm(datetime.fromisoformat(manifest_entry["last_modified"]).timetuple())
         else:
             # Calculate the MD5 checksum of the file payload
             md5_digest = md5_for_path(ingress_path).hexdigest()
 
             # Get the size and last modified time of the file
             file_size = os.stat(ingress_path).st_size
-            last_modified_time = os.path.getmtime(ingress_path)
+            last_modified_time = int(os.path.getmtime(ingress_path))
 
             # Update manifest with new entry
             MANIFEST[trimmed_path] = {
@@ -474,9 +475,17 @@ def ingress_file_to_s3(ingress_response, batch_index, batch_pbar):
         if not ingress_path:
             raise ValueError("No ingress path provided with response for %s", trimmed_path)
 
-        # Include the base64-encoded MD5 hash so AWS can perform its own
-        # integrity check on the uploaded file
-        headers = {"Content-MD5": ingress_response.get("base64_md5")}
+        file_length = os.stat(ingress_path).st_size
+
+        headers = {
+            "Content-Length": str(file_length),
+        }
+
+        # If the file is non-empty, include the base64-encoded MD5 hash so AWS
+        # can perform its own integrity check on the uploaded file (AWS does
+        # not support hash checks for empty files)
+        if file_length > 0:
+            headers["Content-MD5"] = ingress_response.get("base64_md5")
 
         # Initialize the file upload progress subbar attached to the batch progress bar
         upload_pbar = get_upload_progress_bar_for_batch(
@@ -486,7 +495,9 @@ def ingress_file_to_s3(ingress_response, batch_index, batch_pbar):
         with open(ingress_path, "rb") as infile:
             # Wrap file I/O with our upload bar to automatically track file upload progress
             wrapped_file = CallbackIOWrapper(upload_pbar.update, infile, "read")
-            response = requests.put(s3_ingress_url, data=wrapped_file, headers=headers)
+
+            # Only send the file data if the file is non-empty
+            response = requests.put(s3_ingress_url, data=wrapped_file if file_length > 0 else b"", headers=headers)
             response.raise_for_status()
 
         logger.info("Batch %d : %s Ingest complete", batch_index, trimmed_path)

src/pds/ingress/service/pds_ingress_app.py

@@ -211,6 +211,7 @@ def generate_presigned_upload_url(
     object_key,
     md5_digest,
     base64_md5_digest,
+    file_size,
     last_modified,
     client_version,
     service_version,
@@ -231,6 +232,8 @@ def generate_presigned_upload_url(
     base64_md5_digest : str
         Base64 encoded version of the MD5 hash digest corresponding to the file
         to generate a URL for.
+    file_size : int
+        Size in bytes of the incoming version of the file.
     last_modified : float
         Last modified time of the incoming version of the file as a Unix Epoch.
     client_version : str
@@ -252,7 +255,7 @@ def generate_presigned_upload_url(
     method_parameters = {
         "Bucket": bucket_info["name"],
         "Key": object_key,
-        "ContentMD5": base64_md5_digest,
+        "ContentLength": file_size,
         "Metadata": {
             "md5": md5_digest,
             "last_modified": datetime.fromtimestamp(last_modified, tz=timezone.utc).isoformat(),
@@ -261,6 +264,11 @@ def generate_presigned_upload_url(
         },
     }
 
+    # We only want to include an MD5 if the requested file length is non-zero,
+    # AWS will reject the request otherwise.
+    if file_size > 0:
+        method_parameters["ContentMD5"] = base64_md5_digest
+
     if bucket_info.get("storage_class"):
         method_parameters["StorageClass"] = bucket_info["storage_class"]
 
@@ -456,6 +464,7 @@ def lambda_handler(event, context):
                 {
                     "result": HTTPStatus.NOT_FOUND,
                     "trimmed_path": trimmed_path,
+                    "ingress_path": ingress_path,
                     "s3_url": None,
                     "bucket": None,
                     "key": None,
@@ -502,6 +511,7 @@ def lambda_handler(event, context):
                         object_key,
                         md5_digest,
                         base64_md5_digest,
+                        file_size,
                         float(last_modified),
                         client_version,
                         service_version,
@@ -529,6 +539,7 @@ def lambda_handler(event, context):
                     {
                         "result": HTTPStatus.NO_CONTENT,
                         "trimmed_path": trimmed_path,
+                        "ingress_path": ingress_path,
                         "s3_url": None,
                         "bucket": destination_bucket,
                         "key": object_key,

terraform/main.tf

@@ -46,7 +46,7 @@ module "nucleus_dum_lambda_authorizer" {
   lambda_authorizer_cognito_client_id  = module.nucleus_dum_cognito.nucleus_dum_cognito_user_pool_client_id
   lambda_authorizer_localstack_context = var.localstack_context
 
-  depends_on = [module.nucleus_dum_cognito]
+  depends_on = [module.nucleus_dum_ingress_service_lambda, module.nucleus_dum_cognito]
 }
 
 module "nucleus_dum_api" {

terraform/modules/lambda/service/ingress_service.tf

@@ -143,14 +143,14 @@ resource "aws_lambda_layer_version" "lambda_ingress_service_pyyaml_layer" {
   s3_bucket           = module.lambda_bucket.bucket_id
   s3_key              = "layer-PyYAML.zip"
   layer_name          = "PyYAML"
-  compatible_runtimes = ["python3.9","python3.10","python3.11","python3.12","python3.13"]
+  compatible_runtimes = ["python3.9","python3.10","python3.11","python3.12"]
 }
 
 resource "aws_lambda_layer_version" "lambda_ingress_service_yamale_layer" {
   s3_bucket           = module.lambda_bucket.bucket_id
   s3_key              = "layer-Yamale.zip"
   layer_name          = "Yamale"
-  compatible_runtimes = ["python3.9","python3.10","python3.11","python3.12","python3.13"]
+  compatible_runtimes = ["python3.9","python3.10","python3.11","python3.12"]
 }
 
 # Create the Ingress Lambda functions using the zips uploaded to S3