Added retry logic for any failed tranfers during initial ingress attempt

Author: Scott Collins

src/pds/ingress/client/pds_ingress_client.py

@@ -847,11 +847,27 @@ def main(args):
         refresh_thread.start()
 
         try:
-            init_batch_progress_bars(args.num_threads)
+            init_batch_progress_bars(min(args.num_threads, len(request_batchs)))
             perform_ingress(request_batchs, node_id, args.force_overwrite, config["API_GATEWAY"])
         finally:
             close_batch_progress_bars()
 
+        logger.info("All batches processed")
+
+        try:
+            if len(SUMMARY_TABLE["failed"]) > 0:
+                logger.info("----------------------------------------")
+                logger.info("Reattempting ingress for failed files...")
+
+                failed_ingresses = SUMMARY_TABLE["failed"]
+                batched_failed_ingresses = list(batched(failed_ingresses, batch_size))
+                failed_request_batchs = prepare_batches(batched_failed_ingresses, args.prefix)
+
+                init_batch_progress_bars(min(args.num_threads, len(failed_request_batchs)))
+                perform_ingress(failed_request_batchs, node_id, args.force_overwrite, config["API_GATEWAY"])
+        finally:
+            close_batch_progress_bars()
+
             # Flush all logged statements to CloudWatch Logs
             log_util.CLOUDWATCH_HANDLER.flush()
     else:

src/pds/ingress/util/progress_util.py

@@ -279,7 +279,12 @@ def update_upload_pbar_filename(upload_pbar, filename):
 
 def close_batch_progress_bars():
     """Closes all Batch progress bars and associated Upload sub-bars."""
+    global BATCH_BARS, TOTAL_INGRESS_BAR  # noqa F824
+
     with BATCH_SEMAPHORE:
         for batch_pbar in BATCH_BARS:
             batch_pbar.upload_pbar.close()
             batch_pbar.close()
+
+        BATCH_BARS.clear()
+        TOTAL_INGRESS_BAR = None