Merge pull request #195 from cgs-earth/smapcheckpointfix

Smap checkpoint fix

Author: C-Loftus

SMAP_ETL/src/lib.py

@@ -7,6 +7,8 @@
 import logging
 import os
 from pathlib import Path
+import platform
+import resource
 from zoneinfo import ZoneInfo
 
 import earthaccess
@@ -293,6 +295,20 @@ def append_hd5_to_s3_zarr(
         }
     }
 
+    # add a check for how much memory is in use
+    # tracemalloc.start()
+
+    # current, _ = tracemalloc.get_traced_memory()
+    # print(f"Current memory in use: {current} bytes")
+
+    usage = resource.getrusage(resource.RUSAGE_SELF)
+    if platform.system() == "Darwin":
+        rss_mb = usage.ru_maxrss / 1024 / 1024  # macOS: bytes
+    else:
+        rss_mb = usage.ru_maxrss / 1024  # Linux: kilobytes
+
+    print(f"Peak RSS: {rss_mb:.1f} MB")
+
     if store_exists:
         ds.to_zarr(
             store=zarr_mapper,

SMAP_ETL/src/main.py

@@ -16,6 +16,7 @@
     get_previously_downloaded_urls,
     get_smap_data_list_for_arizona,
     get_total_size_in_gb,
+    update_checkpoint,
 )
 import minio
 import s3fs
@@ -109,6 +110,11 @@ def main(
             LOGGER.warning("Stopping early since test mode was specified")
             break
 
+        LOGGER.info(
+            f"Updating checkpoint file with new file {hd5_file_path[0].name}"
+        )
+        update_checkpoint(mc, bucket, hd5_file_path[0].name)
+
     LOGGER.info("Done!")