Release v0.8.0

.github/actions/deploy/action.yml

@@ -8,6 +8,8 @@ inputs:
     required: true
   LANDSAT_TOPIC_ARN:
     required: true
+  NISAR_TOPIC_ARN:
+    required: true
   SENTINEL1_TOPIC_ARN:
     required: true
   SENTINEL2_TOPIC_ARN:
@@ -48,6 +50,7 @@ runs:
           --capabilities CAPABILITY_IAM \
           --parameter-overrides \
             LandsatTopicArn=${{ inputs.LANDSAT_TOPIC_ARN }} \
+            NisarTopicArn=${{ inputs.NISAR_TOPIC_ARN }} \
             Sentinel1TopicArn=${{ inputs.SENTINEL1_TOPIC_ARN }} \
             Sentinel2TopicArn=${{ inputs.SENTINEL2_TOPIC_ARN }} \
             Hyp3Api=${{ inputs.HYP3_API }} \

.github/workflows/build-and-deploy.yml

@@ -49,14 +49,15 @@ jobs:
         with:
           platforms: ${{ env.PLATFORMS }}
 
-      - uses: aws-actions/configure-aws-credentials@v5
+      - uses: aws-actions/configure-aws-credentials@v6
         if: ${{ github.actor != 'dependabot[bot]' }}
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ vars.AWS_REGION }}
 
       - name: Login to Amazon ECR
+        if: ${{ github.actor != 'dependabot[bot]' }}
         uses: aws-actions/amazon-ecr-login@v2
 
       - name: Build, tag, and push image to Amazon ECR
@@ -104,7 +105,7 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - uses: aws-actions/configure-aws-credentials@v5
+      - uses: aws-actions/configure-aws-credentials@v6
         with:
           aws-access-key-id:  ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -117,6 +118,7 @@ jobs:
           EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
           CF_TEMPLATE_BUCKET: cf-templates-3o5lnspmwmzg-us-west-2
           LANDSAT_TOPIC_ARN: ${{ vars.LANDSAT_TOPIC_ARN }}
+          NISAR_TOPIC_ARN: ${{ vars.NISAR_TOPIC_ARN }}
           SENTINEL1_TOPIC_ARN: ${{ vars.SENTINEL1_TOPIC_ARN }}
           SENTINEL2_TOPIC_ARN: ${{ vars.SENTINEL2_TOPIC_ARN }}
           HYP3_API: ${{ vars.HYP3_API }}

CHANGELOG.md

@@ -4,13 +4,33 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0]
+### Added
+- Support for processing NISAR SNS messages and submitting jobs to hyp3-its-live has been added.
+  - NISAR specific functions have been added to a new `nisar` module.
+- An `itslive` module with functions to deduplicate already published pairs based on the STAC catalog.
+
+### Changed
+- All environment variable configuration is now centralized in a new `config` module.
+- HyP3 deduplication of pairs will be skipped if the `JOBS_TABLE_NAME` environment variable is not provided.
+- ITS_LIVE STAC deduplication of pairs will be skipped if the `STAC_ITEMS_ENDPOINT` environment variable is not provided
+- `deduplicate_hyp3_pairs`, `submit_pairs_for_processing`, and other HyP3 specific functionality has been moved to the `hyp3` module from `main`. 
+
+### Fixed
+- Monitoring lambda function now uses reserved concurrency to set a maximum number of concurrent executions so it doesn't overwhelm the STAC catalog instance.
+- pystac-client is now configured with connection and read timeouts when searching the ITS_LIVE catalog so it does not hang indefinitely waiting for a response.
+- Sentinel-1 burst or SLC image pairs are now deduplicated against already published ITS_LIVE pairs in the STAC catalog.See [#331](https://github.com/ASFHyP3/its-live-monitoring/issues/331) for more info.
+
+### Removed
+- `deduplicate_s3_pairs` and supporting functions in favor of `itslive.deduplicate_published_pairs`.
+
 ## [0.7.0]
-## Added
+### Added
 - `StacItemsEndpoint` and `StacExistsOk` cloudformation parameters to allow publishing STAC items directly to a STAC catalog. Accordingly:
   - `STAC_ITEMS_ENDPOINT` and `STAC_EXISTS_OK` environment variables are set for the monitoring lambda.
   - The cloud formation parameters are set in the build and deploy GitHub Actions workflow by the `STAC_ITEMS_ENDPOINT` and `STAC_EXISTS_OK` deploy environment variables.
 
-## Changed
+### Changed
 - Updated the `AUTORIFT_JOB_TEMPLATE` for [HyP3 v10.13.0+](https://github.com/ASFHyP3/hyp3/pull/3003) to allow posting/putting STAC items in the catalog instead of writing them to an alternate ingest location.
 
 ### Removed

Makefile

@@ -13,6 +13,7 @@ IMAGE_TAG = $(subst +,_,$(SDIST_VERSION))
 endif
 
 LANDSAT_TOPIC_ARN ?= arn:aws:sns:us-west-2:986442313181:its-live-notify-landsat-test
+NISAR_TOPIC_ARN ?= arn:aws:sns:us-west-2:986442313181:its-live-notify-nisar-test
 SENTINEL1_TOPIC_ARN ?= arn:aws:sns:us-west-2:986442313181:its-live-notify-sentinel1-test
 SENTINEL2_TOPIC_ARN ?= arn:aws:sns:eu-west-1:986442313181:its-live-notify-sentinel2-test
 
@@ -32,6 +33,10 @@ landsat-integration:
 	export AWS_PAGER='' && \
 	$(foreach file, $(wildcard tests/integration/landsat*.json), aws sns publish --profile saml-pub --topic-arn ${LANDSAT_TOPIC_ARN} --message file://${file} --output json;)
 
+nisar-integration:
+	export AWS_PAGER='' && \
+	$(foreach file, $(wildcard tests/integration/nisar*.json), aws sns publish --profile saml-pub --topic-arn ${NISAR_TOPIC_ARN} --message file://${file} --output json;)
+
 sentinel1-integration:
 	export AWS_PAGER='' && \
 	$(foreach file, $(wildcard tests/integration/sentinel1*.json), aws sns publish --profile saml-pub --topic-arn ${SENTINEL1_TOPIC_ARN} --message file://${file} --output json;)
@@ -40,7 +45,7 @@ sentinel2-integration:
 	export AWS_PAGER='' && \
 	$(foreach file, $(wildcard tests/integration/sentinel2*.json), aws sns publish --region eu-west-1 --profile saml-pub --topic-arn ${SENTINEL2_TOPIC_ARN} --message file://${file} --output json;)
 
-integration: landsat-integration sentinel1-integration sentinel2-integration
+integration: landsat-integration nisar-integration sentinel1-integration sentinel2-integration
 
 static: mypy ruff cfn-lint
 

README.md

@@ -1,11 +1,12 @@
 # ITS_LIVE Monitoring
 
-The ITS_LIVE monitoring stack provides the AWS architecture to support low-latency production of netCDF glacier velocity products produced from Optical (Landsat 8/9, Sentinel-2) and SAR (Sentinel-1) image pairs.
+The ITS_LIVE monitoring stack provides the AWS architecture to support low-latency production of netCDF glacier velocity products produced from Optical (Landsat 8/9, Sentinel-2) and SAR (Sentinel-1, NISAR) image pairs.
 
 ## Architecture overview
 
 ITS_LIVE Monitoring uses a pub-sub model all missions. Data providers publish new scene messages to SNS Topics for each new scene added to the dataset. The SNS Topics for each mission data are described on these pages:
 * Landsat: <https://registry.opendata.aws/usgs-landsat/>
+* NISAR: <https://github.com/ASFHyP3/CMR-notifier>
 * Sentinel-2: <https://registry.opendata.aws/sentinel-2/>
 * Sentinel-1: <https://github.com/ASFHyP3/CMR-notifier>
 
@@ -95,6 +96,10 @@ To submit _just_ the Landsat integration test payloads to the default Landsat te
 ```shell
 make landsat-integration
 ```
+To submit _just_ the NISAR integration test payloads to the default NISAR test SNS topic, run:
+```shell
+make nisar-integration
+```
 To submit _just_ the Sentinel-1 integration test payloads to the default Sentinel-1 test SNS topic, run:
 ```shell
 make Sentinel1-integration
@@ -107,6 +112,7 @@ make Sentinel2-integration
 or, you can submit to an alternative SNS topic like:
 ```shell
 LANDSAT_TOPIC_ARN=foobar make landsat-integration
+NISAR_TOPIC_ARN=foobar make nisar-integration
 SENTINEL1_TOPIC_ARN=foobar make sentinel1-integration
 SENTINEL2_TOPIC_ARN=foobar make sentinel2-integration
 ```

cloudformation.yml

@@ -2,6 +2,9 @@ Parameters:
   LandsatTopicArn:
     Type: String
 
+  NisarTopicArn:
+    Type: String
+
   Sentinel1TopicArn:
     Type: String
 
@@ -78,7 +81,7 @@ Resources:
             Resource: !GetAtt Queue.Arn
             Condition:
               ArnEquals:
-                aws:SourceArn: [!Ref LandsatTopicArn, !Ref Sentinel1TopicArn, !Ref Sentinel2TopicArn]
+                aws:SourceArn: [!Ref LandsatTopicArn, !Ref NisarTopicArn, !Ref Sentinel1TopicArn, !Ref Sentinel2TopicArn]
       Queues:
         - !Ref Queue
 
@@ -96,6 +99,18 @@ Resources:
         s3_location:
           - prefix: 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/'
 
+  NisarSubscription:
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Ref NisarTopicArn
+      Protocol: sqs
+      Endpoint: !GetAtt Queue.Arn
+      FilterPolicyScope: MessageBody
+      FilterPolicy:
+        granule_ur:
+          - prefix: 'NISAR_L'
+
+
   Sentinel1Subscription:
     Type: AWS::SNS::Subscription
     Properties:
@@ -119,6 +134,7 @@ Resources:
       Role: !GetAtt LambdaRole.Arn
       MemorySize: 256
       Timeout: 900
+      ReservedConcurrentExecutions: 10
       Environment:
         Variables:
           HYP3_API: !Ref Hyp3Api

its_live_monitoring/src/config.py

@@ -0,0 +1,28 @@
+"""ITS_LIVE monitoring environment variable configuration."""
+
+import os
+
+
+def _nullable_str(s: str) -> str | None:
+    s = s.replace('None', '').strip()
+    return s if s else None
+
+
+def _string_is_true(s: str) -> bool:
+    return s.lower() == 'true'
+
+
+LOGGING_LEVEL = os.environ.get('LOGGING_LEVEL', 'INFO')
+
+# needed to deduplicate via HyP3
+HYP3_JOBS_TABLE_NAME = os.environ.get('JOBS_TABLE_NAME', '')
+EARTHDATA_USERNAME = os.environ.get('EARTHDATA_USERNAME')
+
+# needed to deduplicate via STAC
+STAC_ITEMS_ENDPOINT = _nullable_str(os.environ.get('STAC_ITEMS_ENDPOINT', 'None'))
+
+# needed to submit HyP3 jobs
+HYP3_API = os.environ.get('HYP3_API', '')
+EARTHDATA_PASSWORD = os.environ.get('EARTHDATA_PASSWORD')
+STAC_EXISTS_OK = _string_is_true(os.environ.get('STAC_EXISTS_OK', 'False'))
+PUBLISH_BUCKET = _nullable_str(os.environ.get('PUBLISH_BUCKET', 'None'))