Merge pull request #328 from ASFHyP3/develop

Release V0.7.0

Author: jhkennedy

.github/actions/deploy/action.yml

@@ -24,10 +24,12 @@ inputs:
     required: true
   PUBLISH_BUCKET:
     required: true
-  MATTERMOST_PAT:
-    required: true
   ECR_IMAGE_URI:
     required: true
+  STAC_ITEMS_ENDPOINT:
+    required: true
+  STAC_EXISTS_OK:
+    required: true
 
 runs:
   using: composite
@@ -54,8 +56,9 @@ runs:
             EarthdataUsername=${{ inputs.EARTHDATA_USERNAME }} \
             EarthdataPassword=${{ inputs.EARTHDATA_PASSWORD }} \
             PublishBucket=${{ inputs.PUBLISH_BUCKET }} \
-            MattermostPAT=${{ inputs.MATTERMOST_PAT }} \
-            EcrImageUri=${{ inputs.ECR_IMAGE_URI }}
+            EcrImageUri=${{ inputs.ECR_IMAGE_URI }} \
+            StacItemsEndpoint=${{ inputs.STAC_ITEMS_ENDPOINT }} \
+            StacExistsOk=${{ inputs.STAC_EXISTS_OK }}
 
         export CF_TEMPLATE_BUCKET=${{ inputs.CF_TEMPLATE_BUCKET }}
         export ITS_LIVE_MONITORING_QUEUE_ARN=$( \

.github/workflows/build-and-deploy.yml

@@ -50,6 +50,7 @@ jobs:
           platforms: ${{ env.PLATFORMS }}
 
       - uses: aws-actions/configure-aws-credentials@v5
+        if: ${{ github.actor != 'dependabot[bot]' }}
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -122,5 +123,6 @@ jobs:
           HYP3_JOBS_TABLE: ${{ vars.HYP3_JOBS_TABLE }}
           LAMBDA_LOGGING_LEVEL: INFO
           PUBLISH_BUCKET: ${{ vars.PUBLISH_BUCKET }}
-          MATTERMOST_PAT: ${{ secrets.MATTERMOST_PAT }}
+          STAC_ITEMS_ENDPOINT: ${{ vars.STAC_ITEMS_ENDPOINT }}
+          STAC_EXISTS_OK: ${{ vars.STAC_EXISTS_OK }}
           ECR_IMAGE_URI: ${{ needs.dockerize.outputs.image_uri }}

CHANGELOG.md

@@ -4,9 +4,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0]
+## Added
+- `StacItemsEndpoint` and `StacExistsOk` cloudformation parameters to allow publishing STAC items directly to a STAC catalog. Accordingly:
+  - `STAC_ITEMS_ENDPOINT` and `STAC_EXISTS_OK` environment variables are set for the monitoring lambda.
+  - The cloud formation parameters are set in the build and deploy GitHub Actions workflow by the `STAC_ITEMS_ENDPOINT` and `STAC_EXISTS_OK` deploy environment variables.
+
+## Changed
+- Updated the `AUTORIFT_JOB_TEMPLATE` for [HyP3 v10.13.0+](https://github.com/ASFHyP3/hyp3/pull/3003) to allow posting/putting STAC items in the catalog instead of writing them to an alternate ingest location.
+
+### Removed
+- Support for dead-letter queue monitoring through Mattermost.
+
 ## [0.6.1]
 ### Added
-- A check to ensure that an orbit file is availble for Sentinel-1 scenes
+- A check to ensure that an orbit file is available for Sentinel-1 scenes
 
 ### Changed
 - Sentinel-1 frame overlap has been removed such that bursts that could occur in multiple opera frames now only occur in one.

Dockerfile

@@ -5,10 +5,5 @@ RUN pip install -r  requirements-its_live_monitoring.txt
 
 COPY its_live_monitoring/src ${LAMBDA_TASK_ROOT}
 
-COPY requirements-status-messages.txt ${LAMBDA_TASK_ROOT}
-RUN pip install -r  requirements-status-messages.txt
-
-COPY status-messages/src ${LAMBDA_TASK_ROOT}
-
 # NOTE: handler set as CMD by  parameter override outside of the Dockerfile
 # CMD [ "lambda_function.handler" ]

cloudformation.yml

@@ -34,9 +34,17 @@ Parameters:
       - its-live-data
       - its-live-data-test
 
-  MattermostPAT:
+  StacItemsEndpoint:
     Type: String
-    NoEcho: true
+    Default: "None"
+
+  StacExistsOk:
+    Type: String
+    Default: "false"
+    AllowedValues:
+      - "false"
+      - "true"
+
 
   EcrImageUri:
     Description: ECR image URI
@@ -119,6 +127,8 @@ Resources:
           EARTHDATA_USERNAME: !Ref EarthdataUsername
           EARTHDATA_PASSWORD: !Ref EarthdataPassword
           PUBLISH_BUCKET: !Ref PublishBucket
+          STAC_ITEMS_ENDPOINT: !Ref StacItemsEndpoint
+          STAC_EXISTS_OK: !Ref StacExistsOk
 
   LambdaEventSourceMapping:
     Type: AWS::Lambda::EventSourceMapping
@@ -186,17 +196,6 @@ Resources:
               - ecr:GetDownloadUrlForLayer
             Resource: !Sub "arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/*"
 
-  StatusMessages:
-    Type: AWS::CloudFormation::Stack
-    Properties:
-      Parameters:
-        QueueURL: !Ref DeadLetterQueue
-        QueueName: !GetAtt DeadLetterQueue.QueueName
-        MattermostPAT: !Ref MattermostPAT
-        LambdaLoggingLevel: !Ref LambdaLoggingLevel
-        EcrImageUri: !Ref EcrImageUri
-      TemplateURL: status-messages/cloudformation.yml
-
 Outputs:
   ItsLiveMonitoringQueueArn:
     Description: "The ARN for its-live-monitoring's SQS queue"

its_live_monitoring/src/main.py

@@ -41,9 +41,10 @@
         # 'secondary': list[str],
         'parameter_file': '/vsicurl/https://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp',
         # 'publish_bucket': str | None,
-        'publish_stac_prefix': 'stac-ingest',
         'use_static_files': True,
         # 'frame_id' = str | None,
+        # 'stac_items_endpoint': str | None,
+        # 'stac_exists_ok': bool,
     },
     'job_type': 'AUTORIFT',
     # 'name': str | None,
@@ -97,7 +98,7 @@ def get_key(tile_prefixes: list[str], reference: str, secondary: str) -> str | N
     for tile_prefix in tile_prefixes:
         prefix = f'{tile_prefix}/{reference}_X_{secondary}'
         response = s3.list_objects_v2(
-            Bucket=os.environ.get('PUBLISH_BUCKET', 'its-live-data'),
+            Bucket=os.environ['PUBLISH_BUCKET'],
             Prefix=prefix,
         )
         for item in response.get('Contents', []):
@@ -233,6 +234,15 @@ def deduplicate_hyp3_pairs(pairs: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
     return pairs.reset_index()
 
 
+def _nullable_str(s: str) -> str | None:
+    s = s.replace('None', '').strip()
+    return s if s else None
+
+
+def _string_is_true(s: str) -> bool:
+    return s.lower() == 'true'
+
+
 def submit_pairs_for_processing(pairs: gpd.GeoDataFrame) -> sdk.Batch:  # noqa: D103
     prepared_jobs = []
     for reference, secondary, name in pairs[['reference', 'secondary', 'job_name']].itertuples(index=False):
@@ -242,7 +252,11 @@ def submit_pairs_for_processing(pairs: gpd.GeoDataFrame) -> sdk.Batch:  # noqa:
         prepared_job['job_parameters']['secondary'] = secondary
 
         if publish_bucket := os.environ.get('PUBLISH_BUCKET', ''):
-            prepared_job['job_parameters']['publish_bucket'] = publish_bucket
+            prepared_job['job_parameters']['publish_bucket'] = _nullable_str(publish_bucket)
+
+        if stac_items_endpoints := os.environ.get('STAC_ITEMS_ENDPOINT', ''):
+            prepared_job['job_parameters']['stac_items_endpoint'] = _nullable_str(stac_items_endpoints)
+            prepared_job['job_parameters']['stac_exists_ok'] = _string_is_true(os.environ.get('STAC_EXISTS_OK', ''))
 
         if name.startswith('OPERA'):
             prepared_job['job_parameters']['frame_id'] = name.split('_')[1]
@@ -311,12 +325,13 @@ def process_scene(
 
     # FIXME: Sentinel-1's file name is not easily predictable from the burst acquisitions so we can't do this yet
     # TODO: Instead of looking in the bucket, we should look in the (pending) STAC ITS_LIVE catalog
-    if len(pairs) > 0 and not scene.startswith('S1'):
-        pairs = deduplicate_s3_pairs(pairs)
+    if os.environ.get('PUBLISH_BUCKET', ''):
+        if len(pairs) > 0 and not scene.startswith('S1'):
+            pairs = deduplicate_s3_pairs(pairs)
 
-        log.info(f'Deduplicated already published pairs; {len(pairs)} remaining')
-        with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):
-            log.debug(pairs.sort_values(by=['secondary'], ascending=False).loc[:, ['reference', 'secondary']])
+            log.info(f'Deduplicated already published pairs; {len(pairs)} remaining')
+            with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):
+                log.debug(pairs.sort_values(by=['secondary'], ascending=False).loc[:, ['reference', 'secondary']])
 
     jobs = sdk.Batch()
     if submit:

requirements-all.txt

@@ -1,5 +1,4 @@
 -r requirements-its_live_monitoring.txt
--r requirements-status-messages.txt
 cfn-lint==1.42.0
 ruff==0.14.7
 moto==5.1.18

requirements-its_live_monitoring.txt

@@ -1,4 +1,4 @@
-geopandas==1.1.1
+geopandas==1.1.2
 hyp3-sdk==7.7.5
 pandas==2.3.3
 pystac-client==0.9.0