User Defined Buckets

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [10.15.0]
+
+### Added
+- `bucket` and `bucket_prefix` top level parameters for all job types.
+  - The `bucket` parameter allows for overwriting the content bucket that a job's products will be placed into.
+  - The `bucket_prefix` parameter:
+    - allows for overwriting the default prefix (the job ID) for the S3 bucket.
+    - can only be used if also using a custom bucket.
+    - allows for inserting the job's ID and name by including `{job_id}` or `{name}`. 
+- `/bucket-policy/<bucket_name>` route for retrieving AWS policy to allow HyP3 to write to a custom bucket
+
 ## [10.14.4]
 
 ### Changed

apps/api/api-cf.yml.j2

@@ -24,6 +24,9 @@ Parameters:
   SystemAvailable:
     Type: String
 
+  ContentBucket:
+    Type: String
+
   {% if security_environment == 'EDC' %}
   VpcId:
     Type: String
@@ -193,6 +196,7 @@ Resources:
           DEFAULT_CREDITS_PER_USER: !Ref DefaultCreditsPerUser
           DEFAULT_APPLICATION_STATUS: !Ref DefaultApplicationStatus
           SYSTEM_AVAILABLE: !Ref SystemAvailable
+          CONTENT_BUCKET: !Ref ContentBucket
       Code: src/
       Handler: hyp3_api.lambda_handler.handler
       MemorySize: 3008

apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2

@@ -54,6 +54,10 @@ components:
             - {{ job_type }}
         name:
           $ref: "./openapi-spec.yml#/components/schemas/name"
+        bucket:
+          $ref: "./openapi-spec.yml#/components/schemas/bucket"
+        bucket_prefix: 
+          $ref: "./openapi-spec.yml#/components/schemas/bucket_prefix"
         job_parameters:
           $ref: "#/components/schemas/{{ job_type }}Parameters"
 

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -354,6 +354,10 @@ components:
           $ref: "#/components/schemas/execution_started"
         name:
           $ref: "#/components/schemas/name"
+        bucket:
+          $ref: "#/components/schemas/bucket"
+        bucket_prefix:
+          $ref: "#/components/schemas/bucket_prefix"
         files:
           $ref: "#/components/schemas/list_of_files"
         browse_images:
@@ -438,6 +442,24 @@ components:
       maxLength: 100
       example: Job Name
 
+    bucket:
+      description: User provided text to define the product bucket, null for default
+      type: string
+      nullable: true
+      minLength: 3
+      maxLength: 63
+      example: "default-s3-bucket"
+      pattern: "(?!(^((2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2}).){3}(2(5[0-5]|[0-4][0-9])|[01]?[0-9]{1,2})$|^xn--|.+-s3alias$))^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"
+
+    bucket_prefix:
+      description: User provided text to define the bucket prefix to place the product in
+      type: string
+      nullable: true
+      minLength: 1
+      maxLength: 100
+      example: "{job_id}"
+      pattern: "^(?!\/)(?!.*\/\/)(?:[A-Za-z0-9._\/-]|{job_id}|{name}){1,1024}$"
+
     start_token:
       description: Token used for fetching subsequent results for large queries
       type: string

apps/api/src/hyp3_api/handlers.py

@@ -5,6 +5,7 @@
 import dynamo
 from dynamo.exceptions import (
     AccessCodeError,
+    CustomPrefixForDefaultBucketError,
     InsufficientCreditsError,
     UnexpectedApplicationStatusError,
     UpdateJobForDifferentUserError,
@@ -24,20 +25,20 @@ def problem_format(status: int, message: str) -> Response:
 
 def post_jobs(body: dict, user: str) -> dict:
     print(body)
-
     try:
         validate_jobs(body['jobs'])
     except CmrError as e:
         abort(problem_format(503, str(e)))
     except (ValidationError, MultiBurstValidationError) as e:
         abort(problem_format(400, str(e)))
-
     try:
         body['jobs'] = dynamo.jobs.put_jobs(user, body['jobs'], dry_run=bool(body.get('validate_only')))
     except UnexpectedApplicationStatusError as e:
         abort(problem_format(403, str(e)))
     except InsufficientCreditsError as e:
         abort(problem_format(400, str(e)))
+    except CustomPrefixForDefaultBucketError as e:
+        abort(problem_format(400, str(e)))
     return body
 
 
@@ -132,3 +133,50 @@ def _get_names_for_user(user: str) -> list[str]:
         jobs.extend(new_jobs)
     names = {job['name'] for job in jobs if 'name' in job}
     return sorted(list(names))
+
+
+def get_bucket_policy(bucket_name: str) -> str:
+    account_arn = util.get_current_account_arn()
+    policy = f"""
+    {{
+        "Version": "2012-10-17",
+        "Statement": [
+            {{
+                "Sid": "write permission",
+                "Effect": "Allow",
+                "Principal": {{ "AWS": "{account_arn}:root" }},
+                "Action": "s3:PutObject",
+                "Resource": "arn:aws:s3:::{bucket_name}/*"
+            }},
+            {{
+                "Sid": "write tagging permission",
+                "Effect": "Allow",
+                "Principal": {{ "AWS": "{account_arn}:root" }},
+                "Action": "s3:PutObjectTagging",
+                "Resource": "arn:aws:s3:::{bucket_name}/*"
+            }},
+            {{
+                "Sid": "read permission",
+                "Effect": "Allow",
+                "Principal": {{ "AWS": "{account_arn}:root" }},
+                "Action": "s3:GetObject",
+                "Resource": "arn:aws:s3:::{bucket_name}/*"
+            }},
+            {{
+                "Sid": "read tagging permission",
+                "Effect": "Allow",
+                "Principal": {{ "AWS": "{account_arn}:root" }},
+                "Action": "s3:GetObjectTagging",
+                "Resource": "arn:aws:s3:::{bucket_name}/*"
+            }},
+            {{
+                "Sid": "get bucket location permission",
+                "Effect": "Allow",
+                "Principal": {{ "AWS": "arn:aws:iam::{account_arn}:root" }},
+                "Action": "s3:GetBucketLocation",
+                "Resource": "arn:aws:s3:::{bucket_name}"
+            }}
+        ]
+    }}
+    """
+    return policy

apps/api/src/hyp3_api/routes.py

@@ -193,3 +193,9 @@ def user_patch() -> Response:
 @openapi
 def user_get() -> Response:
     return jsonify(handlers.get_user(g.user))
+
+
+@app.route('/bucket-policy/<bucket_name>', methods=['GET'])
+@openapi
+def bucket_policy_get(bucket_name: str) -> Response:
+    return jsonify(handlers.get_bucket_policy(bucket_name))

apps/api/src/hyp3_api/util.py

@@ -4,6 +4,8 @@
 from typing import Any
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
+import boto3
+
 
 class TokenDeserializeError(Exception):
     """Raised when paging results and `start_token` fails to deserialize."""
@@ -46,3 +48,9 @@ def build_next_url(url: str, start_token: str, x_forwarded_host: str | None = No
     url_parts[4] = urlencode(query)
 
     return urlunparse(url_parts)
+
+
+def get_current_account_arn() -> str:
+    sts = boto3.client('sts')
+    account = sts.get_caller_identity().split(':user')[0]
+    return account

apps/get-files/src/get_files.py

@@ -91,8 +91,11 @@ def organize_files(s3_objects: list[dict], bucket: str) -> dict:
 
 
 def lambda_handler(event: dict, context: object) -> None:
-    bucket = environ['BUCKET']
+    job = dynamo.jobs.get_job(job_id=event['job_id'])
 
-    response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=event['job_id'])
+    bucket = job['bucket']
+    prefix = job['bucket_prefix']
+
+    response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix)
     files = organize_files(response['Contents'], bucket)
     dynamo.jobs.update_job({'job_id': event['job_id'], **files})

apps/main-cf.yml.j2

@@ -126,6 +126,7 @@ Resources:
         DefaultCreditsPerUser: !Ref DefaultCreditsPerUser
         DefaultApplicationStatus: !Ref DefaultApplicationStatus
         SystemAvailable: !Ref SystemAvailable
+        ContentBucket: !Ref ContentBucket
         {% if security_environment == 'EDC' %}
         VpcId: !Ref VpcId
         SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId

apps/render_cf.py

@@ -130,6 +130,10 @@ def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None =
     for param in step_params:
         if param == 'job_id':
             batch_params['job_id.$'] = '$.job_id'
+        elif param == 'bucket':
+            batch_params['bucket.$'] = '$.bucket'
+        elif param == 'bucket_prefix':
+            batch_params['bucket_prefix.$'] = '$.bucket_prefix'
         elif param == map_item:
             batch_params[f'{map_item}.$'] = "States.Format('{}', $$.Map.Item.Value)"
         else:

job_spec/AK_FIRE_SAFE.yml

@@ -39,9 +39,9 @@ AK_FIRE_SAFE:
       image: ghcr.io/asfhyp3/hyp3-gather-landsat
       command:
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --start-date
         - Ref::start_date
         - --end-date

job_spec/ARIA_AUTORIFT.yml

@@ -148,9 +148,9 @@ AUTORIFT:
       command:
         - Ref::granules
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --parameter-file
         - Ref::parameter_file
         - --naming-scheme

job_spec/ARIA_RAIDER.yml

@@ -30,9 +30,9 @@ ARIA_RAIDER:
         - ++process
         - calcDelaysGUNW
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --weather-model
         - Ref::weather_model
         - --input-bucket-prefix

job_spec/ARIA_S1_COSEIS.yml

@@ -104,9 +104,9 @@ ARIA_S1_COSEIS:
         - ++omp-num-threads
         - '4'  # 8 vCPUs per 16 GB RAM for the C instance family; 4 for M; 2 for R
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --reference-scenes
         - Ref::granules
         - --secondary-scenes

job_spec/ARIA_S1_GUNW.yml

@@ -61,9 +61,9 @@ ARIA_S1_GUNW:
         - ++process
         - calcDelaysGUNW
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --weather-model
         - HRRR
       timeout: 10800  # 3 hr

job_spec/AUTORIFT.yml

@@ -42,9 +42,9 @@ AUTORIFT:
       command:
         - Ref::granules
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --parameter-file
         - '/vsicurl/https://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp'
         - --naming-scheme

job_spec/INSAR_GAMMA.yml

@@ -86,9 +86,9 @@ INSAR_GAMMA:
         - ++omp-num-threads
         - '4'
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --include-look-vectors
         - Ref::include_look_vectors
         - --include-los-displacement

job_spec/INSAR_ISCE.yml

@@ -102,9 +102,9 @@ INSAR_ISCE:
         - ++omp-num-threads
         - '4'  # 8 vCPUs per 16 GB RAM for the C instance family; 4 for M; 2 for R
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --reference-scenes
         - Ref::granules
         - --secondary-scenes

job_spec/INSAR_ISCE_BURST.yml

@@ -50,9 +50,9 @@ INSAR_ISCE_BURST:
         - ++omp-num-threads
         - '1'
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --apply-water-mask
         - Ref::apply_water_mask
         - --looks

job_spec/INSAR_ISCE_MULTI_BURST.yml

@@ -109,9 +109,9 @@ INSAR_ISCE_MULTI_BURST:
         - ++process
         - insar_tops_multi_burst
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --apply-water-mask
         - Ref::apply_water_mask
         - --looks

job_spec/ITS_LIVE_AUTORIFT.yml

@@ -199,9 +199,9 @@ AUTORIFT:
         - ++plugin
         - meta
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
         - --publish-bucket
         - Ref::publish_bucket
         - --stac-items-endpoint

job_spec/ITS_LIVE_CROP.yml

@@ -41,9 +41,10 @@ ITS_LIVE_CROP:
         - crop_netcdf_product
         - Ref::granule_uri
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
+        # FIXME:
         - --publish-bucket
         - Ref::publish_bucket
       timeout: 10800
@@ -61,9 +62,10 @@ ITS_LIVE_CROP:
         - --granule-uri
         - Ref::granule_uri
         - --bucket
-        - '!Ref Bucket'
+        - Ref::bucket
         - --bucket-prefix
-        - Ref::job_id
+        - Ref::bucket_prefix
+        # FIXME:
         - --publish-bucket
         - Ref::publish_bucket
         - --stac-items-endpoint