Skip to content

Commit 95a72b8

Browse files
mikedorfmanpre-commit-ci[bot]
mikedorfman
and
pre-commit-ci[bot]
authored
CUMULUS-4385 (#4229)
* Added Dockerfile to facilitate UV builds Added granule-invalidator Terraform and deployment mechanism Added the `deploy` dir to the create-release-artifacts.sh script so task Terraform is included in release * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated ChangeLog Updated package.json based on updates to scripts * Fixed package.sh script to reference the correct lambda tag * Update to reference correct docker binary path * Updated package.sh to avoid docker * Updated the python-platform flag * Fixed broken path for lambda zipfile * Updated runtime from python3.14 to python3.13 * Updates per PR suggestions * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fixed ruff issues * Updated package.sh to be consistent with other tasks Added ${PWD} arg to package.json * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Temporarily allowed fast-xml-parser to unblock feature branch development * Updated lambda path for granule-invalidator * Trying to pin a non-vulnerable version of fast-xml-parser * Using overrides for a more targeted override of fast-xml-parser * Overriding the dependency in xml-builder instead of core * Moved fast-xml-parser override to the root package.json * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Removed pin of non-vulnerable version since downstream dependencies have now been updated * Added granule_invalidator to outputs of ingest and cumulus modules * Fixed issue with granule-invalidator outputs * Fixed broken output path --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent e9fb43f commit 95a72b8

21 files changed

Lines changed: 376 additions & 21 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,4 @@ yarn.lock
5151
unit-logs/
5252
test_output.txt
5353
__pycache__/
54+
*.egg-info/

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ Please complete the following steps before upgrading Cumulus.
8585
The module also configures the required RDS extensions and parameters necessary for slow query instrumentation.
8686
- **CUMULUS-4382**
8787
- Migrated the granule-invalidator task to the `tasks` directory as part of a coreification task in support of providing rolling archive functionality.
88+
- **CUMULUS-4385**
89+
- Added supporting Terraform for the granule-invalidator task that allows it to be included in the Cumulus terraform zipfile and deployed with Cumulus.
8890

8991
### Changed
9092

‎bamboo/create-release-artifacts.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@ set -ex
99
(cd tf-modules/cumulus-rds-tf && ./bin/build-tf-module.sh && cp ./dist/terraform-aws-cumulus-rds.zip ../../terraform-aws-cumulus-rds.zip)
1010

1111
## Create zipfiles
12-
zip -FS -r -x \*node_modules\* \*internal\* -o terraform-aws-cumulus.zip tf-modules lambdas tasks/**/dist/** packages/**/dist/**
12+
zip -FS -r -x \*node_modules\* \*internal\* -o terraform-aws-cumulus.zip tf-modules lambdas tasks/**/dist/** tasks/**/deploy/** packages/**/dist/**
1313
(cd ./tf-modules/workflow; zip -FS -r -x \*node_modules\* -o ../../terraform-aws-cumulus-workflow.zip .)
1414
(cd ./tf-modules/cumulus_ecs_service; zip -FS -r -x \*node_modules\* -o ../../terraform-aws-cumulus-ecs-service.zip .)

‎tasks/granule-invalidator/bin/Dockerfile.package‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# TODO; this is currently unused since the Cumulus Bamboo build runs in a Docker runner and using docker-in-docker is difficult to configure.
2+
3+
ARG UV_TAG="0.9"
4+
ARG LAMBDA_TAG="3.14"
5+
6+
FROM ghcr.io/astral-sh/uv:${UV_TAG} AS uv
7+
8+
# First, bundle the dependencies into the task root.
9+
FROM public.ecr.aws/lambda/python:${LAMBDA_TAG} AS builder
10+
11+
#ENV OUTPUT_FILENAME="python_reference_task-${PACKAGE_VERSION}-${ARCHITECTURE}.zip"
12+
ARG OUTPUT_FILENAME
13+
14+
# Enable bytecode compilation, to improve cold-start performance.
15+
ENV UV_COMPILE_BYTECODE=1
16+
17+
# Disable installer metadata, to create a deterministic layer.
18+
ENV UV_NO_INSTALLER_METADATA=1
19+
20+
# Enable copy mode to support bind mount caching.
21+
ENV UV_LINK_MODE=copy
22+
23+
# Install a compiler for packages
24+
RUN dnf install -y gcc git
25+
26+
# Copies for debugging
27+
COPY --from=uv /uv /bin/uv
28+
COPY uv.lock pyproject.toml README.md ./
29+
COPY src ./src
30+
COPY schemas ./schemas
31+
32+
# Bundle the dependencies into the Lambda task root via `uv pip install --target`.
33+
#
34+
# Omit any local packages (`--no-emit-workspace`) and development dependencies (`--no-dev`).
35+
# This ensures that the Docker layer cache is only invalidated when the `pyproject.toml` or `uv.lock`
36+
# files change, but remains robust to changes in the application code.
37+
RUN mkdir -p /dist/{dist,packages,final} \
38+
&& ls -ls \
39+
&& uv export \
40+
--frozen \
41+
--no-emit-workspace \
42+
--all-extras \
43+
--no-dev \
44+
--no-editable \
45+
-o /dist/requirements.txt \
46+
&& uv build \
47+
--clear \
48+
--wheel \
49+
-o /dist/dist \
50+
&& uv pip install \
51+
--requirements /dist/requirements.txt \
52+
--target /dist/packages \
53+
&& uv pip install /dist/dist/*.whl --target /dist/packages \
54+
&& cd /dist/packages; zip -r /dist/final/${OUTPUT_FILENAME} .
55+
56+
FROM scratch
57+
COPY --from=builder /dist/final /

‎tasks/granule-invalidator/bin/package.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
set -xe
4+
5+
DIR=$1
6+
# find packages location
7+
echo "Entering $DIR"
8+
echo $PWD
9+
cd "$DIR" || exit 1
10+
11+
# ensure packages are up to date using uv
12+
uv sync --no-dev --frozen
13+
14+
# package dependencies
15+
SITE_PACKAGES=$(find "$DIR"/.venv/lib/python*/site-packages -type d | head -1)
16+
echo "Entering $SITE_PACKAGES"
17+
cd "$SITE_PACKAGES" || exit 1
18+
cp -R ./* "$DIR/dist/"
19+
20+
cd "$DIR" || exit 1
21+
22+
cp -R ./src/* schemas/ ./dist/
23+
24+
cd ./dist || exit 1
25+
26+
node ../../../bin/zip.js lambda.zip $(ls | grep -v lambda.zip)
27+
28+
cd .. || exit 1
29+
30+
# Re-sync dev dependencies for any post-packaging tasks
31+
uv sync

‎tasks/granule-invalidator/bin/package_docker.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# TODO; this is currently unused since the Cumulus Bamboo build runs in a Docker runner and using docker-in-docker is difficult to configure.
2+
3+
#!/bin/bash
4+
CONFIG=$(jq -r '.' build-config.json)
5+
RUNTIME=$(echo $CONFIG | jq -r '.runtime')
6+
PYTHON_VERSION=$(echo $RUNTIME | sed 's/^python//')
7+
ARCHITECTURE=$(echo $CONFIG | jq -r '.architecture')
8+
UV_VERSION=$(echo $CONFIG | jq -r '.uv_version')
9+
10+
docker build \
11+
--platform linux/${ARCHITECTURE} \
12+
--build-arg UV_TAG=${UV_VERSION} \
13+
--build-arg LAMBDA_TAG=${PYTHON_VERSION}-${ARCHITECTURE} \
14+
--build-arg OUTPUT_FILENAME=lambda.zip \
15+
--output type=local,dest=dist/ \
16+
--file bin/Dockerfile.package .

‎tasks/granule-invalidator/build-config.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"runtime": "python3.13",
3+
"uv_version": "0.9.21",
4+
"architecture": "x86_64"
5+
}

‎tasks/granule-invalidator/deploy/README.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Granule-invalidator Terraform Module
2+
3+
This module creates the granule invalidator Lambda function used in Cumulus workflows.
4+
5+
## Usage
6+
7+
```hcl
8+
module "granule_invalidator" {
9+
source = "../lambdas/granule-invalidator/deploy"
10+
11+
prefix = var.prefix
12+
role = module.cumulus.lambda_processing_role_arn
13+
layers = [var.cumulus_message_adapter_lambda_layer_version_arn]
14+
subnet_ids = var.lambda_subnet_ids
15+
security_group_id = aws_security_group.no_ingress_all_egress.id
16+
default_log_retention_days = var.default_log_retention_days
17+
tags = local.tags
18+
}
19+
```
20+
21+
## Requirements
22+
23+
- The Lambda deployment package must be built and available at `../dist/lambda.zip` relative to this module
24+
- Terraform >= 1.0
25+
- AWS Provider >= 5.0
26+
27+
## Inputs
28+
29+
| Name | Description | Type | Default | Required |
30+
| ------ | ------------- | ------ | --------- | :--------: |
31+
| prefix | The prefix for resource names | `string` | n/a | yes |
32+
| role | ARN of the IAM role for Lambda execution | `string` | n/a | yes |
33+
| layers | ARN of the Cumulus Message Adapter Lambda layer | `list(string)` | n/a | yes |
34+
| default_log_retention_days | The number of days to retain logs in CloudWatch | `number` | n/a | yes |
35+
| subnet_ids | List of subnet IDs for Lambda VPC configuration | `list(string)` | `[]` | no |
36+
| security_group_id | Security group ID for Lambda VPC configuration | `string` | `""` | no |
37+
| timeout | Timeout value for the Lambda function in seconds | `number` | `900` | no |
38+
| memory_size | Memory size for the Lambda function in MB | `number` | `4096` | no |
39+
| environment | Environment variables for the Lambda function. This is a map that's merged with a set of defaults. | `map(string)` | `{}` | no |
40+
| tags | Tags to be applied to resources | `map(string)` | `{}` | no |
41+
42+
## Outputs
43+
44+
| Name | Description |
45+
| ------ | ------------- |
46+
| lambda_function_arn | ARN of the Lambda function |
47+
| lambda_function_name | Name of the Lambda function |
48+
| lambda_function_invoke_arn | Invoke ARN of the Lambda function |

‎tasks/granule-invalidator/deploy/main.tf‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
locals {
2+
build_config = jsondecode(file("${path.module}/../build-config.json"))
3+
}
4+
5+
resource "aws_lambda_function" "granule_invalidator_task" {
6+
function_name = "${var.prefix}-granule-invalidator-task"
7+
filename = "${path.module}/../dist/lambda.zip"
8+
source_code_hash = filebase64sha256("${path.module}/../dist/lambda.zip")
9+
handler = "main.handler"
10+
role = var.role
11+
runtime = local.build_config.runtime
12+
architectures = [local.build_config.architecture]
13+
timeout = var.timeout
14+
memory_size = var.memory_size
15+
16+
layers = var.layers
17+
18+
environment {
19+
variables = merge(
20+
{
21+
stackName = var.prefix
22+
CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/"
23+
},
24+
var.environment
25+
)
26+
}
27+
28+
dynamic "vpc_config" {
29+
for_each = length(var.subnet_ids) == 0 ? [] : [1]
30+
content {
31+
subnet_ids = var.subnet_ids
32+
security_group_ids = [var.security_group_id]
33+
}
34+
}
35+
36+
tags = var.tags
37+
}
38+
39+
resource "aws_cloudwatch_log_group" "granule_invalidator_task" {
40+
name = "/aws/lambda/${aws_lambda_function.granule_invalidator_task.function_name}"
41+
retention_in_days = var.default_log_retention_days
42+
tags = var.tags
43+
}

‎tasks/granule-invalidator/deploy/outputs.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
output "lambda_function_arn" {
2+
description = "ARN of the Lambda function"
3+
value = aws_lambda_function.granule_invalidator_task.arn
4+
}
5+
6+
output "lambda_function_name" {
7+
description = "Name of the Lambda function"
8+
value = aws_lambda_function.granule_invalidator_task.function_name
9+
}
10+
11+
output "lambda_function_invoke_arn" {
12+
description = "Invoke ARN of the Lambda function"
13+
value = aws_lambda_function.granule_invalidator_task.invoke_arn
14+
}
15+
16+
output "lambda_function_last_modified" {
17+
description = "Last modified date of the Lambda function"
18+
value = aws_lambda_function.granule_invalidator_task.last_modified
19+
}

0 commit comments

Comments
 (0)