wip: registry deployed in dev with api and sweeper manually added from their own terraform, integration test still fail.

Author: Thomas Loubrieu

terraform/README.md

@@ -114,7 +114,7 @@ Then set you python environment:
 
     python3.12 -m venv venv
     source venv/bin/activate
-    pip install jinja2 requests
+    pip install jinja2 requests boto3
 
 
 And run the script to initialize the registry:

terraform/data.tf

@@ -8,4 +8,12 @@ data "aws_ssm_parameter" "opensearch_readonly_role_arn" {
 
 data "aws_ssm_parameter" "lambda_execution_role_arn" {
   name = "/pds/cds-infra/iam/roles/registry/lambda_execution_role_arn"
-}
+}
+
+data "aws_ssm_parameter" "registry_api_ecs_task_role_arn" {
+  name     = "/pds/cds-infra/iam/roles/registry/ecs_task_role_arn"
+}
+
+data "aws_ssm_parameter" "registry_api_ecs_task_execution_role_arn" {
+  name     = "/pds/cds-infra/iam/roles/registry/ecs_task_execution_role_arn"
+}

terraform/main.tf

@@ -14,7 +14,7 @@ module "opensearch" {
   vpc_id             = var.vpc_id
   subnet_ids         = var.subnet_ids
   admin_roles        = concat(var.admin_roles, [data.aws_ssm_parameter.opensearch_admin_role_arn.value])
-  readonly_roles     = [data.aws_ssm_parameter.opensearch_readonly_role_arn.value]
+  readonly_roles     = [data.aws_ssm_parameter.opensearch_readonly_role_arn.value, data.aws_ssm_parameter.registry_api_ecs_task_role_arn.value]
   node_list          = var.node_list
   aws_region         = var.aws_region
   common_tags        = var.common_tags
@@ -54,3 +54,23 @@ module "api_gateway" {
   aws_region           = var.aws_region
   common_tags          = var.common_tags
 }
+
+# module "registry_api" {
+#   source = "git::https://github.com/NASA-PDS/registry-api.git//terraform?ref=terraform_for_dev"
+#   count  = 0
+#
+#   aws_region = var.aws_region
+#   sprint_boot_args = "--openSearch.host=${module.opensearch.collection_endpoint} --openSearch.CCSEnabled=true --openSearch.username='' --openSearch.disciplineNodes=${join(",", var.node_list)} --registry.service.version=1.6.0-SNAPSHOT"
+#   aws_s3_bucket_logs_id=var.aws_s3_bucket_logs_id
+#   registry_api_docker_image=var.registry_api_docker_image
+#   ecs_task_role = data.aws_ssm_parameter.registry_api_ecs_task_role_arn
+#   ecs_task_execution_role = data.aws_ssm_parameter.registry_api_ecs_task_execution_role_arn
+#   aws_fg_vpc = var.vpc_id
+#   aws_fg_security_group = var.registry_api_ecs_service_security_group
+#   aws_fg_subnets = var.subnet_ids
+#   aws_lb_subnets = var.public_subnet_ids
+#   aws_acm_certificate_arn = var.acm_certificate_arn
+#
+#   common_tags = var.common_tags
+#
+# }

terraform/opensearch/data.tf

@@ -9,3 +9,4 @@ data "aws_ssm_parameter" "opensearch_tenant_core_cloudops_role_arns" {
 }
 
 
+

terraform/run-init-on-aws.py

@@ -14,10 +14,14 @@
 import tarfile
 import zipfile
 import urllib.request
+import urllib.error
 from pathlib import Path
 from typing import Optional, Dict
 from datetime import datetime
 from jinja2 import Template
+import boto3
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
 
 
 class RegistryInitializer:
@@ -187,14 +191,18 @@ def load_env_file(self):
                             self.env_vars[key] = value
         else:
             print("⚠️  Warning: .env file not found, using default values")
-            self.env_vars.setdefault("REG_LOADER_IMAGE", "nasapds/registry-loader:latest")
+            self.env_vars.setdefault("REG_LOADER_IMAGE", "nasapds/registry-loader-lite:latest")
             self.env_vars.setdefault("TEST_DATA_URL",
                 "https://github.com/NASA-PDS/registry-ref-data/releases/download/Latest/custom-datasets.tar.gz")
             self.env_vars.setdefault("TEST_DATA_LIDVID",
                 "urn:nasa:pds:mars2020.spice::1.0 urn:nasa:pds:mars2020.spice::2.0 urn:nasa:pds:mars2020.spice::3.0")
             self.env_vars.setdefault("CONTAINER_HARVEST_DATA_DIR", "/data")
             self.env_vars.setdefault("REG_DATA_VOLUME", "data-volume")
 
+    def add_locally_defined_env_vars(self) -> None:
+        for k , v in os.environ.items():
+            self.env_vars[k] = v
+
     def generate_registry_config(self):
         """Generate registry connection XML configuration."""
         self.print_section("📝", "Generating registry connection configuration...")
@@ -310,6 +318,45 @@ def run_docker_container(self, purpose: str, command: list):
         self.run_command(docker_cmd, capture=False)
         print("-" * 50 + "\n")
 
+    def create_aliases(self):
+        """Create OpenSearch aliases, until registry-manager does it."""
+        self.print_section("🔗", "Create aliases, until registry-manager does it")
+
+        alias_file = self.docker_dir / "scripts" / "aliases" / "alias_registry.json"
+        if not alias_file.exists():
+            print(f"   Warning: Alias file not found: {alias_file}")
+            return
+
+        alias_content = alias_file.read_text()
+        url = f"{self.opensearch_endpoint}/_aliases"
+
+        session = boto3.Session(region_name=self.aws_region)
+        caller = session.client("sts").get_caller_identity()
+        print(f"   Calling as: {caller['Arn']}")
+        credentials = session.get_credentials().get_frozen_credentials()
+
+        aws_request = AWSRequest(
+            method="POST",
+            url=url,
+            data=alias_content,
+            headers={"Content-Type": "application/json"},
+        )
+        SigV4Auth(credentials, "aoss", self.aws_region).add_auth(aws_request)
+
+        req = urllib.request.Request(
+            url,
+            data=alias_content.encode("utf-8"),
+            headers=dict(aws_request.headers),
+            method="POST",
+        )
+
+        try:
+            with urllib.request.urlopen(req) as response:
+                print(f"   Aliases created: {response.read().decode()}\n")
+        except urllib.error.HTTPError as e:
+            print(f"   Error creating aliases: {e.read().decode()}\n")
+            raise
+
     def cleanup_temp_files(self):
         """Clean up temporary authentication configuration files."""
         temp_files = [
@@ -413,6 +460,7 @@ def run(self) -> int:
 
             # Setup
             self.load_env_file()
+            self.add_locally_defined_env_vars()
             self.generate_registry_config()
             self.generate_auth_configs()
 
@@ -426,6 +474,7 @@ def run(self) -> int:
             except subprocess.CalledProcessError:
                 print("\n⚠️  OpenSearch indexes probably already created, skip that step\n")
 
+            self.create_aliases()
             self.download_and_extract_test_data()
             self.generate_harvest_config()
             self.run_docker_container(

terraform/run-init-on-aws.sh

@@ -16,9 +16,12 @@ enable_public_access = true
 # Option 2: VPC-only access via VPC endpoint (recommended for production)
 # enable_public_access = false
 # create_vpc_endpoint = true
-# vpc_id = "vpc-1234567890abcdef0"
-# subnet_ids = ["subnet-1234567890abcdef0", "subnet-0987654321fedcba0"]
-# security_group_ids = []  # Leave empty to create a default security group
+
+# Needed for VPC end-point above and registry API load balancer and ECS service
+vpc_id = "vpc-1234567890abcdef0"
+subnet_ids = ["subnet-1234567890abcdef0", "subnet-0987654321fedcba0"]
+public_subnet_ids = ["subnet-3234567890abcdef0", "subnet-3987654321fedcba0"]
+security_group_id = []  # Leave empty to create a default security group
 
 # Admin roles configuration (see DEPLOYMENT_STAGES.md for deployment workflow)
 # Stage 1 (initial): Deploy without admin role