simplify docker compose, adjust test or latest PR of registry-api #753

Author: thomas loubrieu

docker/docker-compose.yml

@@ -31,20 +31,56 @@
 version: "3.9"
 
 services:
+  # Starts Elasticsearch
+  elasticsearch:
+    profiles: ["dev-api", "dev-loader", "pds-core-registry", "int-registry-batch-loader", "os"]
+    image: ${ES_IMAGE}
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    hostname: elasticsearch
+    container_name: elasticsearch
+    environment:
+      - discovery.type=${ES_DISCOVERY_TYPE}
+      - _JAVA_OPTIONS=-XX:-UseContainerSupport
+    # Uncomment the following environment variable to use OpenSearch without security
+    # - plugins.security.disabled=true
+    volumes:
+      - ./certs/node1.pem:/usr/share/opensearch/config/esnode.pem
+      - ./certs/node1-key.pem:/usr/share/opensearch/config/esnode-key.pem
+      - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
+      - ./default-config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+    ports:
+      - "9200:9200"
+      - "9600:9600" # required for Performance Analyzer
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - curl --insecure -f https://elasticsearch:9200/_cluster/health -u admin:admin
+      interval: 10s
+      timeout: 2s
+      retries: 50
+    networks:
+      - pds
 
-  # Executes the Registry Loader component, which contains Harvest and Registry Manager
-  registry-loader:
-    profiles: ["pds-batch-loader"]
+  # Initializes Elasticsearch by creating registry and data dictionary indices by utilizing the Registry Loader
+  elasticsearch-init:
+    profiles: ["dev-api", "pds-core-registry", "int-registry-batch-loader"]
     image: ${REG_LOADER_IMAGE}
     environment:
       - ES_URL=${ES_URL}
     volumes:
-      - ${HARVEST_JOB_CONFIG_FILE}:/cfg/harvest-config.xml
-      - ${REG_DATA_VOLUME}:${CONTAINER_HARVEST_data-volume}
+      - ./scripts/elasticsearch-init.sh:/usr/local/bin/elasticsearch-init.sh
+      - ./scripts/aliases:/usr/local/bin/aliases
       - ./default-config/es-auth.cfg:/etc/es-auth.cfg
-      - ./default-config/local-registry.xml:/etc/local-registry.xml
+      - ./default-config/local_registry.xml:/etc/local_registry.xml
     networks:
       - pds
+    entrypoint: ["bash", "/usr/local/bin/elasticsearch-init.sh"]
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
 
   # Downloads and harvests test data with the Registry loader
   registry-loader-test-init:
@@ -57,12 +93,13 @@ services:
       - TEST_DATA_LIDVID=${TEST_DATA_LIDVID}
     volumes:
       - ${REG_DATA_VOLUME}:${CONTAINER_HARVEST_DATA_DIR}
-      - ./scripts/registry-loader-waits-for-elasticsearch.sh:/usr/local/bin/registry-loader-waits-for-elasticsearch.sh
       - ./default-config/es-auth.cfg:/etc/es-auth.cfg
       - ./custom-datasets.tar.gz:/etc/custom-datasets.tar.gz
     networks:
       - pds
-    entrypoint: /usr/local/bin/registry-loader-waits-for-elasticsearch.sh
+    depends_on:
+      elasticsearch-init:
+        condition: service_completed_successfully
 
   # Runs provenance script
   registry-sweepers:
@@ -81,46 +118,7 @@ services:
       registry-loader-test-init:
         condition: service_completed_successfully
 
-  # Starts Elasticsearch
-  elasticsearch:
-    profiles: ["dev-api", "dev-loader", "pds-core-registry", "int-registry-batch-loader", "os"]
-    image: ${ES_IMAGE}
-    ulimits:
-      nofile:
-        soft: 65536
-        hard: 65536
-    hostname: elasticsearch
-    container_name: elasticsearch
-    environment:
-      - discovery.type=${ES_DISCOVERY_TYPE}
-      - _JAVA_OPTIONS=-XX:-UseContainerSupport
-    # Uncomment the following environment variable to use OpenSearch without security
-    # - plugins.security.disabled=true
-    volumes:
-      - ./certs/node1.pem:/usr/share/opensearch/config/esnode.pem
-      - ./certs/node1-key.pem:/usr/share/opensearch/config/esnode-key.pem
-      - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
-      - ./default-config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
-    ports:
-      - "9200:9200"
-      - "9600:9600" # required for Performance Analyzer
-    networks:
-      - pds
 
-  # Initializes Elasticsearch by creating registry and data dictionary indices by utilizing the Registry Loader
-  elasticsearch-init:
-    profiles: ["dev-api", "pds-core-registry", "int-registry-batch-loader"]
-    image: ${REG_LOADER_IMAGE}
-    environment:
-      - ES_URL=${ES_URL}
-    volumes:
-      - ./scripts/elasticsearch-init.sh:/usr/local/bin/elasticsearch-init.sh
-      - ./scripts/aliases:/usr/local/bin/aliases
-      - ./default-config/es-auth.cfg:/etc/es-auth.cfg
-      - ./default-config/local_registry.xml:/etc/local_registry.xml
-    networks:
-      - pds
-    entrypoint: ["bash", "/usr/local/bin/elasticsearch-init.sh"]
 
   # Starts the Registry API
   registry-api:
@@ -165,8 +163,8 @@ services:
     networks:
       - pds
     depends_on:
-      registry-api:
-        condition: service_healthy
+      registry-loader-test-init:
+        condition: service_completed_successfully
 
   # Executes Registry API integration test as a Postman collection with test data
   reg-api-integration-test:
@@ -233,6 +231,24 @@ services:
       - "443:443"
     networks:
       - pds
+    depends_on:
+      registry-api:
+        condition: service_healthy
+
+
+  # Executes the Registry Loader component, which contains Harvest and Registry Manager
+  registry-loader:
+    profiles: ["pds-batch-loader"]
+    image: ${REG_LOADER_IMAGE}
+    environment:
+      - ES_URL=${ES_URL}
+    volumes:
+      - ${HARVEST_JOB_CONFIG_FILE}:/cfg/harvest-config.xml
+      - ${REG_DATA_VOLUME}:${CONTAINER_HARVEST_data-volume}
+      - ./default-config/es-auth.cfg:/etc/es-auth.cfg
+      - ./default-config/local-registry.xml:/etc/local-registry.xml
+    networks:
+      - pds
 
 volumes:
   data-volume:

docker/postman/postman_collection.json

@@ -2443,11 +2443,7 @@
               "listen": "test",
               "script": {
                 "exec": [
-                  "pm.test(\"C4438464 Status code is 200\", () => {pm.response.to.have.status(200)});",
-                  "const content = pm.response.json();",
-                  "pm.test.skip(\"C4438464 Correct hits returned\", () => {pm.expect(content.summary.hits).to.eql(5)});",
-                  "",
-                  "pm.test.skip(\"C4443902 data match q param\", () => {pm.expect(content.data[0][\"properties\"][\"pds:Time_Coordinates.pds:start_date_time\"][0]).to.eql(\"2020-07-30T12:51:34Z\")});",
+                  "pm.test(\"C4438464 Status code is 410\", () => {pm.response.to.have.status(410)});",
                   ""
                 ],
                 "type": "text/javascript",
@@ -2487,11 +2483,7 @@
               "listen": "test",
               "script": {
                 "exec": [
-                  "pm.test(\"C4438481 Status code is 200\", () => { pm.response.to.have.status(200) });",
-                  "const content = pm.response.json();",
-                  "const relevantHits = content.data.filter(hit => hit.properties.lid[0] === \"urn:nasa:pds:mars2020.spice:spice_kernels:mk_m2020\")",
-                  "pm.test.skip(\"C4438481 Correct hits returned\", () => { pm.expect(relevantHits.length).to.eql(1) });",
-                  "pm.test.skip(\"C4438481 Correct data returned\", () => {pm.expect(relevantHits[0].id).to.eql(\"urn:nasa:pds:mars2020.spice:spice_kernels:mk_m2020::3.0\")});",
+                  "pm.test(\"C4438481 Status code is 410\", () => { pm.response.to.have.status(410) });",
                   ""
                 ],
                 "type": "text/javascript",

docker/scripts/elasticsearch-init.sh

@@ -40,12 +40,6 @@ if [ -z "$ES_URL" ]; then
     exit 1
 fi
 
-echo "Waiting for Elasticsearch to launch..."  1>&2
-# TODO Warning: Use the default username and password only for testing purposes in local setup
-while ! curl --output /dev/null --silent --head --fail "$ES_URL" -u 'admin:admin' --insecure; do
-  sleep 1
-done
-
 echo "Creating registry and data dictionary indices..." 1>&2
 registry-manager create-registry -registry file:///etc/local_registry.xml  -auth /etc/es-auth.cfg