Remove publication & fix health check (#213)

* Remove publication

Signed-off-by: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com>

* Update cli.sh

Signed-off-by: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com>

* Update docker-compose.yml

Signed-off-by: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com>

* Update docker-compose.yml

Signed-off-by: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com>

* update healthcheck for dataflow-gen and dataflow-gen-worker

---------

Signed-off-by: Peter Hoffmann <954078+p-hoffmann@users.noreply.github.com>
Co-authored-by: mwaiyee <waiyee.man@data4life-asia.care>

Author: p-hoffmann

docker-compose.yml

@@ -717,7 +717,7 @@ services:
     command:
       - sh
       - -c
-      - "apt-get update && apt-get install -y curl && prefect server start"
+      - "prefect server start"
     container_name: &c2 ${PROJECT_NAME:-d2e}-dataflow-gen-1
     depends_on:
       alp-logto-post-init:
@@ -726,11 +726,12 @@ services:
       interval: 30s
       retries: 30
       start_period: 30s
-      test:
-        - CMD
-        - curl
-        - -f
-        - http://localhost:41120/api/health
+      test: [
+        "CMD",
+        "python",
+        "-c",
+        "import urllib.request, sys; sys.exit(0 if 200 <= urllib.request.urlopen('http://localhost:41120/api/health', timeout=5).getcode() < 300 else 1)"
+      ]
       timeout: 30s
     hostname: dataflow-gen-1
     image: prefecthq/prefect:3.0.3-python3.11
@@ -765,7 +766,7 @@ services:
     command:
       - sh
       - -c
-      - "apt-get update && apt-get install -y curl && pip install prefect[docker]==3.0.3 && prefect work-pool create 'docker-pool' --type docker; prefect worker start --pool 'docker-pool' --type docker --name prefect-docker-worker"
+      - "pip install prefect[docker]==3.0.3 && prefect work-pool create 'docker-pool' --type docker; prefect worker start --pool 'docker-pool' --type docker --name prefect-docker-worker --with-healthcheck"
     container_name: &c83 ${PROJECT_NAME:-d2e}-dataflow-gen-worker
     platform: linux/amd64
     depends_on:
@@ -779,11 +780,12 @@ services:
       interval: 30s
       retries: 30
       start_period: 30s
-      test:
-        - CMD
-        - curl
-        - -f
-        - http://${PROJECT_NAME:-d2e}-dataflow-gen-1:41120/api/health
+      test: [
+        "CMD",
+        "python",
+        "-c",
+        "import urllib.request, sys; sys.exit(0 if 200 <= urllib.request.urlopen('http://localhost:8080/health', timeout=5).getcode() < 300 else 1)"
+      ]
       timeout: 30s
     hostname: ${PROJECT_NAME:-d2e}-dataflow-gen-worker.${TLS__INTERNAL__DOMAIN:-alp.local}
     image: prefecthq/prefect:3.0.3-python3.11
@@ -801,6 +803,7 @@ services:
       D2E_MEMORY_LIMIT: ${D2E_MEMORY_LIMIT:-16G}
       D2E_SWAP_LIMIT: ${D2E_SWAP_LIMIT:-64G}
       DOCKER_REGISTRY_URL: ${DOCKER_REGISTRY_URL:-ghcr.io}
+      PREFECT_WORKER_WEBSERVER_HOST: 0.0.0.0
 
   # Needed only for future alterations of schema when version upgrades
   # alp-logto-db-post-alterations-init:

scripts/cli.sh

@@ -239,7 +239,6 @@ case $cmd in
         source "$ENVFILE"
         npx d2e patchdemodb
         database_host=${PROJECT_NAME:-d2e}-demodb
-        docker exec $database_host psql -h localhost -U postgres -d postgres -c "CREATE PUBLICATION demo_database_publication FOR TABLES IN SCHEMA demo_cdm; ALTER TABLE demo_cdm.COHORT REPLICA IDENTITY FULL; ALTER TABLE demo_cdm.COHORT_DEFINITION REPLICA IDENTITY FULL;"
         npx zx $node_modules_path/scripts/load-demodatabase.mjs -v $version -d $function_path -n "$ENVFILE" &&
         npx zx $node_modules_path/scripts/load-demodataset.mjs
         npx zx $node_modules_path/scripts/check-setupdemo-flow.mjs

scripts/load-demodatabase.mjs

@@ -174,13 +174,7 @@ var payload = JSON.stringify({
     "vocabSchemas": [
         DEMO__DB_CDM_SCHEMA
     ], 
-    "authenticationMode": "Password",
-    "publications" : [
-       { 
-            "slot": "data2evidence",
-            "publication": `${DEMO__DB_CODE}_publication`,
-       }
-    ]
+    "authenticationMode": "Password"
 })
 try { 
     var resp = await $`(curl -ks -w "status_code:%{http_code}" --location --request POST 'https://${CADDY__ALP__PUBLIC_FQDN}/trex/db/' \