Patch v0.7.1 (#181)

satish-a0 · mwaiyee · csafreen · web-flow · commit 69888953fd61 · 2025-05-23T16:45:03.000+08:00
* Add wget no proxy flag

* Remove dependency

* add duckdb extensions for offline

* Remove DOCKER_TREX_TAG_NAME env

* Streamline env DOCKER_TAG_NAME

* Revert

* Use auto generated password for demodb

* Create new folder to store csvs used for data loading into d2e dataset

* Remove commented code

* Use default password

* Update scripts/load-demodatabase.mjs

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Signed-off-by: Afreen &lt;reachafreen@hotmail.com&gt;

* Add headers for omop cdm v5.3

* Skip drop column on error

---------

Signed-off-by: Afreen &lt;reachafreen@hotmail.com&gt;
Co-authored-by: mwaiyee &lt;waiyee.man@data4life-asia.care&gt;
Co-authored-by: Afreen Sikandara &lt;afreen.sikandara@data4life-asia.care&gt;
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Co-authored-by: alicia-koh &lt;alicia.koh@data4life-asia.care&gt;
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -110,7 +110,7 @@ services:
       alp-minio-post-init:
         condition: service_completed_successfully
     hostname: ${PROJECT_NAME:-d2e}-trex.${TLS__INTERNAL__DOMAIN:-alp.local}
-    image: ${DOCKER_IMAGE_PREFIX:-ghcr.io/ohdsi/}d2e-trex:${DOCKER_TREX_TAG_NAME:-develop}
+    image: ${DOCKER_IMAGE_PREFIX:-ghcr.io/ohdsi/}d2e-trex:${DOCKER_TAG_NAME:-develop}
     pull_policy: ${TREX_PULL_POLICY:-missing}
     platform: linux/amd64
     logging:
@@ -143,7 +143,7 @@ services:
         - *x-minio
         - *x-pg
         - *x-perseus
-      REP_PG: postgres://postgres:mypass@demodb:5432/postgres
+      REP_PG: postgres://postgres:${DEMO__DB_PASSWORD}@demodb:5432/postgres
       SERVICE_ENV: |-
         {
           "_shared": {
@@ -490,6 +490,7 @@ services:
       PLUGINS_IMAGE_TAG: ${PLUGINS_IMAGE_TAG:-develop}
       PLUGINS_FLOW_CUSTOM_REPO_IMAGE_CONFIG: '${PLUGINS_FLOW_CUSTOM_REPO_IMAGE_CONFIG:-{}}' #{"current": "ghcr.io/ohdsi/d2e", "new": ""}
       PROJECT_NAME: ${PROJECT_NAME:-d2e}
+      DEMO__DB_PASSWORD: ${DEMO__DB_PASSWORD}
 
   alp-minerva-pg-mgmt-init:
     container_name: &c15 ${PROJECT_NAME:-d2e}-minerva-pg-mgmt-init-1
@@ -770,8 +771,6 @@ services:
     depends_on:
       alp-minio-post-init:
         condition: service_completed_successfully
-      trex:
-        condition: service_healthy
       alp-logto-post-init:
         condition: service_completed_successfully
       alp-dataflow-gen:
@@ -860,6 +859,8 @@ services:
       test:
         - CMD
         - wget
+        - --proxy
+        - off
         - -O
         - logto.status
         - http://localhost:3001/api/status
@@ -1048,12 +1049,26 @@ services:
     image: ohdsi/broadsea-atlasdb:2.2.0
     container_name: ${PROJECT_NAME:-d2e}-demodb
     mem_limit: "2G"
-    command:
-      - "postgres"
-      - "-c"
-      - "wal_level=logical"
+    command: >
+      sh -c '
+      set -e;
+      echo "Starting PostgreSQL in background via original docker-entrypoint.sh...";
+      /usr/local/bin/docker-entrypoint.sh postgres -c wal_level=logical &
+      PG_PID=$!;
+      echo "Waiting for PostgreSQL to become ready...";
+        until pg_isready -q -U postgres -h localhost -p 5432; do
+          echo "PostgreSQL is unavailable - sleeping for 1 second...";
+          sleep 1;
+        done;
+      echo "PostgreSQL is ready on localhost:5432.";
+      echo "Attempting to change password for user postgres...";
+      psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d postgres -c "ALTER USER postgres WITH PASSWORD '\''${DEMO__DB_PASSWORD}'\'';";
+      echo "PostgreSQL setup complete. Bringing PostgreSQL process (PID $PG_PID) to foreground...";
+      wait $PG_PID;
+      echo "PostgreSQL process (PID $PG_PID) has exited.";
+      exit $?;'
     environment:
-      - POSTGRES_PASSWORD=mypass
+      POSTGRES_PASSWORD: ${DEMO__DB_PASSWORD}
     networks:
       alp:
         priority: 20
@@ -1067,6 +1082,7 @@ services:
       interval: 5s
       timeout: 5s
       retries: 10
+    restart: ${DOCKER__RESTART_POLICY:-unless-stopped}
     profiles: [demodb]
 
   redis:
@@ -1103,6 +1119,7 @@ services:
       test:
         - CMD
         - wget
+        - --no-proxy
         - -O
         - fhir.status
         - http://localhost:8103/healthcheck
diff --git a/flows/base/Dockerfile b/flows/base/Dockerfile
@@ -144,6 +144,10 @@ RUN chown -R docker:docker /app/synpuf1k
 RUN mkdir -p /app/vocab
 RUN chown -R docker:docker /app/vocab
 
+# Create folder to store data load CSVs
+RUN mkdir -p /app/data_load
+RUN chown -R docker:docker /app/data_load
+
 # Create folder to store duckdb extensions for offline plugins
 # version must match duckdb in requirements.txt
 ENV duckdb_version 1.2.1
diff --git a/flows/base/data_load_plugin/flow.py b/flows/base/data_load_plugin/flow.py
@@ -112,7 +112,12 @@ def format_vocab_synpuf_data(dbdao, data: pd.DataFrame, table_name: str, logger)
             t = ibis.memtable(data, name=table_name)
             
             if table_name in columns_to_drop.keys():
-                t = t.drop(columns_to_drop.get(table_name))
+                for col in columns_to_drop.get(table_name):
+                    try:
+                        t = t.drop(col)
+                    except Exception as e:
+                        logger.error(f"Skipping dropping of column '{col}' from table '{table_name}' with error: {e}")
+                        continue
             
             # rename columns to lower case
             renamed = t.rename({col.lower(): col for col in t.columns})
diff --git a/internal/docs/data-load/headers-synpuf1k/002_LOCATION.csv b/internal/docs/data-load/headers-synpuf1k/002_LOCATION.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/008_CONDITION_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/008_CONDITION_OCCURRENCE.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/014_OBSERVATION.csv b/internal/docs/data-load/headers-synpuf1k/014_OBSERVATION.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/017_PROCEDURE_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/017_PROCEDURE_OCCURRENCE.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/README.md b/internal/docs/data-load/headers-synpuf1k/README.md
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/002_LOCATION.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/002_LOCATION.csv
@@ -0,0 +1 @@
+LOCATION_ID,ADDRESS_1,ADDRESS_2,CITY,STATE,ZIP,COUNTY,LOCATION_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/003_CARE_SITE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/003_CARE_SITE.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/004_PROVIDER.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/004_PROVIDER.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/005_COST.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/005_COST.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/006_PERSON.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/006_PERSON.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/007_DEATH.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/007_DEATH.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/008_CONDITION_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/008_CONDITION_OCCURRENCE.csv
@@ -0,0 +1 @@
+CONDITION_OCCURRENCE_ID,PERSON_ID,CONDITION_CONCEPT_ID,CONDITION_START_DATE,CONDITION_START_DATETIME,CONDITION_END_DATE,CONDITION_END_DATETIME,CONDITION_TYPE_CONCEPT_ID,STOP_REASON,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,CONDITION_SOURCE_VALUE,CONDITION_SOURCE_CONCEPT_ID,CONDITION_STATUS_SOURCE_VALUE,CONDITION_STATUS_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/009_CONDITION_ERA.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/009_CONDITION_ERA.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/010_DEVICE_EXPOSURE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/010_DEVICE_EXPOSURE.csv
@@ -1 +1 @@
-DEVICE_EXPOSURE_ID,PERSON_ID,DEVICE_CONCEPT_ID,DEVICE_EXPOSURE_START_DATE,DEVICE_EXPOSURE_START_DATETIME,DEVICE_EXPOSURE_END_DATE,DEVICE_EXPOSURE_END_DATETIME,DEVICE_TYPE_CONCEPT_ID,UNIQUE_DEVICE_ID,PRODUCTION_ID,QUANTITY,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,DEVICE_SOURCE_VALUE,DEVICE_SOURCE_CONCEPT_ID,UNIT_CONCEPT_ID,UNIT_SOURCE_VALUE,UNIT_SOURCE_CONCEPT_ID
+DEVICE_EXPOSURE_ID,PERSON_ID,DEVICE_CONCEPT_ID,DEVICE_EXPOSURE_START_DATE,DEVICE_EXPOSURE_START_DATETIME,DEVICE_EXPOSURE_END_DATE,DEVICE_EXPOSURE_END_DATETIME,DEVICE_TYPE_CONCEPT_ID,UNIQUE_DEVICE_ID,QUANTITY,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,DEVICE_SOURCE_VALUE,DEVICE_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/011_DRUG_EXPOSURE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/011_DRUG_EXPOSURE.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/012_DRUG_ERA.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/012_DRUG_ERA.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/013_MEASUREMENT.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/013_MEASUREMENT.csv
@@ -1 +1 @@
-MEASUREMENT_ID,PERSON_ID,MEASUREMENT_CONCEPT_ID,MEASUREMENT_DATE,MEASUREMENT_DATETIME,MEASUREMENT_TIME,MEASUREMENT_TYPE_CONCEPT_ID,OPERATOR_CONCEPT_ID,VALUE_AS_NUMBER,VALUE_AS_CONCEPT_ID,UNIT_CONCEPT_ID,RANGE_LOW,RANGE_HIGH,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,MEASUREMENT_SOURCE_VALUE,MEASUREMENT_SOURCE_CONCEPT_ID,UNIT_SOURCE_VALUE,VALUE_SOURCE_VALUE,MEASUREMENT_EVENT_ID,MEAS_EVENT_FIELD_CONCEPT_ID,UNIT_SOURCE_CONCEPT_ID
+MEASUREMENT_ID,PERSON_ID,MEASUREMENT_CONCEPT_ID,MEASUREMENT_DATE,MEASUREMENT_DATETIME,MEASUREMENT_TIME,MEASUREMENT_TYPE_CONCEPT_ID,OPERATOR_CONCEPT_ID,VALUE_AS_NUMBER,VALUE_AS_CONCEPT_ID,UNIT_CONCEPT_ID,RANGE_LOW,RANGE_HIGH,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,MEASUREMENT_SOURCE_VALUE,MEASUREMENT_SOURCE_CONCEPT_ID,UNIT_SOURCE_VALUE,VALUE_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/014_OBSERVATION.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/014_OBSERVATION.csv
@@ -0,0 +1 @@
+OBSERVATION_ID,PERSON_ID,OBSERVATION_CONCEPT_ID,OBSERVATION_DATE,OBSERVATION_DATETIME,OBSERVATION_TYPE_CONCEPT_ID,OBSERVATION_EVENT_ID,VALUE_AS_NUMBER,VALUE_AS_STRING,VALUE_AS_CONCEPT_ID,VALUE_AS_DATETIME,QUALIFIER_CONCEPT_ID,UNIT_CONCEPT_ID,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,OBSERVATION_SOURCE_VALUE,OBSERVATION_SOURCE_CONCEPT_ID,UNIT_SOURCE_VALUE,QUALIFIER_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/015_OBSERVATION_PERIOD.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/015_OBSERVATION_PERIOD.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/016_PAYER_PLAN_PERIOD.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/016_PAYER_PLAN_PERIOD.csv
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/017_PROCEDURE_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/017_PROCEDURE_OCCURRENCE.csv
@@ -0,0 +1 @@
+PROCEDURE_OCCURRENCE_ID,PERSON_ID,PROCEDURE_CONCEPT_ID,PROCEDURE_DATE,PROCEDURE_DATETIME,PROCEDURE_TYPE_CONCEPT_ID,MODIFIER_CONCEPT_ID,QUANTITY,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,PROCEDURE_SOURCE_VALUE,PROCEDURE_SOURCE_CONCEPT_ID,MODIFIER_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/018_VISIT_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.3/018_VISIT_OCCURRENCE.csv
@@ -1 +1 @@
-VISIT_OCCURRENCE_ID,PERSON_ID,VISIT_CONCEPT_ID,VISIT_START_DATE,VISIT_START_DATETIME,VISIT_END_DATE,VISIT_END_DATETIME,VISIT_TYPE_CONCEPT_ID,PROVIDER_ID,CARE_SITE_ID,VISIT_SOURCE_VALUE,VISIT_SOURCE_CONCEPT_ID,ADMITTED_FROM_CONCEPT_ID,ADMITTED_FROM_SOURCE_VALUE,DISCHARGED_TO_CONCEPT_ID,DISCHARGED_TO_SOURCE_VALUE,PRECEDING_VISIT_OCCURRENCE_ID
+VISIT_OCCURRENCE_ID,PERSON_ID,VISIT_CONCEPT_ID,VISIT_START_DATE,VISIT_START_DATETIME,VISIT_END_DATE,VISIT_END_DATETIME,VISIT_TYPE_CONCEPT_ID,PROVIDER_ID,CARE_SITE_ID,VISIT_SOURCE_VALUE,VISIT_SOURCE_CONCEPT_ID,ADMITTING_SOURCE_CONCEPT_ID,ADMITTING_SOURCE_VALUE,DISCHARGE_TO_CONCEPT_ID,DISCHARGE_TO_SOURCE_VALUE,PRECEDING_VISIT_OCCURRENCE_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.3/README.md b/internal/docs/data-load/headers-synpuf1k/v5.3/README.md
@@ -0,0 +1,3 @@
+# synpuf1k headers 
+
+- header files for synpuf1k v5.3 downloaded from public source
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/002_LOCATION.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/002_LOCATION.csv
@@ -0,0 +1 @@
+LOCATION_ID,ADDRESS_1,ADDRESS_2,CITY,STATE,ZIP,COUNTY,LOCATION_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/003_CARE_SITE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/003_CARE_SITE.csv
@@ -0,0 +1 @@
+CARE_SITE_ID,CARE_SITE_NAME,PLACE_OF_SERVICE_CONCEPT_ID,LOCATION_ID,CARE_SITE_SOURCE_VALUE,PLACE_OF_SERVICE_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/004_PROVIDER.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/004_PROVIDER.csv
@@ -0,0 +1 @@
+PROVIDER_ID,PROVIDER_NAME,NPI,DEA,SPECIALTY_CONCEPT_ID,CARE_SITE_ID,YEAR_OF_BIRTH,GENDER_CONCEPT_ID,PROVIDER_SOURCE_VALUE,SPECIALTY_SOURCE_VALUE,SPECIALTY_SOURCE_CONCEPT_ID,GENDER_SOURCE_VALUE,GENDER_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/005_COST.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/005_COST.csv
@@ -0,0 +1 @@
+COST_ID,COST_EVENT_ID,COST_DOMAIN_ID,COST_TYPE_CONCEPT_ID,CURRENCY_CONCEPT_ID,TOTAL_CHARGE,TOTAL_COST,TOTAL_PAID,PAID_BY_PAYER,PAID_BY_PATIENT,PAID_PATIENT_COPAY,PAID_PATIENT_COINSURANCE,PAID_PATIENT_DEDUCTIBLE,PAID_BY_PRIMARY,PAID_INGREDIENT_COST,PAID_DISPENSING_FEE,PAYER_PLAN_PERIOD_ID,AMOUNT_ALLOWED,REVENUE_CODE_CONCEPT_ID,REVENUE_CODE_SOURCE_VALUE,DRG_CONCEPT_ID,DRG_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/006_PERSON.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/006_PERSON.csv
@@ -0,0 +1 @@
+PERSON_ID,GENDER_CONCEPT_ID,YEAR_OF_BIRTH,MONTH_OF_BIRTH,DAY_OF_BIRTH,BIRTH_DATETIME,RACE_CONCEPT_ID,ETHNICITY_CONCEPT_ID,LOCATION_ID,PROVIDER_ID,CARE_SITE_ID,PERSON_SOURCE_VALUE,GENDER_SOURCE_VALUE,GENDER_SOURCE_CONCEPT_ID,RACE_SOURCE_VALUE,RACE_SOURCE_CONCEPT_ID,ETHNICITY_SOURCE_VALUE,ETHNICITY_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/007_DEATH.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/007_DEATH.csv
@@ -0,0 +1 @@
+PERSON_ID,DEATH_DATE,DEATH_DATETIME,DEATH_TYPE_CONCEPT_ID,CAUSE_CONCEPT_ID,CAUSE_SOURCE_VALUE,CAUSE_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/008_CONDITION_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/008_CONDITION_OCCURRENCE.csv
@@ -0,0 +1 @@
+CONDITION_OCCURRENCE_ID,PERSON_ID,CONDITION_CONCEPT_ID,CONDITION_START_DATE,CONDITION_START_DATETIME,CONDITION_END_DATE,CONDITION_END_DATETIME,CONDITION_TYPE_CONCEPT_ID,STOP_REASON,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,CONDITION_SOURCE_VALUE,CONDITION_SOURCE_CONCEPT_ID,CONDITION_STATUS_SOURCE_VALUE,CONDITION_STATUS_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/009_CONDITION_ERA.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/009_CONDITION_ERA.csv
@@ -0,0 +1 @@
+CONDITION_ERA_ID,PERSON_ID,CONDITION_CONCEPT_ID,CONDITION_ERA_START_DATE,CONDITION_ERA_END_DATE,CONDITION_OCCURRENCE_COUNT
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/010_DEVICE_EXPOSURE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/010_DEVICE_EXPOSURE.csv
@@ -0,0 +1 @@
+DEVICE_EXPOSURE_ID,PERSON_ID,DEVICE_CONCEPT_ID,DEVICE_EXPOSURE_START_DATE,DEVICE_EXPOSURE_START_DATETIME,DEVICE_EXPOSURE_END_DATE,DEVICE_EXPOSURE_END_DATETIME,DEVICE_TYPE_CONCEPT_ID,UNIQUE_DEVICE_ID,QUANTITY,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,DEVICE_SOURCE_VALUE,DEVICE_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/011_DRUG_EXPOSURE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/011_DRUG_EXPOSURE.csv
@@ -0,0 +1 @@
+DRUG_EXPOSURE_ID,PERSON_ID,DRUG_CONCEPT_ID,DRUG_EXPOSURE_START_DATE,DRUG_EXPOSURE_START_DATETIME,DRUG_EXPOSURE_END_DATE,DRUG_EXPOSURE_END_DATETIME,VERBATIM_END_DATE,DRUG_TYPE_CONCEPT_ID,STOP_REASON,REFILLS,QUANTITY,DAYS_SUPPLY,SIG,ROUTE_CONCEPT_ID,LOT_NUMBER,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,DRUG_SOURCE_VALUE,DRUG_SOURCE_CONCEPT_ID,ROUTE_SOURCE_VALUE,DOSE_UNIT_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/012_DRUG_ERA.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/012_DRUG_ERA.csv
@@ -0,0 +1 @@
+DRUG_ERA_ID,PERSON_ID,DRUG_CONCEPT_ID,DRUG_ERA_START_DATE,DRUG_ERA_END_DATE,DRUG_EXPOSURE_COUNT,GAP_DAYS
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/013_MEASUREMENT.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/013_MEASUREMENT.csv
@@ -0,0 +1 @@
+MEASUREMENT_ID,PERSON_ID,MEASUREMENT_CONCEPT_ID,MEASUREMENT_DATE,MEASUREMENT_DATETIME,MEASUREMENT_TIME,MEASUREMENT_TYPE_CONCEPT_ID,OPERATOR_CONCEPT_ID,VALUE_AS_NUMBER,VALUE_AS_CONCEPT_ID,UNIT_CONCEPT_ID,RANGE_LOW,RANGE_HIGH,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,MEASUREMENT_SOURCE_VALUE,MEASUREMENT_SOURCE_CONCEPT_ID,UNIT_SOURCE_VALUE,VALUE_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/014_OBSERVATION.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/014_OBSERVATION.csv
@@ -0,0 +1 @@
+OBSERVATION_ID,PERSON_ID,OBSERVATION_CONCEPT_ID,OBSERVATION_DATE,OBSERVATION_DATETIME,OBSERVATION_TYPE_CONCEPT_ID,OBSERVATION_EVENT_ID,VALUE_AS_NUMBER,VALUE_AS_STRING,VALUE_AS_CONCEPT_ID,VALUE_AS_DATETIME,QUALIFIER_CONCEPT_ID,UNIT_CONCEPT_ID,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,OBSERVATION_SOURCE_VALUE,OBSERVATION_SOURCE_CONCEPT_ID,UNIT_SOURCE_VALUE,QUALIFIER_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/015_OBSERVATION_PERIOD.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/015_OBSERVATION_PERIOD.csv
@@ -0,0 +1 @@
+OBSERVATION_PERIOD_ID,PERSON_ID,OBSERVATION_PERIOD_START_DATE,OBSERVATION_PERIOD_END_DATE,PERIOD_TYPE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/016_PAYER_PLAN_PERIOD.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/016_PAYER_PLAN_PERIOD.csv
@@ -0,0 +1 @@
+PAYER_PLAN_PERIOD_ID,PERSON_ID,PAYER_PLAN_PERIOD_START_DATE,PAYER_PLAN_PERIOD_END_DATE,PAYER_CONCEPT_ID,PAYER_SOURCE_VALUE,PAYER_SOURCE_CONCEPT_ID,PLAN_CONCEPT_ID,PLAN_SOURCE_VALUE,PLAN_SOURCE_CONCEPT_ID,SPONSOR_CONCEPT_ID,SPONSOR_SOURCE_VALUE,SPONSOR_SOURCE_CONCEPT_ID,FAMILY_SOURCE_VALUE,STOP_REASON_CONCEPT_ID,STOP_REASON_SOURCE_VALUE,STOP_REASON_SOURCE_CONCEPT_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/017_PROCEDURE_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/017_PROCEDURE_OCCURRENCE.csv
@@ -0,0 +1 @@
+PROCEDURE_OCCURRENCE_ID,PERSON_ID,PROCEDURE_CONCEPT_ID,PROCEDURE_DATE,PROCEDURE_DATETIME,PROCEDURE_TYPE_CONCEPT_ID,MODIFIER_CONCEPT_ID,QUANTITY,PROVIDER_ID,VISIT_OCCURRENCE_ID,VISIT_DETAIL_ID,PROCEDURE_SOURCE_VALUE,PROCEDURE_SOURCE_CONCEPT_ID,MODIFIER_SOURCE_VALUE
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/018_VISIT_OCCURRENCE.csv b/internal/docs/data-load/headers-synpuf1k/v5.4/018_VISIT_OCCURRENCE.csv
@@ -0,0 +1 @@
+VISIT_OCCURRENCE_ID,PERSON_ID,VISIT_CONCEPT_ID,VISIT_START_DATE,VISIT_START_DATETIME,VISIT_END_DATE,VISIT_END_DATETIME,VISIT_TYPE_CONCEPT_ID,PROVIDER_ID,CARE_SITE_ID,VISIT_SOURCE_VALUE,VISIT_SOURCE_CONCEPT_ID,ADMITTING_SOURCE_CONCEPT_ID,ADMITTING_SOURCE_VALUE,DISCHARGE_TO_CONCEPT_ID,DISCHARGE_TO_SOURCE_VALUE,PRECEDING_VISIT_OCCURRENCE_ID
diff --git a/internal/docs/data-load/headers-synpuf1k/v5.4/README.md b/internal/docs/data-load/headers-synpuf1k/v5.4/README.md
@@ -0,0 +1,3 @@
+# synpuf1k headers 
+
+- header files for synpuf1k v5.3 downloaded from public source
diff --git a/internal/docs/env-vars.yml b/internal/docs/env-vars.yml
@@ -60,8 +60,6 @@ DOCKER_TAG_NAME:
   comment: default tag
   publish: true
   type: string
-DOCKER_TREX_TAG_NAME:
-  publish: true
 DOCKER__RESTART_POLICY:
   comment: auto restart on issue. logging debug simpler if off
   publish: false
diff --git a/package.json b/package.json
@@ -21,7 +21,7 @@
     "init": "ENV_TYPE=remote npx d2e -v develop -n .env.remote init",
     "init:minerva": "PROJECT_NAME=alp ENV_TYPE=local npx d2e -v develop -n .env.local init",
     "start": "ENV_TYPE=remote npx d2e -v develop -e -n .env.remote -p 3000 start",
-    "start:ci": "DOCKER_TREX_TAG_NAME=develop npm run start:minerva",
+    "start:ci": "npm run start:minerva",
     "remote:minerva": "ENV_TYPE=remote npx d2e -v develop  -e -n .env.remote -f -j",
     "local:minerva": "PROJECT_NAME=alp ENV_TYPE=local CADDY__CONFIG=./deploy/caddy-config npx d2e -v develop -e -n .env.local -f -j -d ./functions -p 41100",
     "build:minerva": "npm run local:minerva -- build",
diff --git a/scripts/cli.sh b/scripts/cli.sh
@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 set -o errexit
 
-version=0.7.0 #default version
+version=0.7.0 #default/base version
+LATEST_DOCKER_TAG_NAME=0.7.1-beta
+
 
 cmd=""
 script_full_path=$(dirname "$0")
@@ -83,16 +85,14 @@ export ENVFILE=$env
 if [[ $version = "develop" ]]; then
   export PLUGINS_API_VERSION=${PLUGINS_API_VERSION:-latest}
   export DOCKER_TAG_NAME=${DOCKER_TAG_NAME:-develop}
-  export DOCKER_TREX_TAG_NAME=${DOCKER_TREX_TAG_NAME:-develop}
-  export PLUGINS_IMAGE_TAG=${PLUGINS_IMAGE_TAG:-develop}
   #export DOCKER_IMAGE_PREFIX=ghcr.io/ohdsi/
+  export PLUGINS_IMAGE_TAG=${PLUGINS_IMAGE_TAG:-develop}
   export PLUGINS_REGISTRY=${PLUGINS_REGISTRY:-https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/d2e/npm/registry/}
   DOCKER_LOG_LEVEL=INFO
 else
   export PLUGINS_API_VERSION=${PLUGINS_API_VERSION:-~$version}
-  export DOCKER_TAG_NAME=${DOCKER_TAG_NAME:-$version-beta}
-  export DOCKER_TREX_TAG_NAME=${DOCKER_TREX_TAG_NAME:-$version-beta}
-  export PLUGINS_IMAGE_TAG=${PLUGINS_IMAGE_TAG:-$version-beta}
+  export DOCKER_TAG_NAME=${DOCKER_TAG_NAME:-${LATEST_DOCKER_TAG_NAME}}
+  export PLUGINS_IMAGE_TAG=${PLUGINS_IMAGE_TAG:-${LATEST_DOCKER_TAG_NAME}}
   export PLUGINS_REGISTRY=${PLUGINS_REGISTRY:-https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/stable/npm/registry/}
 fi
 
@@ -118,7 +118,7 @@ case $cmd in
         ;;
     build)
         cmd="$dockerbasecmd build"
-        if [ -n "$services" ]; then
+        if [ -n "$services" ]; then      
             cmd="$cmd $services"
         fi
         echo . $cmd
@@ -166,8 +166,6 @@ case $cmd in
         ;;
     init)
         CADDY__ALP__PUBLIC_FQDN=${CADDY__ALP__PUBLIC_FQDN:-localhost}
-        DOCKER_TAG_NAME=${DOCKER_TAG_NAME:-develop}
-        DOCKER_TREX_TAG_NAME=${DOCKER_TREX_TAG_NAME:-develop}
         ENV_TYPE=${ENV_TYPE:-local}
         TLS__CADDY_DIRECTIVE=${TLS__CADDY_DIRECTIVE:-tls internal}
         PROJECT_NAME=${PROJECT_NAME:-d2e}
@@ -177,7 +175,6 @@ case $cmd in
         echo . INPUTS TLS__CADDY_DIRECTIVE=\"$TLS__CADDY_DIRECTIVE\" DOTENV_FILE=$DOTENV_FILE
 
         # vars
-        [ $ENV_TYPE = local ] && [ -z DOCKER_TAG_NAME ] && DOCKER_TAG_NAME=local
 
         source $node_modules_path/scripts/lib.sh # functions here
 
@@ -191,7 +188,6 @@ case $cmd in
         echo -n '' > $DOTENV_FILE
         echo CADDY__ALP__PUBLIC_FQDN=$CADDY__ALP__PUBLIC_FQDN >> $DOTENV_FILE
         echo DOCKER_TAG_NAME=$DOCKER_TAG_NAME >> $DOTENV_FILE
-        echo DOCKER_TREX_TAG_NAME=$DOCKER_TREX_TAG_NAME >> $DOTENV_FILE
         echo ENV_TYPE=$ENV_TYPE >> $DOTENV_FILE
         echo FHIR__CLIENT_ID=$(random-uuid) >> $DOTENV_FILE
         echo FHIR__CLIENT_SECRET=$(random-password 64) >> $DOTENV_FILE
@@ -207,6 +203,7 @@ case $cmd in
         echo PG_ADMIN_PASSWORD=$(random-password $DEFAULT_PASSWORD_LENGTH) >> $DOTENV_FILE
         echo PG_SUPER_PASSWORD=$(random-password $DEFAULT_PASSWORD_LENGTH) >> $DOTENV_FILE
         echo PG_WRITE_PASSWORD=$(random-password $DEFAULT_PASSWORD_LENGTH) >> $DOTENV_FILE
+        echo DEMO__DB_PASSWORD=$(random-password $DEFAULT_PASSWORD_LENGTH) >> $DOTENV_FILE
         echo REDIS_PASSWORD=$(random-uuid) >> $DOTENV_FILE
         echo DICOM__HEALTH_CHECK_PASSWORD=$(random-password $DEFAULT_PASSWORD_LENGTH) >> $DOTENV_FILE
         echo TLS__CADDY_DIRECTIVE=\'"$TLS__CADDY_DIRECTIVE"\' >> $DOTENV_FILE
@@ -231,7 +228,7 @@ case $cmd in
         wc -l $DOTENV_FILE $DOTENV_KEYS | sed '$d'
         ;;
     pull)
-        cmd="docker pull --platform linux/amd64 ${DOCKER_IMAGE_PREFIX:-ghcr.io/ohdsi/}d2e/flow-base:${DOCKER_TAG_NAME:-develop}" # not part of dc.yml
+        cmd="docker pull --platform linux/amd64 ${DOCKER_IMAGE_PREFIX:-ghcr.io/ohdsi/}d2e/flow-base:${PLUGINS_IMAGE_TAG}" # not part of dc.yml
         echo . $cmd
         $cmd
         cmd="$dockerbasecmd pull"
diff --git a/scripts/load-demodatabase.mjs b/scripts/load-demodatabase.mjs
@@ -44,7 +44,7 @@ let database_host = `${project_name}-demodb`; //PostgreSQL container name /or/ e
 let DEMO__DB_CODE = 'demo_database'; //display name
 let DEMO__DB_CDM_SCHEMA = 'demo_cdm';
 let DEMO__DB_USER = 'postgres';
-let DEMO__DB_PASSWORD = 'mypass';
+let DEMO__DB_PASSWORD = process.env.PROJECT_NAME? process.env.DEMO__DB_PASSWORD : 'mypass';
 var db_extra = {"max": 50, "schema":DEMO__DB_CDM_SCHEMA, "queryTimeout":60000,"statementTimeout":60000,"idleTimeoutMillis":300000,"connectionTimeoutMillis":60000,"idleInTransactionSessionTimeout":300000};
 
 const public_key = process.env.DB_CREDENTIALS__INTERNAL__PUBLIC_KEY;
diff --git a/services/trex/Dockerfile b/services/trex/Dockerfile
@@ -31,6 +31,23 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \
     ln -sf /usr/src/node_modules/bun-linux-x64-baseline/bin/bun /usr/src/node_modules/.bin/bun; \
     fi
 
+# Create folder to store duckdb extensions for offline plugins
+ENV duckdb_version=1.2.1
+RUN mkdir -p /root/.duckdb/extensions/v${duckdb_version}/linux_amd64
+WORKDIR /root/.duckdb/extensions/v${duckdb_version}/linux_amd64
+RUN apt-get update && apt-get install -y wget 
+RUN for lib in \
+    autocomplete avro aws azure delta excel fts httpfs iceberg icu inet json \
+    mysql_scanner parquet postgres_scanner spatial sqlite sqlite_scanner tpcds tpch ui vss; do \
+    wget http://extensions.duckdb.org/v${duckdb_version}/linux_amd64/${lib}.duckdb_extension.gz && \
+    gzip -d ${lib}.duckdb_extension.gz; \
+    done
+RUN for lib in \
+    bigquery; do \ 
+    wget http://community-extensions.duckdb.org/v${duckdb_version}/linux_amd64/${lib}.duckdb_extension.gz && \
+    gzip -d ${lib}.duckdb_extension.gz; \
+    done
 
+WORKDIR /usr/src
 CMD ["npm", "run", "start"]
 EXPOSE 33000

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+LOCATION_ID,ADDRESS_1,ADDRESS_2,CITY,STATE,ZIP,COUNTY,LOCATION_SOURCE_VALUE`