bento-platform · v-rocheleau · Oct 20, 2025 · Oct 20, 2025 · Oct 20, 2025 · Oct 21, 2025
diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml
@@ -9,9 +9,9 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v6
         name: Set up Python
         with:
           python-version: "3.10"

diff --git a/.github/workflows/test-actions.yml b/.github/workflows/test-actions.yml
@@ -9,9 +9,9 @@ jobs:
   test_dev_init:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v6
         name: Set up Python
         with:
           python-version: "3.10"

diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml
@@ -47,6 +47,11 @@ services:
           - ${BENTOV2_PORTAL_DOMAIN}
           - ${BENTOV2_AUTH_DOMAIN}
           - ${BENTO_MINIO_DOMAIN}
+      etl-net:
+        aliases:
+          - ${BENTOV2_DOMAIN}
+          - ${BENTOV2_PORTAL_DOMAIN}
+          - ${BENTOV2_AUTH_DOMAIN}
       event-relay-net:
         aliases:
           - ${BENTOV2_DOMAIN}
@@ -172,6 +177,16 @@ services:
       - "${BENTOV2_WES_EXTERNAL_PORT}:${BENTOV2_WES_INTERNAL_PORT}"
       - "${BENTOV2_WES_DEBUGGER_EXTERNAL_PORT}:${BENTOV2_WES_DEBUGGER_INTERNAL_PORT}"
 
+  etl:
+    environment:
+      - BENTO_DEBUG=True
+      - BENTO_VALIDATE_SSL=False
+      - PORT=${BENTO_ETL_INTERNAL_PORT}
+      - DEBUGGER_PORT=${BENTO_ETL_DEBUGGER_EXTERNAL_PORT}
+    ports:
+      - "${BENTO_ETL_EXTERNAL_PORT}:${BENTO_ETL_INTERNAL_PORT}"
+      - "${BENTO_ETL_DEBUGGER_EXTERNAL_PORT}:${BENTO_ETL_DEBUGGER_INTERNAL_PORT}"
+
   drs:
     environment:
       - FLASK_DEBUG=True

diff --git a/docker-compose.local.yaml b/docker-compose.local.yaml
@@ -79,6 +79,15 @@ services:
     volumes:
       - ./repos/wes:/wes
 
+  etl:
+    image: ${BENTO_ETL_IMAGE}:${BENTO_ETL_VERSION_DEV}
+    environment:
+        - BENTO_GIT_NAME
+        - BENTO_GIT_EMAIL
+        - BENTO_GIT_REPOSITORY_DIR=/etl
+    volumes:
+      - ./repos/etl:/etl
+
   drs:
     image: ${BENTOV2_DRS_IMAGE}:${BENTOV2_DRS_VERSION_DEV}
     environment:

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -10,6 +10,7 @@ include:
       - lib/cbioportal/docker-compose.cbioportal.yaml  # Optional feature; controlled by a compose profile
       - lib/drop-box/docker-compose.drop-box.yaml
       - lib/drs/docker-compose.drs.yaml
+      - lib/etl/docker-compose.etl.yaml
       - lib/event-relay/docker-compose.event-relay.yaml
       - lib/gohan/docker-compose.gohan.yaml  # Optional feature; controlled by a compose profile
       - lib/katsu/docker-compose.katsu.yaml

diff --git a/docs/development.md b/docs/development.md
@@ -141,7 +141,7 @@ The following is a list of all host port allocations for Bento services in devel
 | Reference DB     | 9512 | `N/A`         |
 | Service Registry | 5010 | Unimplemented |
 | WES              | 9250 | 5680          |
-
+| ETL              | 6400 | 6456          |
 
 ## Local NPM package development
 

diff --git a/docs/etl.md b/docs/etl.md
@@ -0,0 +1,69 @@
+# Bento ETL (Extract, Transform, Load)
+
+Bento ETL is a service to automate the injestion of external health data into the Bento data services.
+With customizable transformers, ETL can collect and convert a variety of data types from any data source.
+
+## Configuration
+
+Please follow the instructions below to deploy the ETL service in a Bento stack.
+
+### Environment variables
+
+Enable Bento ETL by setting the feature flag in `local.env`.
+
+```bash
+BENTO_ETL_ENABLED='true'
+```
+
+### Initialize networking and directories and generate client secret
+
+```bash
+./bentoctl.bash init-docker  # Creates the Docker network for Bento ETL
+./bentoctl.bash init-dirs  # Creates Bento ETL's data directory
+
+# Generates the client secret
+./bentoctl.bash run auth
+./bentoctl.bash run gateway
+./bentoctl.bash init-auth
+```
+After running `init-auth`, update the following in your `local.env` file!
+
+```bash
+BENTO_ETL_CLIENT_ID=etl
+BENTO_ETL_CLIENT_SECRET={your-newly-generated-secret-here!}
+```
+
+### Create additional grants
+
+As Bento ETL is responsible for the upload of data to other Bento data services, it needs some additional grants to be able to do so.
+
+First, run and shell into the authorization service:
+
+```bash
+./bentoctl.bash run authz
+./bentoctl.bash shell authz
+```
+
+Then, substituting the `<ISSUER_HERE>` field with your issuer (`iss`), run the following:
+
+```bash
+bento_authz create grant \
+  '{"iss": "<ISSUER_HERE>", "client": "etl"}' \
+  '{"everything": true}' \
+  'view:private_portal'
+
+bento_authz create grant \
+  '{"iss": "<ISSUER_HERE>", "client": "etl"}' \
+  '{"everything": true}' \
+  'query:data' 'ingest:data' 'ingest:reference_material' 'delete:reference_material'
+```
+
+## Run Bento ETL
+
+Now that everything's set up, you can start the ETL service container using:
+
+```bash
+./bentoctl.bash run etl  # Runs just the etl container
+# or
+./bentoctl.bash run  # Runs everything, including etl
+```
diff --git a/etc/bento.env b/etc/bento.env
@@ -80,7 +80,7 @@ BENTO_AUTH_DB_NETWORK="${BENTOV2_PREFIX}-auth-db-net"
 
 # - Authz service
 BENTO_AUTHZ_IMAGE=ghcr.io/bento-platform/bento_authorization_service
-BENTO_AUTHZ_VERSION=0.11.0
+BENTO_AUTHZ_VERSION=edge
 BENTO_AUTHZ_VERSION_DEV=${BENTO_AUTHZ_VERSION}-dev
 BENTO_AUTHZ_CONTAINER_NAME=${BENTOV2_PREFIX}-authz
 BENTO_AUTHZ_NETWORK=${BENTOV2_PREFIX}-authz-net
@@ -100,7 +100,7 @@ BENTO_AUTHZ_DB_MEM_LIM=1G
 # Web
 BENTO_WEB_CUSTOM_HEADER=
 BENTOV2_WEB_IMAGE=ghcr.io/bento-platform/bento_web
-BENTOV2_WEB_VERSION=7.1.0
+BENTOV2_WEB_VERSION=edge
 BENTOV2_WEB_VERSION_DEV=${BENTOV2_WEB_VERSION}-dev
 BENTOV2_WEB_CONTAINER_NAME=${BENTOV2_PREFIX}-web
 BENTO_WEB_NETWORK=${BENTOV2_PREFIX}-web-net
@@ -112,7 +112,7 @@ BENTOV2_WEB_CPUS=2
 
 # Drop-Box
 BENTOV2_DROP_BOX_IMAGE=ghcr.io/bento-platform/bento_drop_box_service
-BENTOV2_DROP_BOX_VERSION=2.0.1
+BENTOV2_DROP_BOX_VERSION=edge
 BENTOV2_DROP_BOX_VERSION_DEV=${BENTOV2_DROP_BOX_VERSION}-dev
 BENTOV2_DROP_BOX_CONTAINER_NAME=${BENTOV2_PREFIX}-drop-box
 BENTO_DROP_BOX_NETWORK=${BENTOV2_PREFIX}-drop-box-net
@@ -158,7 +158,7 @@ BENTOV2_NOTIFICATION_CPUS=2
 
 # Aggregation
 BENTOV2_AGGREGATION_IMAGE=ghcr.io/bento-platform/bento_aggregation_service
-BENTOV2_AGGREGATION_VERSION=0.20.3
+BENTOV2_AGGREGATION_VERSION=edge
 BENTOV2_AGGREGATION_VERSION_DEV=${BENTOV2_AGGREGATION_VERSION}-dev
 BENTOV2_AGGREGATION_CONTAINER_NAME=${BENTOV2_PREFIX}-aggregation
 BENTO_AGGREGATION_NETWORK=${BENTOV2_PREFIX}-aggregation-net
@@ -187,7 +187,7 @@ BENTOV2_EVENT_RELAY_CPUS=1
 # Reference
 #  - Service
 BENTO_REFERENCE_IMAGE=ghcr.io/bento-platform/bento_reference_service
-BENTO_REFERENCE_VERSION=0.5.1
+BENTO_REFERENCE_VERSION=edge
 BENTO_REFERENCE_VERSION_DEV=${BENTO_REFERENCE_VERSION}-dev
 BENTO_REFERENCE_CONTAINER_NAME=${BENTOV2_PREFIX}-reference
 BENTO_REFERENCE_NETWORK=${BENTOV2_PREFIX}-reference-net
@@ -209,7 +209,7 @@ BENTO_REFERENCE_DB_USER="reference_user"
 
 # WES
 BENTOV2_WES_IMAGE=ghcr.io/bento-platform/bento_wes
-BENTOV2_WES_VERSION=0.15.1
+BENTOV2_WES_VERSION=edge
 BENTOV2_WES_VERSION_DEV=${BENTOV2_WES_VERSION}-dev
 BENTOV2_WES_CONTAINER_NAME=${BENTOV2_PREFIX}-wes
 BENTO_WES_NETWORK=${BENTOV2_PREFIX}-wes-net
@@ -233,7 +233,7 @@ BENTOV2_WES_WORKFLOW_TIMEOUT=172800
 
 # DRS
 BENTOV2_DRS_IMAGE=ghcr.io/bento-platform/bento_drs
-BENTOV2_DRS_VERSION=0.20.0
+BENTOV2_DRS_VERSION=edge
 BENTOV2_DRS_VERSION_DEV=${BENTOV2_DRS_VERSION}-dev
 BENTOV2_DRS_CONTAINER_NAME=${BENTOV2_PREFIX}-drs
 BENTO_DRS_NETWORK=${BENTOV2_PREFIX}-drs-net
@@ -278,7 +278,7 @@ BENTOV2_KATSU_DB_CPUS=4
 
 # Katsu
 BENTOV2_KATSU_IMAGE=ghcr.io/bento-platform/katsu
-BENTOV2_KATSU_VERSION=11.0.0
+BENTOV2_KATSU_VERSION=edge
 BENTOV2_KATSU_VERSION_DEV=${BENTOV2_KATSU_VERSION}-dev
 BENTOV2_KATSU_CONTAINER_NAME=${BENTOV2_PREFIX}-katsu
 BENTO_KATSU_NETWORK=${BENTOV2_PREFIX}-katsu-net
@@ -381,7 +381,7 @@ BENTOV2_GOHAN_PRIVATE_AUTHZ_URL=http://${BENTOV2_GOHAN_AUTHZ_OPA_CONTAINER_NAME}
 # Bento-Public
 
 BENTO_PUBLIC_IMAGE=ghcr.io/bento-platform/bento_public
-BENTO_PUBLIC_VERSION=0.24.0
+BENTO_PUBLIC_VERSION=pr-285
 BENTO_PUBLIC_VERSION_DEV=${BENTO_PUBLIC_VERSION}-dev
 BENTO_PUBLIC_CONTAINER_NAME=${BENTOV2_PREFIX}-public
 BENTO_PUBLIC_NETWORK=${BENTOV2_PREFIX}-public-net
@@ -476,3 +476,18 @@ BENTO_MINIO_NETWORK=${BENTOV2_PREFIX}-minio-net
 BENTO_MINIO_ROOT_USER=root
 BENTO_MINIO_INTERNAL_PORT=9000
 BENTO_MINIO_CONSOLE_PORT=9001
+
+# ETL
+BENTO_ETL_IMAGE=ghcr.io/bento-platform/bento_etl
+BENTO_ETL_VERSION=pr-9
+BENTO_ETL_VERSION_DEV=${BENTO_ETL_VERSION}-dev
+BENTO_ETL_CONTAINER_NAME=${BENTOV2_PREFIX}-etl
+BENTO_ETL_VOL_DATA_DIR=${BENTO_FAST_DATA_DIR}/etl/data
+BENTO_ETL_INTERNAL_PROJECT_HOME=/etl
+BENTO_ETL_INTERNAL_DATA_DIR=${BENTO_ETL_INTERNAL_PROJECT_HOME}/data
+BENTO_ETL_SQLITE_DB_NAME=bento_etl.db
+BENTO_ETL_NETWORK=${BENTOV2_PREFIX}-etl-net
+BENTO_ETL_INTERNAL_PORT=${BENTO_STD_SERVICE_INTERNAL_PORT}
+BENTO_ETL_EXTERNAL_PORT=6400
+BENTO_ETL_DEBUGGER_EXTERNAL_PORT=6456
+BENTO_ETL_DEBUGGER_INTERNAL_PORT=6456
diff --git a/etc/bento_deploy.env b/etc/bento_deploy.env
@@ -16,6 +16,7 @@ BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
 BENTO_MINIO_ENABLED='false'
+BENTO_ETL_ENABLED='false'
 
 #  - Display flags for Bento portals
 #    - Switch to enable French translation in Bento Public
@@ -70,6 +71,10 @@ BENTO_AGGREGATION_CLIENT_SECRET=  # TODO: SET ME WHEN DEPLOYING!
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=  # TODO: SET ME WHEN DEPLOYING!
 
+#  - ETL Client ID/secret; client within BENTOV2_AUTH_REALM
+BENTO_ETL_CLIENT_ID=etl
+BENTO_ETL_CLIENT_SECRET=  # TODO: SET ME WHEN DEPLOYING!
+
 #  - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM
 BENTO_GRAFANA_CLIENT_ID=grafana
 BENTO_GRAFANA_CLIENT_SECRET=  # TODO: SET ME WHEN DEPLOYING IF GRAFANA IS ENABLED!

diff --git a/etc/bento_dev.env b/etc/bento_dev.env
@@ -16,6 +16,7 @@ BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
 BENTO_MINIO_ENABLED='false'
+BENTO_ETL_ENABLED='false'
 
 #  - Display flags for Bento portals
 #    - Switch to enable French translation in Bento Public
@@ -70,6 +71,10 @@ BENTO_AGGREGATION_CLIENT_SECRET=
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=
 
+#  - ETL Client ID/secret; client within BENTOV2_AUTH_REALM
+BENTO_ETL_CLIENT_ID=etl
+BENTO_ETL_CLIENT_SECRET=
+
 #  - Grafana Client ID/secret; client within BENTOV2_AUTH_REALM
 BENTO_GRAFANA_CLIENT_ID=grafana
 BENTO_GRAFANA_CLIENT_SECRET=

diff --git a/etc/bento_services.json b/etc/bento_services.json
@@ -14,6 +14,11 @@
     "url_template": "{BENTO_PUBLIC_URL}/api/{service_kind}",
     "repository": "git@github.com:bento-platform/bento_wes"
   },
+  "etl": {
+    "service_kind": "etl",
+    "url_template": "{BENTO_PUBLIC_URL}/api/{service_kind}",
+    "repository": "git@github.com:bento-platform/bento_etl"
+  },
   "aggregation": {
     "service_kind": "aggregation",
     "url_template": "{BENTO_PUBLIC_URL}/api/{service_kind}",

diff --git a/etc/default_config.env b/etc/default_config.env
@@ -21,6 +21,7 @@ BENTO_CBIOPORTAL_ENABLED='false'
 BENTO_GOHAN_ENABLED='true'
 BENTO_MONITORING_ENABLED='false'
 BENTO_MINIO_ENABLED='false'
+BENTO_ETL_ENABLED='false'
 
 #  - Display flags for Bento portals
 #    - Switch to enable French translation in Bento Public
@@ -104,6 +105,9 @@ BENTO_GRAFANA_CLIENT_SECRET=
 #  - WES Client ID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM
 BENTO_WES_CLIENT_ID=wes
 BENTO_WES_CLIENT_SECRET=
+#  - ETL ClientID/secret; secret to be filled by local.env - client within BENTOV2_AUTH_REALM
+BENTO_ETL_CLIENT_ID=etl
+BENTO_ETL_CLIENT_SECRET=
 # --------------------------------------------------------------------
 
 # Drop-Box

diff --git a/lib/etl/docker-compose.etl.yaml b/lib/etl/docker-compose.etl.yaml
@@ -0,0 +1,39 @@
+services:
+  etl:
+    image: ${BENTO_ETL_IMAGE}:${BENTO_ETL_VERSION}
+    container_name: ${BENTO_ETL_CONTAINER_NAME}
+    networks:
+      - etl-net
+    expose:
+      - ${BENTO_ETL_INTERNAL_PORT}
+    environment:
+      - BENTO_UID
+      - BENTO_DEBUG=False
+      - BENTO_AUTHZ_ENABLED=True
+      - BENTO_AUTHZ_SERVICE_URL
+      - CORS_ORIGINS=${BENTO_CORS_ORIGINS}
+      - KATSU_URL=${BENTO_KATSU_URL}/
+      - SERVICE_URL_BASE_PATH=${BENTOV2_PUBLIC_URL}/api/etl
+      - BENTO_OPENID_CONFIG_URL
+      - ETL_CLIENT_ID=${BENTO_ETL_CLIENT_ID}
+      - ETL_CLIENT_SECRET=${BENTO_ETL_CLIENT_SECRET}
+      - DB_NAME=${BENTO_ETL_SQLITE_DB_NAME}
+      - DATABASE_PATH=${BENTO_ETL_INTERNAL_DATA_DIR}/{BENTO_ETL_SQLITE_DB_NAME}
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "http://localhost:${BENTO_ETL_INTERNAL_PORT}/service-info",
+        ]
+      timeout: ${BENTO_HEALTHCHECK_TIMEOUT}
+      interval: ${BENTO_HEALTHCHECK_INTERVAL}
+      start_period: ${BENTO_HEALTHCHECK_START_PERIOD}
+      start_interval: ${BENTO_HEALTHCHECK_START_INTERVAL}
+    volumes:
+      - ${BENTO_ETL_VOL_DATA_DIR}:${BENTO_ETL_INTERNAL_DATA_DIR}
+
+networks:
+  etl-net:
+    external: true
+    name: ${BENTO_ETL_NETWORK}