update helm charts to dev

mwaiyee · mwaiyee · commit 92dd517b0baa · 2026-07-24T16:02:20.000+08:00
diff --git a/charts/d2e-services/charts/d2e-core/templates/core-deployment.yaml b/charts/d2e-services/charts/d2e-core/templates/core-deployment.yaml
@@ -285,7 +285,7 @@ spec:
           args:
             [
               "-c",
-              "npm run cli db seed -- --swe && npx @logto/cli db alteration rollback 1.23.0 && npx @logto/cli db alteration deploy 1.23.1",
+              "npm run cli db seed -- --swe && npm run cli db alteration deploy latest && node packages/core/d2e-grants.mjs",
             ]
           resources:
             {{- toYaml .Values.global.resources.logtoSeedInit | nindent 12 }}
diff --git a/charts/d2e-services/ci/test-values.yaml b/charts/d2e-services/ci/test-values.yaml
@@ -170,6 +170,8 @@ global:
     LOGTO__CLIENTID_PASSWORD__BASIC_AUTH: "Y2ktbTJtOmNpLW0ybS1zZWNyZXQ="
     SUPABASE_STORAGE_JWT_SECRET: "ci-supabase-jwt-secret-not-real-0000000000"
     SUPABASE_STORAGE_JWT_TOKEN: ""
+    TREX__SQL__PASSWORD: "pencil"
+    TREX_ROOT_KEY: "ndDfri0G85Aw/P+s5jYqdnuBFdfbbY8irhrI9gFSE1Q="
   resources:
     caddy:
       requests: { cpu: 25m, memory: 50Mi }
@@ -189,6 +191,9 @@ global:
     dataflowGen:
       requests: { cpu: 50m, memory: 150Mi }
       limits:   { cpu: 500m, memory: 500Mi }
+    dataflowGenWorker:
+      requests: { cpu: 100m, memory: 250Mi }
+      limits:   { cpu: 250m, memory: 1000Mi }
 
 features:
   materializeCohorts: false
diff --git a/charts/d2e-services/templates/d2e-deployment.yaml b/charts/d2e-services/templates/d2e-deployment.yaml
@@ -385,6 +385,13 @@ spec:
               echo "webapi-init: SQL scripts complete";
               tail -f /dev/null
         - name: trex
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              mkdir -p /usr/src/cert
+              printf '%s' "$TLS__INTERNAL__CRT" > /usr/src/cert/cert.pem
+              printf '%s' "$TLS__INTERNAL__KEY" > /usr/src/cert/key.pem
+              exec /usr/src/entrypoint.sh
           env:
             - name: RUST_BACKTRACE
               value: "full"
@@ -404,6 +411,24 @@ spec:
                   key: AZ_AUTO_GRANT_RESEARCHER_BY_DATASET_CODES
             - name: USER_MGMT__ROLE_SOURCE
               value: "logto"
+            - name: NETWORK_COGNITO_DOMAIN
+              value: ""
+            - name: NETWORK_CENTRAL_API_URL
+              value: ""
+            - name: NETWORK_MACHINE_CLIENT_ID
+              value: ""
+            - name: NETWORK_TOKEN_SCOPE
+              value: ""
+            - name: NETWORK_COORDINATOR_CLIENT_ID
+              value: ""
+            - name: NETWORK_COORDINATOR_TOKEN_SCOPE
+              value: ""
+            - name: NETWORK_CLIENT_SECRET
+              value: ""
+            - name: NETWORK_ENC_KEY
+              value: ""
+            - name: NETWORK_COORDINATOR_CLIENT_SECRET
+              value: ""
             - name: CADDY__D2E__PUBLIC_FQDN
               valueFrom:
                 configMapKeyRef:
@@ -736,6 +761,11 @@ spec:
               value: postgres
             - name: TREX__SQL__DBNAME
               value: postgres
+            - name: TREX__SQL__PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-{{ .Chart.Name }}"
+                  key: TREX__SQL__PASSWORD
             - name: TREX__ENDPOINT_URL
               value: "http://localhost:33001"
             - name: TREX_ROUTES
@@ -746,6 +776,14 @@ spec:
               value: "1024"
             - name: TREXSQL_ENABLED
               value: "true"
+            - name: "PG_SCHEMA"
+              value: "trexdb,notebook"
+            - name: HADES_ENVS_BASE_DIR
+              value: /data/hades/envs
+            - name: HADES_OUTPUT_BASE_DIR
+              value: /data/hades/runs
+            - name: HADES_WORK_SCHEMA
+              value: _config.work
             - name: SWARM_NODE
               value: d2e
             - name: SWARM_CONFIG
@@ -770,7 +808,7 @@ spec:
                         },
                         {
                           "name": "pgwire",
-                          "config": { "host": "0.0.0.0", "port": 5433, "password": "dummy" }
+                          "config": { "host": "0.0.0.0", "port": 5433, "password": "$(TREX__SQL__PASSWORD)" }
                         }
                       ]
                     }
@@ -872,6 +910,20 @@ spec:
                   key: PG__SSL
             - name: PREFECT_API_URL
               value: http://dataflow-gen.d2e.svc.cluster.local:443/d2e/api
+            - name: PREFECT_HEALTH_CHECK
+              value: "http://dataflow-gen.d2e.svc.cluster.local:443/d2e/api/health"
+            - name: PREFECT_POOL
+              value: "process-pool"
+            - name: WORKPOOL_TYPE
+              value: "process"
+            - name: DEFAULT_FLOW_COMMAND
+              value: "/app/run-flow.sh d2e-flows"
+            - name: STORAGE_FILE_BACKEND_PATH
+              value: "/usr/src/data/storage"
+            - name: TREX_STORAGE_URL
+              value: "http://localhost:33001/plugins/trex/storage-api"
+            - name: TREX_STORAGE_SERVICE_KEY
+              value: ""
             - name: PROJECT_NAME
               value: d2e
             - name: SERVICE_ROUTES
@@ -920,8 +972,6 @@ spec:
                   key: TLS__INTERNAL__KEY
             - name: TREX__SQL__PORT
               value: "443"
-            - name: TREX__SQL__PASSWORD
-              value: "dummy"
             - name: TREX_SQL_PASSWORD
               value: "$(TREX__SQL__PASSWORD)"
             - name: JASYPT_ENCRYPTOR_ENABLED
@@ -938,6 +988,28 @@ spec:
               value: "false"
             - name: ANALYTICS_HANA_STREAMING_ENABLED
               value: {{ .Values.features.materializeCohorts | quote }}
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-{{ .Chart.Name }}"
+                  key: STORAGE__JDBC__URL
+            - name: TREX_ROOT_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-{{ .Chart.Name }}"
+                  key: TREX_ROOT_KEY
+            - name: BETTER_AUTH_SECRET
+              value: "$(TREX_ROOT_KEY)"
+            - name: BETTER_AUTH_URL
+              value: "http://localhost:33001/trex"
+            - name: BASE_PATH
+              value: /trex
+            - name: SCHEMA_DIR
+              value: /usr/src/core/schema
+            - name: PLUGINS_DEV_PATH
+              value: "/usr/src/plugins-dev:/usr/src/bundled-plugins"
+            - name: D2E_COMPAT
+              value: "true"
             - name: ANALYTICS_STREAMING_CHUNK_SIZE_BY_DIALECT
               value: '{"hana": 10000}'
             - name: IS_SELF_MANAGED_ROLES
@@ -977,11 +1049,11 @@ spec:
               command:
                 - /bin/sh
                 - -c
-                - curl -f http://localhost:33001/_internal/health || exit 1
-            failureThreshold: 5
-            initialDelaySeconds: 200
-            periodSeconds: 5
+                - curl -fk https://localhost:33000/trex/_internal/health || curl -f http://localhost:33001/trex/_internal/health || exit 1
+            initialDelaySeconds: 120
+            periodSeconds: 10
             timeoutSeconds: 10
+            failureThreshold: 30
         {{- if .Values.features.materializeCohorts }}
         - name: materialize-cohorts
           command: ["npm", "run", "start"]
@@ -1015,7 +1087,8 @@ spec:
               protocol: TCP
           resources:
             requests:
-              cpu: 100m
+              # cpu: 100m
+              cpu: 50m # for dev
               memory: 200Mi
             limits:
               cpu: 1000m
diff --git a/charts/d2e-services/templates/d2e-secrets.yaml b/charts/d2e-services/templates/d2e-secrets.yaml
@@ -37,6 +37,8 @@ data:
   SECURITY_DB_DATASOURCE_URL: '{{ printf "jdbc:postgresql://%s:5432/%s?currentSchema=webapi_security&sslmode=%s" .Values.global.secrets.POSTGRES_HOST .Values.global.secrets.POSTGRES_CONFIG_DB .Values.global.pgSSLMode  | b64enc }}'
   SUPABASE_STORAGE_JWT_SECRET: "{{ .Values.global.secrets.SUPABASE_STORAGE_JWT_SECRET | b64enc }}"
   SUPABASE_STORAGE_JWT_TOKEN: "{{ .Values.global.secrets.SUPABASE_STORAGE_JWT_TOKEN | b64enc }}"
+  TREX__SQL__PASSWORD: "{{ .Values.global.secrets.TREX__SQL__PASSWORD | b64enc }}"
+  TREX_ROOT_KEY: "{{ .Values.global.secrets.TREX_ROOT_KEY | b64enc }}"
 {{- if .Values.features.materializeCohorts }}
 ---
 apiVersion: v1
diff --git a/charts/d2e-services/templates/dataflow-worker-deployment.yaml b/charts/d2e-services/templates/dataflow-worker-deployment.yaml
@@ -19,6 +19,13 @@ spec:
     spec:
       serviceAccountName: dataflow-gen-svc-acc
       restartPolicy: Always
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  profile: d2e-functions
+              topologyKey: kubernetes.io/hostname
       # nodeSelector:
       #   "kubernetes.io/os": linux
       #   "system": "d2e"
@@ -38,7 +45,7 @@ spec:
             [
               "sh",
               "-c",
-              "while :;do sleep {{ .Values.dataflowWorker.readinessSleep }} && wget -O success-trex http://trex-http.d2e.svc.cluster.local:443/_internal/health && wget -O success-dataflow http://dataflow-gen.d2e.svc.cluster.local:443/d2e/api/health && if test -f success-trex; then if test -f success-dataflow; then exit 0; fi; fi; done",
+              "while :;do sleep {{ .Values.dataflowWorker.readinessSleep }} && wget -O success-trex http://trex-http.d2e.svc.cluster.local:443/trex/_internal/health && wget -O success-dataflow http://dataflow-gen.d2e.svc.cluster.local:443/d2e/api/health && if test -f success-trex; then if test -f success-dataflow; then exit 0; fi; fi; done",
             ]
           resources:
             requests:
@@ -140,6 +147,9 @@ spec:
         - name: d2e-worker-cm
           configMap:
             name: "{{ .Release.Name }}-{{ .Chart.Name }}"
+        - name: trex-volume
+          persistentVolumeClaim:
+            claimName: trex-pvc
       containers:
         - name: dataflow-gen-worker
           env:
@@ -211,6 +221,12 @@ spec:
               value: postgresql+asyncpg://$(PG__USER):$(PG__PASSWORD)@$(PG__HOST):5432/$(PG__DB_NAME)
             - name: PREFECT_API_URL
               value: http://dataflow-gen.d2e.svc.cluster.local:443/d2e/api
+            - name: INSTALL_SQLALCHEMY_HANA
+              value: "true"
+            - name: TREX_STORAGE_URL
+              value: "http://trex-http.d2e.svc.cluster.local:443/plugins/trex/storage-api"
+            - name: TREX_STORAGE_SERVICE_KEY
+              value: ""
             - name: PREFECT_API_ENABLE_HTTP2
               value: "False"
             - name: PREFECT_WORKER_WEBSERVER_HOST
@@ -258,32 +274,40 @@ spec:
                 secretKeyRef:
                   name: "{{ .Release.Name }}-{{ .Chart.Name }}"
                   key: TLS__INTERNAL__CA_CRT
-          command: ["/bin/sh"]
+          command:
+            - pixi
+            - run
+            - --manifest-path
+            - /opt/worker/pyproject.toml
+            - sh
+            - -c
           args:
-            [
-              "-c",
-              "rm -rf /root/.prefect/storage/run_cache.db /root/.prefect/storage/events.db; pip install prefect[kubernetes]==3.6.10 && prefect work-pool create 'docker-pool' --type kubernetes; prefect worker start --pool 'docker-pool' --type kubernetes --name prefect-docker-worker --with-healthcheck",
-            ]
-          image: prefecthq/prefect:3.6.10-python3.12
-          imagePullPolicy: IfNotPresent
+            - >-
+              rm -rf /root/.prefect/storage/run_cache.db /root/.prefect/storage/events.db;
+              prefect work-pool create '${PREFECT_POOL:-process-pool}' --type ${WORKPOOL_TYPE:-process};
+              prefect worker start --pool '${PREFECT_POOL:-process-pool}' --type ${WORKPOOL_TYPE:-process}
+              --name prefect-${WORKPOOL_TYPE:-process}-worker --with-healthcheck
+              --limit {{ .Values.dataflowWorker.flowRunLimit | default 3 }}
+          image: ghcr.io/ohdsi/d2e-dataflow-gen-worker:{{ .Values.global.image.tag }}
+          imagePullPolicy: Always
           volumeMounts:
             - mountPath: /opt/prefect/libs/baseJobTemplate.json
               name: d2e-worker-cm
               subPath: baseJobTemplate.json
+            - mountPath: /app/duckdb_data
+              name: trex-volume
           resources:
-            requests:
-              cpu: 100m
-              memory: 250Mi
-            limits:
-              cpu: 250m
-              memory: 1000Mi
+            {{- toYaml .Values.global.resources.dataflowGenWorker | nindent 12 }}
           livenessProbe:
             exec:
               command:
-                - wget
-                - -O
-                - liveness
-                - http://localhost:8080/health
+                - pixi
+                - run
+                - --manifest-path
+                - /opt/worker/pyproject.toml
+                - python
+                - -c
+                - "import urllib.request, sys; sys.exit(0 if 200 <= urllib.request.urlopen('http://localhost:8080/health', timeout=5).getcode() < 300 else 1)"
             failureThreshold: 5
             initialDelaySeconds: 120
             periodSeconds: 10

Original file line number	Diff line number	Diff line change
`@@ -285,7 +285,7 @@ spec:`
`285`	`285`	`args:`
`286`	`286`	`[`
`287`	`287`	`"-c",`
`288`		`- "npm run cli db seed -- --swe && npx @logto/cli db alteration rollback 1.23.0 && npx @logto/cli db alteration deploy 1.23.1",`
	`288`	`+ "npm run cli db seed -- --swe && npm run cli db alteration deploy latest && node packages/core/d2e-grants.mjs",`
`289`	`289`	`]`
`290`	`290`	`resources:`
`291`	`291`	`{{- toYaml .Values.global.resources.logtoSeedInit \| nindent 12 }}`