Merge pull request #208 from bcgov/chore/database-migration

Chore/database migration

Author: joshgamache

.github/workflows/release.yaml

@@ -52,7 +52,7 @@ jobs:
           context: .
           dockerfile_path: Dockerfile
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          chart_version: 2.0.20
+          chart_version: 2.1.0
           is_chart_release: true
   cas-airflow-dag-trigger-build:
     needs: [release]

deploy.sh

@@ -7,6 +7,14 @@ set -euo pipefail
 git_sha1=$(git rev-parse HEAD)
 
 helm dep up ./helm/cas-airflow
+helm repo add cas-postgres https://bcgov.github.io/cas-postgres/
+helm repo update
+
+helm upgrade --install \
+  -f ./helm/cas-airflow-postgres-cluster/values.yaml \
+  -f "./helm/cas-airflow/values-$ENVIRONMENT.yaml" \
+  cas-airflow-db cas-postgres/cas-postgres-cluster
+
 helm upgrade --install --timeout 900s \
   --namespace "$AIRFLOW_NAMESPACE_PREFIX-$ENVIRONMENT" \
   -f ./helm/cas-airflow/values.yaml \

helm/cas-airflow-postgres-cluster/values-dev.yaml

@@ -0,0 +1 @@
+environment: dev

helm/cas-airflow-postgres-cluster/values-prod.yaml

@@ -0,0 +1 @@
+environment: prod

helm/cas-airflow-postgres-cluster/values-test.yaml

@@ -0,0 +1 @@
+environment: test

helm/cas-airflow-postgres-cluster/values.yaml

@@ -0,0 +1,43 @@
+# Values file to be passed to a deployment of https://github.com/bcgov/cas-postgres/tree/develop/helm/cas-postgres-cluster
+
+# dev, test, prod
+environment: ~
+
+postgresCluster:
+  postgresVersion: 17
+  postgres:
+    replicaCount: 3
+  pgbouncer:
+    replicaCount: 2
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+
+  # The "users" value(s) is passed to the crunch postgres operator to create the database.
+  # See http://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/basic-setup/user-management
+  users:
+    - name: postgres
+      options: "SUPERUSER"
+      databases:
+        - postgres
+    - name: airflow
+      databases:
+        - postgres
+
+gcsBackups:
+  enable: true
+  # Needs to match the "namespace_apps" value in the terraform provisioning chart.
+  # example syntax: bucketName
+  bucketName: af-pgo-backups
+
+terraform-bucket-provision:
+  terraform:
+    # example syntax: '["bucketName"]'
+    namespace_apps: '["af-pgo-backups"]'
+    # !important: unique for the deployment
+    workspace: airflow-pgo
+
+# To configure a KNP allowing external access, for metabase for example
+external-access:
+  enabled: false

helm/cas-airflow/Chart.lock

@@ -8,5 +8,8 @@ dependencies:
 - name: terraform-bucket-provision
   repository: https://bcgov.github.io/cas-pipeline/
   version: 0.1.3
-digest: sha256:fa547b34056cf2431a0354054e48b4efeefc39cd74be257e4e6a64b14cc9174e
-generated: "2024-11-15T13:11:12.776004708-08:00"
+- name: patroni-migration
+  repository: https://bcgov.github.io/cas-postgres/
+  version: 1.0.1
+digest: sha256:164d57773acfb1c55c545e2bede23068136614ae2a2a52b9bf8d26bacd375914
+generated: "2025-05-13T08:52:20.688824737-06:00"

helm/cas-airflow/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: cas-airflow
 type: application
-version: 2.0.20 # Changing this requires updating the image tag in .github/workflows/release.yaml
+version: 2.1.0 # Changing this requires updating the image tag in .github/workflows/release.yaml
 appVersion: 2.10.5 # The airflow version
 description: Helm chart to deploy cas' flavour of airflow, compatible with OpenShift 4. This chart uses the vanilla airflow chart and adds cas' own templates and values.
 icon: https://www.astronomer.io/static/airflowNewA.png
@@ -20,3 +20,6 @@ dependencies:
   - name: terraform-bucket-provision
     version: "0.1.3"
     repository: https://bcgov.github.io/cas-pipeline/
+  - name: patroni-migration
+    version: 1.0.1
+    repository: https://bcgov.github.io/cas-postgres/

helm/cas-airflow/templates/networkPolicy/postgres-cluster-access.yaml

@@ -0,0 +1,17 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: {{ include "cas-airflow.fullname" . }}-postgres-cluster-access
+  labels: {{ include "cas-airflow.labels" . | nindent 4 }}
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Values.database.clusterReleaseName }}
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              release: {{ include "cas-airflow.name" . }}
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/instance: {{ include "cas-airflow.name" . }}

helm/cas-airflow/templates/secrets/metadata.yaml

@@ -1,10 +1,8 @@
 ################################
 ## Airflow Metadata Secret
 #################################
-{{- $postgresHost := (printf "%s-%s.%s.svc.cluster.local" .Release.Name "patroni" .Release.Namespace) }}
-{{- $host := $postgresHost }}
-{{- $port := .Values.airflow.data.metadataConnection.port | toString }}
-{{- $database := .Values.airflow.data.metadataConnection.db }}
+## Uses the PGBOUNCER_URI environment variable to connect to the database, which ensures characters are URI encoded
+#################################
 
 kind: Secret
 apiVersion: v1
@@ -16,4 +14,4 @@ metadata:
 {{ include "cas-airflow.labels" $ | nindent 4 }}
 type: Opaque
 data:
-  connection: {{ (printf "postgresql://%s:%s@%s:%s/%s?sslmode=%s" .Values.airflow.data.metadataConnection.user .Values.airflow.data.metadataConnection.pass $host $port $database .Values.airflow.data.metadataConnection.sslmode) | b64enc | quote }}
+  connection: {{ (printf "$PGBOUNCER_URI?sslmode=%s"  .Values.airflow.data.metadataConnection.sslmode) | b64enc | quote }}