Merge pull request #213 from bcgov/chore/update-to-v3

chore: upgrade to Airflow 3

Author: joshgamache

.github/workflows/release.yaml

@@ -52,7 +52,7 @@ jobs:
           context: .
           dockerfile_path: Dockerfile
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          chart_version: 2.2.1
+          chart_version: 3.0.0
           is_chart_release: true
   cas-airflow-dag-trigger-build:
     needs: [release]
@@ -65,5 +65,5 @@ jobs:
           context: dag-trigger
           dockerfile_path: dag-trigger/Dockerfile
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          chart_version: 1.2.1
+          chart_version: 1.2.2
           is_chart_release: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.10.5
+FROM apache/airflow:3.1.2
 
 # Install the standard providers for Airflow 2 -- can be removed when we upgrade to Airflow 3
 RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" apache-airflow-providers-standard

dag-trigger/airflow-dag-trigger.sh

@@ -81,9 +81,11 @@ if [ "$AIRFLOW_API_VERSION" = "v2" ]; then
     -d "{\"username\": \"$AIRFLOW_USERNAME\", \"password\": \"$AIRFLOW_PASSWORD\"}" \
     | jq -r .access_token)
   auth_params=(-H "Authorization: Bearer $jwt_token")
+  extra_arguments="\"logical_date\": null, "
 elif [ "$AIRFLOW_API_VERSION" = "v1" ]; then
   dag_url="$AIRFLOW_ENDPOINT/api/v1/dags/${dag_id}"
   auth_params=(-u "$AIRFLOW_USERNAME:$AIRFLOW_PASSWORD")
+  extra_arguments=""
 fi
 
 is_paused=$(_curl "${auth_params[@]}" "$dag_url" | jq .is_paused)
@@ -100,7 +102,7 @@ run_json=$(_curl "${auth_params[@]}" -X POST \
   "$dag_run_url" \
   -H 'Cache-Control: no-cache' \
   -H 'Content-Type: application/json' \
-  -d "{\"conf\": $dag_config}")
+  -d "{$extra_arguments\"conf\": $dag_config}")
 dag_run_id=$(echo "$run_json" | jq -r .dag_run_id)
 
 echo "Started dag run ID: $dag_run_id"

dags/fetch_and_save_dag_from_github.py

@@ -1,4 +1,4 @@
-from airflow.decorators import dag, task
+from airflow.sdk import dag, task
 from airflow import settings
 from dag_configuration import default_dag_args
 from datetime import datetime, timedelta

deploy.sh

@@ -18,7 +18,7 @@ helm upgrade --install \
 helm upgrade --install --timeout 900s \
   --namespace "$AIRFLOW_NAMESPACE_PREFIX-$ENVIRONMENT" \
   -f ./helm/cas-airflow/values.yaml \
-  --set-file airflow.webserver.webserverConfig=./helm/cas-airflow/webserver_config.py \
+  --set-file airflow.apiServer.apiServerConfig=./helm/cas-airflow/webserver_config.py \
   -f "./helm/cas-airflow/values-$ENVIRONMENT.yaml" \
   --set namespaces.airflow="$AIRFLOW_NAMESPACE_PREFIX-$ENVIRONMENT" \
   --set namespaces.ggircs="$GGIRCS_NAMESPACE_PREFIX-$ENVIRONMENT" \

helm/cas-airflow-dag-trigger/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v2
 name: cas-airflow-dag-trigger
 description: A Helm chart for triggering remote DAGs on an existing airflow instance
 type: application
-version: 1.2.1  # Changing this requires updating the image tag in .github/workflows/release.yaml
+version: 1.2.2  # Changing this requires updating the image tag in .github/workflows/release.yaml

helm/cas-airflow-postgres-cluster/values.yaml

@@ -21,7 +21,11 @@ postgresCluster:
       options: "SUPERUSER"
       databases:
         - postgres
+      password:
+        type: AlphaNumeric
     - name: airflow
+      password:
+        type: AlphaNumeric
       databases:
         - postgres
 

helm/cas-airflow/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: cas-airflow
 type: application
-version: 2.2.1 # Changing this requires updating the image tag in .github/workflows/release.yaml
-appVersion: 2.10.5 # The airflow version
+version: 3.0.0 # Changing this requires updating the image tag in .github/workflows/release.yaml
+appVersion: 3.1.2 # The airflow version
 description: Helm chart to deploy cas' flavour of airflow, compatible with OpenShift 4. This chart uses the vanilla airflow chart and adds cas' own templates and values.
 icon: https://www.astronomer.io/static/airflowNewA.png
 keywords:

helm/cas-airflow/templates/routes/route.yaml

@@ -8,10 +8,10 @@ spec:
   host: {{ .Release.Name }}-{{ include "cas-airflow.namespaceSuffix" . }}.{{ .Values.route.clusterDomain }}
   to:
     kind: Service
-    name: {{ .Release.Name }}-webserver
+    name: {{ .Release.Name }}-api-server
     weight: 100
   port:
-    targetPort: airflow-ui
+    targetPort: api-server
   tls:
     termination: edge
     insecureEdgeTerminationPolicy: Allow

…airflow/templates/secrets/webserver.yaml …m/cas-airflow/templates/secrets/api.yamlhelm/cas-airflow/templates/secrets/webserver.yaml renamed to helm/cas-airflow/templates/secrets/api.yaml helm/cas-airflow/templates/secrets/webserver.yaml renamed to helm/cas-airflow/templates/secrets/api.yaml

@@ -1,9 +1,9 @@
-{{- $webserverSecret := (randAlphaNum 32) | b64enc | quote }}
+{{- $apiSecret := (randAlphaNum 32) | b64enc | quote }}
 
-{{- $secretName := printf "%s-%s" .Release.Name "webserver-secret" }}
+{{- $secretName := printf "%s-%s" .Release.Name "api-secret" }}
 {{- $secret := (lookup "v1" "Secret" .Release.Namespace $secretName ) }}
 {{- if $secret }}
-{{- $webserverSecret = index $secret.data "webserver-secret-key" }}
+{{- $apiSecret = index $secret.data "api-secret-key" }}
 {{- end -}}
 
 
@@ -16,4 +16,4 @@ metadata:
     "helm.sh/hook": "pre-install,pre-upgrade"
 type: Opaque
 data:
-  webserver-secret-key: {{ $webserverSecret }}
+  api-secret-key: {{ $apiSecret }}