Merge pull request #311 from cityofaustin/john/25412-deploy-ems-match-etl

Add EMS person matcher to cris import dag

Author: johnclary

dags/vz_cris_import.py

@@ -64,16 +64,18 @@
     },
 }
 
-docker_image = f"atddocker/vz-cris-import:{'production' if DEPLOYMENT_ENVIRONMENT == 'production' else 'latest'}"
-
+docker_image_cris_import = f"atddocker/vz-cris-import:{'production' if DEPLOYMENT_ENVIRONMENT == 'production' else 'latest'}"
+docker_image_ems_match = f"atddocker/vz-ems-person-match:{'production' if DEPLOYMENT_ENVIRONMENT == 'production' else 'development'}"
 
 DEFAULT_ARGS = {
     "depends_on_past": False,
     "email_on_failure": False,
     "email_on_retry": False,
     "retries": 0,
     "execution_timeout": duration(minutes=60),
-    "on_failure_callback": task_fail_slack_alert,
+    "on_failure_callback": (
+        task_fail_slack_alert if DEPLOYMENT_ENVIRONMENT != "development" else None
+    ),
 }
 
 
@@ -93,7 +95,7 @@
     cris_import = DockerOperator(
         task_id="run_cris_import",
         docker_conn_id="docker_default",
-        image=docker_image,
+        image=docker_image_cris_import,
         command=f"./cris_import.py --csv --pdf --s3-download --s3-upload --s3-archive --workers 2",
         environment=env_vars,
         auto_remove="force",
@@ -104,11 +106,22 @@
     ocr_crash_narratives = DockerOperator(
         task_id="ocr_crash_narratives",
         docker_conn_id="docker_default",
-        image=docker_image,
+        image=docker_image_cris_import,
         command=f"./cr3_ocr_narrative.py --workers 2",
         environment=env_vars,
         auto_remove="force",
         tty=True,
     )
 
-    cris_import >> ocr_crash_narratives
+    match_ems_to_people = DockerOperator(
+        task_id="match_ems_to_people",
+        docker_conn_id="docker_default",
+        image=docker_image_ems_match,
+        command=f"python match_ems_to_people.py",
+        environment=env_vars,
+        auto_remove="force",
+        tty=True,
+        force_pull=True,
+    )
+
+    cris_import >> ocr_crash_narratives >> match_ems_to_people