WG-62: Set up celery + redis on staging/prod servers

Author: bmispelon

ansible/roles/hypha/defaults/main.yml

@@ -25,5 +25,8 @@ hypha_docker_tag: "wtsh-main"
 # The postgres version used
 hypha_postgres_version: 16
 
+# The redis version used (for celery)
+hypha_redis_version: "8.4"
+
 # The name of the system user that can deploy the site (used in github action)
 hypha_deploy_user: deploy-{{ app_name }}

ansible/roles/hypha/tasks/main.yml

@@ -35,6 +35,8 @@
   loop:
     - web
     - db
+    - redis
+    - celery
 
 - name: Enable systemd service {{ hypha_project_name }}-{{ item }}
   ansible.builtin.systemd:
@@ -43,11 +45,16 @@
   loop:
     - web
     - db
+    - redis
+    - celery
 
-- name: Start service {{ hypha_project_name }}-db
+- name: Start service {{ hypha_project_name }}-{{ item }}
   ansible.builtin.systemd:
-    name: "{{ hypha_project_name }}-db"
+    name: "{{ hypha_project_name }}-{{ item }}"
     state: started
+  loop:
+    - db
+    - redis
 
 - name: Make sure .env file exists
   ansible.builtin.template:

ansible/roles/hypha/templates/deploy.sh.jinja2

@@ -3,6 +3,7 @@ set -euo pipefail
 
 CONTAINER_WEB={{hypha_project_name}}-web
 SERVICE_WEB={{hypha_project_name}}-web
+SERVICE_CELERY={{hypha_project_name}}-celery
 IMAGE={{hypha_docker_image}}{% if hypha_docker_tag %}:{{ hypha_docker_tag }}{% endif +%}
 LOGFILE="{{ hypha_root_dir }}/deploy_$(date --iso).log"
 
@@ -28,7 +29,11 @@ systemctl daemon-reload
 
 echolog "Restarting service $SERVICE_WEB..."
 systemctl restart $SERVICE_WEB
-echolog "Service restarted. Sleeping for a bit..."
+
+echolog "Restarting celery $SERVICE_CELERY..."
+systemctl restart $SERVICE_CELERY
+
+echolog "Services restarted. Sleeping for a bit..."
 sleep 5
 echolog "Done."
 

ansible/roles/hypha/templates/hypha.env.jinja2

@@ -4,6 +4,10 @@ SECRET_KEY="{{ lookup('community.general.random_string', length=50) }}"
 ALLOWED_HOSTS={{ hypha_external_domain }}
 PRIMARY_HOST={{ hypha_external_domain }}
 DATABASE_URL="postgres://hypha@{{ hypha_project_name }}-db/hypha"
+REDIS_URL="redis://{{ hypha_project_name }}-redis:6379"
+
+# Activate background tasks:
+CELERY_TASK_ALWAYS_EAGER=0
 
 
 # Enable basic auth out of the box

ansible/roles/hypha/templates/systemd/celery.service.jinja2

@@ -0,0 +1,28 @@
+[Unit]
+Description=Hypha celery docker container ({{ app_name }})
+After=docker.service
+Wants=network-online.target docker.socket
+Requires=docker.socket
+
+[Service]
+# Automatically restart this unit when web is restarted:
+PartOf={{ hypha_project_name }}-web
+Restart=on-failure
+ExecStart=/usr/bin/docker run \
+    --name {{ hypha_project_name }}-celery \
+    --pull never \
+    --rm \
+    --env PYTHONDONTWRITEBYTECODE=1 \
+    --env PYTHONUNBUFFERED=1 \
+    --network {{ hypha_docker_network }} \
+    --volume {{ hypha_root_dir }}/media:/app/media \
+    --mount type=bind,src={{ hypha_root_dir }}/hypha.env,dst=/app/.env,ro \
+    --mount type=bind,src={{ hypha_root_dir }}/local_settings.py,dst=/app/hypha/settings/local.py,ro \
+    {{ hypha_docker_image }}{% if hypha_docker_tag %}:{{ hypha_docker_tag }}{% endif +%}
+    python -m celery --app={{ app_name }} worker -l INFO
+ExecStop=/usr/bin/docker container stop \
+    --timeout 10 \
+    {{ hypha_project_name }}-celery
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/hypha/templates/systemd/redis.service.jinja2

@@ -0,0 +1,20 @@
+[Unit]
+Description=Hypha redis docker container ({{ app_name }})
+After=docker.service
+Wants=network-online.target docker.socket
+Requires=docker.socket
+
+[Service]
+Restart=on-failure
+ExecStart=/usr/bin/docker run \
+    --name {{ hypha_project_name }}-redis \
+    --pull missing \
+    --rm \
+    --network {{ hypha_docker_network }} \
+    redis:{{ hypha_redis_version }}
+ExecStop=/usr/bin/docker container stop \
+    --timeout 10 \
+    {{ hypha_project_name }}-redis
+
+[Install]
+WantedBy=multi-user.target