Backend/requests: Support .ics attachments (#8471)

* Backend/requests: Support .ics attachments

* Fix tests and mypy

* Fix mypy

* Added caveat comments

* PR feedback

* Formatting

Author: tristanlabelle

app/backend.dev.env

@@ -38,6 +38,7 @@ NOTIFICATION_EMAIL_SENDER=Couchers.org
 NOTIFICATION_EMAIL_ADDRESS=notify@couchers.org.invalid
 NOTIFICATION_PREFIX='[DEV] '
 ENABLE_NOTIFICATION_TRANSLATIONS=1
+ENABLE_EMAIL_ICS_ATTACHMENTS=1
 
 REPORTS_EMAIL_RECIPIENT=reports@couchers.org.invalid
 CONTRIBUTOR_FORM_EMAIL_RECIPIENT=forms@couchers.org.invalid

app/backend/proto/internal/jobs.proto

@@ -14,6 +14,13 @@ message SendEmailPayload {
   string list_unsubscribe_header = 7;
   // source data as to where this email came from
   string source_data = 8;
+  repeated EmailAttachment attachments = 9;
+}
+
+message EmailAttachment {
+  string filename = 1;
+  string mime_type = 2; // Both the type and subtype, e.g. "application/pdf"
+  bytes data = 3;
 }
 
 message HandleNotificationPayload {

app/backend/pyproject.toml

@@ -13,6 +13,7 @@ dependencies = [
     "geoip2>=5.1.0",
     "grpcio>=1.76.0",
     "http-ece>=1.2.1",
+    "ics>=0.7.3",
     "Jinja2>=3.1.6",
     "luhn>=0.2.0",
     "markdown-it-py>=4.0.0",

app/backend/src/couchers/config.py

@@ -70,6 +70,7 @@
     # An optional prefix for email subject, e.g. [STAGING]
     ("NOTIFICATION_PREFIX", str, ""),
     ("ENABLE_NOTIFICATION_TRANSLATIONS", bool),
+    ("ENABLE_EMAIL_ICS_ATTACHMENTS", bool),
     # Address to send emails about reported users
     ("REPORTS_EMAIL_RECIPIENT", str),
     # Address to send contributor forms when users sign up/fill the form

app/backend/src/couchers/email/calendar_events.py

@@ -0,0 +1,104 @@
+from email.headerregistry import Address
+from typing import cast
+
+from ics import Calendar, Event  # type: ignore[import-untyped]
+from ics.grammar.parse import ContentLine  # type: ignore[import-untyped]
+
+from couchers import urls
+from couchers.config import config
+from couchers.email.rendering import get_emails_i18next
+from couchers.i18n import LocalizationContext
+from couchers.proto.internal.jobs_pb2 import EmailAttachment
+from couchers.proto.requests_pb2 import HostRequest
+
+HOST_REQUEST_ICS_FILENAME = "host_request.ics"
+
+
+def create_host_request_attachment(
+    host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
+) -> EmailAttachment:
+    ics = create_host_request_ics(host_request, other_name, hosting, loc_context)
+    return ics_to_attachment(ics, HOST_REQUEST_ICS_FILENAME)
+
+
+def create_host_request_ics(
+    host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
+) -> str:
+    event = create_host_request_event(host_request, other_name, hosting, loc_context)
+    return event_to_ics(event, loc_context)
+
+
+def create_host_request_event(
+    host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
+) -> Event:
+    """Creates an ics event for a host request."""
+
+    event = Event()
+    event.uid = get_host_request_event_uid(host_request.host_request_id)
+
+    if hosting:
+        event.name = get_emails_i18next().localize(
+            "calendar_events.host_requests.title_host", loc_context.locale, {"name": other_name}
+        )
+    else:
+        event.name = get_emails_i18next().localize(
+            "calendar_events.host_requests.title_surfer", loc_context.locale, {"name": other_name}
+        )
+
+    # Our to_date is inclusive, iCalendar's DTEND is exclusive (for full-day events)
+    # make_all_day will adjust the end date by one day accordingly.
+    event.begin = host_request.from_date
+    event.end = host_request.to_date
+    event.make_all_day()
+
+    event.location = host_request.hosting_city
+    event.url = urls.host_request(host_request_id=str(host_request.host_request_id))
+
+    # Google Calendar™ will hide the URL if there is a location, so also include it in the description
+    event.description = event.url
+
+    return event
+
+
+def create_host_request_cancellation_attachment(
+    host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
+) -> EmailAttachment:
+    ics = create_host_request_cancellation_ics(host_request, other_name, hosting, loc_context)
+    return ics_to_attachment(ics, HOST_REQUEST_ICS_FILENAME)
+
+
+def create_host_request_cancellation_ics(
+    host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
+) -> str:
+    event = create_host_request_event(host_request, other_name, hosting, loc_context)
+    event.name = get_emails_i18next().localize(
+        "calendar_events.title_cancelled", loc_context.locale, {"title": event.name}
+    )
+    event.status = "CANCELLED"
+
+    # Cancellation is sequenced after creation (which defaults to sequence number 0)
+    event.extra.append(ContentLine(name="SEQUENCE", value="1"))
+
+    return event_to_ics(event, loc_context)
+
+
+def event_to_ics(event: Event, loc_context: LocalizationContext) -> str:
+    # PRODID is mandatory and generally follows "-//[Organization]//[Product Name]//[Language]"
+    calendar = Calendar(creator=f"-//Couchers.org//Couchers//{loc_context.locale.upper()}")
+    if event.status == "CANCELLED":
+        calendar.method = "CANCEL"
+    calendar.events.add(event)
+    return cast(str, calendar.serialize())
+
+
+def ics_to_attachment(ics: str, filename: str) -> EmailAttachment:
+    return EmailAttachment(
+        filename=filename,
+        mime_type="text/calendar",
+        data=ics.encode("utf-8"),
+    )
+
+
+def get_host_request_event_uid(host_request_id: int) -> str:
+    uid_domain = Address(addr_spec=config["NOTIFICATION_EMAIL_ADDRESS"]).domain
+    return f"host_request.{host_request_id}@{uid_domain}"

app/backend/src/couchers/email/locales/en.json

@@ -2,6 +2,13 @@
   "generic": {
     "greeting_line": "Hi {{name}},"
   },
+  "calendar_events": {
+    "title_cancelled": "Cancelled: {{ title }}",
+    "host_requests": {
+      "title_host": "Hosting {{ name }}",
+      "title_surfer": "Surfing with {{ name }}"
+    }
+  },
   "plaintext_formats": {
     "action": "{{text}}: {{url}}",
     "user": "{{name}}, {{age}}, {{city}}"

app/backend/src/couchers/email/smtp.py

@@ -45,12 +45,12 @@ def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
     if updated_html:
         # for any png files in attachment_imgs/, goes through and replaces instances of the filename with attachment
         used_attachments = []
-        for attachment in (template_base / "attachment_imgs").glob("*.png"):
-            attachment_html_path = str(attachment.relative_to(template_base))
+        for attachment_full_path in (template_base / "attachment_imgs").glob("*.png"):
+            attachment_html_path = str(attachment_full_path.relative_to(template_base))
             if attachment_html_path not in updated_html:
                 continue
             # it's used in this template, so attach and replace it
-            data = attachment.read_bytes()
+            data = attachment_full_path.read_bytes()
             cid, wcid = make_cid(payload.sender_email)
             updated_html = updated_html.replace(attachment_html_path, f"cid:{wcid}")
             used_attachments.append((cid, "image", "png", data))
@@ -61,6 +61,13 @@ def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
             payloads = cast(list[MIMEPart], msg.get_payload())
             payloads[1].add_related(data, mime_type, mime_subtype, cid=cid)
 
+    if payload.attachments:
+        for attachment in payload.attachments:
+            mime_maintype, mime_subtype = attachment.mime_type.split("/")
+            msg.add_attachment(
+                attachment.data, maintype=mime_maintype, subtype=mime_subtype, filename=attachment.filename
+            )
+
     with smtplib.SMTP(config["SMTP_HOST"], config["SMTP_PORT"]) as server:
         server.ehlo()
         if not config["DEV"]:

app/backend/src/couchers/notifications/background.py

@@ -58,6 +58,7 @@ def _send_email_notification(session: Session, user: User, notification: Notific
             html=rendered.body_html,
             source_data=rendered.source_data,
             list_unsubscribe_header=rendered.list_unsubscribe_header,
+            attachments=rendered.attachments,
         ),
     )
 

app/backend/src/couchers/notifications/render_email.py

@@ -1,9 +1,10 @@
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from couchers import urls
 from couchers.config import config
+from couchers.email.calendar_events import create_host_request_attachment, create_host_request_cancellation_attachment
 from couchers.email.rendering import EmailFooter, UnsubscribeInfo, UnsubscribeLink
 from couchers.i18n import LocalizationContext
 from couchers.i18n.localize import format_phone_number
@@ -16,6 +17,7 @@
     generate_unsub_topic_key,
 )
 from couchers.proto import api_pb2, notification_data_pb2
+from couchers.proto.internal.jobs_pb2 import EmailAttachment
 from couchers.templating import Jinja2Template, template_folder
 from couchers.utils import now, to_aware_datetime
 
@@ -29,6 +31,7 @@ class RenderedEmailNotification:
     body_html: str | None
     source_data: str | None
     list_unsubscribe_header: str | None
+    attachments: list[EmailAttachment] = field(default_factory=list)
 
 
 def render_email_notification(
@@ -64,12 +67,18 @@ def render_email_notification(
     list_unsubscribe_header = get_list_unsubscribe_header(notification)
     source_data = config["VERSION"] + f"/{custom_templated.template_name}"
 
+    if config.get("ENABLE_EMAIL_ICS_ATTACHMENTS"):
+        attachment = get_ics_attachment(notification, loc_context)
+    else:
+        attachment = None
+
     return RenderedEmailNotification(
         subject=custom_templated.subject,
         body_plaintext=plain,
         body_html=html,
         source_data=source_data,
         list_unsubscribe_header=list_unsubscribe_header,
+        attachments=[attachment] if attachment else [],
     )
 
 
@@ -761,6 +770,28 @@ def _get_custom_templated_email(notification: Notification, loc_context: Localiz
     raise NotImplementedError(f"Unknown topic-action: {notification.topic}:{notification.action}")
 
 
+def get_ics_attachment(notification: Notification, loc_context: LocalizationContext) -> EmailAttachment | None:
+    data = notification.topic_action.data_type.FromString(notification.data)  # type: ignore[attr-defined]
+    if notification.topic_action == NotificationTopicAction.host_request__accept:
+        # Caveat: The surfer technically still hasn't confirmed, but when they do they don't receive an email,
+        # so the accept notification is our last opportunity to provide them with a calendar event.
+        return create_host_request_attachment(
+            data.host_request, other_name=data.host.name, hosting=False, loc_context=loc_context
+        )
+    elif notification.topic_action == NotificationTopicAction.host_request__confirm:
+        return create_host_request_attachment(
+            data.host_request, other_name=data.surfer.name, hosting=True, loc_context=loc_context
+        )
+    elif notification.topic_action == NotificationTopicAction.host_request__cancel:
+        # Caveat: only the party getting cancelled receives this notification,
+        # we have no opportunity to provide the cancelling party with a cancelled ics attachment.
+        return create_host_request_cancellation_attachment(
+            data.host_request, other_name=data.surfer.name, hosting=True, loc_context=loc_context
+        )
+    else:
+        return None
+
+
 def get_list_unsubscribe_header(notification: Notification) -> str | None:
     if notification.topic_action.is_critical:
         return None

app/backend/src/tests/conftest.py

@@ -172,6 +172,7 @@ def testconfig():
     config["REPORTS_EMAIL_RECIPIENT"] = "reports@couchers.org.invalid"
     config["CONTRIBUTOR_FORM_EMAIL_RECIPIENT"] = "forms@couchers.org.invalid"
     config["MODS_EMAIL_RECIPIENT"] = "mods@couchers.org.invalid"
+    config["ENABLE_EMAIL_ICS_ATTACHMENTS"] = True
 
     config["ENABLE_DONATIONS"] = False
     config["STRIPE_API_KEY"] = ""