Backend/emails: Support custom SMTP headers for attachments (#8665)

Author: tristanlabelle

app/backend/proto/internal/jobs.proto

@@ -14,13 +14,14 @@ message SendEmailPayload {
   string list_unsubscribe_header = 7;
   // source data as to where this email came from
   string source_data = 8;
-  repeated EmailAttachment attachments = 9;
+  reserved 9; // Previous "attachments" field, which had mime_type+filename subfields.
+  repeated EmailAttachmentV2 attachments = 10;
 }
 
-message EmailAttachment {
-  string filename = 1;
-  string mime_type = 2; // Both the type and subtype, e.g. "application/pdf"
-  bytes data = 3;
+message EmailAttachmentV2 {
+  bytes data = 1;
+  string content_disposition = 2; // The Content-Disposition header, including parameters
+  string content_type = 3; // The Content-Type header, including parameters
 }
 
 message HandleNotificationPayload {

app/backend/src/couchers/email/calendar_events.py

@@ -1,5 +1,4 @@
 from email.headerregistry import Address
-from typing import cast
 
 from ics import Calendar, Event  # type: ignore[import-untyped]
 from ics.grammar.parse import ContentLine  # type: ignore[import-untyped]
@@ -8,27 +7,27 @@
 from couchers.config import config
 from couchers.email.rendering import get_emails_i18next
 from couchers.i18n import LocalizationContext
-from couchers.proto.internal.jobs_pb2 import EmailAttachment
+from couchers.proto.internal.jobs_pb2 import EmailAttachmentV2
 from couchers.proto.requests_pb2 import HostRequest
 
 HOST_REQUEST_ICS_FILENAME = "host_request.ics"
 
 
 def create_host_request_attachment(
     host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
-) -> EmailAttachment:
-    ics = create_host_request_ics(host_request, other_name, hosting, loc_context)
-    return ics_to_attachment(ics, HOST_REQUEST_ICS_FILENAME)
+) -> EmailAttachmentV2:
+    calendar = create_host_request_calendar(host_request, other_name, hosting, loc_context)
+    return calendar_to_attachment(calendar, HOST_REQUEST_ICS_FILENAME)
 
 
-def create_host_request_ics(
+def create_host_request_calendar(
     host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
-) -> str:
+) -> Calendar:
     event = create_host_request_event(host_request, other_name, hosting, loc_context)
 
     # METHOD:PUBLISH means this is part of a stream of calendar event information.
     # It allows for later cancellation, and doesn't expose accept/decline functionality.
-    return event_to_ics(event, "PUBLISH", loc_context)
+    return event_to_calendar(event, "PUBLISH", loc_context)
 
 
 def create_host_request_event(
@@ -68,14 +67,14 @@ def create_host_request_event(
 
 def create_host_request_cancellation_attachment(
     host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
-) -> EmailAttachment:
-    ics = create_host_request_cancellation_ics(host_request, other_name, hosting, loc_context)
-    return ics_to_attachment(ics, HOST_REQUEST_ICS_FILENAME)
+) -> EmailAttachmentV2:
+    calendar = create_host_request_cancellation_calendar(host_request, other_name, hosting, loc_context)
+    return calendar_to_attachment(calendar, HOST_REQUEST_ICS_FILENAME)
 
 
-def create_host_request_cancellation_ics(
+def create_host_request_cancellation_calendar(
     host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
-) -> str:
+) -> Calendar:
     event = create_host_request_event(host_request, other_name, hosting, loc_context, sequence=1)
     event.name = get_emails_i18next().localize(
         "calendar_events.title_cancelled", loc_context.locale, {"title": event.name}
@@ -85,24 +84,28 @@ def create_host_request_cancellation_ics(
     # METHOD:PUBLISH means this is part of a stream of calendar event information.
     # Gmail™ will immediately remove the event from the user's calendar.
     # METHOD:CANCEL might leave the event in cancelled state or not work.
-    return event_to_ics(event, "PUBLISH", loc_context)
+    return event_to_calendar(event, "PUBLISH", loc_context)
 
 
-def event_to_ics(event: Event, method: str | None, loc_context: LocalizationContext) -> str:
+def event_to_calendar(event: Event, method: str | None, loc_context: LocalizationContext) -> Calendar:
     # PRODID is mandatory and generally follows "-//[Organization]//[Product Name]//[Language]"
     calendar = Calendar(creator=f"-//Couchers.org//Couchers//{loc_context.locale.upper()}")
     if method:
         calendar.method = method
     calendar.events.add(event)
-    return cast(str, calendar.serialize())
+    return calendar
 
 
-def ics_to_attachment(ics: str, filename: str) -> EmailAttachment:
-    return EmailAttachment(
-        filename=filename,
-        mime_type="text/calendar",
-        data=ics.encode("utf-8"),
-    )
+def calendar_to_attachment(calendar: Calendar, filename: str) -> EmailAttachmentV2:
+    data = calendar.serialize().encode("utf-8")
+    content_disposition = f'attachment; filename="{filename}"'
+    content_type = 'text/calendar; charset="utf-8"'
+    if calendar.method:
+        # The SMTP Content-Type "method" parameter must match the value in the ics file.
+        # AI recommends avoiding quotes on this parameter for backwards compatibility with old email clients.
+        content_type += f"; method={calendar.method}"
+
+    return EmailAttachmentV2(data=data, content_disposition=content_disposition, content_type=content_type)
 
 
 def get_host_request_event_uid(host_request_id: int) -> str:

app/backend/src/couchers/email/smtp.py

@@ -19,18 +19,12 @@ def make_cid(sender_email: str) -> tuple[str, str]:
     return cid, without_tag
 
 
-def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
-    """
-    Sends out the email through SMTP, settings from config.
-
-    Returns a models.Email object that can be straight away added to the database.
-    """
-    message_id = random_hex()
+def email_proto_to_message(payload: jobs_pb2.SendEmailPayload, couchers_id: str) -> tuple[EmailMessage, str | None]:
     msg = EmailMessage()
     msg["Subject"] = payload.subject
     msg["From"] = Address(payload.sender_name, addr_spec=payload.sender_email)
     msg["To"] = Address(addr_spec=payload.recipient)
-    msg["X-Couchers-ID"] = message_id
+    msg["X-Couchers-ID"] = couchers_id
 
     if payload.list_unsubscribe_header:
         msg["List-Unsubscribe"] = payload.list_unsubscribe_header
@@ -41,7 +35,7 @@ def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
 
     msg.set_content(payload.plain)
 
-    updated_html = payload.html
+    updated_html: str | None = payload.html
     if updated_html:
         # for any png files in attachment_imgs/, goes through and replaces instances of the filename with attachment
         used_attachments = []
@@ -58,15 +52,31 @@ def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
         msg.add_alternative(updated_html, subtype="html")
 
         for cid, mime_type, mime_subtype, data in used_attachments:
-            payloads = cast(list[MIMEPart], msg.get_payload())
-            payloads[1].add_related(data, mime_type, mime_subtype, cid=cid)
+            html_part = cast(list[MIMEPart], msg.get_payload())[-1]
+            html_part.add_related(data, mime_type, mime_subtype, cid=cid)
 
     if payload.attachments:
         for attachment in payload.attachments:
-            mime_maintype, mime_subtype = attachment.mime_type.split("/")
+            # Create with generic Content-Type/Content-Disposition headers,
+            # then overwrite them with the headers specified by the caller.
             msg.add_attachment(
-                attachment.data, maintype=mime_maintype, subtype=mime_subtype, filename=attachment.filename
+                attachment.data, maintype="application", subtype="octet-stream", disposition="attachment"
             )
+            attachment_part = cast(list[MIMEPart], msg.get_payload())[-1]
+            _replace_header_verbatim(attachment_part, "Content-Type", attachment.content_type)
+            _replace_header_verbatim(attachment_part, "Content-Disposition", attachment.content_disposition)
+
+    return msg, updated_html
+
+
+def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
+    """
+    Sends out the email through SMTP, settings from config.
+
+    Returns a models.Email object that can be straight away added to the database.
+    """
+    message_id = random_hex()
+    msg, updated_html = email_proto_to_message(payload, message_id)
 
     with smtplib.SMTP(config["SMTP_HOST"], config["SMTP_PORT"]) as server:
         server.ehlo()
@@ -84,7 +94,24 @@ def send_smtp_email(payload: jobs_pb2.SendEmailPayload) -> Email:
         recipient=payload.recipient,
         subject=payload.subject,
         plain=payload.plain,
-        html=updated_html,
+        html=updated_html or "",
         list_unsubscribe_header=payload.list_unsubscribe_header,
         source_data=payload.source_data,
     )
+
+
+def _replace_header_verbatim(part: MIMEPart, name: str, value: str) -> None:
+    # MIMEPart.replace_header will parse the value and reformat it,
+    # resulting in additional quoting for an .ics "method=PUBLISH" parameter,
+    # which are not as backwards compatible with older email clients.
+
+    if hasattr(part, "_headers"):
+        # Replace the header in the internal data structure to avoid reformatting.
+        header_index = next((i for i, val in enumerate(part._headers) if val[0] == name), None)
+        if isinstance(header_index, int):
+            part._headers[header_index] = (name, value)
+        else:
+            part._headers.append((name, value))
+    else:
+        # Non-verbatim fallback, in case the internals change
+        part.replace_header(name, value)

app/backend/src/couchers/notifications/render_email.py

@@ -22,7 +22,7 @@
     generate_unsub_topic_key,
 )
 from couchers.proto import api_pb2
-from couchers.proto.internal.jobs_pb2 import EmailAttachment
+from couchers.proto.internal.jobs_pb2 import EmailAttachmentV2
 from couchers.templating import Jinja2Template, template_folder
 from couchers.utils import now, to_aware_datetime
 
@@ -36,7 +36,7 @@ class RenderedEmailNotification:
     body_html: str | None
     source_data: str | None
     list_unsubscribe_header: str | None
-    attachments: list[EmailAttachment] = field(default_factory=list)
+    attachments: list[EmailAttachmentV2] = field(default_factory=list)
 
 
 def render_email_notification(
@@ -450,7 +450,7 @@ def _get_custom_templated_email(notification: Notification, loc_context: Localiz
     raise NotImplementedError(f"Unknown topic-action: {notification.topic}:{notification.action}")
 
 
-def get_ics_attachment(notification: Notification, loc_context: LocalizationContext) -> EmailAttachment | None:
+def get_ics_attachment(notification: Notification, loc_context: LocalizationContext) -> EmailAttachmentV2 | None:
     data = notification.topic_action.data_type.FromString(notification.data)  # type: ignore[attr-defined]
     if notification.topic_action == NotificationTopicAction.host_request__accept:
         # Caveat: The surfer technically still hasn't confirmed, but when they do they don't receive an email,

app/backend/src/tests/test_calendar_events.py

@@ -4,7 +4,7 @@
 import ics
 import pytest
 
-from couchers.email.calendar_events import create_host_request_cancellation_ics, create_host_request_ics
+from couchers.email.calendar_events import create_host_request_calendar, create_host_request_cancellation_calendar
 from couchers.i18n.context import LocalizationContext
 from couchers.proto import conversations_pb2, requests_pb2
 from couchers.proto.requests_pb2 import HostRequest
@@ -25,9 +25,9 @@ def test_initial_ics_content():
         host_request_id=42, from_date="2000-01-01", to_date="2000-01-02", hosting_city="New York"
     )
 
-    ics = create_host_request_ics(
+    ics: str = create_host_request_calendar(
         host_request, other_name="Bob", hosting=True, loc_context=LocalizationContext.en_utc()
-    )
+    ).serialize()
     assert _normalize_ics(ics) == _normalize_ics("""
 BEGIN:VCALENDAR
 VERSION:2.0
@@ -52,9 +52,9 @@ def test_cancellation_ics_content():
         host_request_id=42, from_date="2000-01-01", to_date="2000-01-02", hosting_city="New York"
     )
 
-    ics = create_host_request_cancellation_ics(
+    ics: str = create_host_request_cancellation_calendar(
         host_request, other_name="Bob", hosting=True, loc_context=LocalizationContext.en_utc()
-    )
+    ).serialize()
     assert _normalize_ics(ics) == _normalize_ics("""
 BEGIN:VCALENDAR
 VERSION:2.0
@@ -216,8 +216,9 @@ def test_host_request_attachments_disabled(db, feature_flags, moderator: Moderat
 def _get_email_ics_attachment_calendar_event(e) -> ics.Event:
     assert len(e.attachments or []) == 1
     ics_attachment = e.attachments[0]
-    assert ics_attachment.filename.endswith(".ics")
+    assert ics_attachment.content_type.startswith("text/calendar")
     ics_calendar = ics.Calendar(ics_attachment.data.decode("utf-8"))
+    assert f"method={ics_calendar.method}" in ics_attachment.content_type
     assert len(ics_calendar.events) == 1
     return next(iter(ics_calendar.events))
 

app/backend/src/tests/test_smtp.py

@@ -0,0 +1,36 @@
+from couchers.email.smtp import email_proto_to_message
+from couchers.proto.internal.jobs_pb2 import EmailAttachmentV2, SendEmailPayload
+
+
+def test_verbatim_attachment_headers():
+    """
+    Test that specified Content-Type/Content-Disposition are preserved verbatim (with original quoting).
+    This ensures we can create attachments compatible with older email clients.
+    """
+
+    send_email_payload = SendEmailPayload(
+        sender_name="alice",
+        sender_email="alice@couchers.org",
+        recipient="bob@couchers.org",
+        subject="greeting",
+        plain="hello",
+        attachments=[
+            EmailAttachmentV2(
+                data=bytes([0, 255]),  # Force base64 encoding
+                content_type='maintype/subtype; quoted-header="value"; unquoted-header=value',
+                content_disposition="attachment",
+            )
+        ],
+    )
+
+    email_message, _ = email_proto_to_message(send_email_payload, couchers_id="42")
+    smtp_str = email_message.as_string()
+
+    expected_snippet = """
+Content-Type: maintype/subtype; quoted-header="value"; unquoted-header=value
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment
+MIME-Version: 1.0
+    """.strip()
+
+    assert expected_snippet in smtp_str