Backend/emails: Use user content as the preview line (#9122)

Author: tristanlabelle

app/backend/src/couchers/email/emails.py

@@ -23,6 +23,7 @@
 from couchers.email.locales import get_emails_i18next
 from couchers.i18n import LocalizationContext
 from couchers.i18n.localize import format_phone_number
+from couchers.markup import markdown_to_plaintext
 from couchers.notifications.quick_links import generate_quick_decline_link
 from couchers.proto import conversations_pb2, events_pb2, notification_data_pb2
 from couchers.utils import now, to_aware_datetime
@@ -271,6 +272,9 @@ def get_subject_line(self, loc_context: LocalizationContext) -> str:
             loc_context, ".subject", {"author": self.author.name, "group": self.group_chat_title or ""}
         )
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return self.text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"author": self.author.name, "group": self.group_chat_title or ""})
@@ -327,6 +331,11 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject")
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        if len(self.entries) != 1:
+            return None
+        return self.entries[0].latest_message_text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         for entry in self.entries:
@@ -391,6 +400,9 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"author": self.author.name, "title": self.title})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return markdown_to_plaintext(self.markdown_text)
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(
@@ -451,6 +463,9 @@ def get_subject_line(self, loc_context: LocalizationContext) -> str:
             loc_context, ".subject", {"author": self.author.name, "discussion_title": self.discussion_title}
         )
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return markdown_to_plaintext(self.markdown_text)
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(
@@ -503,9 +518,6 @@ class DonationReceivedEmail(EmailBase):
     def string_key_base(self) -> str:
         return "donation_received"
 
-    def get_preview_line(self, loc_context: LocalizationContext) -> str:
-        return self._localize(loc_context, ".thanks_amount", {"amount": self.amount})
-
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context, standard_closing=False)
         builder.para(".thanks_amount", {"amount": self.amount})
@@ -680,6 +692,9 @@ def get_subject_line(self, loc_context: LocalizationContext) -> str:
             loc_context, ".subject", {"user": self.inviting_user.name, "title": self.event_info.title}
         )
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return markdown_to_plaintext(self.event_info.description_markdown)
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         if self.community_name:
@@ -882,6 +897,9 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"author": self.author.name, "title": self.event_info.title})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return markdown_to_plaintext(self.comment_markdown)
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"author": self.author.name, "title": self.event_info.title})
@@ -1036,6 +1054,9 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"name": self.from_user.name})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return self.text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"name": self.from_user.name})
@@ -1072,9 +1093,6 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"name": self.befriender.name})
 
-    def get_preview_line(self, loc_context: LocalizationContext) -> str:
-        return self._localize(loc_context, ".body", {"name": self.befriender.name})
-
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"name": self.befriender.name})
@@ -1111,9 +1129,6 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"name": self.new_friend.name})
 
-    def get_preview_line(self, loc_context: LocalizationContext) -> str:
-        return self._localize(loc_context, ".body", {"name": self.new_friend.name})
-
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"name": self.new_friend.name})
@@ -1183,6 +1198,9 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"surfer_name": self.surfer.name})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return self.text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"surfer_name": self.surfer.name})
@@ -1301,6 +1319,9 @@ def string_key_base(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"other_name": self.other_user.name})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return self.text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"other_name": self.other_user.name})
@@ -1519,6 +1540,9 @@ def string_role_subkey(self) -> str:
     def get_subject_line(self, loc_context: LocalizationContext) -> str:
         return self._localize(loc_context, ".subject", {"name": self.from_user.name})
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return self.text
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(f".{self.string_role_subkey}.body", {"name": self.from_user.name})
@@ -2045,6 +2069,9 @@ def get_subject_line(self, loc_context: LocalizationContext) -> str:
             loc_context, ".subject", {"author": self.author.name, "parent_context": self.parent_context}
         )
 
+    def get_preview_line(self, loc_context: LocalizationContext) -> str | None:
+        return markdown_to_plaintext(self.markdown_text)
+
     def get_body_blocks(self, loc_context: LocalizationContext) -> list[EmailBlock]:
         builder = self._body_builder(loc_context)
         builder.para(".body", {"author": self.author.name, "parent_context": self.parent_context})

app/backend/src/couchers/email/rendering.py

@@ -6,11 +6,9 @@
 from dataclasses import asdict, dataclass
 from email.headerregistry import Address
 from functools import cache
-from html import unescape
 from pathlib import Path
 from typing import Any
 
-from markdown_it import MarkdownIt
 from markupsafe import Markup
 
 from couchers.config import config
@@ -19,15 +17,12 @@
 from couchers.email.smtp import embed_html_relative_images
 from couchers.i18n import LocalizationContext
 from couchers.i18n.i18next import SubstitutionDict, full_string_key
+from couchers.markup import html_to_plaintext, markdown_to_html
 from couchers.proto.internal import jobs_pb2
 from couchers.templating import Jinja2Template
 
 template_folder = Path(__file__).parent.parent.parent.parent / "templates" / "v2"
 
-_markdown = MarkdownIt("zero", {"typographer": True}).enable(
-    ["smartquotes", "heading", "hr", "list", "link", "emphasis"]
-)
-
 
 @dataclass(kw_only=True, slots=True)
 class RenderedEmail:
@@ -116,25 +111,11 @@ def _to_plaintext(text: str | Markup) -> str:
     Converts any markup in its plaintext equivalent, allowing reuse of translations that have span-level markup
     like <b> when formatting as plaintext email bodies.
     """
-    if not isinstance(text, Markup):  # Markup derives from str so can't test for isinstance(, str)
+    if isinstance(text, Markup):
+        return html_to_plaintext(text)
+    else:
         return text
 
-    # Convert markup to its plaintext equivalent.
-    # This code is not security-sensitive since we're producing a plaintext string where markup will not be evaluated.
-
-    # Strip/convert any markup since we can't render it in plaintext.
-    text = text.replace("\n", "")  # Newlines are irrelevant in markup
-    text = re.sub(r"<br\s*/?>", "\n", text)  # But <br>'s should be newlines in plaintext
-
-    # Keep the content of span-level markup (assume no nesting)
-    text = re.sub(
-        r"<(?P<name>\w+)(?P<attrs>[^>]*)>(?P<inner>.*?)</(?P=name)>", lambda match: match.group("inner"), text
-    )
-    text = re.sub(r"<\w+[^/>]*/>", "", text)  # Remove any other self-closing tag
-
-    # We've handled tags but still have escapes like "&gt;", convert those to plaintext.
-    return unescape(text)
-
 
 def _get_footer_template_args(footer: EmailFooter, loc_context: LocalizationContext) -> dict[str, Any]:
     i18n = get_emails_i18next()
@@ -264,7 +245,7 @@ def render(
                         )
                     )
                 case QuoteBlock():
-                    args = {"text": Markup(_markdown.render(block.text)) if block.markdown else block.text}
+                    args = {"text": Markup(markdown_to_html(block.text)) if block.markdown else block.text}
                     concats.append(self.quote_block_template.render(args))
                 case ActionBlock():
                     concats.append(self.action_block_template.render(asdict(block)))

app/backend/src/couchers/markup.py

@@ -0,0 +1,62 @@
+from html.parser import HTMLParser
+from typing import Any
+
+from markdown_it import MarkdownIt
+from markupsafe import Markup
+
+# Markdown config should match frontend's MarkdownNoSSR component.
+_markdown = MarkdownIt(
+    "zero",  # Base configuration disables all features
+    options_update={
+        "typographer": True,  # Enable some language-neutral replacement + quotes beautification
+        "breaks": True,  # Convert '\n' in paragraphs into <br>
+    },
+).enable(
+    [
+        "emphasis",  # Process *this* and _that_
+        "heading",  # Headings (#, ##, ...)
+        "hr",  # Horizontal rule
+        "link",  # Process [link](<to> "stuff")
+        "list",  # Lists
+        "newline",  # Process '\n'
+        "smartquotes",  # Convert straight quotation marks to typographic ones
+    ]
+)
+
+
+def markdown_to_html(text: str) -> Markup:
+    return Markup(_markdown.render(text))
+
+
+def markdown_to_plaintext(text: str) -> str:
+    return html_to_plaintext(markdown_to_html(text))
+
+
+def html_to_plaintext(html: str | Markup) -> str:
+    """
+    Renders a plaintext version of HTML by extracting inner HTML and converting entities+newlines.
+    Do not use for sanitization. The resulting string may not be markup-safe.
+    """
+
+    if isinstance(html, Markup):
+        html = str(html)
+
+    converter = _HTMLToPlaintext()
+    converter.feed(html)
+    return converter.plaintext
+
+
+class _HTMLToPlaintext(HTMLParser):
+    plaintext: str
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.plaintext = ""
+
+    def handle_starttag(self, tag: str, attrs: Any) -> None:
+        if tag == "br":
+            self.plaintext += "\n"
+
+    def handle_data(self, data: str) -> None:
+        # Escapes have already been unescaped
+        self.plaintext += data.replace("\n", "")  # Newlines in html are meaningless

app/backend/src/tests/test_markup.py

@@ -0,0 +1,44 @@
+from couchers.markup import html_to_plaintext, markdown_to_html, markdown_to_plaintext
+
+
+def test_markdown_to_html() -> None:
+    def to_para(span: str) -> str:
+        return f"<p>{span}</p>\n"
+
+    assert markdown_to_html("new\nline") == to_para("new<br>\nline")
+    assert markdown_to_html("a & b") == to_para("a &amp; b")
+
+    assert markdown_to_html("**bold**") == to_para("<strong>bold</strong>")
+    assert markdown_to_html("*italic*") == to_para("<em>italic</em>")
+    assert markdown_to_html("_italic_") == to_para("<em>italic</em>")
+    assert markdown_to_html("[link](url)") == to_para('<a href="url">link</a>')
+    assert markdown_to_html('"quoted"') == to_para("“quoted”")
+
+    assert markdown_to_html("<script/>") == to_para("&lt;script/&gt;")
+
+    assert markdown_to_html("# title") == "<h1>title</h1>\n"
+    assert markdown_to_html("- a\n- b") == "<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n"
+    assert markdown_to_html("---") == "<hr>\n"
+
+
+def test_markdown_to_plaintext() -> None:
+    assert markdown_to_plaintext("new\nline") == "new\nline"
+    assert markdown_to_plaintext("a & b") == "a & b"
+
+    assert markdown_to_plaintext("**bold**") == "bold"
+    assert markdown_to_plaintext("_italic_") == "italic"
+    assert markdown_to_plaintext("[link](url)") == "link"
+
+    # By design since plaintext should never be interpreted as markup.
+    assert markdown_to_plaintext("<script/>") == "<script/>"
+
+    assert markdown_to_plaintext("# title") == "title"
+
+
+def test_html_to_plaintext() -> None:
+    assert html_to_plaintext("new<br>line") == "new\nline"
+    assert html_to_plaintext("entity&excl;") == "entity!"
+    assert html_to_plaintext("<b>stripped</b>") == "stripped"
+    assert html_to_plaintext("a<script/>b") == "ab"
+    assert html_to_plaintext('<a href="https://example.com">attributes</a>') == "attributes"
+    assert html_to_plaintext("</b>malformed<a>") == "malformed"