Backend/i18n: Externalize fallbacks languages from I18Next

Author: tristanlabelle

app/backend/src/couchers/email/blocks.py

@@ -51,7 +51,7 @@ def _body_builder(
         standard_closing: bool = True,
         security_warning: bool = False,
     ) -> EmailBlocksBuilder:
-        builder = EmailBlocksBuilder(locale=loc_context.locale, string_key_base=self.string_key_base)
+        builder = EmailBlocksBuilder(locales=loc_context.locale_list, string_key_base=self.string_key_base)
         if standard_greeting:
             builder.para("generic.greeting_line", {"name": self.user_name})
         if standard_closing:
@@ -78,7 +78,7 @@ def _localize(
         self, loc_context: LocalizationContext, key: str, substitutions: SubstitutionDict | None = None
     ) -> str:
         key = full_string_key(key, relative_base=self.string_key_base)
-        return get_emails_i18next().localize(key, loc_context.locale, substitutions)
+        return loc_context.localize_string(key, i18next=get_emails_i18next(), substitutions=substitutions)
 
 
 @dataclass
@@ -153,13 +153,13 @@ class EmailBlocksBuilder:
     Builder object for constructing a list of localized EmailBlock's to form the body of an email.
     """
 
-    _locale: str
+    _locales: list[str]
     _string_key_base: str
     _blocks: list[EmailBlock]
     _epilogue: list[EmailBlock]
 
-    def __init__(self, locale: str, string_key_base: str):
-        self._locale = locale
+    def __init__(self, locales: list[str], string_key_base: str):
+        self._locales = locales
         self._string_key_base = string_key_base
         self._blocks = []
         self._epilogue = []
@@ -194,11 +194,11 @@ def block(self, block: EmailBlock, epilogue: bool = False) -> Self:
 
     def _text(self, key: str, substitutions: SubstitutionDict | None = None) -> str:
         key = full_string_key(key, relative_base=self._string_key_base)
-        return get_emails_i18next().localize(key, self._locale, substitutions)
+        return get_emails_i18next().localize(key, self._locales, substitutions)
 
     def _markup(self, key: str, substitutions: SubstitutionDict | None = None) -> Markup:
         key = full_string_key(key, relative_base=self._string_key_base)
-        return get_emails_i18next().localize_with_markup(key, self._locale, substitutions)
+        return get_emails_i18next().localize_with_markup(key, self._locales, substitutions)
 
 
 @dataclass(kw_only=True)

app/backend/src/couchers/email/calendar_events.py

@@ -42,12 +42,14 @@ def create_host_request_event(
     event.extra.append(ContentLine(name="SEQUENCE", value=str(sequence)))
 
     if hosting:
-        event.name = get_emails_i18next().localize(
-            "calendar_events.host_requests.title_host", loc_context.locale, {"name": other_name}
+        event.name = loc_context.localize_string(
+            "calendar_events.host_requests.title_host", i18next=get_emails_i18next(), substitutions={"name": other_name}
         )
     else:
-        event.name = get_emails_i18next().localize(
-            "calendar_events.host_requests.title_surfer", loc_context.locale, {"name": other_name}
+        event.name = loc_context.localize_string(
+            "calendar_events.host_requests.title_surfer",
+            i18next=get_emails_i18next(),
+            substitutions={"name": other_name},
         )
 
     # Our to_date is inclusive, iCalendar's DTEND is exclusive (for full-day events)
@@ -76,8 +78,8 @@ def create_host_request_cancellation_calendar(
     host_request: HostRequest, other_name: str, hosting: bool, loc_context: LocalizationContext
 ) -> Calendar:
     event = create_host_request_event(host_request, other_name, hosting, loc_context, sequence=1)
-    event.name = get_emails_i18next().localize(
-        "calendar_events.title_cancelled", loc_context.locale, {"title": event.name}
+    event.name = loc_context.localize_string(
+        "calendar_events.title_cancelled", i18next=get_emails_i18next(), substitutions={"title": event.name}
     )
     event.status = "CANCELLED"
 

app/backend/src/couchers/email/emails.py

@@ -621,7 +621,7 @@ def get_details_block(self, loc_context: LocalizationContext) -> EmailBlock:
         html += time_range_display
         if self.online_link:
             html += "<br>"
-            online_link_text = get_emails_i18next().localize("events.generic.online_link", loc_context.locale)
+            online_link_text = loc_context.localize_string("events.generic.online_link", i18next=get_emails_i18next())
             html += f'<i><a href="{escape(self.online_link)}">{escape(online_link_text)}</a></i>'
         elif self.address:
             html += "<br>"
@@ -633,7 +633,7 @@ def get_description_block(self) -> EmailBlock:
         return QuoteBlock(text=Markup(self.description_markdown), markdown=True)
 
     def get_view_action_block(self, loc_context: LocalizationContext) -> EmailBlock:
-        view_action_text = get_emails_i18next().localize("events.generic.view_action", loc_context.locale)
+        view_action_text = loc_context.localize_string("events.generic.view_action", i18next=get_emails_i18next())
         return ActionBlock(text=view_action_text, target_url=self.view_url)
 
     @classmethod

app/backend/src/couchers/email/rendering.py

@@ -77,10 +77,10 @@ def render_plaintext_body(*, blocks: list[EmailBlock], footer: EmailFooter, loc_
             case ParaBlock():
                 concat.append(_to_plaintext(block.text))
             case UserBlock():
-                line = get_emails_i18next().localize(
+                line = loc_context.localize_string(
                     "plaintext_formats.user",
-                    loc_context.locale,
-                    {"name": block.info.name, "age": str(block.info.age), "city": block.info.city},
+                    i18next=get_emails_i18next(),
+                    substitutions={"name": block.info.name, "age": str(block.info.age), "city": block.info.city},
                 )
                 concat.append(line)
                 if block.comment:
@@ -90,8 +90,10 @@ def render_plaintext_body(*, blocks: list[EmailBlock], footer: EmailFooter, loc_
                 for line in block.text.splitlines():
                     concat.append(f"> {line}")
             case ActionBlock():
-                line = get_emails_i18next().localize(
-                    "plaintext_formats.action", loc_context.locale, {"text": block.text, "url": block.target_url}
+                line = loc_context.localize_string(
+                    "plaintext_formats.action",
+                    i18next=get_emails_i18next(),
+                    substitutions={"text": block.text, "url": block.target_url},
                 )
                 concat.append(line)
             case _:

app/backend/src/couchers/i18n/context.py

@@ -6,11 +6,11 @@
 import babel
 from google.protobuf.timestamp_pb2 import Timestamp
 
-from couchers.i18n.i18next import I18Next
+from couchers.i18n.i18next import I18Next, SubstitutionDict
 from couchers.i18n.locales import (
     DEFAULT_LOCALE,
     get_babel_locale,
-    get_locale_fallbacks,
+    get_locale_chain,
     get_main_i18next,
     is_supported_locale,
 )
@@ -48,7 +48,7 @@ def __init__(self, locale: str, timezone: tzinfo) -> None:
             raise ValueError(f"Unsupported locale {locale}.")
 
         self.locale = locale
-        self.locale_list = [self.locale] + get_locale_fallbacks(self.locale)
+        self.locale_list = get_locale_chain(self.locale)
         self.timezone = timezone
         self.babel_locale = get_babel_locale(locale)
 
@@ -64,10 +64,10 @@ def localized_timezone(self) -> str:
         return localize_timezone(self.timezone, self.babel_locale)
 
     def localize_string(
-        self, key: str, *, i18next: I18Next | None = None, substitutions: dict[str, str | int] | None = None
+        self, key: str, *, i18next: I18Next | None = None, substitutions: SubstitutionDict | None = None
     ) -> str:
         i18next = i18next or get_main_i18next()
-        return i18next.localize(key, self.locale, substitutions=substitutions)
+        return i18next.localize(key, self.locale_list, substitutions=substitutions)
 
     def localize_date(
         self, value: date | datetime, *, abbrev: bool = False, with_year: bool = True, with_day_of_week: bool = False

app/backend/src/couchers/i18n/i18next.py

@@ -28,45 +28,41 @@ class I18Next:
     def __init__(self) -> None:
         self.translations_by_locale: dict[str, Translation] = dict()
 
-        # The translation used to look up strings in unsupported locales.
-        self.default_translation: Translation | None = None
-
     def add_translation(self, locale: str, *, json_dict: dict[str, Any] | None = None) -> Translation:
         translation = Translation(babel_locale=babel.Locale.parse(locale, sep="-"))
         self.translations_by_locale[locale] = translation
         if json_dict:
             translation.load_json_dict(json_dict)
         return translation
 
-    def find_string(self, key: str, locale: str, substitutions: SubstitutionDict | None = None) -> String | None:
-        """Find the string that will be localized, applying fallbacks and variant selection."""
-        translation = self.translations_by_locale.get(locale, self.default_translation)
-        candidate_translations = [translation] + translation.fallbacks if translation else []
-        for candidate_translation in candidate_translations:
-            if string := candidate_translation.find_string(key, substitutions):
-                if string.template.can_render(substitutions):
-                    return string
+    def find_string(self, key: str, locales: list[str], substitutions: SubstitutionDict | None = None) -> String | None:
+        """Find the string that will be localized, trying each locale in order."""
+        for locale in locales:
+            if t := self.translations_by_locale.get(locale):
+                if string := t.find_string(key, substitutions):
+                    if string.template.can_render(substitutions):
+                        return string
 
-        raise LocalizationError(locale, key)
+        raise LocalizationError(locales, key)
 
-    def localize(self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None) -> str:
+    def localize(self, string_key: str, locales: list[str], substitutions: SubstitutionDict | None = None) -> str:
         """Finds a translated string in the best matching locale and performs substitutions."""
-        if string := self.find_string(string_key, locale, substitutions):
+        if string := self.find_string(string_key, locales, substitutions):
             return string.render(substitutions)
         else:
-            raise LocalizationError(locale, string_key)
+            raise LocalizationError(locales, string_key)
 
     def localize_with_markup(
-        self, string_key: str, locale: str, substitutions: SubstitutionDict | None = None
+        self, string_key: str, locales: list[str], substitutions: SubstitutionDict | None = None
     ) -> Markup:
         """
         Finds a translated string that might contain markup in the best matching locale,
         and performs substitutions in a way that results in a markup-safe string.
         """
-        if string := self.find_string(string_key, locale, substitutions):
+        if string := self.find_string(string_key, locales, substitutions):
             return string.render_with_markup(substitutions)
         else:
-            raise LocalizationError(locale, string_key)
+            raise LocalizationError(locales, string_key)
 
 
 class Translation:
@@ -76,7 +72,6 @@ def __init__(self, *, babel_locale: babel.Locale) -> None:
         # The Babel library locale for this translation. Used to resolved plural forms.
         self.babel_locale = babel_locale
         self.strings_by_key: dict[str, String] = dict()
-        self.fallbacks: list[Translation] = []
 
     @property
     def locale(self) -> str:
@@ -116,13 +111,13 @@ def find_string(self, key: str, substitutions: SubstitutionDict | None = None) -
     def localize(self, string_key: str, substitutions: SubstitutionDict | None = None) -> str:
         string = self.find_string(string_key, substitutions)
         if string is None:
-            raise LocalizationError(locale=self.locale, string_key=string_key)
+            raise LocalizationError(locales=[self.locale], string_key=string_key)
         return string.render(substitutions)
 
     def localize_with_markup(self, string_key: str, substitutions: SubstitutionDict | None = None) -> Markup:
         string = self.find_string(string_key, substitutions)
         if string is None:
-            raise LocalizationError(locale=self.locale, string_key=string_key)
+            raise LocalizationError(locales=[self.locale], string_key=string_key)
         return string.render_with_markup(substitutions)
 
 
@@ -203,12 +198,12 @@ class StringSegment:
 
 
 class LocalizationError(Exception):
-    """Raised failing to localize a string, e.g. if it is not found in any fallback locale."""
+    """Raised failing to localize a string, e.g. if it is not found in any of the given locales."""
 
-    def __init__(self, locale: str, string_key: str):
-        self.locale = locale
+    def __init__(self, locales: list[str], string_key: str):
+        self.locales = locales
         self.string_key = string_key
-        super().__init__(f"Could not localize string {string_key} for locale {locale}")
+        super().__init__(f"Could not localize string {string_key} for locales {locales}")
 
 
 def full_string_key(key: str, *, relative_base: str | None) -> str:

app/backend/src/couchers/i18n/locales.py

@@ -63,13 +63,13 @@ def to_supported_locale(locale: str) -> str:
     return DEFAULT_LOCALE
 
 
-def get_locale_fallbacks(locale: str) -> list[str]:
-    """Gets the list of locales to which to fallback to if the given one is unavailable."""
+def get_locale_chain(locale: str) -> list[str]:
+    """Gets the ordered list of locales to try when looking up a string, starting with the given locale."""
     if fallback := _LOCALE_FALLBACKS.get(locale):
-        return [fallback, DEFAULT_LOCALE]
+        return [locale, fallback, DEFAULT_LOCALE]
     if locale == DEFAULT_LOCALE:
-        return []
-    return [DEFAULT_LOCALE]
+        return [locale]
+    return [locale, DEFAULT_LOCALE]
 
 
 def get_babel_locale(locale: str) -> babel.Locale:
@@ -99,12 +99,6 @@ def load_locales(directory: Path) -> I18Next:
     default_translation = i18next.translations_by_locale.get(DEFAULT_LOCALE)
     if default_translation is None:
         raise RuntimeError("English translations must be loaded")
-    i18next.default_translation = default_translation
-
-    # Apply fallbacks
-    for translation in i18next.translations_by_locale.values():
-        for fallback_locale in get_locale_fallbacks(translation.locale):
-            translation.fallbacks.append(i18next.translations_by_locale[fallback_locale])
 
     return i18next
 

app/backend/src/couchers/i18n/localize.py

@@ -4,31 +4,13 @@
 """
 
 import re
-from collections.abc import Mapping
 from datetime import date, datetime, time, tzinfo
 from typing import cast
 
 import babel
 import phonenumbers
 from babel.dates import get_datetime_format, get_timezone_name, match_skeleton, parse_pattern
 
-from couchers.i18n.locales import DEFAULT_LOCALE, get_main_i18next
-
-
-def localize_string(lang: str | None, key: str, *, substitutions: Mapping[str, str | int] | None = None) -> str:
-    """
-    Retrieves a translated string and performs substitutions.
-
-    Args:
-        lang: Language code (e.g., "en", "pt-BR"). If None, defaults to the default fallback language ("en")
-        key: The key for the string to be looked up.
-        substitutions: Dictionary of variable substitutions for the string (e.g., {"hours": 24})
-
-    Returns:
-        The translated string with substitutions applied
-    """
-    return get_main_i18next().localize(key, lang or DEFAULT_LOCALE, substitutions)
-
 
 def localize_date(
     value: date, locale: babel.Locale, *, abbrev: bool = False, with_year: bool = True, with_day_of_week: bool = False