Backend/i18n: Localize region & spoken language names (#9173)

Author: tristanlabelle

app/backend/src/couchers/i18n/context.py

@@ -21,6 +21,8 @@
     localize_list,
     localize_time,
     localize_timezone,
+    try_localize_language_name_from_iso639,
+    try_localize_region_name_from_iso3166,
 )
 from couchers.models.users import User
 from couchers.utils import to_timezone
@@ -65,6 +67,12 @@ def __setattr__(self, name: str, value: Any) -> None:
     def localized_timezone(self) -> str:
         return localize_timezone(self.timezone, self.babel_locale)
 
+    def try_localize_language_name_from_iso639(self, code: str, standalone: bool = False) -> str | None:
+        return try_localize_language_name_from_iso639(code, self.babel_locale, standalone=standalone)
+
+    def try_localize_region_name_from_iso3166(self, code: str) -> str | None:
+        return try_localize_region_name_from_iso3166(code, self.babel_locale)
+
     def localize_string(
         self, key: str, *, i18next: I18Next | None = None, substitutions: dict[str, str | int] | None = None
     ) -> str:

app/backend/src/couchers/i18n/localize.py

@@ -14,6 +14,7 @@
 from babel.lists import format_list
 
 from couchers.i18n.locales import DEFAULT_LOCALE, get_main_i18next
+from couchers.resources import get_region_code_iso3166_alpha3_to_alpha2
 
 
 def localize_string(lang: str | None, key: str, *, substitutions: Mapping[str, str | int] | None = None) -> str:
@@ -35,6 +36,43 @@ def localize_list(items: Sequence[str], locale: babel.Locale) -> str:
     return format_list(items, locale=locale)
 
 
+def try_localize_language_name_from_iso639(code: str, locale: babel.Locale, standalone: bool = False) -> str | None:
+    """
+    Attempts to localize the name of a language expressed as an ISO639 code.
+
+    Args:
+        code: The ISO639 language code.
+        locale: The locale to render the language name in.
+        standalone: The result won't be part of a larger sentence and should be capitalized if the language has capitals.
+
+    Returns:
+        The localized name, or None if no localized name is available.
+    """
+    try:
+        name = babel.Locale.parse(code).get_language_name(locale)
+        if name is None:
+            return None
+        if standalone:
+            # The Unicode CLDR returns a casing that allows embedding in a larger sentence, e.g. "español".
+            # If we're displaying the language name on its own, capitalize its first letter if applicable.
+            # An LLM prompt revealed that this holds for all major languages.
+            # It is a no-op for scripts that don't have capital letters.
+            name = name[:1].title() + name[1:]
+        return name
+    except (ValueError, babel.UnknownLocaleError):
+        return None
+
+
+def try_localize_region_name_from_iso3166(code: str, locale: babel.Locale) -> str | None:
+    """
+    Gets a region name specified as an ISO3166 alpha2 or alpha3 code, localized in the given locale.
+    """
+    # The Unicode CLDR uses alpha2 codes as keys (all alpha3 codes have a corresponding alpha2 code)
+    code = get_region_code_iso3166_alpha3_to_alpha2().get(code, code)
+    region_name: str | None = locale.territories.get(code, None)
+    return region_name
+
+
 def localize_date(
     value: date, locale: babel.Locale, *, abbrev: bool = False, with_year: bool = True, with_day_of_week: bool = False
 ) -> str:

app/backend/src/couchers/resources.py

@@ -1,6 +1,7 @@
 import functools
 import json
 import logging
+from collections.abc import Mapping
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, cast
@@ -36,14 +37,26 @@ def get_icon(name: str) -> str:
 
 
 @functools.cache
-def get_region_dict() -> dict[str, str]:
+def get_region_dict() -> Mapping[str, str]:
     """
     Get a list of allowed regions as a dictionary of {alpha3: name}.
     """
     with session_scope() as session:
         return {region.code: region.name for region in session.execute(select(Region)).scalars().all()}
 
 
+@functools.cache
+def get_region_code_iso3166_alpha3_to_alpha2() -> Mapping[str, str]:
+    """
+    Gets a best-effort mapping table from ISO-3166 alpha3 to alpha2 codes.
+    The alpha3 keys of get_region_dict() are not guaranteed to be present in this table.
+    Valid region codes, this mapping, and localized strings are all versioned separately (database / repo / CLDR).
+    """
+    with open(resources_folder / "regions.json", "r") as f:
+        json_regions = json.load(f)
+    return {region["alpha3"]: region["alpha2"] for region in json_regions if "alpha2" in region}
+
+
 def region_is_allowed(code: str) -> bool:
     """
     Check a region code is valid
@@ -52,7 +65,7 @@ def region_is_allowed(code: str) -> bool:
 
 
 @functools.cache
-def get_language_dict() -> dict[str, str]:
+def get_language_dict() -> Mapping[str, str]:
     """
     Get a list of allowed languages as a dictionary of {code: name}.
     """
@@ -61,7 +74,7 @@ def get_language_dict() -> dict[str, str]:
 
 
 @functools.cache
-def get_badge_data() -> dict[str, Any]:
+def get_badge_data() -> Mapping[str, Any]:
     """
     Get a list of profile badges in form {id: Badge}
     """
@@ -83,7 +96,7 @@ class Badge:
 
 
 @functools.cache
-def get_badge_dict() -> dict[str, Badge]:
+def get_badge_dict() -> Mapping[str, Badge]:
     """
     Get a list of profile badges in form {id: Badge}
     """
@@ -92,7 +105,7 @@ def get_badge_dict() -> dict[str, Badge]:
 
 
 @functools.cache
-def get_static_badge_dict() -> dict[str, list[int]]:
+def get_static_badge_dict() -> Mapping[str, list[int]]:
     """
     Get a list of static badges in form {id: list(user_ids)}
     """
@@ -124,7 +137,7 @@ def get_postcard_font() -> bytes:
 
 
 @functools.cache
-def get_postcard_metadata() -> dict[str, Any]:
+def get_postcard_metadata() -> Mapping[str, Any]:
     """
     Returns the postcard metadata (coordinates, sizes, etc.) from postcard-metadata.json.
     """

app/backend/src/couchers/servicers/resources.py

@@ -48,7 +48,11 @@ def GetRegions(
     ) -> resources_pb2.GetRegionsRes:
         return resources_pb2.GetRegionsRes(
             regions=[
-                resources_pb2.Region(alpha3=alpha3, name=name) for alpha3, name in sorted(get_region_dict().items())
+                resources_pb2.Region(
+                    alpha3=alpha3,
+                    name=context.localization.try_localize_region_name_from_iso3166(alpha3) or english_name,
+                )
+                for alpha3, english_name in sorted(get_region_dict().items())
             ]
         )
 
@@ -57,7 +61,12 @@ def GetLanguages(
     ) -> resources_pb2.GetLanguagesRes:
         return resources_pb2.GetLanguagesRes(
             languages=[
-                resources_pb2.Language(code=code, name=name) for code, name in sorted(get_language_dict().items())
+                resources_pb2.Language(
+                    code=code,
+                    name=context.localization.try_localize_language_name_from_iso639(code, standalone=True)
+                    or english_name,
+                )
+                for code, english_name in sorted(get_language_dict().items())
             ]
         )
 

app/backend/src/tests/fixtures/sessions.py

@@ -153,9 +153,10 @@ class FakeChannel:
     instances when handlers are invoked.
     """
 
-    def __init__(self, token: str | None = None):
+    def __init__(self, token: str | None = None, *, locale: str | None = None):
         self.handlers: dict[str, Any] = {}
         self._token = token
+        self._locale = locale or DEFAULT_LOCALE
         self._pool = get_descriptor_pool()
 
     def add_generic_rpc_handlers(self, generic_rpc_handlers: Any):
@@ -188,7 +189,7 @@ def fake_handler(request):
                     is_api_key=False,
                     token=self._token if auth_info else None,
                     localization=LocalizationContext(
-                        locale=(auth_info and auth_info.ui_language_preference) or DEFAULT_LOCALE,
+                        locale=(auth_info and auth_info.ui_language_preference) or self._locale,
                         timezone=ZoneInfo((auth_info and auth_info.timezone) or "Etc/UTC"),
                     ),
                     sofa="test_sofa_cookie_value",
@@ -528,7 +529,7 @@ def bugs_session(token: str | None = None):
 
 
 @contextmanager
-def resources_session():
-    channel = FakeChannel()
+def resources_session(*, locale: str | None = None):
+    channel = FakeChannel(locale=locale)
     resources_pb2_grpc.add_ResourcesServicer_to_server(Resources(), channel)
     yield resources_pb2_grpc.ResourcesStub(channel)

app/backend/src/tests/test_localize.py

@@ -3,14 +3,46 @@
 
 import babel
 
-from couchers.i18n.localize import localize_date, localize_datetime, localize_list, localize_time, localize_timezone
+from couchers.i18n.localize import (
+    localize_date,
+    localize_datetime,
+    localize_list,
+    localize_time,
+    localize_timezone,
+    try_localize_language_name_from_iso639,
+    try_localize_region_name_from_iso3166,
+)
 
 babel_en = babel.Locale.parse("en")
 babel_es = babel.Locale.parse("es")
 babel_fr = babel.Locale.parse("fr")
 babel_zh = babel.Locale.parse("zh")
 
 
+def test_localize_language_name() -> None:
+    assert try_localize_language_name_from_iso639("en", babel_en) == "English"
+    assert try_localize_language_name_from_iso639("es", babel_en) == "Spanish"
+
+    assert try_localize_language_name_from_iso639("en", babel_es) == "inglés"
+    assert try_localize_language_name_from_iso639("es", babel_es) == "español"
+
+    assert try_localize_language_name_from_iso639("en", babel_es, standalone=True) == "Inglés"  # Sentence case
+    assert try_localize_language_name_from_iso639("eng", babel_es) == "inglés"  # ISO639-3 code
+    assert try_localize_language_name_from_iso639("xx", babel_en) is None
+
+
+def test_localize_region_name() -> None:
+    assert try_localize_region_name_from_iso3166("US", babel_en) == "United States"
+    assert try_localize_region_name_from_iso3166("DE", babel_en) == "Germany"
+
+    assert try_localize_region_name_from_iso3166("US", babel_es) == "Estados Unidos"
+    assert try_localize_region_name_from_iso3166("DE", babel_es) == "Alemania"
+
+    assert try_localize_region_name_from_iso3166("USA", babel_en) == "United States"  # alpha3 code
+
+    assert try_localize_region_name_from_iso3166("xx", babel_en) is None
+
+
 def test_localize_date() -> None:
     assert localize_date(date(2000, 1, 2), babel_en) == "January 2, 2000"
     assert localize_date(date(2000, 1, 2), babel_es, abbrev=True) == "2 ene 2000"

app/backend/src/tests/test_resources.py

@@ -35,6 +35,12 @@ def test_GetRegions(db):
         assert ("SWE", "Sweden") in regions_list
         assert ("???", "Nonexistent region") not in regions_list
 
+    with resources_session(locale="es") as api:
+        regions = api.GetRegions(empty_pb2.Empty()).regions
+        regions_list = [(r.alpha3, r.name) for r in regions]
+        assert ("FIN", "Finlandia") in regions_list
+        assert ("FIN", "Finland") not in regions_list
+
 
 def test_GetLanguages(db):
     with resources_session() as api:
@@ -44,6 +50,12 @@ def test_GetLanguages(db):
         assert ("swe", "Swedish") in languages_list
         assert ("???", "Nonexistent language") not in languages_list
 
+    with resources_session(locale="es") as api:
+        languages = api.GetLanguages(empty_pb2.Empty()).languages
+        languages_list = [(r.code, r.name) for r in languages]
+        assert ("swe", "Sueco") in languages_list
+        assert ("swe", "Swedish") not in languages_list
+
 
 def test_GetBadges(db):
     with resources_session() as api:

app/web/features/profile/hooks/useLanguages.ts

@@ -1,10 +1,12 @@
 import { useQuery } from "@tanstack/react-query";
 import { languagesKey } from "features/queryKeys";
+import { useTranslation } from "i18n";
 import { service } from "service";
 
 export const useLanguages = () => {
+  const locale = useTranslation().i18n.language;
   const { data: languages, ...rest } = useQuery({
-    queryKey: [languagesKey],
+    queryKey: [languagesKey, locale],
     queryFn: () =>
       service.resources
         .getLanguages()

app/web/features/profile/hooks/useRegions.ts

@@ -1,10 +1,12 @@
 import { useQuery } from "@tanstack/react-query";
 import { regionsKey } from "features/queryKeys";
+import { useTranslation } from "i18n";
 import { service } from "service";
 
 export const useRegions = () => {
+  const locale = useTranslation().i18n.language;
   const { data: regions, ...rest } = useQuery({
-    queryKey: [regionsKey],
+    queryKey: [regionsKey, locale],
     queryFn: () =>
       service.resources
         .getRegions()