validate idna-encoded host against reg-name in _encode_host

CHANGES/955.bugfix.rst

@@ -0,0 +1,7 @@
+Fixed non-ASCII hosts bypassing reg-name validation. The IDNA 2003 fallback
+used by host encoding did not enforce the :rfc:`3986#section-3.2.2`
+``reg-name`` grammar, so reserved characters such as ``@``, ``:`` or ``/`` in a
+non-ASCII host (e.g. ``URL.build(host="user:pass@bücher.example")``) survived
+encoding and produced a misleading authority. The encoded host is now
+validated and such values raise :exc:`ValueError`
+-- by :user:`alhudz`.

tests/test_url_update_netloc.py

@@ -206,6 +206,28 @@ def test_with_invalid_host(host: str, is_authority: bool) -> None:
         url.with_host(host=host)
 
 
+@pytest.mark.parametrize(
+    ("host", "is_authority"),
+    [
+        ("user:pass@истори.рф", True),
+        ("evil.com@истори.рф", True),
+        ("a/b.истори.рф", False),
+        ("исто?ри.рф", False),
+        ("исто#ри.рф", False),
+    ],
+)
+def test_with_invalid_non_ascii_host(host: str, is_authority: bool) -> None:
+    # The IDNA 2003 codec fallback in _idna_encode() does not enforce the
+    # reg-name grammar, so reserved characters could previously survive in a
+    # non-ASCII host and produce a confusable authority (gh#955).
+    url = URL("http://example.com:123")
+    match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)"
+    if is_authority:
+        match += ", if .* use 'authority' instead of 'host'"
+    with pytest.raises(ValueError, match=f"{match}$"):
+        url.with_host(host=host)
+
+
 def test_with_host_percent_encoded() -> None:
     url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%80@example.com:123")
     url2 = url.with_host("%cf%80.org")

yarl/_url.py

@@ -1635,23 +1635,29 @@ def _encode_host(host: str, validate_host: bool) -> str:
 
     # IDNA encoding is slow, skip it for ASCII-only strings
     if host.isascii():
-        # Check for invalid characters explicitly; _idna_encode() does this
-        # for non-ascii host names.
         host = host.lower()
-        if validate_host and (invalid := NOT_REG_NAME.search(host)):
-            value, pos, extra = invalid.group(), invalid.start(), ""
-            if value == "@" or (value == ":" and "@" in host[pos:]):
-                # this looks like an authority string
-                extra = (
-                    ", if the value includes a username or password, "
-                    "use 'authority' instead of 'host'"
-                )
-            raise ValueError(
-                f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
-            ) from None
-        return host
-
-    return _idna_encode(host)
+    else:
+        # _idna_encode() falls back to the stdlib 'idna' codec for hosts the
+        # idna package rejects (IDNA 2003 compatibility, see #152). That codec
+        # does not enforce the reg-name grammar, so reserved characters such
+        # as '@', ':' or '/' can survive in the encoded (now ASCII) host and
+        # produce a confusable authority. Validate the encoded result below.
+        host = _idna_encode(host)
+
+    # Check for invalid characters explicitly; both branches above yield an
+    # ASCII host, so the reg-name check applies uniformly.
+    if validate_host and (invalid := NOT_REG_NAME.search(host)):
+        value, pos, extra = invalid.group(), invalid.start(), ""
+        if value == "@" or (value == ":" and "@" in host[pos:]):
+            # this looks like an authority string
+            extra = (
+                ", if the value includes a username or password, "
+                "use 'authority' instead of 'host'"
+            )
+        raise ValueError(
+            f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
+        ) from None
+    return host
 
 
 @rewrite_module