Look up the IDNA-encoded TLD when consider_tld is set

The IANA list stores internationalized TLDs in punycode (XN--P1AI),
but domain() checked the raw Unicode TLD against it. Unicode domains
such as пример.рф were rejected with consider_tld=True while their
punycode form was accepted. hostname() and url() inherit this through
domain().

Run the TLD check after IDNA encoding, on the encoded TLD.

Author: DevKaurav-14

src/validators/domain.py

@@ -93,9 +93,6 @@ def domain(
     if not value:
         return False
 
-    if consider_tld and not _IanaTLD.check(value.rstrip(".").rsplit(".", 1)[-1].upper()):
-        return False
-
     try:
         if _relies_on_idna_mapping(value):
             return False
@@ -108,6 +105,11 @@ def domain(
             # ref: RFC 1035, presentation form is limited to 253 characters
             return False
 
+        # IANA lists IDN TLDs in their ASCII (punycode) form,
+        # so look up the encoded TLD, not the raw one.
+        if consider_tld and not _IanaTLD.check(encoded.rstrip(".").rsplit(".", 1)[-1].upper()):
+            return False
+
         return not re.search(r"\s|__+", value) and re.match(
             # First character of the domain
             rf"^(?:[a-z0-9{service_record}]"

tests/test_domain.py

@@ -47,6 +47,9 @@ def test_returns_true_on_valid_domain(value: str, rfc_1034: bool, rfc_2782: bool
         ("example_.com", True, False, True),
         ("somerandomexample.xn--fiqs8s", True, False, False),
         ("somerandomexample.onion", True, False, False),
+        ("пример.рф", True, False, False),
+        ("пример.рф.", True, True, False),
+        ("例子.中国", True, False, False),
     ],
 )
 def test_returns_true_on_valid_top_level_domain(
@@ -100,6 +103,7 @@ def test_returns_failed_validation_on_invalid_domain(value: str, rfc_1034: bool,
         ("_example.#13", True, False, True),
         ("example_.fo-ul", True, False, True),
         ("somerandomexample.xn-n-fiqs8s", True, False, False),
+        ("пример.рфы", True, False, False),
     ],
 )
 def test_returns_failed_validation_invalid_top_level_domain(

tests/test_hostname.py

@@ -137,3 +137,29 @@ def test_returns_failed_validation_on_non_ascii_simple_hostname(value: str):
 def test_returns_failed_validation_on_idna_mapped_lookalike_hostname(value: str):
     """Test returns failed validation on hostnames that only IDNA 2003 mapping makes valid."""
     assert isinstance(hostname(value), ValidationError)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "example.com:443",
+        "xn--e1afmkfd.xn--p1ai",
+        "пример.рф",
+        "пример.рф:8080",
+    ],
+)
+def test_returns_true_on_valid_top_level_domain_hostname(value: str):
+    """Test returns true on hostname with a TLD allowed by IANA."""
+    assert hostname(value, consider_tld=True)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "example.flat:443",
+        "пример.рфы",
+    ],
+)
+def test_returns_failed_validation_on_invalid_top_level_domain_hostname(value: str):
+    """Test returns failed validation on hostname with a TLD not allowed by IANA."""
+    assert isinstance(hostname(value, consider_tld=True), ValidationError)

tests/test_url.py

@@ -266,3 +266,29 @@ def test_url_accepts_percent_encoded_controls(encoded: str, template: str):
 def test_returns_failed_validation_on_idna_mapped_lookalike_url(value: str):
     """Test returns failed validation on url hosts that only IDNA 2003 mapping makes valid."""
     assert isinstance(url(value, simple_host=True), ValidationError)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "https://example.com/path",
+        "https://xn--e1afmkfd.xn--p1ai/",
+        "https://пример.рф/",
+        "http://例子.中国:8080/index.html",
+    ],
+)
+def test_returns_true_on_valid_top_level_domain_url(value: str):
+    """Test returns true on url with a TLD allowed by IANA."""
+    assert url(value, consider_tld=True)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "https://example.flat/",
+        "https://пример.рфы/",
+    ],
+)
+def test_returns_failed_validation_on_invalid_top_level_domain_url(value: str):
+    """Test returns failed validation on url with a TLD not allowed by IANA."""
+    assert isinstance(url(value, consider_tld=True), ValidationError)