gh-155292: Don't consider Unicode codepoint attributes outside RFC 3454 (GH-155293)

Due to a bug, some Unicode codepoint attributes were considered
for characters not yet defined in Unicode 3.2.0 or attributes
which changed in later Unicode versions. RFC 3454 (StringPrep)
requires using Unicode 3.2.0 strictly.


Co-authored-by: Stan Ulbrych <89152624+stanfromireland@users.noreply.github.com>
Co-authored-by: Petr Viktorin <encukou@gmail.com>

Author: 3 people

Lib/stringprep.py

@@ -1695,6 +1695,15 @@ def test_builtin_encode(self):
         self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
         self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
 
+    @support.subTests(['unicode', 'encoded'], [
+        ('\N{CHEROKEE LETTER A}\N{CHEROKEE LETTER A}', b"xn--58da"),
+        ('\N{GEORGIAN CAPITAL LETTER AN}.', b"xn--7md."),
+        ('\N{CYRILLIC LETTER PALOCHKA}.example', b"xn--d5a.example"),
+        ('\N{ROMAN NUMERAL REVERSED ONE HUNDRED}.example.', b"xn--q5g.example."),
+    ])
+    def test_new_unicode_case_folding(self, unicode, encoded):
+        self.assertEqual(unicode.encode("idna"), encoded)
+
     def test_builtin_encode_invalid(self):
         for case, expected in self.invalid_encode_testcases:
             with self.subTest(case=case, expected=expected):

Lib/test/test_codecs.py

@@ -321,7 +321,7 @@ def test_category(self):
         self.assertRaises(TypeError, self.db.category, 'xx')
 
     def test_bidirectional(self):
-        self.assertEqual(self.db.bidirectional('\uFFFE'), 'BN')
+        self.assertEqual(self.db.bidirectional('\uFFFE'), '' if self.old else 'BN')
         self.assertEqual(self.db.bidirectional(' '), 'WS')
         self.assertEqual(self.db.bidirectional('A'), 'L')
         self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
@@ -350,15 +350,13 @@ def test_bidirectional(self):
         self.assertRaises(TypeError, self.db.bidirectional, 'xx')
 
     def test_bidirectional_unassigned(self):
-        if self.old:
-            return
-        self.assertEqual(self.db.bidirectional('\u0378'), 'L')
-        self.assertEqual(self.db.bidirectional('\u077F'), 'AL')
-        self.assertEqual(self.db.bidirectional('\u20CF'), 'ET')
-        self.assertEqual(self.db.bidirectional('\u0590'), 'R')
-        self.assertEqual(self.db.bidirectional('\uFFFF'), 'BN')
-        self.assertEqual(self.db.bidirectional('\U0001FFFE'), 'BN')
-        self.assertEqual(self.db.bidirectional('\U00010D01'), 'AL')
+        self.assertEqual(self.db.bidirectional('\u0378'), '' if self.old else 'L')
+        self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
+        self.assertEqual(self.db.bidirectional('\u20CF'), '' if self.old else 'ET')
+        self.assertEqual(self.db.bidirectional('\u0590'), '' if self.old else 'R')
+        self.assertEqual(self.db.bidirectional('\uFFFF'), '' if self.old else 'BN')
+        self.assertEqual(self.db.bidirectional('\U0001FFFE'), '' if self.old else 'BN')
+        self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')
 
     def test_decomposition(self):
         self.assertEqual(self.db.decomposition('\uFFFE'),'')
@@ -1104,9 +1102,9 @@ def test_block_invalid_input(self):
 class Unicode_3_2_0_FunctionsTest(unittest.TestCase, BaseUnicodeFunctionsTest):
     db = unicodedata.ucd_3_2_0
     old = True
-    expectedchecksum = ('cb5bbbd1f55b67371e18222b90a8e21c87f16b72'
+    expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47'
                         if quicktest else
-                        '74936dffe949d99203a47e6a66565b2fc337bae7')
+                        '68cd01e2c680b851c1fcab012efb5635b2229c2b')
 
 
 class UnicodeMiscTest(unittest.TestCase):

Lib/test/test_unicodedata.py

@@ -0,0 +1,2 @@
+Change the :mod:`stringprep` module and :mod:`encodings.idna` codec to not
+consider Unicode codepoint attributes beyond those defined in :rfc:`3454`.