gh-155985: Do not fuse a set difference under IGNORECASE with LOCALE

_fuse_difference() rewrites <flat charset A> (?<![B]) into the single
charset [NEGATE] B [NEGATE] A.  That is sound where the engine walks a
set once, but SRE(charset_loc_ignore) walks it twice, once with the
lowercased character and once with the uppercased one, and matches if
either walk does.  The second walk starts with the NEGATE polarity
reset, so a character that matches B in one case and A in the other
escapes the subtraction.

The pass did not see the compile flags, so it could not tell the two
apart.  Thread the effective flags through optimize() and _walk(),
combining them for a scoped group, and skip the fusion when IGNORECASE
and LOCALE are both set, the condition _compiler.py already uses to
emit IN_LOC_IGNORE.

Author: fedonman

Lib/re/_compiler.py

@@ -221,7 +221,7 @@ def _code(p, flags):
     flags = p.state.flags | flags
 
     # run the optimizer passes over the parsed pattern
-    optimize(p)
+    optimize(p, flags)
 
     code = []
 

Lib/re/_optimizer.py

@@ -467,10 +467,12 @@ def _fuse_branch(av):
             items += cs
     return items if tail is None else items + tail
 
-def _fuse_difference(data):
+def _fuse_difference(data, flags):
     # Replace  <flat charset A> (?<![B1]) (?<![B2]) ...  with the single charset
     # [NEGATE] B1 B2 ... [NEGATE] A.  Each negative lookbehind over a flat
     # charset subtracts its set from the character A matches.
+    if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE:
+        return
     out = []
     head = None        # _flat_items(A) for the fused difference now at out[-1]
     subtrahend = None  # its accumulated B items, or None when not fusing
@@ -493,17 +495,20 @@ def _fuse_difference(data):
         out.append((op, av))
     data[:] = out
 
-def _walk(seq):
+def _walk(seq, flags):
     for i, (op, av) in enumerate(seq):
+        subflags = flags
+        if op is SUBPATTERN:
+            subflags = _combine_flags(flags, av[1], av[2])
         for sub in _subpatterns(op, av):
-            _walk(sub.data)
+            _walk(sub.data, subflags)
         if op is BRANCH:
             items = _fuse_branch(av)
             if items is not None:
                 seq[i] = (IN, items)
-    _fuse_difference(seq)
+    _fuse_difference(seq, flags)
 
-def optimize(pattern):
+def optimize(pattern, flags):
     """Rewrite a parsed pattern in place and return it."""
-    _walk(pattern.data)
+    _walk(pattern.data, flags)
     return pattern

Lib/test/test_re.py

@@ -1558,6 +1558,22 @@ def test_set_operations(self):
         self.assertEqual(w.filename, __file__)
         self.assertEqual(re.findall(r'[~~1]', s), list('1~'))
 
+    def test_difference_not_fused_with_locale_ignorecase(self):
+        # IN_LOC_IGNORE tests the whole set once per locale case, so a fused
+        # difference, which relies on an embedded NEGATE, cannot be used.
+        IL = re.IGNORECASE | re.LOCALE
+        # [b] matches b'B' under these flags, so the lookbehind must reject it.
+        self.assertTrue(re.fullmatch(rb'[b]', b'B', IL))
+        self.assertIsNone(re.fullmatch(rb'\w(?<!b)', b'B', IL))
+        # IGNORECASE may also be scoped to the group.
+        self.assertIsNone(re.fullmatch(rb'(?i:\w(?<!b))', b'B', re.LOCALE))
+        # [0-m] matches b'N', so the difference operator must exclude it.
+        self.assertTrue(re.fullmatch(rb'[0-m]', b'N', IL))
+        self.assertIsNone(re.fullmatch(rb'[A-z--[0-m]]', b'N', IL))
+        # Without both flags the fusion is still applied.
+        self.assertTrue(re.fullmatch(rb'\w(?<!b)', b'B'))
+        self.assertTrue(re.fullmatch(rb'[a-c--[b]]', b'a'))
+
     def test_search_coverage(self):
         self.assertEqual(re.search(r"\s(b)", " b").group(1), "b")
         self.assertEqual(re.search(r"a\s", "a ").group(0), "a ")