jslexer: stop regexp literals from spanning lines

babel/messages/jslexer.py

@@ -27,7 +27,13 @@
 name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE)
 dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE)
 division_re = re.compile(r'/=?')
-regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL)
+# A regexp literal cannot contain an ECMAScript line terminator, and a backslash
+# cannot escape one either: `\` followed by a line terminator is a syntax error
+# inside a regexp literal, so it must not continue the literal on the next line.
+# https://tc39.es/ecma262/multipage/ecmascript-language-lexical-grammar.html#sec-line-terminators
+regex_re = re.compile(
+    r'/(?:[^/\\\r\n\u2028\u2029]*(?:\\[^\r\n\u2028\u2029][^/\\\r\n\u2028\u2029]*)*)/[a-zA-Z]*',
+)
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')

tests/messages/test_js_extract.py

@@ -126,7 +126,10 @@ def test_jsx_extraction(jsx_enabled):
     if jsx_enabled:
         assert messages == EXPECTED_JSX_MESSAGES
     else:
-        assert messages != EXPECTED_JSX_MESSAGES
+        # Without the jsx option the regex literals in the source used to
+        # swallow following messages (issue #640); with that lexer bug
+        # fixed, plain JavaScript extraction finds them all too.
+        assert messages == EXPECTED_JSX_MESSAGES
 
 
 def test_dotted_keyword_extract():
@@ -211,3 +214,20 @@ def test_template_string_with_literal_braces() -> None:
     )
     messages = list(extract.extract("javascript", buf, {"gettext": None}, [], {"parse_template_string": True}))
     assert messages == []
+
+
+@pytest.mark.parametrize("terminator", ["\n", "\r", "\r\n", "\u2028", "\u2029"])
+def test_regexp_literals_do_not_swallow_messages(terminator):
+    # https://github.com/python-babel/babel/issues/640: a bogus fallback
+    # regex token used to span line terminators and silently drop every
+    # gettext() call inside the swallowed region.  Applies to every ECMAScript
+    # line terminator, not just U+000A.
+    source = terminator.join((
+        'alert(gettext("First message"));',
+        'var expr = />/g;',
+        'alert(gettext("Second message"));',
+        "var expr2 = /'/g;",
+        'alert(gettext("Third message"));',
+    )).encode()
+    messages = [m[2] for m in extract.extract_javascript(BytesIO(source), ("gettext",), {}, {})]
+    assert messages == ["First message", "Second message", "Third message"]