fix(async): linearize await inside catch; complete the RegExp test262 worklist

changelog.d/8667-regexp-repeat-matcher.md

@@ -0,0 +1,8 @@
+Fixed RegExp backtracking behavior for quantified capture groups, nullable
+iterations, and captures referenced across lookaround assertions. Perry now
+uses an ECMAScript matcher for the affected patterns while retaining the
+existing linear-time engines for the common path.
+
+Regular-expression replacement also now matches JavaScript strings containing
+lone surrogates as UTF-16 code units without losing their WTF-8 representation.
+Together these changes make all 89 test262 cases tracked by #5897 pass.

changelog.d/8707-await-in-catch-blockwait.md

@@ -0,0 +1 @@
+Fix a deadlock where an `await` inside a `catch` block of an async function or closure compiled to a blocking busy-wait instead of an async suspend. Reached re-entrantly from inside the async-step / async-generator pull cascade (a rejected awaited promise in a `try` routing to a `catch` that itself awaits — a common stream retry/cleanup shape), it monopolised the runtime thread and hung — the natively-compiled Claude Code `-p` streaming path being the motivating case. The async-step throw handler now routes the delivered error into the catch's already-linearized dispatch states (and lets a `throw` raised inside the catch escape correctly) instead of re-inlining the catch body as blocking awaits. (#8681)

crates/perry-runtime/Cargo.toml

@@ -261,7 +261,7 @@ libc.workspace = true
 gimli = { version = "0.34", default-features = false, features = ["read"] }
 rand = "0.10"
 regex = { workspace = true, optional = true }
-regress = { workspace = true, optional = true }
+regress = { workspace = true, features = ["utf16"], optional = true }
 # Taffy — flexbox / grid layout engine for the perry/tui module
 # (#358 Phase 3). Same crate Bevy and Dioxus use; pure Rust, no FFI.
 taffy = { version = "0.13", default-features = false, features = ["std", "flexbox", "taffy_tree"] }

crates/perry-runtime/src/regex.rs

@@ -636,11 +636,16 @@ pub fn is_registered_regex(addr: usize) -> bool {
 
 /// Internal helper: Get string data from StringHeader
 pub(crate) fn string_as_str<'a>(s: *const StringHeader) -> &'a str {
+    unsafe { std::str::from_utf8_unchecked(string_as_bytes(s)) }
+}
+
+/// Internal helper: get the byte payload without assuming it is Unicode
+/// scalar UTF-8. JavaScript strings containing lone surrogates use WTF-8.
+pub(crate) fn string_as_bytes<'a>(s: *const StringHeader) -> &'a [u8] {
     unsafe {
         let len = (*s).byte_len as usize;
         let data = (s as *const u8).add(std::mem::size_of::<StringHeader>());
-        let bytes = std::slice::from_raw_parts(data, len);
-        std::str::from_utf8_unchecked(bytes)
+        std::slice::from_raw_parts(data, len)
     }
 }
 
@@ -1397,19 +1402,39 @@ pub extern "C" fn js_string_replace_regex(
         return js_string_from_str("");
     }
 
-    let str_data = string_as_str(s);
-    let repl_str = if is_valid_ptr(replacement) {
-        string_as_str(replacement)
-    } else {
-        "undefined"
-    };
-
     if !is_valid_regex_ptr(re) {
         // If regex is null, return original string
         return copy_replace_source(s);
     }
 
     unsafe {
+        // The Rust string engines require scalar-value UTF-8, while Perry
+        // stores lone JavaScript surrogates as WTF-8. Match those subjects as
+        // UTF-16 code units with the ECMAScript engine and rebuild the result
+        // through the WTF-8-aware string builder.
+        if (*s).flags & crate::string::STRING_FLAG_HAS_LONE_SURROGATES != 0 {
+            let replacement_bytes = if is_valid_ptr(replacement) {
+                string_as_bytes(replacement)
+            } else {
+                b"undefined"
+            };
+            if let Some(result) = repeat_matcher::replace_wtf8_subject(
+                re,
+                string_as_bytes(s),
+                replacement_bytes,
+                (*re).global,
+            ) {
+                return finish_replace_bytes(&result);
+            }
+        }
+
+        let str_data = string_as_str(s);
+        let repl_str = if is_valid_ptr(replacement) {
+            string_as_str(replacement)
+        } else {
+            "undefined"
+        };
+
         if let Some(repeat_matcher) = lookup_repeat_matcher(re) {
             let result = repeat_matcher.replace(str_data, repl_str, (*re).global);
             return finish_replace_bytes(result.as_bytes());

crates/perry-runtime/src/regex/repeat_matcher.rs

@@ -185,6 +185,7 @@ impl RepeatMatcherRegex {
 #[derive(Clone, Copy)]
 struct GroupFrame {
     captures_before: usize,
+    negative_lookaround: bool,
 }
 
 fn named_capture_end(bytes: &[u8], open: usize) -> Option<usize> {
@@ -204,6 +205,12 @@ fn is_capturing_group(bytes: &[u8], open: usize) -> bool {
     bytes.get(open + 1) != Some(&b'?') || named_capture_end(bytes, open).is_some()
 }
 
+fn is_negative_lookaround(bytes: &[u8], open: usize) -> bool {
+    bytes.get(open + 1) == Some(&b'?')
+        && (bytes.get(open + 2) == Some(&b'!')
+            || (bytes.get(open + 2) == Some(&b'<') && bytes.get(open + 3) == Some(&b'!')))
+}
+
 fn has_braced_quantifier(bytes: &[u8], mut index: usize) -> bool {
     if bytes.get(index) != Some(&b'{') {
         return false;
@@ -230,10 +237,10 @@ fn quantifier_follows(bytes: &[u8], index: usize) -> bool {
         || has_braced_quantifier(bytes, index)
 }
 
-/// Return the capture-name layout when a pattern has a capture inside a
-/// quantified group. That is precisely the shape for which the linear engine's
-/// leftmost-first result can expose stale captures or stop after the wrong
-/// nullable iteration.
+/// Return the capture-name layout when a pattern needs ECMAScript backtracking
+/// capture semantics. Besides quantified captures, this includes captures in a
+/// negative lookaround: after a successful negative assertion those captures
+/// are unmatched, so a later backreference must match the empty string.
 fn quantified_capture_layout(pattern: &str) -> Option<Vec<Option<String>>> {
     let bytes = pattern.as_bytes();
     let mut captures = Vec::new();
@@ -260,15 +267,20 @@ fn quantified_capture_layout(pattern: &str) -> Option<Vec<Option<String>>> {
                         .map(|end| pattern[index + 3..end].to_string());
                     captures.push(name);
                 }
-                groups.push(GroupFrame { captures_before });
+                groups.push(GroupFrame {
+                    captures_before,
+                    negative_lookaround: is_negative_lookaround(bytes, index),
+                });
                 index += 1;
             }
             b')' if !in_class => {
                 let Some(group) = groups.pop() else {
                     index += 1;
                     continue;
                 };
-                if captures.len() > group.captures_before && quantifier_follows(bytes, index + 1) {
+                if captures.len() > group.captures_before
+                    && (quantifier_follows(bytes, index + 1) || group.negative_lookaround)
+                {
                     needs_repeat_matcher = true;
                 }
                 index += 1;
@@ -288,6 +300,169 @@ pub(super) fn compile(pattern: &str, flags: &str) -> Option<RepeatMatcherRegex>
     })
 }
 
+fn source_and_flags(re: *const super::RegExpHeader) -> (String, String) {
+    if let Some(source) =
+        super::REGEX_SOURCE_TABLE.with(|table| table.borrow().get(&(re as usize)).cloned())
+    {
+        return source;
+    }
+    unsafe {
+        (
+            super::string_as_str((*re).pattern_ptr).to_string(),
+            super::string_as_str((*re).flags_ptr).to_string(),
+        )
+    }
+}
+
+fn decode_wtf8_units(bytes: &[u8]) -> Vec<u16> {
+    let mut units = Vec::new();
+    let mut offset = 0usize;
+    while offset < bytes.len() {
+        let (advance, utf16_units, code_point) = crate::string::wtf8_step(bytes, offset);
+        if utf16_units == 2 && code_point >= 0x10000 {
+            let astral = code_point - 0x10000;
+            units.push(0xD800 + (astral >> 10) as u16);
+            units.push(0xDC00 + (astral & 0x3FF) as u16);
+        } else if utf16_units == 1 {
+            units.push(code_point as u16);
+        }
+        offset = (offset + advance).min(bytes.len());
+    }
+    units
+}
+
+fn append_wtf8_unit(out: &mut Vec<u8>, unit: u16) {
+    if let Some(ch) = char::from_u32(unit as u32) {
+        let mut encoded = [0u8; 3];
+        out.extend_from_slice(ch.encode_utf8(&mut encoded).as_bytes());
+    } else {
+        out.extend_from_slice(&[
+            0xE0 | ((unit >> 12) as u8),
+            0x80 | (((unit >> 6) & 0x3F) as u8),
+            0x80 | ((unit & 0x3F) as u8),
+        ]);
+    }
+}
+
+fn append_unit_range(out: &mut Vec<u8>, units: &[u16], range: std::ops::Range<usize>) {
+    for &unit in &units[range] {
+        append_wtf8_unit(out, unit);
+    }
+}
+
+fn append_replacement(
+    out: &mut Vec<u8>,
+    replacement: &[u8],
+    units: &[u16],
+    matched: &regress::Match,
+) {
+    let group_count = matched.groups().len();
+    let has_named_groups = matched.named_groups().next().is_some();
+    let mut index = 0usize;
+    while index < replacement.len() {
+        if replacement[index] != b'$' || index + 1 == replacement.len() {
+            out.push(replacement[index]);
+            index += 1;
+            continue;
+        }
+        match replacement[index + 1] {
+            b'$' => {
+                out.push(b'$');
+                index += 2;
+            }
+            b'&' => {
+                append_unit_range(out, units, matched.range());
+                index += 2;
+            }
+            b'`' => {
+                append_unit_range(out, units, 0..matched.start());
+                index += 2;
+            }
+            b'\'' => {
+                append_unit_range(out, units, matched.end()..units.len());
+                index += 2;
+            }
+            b'0'..=b'9' => {
+                let first = (replacement[index + 1] - b'0') as usize;
+                let (group, consumed) =
+                    if index + 2 < replacement.len() && replacement[index + 2].is_ascii_digit() {
+                        let two = first * 10 + (replacement[index + 2] - b'0') as usize;
+                        if (1..group_count).contains(&two) {
+                            (Some(two), 2)
+                        } else if (1..group_count).contains(&first) {
+                            (Some(first), 1)
+                        } else {
+                            (None, 0)
+                        }
+                    } else if (1..group_count).contains(&first) {
+                        (Some(first), 1)
+                    } else {
+                        (None, 0)
+                    };
+                if let Some(group) = group {
+                    if let Some(range) = matched.group(group) {
+                        append_unit_range(out, units, range);
+                    }
+                    index += 1 + consumed;
+                } else {
+                    out.push(b'$');
+                    index += 1;
+                }
+            }
+            b'<' if has_named_groups => {
+                if let Some(relative_end) = replacement[index + 2..]
+                    .iter()
+                    .position(|&byte| byte == b'>')
+                {
+                    let name =
+                        std::str::from_utf8(&replacement[index + 2..index + 2 + relative_end])
+                            .unwrap_or_default();
+                    if let Some(range) = matched.named_group(name) {
+                        append_unit_range(out, units, range);
+                    }
+                    index += 3 + relative_end;
+                } else {
+                    out.push(b'$');
+                    index += 1;
+                }
+            }
+            _ => {
+                out.push(b'$');
+                index += 1;
+            }
+        }
+    }
+}
+
+/// Replace on a WTF-8 subject by exposing its exact JavaScript UTF-16 code
+/// units to the ECMAScript matcher. The returned bytes remain WTF-8 and are
+/// canonicalized by the caller's string builder.
+pub(super) fn replace_wtf8_subject(
+    re: *const super::RegExpHeader,
+    subject: &[u8],
+    replacement: &[u8],
+    global: bool,
+) -> Option<Vec<u8>> {
+    let (source, flags) = source_and_flags(re);
+    let regex = regress::Regex::with_flags(&source, flags.as_str()).ok()?;
+    let units = decode_wtf8_units(subject);
+    let matches: Vec<regress::Match> = if flags.contains('u') || flags.contains('v') {
+        regex.find_from_utf16(&units, 0).collect()
+    } else {
+        regex.find_from_ucs2(&units, 0).collect()
+    };
+
+    let mut out = Vec::with_capacity(subject.len().saturating_add(replacement.len()));
+    let mut last_end = 0usize;
+    for matched in matches.iter().take(if global { usize::MAX } else { 1 }) {
+        append_unit_range(&mut out, &units, last_end..matched.start());
+        append_replacement(&mut out, replacement, &units, matched);
+        last_end = matched.end();
+    }
+    append_unit_range(&mut out, &units, last_end..units.len());
+    Some(out)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -296,6 +471,8 @@ mod tests {
     fn detects_only_quantified_groups_with_captures() {
         assert!(quantified_capture_layout(r"(a?b??)*").is_some());
         assert!(quantified_capture_layout(r"(?:(?=(abc))){0,1}a").is_some());
+        assert!(quantified_capture_layout(r"(?!(a)b)\1").is_some());
+        assert!(quantified_capture_layout(r"(?<!(a)b)\1").is_some());
         assert!(quantified_capture_layout(r"[()]\\(literal\\)").is_none());
         assert!(quantified_capture_layout(r"(?:ab)*").is_none());
         assert!(quantified_capture_layout(r"(ab)c").is_none());

crates/perry-runtime/src/regex/tests.rs

@@ -316,6 +316,25 @@ fn repeat_matcher_clears_captures_when_optional_lookahead_is_skipped() {
     }
 }
 
+#[test]
+fn repeat_matcher_preserves_negative_lookahead_capture_semantics() {
+    let re = js_regexp_new(make_string(r"(.*?)a(?!(a+)b\2c)\2(.*)"), make_string(""));
+    let result = js_regexp_exec(re, make_string("baaabaac"));
+    assert!(!result.is_null());
+    assert_eq!(match_capture_text(result, 0).as_deref(), Some("baaabaac"));
+    assert_eq!(match_capture_text(result, 1).as_deref(), Some("ba"));
+    assert_eq!(match_capture_text(result, 2), None);
+    assert_eq!(match_capture_text(result, 3).as_deref(), Some("abaac"));
+}
+
+#[test]
+fn regex_replace_matches_lone_surrogates_as_utf16_units() {
+    let source = make_wtf8(&[0xED, 0xA0, 0x80]);
+    let re = js_regexp_new(make_string(r"\S+"), make_string("g"));
+    let result = js_string_replace_regex(source, re, make_string("test262"));
+    assert_eq!(string_as_str(result), "test262");
+}
+
 #[test]
 fn test_regexp_test_basic() {
     let pattern = make_string("hello");