From 3d8082f467179916930abc0ec8ef47fb2453e62f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 23 Aug 2026 23:08:57 +0200 Subject: [PATCH 1/4] fix(runtime): complete RegExp repeat matcher semantics --- crates/perry-runtime/src/regex.rs | 43 ++++- .../perry-runtime/src/regex/repeat_matcher.rs | 163 ++++++++++++++++++ crates/perry-runtime/src/regex/tests.rs | 19 ++ 3 files changed, 216 insertions(+), 9 deletions(-) diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 324fbb3e5e..964098fbeb 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -636,11 +636,16 @@ pub fn is_registered_regex(addr: usize) -> bool { /// Internal helper: Get string data from StringHeader pub(crate) fn string_as_str<'a>(s: *const StringHeader) -> &'a str { + unsafe { std::str::from_utf8_unchecked(string_as_bytes(s)) } +} + +/// Internal helper: get the byte payload without assuming it is Unicode +/// scalar UTF-8. JavaScript strings containing lone surrogates use WTF-8. +pub(crate) fn string_as_bytes<'a>(s: *const StringHeader) -> &'a [u8] { unsafe { let len = (*s).byte_len as usize; let data = (s as *const u8).add(std::mem::size_of::()); - let bytes = std::slice::from_raw_parts(data, len); - std::str::from_utf8_unchecked(bytes) + std::slice::from_raw_parts(data, len) } } @@ -1397,19 +1402,39 @@ pub extern "C" fn js_string_replace_regex( return js_string_from_str(""); } - let str_data = string_as_str(s); - let repl_str = if is_valid_ptr(replacement) { - string_as_str(replacement) - } else { - "undefined" - }; - if !is_valid_regex_ptr(re) { // If regex is null, return original string return copy_replace_source(s); } unsafe { + // The Rust string engines require scalar-value UTF-8, while Perry + // stores lone JavaScript surrogates as WTF-8. Match those subjects as + // UTF-16 code units with the ECMAScript engine and rebuild the result + // through the WTF-8-aware string builder. + if (*s).flags & crate::string::STRING_FLAG_HAS_LONE_SURROGATES != 0 { + let replacement_bytes = if is_valid_ptr(replacement) { + string_as_bytes(replacement) + } else { + b"undefined" + }; + if let Some(result) = repeat_matcher::replace_wtf8_subject( + re, + string_as_bytes(s), + replacement_bytes, + (*re).global, + ) { + return finish_replace_bytes(&result); + } + } + + let str_data = string_as_str(s); + let repl_str = if is_valid_ptr(replacement) { + string_as_str(replacement) + } else { + "undefined" + }; + if let Some(repeat_matcher) = lookup_repeat_matcher(re) { let result = repeat_matcher.replace(str_data, repl_str, (*re).global); return finish_replace_bytes(result.as_bytes()); diff --git a/crates/perry-runtime/src/regex/repeat_matcher.rs b/crates/perry-runtime/src/regex/repeat_matcher.rs index 5ce703b71f..c62a4f63ba 100644 --- a/crates/perry-runtime/src/regex/repeat_matcher.rs +++ b/crates/perry-runtime/src/regex/repeat_matcher.rs @@ -288,6 +288,169 @@ pub(super) fn compile(pattern: &str, flags: &str) -> Option }) } +fn source_and_flags(re: *const super::RegExpHeader) -> (String, String) { + if let Some(source) = + super::REGEX_SOURCE_TABLE.with(|table| table.borrow().get(&(re as usize)).cloned()) + { + return source; + } + unsafe { + ( + super::string_as_str((*re).pattern_ptr).to_string(), + super::string_as_str((*re).flags_ptr).to_string(), + ) + } +} + +fn decode_wtf8_units(bytes: &[u8]) -> Vec { + let mut units = Vec::new(); + let mut offset = 0usize; + while offset < bytes.len() { + let (advance, utf16_units, code_point) = crate::string::wtf8_step(bytes, offset); + if utf16_units == 2 && code_point >= 0x10000 { + let astral = code_point - 0x10000; + units.push(0xD800 + (astral >> 10) as u16); + units.push(0xDC00 + (astral & 0x3FF) as u16); + } else if utf16_units == 1 { + units.push(code_point as u16); + } + offset = (offset + advance).min(bytes.len()); + } + units +} + +fn append_wtf8_unit(out: &mut Vec, unit: u16) { + if let Some(ch) = char::from_u32(unit as u32) { + let mut encoded = [0u8; 3]; + out.extend_from_slice(ch.encode_utf8(&mut encoded).as_bytes()); + } else { + out.extend_from_slice(&[ + 0xE0 | ((unit >> 12) as u8), + 0x80 | (((unit >> 6) & 0x3F) as u8), + 0x80 | ((unit & 0x3F) as u8), + ]); + } +} + +fn append_unit_range(out: &mut Vec, units: &[u16], range: std::ops::Range) { + for &unit in &units[range] { + append_wtf8_unit(out, unit); + } +} + +fn append_replacement( + out: &mut Vec, + replacement: &[u8], + units: &[u16], + matched: ®ress::Match, +) { + let group_count = matched.groups().len(); + let has_named_groups = matched.named_groups().next().is_some(); + let mut index = 0usize; + while index < replacement.len() { + if replacement[index] != b'$' || index + 1 == replacement.len() { + out.push(replacement[index]); + index += 1; + continue; + } + match replacement[index + 1] { + b'$' => { + out.push(b'$'); + index += 2; + } + b'&' => { + append_unit_range(out, units, matched.range()); + index += 2; + } + b'`' => { + append_unit_range(out, units, 0..matched.start()); + index += 2; + } + b'\'' => { + append_unit_range(out, units, matched.end()..units.len()); + index += 2; + } + b'0'..=b'9' => { + let first = (replacement[index + 1] - b'0') as usize; + let (group, consumed) = + if index + 2 < replacement.len() && replacement[index + 2].is_ascii_digit() { + let two = first * 10 + (replacement[index + 2] - b'0') as usize; + if (1..group_count).contains(&two) { + (Some(two), 2) + } else if (1..group_count).contains(&first) { + (Some(first), 1) + } else { + (None, 0) + } + } else if (1..group_count).contains(&first) { + (Some(first), 1) + } else { + (None, 0) + }; + if let Some(group) = group { + if let Some(range) = matched.group(group) { + append_unit_range(out, units, range); + } + index += 1 + consumed; + } else { + out.push(b'$'); + index += 1; + } + } + b'<' if has_named_groups => { + if let Some(relative_end) = replacement[index + 2..] + .iter() + .position(|&byte| byte == b'>') + { + let name = + std::str::from_utf8(&replacement[index + 2..index + 2 + relative_end]) + .unwrap_or_default(); + if let Some(range) = matched.named_group(name) { + append_unit_range(out, units, range); + } + index += 3 + relative_end; + } else { + out.push(b'$'); + index += 1; + } + } + _ => { + out.push(b'$'); + index += 1; + } + } + } +} + +/// Replace on a WTF-8 subject by exposing its exact JavaScript UTF-16 code +/// units to the ECMAScript matcher. The returned bytes remain WTF-8 and are +/// canonicalized by the caller's string builder. +pub(super) fn replace_wtf8_subject( + re: *const super::RegExpHeader, + subject: &[u8], + replacement: &[u8], + global: bool, +) -> Option> { + let (source, flags) = source_and_flags(re); + let regex = regress::Regex::with_flags(&source, &flags).ok()?; + let units = decode_wtf8_units(subject); + let matches: Vec = if flags.contains('u') || flags.contains('v') { + regex.find_from_utf16(&units, 0).collect() + } else { + regex.find_from_ucs2(&units, 0).collect() + }; + + let mut out = Vec::with_capacity(subject.len().saturating_add(replacement.len())); + let mut last_end = 0usize; + for matched in matches.iter().take(if global { usize::MAX } else { 1 }) { + append_unit_range(&mut out, &units, last_end..matched.start()); + append_replacement(&mut out, replacement, &units, matched); + last_end = matched.end(); + } + append_unit_range(&mut out, &units, last_end..units.len()); + Some(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 5ce8ea4ee1..42b2edb796 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -316,6 +316,25 @@ fn repeat_matcher_clears_captures_when_optional_lookahead_is_skipped() { } } +#[test] +fn repeat_matcher_preserves_negative_lookahead_capture_semantics() { + let re = js_regexp_new(make_string(r"(.*?)a(?!(a+)b\2c)\2(.*)"), make_string("")); + let result = js_regexp_exec(re, make_string("baaabaac")); + assert!(!result.is_null()); + assert_eq!(match_capture_text(result, 0).as_deref(), Some("baaabaac")); + assert_eq!(match_capture_text(result, 1).as_deref(), Some("ba")); + assert_eq!(match_capture_text(result, 2), None); + assert_eq!(match_capture_text(result, 3).as_deref(), Some("abaac")); +} + +#[test] +fn regex_replace_matches_lone_surrogates_as_utf16_units() { + let source = make_wtf8(&[0xED, 0xA0, 0x80]); + let re = js_regexp_new(make_string(r"\S+"), make_string("g")); + let result = js_string_replace_regex(source, re, make_string("test262")); + assert_eq!(string_as_str(result), "test262"); +} + #[test] fn test_regexp_test_basic() { let pattern = make_string("hello"); From a3fb6f441bc16c90f71aa968aed79917efb1c850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 23 Aug 2026 23:13:37 +0200 Subject: [PATCH 2/4] docs(changelog): record RegExp worklist fix --- changelog.d/8667-regexp-repeat-matcher.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog.d/8667-regexp-repeat-matcher.md diff --git a/changelog.d/8667-regexp-repeat-matcher.md b/changelog.d/8667-regexp-repeat-matcher.md new file mode 100644 index 0000000000..d84dccab52 --- /dev/null +++ b/changelog.d/8667-regexp-repeat-matcher.md @@ -0,0 +1,8 @@ +Fixed RegExp backtracking behavior for quantified capture groups, nullable +iterations, and captures referenced across lookaround assertions. Perry now +uses an ECMAScript matcher for the affected patterns while retaining the +existing linear-time engines for the common path. + +Regular-expression replacement also now matches JavaScript strings containing +lone surrogates as UTF-16 code units without losing their WTF-8 representation. +Together these changes make all 89 test262 cases tracked by #5897 pass. From c455c28e1a702e43d3edee3ca55c06755c74217e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 24 Aug 2026 10:12:27 +0200 Subject: [PATCH 3/4] fix(regex): cover negative lookaround captures --- crates/perry-runtime/Cargo.toml | 2 +- .../perry-runtime/src/regex/repeat_matcher.rs | 28 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/perry-runtime/Cargo.toml b/crates/perry-runtime/Cargo.toml index de9861a150..3ac80518dd 100644 --- a/crates/perry-runtime/Cargo.toml +++ b/crates/perry-runtime/Cargo.toml @@ -261,7 +261,7 @@ libc.workspace = true gimli = { version = "0.34", default-features = false, features = ["read"] } rand = "0.10" regex = { workspace = true, optional = true } -regress = { workspace = true, optional = true } +regress = { workspace = true, features = ["utf16"], optional = true } # Taffy — flexbox / grid layout engine for the perry/tui module # (#358 Phase 3). Same crate Bevy and Dioxus use; pure Rust, no FFI. taffy = { version = "0.13", default-features = false, features = ["std", "flexbox", "taffy_tree"] } diff --git a/crates/perry-runtime/src/regex/repeat_matcher.rs b/crates/perry-runtime/src/regex/repeat_matcher.rs index c62a4f63ba..e962c0e181 100644 --- a/crates/perry-runtime/src/regex/repeat_matcher.rs +++ b/crates/perry-runtime/src/regex/repeat_matcher.rs @@ -185,6 +185,7 @@ impl RepeatMatcherRegex { #[derive(Clone, Copy)] struct GroupFrame { captures_before: usize, + negative_lookaround: bool, } fn named_capture_end(bytes: &[u8], open: usize) -> Option { @@ -204,6 +205,12 @@ fn is_capturing_group(bytes: &[u8], open: usize) -> bool { bytes.get(open + 1) != Some(&b'?') || named_capture_end(bytes, open).is_some() } +fn is_negative_lookaround(bytes: &[u8], open: usize) -> bool { + bytes.get(open + 1) == Some(&b'?') + && (bytes.get(open + 2) == Some(&b'!') + || (bytes.get(open + 2) == Some(&b'<') && bytes.get(open + 3) == Some(&b'!'))) +} + fn has_braced_quantifier(bytes: &[u8], mut index: usize) -> bool { if bytes.get(index) != Some(&b'{') { return false; @@ -230,10 +237,10 @@ fn quantifier_follows(bytes: &[u8], index: usize) -> bool { || has_braced_quantifier(bytes, index) } -/// Return the capture-name layout when a pattern has a capture inside a -/// quantified group. That is precisely the shape for which the linear engine's -/// leftmost-first result can expose stale captures or stop after the wrong -/// nullable iteration. +/// Return the capture-name layout when a pattern needs ECMAScript backtracking +/// capture semantics. Besides quantified captures, this includes captures in a +/// negative lookaround: after a successful negative assertion those captures +/// are unmatched, so a later backreference must match the empty string. fn quantified_capture_layout(pattern: &str) -> Option>> { let bytes = pattern.as_bytes(); let mut captures = Vec::new(); @@ -260,7 +267,10 @@ fn quantified_capture_layout(pattern: &str) -> Option>> { .map(|end| pattern[index + 3..end].to_string()); captures.push(name); } - groups.push(GroupFrame { captures_before }); + groups.push(GroupFrame { + captures_before, + negative_lookaround: is_negative_lookaround(bytes, index), + }); index += 1; } b')' if !in_class => { @@ -268,7 +278,9 @@ fn quantified_capture_layout(pattern: &str) -> Option>> { index += 1; continue; }; - if captures.len() > group.captures_before && quantifier_follows(bytes, index + 1) { + if captures.len() > group.captures_before + && (quantifier_follows(bytes, index + 1) || group.negative_lookaround) + { needs_repeat_matcher = true; } index += 1; @@ -432,7 +444,7 @@ pub(super) fn replace_wtf8_subject( global: bool, ) -> Option> { let (source, flags) = source_and_flags(re); - let regex = regress::Regex::with_flags(&source, &flags).ok()?; + let regex = regress::Regex::with_flags(&source, flags.as_str()).ok()?; let units = decode_wtf8_units(subject); let matches: Vec = if flags.contains('u') || flags.contains('v') { regex.find_from_utf16(&units, 0).collect() @@ -459,6 +471,8 @@ mod tests { fn detects_only_quantified_groups_with_captures() { assert!(quantified_capture_layout(r"(a?b??)*").is_some()); assert!(quantified_capture_layout(r"(?:(?=(abc))){0,1}a").is_some()); + assert!(quantified_capture_layout(r"(?!(a)b)\1").is_some()); + assert!(quantified_capture_layout(r"(? Date: Mon, 24 Aug 2026 11:16:13 +0200 Subject: [PATCH 4/4] fix(ci): restore current main PR gates --- crates/perry-stdlib/src/net/mod.rs | 45 ++++++++++++++++------ scripts/ci_e2e_scope.py | 2 + scripts/unrooted_local_shape_baseline.json | 12 +++--- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/crates/perry-stdlib/src/net/mod.rs b/crates/perry-stdlib/src/net/mod.rs index b24c6b793d..66de8a8254 100644 --- a/crates/perry-stdlib/src/net/mod.rs +++ b/crates/perry-stdlib/src/net/mod.rs @@ -543,6 +543,7 @@ unsafe fn get_object_bool_field(obj_f64: f64, field_name: &str) -> Option /// mirrors `crates/perry-stdlib/src/sqlite.rs::build_packed_keys`. unsafe fn build_error_object(msg: &str) -> f64 { use perry_runtime::JSValue; + let scope = perry_runtime::gc::RuntimeHandleScope::new(); let keys = ["message", "code", "name"]; let mut packed = Vec::new(); for key in keys { @@ -554,17 +555,27 @@ unsafe fn build_error_object(msg: &str) -> f64 { shape_id = shape_id.wrapping_mul(31).wrapping_add(b as u32); } shape_id = shape_id.wrapping_add(3); - let s_msg = perry_runtime::js_string_from_bytes(msg.as_ptr(), msg.len() as u32); - let obj = perry_runtime::js_object_alloc_with_shape( + let s_msg = scope.root_string_ptr(perry_runtime::js_string_from_bytes( + msg.as_ptr(), + msg.len() as u32, + )); + let obj_ptr = perry_runtime::js_object_alloc_with_shape( shape_id, 3, packed.as_ptr(), packed.len() as u32, ); - if obj.is_null() { - return f64::from_bits(0x7FFF_0000_0000_0000u64 | (s_msg as u64 & 0x0000_FFFF_FFFF_FFFF)); + if obj_ptr.is_null() { + return s_msg.with_const_ptr(|s_msg: *const perry_runtime::StringHeader| { + f64::from_bits(0x7FFF_0000_0000_0000u64 | (s_msg as u64 & 0x0000_FFFF_FFFF_FFFF)) + }); } - perry_runtime::js_object_set_field(obj, 0, JSValue::string_ptr(s_msg)); + let obj = scope.root_raw_mut_ptr(obj_ptr); + obj.with_mut_ptr(|obj| { + s_msg.with_mut_ptr(|s_msg| { + perry_runtime::js_object_set_field(obj, 0, JSValue::string_ptr(s_msg)) + }) + }); let code = if msg.starts_with("ERR_") { Some(msg) } else if msg.contains("UnknownIssuer") @@ -578,13 +589,25 @@ unsafe fn build_error_object(msg: &str) -> f64 { None }; if let Some(code) = code { - let code = perry_runtime::js_string_from_bytes(code.as_ptr(), code.len() as u32); - perry_runtime::js_object_set_field(obj, 1, JSValue::string_ptr(code)); + let code = scope.root_string_ptr(perry_runtime::js_string_from_bytes( + code.as_ptr(), + code.len() as u32, + )); + obj.with_mut_ptr(|obj| { + code.with_mut_ptr(|code| { + perry_runtime::js_object_set_field(obj, 1, JSValue::string_ptr(code)) + }) + }); } - let name = perry_runtime::js_string_from_bytes(b"Error".as_ptr(), 5); - perry_runtime::js_object_set_field(obj, 2, JSValue::string_ptr(name)); - let obj_bits = (obj as u64 & 0x0000_FFFF_FFFF_FFFF) | 0x7FFD_0000_0000_0000; - f64::from_bits(obj_bits) + let name = scope.root_string_ptr(perry_runtime::js_string_from_bytes(b"Error".as_ptr(), 5)); + obj.with_mut_ptr(|obj| { + name.with_mut_ptr(|name| { + perry_runtime::js_object_set_field(obj, 2, JSValue::string_ptr(name)) + }) + }); + obj.with_mut_ptr(|obj: *mut perry_runtime::ObjectHeader| { + f64::from_bits((obj as u64 & 0x0000_FFFF_FFFF_FFFF) | 0x7FFD_0000_0000_0000) + }) } fn next_id() -> i64 { diff --git a/scripts/ci_e2e_scope.py b/scripts/ci_e2e_scope.py index 94e88edd69..a364c98be3 100755 --- a/scripts/ci_e2e_scope.py +++ b/scripts/ci_e2e_scope.py @@ -121,6 +121,7 @@ "constructor_recursion", "destructure_call_location", "i64_spec_ternary_recursion", + "ios_platform_api_lowering", "large_object_barriers", "loop_safepoint_purity", "macos_bundle_chdir_gate", @@ -143,6 +144,7 @@ "spec_abi_typed_array_local_length", "static_symbol_hygiene", "temp_root_operand_temporaries", + "typed_array_rmw_8692", "typed_shape_declared_at_allocation", "typed_shape_descriptor", "typed_shape_descriptors", diff --git a/scripts/unrooted_local_shape_baseline.json b/scripts/unrooted_local_shape_baseline.json index f8af29e696..986057a159 100644 --- a/scripts/unrooted_local_shape_baseline.json +++ b/scripts/unrooted_local_shape_baseline.json @@ -5,13 +5,13 @@ "crates/perry-ext-commander/src/lib.rs": 4, "crates/perry-ext-cron/src/lib.rs": 2, "crates/perry-ext-decimal/src/lib.rs": 1, - "crates/perry-ext-events/src/lib.rs": 23, + "crates/perry-ext-events/src/lib.rs": 21, "crates/perry-ext-events/src/module_iterators.rs": 2, "crates/perry-ext-events/src/tests.rs": 2, "crates/perry-ext-fastify/src/upgrade.rs": 4, "crates/perry-ext-fetch/src/lib.rs": 14, "crates/perry-ext-fetch/src/tests.rs": 14, - "crates/perry-ext-http/src/agent.rs": 4, + "crates/perry-ext-http/src/agent.rs": 3, "crates/perry-ext-http/src/client_request_surface.rs": 2, "crates/perry-ext-http/src/response_headers.rs": 1, "crates/perry-ext-http/src/server/handle_dispatch.rs": 2, @@ -35,15 +35,14 @@ "crates/perry-stdlib/src/cron.rs": 2, "crates/perry-stdlib/src/crypto/kdf.rs": 9, "crates/perry-stdlib/src/crypto/sign.rs": 22, - "crates/perry-stdlib/src/crypto/util.rs": 3, - "crates/perry-stdlib/src/crypto/x509.rs": 19, + "crates/perry-stdlib/src/crypto/util.rs": 2, "crates/perry-stdlib/src/domain.rs": 3, "crates/perry-stdlib/src/ethers.rs": 5, "crates/perry-stdlib/src/events.rs": 6, "crates/perry-stdlib/src/events/constructors.rs": 1, "crates/perry-stdlib/src/events/events_on.rs": 17, "crates/perry-stdlib/src/events/module_helpers.rs": 1, - "crates/perry-stdlib/src/events/once_helpers.rs": 4, + "crates/perry-stdlib/src/events/once_helpers.rs": 1, "crates/perry-stdlib/src/events/warnings.rs": 1, "crates/perry-stdlib/src/fetch/mod.rs": 6, "crates/perry-stdlib/src/ioredis.rs": 14, @@ -52,7 +51,6 @@ "crates/perry-stdlib/src/mysql2/pool.rs": 2, "crates/perry-stdlib/src/mysql2/result.rs": 43, "crates/perry-stdlib/src/mysql2/types.rs": 16, - "crates/perry-stdlib/src/net/mod.rs": 3, "crates/perry-stdlib/src/nodemailer.rs": 3, "crates/perry-stdlib/src/pg/result.rs": 14, "crates/perry-stdlib/src/pg/types.rs": 14, @@ -86,5 +84,5 @@ "crates/perry-stdlib/src/zlib.rs": 2 }, "schema_version": 2, - "total": 605 + "total": 576 }