From 00be9741bfba741d0f2a43a89eaad7c9abe7a468 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:15:53 -0400 Subject: [PATCH 01/16] std: reduce visibility of some internal OsStr related types The std::sys::os_str::{Buf, Slice} types are only used within the std crate and not actually exported. Whole `sys` module is private. They don't need to be public. This might result in a better generated code, but more importantly it avoids some compile errors down the line. --- library/std/src/sys/os_str/bytes.rs | 4 ++-- library/std/src/sys/os_str/mod.rs | 6 +++--- library/std/src/sys/os_str/utf8.rs | 4 ++-- library/std/src/sys/os_str/wtf8.rs | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/library/std/src/sys/os_str/bytes.rs b/library/std/src/sys/os_str/bytes.rs index a57da01a5d85d..048e5def1369b 100644 --- a/library/std/src/sys/os_str/bytes.rs +++ b/library/std/src/sys/os_str/bytes.rs @@ -16,12 +16,12 @@ mod tests; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: Vec, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: [u8], } diff --git a/library/std/src/sys/os_str/mod.rs b/library/std/src/sys/os_str/mod.rs index f7007cbf18b4c..dd37b558a096e 100644 --- a/library/std/src/sys/os_str/mod.rs +++ b/library/std/src/sys/os_str/mod.rs @@ -3,14 +3,14 @@ cfg_select! { any(target_os = "windows", target_os = "uefi") => { mod wtf8; - pub use wtf8::{Buf, Slice}; + pub(crate) use wtf8::{Buf, Slice}; } any(target_os = "motor") => { mod utf8; - pub use utf8::{Buf, Slice}; + pub(crate) use utf8::{Buf, Slice}; } _ => { mod bytes; - pub use bytes::{Buf, Slice}; + pub(crate) use bytes::{Buf, Slice}; } } diff --git a/library/std/src/sys/os_str/utf8.rs b/library/std/src/sys/os_str/utf8.rs index 289f58aa480f7..3648962da5eeb 100644 --- a/library/std/src/sys/os_str/utf8.rs +++ b/library/std/src/sys/os_str/utf8.rs @@ -11,12 +11,12 @@ use crate::{fmt, mem}; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: String, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: str, } diff --git a/library/std/src/sys/os_str/wtf8.rs b/library/std/src/sys/os_str/wtf8.rs index 9a32ab3f3ea12..5f9d64dc5d09c 100644 --- a/library/std/src/sys/os_str/wtf8.rs +++ b/library/std/src/sys/os_str/wtf8.rs @@ -13,12 +13,12 @@ use crate::{fmt, mem}; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: Wtf8Buf, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: Wtf8, } From 8662be8340e5e2bc06fe16b7df0a7d02ebdf5e64 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:48:54 -0400 Subject: [PATCH 02/16] core: refactor tests/pattern.rs tests Firstly, combine functions and results lists into a single list with 'function => result' pairs. This makes it easier to match function with its result. Secondly, eliminate InRange step so that it's easier to notice series of matches or rejects. @pacak: I added a variant to test_stress_indices that matches stuff --- library/coretests/tests/pattern.rs | 485 ++++++++++++++++------------- 1 file changed, 271 insertions(+), 214 deletions(-) diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index d4bec996d89a1..d76d67771631c 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -3,10 +3,10 @@ use std::str::pattern::*; // This macro makes it easier to write // tests that do a series of iterations macro_rules! search_asserts { - ($haystack:expr, $needle:expr, $testname:expr, [$($func:ident),*], $result:expr) => { + ($haystack:expr, $needle:expr, $testname:literal, $($func:ident => $result:expr),*) => { let mut searcher = $needle.into_searcher($haystack); - let arr = [$( Step::from(searcher.$func()) ),*]; - assert_eq!(&arr[..], &$result, $testname); + let arr = [$( searcher.$func().into_step(stringify!($func)) ),*]; + assert_eq!(&arr[..], &[$($result),*], $testname); } } @@ -17,26 +17,31 @@ enum Step { // be the same length for easy alignment Matches(usize, usize), Rejects(usize, usize), - InRange(usize, usize), Done, } -use self::Step::*; +use Step::*; -impl From for Step { - fn from(x: SearchStep) -> Self { - match x { - SearchStep::Match(a, b) => Matches(a, b), - SearchStep::Reject(a, b) => Rejects(a, b), +trait IntoStep { + fn into_step(self, method_name: &str) -> Step; +} + +impl IntoStep for SearchStep { + fn into_step(self, _name: &str) -> Step { + match self { + SearchStep::Match(s, e) => Matches(s, e), + SearchStep::Reject(s, e) => Rejects(s, e), SearchStep::Done => Done, } } } -impl From> for Step { - fn from(x: Option<(usize, usize)>) -> Self { - match x { - Some((a, b)) => InRange(a, b), +impl IntoStep for Option<(usize, usize)> { + fn into_step(self, method_name: &str) -> Step { + let is_reject = method_name.starts_with("next_reject"); + match self { + Some((s, e)) if is_reject => Rejects(s, e), + Some((s, e)) => Matches(s, e), None => Done, } } @@ -54,93 +59,74 @@ fn test_simple_iteration() { "abcdeabcd", 'a', "forward iteration for ASCII string", - // a b c d e a b c d EOF - [next, next, next, next, next, next, next, next, next, next], - [ - Matches(0, 1), - Rejects(1, 2), - Rejects(2, 3), - Rejects(3, 4), - Rejects(4, 5), - Matches(5, 6), - Rejects(6, 7), - Rejects(7, 8), - Rejects(8, 9), - Done - ] + next => Matches(0, 1), + next => Rejects(1, 2), + next => Rejects(2, 3), + next => Rejects(3, 4), + next => Rejects(4, 5), + next => Matches(5, 6), + next => Rejects(6, 7), + next => Rejects(7, 8), + next => Rejects(8, 9), + next => Done ); search_asserts!( "abcdeabcd", 'a', "reverse iteration for ASCII string", - // d c b a e d c b a EOF - [ - next_back, next_back, next_back, next_back, next_back, next_back, next_back, next_back, - next_back, next_back - ], - [ - Rejects(8, 9), - Rejects(7, 8), - Rejects(6, 7), - Matches(5, 6), - Rejects(4, 5), - Rejects(3, 4), - Rejects(2, 3), - Rejects(1, 2), - Matches(0, 1), - Done - ] + next_back => Rejects(8, 9), + next_back => Rejects(7, 8), + next_back => Rejects(6, 7), + next_back => Matches(5, 6), + next_back => Rejects(4, 5), + next_back => Rejects(3, 4), + next_back => Rejects(2, 3), + next_back => Rejects(1, 2), + next_back => Matches(0, 1), + next_back => Done ); search_asserts!( "我爱我的猫", '我', "forward iteration for Chinese string", - // 我 愛 我 的 貓 EOF - [next, next, next, next, next, next], - [Matches(0, 3), Rejects(3, 6), Matches(6, 9), Rejects(9, 12), Rejects(12, 15), Done] + next => Matches(0, 3), + next => Rejects(3, 6), + next => Matches(6, 9), + next => Rejects(9, 12), + next => Rejects(12, 15), + next => Done ); search_asserts!( "我的猫说meow", 'm', "forward iteration for mixed string", - // 我 的 猫 说 m e o w EOF - [next, next, next, next, next, next, next, next, next], - [ - Rejects(0, 3), - Rejects(3, 6), - Rejects(6, 9), - Rejects(9, 12), - Matches(12, 13), - Rejects(13, 14), - Rejects(14, 15), - Rejects(15, 16), - Done - ] + next => Rejects(0, 3), + next => Rejects(3, 6), + next => Rejects(6, 9), + next => Rejects(9, 12), + next => Matches(12, 13), + next => Rejects(13, 14), + next => Rejects(14, 15), + next => Rejects(15, 16), + next => Done ); search_asserts!( "我的猫说meow", '猫', "reverse iteration for mixed string", - // w o e m 说 猫 的 我 EOF - [ - next_back, next_back, next_back, next_back, next_back, next_back, next_back, next_back, - next_back - ], - [ - Rejects(15, 16), - Rejects(14, 15), - Rejects(13, 14), - Rejects(12, 13), - Rejects(9, 12), - Matches(6, 9), - Rejects(3, 6), - Rejects(0, 3), - Done - ] + next_back => Rejects(15, 16), + next_back => Rejects(14, 15), + next_back => Rejects(13, 14), + next_back => Rejects(12, 13), + next_back => Rejects(9, 12), + next_back => Matches(6, 9), + next_back => Rejects(3, 6), + next_back => Rejects(0, 3), + next_back => Done ); } @@ -150,46 +136,43 @@ fn test_simple_search() { "abcdeabcdeabcde", 'a', "next_match for ASCII string", - [next_match, next_match, next_match, next_match], - [InRange(0, 1), InRange(5, 6), InRange(10, 11), Done] + next_match => Matches(0, 1), + next_match => Matches(5, 6), + next_match => Matches(10, 11), + next_match => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "next_match_back for ASCII string", - [next_match_back, next_match_back, next_match_back, next_match_back], - [InRange(10, 11), InRange(5, 6), InRange(0, 1), Done] + next_match_back => Matches(10, 11), + next_match_back => Matches(5, 6), + next_match_back => Matches(0, 1), + next_match_back => Done ); search_asserts!( "abcdeab", 'a', "next_reject for ASCII string", - [next_reject, next_reject, next_match, next_reject, next_reject], - [InRange(1, 2), InRange(2, 3), InRange(5, 6), InRange(6, 7), Done] + next_reject => Rejects(1, 2), + next_reject => Rejects(2, 3), + next_match => Matches(5, 6), + next_reject => Rejects(6, 7), + next_reject => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "next_reject_back for ASCII string", - [ - next_reject_back, - next_reject_back, - next_match_back, - next_reject_back, - next_reject_back, - next_reject_back - ], - [ - InRange(14, 15), - InRange(13, 14), - InRange(10, 11), - InRange(9, 10), - InRange(8, 9), - InRange(7, 8) - ] + next_reject_back => Rejects(14, 15), + next_reject_back => Rejects(13, 14), + next_match_back => Matches(10, 11), + next_reject_back => Rejects(9, 10), + next_reject_back => Rejects(8, 9), + next_reject_back => Rejects(7, 8) ); } @@ -207,38 +190,58 @@ const STRESS: &str = "Áa🁀bÁꁁfg😁각กᘀ각aÁ각ꁁก😁a"; #[test] fn test_stress_indices() { // this isn't really a test, more of documentation on the indices of each character in the stresstest string + search_asserts!( + STRESS, + |_| true, + "Indices of characters in stress test", + next => Matches(0, 2), // Á + next => Matches(2, 3), // a + next => Matches(3, 7), // 🁀 + next => Matches(7, 8), // b + next => Matches(8, 10), // Á + next => Matches(10, 13), // ꁁ + next => Matches(13, 14), // f + next => Matches(14, 15), // g + next => Matches(15, 19), // 😀 + next => Matches(19, 22), // 각 + next => Matches(22, 25), // ก + next => Matches(25, 28), // ᘀ + next => Matches(28, 31), // 각 + next => Matches(31, 32), // a + next => Matches(32, 34), // Á + next => Matches(34, 37), // 각 + next => Matches(37, 40), // ꁁ + next => Matches(40, 43), // ก + next => Matches(43, 47), // 😀 + next => Matches(47, 48), // a + next => Done + ); search_asserts!( STRESS, 'x', "Indices of characters in stress test", - [ - next, next, next, next, next, next, next, next, next, next, next, next, next, next, - next, next, next, next, next, next, next - ], - [ - Rejects(0, 2), // Á - Rejects(2, 3), // a - Rejects(3, 7), // 🁀 - Rejects(7, 8), // b - Rejects(8, 10), // Á - Rejects(10, 13), // ꁁ - Rejects(13, 14), // f - Rejects(14, 15), // g - Rejects(15, 19), // 😀 - Rejects(19, 22), // 각 - Rejects(22, 25), // ก - Rejects(25, 28), // ᘀ - Rejects(28, 31), // 각 - Rejects(31, 32), // a - Rejects(32, 34), // Á - Rejects(34, 37), // 각 - Rejects(37, 40), // ꁁ - Rejects(40, 43), // ก - Rejects(43, 47), // 😀 - Rejects(47, 48), // a - Done - ] + next => Rejects(0, 2), // Á + next => Rejects(2, 3), // a + next => Rejects(3, 7), // 🁀 + next => Rejects(7, 8), // b + next => Rejects(8, 10), // Á + next => Rejects(10, 13), // ꁁ + next => Rejects(13, 14), // f + next => Rejects(14, 15), // g + next => Rejects(15, 19), // 😀 + next => Rejects(19, 22), // 각 + next => Rejects(22, 25), // ก + next => Rejects(25, 28), // ᘀ + next => Rejects(28, 31), // 각 + next => Rejects(31, 32), // a + next => Rejects(32, 34), // Á + next => Rejects(34, 37), // 각 + next => Rejects(37, 40), // ꁁ + next => Rejects(40, 43), // ก + next => Rejects(43, 47), // 😀 + next => Rejects(47, 48), // a + next => Done ); } @@ -248,96 +251,113 @@ fn test_forward_search_shared_bytes() { STRESS, 'Á', "Forward search for two-byte Latin character", - [next_match, next_match, next_match, next_match], - [InRange(0, 2), InRange(8, 10), InRange(32, 34), Done] + next_match => Matches(0, 2), + next_match => Matches(8, 10), + next_match => Matches(32, 34), + next_match => Done ); search_asserts!( STRESS, 'Á', "Forward search for two-byte Latin character; check if next() still works", - [next_match, next, next_match, next, next_match, next, next_match], - [ - InRange(0, 2), - Rejects(2, 3), - InRange(8, 10), - Rejects(10, 13), - InRange(32, 34), - Rejects(34, 37), - Done - ] + next_match => Matches(0, 2), + next => Rejects(2, 3), + next_match => Matches(8, 10), + next => Rejects(10, 13), + next_match => Matches(32, 34), + next => Rejects(34, 37), + next_match => Done ); search_asserts!( STRESS, '각', "Forward search for three-byte Hangul character", - [next_match, next, next_match, next_match, next_match], - [InRange(19, 22), Rejects(22, 25), InRange(28, 31), InRange(34, 37), Done] + next_match => Matches(19, 22), + next => Rejects(22, 25), + next_match => Matches(28, 31), + next_match => Matches(34, 37), + next_match => Done ); search_asserts!( STRESS, '각', "Forward search for three-byte Hangul character; check if next() still works", - [next_match, next, next_match, next, next_match, next, next_match], - [ - InRange(19, 22), - Rejects(22, 25), - InRange(28, 31), - Rejects(31, 32), - InRange(34, 37), - Rejects(37, 40), - Done - ] + next_match => Matches(19, 22), + next => Rejects(22, 25), + next_match => Matches(28, 31), + next => Rejects(31, 32), + next_match => Matches(34, 37), + next => Rejects(37, 40), + next_match => Done ); search_asserts!( STRESS, 'ก', "Forward search for three-byte Thai character", - [next_match, next, next_match, next, next_match], - [InRange(22, 25), Rejects(25, 28), InRange(40, 43), Rejects(43, 47), Done] + next_match => Matches(22, 25), + next => Rejects(25, 28), + next_match => Matches(40, 43), + next => Rejects(43, 47), + next_match => Done ); search_asserts!( STRESS, 'ก', "Forward search for three-byte Thai character; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(22, 25), Rejects(25, 28), InRange(40, 43), Rejects(43, 47), Done] + next_match => Matches(22, 25), + next => Rejects(25, 28), + next_match => Matches(40, 43), + next => Rejects(43, 47), + next_match => Done ); search_asserts!( STRESS, '😁', "Forward search for four-byte emoji", - [next_match, next, next_match, next, next_match], - [InRange(15, 19), Rejects(19, 22), InRange(43, 47), Rejects(47, 48), Done] + next_match => Matches(15, 19), + next => Rejects(19, 22), + next_match => Matches(43, 47), + next => Rejects(47, 48), + next_match => Done ); search_asserts!( STRESS, '😁', "Forward search for four-byte emoji; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(15, 19), Rejects(19, 22), InRange(43, 47), Rejects(47, 48), Done] + next_match => Matches(15, 19), + next => Rejects(19, 22), + next_match => Matches(43, 47), + next => Rejects(47, 48), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Forward search for three-byte Yi character with repeated bytes", - [next_match, next, next_match, next, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(40, 43), Done] + next_match => Matches(10, 13), + next => Rejects(13, 14), + next_match => Matches(37, 40), + next => Rejects(40, 43), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Forward search for three-byte Yi character with repeated bytes; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(40, 43), Done] + next_match => Matches(10, 13), + next => Rejects(13, 14), + next_match => Matches(37, 40), + next => Rejects(40, 43), + next_match => Done ); } @@ -347,96 +367,112 @@ fn test_reverse_search_shared_bytes() { STRESS, 'Á', "Reverse search for two-byte Latin character", - [next_match_back, next_match_back, next_match_back, next_match_back], - [InRange(32, 34), InRange(8, 10), InRange(0, 2), Done] + next_match_back => Matches(32, 34), + next_match_back => Matches(8, 10), + next_match_back => Matches(0, 2), + next_match_back => Done ); search_asserts!( STRESS, 'Á', "Reverse search for two-byte Latin character; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back, next_back], - [InRange(32, 34), Rejects(31, 32), InRange(8, 10), Rejects(7, 8), InRange(0, 2), Done] + next_match_back => Matches(32, 34), + next_back => Rejects(31, 32), + next_match_back => Matches(8, 10), + next_back => Rejects(7, 8), + next_match_back => Matches(0, 2), + next_back => Done ); search_asserts!( STRESS, '각', "Reverse search for three-byte Hangul character", - [next_match_back, next_back, next_match_back, next_match_back, next_match_back], - [InRange(34, 37), Rejects(32, 34), InRange(28, 31), InRange(19, 22), Done] + next_match_back => Matches(34, 37), + next_back => Rejects(32, 34), + next_match_back => Matches(28, 31), + next_match_back => Matches(19, 22), + next_match_back => Done ); search_asserts!( STRESS, '각', "Reverse search for three-byte Hangul character; check if next_back() still works", - [ - next_match_back, - next_back, - next_match_back, - next_back, - next_match_back, - next_back, - next_match_back - ], - [ - InRange(34, 37), - Rejects(32, 34), - InRange(28, 31), - Rejects(25, 28), - InRange(19, 22), - Rejects(15, 19), - Done - ] + next_match_back => Matches(34, 37), + next_back => Rejects(32, 34), + next_match_back => Matches(28, 31), + next_back => Rejects(25, 28), + next_match_back => Matches(19, 22), + next_back => Rejects(15, 19), + next_match_back => Done ); search_asserts!( STRESS, 'ก', "Reverse search for three-byte Thai character", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(40, 43), Rejects(37, 40), InRange(22, 25), Rejects(19, 22), Done] + next_match_back => Matches(40, 43), + next_back => Rejects(37, 40), + next_match_back => Matches(22, 25), + next_back => Rejects(19, 22), + next_match_back => Done ); search_asserts!( STRESS, 'ก', "Reverse search for three-byte Thai character; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(40, 43), Rejects(37, 40), InRange(22, 25), Rejects(19, 22), Done] + next_match_back => Matches(40, 43), + next_back => Rejects(37, 40), + next_match_back => Matches(22, 25), + next_back => Rejects(19, 22), + next_match_back => Done ); search_asserts!( STRESS, '😁', "Reverse search for four-byte emoji", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(43, 47), Rejects(40, 43), InRange(15, 19), Rejects(14, 15), Done] + next_match_back => Matches(43, 47), + next_back => Rejects(40, 43), + next_match_back => Matches(15, 19), + next_back => Rejects(14, 15), + next_match_back => Done ); search_asserts!( STRESS, '😁', "Reverse search for four-byte emoji; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(43, 47), Rejects(40, 43), InRange(15, 19), Rejects(14, 15), Done] + next_match_back => Matches(43, 47), + next_back => Rejects(40, 43), + next_match_back => Matches(15, 19), + next_back => Rejects(14, 15), + next_match_back => Done ); search_asserts!( STRESS, 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(37, 40), Rejects(34, 37), InRange(10, 13), Rejects(8, 10), Done] + next_match_back => Matches(37, 40), + next_back => Rejects(34, 37), + next_match_back => Matches(10, 13), + next_back => Rejects(8, 10), + next_match_back => Done ); search_asserts!( STRESS, 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(37, 40), Rejects(34, 37), InRange(10, 13), Rejects(8, 10), Done] + next_match_back => Matches(37, 40), + next_back => Rejects(34, 37), + next_match_back => Matches(10, 13), + next_back => Rejects(8, 10), + next_match_back => Done ); } @@ -448,56 +484,77 @@ fn double_ended_regression_test() { "abcdeabcdeabcde", 'a', "alternating double ended search", - [next_match, next_match_back, next_match, next_match_back], - [InRange(0, 1), InRange(10, 11), InRange(5, 6), Done] + next_match => Matches(0, 1), + next_match_back => Matches(10, 11), + next_match => Matches(5, 6), + next_match_back => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "triple double ended search for a", - [next_match, next_match_back, next_match_back, next_match_back], - [InRange(0, 1), InRange(10, 11), InRange(5, 6), Done] + next_match => Matches(0, 1), + next_match_back => Matches(10, 11), + next_match_back => Matches(5, 6), + next_match_back => Done ); search_asserts!( "abcdeabcdeabcde", 'd', "triple double ended search for d", - [next_match, next_match_back, next_match_back, next_match_back], - [InRange(3, 4), InRange(13, 14), InRange(8, 9), Done] + next_match => Matches(3, 4), + next_match_back => Matches(13, 14), + next_match_back => Matches(8, 9), + next_match_back => Done ); search_asserts!( STRESS, 'Á', "Double ended search for two-byte Latin character", - [next_match, next_match_back, next_match, next_match_back], - [InRange(0, 2), InRange(32, 34), InRange(8, 10), Done] + next_match => Matches(0, 2), + next_match_back => Matches(32, 34), + next_match => Matches(8, 10), + next_match_back => Done ); search_asserts!( STRESS, '각', "Reverse double ended search for three-byte Hangul character", - [next_match_back, next_back, next_match, next, next_match_back, next_match], - [InRange(34, 37), Rejects(32, 34), InRange(19, 22), Rejects(22, 25), InRange(28, 31), Done] + next_match_back => Matches(34, 37), + next_back => Rejects(32, 34), + next_match => Matches(19, 22), + next => Rejects(22, 25), + next_match_back => Matches(28, 31), + next_match => Done ); search_asserts!( STRESS, 'ก', "Double ended search for three-byte Thai character", - [next_match, next_back, next, next_match_back, next_match], - [InRange(22, 25), Rejects(47, 48), Rejects(25, 28), InRange(40, 43), Done] + next_match => Matches(22, 25), + next_back => Rejects(47, 48), + next => Rejects(25, 28), + next_match_back => Matches(40, 43), + next_match => Done ); search_asserts!( STRESS, '😁', "Double ended search for four-byte emoji", - [next_match_back, next, next_match, next_back, next_match], - [InRange(43, 47), Rejects(0, 2), InRange(15, 19), Rejects(40, 43), Done] + next_match_back => Matches(43, 47), + next => Rejects(0, 2), + next_match => Matches(15, 19), + next_back => Rejects(40, 43), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Double ended search for three-byte Yi character with repeated bytes", - [next_match, next, next_match_back, next_back, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(34, 37), Done] + next_match => Matches(10, 13), + next => Rejects(13, 14), + next_match_back => Matches(37, 40), + next_back => Rejects(34, 37), + next_match => Done ); } From 13543ff2c81ac7076867755446e586452b7f2a04 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 10 Aug 2026 06:20:59 -0400 Subject: [PATCH 03/16] coretests: Add more pattern tests. Right now things are undertested and underspecified. Some of the library code would get in a loop if searcher starts returning empty rejects. And there's no tests for backwards multi byte char matchers. Pull request I'm reviving had a problem implementing that, so making sure it's tested before the actual code lands. Right now it is possible to break both tests (and user code) without breaking anything else in the test suite I think. --- library/coretests/tests/pattern.rs | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index d4bec996d89a1..30d22daae100b 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -501,3 +501,81 @@ fn double_ended_regression_test() { [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(34, 37), Done] ); } + +#[test] +fn two_way_next_reject_skips_matches() { + // `next_reject` must not report the matched regions as rejects + + // plen - pattern len + // ulen - unmatch len + // haystack is always a concatenation of [Match, Reject, Match] + #[track_caller] + fn check_fw_bw<'a, T>(haystack: &'a str, pat: T, plen: usize, ulen: usize) + where + T: Pattern + Copy, + T::Searcher<'a>: ReverseSearcher<'a>, + { + // fragments + let f1 = (0, plen); + let f2 = (f1.1, f1.1 + ulen); + let f3 = (f2.1, f2.1 + plen); + + let mut searcher = pat.into_searcher(haystack); + let (start, end) = searcher.next_reject().expect("fw reject"); + assert_eq!(start, f2.0, "fw reject start"); + assert!(start < end && end <= f2.1, "fw reject must be a non-empty part of f2"); + assert_eq!(searcher.next_match(), Some(f3), "fw match"); + assert_eq!(searcher.next_match(), None, "fw end"); + + let mut searcher = pat.into_searcher(haystack); + let (start, end) = searcher.next_reject_back().expect("bw reject"); + assert_eq!(end, f2.1, "bw reject end"); + assert!(f2.0 <= start && start < end, "bw reject must be a non-empty part of f2"); + assert_eq!(searcher.next_match_back(), Some(f1), "bw match"); + assert_eq!(searcher.next_match_back(), None, "bw end"); + } + check_fw_bw("XYZabcXYZ", "XYZ", 3, 3); + + // single char 1..4 byte str + check_fw_bw("XabcX", "X", 1, 3); + check_fw_bw("\u{00e9}abc\u{00e9}", "\u{00e9}", 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", "\u{20ac}", 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", "\u{1f60a}", 4, 3); + + // single char 2..4 byte + check_fw_bw("\u{00e9}abc\u{00e9}", '\u{00e9}', 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", '\u{20ac}', 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", '\u{1f60a}', 4, 3); + + check_fw_bw("\u{00e9}abc\u{00e9}", ['\u{00e9}'], 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", ['\u{20ac}'], 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", ['\u{1f60a}'], 4, 3); + + check_fw_bw("\u{00e9}abc\u{00e9}", |c| c == '\u{00e9}', 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", |c| c == '\u{20ac}', 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", |c| c == '\u{1f60a}', 4, 3); +} + +#[test] +fn two_way_next_reject_never_reports_empty_reject() { + // A reject must never be an empty range + + // Haystack fully covered by the pattern: there are no rejects at all. + let mut searcher = "XYZ".into_searcher("XYZ"); + assert_eq!(searcher.next_reject(), None); + assert_eq!(searcher.next_reject(), None); + + let mut searcher = "XYZ".into_searcher("XYZ"); + assert_eq!(searcher.next_reject_back(), None); + assert_eq!(searcher.next_reject_back(), None); + + // Haystack ending with a match: the last reject is followed by `None`, + // not by an empty reject after the final match. + let mut searcher = "XYZ".into_searcher("XYZabcXYZ"); + assert_eq!(searcher.next_reject(), Some((3, 6))); + assert_eq!(searcher.next_reject(), None); + + let mut searcher = "XYZ".into_searcher("XYZabcXYZ"); + assert_eq!(searcher.next_reject_back(), Some((3, 6))); + assert_eq!(searcher.next_reject_back(), None); +} From df3653a54a609c49380b9ed02fa24393cd4738e2 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Tue, 11 Aug 2026 18:52:47 -0400 Subject: [PATCH 04/16] coretests: Add a few tests for backward multibyte predicate Surprisingly enough there's no rfind tests for multibyte needles, at least it is possible to break this test without breaking anything other test. --- library/coretests/tests/pattern.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index d4bec996d89a1..bf2eb20220d0f 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -144,6 +144,20 @@ fn test_simple_iteration() { ); } +#[test] +fn backward_search_predicate() { + assert_eq!("abc".rfind(|c| c == 'a'), Some(0)); + assert_eq!("abcabc".rfind(|c| c == 'c'), Some(5)); + assert_eq!("éabc".rfind(['é']), Some(0)); + assert_eq!("éabc".rfind(|c| c == 'é'), Some(0)); + assert_eq!("éabcé".rfind(|c| c == 'é'), Some(5)); + assert_eq!("€abc".rfind(|c| c == '€'), Some(0)); + assert_eq!("😀abc".rfind(|c| c == '😀'), Some(0)); + + assert_eq!("abc".strip_suffix(|c| c == 'c'), Some("ab")); + assert_eq!("éabc".strip_suffix(|c| c == 'é'), None); +} + #[test] fn test_simple_search() { search_asserts!( From e0156dac25329ae8864e2afecfed61a55adc21b3 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:26:03 -0400 Subject: [PATCH 05/16] core: convert Pattern<'a> into Pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Haystack trait describing something that can be searched in and make core::str::Pattern (and related types) generic on that trait. This will allow Pattern to be used for types other than str (most notably OsStr). This somewhat follows the Pattern API 2.0 design. While that design is apparently abandoned (?), it is somewhat helpful when going for patterns on OsStr, so I’m going with it unless someone tells me otherwise. ;) For now leave Pattern, Haystack et al in core::str::pattern. Since they are no longer str-specific, I’ll move them to core::pattern in future commit. This one leaves them in place to make the diff smaller. @pacak: I moved some (or all new) of the `P: Pattern<&'a str> constraints into where clause to keep things narrower: ``` pub fn foo<'a, P: Pattern<&'a str>>(&'a self, pat: P, ...) ... ``` to ``` pub fn replacen<'a, P>(&'a self, pat: P, ...) ... where P: Pattern<&'a str>, ``` Original code had indices in Haystack abstracted as an associated type Cursor. Replaced with usize - Cursor adds noise with not much value. Changed wording in 2-3 places - for example Searcher is generic over a few types so it makes more sense to talk about split points in general with utf8 split points as an example for `&str`. --- library/alloc/src/str.rs | 10 +- library/alloc/src/string.rs | 28 +- library/alloctests/tests/str.rs | 6 +- library/core/src/str/iter.rs | 90 +++--- library/core/src/str/mod.rs | 100 ++++--- library/core/src/str/pattern.rs | 277 +++++++++++------- library/coretests/tests/pattern.rs | 7 +- tests/ui/suggestions/issue-104961.fixed | 4 +- tests/ui/suggestions/issue-104961.rs | 4 +- tests/ui/suggestions/issue-104961.stderr | 12 +- tests/ui/suggestions/issue-62843.stderr | 6 +- .../bound/assoc-fn-bound-root-obligation.rs | 6 +- .../assoc-fn-bound-root-obligation.stderr | 6 +- .../root-obligation.fixed | 2 +- .../suggest-dereferences/root-obligation.rs | 2 +- .../root-obligation.stderr | 4 +- 16 files changed, 334 insertions(+), 230 deletions(-) diff --git a/library/alloc/src/str.rs b/library/alloc/src/str.rs index 4c86d7e06ae44..ccaded8331b05 100644 --- a/library/alloc/src/str.rs +++ b/library/alloc/src/str.rs @@ -305,7 +305,10 @@ impl str { without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn replace(&self, from: P, to: &str) -> String { + pub fn replace<'a, P>(&'a self, from: P, to: &str) -> String + where + P: Pattern<&'a str>, + { // Fast path for replacing a single ASCII character with another. if let Some(from_byte) = match from.as_utf8_pattern() { Some(Utf8Pattern::StringPattern(s)) => match s.as_bytes() { @@ -363,7 +366,10 @@ impl str { #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "str_replacen", since = "1.16.0")] - pub fn replacen(&self, pat: P, to: &str, count: usize) -> String { + pub fn replacen<'a, P>(&'a self, pat: P, to: &str, count: usize) -> String + where + P: Pattern<&'a str>, + { // Hope to reduce the times of re-allocation let mut result = String::with_capacity(32); let mut last_end = 0; diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index b3fb35a086dcd..1a00ba901fb30 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -1593,7 +1593,10 @@ impl String { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_remove_matches", issue = "72826")] - pub fn remove_matches(&mut self, pat: P) { + pub fn remove_matches<'a, P>(&'a mut self, pat: P) + where + P: for<'x> Pattern<&'x str>, + { use core::str::pattern::Searcher; let rejections = { @@ -2143,7 +2146,10 @@ impl String { /// [replacen]: ../../std/primitive.str.html#method.replacen #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_replace_in_place", issue = "147949")] - pub fn replace_first(&mut self, from: P, to: &str) { + pub fn replace_first<'a, P>(&'a mut self, from: P, to: &str) + where + P: for<'x> Pattern<&'x str>, + { let range = match self.match_indices(from).next() { Some((start, match_str)) => start..start + match_str.len(), None => return, @@ -2169,9 +2175,9 @@ impl String { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_replace_in_place", issue = "147949")] - pub fn replace_last(&mut self, from: P, to: &str) + pub fn replace_last<'a, P>(&'a mut self, from: P, to: &str) where - for<'a> P::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + P: for<'x> Pattern<&'x str, Searcher: core::str::pattern::ReverseSearcher<&'x str>>, { let range = match self.rmatch_indices(from).next() { Some((start, match_str)) => start..start + match_str.len(), @@ -2664,10 +2670,10 @@ impl<'a> Extend<&'a core::ascii::Char> for String { reason = "API not fully fleshed out and ready to be stabilized", issue = "27721" )] -impl<'b> Pattern for &'b String { - type Searcher<'a> = <&'b str as Pattern>::Searcher<'a>; +impl<'a, 'b> Pattern<&'a str> for &'b String { + type Searcher = <&'b str as Pattern<&'a str>>::Searcher; - fn into_searcher(self, haystack: &str) -> <&'b str as Pattern>::Searcher<'_> { + fn into_searcher(self, haystack: &'a str) -> <&'b str as Pattern<&'a str>>::Searcher { self[..].into_searcher(haystack) } @@ -2687,17 +2693,17 @@ impl<'b> Pattern for &'b String { } #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool + fn is_suffix_of(self, haystack: &'a str) -> bool where - Self::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + Self::Searcher: core::str::pattern::ReverseSearcher<&'a str>, { self[..].is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> where - Self::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + Self::Searcher: core::str::pattern::ReverseSearcher<&'a str>, { self[..].strip_suffix_of(haystack) } diff --git a/library/alloctests/tests/str.rs b/library/alloctests/tests/str.rs index 830f6972f5af5..06825794ac145 100644 --- a/library/alloctests/tests/str.rs +++ b/library/alloctests/tests/str.rs @@ -2032,7 +2032,7 @@ mod pattern { fn cmp_search_to_vec

(rev: bool, pat: P, haystack: &str, right: Vec) where - P: for<'a> Pattern: ReverseSearcher<'a>>, + P: for<'a> Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let mut searcher = pat.into_searcher(haystack); let mut v = vec![]; @@ -2292,9 +2292,9 @@ generate_iterator_test! { fn different_str_pattern_forwarding_lifetimes() { use std::str::pattern::Pattern; - fn foo

(p: P) + fn foo<'a, P>(p: P) where - for<'b> &'b P: Pattern, + for<'b> &'b P: Pattern<&'a str>, { for _ in 0..3 { "asdf".find(&p); diff --git a/library/core/src/str/iter.rs b/library/core/src/str/iter.rs index 26c48d48d211e..21015c8c04aba 100644 --- a/library/core/src/str/iter.rs +++ b/library/core/src/str/iter.rs @@ -416,7 +416,7 @@ macro_rules! derive_pattern_clone { (clone $t:ident with |$s:ident| $e:expr) => { impl<'a, P> Clone for $t<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { let $s = self; @@ -429,7 +429,7 @@ macro_rules! derive_pattern_clone { /// This macro generates two public iterator structs /// wrapping a private internal one that makes use of the `Pattern` API. /// -/// For all patterns `P: Pattern` the following items will be +/// For all patterns `P: Pattern` the following items will be /// generated (generics omitted): /// /// struct $forward_iterator($internal_iterator); @@ -489,12 +489,12 @@ macro_rules! generate_pattern_iterators { } => { $(#[$forward_iterator_attribute])* $(#[$common_stability_attribute])* - pub struct $forward_iterator<'a, P: Pattern>(pub(super) $internal_iterator<'a, P>); + pub struct $forward_iterator<'a, P: Pattern<&'a str>>(pub(super) $internal_iterator<'a, P>); $(#[$common_stability_attribute])* impl<'a, P> fmt::Debug for $forward_iterator<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple(stringify!($forward_iterator)) @@ -504,7 +504,7 @@ macro_rules! generate_pattern_iterators { } $(#[$common_stability_attribute])* - impl<'a, P: Pattern> Iterator for $forward_iterator<'a, P> { + impl<'a, P: Pattern<&'a str>> Iterator for $forward_iterator<'a, P> { type Item = $iterty; #[inline] @@ -516,7 +516,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Clone for $forward_iterator<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { $forward_iterator(self.0.clone()) @@ -525,12 +525,12 @@ macro_rules! generate_pattern_iterators { $(#[$reverse_iterator_attribute])* $(#[$common_stability_attribute])* - pub struct $reverse_iterator<'a, P: Pattern>(pub(super) $internal_iterator<'a, P>); + pub struct $reverse_iterator<'a, P: Pattern<&'a str>>(pub(super) $internal_iterator<'a, P>); $(#[$common_stability_attribute])* impl<'a, P> fmt::Debug for $reverse_iterator<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple(stringify!($reverse_iterator)) @@ -542,7 +542,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Iterator for $reverse_iterator<'a, P> where - P: Pattern: ReverseSearcher<'a>>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { type Item = $iterty; @@ -555,7 +555,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Clone for $reverse_iterator<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { $reverse_iterator(self.0.clone()) @@ -563,12 +563,12 @@ macro_rules! generate_pattern_iterators { } #[stable(feature = "fused", since = "1.26.0")] - impl<'a, P: Pattern> FusedIterator for $forward_iterator<'a, P> {} + impl<'a, P: Pattern<&'a str>> FusedIterator for $forward_iterator<'a, P> {} #[stable(feature = "fused", since = "1.26.0")] impl<'a, P> FusedIterator for $reverse_iterator<'a, P> where - P: Pattern: ReverseSearcher<'a>>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, {} generate_pattern_iterators!($($t)* with $(#[$common_stability_attribute])*, @@ -583,7 +583,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> DoubleEndedIterator for $forward_iterator<'a, P> where - P: Pattern: DoubleEndedSearcher<'a>>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { #[inline] fn next_back(&mut self) -> Option<$iterty> { @@ -594,7 +594,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> DoubleEndedIterator for $reverse_iterator<'a, P> where - P: Pattern: DoubleEndedSearcher<'a>>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { #[inline] fn next_back(&mut self) -> Option<$iterty> { @@ -614,17 +614,17 @@ derive_pattern_clone! { with |s| SplitInternal { matcher: s.matcher.clone(), ..*s } } -pub(super) struct SplitInternal<'a, P: Pattern> { +pub(super) struct SplitInternal<'a, P: Pattern<&'a str>> { pub(super) start: usize, pub(super) end: usize, - pub(super) matcher: P::Searcher<'a>, + pub(super) matcher: P::Searcher, pub(super) allow_trailing_empty: bool, pub(super) finished: bool, } impl<'a, P> fmt::Debug for SplitInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SplitInternal") @@ -637,7 +637,7 @@ where } } -impl<'a, P: Pattern> SplitInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitInternal<'a, P> { #[inline] fn get_end(&mut self) -> Option<&'a str> { if !self.finished { @@ -694,7 +694,7 @@ impl<'a, P: Pattern> SplitInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { if self.finished { return None; @@ -731,7 +731,7 @@ impl<'a, P: Pattern> SplitInternal<'a, P> { #[inline] fn next_back_inclusive(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { if self.finished { return None; @@ -801,7 +801,7 @@ generate_pattern_iterators! { delegate double ended; } -impl<'a, P: Pattern> Split<'a, P> { +impl<'a, P: Pattern<&'a str>> Split<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -824,7 +824,7 @@ impl<'a, P: Pattern> Split<'a, P> { } } -impl<'a, P: Pattern> RSplit<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplit<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -865,7 +865,7 @@ generate_pattern_iterators! { delegate double ended; } -impl<'a, P: Pattern> SplitTerminator<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitTerminator<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -888,7 +888,7 @@ impl<'a, P: Pattern> SplitTerminator<'a, P> { } } -impl<'a, P: Pattern> RSplitTerminator<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplitTerminator<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -916,7 +916,7 @@ derive_pattern_clone! { with |s| SplitNInternal { iter: s.iter.clone(), ..*s } } -pub(super) struct SplitNInternal<'a, P: Pattern> { +pub(super) struct SplitNInternal<'a, P: Pattern<&'a str>> { pub(super) iter: SplitInternal<'a, P>, /// The number of splits remaining pub(super) count: usize, @@ -924,7 +924,7 @@ pub(super) struct SplitNInternal<'a, P: Pattern> { impl<'a, P> fmt::Debug for SplitNInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SplitNInternal") @@ -934,7 +934,7 @@ where } } -impl<'a, P: Pattern> SplitNInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitNInternal<'a, P> { #[inline] fn next(&mut self) -> Option<&'a str> { match self.count { @@ -953,7 +953,7 @@ impl<'a, P: Pattern> SplitNInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { match self.count { 0 => None, @@ -992,7 +992,7 @@ generate_pattern_iterators! { delegate single ended; } -impl<'a, P: Pattern> SplitN<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitN<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -1015,7 +1015,7 @@ impl<'a, P: Pattern> SplitN<'a, P> { } } -impl<'a, P: Pattern> RSplitN<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplitN<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -1043,18 +1043,18 @@ derive_pattern_clone! { with |s| MatchIndicesInternal(s.0.clone()) } -pub(super) struct MatchIndicesInternal<'a, P: Pattern>(pub(super) P::Searcher<'a>); +pub(super) struct MatchIndicesInternal<'a, P: Pattern<&'a str>>(pub(super) P::Searcher); impl<'a, P> fmt::Debug for MatchIndicesInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("MatchIndicesInternal").field(&self.0).finish() } } -impl<'a, P: Pattern> MatchIndicesInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> MatchIndicesInternal<'a, P> { #[inline] fn next(&mut self) -> Option<(usize, &'a str)> { self.0 @@ -1066,7 +1066,7 @@ impl<'a, P: Pattern> MatchIndicesInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<(usize, &'a str)> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { self.0 .next_match_back() @@ -1098,18 +1098,18 @@ derive_pattern_clone! { with |s| MatchesInternal(s.0.clone()) } -pub(super) struct MatchesInternal<'a, P: Pattern>(pub(super) P::Searcher<'a>); +pub(super) struct MatchesInternal<'a, P: Pattern<&'a str>>(pub(super) P::Searcher); impl<'a, P> fmt::Debug for MatchesInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("MatchesInternal").field(&self.0).finish() } } -impl<'a, P: Pattern> MatchesInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> MatchesInternal<'a, P> { #[inline] fn next(&mut self) -> Option<&'a str> { // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. @@ -1122,7 +1122,7 @@ impl<'a, P: Pattern> MatchesInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. self.0.next_match_back().map(|(a, b)| unsafe { @@ -1293,7 +1293,7 @@ pub struct SplitAsciiWhitespace<'a> { /// /// [`split_inclusive`]: str::split_inclusive #[stable(feature = "split_inclusive", since = "1.51.0")] -pub struct SplitInclusive<'a, P: Pattern>(pub(super) SplitInternal<'a, P>); +pub struct SplitInclusive<'a, P: Pattern<&'a str>>(pub(super) SplitInternal<'a, P>); #[stable(feature = "split_whitespace", since = "1.1.0")] impl<'a> Iterator for SplitWhitespace<'a> { @@ -1415,7 +1415,7 @@ impl<'a> SplitAsciiWhitespace<'a> { } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern> Iterator for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str>> Iterator for SplitInclusive<'a, P> { type Item = &'a str; #[inline] @@ -1425,7 +1425,7 @@ impl<'a, P: Pattern> Iterator for SplitInclusive<'a, P> { } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str, Searcher: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SplitInclusive").field("0", &self.0).finish() } @@ -1433,14 +1433,14 @@ impl<'a, P: Pattern: fmt::Debug>> fmt::Debug for SplitInclusive<'a, // FIXME(#26925) Remove in favor of `#[derive(Clone)]` #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: Clone>> Clone for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str, Searcher: Clone>> Clone for SplitInclusive<'a, P> { fn clone(&self) -> Self { SplitInclusive(self.0.clone()) } } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: DoubleEndedSearcher<'a>>> DoubleEndedIterator +impl<'a, P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>> DoubleEndedIterator for SplitInclusive<'a, P> { #[inline] @@ -1450,9 +1450,9 @@ impl<'a, P: Pattern: DoubleEndedSearcher<'a>>> DoubleEndedIterator } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern> FusedIterator for SplitInclusive<'a, P> {} +impl<'a, P: Pattern<&'a str>> FusedIterator for SplitInclusive<'a, P> {} -impl<'a, P: Pattern> SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitInclusive<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 79f4f29da2d43..c7c2cfd34c4af 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -1380,7 +1380,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn contains(&self, pat: P) -> bool { + pub fn contains<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> bool { pat.is_contained_in(self) } @@ -1418,7 +1418,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_diagnostic_item = "str_starts_with"] - pub fn starts_with(&self, pat: P) -> bool { + pub fn starts_with<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> bool { pat.is_prefix_of(self) } @@ -1443,9 +1443,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_diagnostic_item = "str_ends_with"] - pub fn ends_with(&self, pat: P) -> bool + pub fn ends_with<'a, P>(&'a self, pat: P) -> bool where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { pat.is_suffix_of(self) } @@ -1494,7 +1494,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn find(&self, pat: P) -> Option { + pub fn find<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Option { pat.into_searcher(self).next_match().map(|(i, _)| i) } @@ -1540,9 +1540,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rfind(&self, pat: P) -> Option + pub fn rfind<'a, P>(&'a self, pat: P) -> Option where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { pat.into_searcher(self).next_match_back().map(|(i, _)| i) } @@ -1668,7 +1668,7 @@ impl str { /// [`split_whitespace`]: str::split_whitespace #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn split(&self, pat: P) -> Split<'_, P> { + pub fn split<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Split<'a, P> { Split(SplitInternal { start: 0, end: self.len(), @@ -1709,7 +1709,7 @@ impl str { /// ``` #[stable(feature = "split_inclusive", since = "1.51.0")] #[inline] - pub fn split_inclusive(&self, pat: P) -> SplitInclusive<'_, P> { + pub fn split_inclusive<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> SplitInclusive<'a, P> { SplitInclusive(SplitInternal { start: 0, end: self.len(), @@ -1764,9 +1764,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplit(&self, pat: P) -> RSplit<'_, P> + pub fn rsplit<'a, P>(&'a self, pat: P) -> RSplit<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplit(self.split(pat).0) } @@ -1813,7 +1813,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn split_terminator(&self, pat: P) -> SplitTerminator<'_, P> { + pub fn split_terminator<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> SplitTerminator<'a, P> { SplitTerminator(SplitInternal { allow_trailing_empty: false, ..self.split(pat).0 }) } @@ -1859,9 +1859,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplit_terminator(&self, pat: P) -> RSplitTerminator<'_, P> + pub fn rsplit_terminator<'a, P>(&'a self, pat: P) -> RSplitTerminator<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplitTerminator(self.split_terminator(pat).0) } @@ -1914,7 +1914,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn splitn(&self, n: usize, pat: P) -> SplitN<'_, P> { + pub fn splitn<'a, P: Pattern<&'a str>>(&'a self, n: usize, pat: P) -> SplitN<'a, P> { SplitN(SplitNInternal { iter: self.split(pat).0, count: n }) } @@ -1963,9 +1963,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplitn(&self, n: usize, pat: P) -> RSplitN<'_, P> + pub fn rsplitn<'a, P>(&'a self, n: usize, pat: P) -> RSplitN<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplitN(self.splitn(n, pat).0) } @@ -1983,7 +1983,10 @@ impl str { /// ``` #[stable(feature = "str_split_once", since = "1.52.0")] #[inline] - pub fn split_once(&self, delimiter: P) -> Option<(&'_ str, &'_ str)> { + pub fn split_once<'a, P>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> + where + P: Pattern<&'a str>, + { let (start, end) = delimiter.into_searcher(self).next_match()?; // SAFETY: `Searcher` is known to return valid indices. unsafe { Some((self.get_unchecked(..start), self.get_unchecked(end..))) } @@ -2002,9 +2005,9 @@ impl str { /// ``` #[stable(feature = "str_split_once", since = "1.52.0")] #[inline] - pub fn rsplit_once(&self, delimiter: P) -> Option<(&'_ str, &'_ str)> + pub fn rsplit_once<'a, P>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let (start, end) = delimiter.into_searcher(self).next_match_back()?; // SAFETY: `Searcher` is known to return valid indices. @@ -2042,7 +2045,7 @@ impl str { /// ``` #[stable(feature = "str_matches", since = "1.2.0")] #[inline] - pub fn matches(&self, pat: P) -> Matches<'_, P> { + pub fn matches<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Matches<'a, P> { Matches(MatchesInternal(pat.into_searcher(self))) } @@ -2076,9 +2079,9 @@ impl str { /// ``` #[stable(feature = "str_matches", since = "1.2.0")] #[inline] - pub fn rmatches(&self, pat: P) -> RMatches<'_, P> + pub fn rmatches<'a, P>(&'a self, pat: P) -> RMatches<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RMatches(self.matches(pat).0) } @@ -2120,7 +2123,10 @@ impl str { /// ``` #[stable(feature = "str_match_indices", since = "1.5.0")] #[inline] - pub fn match_indices(&self, pat: P) -> MatchIndices<'_, P> { + pub fn match_indices<'a, P>(&'a self, pat: P) -> MatchIndices<'a, P> + where + P: Pattern<&'a str>, + { MatchIndices(MatchIndicesInternal(pat.into_searcher(self))) } @@ -2160,9 +2166,9 @@ impl str { /// ``` #[stable(feature = "str_match_indices", since = "1.5.0")] #[inline] - pub fn rmatch_indices(&self, pat: P) -> RMatchIndices<'_, P> + pub fn rmatch_indices<'a, P>(&'a self, pat: P) -> RMatchIndices<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RMatchIndices(self.match_indices(pat).0) } @@ -2375,9 +2381,9 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] - pub fn trim_matches(&self, pat: P) -> &str + pub fn trim_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: DoubleEndedSearcher<'a>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { let mut i = 0; let mut j = 0; @@ -2422,7 +2428,7 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] - pub fn trim_start_matches(&self, pat: P) -> &str { + pub fn trim_start_matches<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> &'a str { let mut i = self.len(); let mut matcher = pat.into_searcher(self); if let Some((a, _)) = matcher.next_reject() { @@ -2456,7 +2462,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "str_strip", since = "1.45.0")] - pub fn strip_prefix(&self, prefix: P) -> Option<&str> { + pub fn strip_prefix<'a, P>(&'a self, prefix: P) -> Option<&'a str> + where + P: Pattern<&'a str>, + { prefix.strip_prefix_of(self) } @@ -2484,9 +2493,9 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "str_strip", since = "1.45.0")] - pub fn strip_suffix(&self, suffix: P) -> Option<&str> + pub fn strip_suffix<'a, P>(&'a self, suffix: P) -> Option<&'a str> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { suffix.strip_suffix_of(self) } @@ -2521,9 +2530,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "strip_circumfix", since = "1.98.0")] - pub fn strip_circumfix(&self, prefix: P, suffix: S) -> Option<&str> + pub fn strip_circumfix<'a, P, S>(&'a self, prefix: P, suffix: S) -> Option<&'a str> where - for<'a> S::Searcher<'a>: ReverseSearcher<'a>, + S: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, + P: Pattern<&'a str>, { self.strip_prefix(prefix)?.strip_suffix(suffix) } @@ -2561,7 +2571,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[unstable(feature = "trim_prefix_suffix", issue = "142312")] - pub fn trim_prefix(&self, prefix: P) -> &str { + pub fn trim_prefix<'a, P>(&'a self, prefix: P) -> &'a str + where + P: Pattern<&'a str>, + { prefix.strip_prefix_of(self).unwrap_or(self) } @@ -2598,9 +2611,9 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[unstable(feature = "trim_prefix_suffix", issue = "142312")] - pub fn trim_suffix(&self, suffix: P) -> &str + pub fn trim_suffix<'a, P>(&'a self, suffix: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { suffix.strip_suffix_of(self).unwrap_or(self) } @@ -2641,9 +2654,9 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] - pub fn trim_end_matches(&self, pat: P) -> &str + pub fn trim_end_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let mut j = 0; let mut matcher = pat.into_searcher(self); @@ -2685,7 +2698,10 @@ impl str { note = "superseded by `trim_start_matches`", suggestion = "trim_start_matches" )] - pub fn trim_left_matches(&self, pat: P) -> &str { + pub fn trim_left_matches<'a, P>(&'a self, pat: P) -> &'a str + where + P: Pattern<&'a str>, + { self.trim_start_matches(pat) } @@ -2728,9 +2744,9 @@ impl str { note = "superseded by `trim_end_matches`", suggestion = "trim_end_matches" )] - pub fn trim_right_matches(&self, pat: P) -> &str + pub fn trim_right_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { self.trim_end_matches(pat) } diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index af48c2eae8dd6..1146e2b0962c3 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -40,22 +40,23 @@ use crate::cmp::Ordering; use crate::convert::TryInto as _; +use crate::ops::Range; use crate::slice::memchr; use crate::{cmp, fmt}; // Pattern -/// A string pattern. +/// A pattern which can be matched against a [`Haystack`]. /// -/// A `Pattern` expresses that the implementing type -/// can be used as a string pattern for searching in a [`&str`][str]. +/// A `Pattern` expresses that the implementing type can be used as a pattern +/// for searching in an `H`. /// -/// For example, both `'a'` and `"aa"` are patterns that -/// would match at index `1` in the string `"baaaab"`. +/// For example, both character `'a'` and string `"aa"` are patterns that would match +/// at index `1` in the string `"baaaab"`. /// /// The trait itself acts as a builder for an associated /// [`Searcher`] type, which does the actual work of finding -/// occurrences of the pattern in a string. +/// occurrences of the pattern in a haystack. /// /// Depending on the type of the pattern, the behavior of methods like /// [`str::find`] and [`str::contains`] can change. The table below describes @@ -96,66 +97,75 @@ use crate::{cmp, fmt}; /// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); /// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); /// ``` -pub trait Pattern: Sized { +pub trait Pattern: Sized { /// Associated searcher for this pattern - type Searcher<'a>: Searcher<'a>; + type Searcher: Searcher; /// Constructs the associated searcher from /// `self` and the `haystack` to search in. - fn into_searcher(self, haystack: &str) -> Self::Searcher<'_>; + fn into_searcher(self, haystack: H) -> Self::Searcher; /// Checks whether the pattern matches anywhere in the haystack #[inline] - fn is_contained_in(self, haystack: &str) -> bool { + fn is_contained_in(self, haystack: H) -> bool { self.into_searcher(haystack).next_match().is_some() } /// Checks whether the pattern matches at the front of the haystack #[inline] - fn is_prefix_of(self, haystack: &str) -> bool { - matches!(self.into_searcher(haystack).next(), SearchStep::Match(0, _)) + fn is_prefix_of(self, haystack: H) -> bool { + matches!( + self.into_searcher(haystack).next(), + SearchStep::Match(start, _) if start == haystack.cursor_at_front() + ) } /// Checks whether the pattern matches at the back of the haystack #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool + fn is_suffix_of(self, haystack: H) -> bool where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher, { - matches!(self.into_searcher(haystack).next_back(), SearchStep::Match(_, j) if haystack.len() == j) + matches!( + self.into_searcher(haystack).next_back(), + SearchStep::Match(_, end) if end == haystack.cursor_at_back() + ) } - /// Removes the pattern from the front of haystack, if it matches. + /// Removes the pattern from the front of a haystack, if it matches #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { - if let SearchStep::Match(start, len) = self.into_searcher(haystack).next() { + fn strip_prefix_of(self, haystack: H) -> Option { + if let SearchStep::Match(start, pos) = self.into_searcher(haystack).next() { debug_assert_eq!( - start, 0, + start, + haystack.cursor_at_front(), "The first search step from Searcher \ must include the first character" ); + let end = haystack.cursor_at_back(); // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(haystack.get_unchecked(len..)) } + Some(unsafe { haystack.get_unchecked(pos..end) }) } else { None } } - /// Removes the pattern from the back of haystack, if it matches. + /// Removes the pattern from the back of a haystack, if it matches. #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> + fn strip_suffix_of(self, haystack: H) -> Option where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher, { - if let SearchStep::Match(start, end) = self.into_searcher(haystack).next_back() { + if let SearchStep::Match(pos, end) = self.into_searcher(haystack).next_back() { debug_assert_eq!( end, - haystack.len(), + haystack.cursor_at_back(), "The first search step from ReverseSearcher \ must include the last character" ); + let start = haystack.cursor_at_front(); // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(haystack.get_unchecked(..start)) } + Some(unsafe { haystack.get_unchecked(start..pos) }) } else { None } @@ -179,6 +189,40 @@ pub enum Utf8Pattern<'a> { CharPattern(char), } +// Haystack + +/// A type which can be searched in using a [`Pattern`]. +/// +/// The trait is used in combination with [`Pattern`] trait to express a pattern +/// that can be used to search for elements in given haystack. +pub trait Haystack: Sized + Copy { + /// Returns cursor pointing at the beginning of the haystack. + fn cursor_at_front(self) -> usize; + + /// Returns cursor pointing at the end of the haystack. + fn cursor_at_back(self) -> usize; + + /// Returns whether the haystack is empty. + fn is_empty(self) -> bool { + self.cursor_at_front() == self.cursor_at_back() + } + + /// Returns portions of the haystack indicated by the cursor range. + /// + /// # Safety + /// + /// The range's start and end must be valid haystack split positions, + /// and start must not point to a position after end. + /// + /// Valid split positions are: + /// - the front of the haystack (as returned by + /// [`cursor_at_front()`][Self::cursor_at_front]), + /// - the back of the haystack (as returned by + /// [`cursor_at_back()`][Self::cursor_at_back]), or + /// - any cursor returned by a [`Searcher`] or [`ReverseSearcher`]. + unsafe fn get_unchecked(self, range: Range) -> Self; +} + // Searcher /// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. @@ -207,14 +251,15 @@ pub enum SearchStep { /// types of the [`Pattern`] trait. /// /// The trait is marked unsafe because the indices returned by the -/// [`next()`][Searcher::next] methods are required to lie on valid utf8 -/// boundaries in the haystack. This enables consumers of this trait to -/// slice the haystack without additional runtime checks. -pub unsafe trait Searcher<'a> { - /// Getter for the underlying string to be searched in +/// [`next()`][Searcher::next] methods are required to lie on valid split +/// positions of the haystack (for `&str` haystacks, these are utf8 character +/// boundaries). This enables consumers of this trait to slice the haystack +/// without additional runtime checks. +pub unsafe trait Searcher { + /// Getter for the underlying haystack to be searched in /// - /// Will always return the same [`&str`][str]. - fn haystack(&self) -> &'a str; + /// Will always return the same haystack. + fn haystack(&self) -> H; /// Performs the next search step starting from the front. /// @@ -228,7 +273,8 @@ pub unsafe trait Searcher<'a> { /// The stream of [`Match`][SearchStep::Match] and /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on utf8 boundaries. + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. /// /// A [`Match`][SearchStep::Match] result needs to contain the whole matched /// pattern, however [`Reject`][SearchStep::Reject] results may be split up @@ -287,7 +333,7 @@ pub unsafe trait Searcher<'a> { /// /// For the reason why this trait is marked unsafe, see the /// parent trait [`Searcher`]. -pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { +pub unsafe trait ReverseSearcher: Searcher { /// Performs the next search step starting from the back. /// /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` @@ -300,7 +346,8 @@ pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { /// The stream of [`Match`][SearchStep::Match] and /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on utf8 boundaries. + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. /// /// A [`Match`][SearchStep::Match] result needs to contain the whole matched /// pattern, however [`Reject`][SearchStep::Reject] results may be split up @@ -359,13 +406,39 @@ pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { /// `(&str)::Searcher` is not a `DoubleEndedSearcher` because /// the pattern `"aa"` in the haystack `"aaa"` matches as either /// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. -pub trait DoubleEndedSearcher<'a>: ReverseSearcher<'a> {} +pub trait DoubleEndedSearcher: ReverseSearcher {} + +///////////////////////////////////////////////////////////////////////////// +// Impl for Haystack +///////////////////////////////////////////////////////////////////////////// + +impl<'a> Haystack for &'a str { + #[inline(always)] + fn cursor_at_front(self) -> usize { + 0 + } + #[inline(always)] + fn cursor_at_back(self) -> usize { + self.len() + } + + #[inline(always)] + fn is_empty(self) -> bool { + self.is_empty() + } + + #[inline(always)] + unsafe fn get_unchecked(self, range: Range) -> Self { + // SAFETY: Caller promises position is a character boundary. + unsafe { self.get_unchecked(range) } + } +} ///////////////////////////////////////////////////////////////////////////// // Impl for char ///////////////////////////////////////////////////////////////////////////// -/// Associated type for `::Searcher<'a>`. +/// Associated type for `>::Searcher`. #[derive(Clone, Debug)] pub struct CharSearcher<'a> { haystack: &'a str, @@ -398,7 +471,7 @@ impl CharSearcher<'_> { } } -unsafe impl<'a> Searcher<'a> for CharSearcher<'a> { +unsafe impl<'a> Searcher<&'a str> for CharSearcher<'a> { #[inline] fn haystack(&self) -> &'a str { self.haystack @@ -476,7 +549,7 @@ unsafe impl<'a> Searcher<'a> for CharSearcher<'a> { // let next_reject use the default implementation from the Searcher trait } -unsafe impl<'a> ReverseSearcher<'a> for CharSearcher<'a> { +unsafe impl<'a> ReverseSearcher<&'a str> for CharSearcher<'a> { #[inline] fn next_back(&mut self) -> SearchStep { let old_finger = self.finger_back; @@ -550,7 +623,7 @@ unsafe impl<'a> ReverseSearcher<'a> for CharSearcher<'a> { // let next_reject_back use the default implementation from the Searcher trait } -impl<'a> DoubleEndedSearcher<'a> for CharSearcher<'a> {} +impl<'a> DoubleEndedSearcher<&'a str> for CharSearcher<'a> {} /// Searches for chars that are equal to a given [`char`]. /// @@ -559,11 +632,11 @@ impl<'a> DoubleEndedSearcher<'a> for CharSearcher<'a> {} /// ``` /// assert_eq!("Hello world".find('o'), Some(4)); /// ``` -impl Pattern for char { - type Searcher<'a> = CharSearcher<'a>; +impl<'a> Pattern<&'a str> for char { + type Searcher = CharSearcher<'a>; #[inline] - fn into_searcher<'a>(self, haystack: &'a str) -> Self::Searcher<'a> { + fn into_searcher(self, haystack: &'a str) -> Self::Searcher { let mut utf8_encoded = [0; char::MAX_LEN_UTF8]; let utf8_size = self .encode_utf8(&mut utf8_encoded) @@ -602,17 +675,17 @@ impl Pattern for char { } #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool + fn is_suffix_of(self, haystack: &'a str) -> bool where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher<&'a str>, { self.encode_utf8(&mut [0u8; 4]).is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher<&'a str>, { self.encode_utf8(&mut [0u8; 4]).strip_suffix_of(haystack) } @@ -672,16 +745,16 @@ struct MultiCharEqSearcher<'a, C: MultiCharEq> { char_indices: super::CharIndices<'a>, } -impl Pattern for MultiCharEqPattern { - type Searcher<'a> = MultiCharEqSearcher<'a, C>; +impl<'a, C: MultiCharEq> Pattern<&'a str> for MultiCharEqPattern { + type Searcher = MultiCharEqSearcher<'a, C>; #[inline] - fn into_searcher(self, haystack: &str) -> MultiCharEqSearcher<'_, C> { + fn into_searcher(self, haystack: &'a str) -> MultiCharEqSearcher<'a, C> { MultiCharEqSearcher { haystack, char_eq: self.0, char_indices: haystack.char_indices() } } } -unsafe impl<'a, C: MultiCharEq> Searcher<'a> for MultiCharEqSearcher<'a, C> { +unsafe impl<'a, C: MultiCharEq> Searcher<&'a str> for MultiCharEqSearcher<'a, C> { #[inline] fn haystack(&self) -> &'a str { self.haystack @@ -706,7 +779,7 @@ unsafe impl<'a, C: MultiCharEq> Searcher<'a> for MultiCharEqSearcher<'a, C> { } } -unsafe impl<'a, C: MultiCharEq> ReverseSearcher<'a> for MultiCharEqSearcher<'a, C> { +unsafe impl<'a, C: MultiCharEq> ReverseSearcher<&'a str> for MultiCharEqSearcher<'a, C> { #[inline] fn next_back(&mut self) -> SearchStep { let s = &mut self.char_indices; @@ -726,46 +799,46 @@ unsafe impl<'a, C: MultiCharEq> ReverseSearcher<'a> for MultiCharEqSearcher<'a, } } -impl<'a, C: MultiCharEq> DoubleEndedSearcher<'a> for MultiCharEqSearcher<'a, C> {} +impl<'a, C: MultiCharEq> DoubleEndedSearcher<&'a str> for MultiCharEqSearcher<'a, C> {} ///////////////////////////////////////////////////////////////////////////// macro_rules! pattern_methods { ($a:lifetime, $t:ty, $pmap:expr, $smap:expr) => { - type Searcher<$a> = $t; + type Searcher = $t; #[inline] - fn into_searcher<$a>(self, haystack: &$a str) -> $t { + fn into_searcher(self, haystack: &$a str) -> $t { ($smap)(($pmap)(self).into_searcher(haystack)) } #[inline] - fn is_contained_in<$a>(self, haystack: &$a str) -> bool { + fn is_contained_in(self, haystack: &$a str) -> bool { ($pmap)(self).is_contained_in(haystack) } #[inline] - fn is_prefix_of<$a>(self, haystack: &$a str) -> bool { + fn is_prefix_of(self, haystack: &$a str) -> bool { ($pmap)(self).is_prefix_of(haystack) } #[inline] - fn strip_prefix_of<$a>(self, haystack: &$a str) -> Option<&$a str> { + fn strip_prefix_of(self, haystack: &$a str) -> Option<&$a str> { ($pmap)(self).strip_prefix_of(haystack) } #[inline] - fn is_suffix_of<$a>(self, haystack: &$a str) -> bool + fn is_suffix_of(self, haystack: &$a str) -> bool where - $t: ReverseSearcher<$a>, + $t: ReverseSearcher<&$a str>, { ($pmap)(self).is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<$a>(self, haystack: &$a str) -> Option<&$a str> + fn strip_suffix_of(self, haystack: &$a str) -> Option<&$a str> where - $t: ReverseSearcher<$a>, + $t: ReverseSearcher<&$a str>, { ($pmap)(self).strip_suffix_of(haystack) } @@ -807,16 +880,16 @@ macro_rules! searcher_methods { }; } -/// Associated type for `<[char; N] as Pattern>::Searcher<'a>`. +/// Associated type for `<[char; N] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] pub struct CharArraySearcher<'a, const N: usize>( - as Pattern>::Searcher<'a>, + as Pattern<&'a str>>::Searcher, ); -/// Associated type for `<&[char; N] as Pattern>::Searcher<'a>`. +/// Associated type for `<&[char; N] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] pub struct CharArrayRefSearcher<'a, 'b, const N: usize>( - as Pattern>::Searcher<'a>, + as Pattern<&'a str>>::Searcher, ); /// Searches for chars that are equal to any of the [`char`]s in the array. @@ -827,19 +900,19 @@ pub struct CharArrayRefSearcher<'a, 'b, const N: usize>( /// assert_eq!("Hello world".find(['o', 'l']), Some(2)); /// assert_eq!("Hello world".find(['h', 'w']), Some(6)); /// ``` -impl Pattern for [char; N] { +impl<'a, const N: usize> Pattern<&'a str> for [char; N] { pattern_methods!('a, CharArraySearcher<'a, N>, MultiCharEqPattern, CharArraySearcher); } -unsafe impl<'a, const N: usize> Searcher<'a> for CharArraySearcher<'a, N> { +unsafe impl<'a, const N: usize> Searcher<&'a str> for CharArraySearcher<'a, N> { searcher_methods!(forward); } -unsafe impl<'a, const N: usize> ReverseSearcher<'a> for CharArraySearcher<'a, N> { +unsafe impl<'a, const N: usize> ReverseSearcher<&'a str> for CharArraySearcher<'a, N> { searcher_methods!(reverse); } -impl<'a, const N: usize> DoubleEndedSearcher<'a> for CharArraySearcher<'a, N> {} +impl<'a, const N: usize> DoubleEndedSearcher<&'a str> for CharArraySearcher<'a, N> {} /// Searches for chars that are equal to any of the [`char`]s in the array. /// @@ -849,19 +922,19 @@ impl<'a, const N: usize> DoubleEndedSearcher<'a> for CharArraySearcher<'a, N> {} /// assert_eq!("Hello world".find(&['o', 'l']), Some(2)); /// assert_eq!("Hello world".find(&['h', 'w']), Some(6)); /// ``` -impl<'b, const N: usize> Pattern for &'b [char; N] { +impl<'a, 'b, const N: usize> Pattern<&'a str> for &'b [char; N] { pattern_methods!('a, CharArrayRefSearcher<'a, 'b, N>, MultiCharEqPattern, CharArrayRefSearcher); } -unsafe impl<'a, 'b, const N: usize> Searcher<'a> for CharArrayRefSearcher<'a, 'b, N> { +unsafe impl<'a, 'b, const N: usize> Searcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> { searcher_methods!(forward); } -unsafe impl<'a, 'b, const N: usize> ReverseSearcher<'a> for CharArrayRefSearcher<'a, 'b, N> { +unsafe impl<'a, 'b, const N: usize> ReverseSearcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> { searcher_methods!(reverse); } -impl<'a, 'b, const N: usize> DoubleEndedSearcher<'a> for CharArrayRefSearcher<'a, 'b, N> {} +impl<'a, 'b, const N: usize> DoubleEndedSearcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> {} ///////////////////////////////////////////////////////////////////////////// // Impl for &[char] @@ -869,19 +942,21 @@ impl<'a, 'b, const N: usize> DoubleEndedSearcher<'a> for CharArrayRefSearcher<'a // Todo: Change / Remove due to ambiguity in meaning. -/// Associated type for `<&[char] as Pattern>::Searcher<'a>`. +/// Associated type for `<&[char] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] -pub struct CharSliceSearcher<'a, 'b>( as Pattern>::Searcher<'a>); +pub struct CharSliceSearcher<'a, 'b>( + as Pattern<&'a str>>::Searcher, +); -unsafe impl<'a, 'b> Searcher<'a> for CharSliceSearcher<'a, 'b> { +unsafe impl<'a, 'b> Searcher<&'a str> for CharSliceSearcher<'a, 'b> { searcher_methods!(forward); } -unsafe impl<'a, 'b> ReverseSearcher<'a> for CharSliceSearcher<'a, 'b> { +unsafe impl<'a, 'b> ReverseSearcher<&'a str> for CharSliceSearcher<'a, 'b> { searcher_methods!(reverse); } -impl<'a, 'b> DoubleEndedSearcher<'a> for CharSliceSearcher<'a, 'b> {} +impl<'a, 'b> DoubleEndedSearcher<&'a str> for CharSliceSearcher<'a, 'b> {} /// Searches for chars that are equal to any of the [`char`]s in the slice. /// @@ -891,7 +966,7 @@ impl<'a, 'b> DoubleEndedSearcher<'a> for CharSliceSearcher<'a, 'b> {} /// assert_eq!("Hello world".find(&['o', 'l'][..]), Some(2)); /// assert_eq!("Hello world".find(&['h', 'w'][..]), Some(6)); /// ``` -impl<'b> Pattern for &'b [char] { +impl<'a, 'b> Pattern<&'a str> for &'b [char] { pattern_methods!('a, CharSliceSearcher<'a, 'b>, MultiCharEqPattern, CharSliceSearcher); } @@ -899,9 +974,9 @@ impl<'b> Pattern for &'b [char] { // Impl for F: FnMut(char) -> bool ///////////////////////////////////////////////////////////////////////////// -/// Associated type for `::Searcher<'a>`. +/// Associated type for `>::Searcher`. #[derive(Clone)] -pub struct CharPredicateSearcher<'a, F>( as Pattern>::Searcher<'a>) +pub struct CharPredicateSearcher<'a, F>( as Pattern<&'a str>>::Searcher) where F: FnMut(char) -> bool; @@ -916,21 +991,21 @@ where .finish() } } -unsafe impl<'a, F> Searcher<'a> for CharPredicateSearcher<'a, F> +unsafe impl<'a, F> Searcher<&'a str> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool, { searcher_methods!(forward); } -unsafe impl<'a, F> ReverseSearcher<'a> for CharPredicateSearcher<'a, F> +unsafe impl<'a, F> ReverseSearcher<&'a str> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool, { searcher_methods!(reverse); } -impl<'a, F> DoubleEndedSearcher<'a> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool {} +impl<'a, F: FnMut(char) -> bool> DoubleEndedSearcher<&'a str> for CharPredicateSearcher<'a, F> {} /// Searches for [`char`]s that match the given predicate. /// @@ -940,7 +1015,7 @@ impl<'a, F> DoubleEndedSearcher<'a> for CharPredicateSearcher<'a, F> where F: Fn /// assert_eq!("Hello world".find(char::is_uppercase), Some(0)); /// assert_eq!("Hello world".find(|c| "aeiou".contains(c)), Some(1)); /// ``` -impl Pattern for F +impl<'a, F> Pattern<&'a str> for F where F: FnMut(char) -> bool, { @@ -952,7 +1027,7 @@ where ///////////////////////////////////////////////////////////////////////////// /// Delegates to the `&str` impl. -impl<'b, 'c> Pattern for &'c &'b str { +impl<'a, 'b, 'c> Pattern<&'a str> for &'c &'b str { pattern_methods!('a, StrSearcher<'a, 'b>, |&s| s, |s| s); } @@ -970,23 +1045,23 @@ impl<'b, 'c> Pattern for &'c &'b str { /// ``` /// assert_eq!("Hello world".find("world"), Some(6)); /// ``` -impl<'b> Pattern for &'b str { - type Searcher<'a> = StrSearcher<'a, 'b>; +impl<'a, 'b> Pattern<&'a str> for &'b str { + type Searcher = StrSearcher<'a, 'b>; #[inline] - fn into_searcher(self, haystack: &str) -> StrSearcher<'_, 'b> { + fn into_searcher(self, haystack: &'a str) -> StrSearcher<'a, 'b> { StrSearcher::new(haystack, self) } /// Checks whether the pattern matches at the front of the haystack. #[inline] - fn is_prefix_of(self, haystack: &str) -> bool { + fn is_prefix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().starts_with(self.as_bytes()) } /// Checks whether the pattern matches anywhere in the haystack #[inline] - fn is_contained_in(self, haystack: &str) -> bool { + fn is_contained_in(self, haystack: &'a str) -> bool { if self.is_empty() { return true; } @@ -1016,7 +1091,7 @@ impl<'b> Pattern for &'b str { /// Removes the pattern from the front of haystack, if it matches. #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { if self.is_prefix_of(haystack) { // SAFETY: prefix was just verified to exist. unsafe { Some(haystack.get_unchecked(self.len()..)) } @@ -1027,18 +1102,18 @@ impl<'b> Pattern for &'b str { /// Checks whether the pattern matches at the back of the haystack. #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool + fn is_suffix_of(self, haystack: &'a str) -> bool where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher<&'a str>, { haystack.as_bytes().ends_with(self.as_bytes()) } /// Removes the pattern from the back of haystack, if it matches. #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> where - Self::Searcher<'a>: ReverseSearcher<'a>, + Self::Searcher: ReverseSearcher<&'a str>, { if self.is_suffix_of(haystack) { let i = haystack.len() - self.len(); @@ -1060,7 +1135,7 @@ impl<'b> Pattern for &'b str { ///////////////////////////////////////////////////////////////////////////// #[derive(Clone, Debug)] -/// Associated type for `<&str as Pattern>::Searcher<'a>`. +/// Associated type for `<&str as Pattern<&'a str>>::Searcher`. pub struct StrSearcher<'a, 'b> { haystack: &'a str, needle: &'b str, @@ -1128,7 +1203,7 @@ impl<'a, 'b> StrSearcher<'a, 'b> { } } -unsafe impl<'a, 'b> Searcher<'a> for StrSearcher<'a, 'b> { +unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn haystack(&self) -> &'a str { self.haystack @@ -1250,7 +1325,7 @@ unsafe impl<'a, 'b> Searcher<'a> for StrSearcher<'a, 'b> { } } -unsafe impl<'a, 'b> ReverseSearcher<'a> for StrSearcher<'a, 'b> { +unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn next_back(&mut self) -> SearchStep { match self.searcher { diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index 30d22daae100b..9546007bc942e 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -510,10 +510,11 @@ fn two_way_next_reject_skips_matches() { // ulen - unmatch len // haystack is always a concatenation of [Match, Reject, Match] #[track_caller] - fn check_fw_bw<'a, T>(haystack: &'a str, pat: T, plen: usize, ulen: usize) + fn check_fw_bw(haystack: H, pat: T, plen: usize, ulen: usize) where - T: Pattern + Copy, - T::Searcher<'a>: ReverseSearcher<'a>, + H: Haystack, + T: Pattern + Copy, + T::Searcher: ReverseSearcher, { // fragments let f1 = (0, plen); diff --git a/tests/ui/suggestions/issue-104961.fixed b/tests/ui/suggestions/issue-104961.fixed index 3019242880f82..7c76b254578d4 100644 --- a/tests/ui/suggestions/issue-104961.fixed +++ b/tests/ui/suggestions/issue-104961.fixed @@ -2,12 +2,12 @@ fn foo(x: &str) -> bool { x.starts_with(&("hi".to_string() + " you")) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn foo2(x: &str) -> bool { x.starts_with(&"hi".to_string()) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn main() { diff --git a/tests/ui/suggestions/issue-104961.rs b/tests/ui/suggestions/issue-104961.rs index b315e9bab0d1a..8d796f96d73f7 100644 --- a/tests/ui/suggestions/issue-104961.rs +++ b/tests/ui/suggestions/issue-104961.rs @@ -2,12 +2,12 @@ fn foo(x: &str) -> bool { x.starts_with("hi".to_string() + " you") - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn foo2(x: &str) -> bool { x.starts_with("hi".to_string()) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn main() { diff --git a/tests/ui/suggestions/issue-104961.stderr b/tests/ui/suggestions/issue-104961.stderr index 0d229e6dada5f..35f166868d3aa 100644 --- a/tests/ui/suggestions/issue-104961.stderr +++ b/tests/ui/suggestions/issue-104961.stderr @@ -1,12 +1,12 @@ -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-104961.rs:4:19 | LL | x.starts_with("hi".to_string() + " you") - | ----------- ^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Pattern` is not implemented for `String` + | ----------- ^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::starts_with` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here @@ -14,15 +14,15 @@ help: consider borrowing here LL | x.starts_with(&("hi".to_string() + " you")) | ++ + -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-104961.rs:9:19 | LL | x.starts_with("hi".to_string()) - | ----------- ^^^^^^^^^^^^^^^^ the trait `Pattern` is not implemented for `String` + | ----------- ^^^^^^^^^^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::starts_with` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here diff --git a/tests/ui/suggestions/issue-62843.stderr b/tests/ui/suggestions/issue-62843.stderr index c3c0360b3a9d1..9a558d06b21cf 100644 --- a/tests/ui/suggestions/issue-62843.stderr +++ b/tests/ui/suggestions/issue-62843.stderr @@ -1,12 +1,12 @@ -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-62843.rs:4:32 | LL | println!("{:?}", line.find(pattern)); - | ---- ^^^^^^^ the trait `Pattern` is not implemented for `String` + | ---- ^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::find` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here diff --git a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs index 667d283bea3e4..22126705a6840 100644 --- a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs +++ b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs @@ -1,10 +1,10 @@ fn strip_lf(s: &str) -> &str { s.strip_suffix(b'\n').unwrap_or(s) - //~^ ERROR the trait bound `u8: Pattern` is not satisfied + //~^ ERROR the trait bound `u8: Pattern<&str>` is not satisfied //~| NOTE required by a bound introduced by this call //~| NOTE the trait `FnMut(char)` is not implemented for `u8` - //~| HELP the following other types implement trait `Pattern`: - //~| NOTE required for `u8` to implement `Pattern` + //~| HELP the following other types implement trait `Pattern`: + //~| NOTE required for `u8` to implement `Pattern<&str>` //~| NOTE required by a bound in `core::str::::strip_suffix` } diff --git a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr index 1cd62d2cbdbaa..ec33e71dfe683 100644 --- a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr +++ b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr @@ -1,4 +1,4 @@ -error[E0277]: the trait bound `u8: Pattern` is not satisfied +error[E0277]: the trait bound `u8: Pattern<&str>` is not satisfied --> $DIR/assoc-fn-bound-root-obligation.rs:2:20 | LL | s.strip_suffix(b'\n').unwrap_or(s) @@ -6,7 +6,7 @@ LL | s.strip_suffix(b'\n').unwrap_or(s) | | | required by a bound introduced by this call | - = help: the following other types implement trait `Pattern`: + = help: the following other types implement trait `Pattern`: &'b String &'b [char; N] &'b [char] @@ -14,7 +14,7 @@ LL | s.strip_suffix(b'\n').unwrap_or(s) &'c &'b str [char; N] char - = note: required for `u8` to implement `Pattern` + = note: required for `u8` to implement `Pattern<&str>` note: required by a bound in `core::str::::strip_suffix` --> $SRC_DIR/core/src/str/mod.rs:LL:COL diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.fixed b/tests/ui/traits/suggest-dereferences/root-obligation.fixed index ad0f184dc9a42..a6edc29244e46 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.fixed +++ b/tests/ui/traits/suggest-dereferences/root-obligation.fixed @@ -4,7 +4,7 @@ fn get_vowel_count(string: &str) -> usize { string .chars() .filter(|c| "aeiou".contains(*c)) - //~^ ERROR the trait bound `&char: Pattern` is not satisfied + //~^ ERROR the trait bound `&char: Pattern<&str>` is not satisfied .count() } diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.rs b/tests/ui/traits/suggest-dereferences/root-obligation.rs index a31a9955d313a..d14e58f4af01a 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.rs +++ b/tests/ui/traits/suggest-dereferences/root-obligation.rs @@ -4,7 +4,7 @@ fn get_vowel_count(string: &str) -> usize { string .chars() .filter(|c| "aeiou".contains(c)) - //~^ ERROR the trait bound `&char: Pattern` is not satisfied + //~^ ERROR the trait bound `&char: Pattern<&str>` is not satisfied .count() } diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.stderr b/tests/ui/traits/suggest-dereferences/root-obligation.stderr index dafd3469b6f03..fa4cf22e463c6 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.stderr +++ b/tests/ui/traits/suggest-dereferences/root-obligation.stderr @@ -1,4 +1,4 @@ -error[E0277]: the trait bound `&char: Pattern` is not satisfied +error[E0277]: the trait bound `&char: Pattern<&str>` is not satisfied --> $DIR/root-obligation.rs:6:38 | LL | .filter(|c| "aeiou".contains(c)) @@ -7,7 +7,7 @@ LL | .filter(|c| "aeiou".contains(c)) | required by a bound introduced by this call | = note: required for `&char` to implement `FnOnce(char)` - = note: required for `&char` to implement `Pattern` + = note: required for `&char` to implement `Pattern<&str>` note: required by a bound in `core::str::::contains` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider dereferencing here From 9471adf50a93760bed23fc5ae89d215c61d3f0a8 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:31:43 -0400 Subject: [PATCH 06/16] core: move Pattern et al to core::pattern module Pattern is no longer str-specific, so move it from core::str::pattern module to a new core::pattern module. This introduces no changes in behaviour or implementation. Just moves stuff around and adjusts documentation. --- library/alloc/src/str.rs | 2 +- library/alloc/src/string.rs | 10 +- library/alloctests/tests/str.rs | 8 +- library/core/src/lib.rs | 1 + library/core/src/pattern.rs | 401 +++++++++++++++++ library/core/src/str/iter.rs | 2 +- library/core/src/str/mod.rs | 2 +- library/core/src/str/pattern.rs | 408 ++---------------- library/coretests/tests/pattern.rs | 2 +- library/std/src/lib.rs | 2 + .../crates/stdarch-gen-arm/src/wildstring.rs | 4 +- 11 files changed, 448 insertions(+), 394 deletions(-) create mode 100644 library/core/src/pattern.rs diff --git a/library/alloc/src/str.rs b/library/alloc/src/str.rs index ccaded8331b05..4e8686d720579 100644 --- a/library/alloc/src/str.rs +++ b/library/alloc/src/str.rs @@ -10,6 +10,7 @@ use core::borrow::{Borrow, BorrowMut}; use core::iter::FusedIterator; use core::mem::MaybeUninit; +use core::pattern::{Pattern, Utf8Pattern}; #[stable(feature = "encode_utf16", since = "1.8.0")] pub use core::str::EncodeUtf16; #[stable(feature = "split_ascii_whitespace", since = "1.34.0")] @@ -20,7 +21,6 @@ pub use core::str::SplitInclusive; pub use core::str::SplitWhitespace; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::pattern; -use core::str::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher, Utf8Pattern}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::{Bytes, CharIndices, Chars, from_utf8, from_utf8_mut}; #[stable(feature = "str_escape", since = "1.34.0")] diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 1a00ba901fb30..787c6acb42945 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -53,7 +53,7 @@ use core::ops::Add; #[cfg(not(no_global_oom_handling))] use core::ops::AddAssign; use core::ops::{self, Range, RangeBounds}; -use core::str::pattern::{Pattern, Utf8Pattern}; +use core::pattern::{Pattern, Utf8Pattern}; use core::{fmt, hash, ptr, slice}; #[cfg(not(no_global_oom_handling))] @@ -1597,7 +1597,7 @@ impl String { where P: for<'x> Pattern<&'x str>, { - use core::str::pattern::Searcher; + use core::pattern::Searcher; let rejections = { let mut searcher = pat.into_searcher(self); @@ -2177,7 +2177,7 @@ impl String { #[unstable(feature = "string_replace_in_place", issue = "147949")] pub fn replace_last<'a, P>(&'a mut self, from: P, to: &str) where - P: for<'x> Pattern<&'x str, Searcher: core::str::pattern::ReverseSearcher<&'x str>>, + P: for<'x> Pattern<&'x str, Searcher: core::pattern::ReverseSearcher<&'x str>>, { let range = match self.rmatch_indices(from).next() { Some((start, match_str)) => start..start + match_str.len(), @@ -2695,7 +2695,7 @@ impl<'a, 'b> Pattern<&'a str> for &'b String { #[inline] fn is_suffix_of(self, haystack: &'a str) -> bool where - Self::Searcher: core::str::pattern::ReverseSearcher<&'a str>, + Self::Searcher: core::pattern::ReverseSearcher<&'a str>, { self[..].is_suffix_of(haystack) } @@ -2703,7 +2703,7 @@ impl<'a, 'b> Pattern<&'a str> for &'b String { #[inline] fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> where - Self::Searcher: core::str::pattern::ReverseSearcher<&'a str>, + Self::Searcher: core::pattern::ReverseSearcher<&'a str>, { self[..].strip_suffix_of(haystack) } diff --git a/library/alloctests/tests/str.rs b/library/alloctests/tests/str.rs index 06825794ac145..86967f8ae5692 100644 --- a/library/alloctests/tests/str.rs +++ b/library/alloctests/tests/str.rs @@ -2009,14 +2009,14 @@ fn test_repeat() { } mod pattern { - use std::str::pattern::SearchStep::{self, Done, Match, Reject}; - use std::str::pattern::{Pattern, ReverseSearcher, Searcher}; + use std::pattern::SearchStep::{self, Done, Match, Reject}; + use std::pattern::{Pattern, ReverseSearcher, Searcher}; macro_rules! make_test { ($name:ident, $p:expr, $h:expr, [$($e:expr,)*]) => { #[allow(unused_imports)] mod $name { - use std::str::pattern::SearchStep::{Match, Reject}; + use std::pattern::SearchStep::{Match, Reject}; use super::{cmp_search_to_vec}; #[test] fn fwd() { @@ -2290,7 +2290,7 @@ generate_iterator_test! { #[test] fn different_str_pattern_forwarding_lifetimes() { - use std::str::pattern::Pattern; + use std::pattern::Pattern; fn foo<'a, P>(p: P) where diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index f026434acbbc1..c5bf8bea6accc 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -318,6 +318,7 @@ pub mod unsafe_binder; pub mod fmt; pub mod hash; +pub mod pattern; pub mod slice; pub mod str; pub mod time; diff --git a/library/core/src/pattern.rs b/library/core/src/pattern.rs new file mode 100644 index 0000000000000..ca413ee3b56df --- /dev/null +++ b/library/core/src/pattern.rs @@ -0,0 +1,401 @@ +//! The Pattern API. +//! +//! The Pattern API provides a generic mechanism for using different pattern +//! types when searching through different objects. +//! +//! For more details, see the traits [`Pattern`], [`Haystack`], [`Searcher`], +//! [`ReverseSearcher`] and [`DoubleEndedSearcher`]. Although this API is +//! unstable, it is exposed via stable methods on corresponding haystack types. +//! +//! # Examples +//! +//! [`Pattern<&str>`] is [implemented][pattern-impls] in the stable API for +//! [`&str`][`str`], [`char`], slices of [`char`], and functions and closures +//! implementing `FnMut(char) -> bool`. +//! +//! ``` +//! let s = "Can you find a needle in a haystack?"; +//! +//! // &str pattern +//! assert_eq!(s.find("you"), Some(4)); +//! // char pattern +//! assert_eq!(s.find('n'), Some(2)); +//! // array of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u']), Some(1)); +//! // slice of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u'][..]), Some(1)); +//! // closure pattern +//! assert_eq!(s.find(|c: char| c.is_ascii_punctuation()), Some(35)); +//! ``` +//! +//! [pattern-impls]: Pattern#implementors + +#![unstable( + feature = "pattern", + reason = "API not fully fleshed out and ready to be stabilized", + issue = "27721" +)] + +use crate::ops::Range; + +// Pattern + +/// A pattern which can be matched against a [`Haystack`]. +/// +/// A `Pattern` expresses that the implementing type can be used as +/// a pattern for searching in an `H`. For example, both the character `'a'` +/// and the string `"aa"` are patterns that would match at index `1` in +/// the string `"baaaab"`. +/// +/// The trait itself acts as a builder for an associated [`Searcher`] type, +/// which does the actual work of finding occurrences of the pattern in a haystack. +/// +/// Depending on the type of the pattern, the behavior of methods like +/// [`str::find`] and [`str::contains`] can change. The table below describes +/// some of those behaviors. +/// +/// | Pattern type | Match condition | +/// |--------------------------|-------------------------------------------| +/// | `&str` | is substring | +/// | `char` | is contained in string | +/// | `&[char]` | any char in slice is contained in string | +/// | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | +/// | `&&str` | is substring | +/// | `&String` | is substring | +/// +/// # Examples +/// +/// ``` +/// // &str +/// assert_eq!("abaaa".find("ba"), Some(1)); +/// assert_eq!("abaaa".find("bac"), None); +/// +/// // char +/// assert_eq!("abaaa".find('a'), Some(0)); +/// assert_eq!("abaaa".find('b'), Some(1)); +/// assert_eq!("abaaa".find('c'), None); +/// +/// // &[char; N] +/// assert_eq!("ab".find(&['b', 'a']), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z']), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd']), None); +/// +/// // &[char] +/// assert_eq!("ab".find(&['b', 'a'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd'][..]), None); +/// +/// // FnMut(char) -> bool +/// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); +/// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); +/// ``` +pub trait Pattern: Sized { + /// Associated searcher for this pattern + type Searcher: Searcher; + + /// Constructs the associated searcher from + /// `self` and the `haystack` to search in. + fn into_searcher(self, haystack: H) -> Self::Searcher; + + /// Checks whether the pattern matches anywhere in the haystack + #[inline] + fn is_contained_in(self, haystack: H) -> bool { + self.into_searcher(haystack).next_match().is_some() + } + + /// Checks whether the pattern matches at the front of the haystack + #[inline] + fn is_prefix_of(self, haystack: H) -> bool { + matches!( + self.into_searcher(haystack).next(), + SearchStep::Match(start, _) if start == haystack.cursor_at_front() + ) + } + + /// Checks whether the pattern matches at the back of the haystack + #[inline] + fn is_suffix_of(self, haystack: H) -> bool + where + Self::Searcher: ReverseSearcher, + { + matches!( + self.into_searcher(haystack).next_back(), + SearchStep::Match(_, end) if end == haystack.cursor_at_back() + ) + } + + /// Removes the pattern from the front of a haystack, if it matches + #[inline] + fn strip_prefix_of(self, haystack: H) -> Option { + if let SearchStep::Match(start, pos) = self.into_searcher(haystack).next() { + debug_assert_eq!( + start, + haystack.cursor_at_front(), + "The first search step from Searcher \ + must include the first character" + ); + let end = haystack.cursor_at_back(); + // SAFETY: `Searcher` is known to return valid indices. + Some(unsafe { haystack.get_unchecked(pos..end) }) + } else { + None + } + } + + /// Removes the pattern from the back of a haystack, if it matches. + #[inline] + fn strip_suffix_of(self, haystack: H) -> Option + where + Self::Searcher: ReverseSearcher, + { + if let SearchStep::Match(pos, end) = self.into_searcher(haystack).next_back() { + debug_assert_eq!( + end, + haystack.cursor_at_back(), + "The first search step from ReverseSearcher \ + must include the last character" + ); + let start = haystack.cursor_at_front(); + // SAFETY: `Searcher` is known to return valid indices. + Some(unsafe { haystack.get_unchecked(start..pos) }) + } else { + None + } + } + + /// Returns the pattern as UTF-8 if possible. + fn as_utf8_pattern(&self) -> Option> { + None + } +} +/// Result of calling [`Pattern::as_utf8_pattern()`]. +/// Can be used for inspecting the contents of a [`Pattern`] in cases +/// where the underlying representation can be represented as UTF-8. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum Utf8Pattern<'a> { + /// Type returned by String and str types. + /// This stores `str` rather than bytes so callers cannot describe + /// non-UTF-8 string patterns through this API. + StringPattern(&'a str), + /// Type returned by char types. + CharPattern(char), +} + +// Haystack + +/// A type which can be searched in using a [`Pattern`]. +/// +/// The trait is used in combination with the [`Pattern`] trait to express a pattern +/// that can be used to search for elements in a given haystack. +pub trait Haystack: Sized + Copy { + /// Returns a cursor pointing at the beginning of the haystack. + fn cursor_at_front(self) -> usize; + + /// Returns a cursor pointing at the end of the haystack. + fn cursor_at_back(self) -> usize; + + /// Returns whether the haystack is empty. + fn is_empty(self) -> bool { + self.cursor_at_front() == self.cursor_at_back() + } + + /// Returns portions of the haystack indicated by the cursor range. + /// + /// # Safety + /// + /// The range's start and end must be valid haystack split positions, + /// and start must not point to a position after end. + /// + /// Valid split positions are: + /// - the front of the haystack (as returned by + /// [`cursor_at_front()`][Self::cursor_at_front]), + /// - the back of the haystack (as returned by + /// [`cursor_at_back()`][Self::cursor_at_back]), or + /// - any cursor returned by a [`Searcher`] or [`ReverseSearcher`]. + unsafe fn get_unchecked(self, range: Range) -> Self; +} + +// Searcher + +/// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum SearchStep { + /// Expresses that a match of the pattern has been found at + /// `haystack[a..b]`. + Match(usize, usize), + /// Expresses that `haystack[a..b]` has been rejected as a possible match + /// of the pattern. + /// + /// Note that there might be more than one `Reject` between two `Match`es, + /// there is no requirement for them to be combined into one. + Reject(usize, usize), + /// Expresses that every byte of the haystack has been visited, ending + /// the iteration. + Done, +} + +/// A searcher for a string pattern. +/// +/// This trait provides methods for searching for non-overlapping +/// matches of a pattern starting from the front (left) of a string. +/// +/// It will be implemented by associated `Searcher` +/// types of the [`Pattern`] trait. +/// +/// The trait is marked unsafe because the indices returned by the +/// [`next()`][Searcher::next] methods are required to lie on valid split +/// positions of the haystack (for `&str` haystacks, these are utf8 character +/// boundaries). This enables consumers of this trait to slice the haystack +/// without additional runtime checks. +pub unsafe trait Searcher { + /// Getter for the underlying haystack to be searched in + /// + /// Will always return the same haystack. + fn haystack(&self) -> H; + + /// Performs the next search step starting from the front. + /// + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` matches + /// the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` can + /// not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack has + /// been visited. + /// + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] + /// will contain index ranges that are adjacent, non-overlapping, + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. + /// + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. + /// + /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` + /// might produce the stream + /// `[Reject(0, 1), Reject(1, 2), Match(2, 5), Reject(5, 8)]` + fn next(&mut self) -> SearchStep; + + /// Finds the next [`Match`][SearchStep::Match] result. See [`next()`][Searcher::next]. + /// + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_reject`][Searcher::next_reject] will overlap. This will return + /// `(start_match, end_match)`, where start_match is the index of where + /// the match begins, and end_match is the index after the end of the match. + #[inline] + fn next_match(&mut self) -> Option<(usize, usize)> { + loop { + match self.next() { + SearchStep::Match(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } + + /// Finds the next [`Reject`][SearchStep::Reject] result. See [`next()`][Searcher::next] + /// and [`next_match()`][Searcher::next_match]. + /// + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_match`][Searcher::next_match] will overlap. + #[inline] + fn next_reject(&mut self) -> Option<(usize, usize)> { + loop { + match self.next() { + SearchStep::Reject(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } +} + +/// A reverse searcher for a string pattern. +/// +/// This trait provides methods for searching for non-overlapping +/// matches of a pattern starting from the back (right) of a string. +/// +/// It will be implemented by associated [`Searcher`] +/// types of the [`Pattern`] trait if the pattern supports searching +/// for it from the back. +/// +/// The index ranges returned by this trait are not required +/// to exactly match those of the forward search in reverse. +/// +/// For the reason why this trait is marked unsafe, see the +/// parent trait [`Searcher`]. +pub unsafe trait ReverseSearcher: Searcher { + /// Performs the next search step starting from the back. + /// + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` + /// matches the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` + /// can not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack + /// has been visited + /// + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] + /// will contain index ranges that are adjacent, non-overlapping, + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. + /// + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. + /// + /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` + /// might produce the stream + /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]`. + fn next_back(&mut self) -> SearchStep; + + /// Finds the next [`Match`][SearchStep::Match] result. + /// See [`next_back()`][ReverseSearcher::next_back]. + #[inline] + fn next_match_back(&mut self) -> Option<(usize, usize)> { + loop { + match self.next_back() { + SearchStep::Match(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } + + /// Finds the next [`Reject`][SearchStep::Reject] result. + /// See [`next_back()`][ReverseSearcher::next_back]. + #[inline] + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + loop { + match self.next_back() { + SearchStep::Reject(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } +} + +/// A marker trait to express that a [`ReverseSearcher`] +/// can be used for a [`DoubleEndedIterator`] implementation. +/// +/// For this, the impl of [`Searcher`] and [`ReverseSearcher`] need +/// to follow these conditions: +/// +/// - All results of `next()` need to be identical +/// to the results of `next_back()` in reverse order. +/// - `next()` and `next_back()` need to behave as +/// the two ends of a range of values, that is they +/// can not "walk past each other". +/// +/// # Examples +/// +/// `char::Searcher` is a `DoubleEndedSearcher` because searching for a +/// [`char`] only requires looking at one at a time, which behaves the same +/// from both ends. +/// +/// `(&str)::Searcher` is not a `DoubleEndedSearcher` because +/// the pattern `"aa"` in the haystack `"aaa"` matches as either +/// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. +pub trait DoubleEndedSearcher: ReverseSearcher {} diff --git a/library/core/src/str/iter.rs b/library/core/src/str/iter.rs index 21015c8c04aba..74c3bed702de1 100644 --- a/library/core/src/str/iter.rs +++ b/library/core/src/str/iter.rs @@ -1,6 +1,5 @@ //! Iterators for `str` methods. -use super::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use super::validations::{next_code_point, next_code_point_reverse}; use super::{ BytesIsNotEmpty, CharEscapeDebugContinue, CharEscapeDefault, CharEscapeUnicode, @@ -13,6 +12,7 @@ use crate::iter::{ }; use crate::num::NonZero; use crate::ops::Try; +use crate::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::slice::{self, Split as SliceSplit}; use crate::{char as char_mod, option}; diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index c7c2cfd34c4af..d2cc0348712a3 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -13,9 +13,9 @@ mod iter; mod traits; mod validations; -use self::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::char::{self, EscapeDebugExtArgs}; use crate::hint::assert_unchecked; +use crate::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::range::Range; use crate::slice::{self, SliceIndex}; use crate::ub_checks::assert_unsafe_precondition; diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index 1146e2b0962c3..6a97ba3d3d55f 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -1,36 +1,47 @@ -//! The string Pattern API. +//! [The Pattern API] implementation for searching in `&str`. //! -//! The Pattern API provides a generic mechanism for using different pattern -//! types when searching through a string. +//! The implementation provides generic mechanism for using different pattern +//! types when searching through a string. Although this API is unstable, it is +//! exposed via stable APIs on the [`str`] type. //! -//! For more details, see the traits [`Pattern`], [`Searcher`], -//! [`ReverseSearcher`], and [`DoubleEndedSearcher`]. +//! Depending on the type of the pattern, the behaviour of methods like +//! [`str::find`] and [`str::contains`] can change. The table below describes +//! some of those behaviours. //! -//! Although this API is unstable, it is exposed via stable APIs on the -//! [`str`] type. +//! | Pattern type | Match condition | +//! |--------------------------|-------------------------------------------| +//! | `&str` | is substring | +//! | `char` | is contained in string | +//! | `&[char]` | any char in slice is contained in string | +//! | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | +//! | `&&str` | is substring | +//! | `&String` | is substring | //! //! # Examples //! -//! [`Pattern`] is [implemented][pattern-impls] in the stable API for -//! [`&str`][`str`], [`char`], slices of [`char`], and functions and closures -//! implementing `FnMut(char) -> bool`. -//! //! ``` //! let s = "Can you find a needle in a haystack?"; //! //! // &str pattern //! assert_eq!(s.find("you"), Some(4)); +//! assert_eq!(s.find("thou"), None); +//! //! // char pattern //! assert_eq!(s.find('n'), Some(2)); -//! // array of chars pattern +//! assert_eq!(s.find('N'), None); +//! +//! // Array of chars pattern and slices thereof //! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u']), Some(1)); -//! // slice of chars pattern //! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u'][..]), Some(1)); -//! // closure pattern +//! assert_eq!(s.find(&['q', 'v', 'x']), None); +//! +//! // Predicate closure //! assert_eq!(s.find(|c: char| c.is_ascii_punctuation()), Some(35)); +//! assert_eq!(s.find(|c: char| c.is_lowercase()), Some(1)); +//! assert_eq!(s.find(|c: char| !c.is_ascii()), None); //! ``` //! -//! [pattern-impls]: Pattern#implementors +//! [The Pattern API]: crate::pattern #![unstable( feature = "pattern", @@ -41,373 +52,12 @@ use crate::cmp::Ordering; use crate::convert::TryInto as _; use crate::ops::Range; +pub use crate::pattern::{ + DoubleEndedSearcher, Haystack, Pattern, ReverseSearcher, SearchStep, Searcher, Utf8Pattern, +}; use crate::slice::memchr; use crate::{cmp, fmt}; -// Pattern - -/// A pattern which can be matched against a [`Haystack`]. -/// -/// A `Pattern` expresses that the implementing type can be used as a pattern -/// for searching in an `H`. -/// -/// For example, both character `'a'` and string `"aa"` are patterns that would match -/// at index `1` in the string `"baaaab"`. -/// -/// The trait itself acts as a builder for an associated -/// [`Searcher`] type, which does the actual work of finding -/// occurrences of the pattern in a haystack. -/// -/// Depending on the type of the pattern, the behavior of methods like -/// [`str::find`] and [`str::contains`] can change. The table below describes -/// some of those behaviors. -/// -/// | Pattern type | Match condition | -/// |--------------------------|-------------------------------------------| -/// | `&str` | is substring | -/// | `char` | is contained in string | -/// | `&[char]` | any char in slice is contained in string | -/// | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | -/// | `&&str` | is substring | -/// | `&String` | is substring | -/// -/// # Examples -/// -/// ``` -/// // &str -/// assert_eq!("abaaa".find("ba"), Some(1)); -/// assert_eq!("abaaa".find("bac"), None); -/// -/// // char -/// assert_eq!("abaaa".find('a'), Some(0)); -/// assert_eq!("abaaa".find('b'), Some(1)); -/// assert_eq!("abaaa".find('c'), None); -/// -/// // &[char; N] -/// assert_eq!("ab".find(&['b', 'a']), Some(0)); -/// assert_eq!("abaaa".find(&['a', 'z']), Some(0)); -/// assert_eq!("abaaa".find(&['c', 'd']), None); -/// -/// // &[char] -/// assert_eq!("ab".find(&['b', 'a'][..]), Some(0)); -/// assert_eq!("abaaa".find(&['a', 'z'][..]), Some(0)); -/// assert_eq!("abaaa".find(&['c', 'd'][..]), None); -/// -/// // FnMut(char) -> bool -/// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); -/// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); -/// ``` -pub trait Pattern: Sized { - /// Associated searcher for this pattern - type Searcher: Searcher; - - /// Constructs the associated searcher from - /// `self` and the `haystack` to search in. - fn into_searcher(self, haystack: H) -> Self::Searcher; - - /// Checks whether the pattern matches anywhere in the haystack - #[inline] - fn is_contained_in(self, haystack: H) -> bool { - self.into_searcher(haystack).next_match().is_some() - } - - /// Checks whether the pattern matches at the front of the haystack - #[inline] - fn is_prefix_of(self, haystack: H) -> bool { - matches!( - self.into_searcher(haystack).next(), - SearchStep::Match(start, _) if start == haystack.cursor_at_front() - ) - } - - /// Checks whether the pattern matches at the back of the haystack - #[inline] - fn is_suffix_of(self, haystack: H) -> bool - where - Self::Searcher: ReverseSearcher, - { - matches!( - self.into_searcher(haystack).next_back(), - SearchStep::Match(_, end) if end == haystack.cursor_at_back() - ) - } - - /// Removes the pattern from the front of a haystack, if it matches - #[inline] - fn strip_prefix_of(self, haystack: H) -> Option { - if let SearchStep::Match(start, pos) = self.into_searcher(haystack).next() { - debug_assert_eq!( - start, - haystack.cursor_at_front(), - "The first search step from Searcher \ - must include the first character" - ); - let end = haystack.cursor_at_back(); - // SAFETY: `Searcher` is known to return valid indices. - Some(unsafe { haystack.get_unchecked(pos..end) }) - } else { - None - } - } - - /// Removes the pattern from the back of a haystack, if it matches. - #[inline] - fn strip_suffix_of(self, haystack: H) -> Option - where - Self::Searcher: ReverseSearcher, - { - if let SearchStep::Match(pos, end) = self.into_searcher(haystack).next_back() { - debug_assert_eq!( - end, - haystack.cursor_at_back(), - "The first search step from ReverseSearcher \ - must include the last character" - ); - let start = haystack.cursor_at_front(); - // SAFETY: `Searcher` is known to return valid indices. - Some(unsafe { haystack.get_unchecked(start..pos) }) - } else { - None - } - } - - /// Returns the pattern as UTF-8 if possible. - fn as_utf8_pattern(&self) -> Option> { - None - } -} -/// Result of calling [`Pattern::as_utf8_pattern()`]. -/// Can be used for inspecting the contents of a [`Pattern`] in cases -/// where the underlying representation can be represented as UTF-8. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum Utf8Pattern<'a> { - /// Type returned by String and str types. - /// This stores `str` rather than bytes so callers cannot describe - /// non-UTF-8 string patterns through this API. - StringPattern(&'a str), - /// Type returned by char types. - CharPattern(char), -} - -// Haystack - -/// A type which can be searched in using a [`Pattern`]. -/// -/// The trait is used in combination with [`Pattern`] trait to express a pattern -/// that can be used to search for elements in given haystack. -pub trait Haystack: Sized + Copy { - /// Returns cursor pointing at the beginning of the haystack. - fn cursor_at_front(self) -> usize; - - /// Returns cursor pointing at the end of the haystack. - fn cursor_at_back(self) -> usize; - - /// Returns whether the haystack is empty. - fn is_empty(self) -> bool { - self.cursor_at_front() == self.cursor_at_back() - } - - /// Returns portions of the haystack indicated by the cursor range. - /// - /// # Safety - /// - /// The range's start and end must be valid haystack split positions, - /// and start must not point to a position after end. - /// - /// Valid split positions are: - /// - the front of the haystack (as returned by - /// [`cursor_at_front()`][Self::cursor_at_front]), - /// - the back of the haystack (as returned by - /// [`cursor_at_back()`][Self::cursor_at_back]), or - /// - any cursor returned by a [`Searcher`] or [`ReverseSearcher`]. - unsafe fn get_unchecked(self, range: Range) -> Self; -} - -// Searcher - -/// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum SearchStep { - /// Expresses that a match of the pattern has been found at - /// `haystack[a..b]`. - Match(usize, usize), - /// Expresses that `haystack[a..b]` has been rejected as a possible match - /// of the pattern. - /// - /// Note that there might be more than one `Reject` between two `Match`es, - /// there is no requirement for them to be combined into one. - Reject(usize, usize), - /// Expresses that every byte of the haystack has been visited, ending - /// the iteration. - Done, -} - -/// A searcher for a string pattern. -/// -/// This trait provides methods for searching for non-overlapping -/// matches of a pattern starting from the front (left) of a string. -/// -/// It will be implemented by associated `Searcher` -/// types of the [`Pattern`] trait. -/// -/// The trait is marked unsafe because the indices returned by the -/// [`next()`][Searcher::next] methods are required to lie on valid split -/// positions of the haystack (for `&str` haystacks, these are utf8 character -/// boundaries). This enables consumers of this trait to slice the haystack -/// without additional runtime checks. -pub unsafe trait Searcher { - /// Getter for the underlying haystack to be searched in - /// - /// Will always return the same haystack. - fn haystack(&self) -> H; - - /// Performs the next search step starting from the front. - /// - /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` matches - /// the pattern. - /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` can - /// not match the pattern, even partially. - /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack has - /// been visited. - /// - /// The stream of [`Match`][SearchStep::Match] and - /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] - /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on valid split positions of the - /// haystack. - /// - /// A [`Match`][SearchStep::Match] result needs to contain the whole matched - /// pattern, however [`Reject`][SearchStep::Reject] results may be split up - /// into arbitrary many adjacent fragments. Both ranges may have zero length. - /// - /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` - /// might produce the stream - /// `[Reject(0, 1), Reject(1, 2), Match(2, 5), Reject(5, 8)]` - fn next(&mut self) -> SearchStep; - - /// Finds the next [`Match`][SearchStep::Match] result. See [`next()`][Searcher::next]. - /// - /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges - /// of this and [`next_reject`][Searcher::next_reject] will overlap. This will return - /// `(start_match, end_match)`, where start_match is the index of where - /// the match begins, and end_match is the index after the end of the match. - #[inline] - fn next_match(&mut self) -> Option<(usize, usize)> { - loop { - match self.next() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } - } - - /// Finds the next [`Reject`][SearchStep::Reject] result. See [`next()`][Searcher::next] - /// and [`next_match()`][Searcher::next_match]. - /// - /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges - /// of this and [`next_match`][Searcher::next_match] will overlap. - #[inline] - fn next_reject(&mut self) -> Option<(usize, usize)> { - loop { - match self.next() { - SearchStep::Reject(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } - } -} - -/// A reverse searcher for a string pattern. -/// -/// This trait provides methods for searching for non-overlapping -/// matches of a pattern starting from the back (right) of a string. -/// -/// It will be implemented by associated [`Searcher`] -/// types of the [`Pattern`] trait if the pattern supports searching -/// for it from the back. -/// -/// The index ranges returned by this trait are not required -/// to exactly match those of the forward search in reverse. -/// -/// For the reason why this trait is marked unsafe, see the -/// parent trait [`Searcher`]. -pub unsafe trait ReverseSearcher: Searcher { - /// Performs the next search step starting from the back. - /// - /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` - /// matches the pattern. - /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` - /// can not match the pattern, even partially. - /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack - /// has been visited - /// - /// The stream of [`Match`][SearchStep::Match] and - /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] - /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on valid split positions of the - /// haystack. - /// - /// A [`Match`][SearchStep::Match] result needs to contain the whole matched - /// pattern, however [`Reject`][SearchStep::Reject] results may be split up - /// into arbitrary many adjacent fragments. Both ranges may have zero length. - /// - /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` - /// might produce the stream - /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]`. - fn next_back(&mut self) -> SearchStep; - - /// Finds the next [`Match`][SearchStep::Match] result. - /// See [`next_back()`][ReverseSearcher::next_back]. - #[inline] - fn next_match_back(&mut self) -> Option<(usize, usize)> { - loop { - match self.next_back() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } - } - - /// Finds the next [`Reject`][SearchStep::Reject] result. - /// See [`next_back()`][ReverseSearcher::next_back]. - #[inline] - fn next_reject_back(&mut self) -> Option<(usize, usize)> { - loop { - match self.next_back() { - SearchStep::Reject(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } - } -} - -/// A marker trait to express that a [`ReverseSearcher`] -/// can be used for a [`DoubleEndedIterator`] implementation. -/// -/// For this, the impl of [`Searcher`] and [`ReverseSearcher`] need -/// to follow these conditions: -/// -/// - All results of `next()` need to be identical -/// to the results of `next_back()` in reverse order. -/// - `next()` and `next_back()` need to behave as -/// the two ends of a range of values, that is they -/// can not "walk past each other". -/// -/// # Examples -/// -/// `char::Searcher` is a `DoubleEndedSearcher` because searching for a -/// [`char`] only requires looking at one at a time, which behaves the same -/// from both ends. -/// -/// `(&str)::Searcher` is not a `DoubleEndedSearcher` because -/// the pattern `"aa"` in the haystack `"aaa"` matches as either -/// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. -pub trait DoubleEndedSearcher: ReverseSearcher {} - ///////////////////////////////////////////////////////////////////////////// // Impl for Haystack ///////////////////////////////////////////////////////////////////////////// diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index 9546007bc942e..021fcb29b0131 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -1,4 +1,4 @@ -use std::str::pattern::*; +use std::pattern::*; // This macro makes it easier to write // tests that do a series of iterations diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index 980ec4416f04a..56ff07a001e86 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -568,6 +568,8 @@ pub use core::mem; pub use core::ops; #[stable(feature = "rust1", since = "1.0.0")] pub use core::option; +#[unstable(feature = "pattern", issue = "27721")] +pub use core::pattern; #[stable(feature = "pin", since = "1.33.0")] pub use core::pin; #[stable(feature = "rust1", since = "1.0.0")] diff --git a/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs index 4f8cc67f5e019..5ff45e5ae7d53 100644 --- a/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs +++ b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt, quote}; use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::str::pattern::Pattern; +use std::pattern::Pattern; use std::{fmt, str::FromStr}; use crate::context::LocalContext; @@ -68,7 +68,7 @@ impl WildString { pub fn replace

(&self, from: P, to: &str) -> WildString where - P: Pattern + Copy, + P: for<'a> Pattern<&'a str> + Copy, { WildString( self.0 From b89196bf52057e4a5bc5be068fdecd1e4912e111 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 17 Aug 2026 19:21:13 -0400 Subject: [PATCH 07/16] Revert "Add ByteNeedle search in StrSearcherImpl" This reverts commit 85cf233ced0d0fe02734c8a83b6d79ccc5432d06. Gone for now, I'll reimplement it later in str_bytes.rs, will confirm with the benchmarks included that the optimization still applies --- library/core/src/str/pattern.rs | 96 ++++----------------------------- 1 file changed, 9 insertions(+), 87 deletions(-) diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index 6a97ba3d3d55f..22f36c7297a2a 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -796,7 +796,6 @@ pub struct StrSearcher<'a, 'b> { #[derive(Clone, Debug)] enum StrSearcherImpl { Empty(EmptyNeedle), - Byte(ByteNeedle), TwoWay(TwoWaySearcher), } @@ -810,16 +809,6 @@ struct EmptyNeedle { is_finished: bool, } -/// Fast searcher for a single-byte needle using `memchr`/`memrchr`. -#[derive(Clone, Debug)] -struct ByteNeedle { - b: u8, - /// Forward cursor: `haystack[..position]` has already been reported. - position: usize, - /// Backward cursor: `haystack[end..]` has already been reported. - end: usize, -} - impl<'a, 'b> StrSearcher<'a, 'b> { fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { if needle.is_empty() { @@ -834,12 +823,6 @@ impl<'a, 'b> StrSearcher<'a, 'b> { is_finished: false, }), } - } else if let &[b] = needle.as_bytes() { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::Byte(ByteNeedle { b, position: 0, end: haystack.len() }), - } } else { StrSearcher { haystack, @@ -882,23 +865,6 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { } } } - StrSearcherImpl::Byte(ref mut searcher) => { - let bytes = self.haystack.as_bytes(); - let pos = searcher.position; - if pos >= bytes.len() { - return SearchStep::Done; - } - if bytes[pos] == searcher.b { - searcher.position = pos + 1; - SearchStep::Match(pos, pos + 1) - } else { - // `pos` is always on a char boundary, so this rejects - // exactly the char starting at `pos`. - let end = self.haystack.ceil_char_boundary(pos + 1); - searcher.position = end; - SearchStep::Reject(pos, end) - } - } StrSearcherImpl::TwoWay(ref mut searcher) => { // TwoWaySearcher produces valid *Match* indices that split at char boundaries // as long as it does correct matching and that haystack and needle are @@ -914,9 +880,11 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { self.needle.as_bytes(), is_long, ) { - SearchStep::Reject(a, b) => { + SearchStep::Reject(a, mut b) => { // skip to next char boundary - let b = self.haystack.ceil_char_boundary(b); + while !self.haystack.is_char_boundary(b) { + b += 1; + } searcher.position = cmp::max(b, searcher.position); SearchStep::Reject(a, b) } @@ -936,23 +904,6 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { SearchStep::Reject(..) => {} } }, - StrSearcherImpl::Byte(ref mut searcher) => { - let bytes = self.haystack.as_bytes(); - if searcher.position >= bytes.len() { - return None; - } - match memchr::memchr(searcher.b, &bytes[searcher.position..]) { - Some(i) => { - let pos = searcher.position + i; - searcher.position = pos + 1; - Some((pos, pos + 1)) - } - None => { - searcher.position = bytes.len(); - None - } - } - } StrSearcherImpl::TwoWay(ref mut searcher) => { let is_long = searcher.memory == usize::MAX; // write out `true` and `false` cases to encourage the compiler @@ -998,21 +949,6 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { } } } - StrSearcherImpl::Byte(ref mut searcher) => { - let end = searcher.end; - if end == 0 { - return SearchStep::Done; - } - let bytes = self.haystack.as_bytes(); - if bytes[end - 1] == searcher.b { - searcher.end = end - 1; - SearchStep::Match(end - 1, end) - } else { - let start = self.haystack.floor_char_boundary(end - 1); - searcher.end = start; - SearchStep::Reject(start, end) - } - } StrSearcherImpl::TwoWay(ref mut searcher) => { if searcher.end == 0 { return SearchStep::Done; @@ -1023,9 +959,11 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { self.needle.as_bytes(), is_long, ) { - SearchStep::Reject(a, b) => { - // skip to previous char boundary - let a = self.haystack.floor_char_boundary(a); + SearchStep::Reject(mut a, b) => { + // skip to next char boundary + while !self.haystack.is_char_boundary(a) { + a -= 1; + } searcher.end = cmp::min(a, searcher.end); SearchStep::Reject(a, b) } @@ -1045,22 +983,6 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { SearchStep::Reject(..) => {} } }, - StrSearcherImpl::Byte(ref mut searcher) => { - if searcher.end == 0 { - return None; - } - let bytes = self.haystack.as_bytes(); - match memchr::memrchr(searcher.b, &bytes[..searcher.end]) { - Some(i) => { - searcher.end = i; - Some((i, i + 1)) - } - None => { - searcher.end = 0; - None - } - } - } StrSearcherImpl::TwoWay(ref mut searcher) => { let is_long = searcher.memory == usize::MAX; // write out `true` and `false`, like `next_match` From 5f861c100efe2db020ebe823c577f49011f69762 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:40:09 -0400 Subject: [PATCH 08/16] core: add core::pattern::EmptyNeedleSearcher internal type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce core::pattern::EmptyNeedleSearcher internal type which implements logic for matching an empty pattern against a haystack. Convert core::str::pattern::StrSearcher to use it. In future more implementations will take advantage of it. Also adapt and rework TwoWayStrategy into an internal SearchResult trait which abstracts differences between Searcher’s next, next_match and next_rejects methods. It makes it simpler to write a single generic method implementing optimised versions of all those calls. @pacak: - Fixed a few typos. - There's no H::Cursor parameter so code gets a bit simplified. - Added a test to assert how TwoWaySearcher runs with EmptyNeedleSearcher --- library/alloctests/tests/str.rs | 15 ++ library/core/src/pattern.rs | 239 +++++++++++++++++++++++++++-- library/core/src/str/pattern.rs | 256 ++++++++++---------------------- 3 files changed, 323 insertions(+), 187 deletions(-) diff --git a/library/alloctests/tests/str.rs b/library/alloctests/tests/str.rs index 86967f8ae5692..a9255ee705de5 100644 --- a/library/alloctests/tests/str.rs +++ b/library/alloctests/tests/str.rs @@ -2196,6 +2196,21 @@ mod pattern { assert_eq!(searcher.next_back(), SearchStep::Done); } } + + #[test] + fn str_searcher_empty_needle_interleaved() { + let mut searcher = "".into_searcher("abc"); + + assert_eq!(searcher.next(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next_back(), SearchStep::Match(3, 3)); + assert_eq!(searcher.next(), SearchStep::Reject(0, 1)); + assert_eq!(searcher.next_back(), SearchStep::Reject(2, 3)); + assert_eq!(searcher.next(), SearchStep::Match(1, 1)); + assert_eq!(searcher.next_back(), SearchStep::Match(2, 2)); + assert_eq!(searcher.next(), SearchStep::Reject(1, 2)); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + } } macro_rules! generate_iterator_test { diff --git a/library/core/src/pattern.rs b/library/core/src/pattern.rs index 2b691c066c4f8..b894e9a911b4d 100644 --- a/library/core/src/pattern.rs +++ b/library/core/src/pattern.rs @@ -4,7 +4,7 @@ //! types when searching through different objects. //! //! For more details, see the traits [`Pattern`], [`Haystack`], [`Searcher`], -//! [`ReverseSearcher`] and [`DoubleEndedSearcher`]. Although this API is +//! [`ReverseSearcher`] and [`DoubleEndedSearcher`]. Although this API is //! unstable, it is exposed via stable methods on corresponding haystack types. //! //! # Examples @@ -37,7 +37,7 @@ )] use crate::fmt; -use crate::mem::replace; +use crate::mem::{replace, take}; use crate::ops::Range; // Pattern @@ -236,6 +236,84 @@ pub enum SearchStep { Done, } +/// Possible return type of a search. +/// +/// It abstracts the differences between `next`, `next_match` and `next_reject` methods. Depending +/// on return type an implementation for those functions will generate matches and rejects, only +/// matches or only rejects. +#[unstable(feature = "pattern_internals", issue = "none")] +pub trait SearchResult: Sized + sealed::Sealed { + /// Value indicating searching has finished. + const DONE: Self; + + /// Returns value describing a match or `None` if this implementation + /// doesn't care about matches. + fn matching(start: usize, end: usize) -> Option; + + /// Returns value describing a reject or `None` if this implementation + /// doesn't care about rejects. + fn rejecting(start: usize, end: usize) -> Option; +} + +/// A wrapper for result type which only carries information about matches. +#[unstable(feature = "pattern_internals", issue = "none")] +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub struct MatchOnly(pub Option<(usize, usize)>); + +/// A wrapper for result type which only carries information about rejects. +#[unstable(feature = "pattern_internals", issue = "none")] +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub struct RejectOnly(pub Option<(usize, usize)>); + +impl SearchResult for SearchStep { + const DONE: Self = SearchStep::Done; + + #[inline(always)] + fn matching(s: usize, e: usize) -> Option { + Some(SearchStep::Match(s, e)) + } + + #[inline(always)] + fn rejecting(s: usize, e: usize) -> Option { + Some(SearchStep::Reject(s, e)) + } +} + +impl SearchResult for MatchOnly { + const DONE: Self = Self(None); + + #[inline(always)] + fn matching(s: usize, e: usize) -> Option { + Some(Self(Some((s, e)))) + } + + #[inline(always)] + fn rejecting(_s: usize, _e: usize) -> Option { + None + } +} + +impl SearchResult for RejectOnly { + const DONE: Self = Self(None); + + #[inline(always)] + fn matching(_s: usize, _e: usize) -> Option { + None + } + + #[inline(always)] + fn rejecting(s: usize, e: usize) -> Option { + Some(Self(Some((s, e)))) + } +} + +mod sealed { + pub trait Sealed {} + impl Sealed for super::SearchStep {} + impl Sealed for super::MatchOnly {} + impl Sealed for super::RejectOnly {} +} + /// A searcher for a string pattern. /// /// This trait provides methods for searching for non-overlapping @@ -402,6 +480,145 @@ pub unsafe trait ReverseSearcher: Searcher { /// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. pub trait DoubleEndedSearcher: ReverseSearcher {} +////////////////////////////////////////////////////////////////////////////// +// Internal EmptyNeedleSearcher helper +////////////////////////////////////////////////////////////////////////////// + +/// Helper for implementing searchers looking for empty patterns. +/// +/// An empty pattern matches around every element of a haystack. For example, +/// within a `&str` it matches around every character. (This includes at the +/// beginning and end of the string). +/// +/// This struct helps implement searchers for empty patterns for various +/// haystacks. The only requirement is a function which advances the start +/// position or end position of the haystack range. +/// +/// # Examples +/// +/// ``` +/// #![feature(pattern, pattern_internals)] +/// # #![allow(internal_features)] +/// +/// use core::pattern::{EmptyNeedleSearcher, SearchStep}; +/// +/// let haystack = "fóó"; +/// let mut searcher = EmptyNeedleSearcher::new(haystack); +/// let advance = |range: core::ops::Range| { +/// range.start + haystack[range].chars().next().unwrap().len_utf8() +/// }; +/// let steps = core::iter::from_fn(|| { +/// match searcher.next_fwd(advance) { +/// SearchStep::Done => None, +/// step => Some(step) +/// } +/// }).collect::>(); +/// assert_eq!(&[ +/// SearchStep::Match(0, 0), +/// SearchStep::Reject(0, 1), +/// SearchStep::Match(1, 1), +/// SearchStep::Reject(1, 3), +/// SearchStep::Match(3, 3), +/// SearchStep::Reject(3, 5), +/// SearchStep::Match(5, 5), +/// ], steps.as_slice()); +/// ``` +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[unstable(feature = "pattern_internals", issue = "none")] +pub struct EmptyNeedleSearcher { + start: usize, + end: usize, + is_match_fwd: bool, + is_match_bwd: bool, + // Needed in case of an empty haystack, see #85462 + is_finished: bool, +} + +impl EmptyNeedleSearcher { + /// Creates a new empty needle searcher for given haystack. + /// + /// The haystack is used to initialise the range of valid cursors positions. + pub fn new(haystack: H) -> Self { + Self { + start: haystack.cursor_at_front(), + end: haystack.cursor_at_back(), + is_match_bwd: true, + is_match_fwd: true, + is_finished: false, + } + } + + /// Returns next search result. + /// + /// The callback function is used to advance the **start** of the range the + /// searcher is working on. It is passed the current range of cursor + /// positions that weren't visited yet and it must return the new start + /// cursor position. It's never called with an empty range. For some + /// haystacks the callback may be as simple as a closure returning the start + /// incremented by one; others might require looking for a new valid + /// boundary. + #[inline] + pub fn next_fwd(&mut self, advance_fwd: F) -> R + where + F: FnOnce(crate::ops::Range) -> usize, + { + if self.is_finished { + return R::DONE; + } + if take(&mut self.is_match_fwd) { + if let Some(ret) = R::matching(self.start, self.start) { + return ret; + } + } + if self.start < self.end { + let pos = self.start; + self.start = advance_fwd(self.start..self.end); + if let Some(ret) = R::rejecting(pos, self.start) { + self.is_match_fwd = true; + return ret; + } + return R::matching(self.start, self.start).unwrap(); + } + self.is_finished = true; + R::DONE + } + + /// Returns next search result. + /// + /// The callback function is used to advance the **end** of the range the + /// searcher is working on backwards. It is passed the current range of + /// cursor positions that weren't visited yet and it must return the new end + /// cursor position. It's never called with an empty range. For some + /// haystacks the callback may be as simple as a closure returning the end + /// decremented by one; others might require looking for a new valid + /// boundary. + #[inline] + pub fn next_bwd(&mut self, advance_bwd: F) -> R + where + F: FnOnce(crate::ops::Range) -> usize, + { + if self.is_finished { + return R::DONE; + } + if take(&mut self.is_match_bwd) { + if let Some(ret) = R::matching(self.end, self.end) { + return ret; + } + } + if self.start < self.end { + let pos = self.end; + self.end = advance_bwd(self.start..self.end); + if let Some(ret) = R::rejecting(self.end, pos) { + self.is_match_bwd = true; + return ret; + } + return R::matching(self.end, self.end).unwrap(); + } + self.is_finished = true; + R::DONE + } +} + ////////////////////////////////////////////////////////////////////////////// // Internal Split and SplitN implementations ////////////////////////////////////////////////////////////////////////////// @@ -523,7 +740,7 @@ impl> Split { impl> Split { /// Returns the next part of the haystack or `None` if splitting is done. /// - /// If `INCLUSIVE` is `true`, returned value will include the matching + /// If `INCLUSIVE` is `true`, the returned value will include the matching /// pattern. #[inline] pub fn next_fwd(&mut self) -> Option { @@ -545,7 +762,7 @@ impl> Split { /// Returns the next part of the haystack looking from the back or /// `None` if splitting is done. /// - /// If `INCLUSIVE` is `true`, returned value will include the matching + /// If `INCLUSIVE` is `true`, the returned value will include the matching /// pattern. #[inline] pub fn next_bwd(&mut self) -> Option @@ -574,12 +791,12 @@ impl> Split { self.finished = true; self.start..self.end }; - // SAFETY: All indices come from Haystack or Searcher which guarantee - // that they are valid split positions. + // SAFETY: All indices come from Haystack or Searcher. + // They are known to return good indices Some(unsafe { self.searcher.haystack().get_unchecked(range) }) } - /// Returns remaining part of the haystack that hasn't been processed yet. + /// Returns the remaining part of the haystack that hasn't been processed yet. #[inline] pub fn remainder(&self) -> Option { (!self.finished).then(|| { @@ -591,7 +808,7 @@ impl> Split { /// Returns the final haystack part. /// - /// Sets `finished` flag so any further calls to this or other methods will + /// Sets the `finished` flag so any further calls to this or other methods will /// return `None`. #[inline] fn get_end(&mut self) -> Option { @@ -612,7 +829,7 @@ impl> Split { impl> SplitN { /// Returns next part of the haystack or `None` if splitting is done. /// - /// If `INCLUSIVE` is `true`, returned value will include the matching + /// If `INCLUSIVE` is `true`, the returned value will include the matching /// pattern. #[inline] pub fn next_fwd(&mut self) -> Option { @@ -625,7 +842,7 @@ impl> SplitN { /// Returns next looking from back of the haystack part of the haystack or /// `None` if splitting is done. /// - /// If `INCLUSIVE` is `true`, returned value will include the matching + /// If `INCLUSIVE` is `true`, the returned value will include the matching /// pattern. #[inline] pub fn next_bwd(&mut self) -> Option @@ -638,7 +855,7 @@ impl> SplitN { } } - /// Returns remaining part of the haystack that hasn't been processed yet. + /// Returns the remaining part of the haystack that hasn't been processed yet. #[inline] pub fn remainder(&self) -> Option { self.inner.remainder() diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index 22f36c7297a2a..ba82e8db0201c 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -53,7 +53,8 @@ use crate::cmp::Ordering; use crate::convert::TryInto as _; use crate::ops::Range; pub use crate::pattern::{ - DoubleEndedSearcher, Haystack, Pattern, ReverseSearcher, SearchStep, Searcher, Utf8Pattern, + DoubleEndedSearcher, Haystack, MatchOnly, Pattern, ReverseSearcher, SearchResult, SearchStep, + Searcher, Utf8Pattern, }; use crate::slice::memchr; use crate::{cmp, fmt}; @@ -795,43 +796,36 @@ pub struct StrSearcher<'a, 'b> { #[derive(Clone, Debug)] enum StrSearcherImpl { - Empty(EmptyNeedle), + Empty(core::pattern::EmptyNeedleSearcher), TwoWay(TwoWaySearcher), } -#[derive(Clone, Debug)] -struct EmptyNeedle { - position: usize, - end: usize, - is_match_fw: bool, - is_match_bw: bool, - // Needed in case of an empty haystack, see #85462 - is_finished: bool, -} - impl<'a, 'b> StrSearcher<'a, 'b> { fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { - if needle.is_empty() { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::Empty(EmptyNeedle { - position: 0, - end: haystack.len(), - is_match_fw: true, - is_match_bw: true, - is_finished: false, - }), - } + let searcher = if needle.is_empty() { + StrSearcherImpl::Empty(core::pattern::EmptyNeedleSearcher::new(haystack)) } else { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::TwoWay(TwoWaySearcher::new( - needle.as_bytes(), - haystack.len(), - )), - } + StrSearcherImpl::TwoWay(TwoWaySearcher::new(needle.as_bytes(), haystack.len())) + }; + StrSearcher { haystack, needle, searcher } + } + + fn fwd_char(haystack: &str, pos: usize) -> usize { + pos + super::utf8_char_width(haystack.as_bytes()[pos]) + } + + fn bwd_char(haystack: &str, pos: usize) -> usize { + // Note: we are guaranteed to operate on valid UTF-8 thus we will never + // need to go further than four bytes back. + let bytes = haystack.as_bytes(); + if bytes[pos - 1].is_utf8_char_boundary() { + pos - 1 + } else if bytes[pos - 2].is_utf8_char_boundary() { + pos - 2 + } else if bytes[pos - 3].is_utf8_char_boundary() { + pos - 3 + } else { + pos - 4 } } } @@ -846,24 +840,7 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { fn next(&mut self) -> SearchStep { match self.searcher { StrSearcherImpl::Empty(ref mut searcher) => { - if searcher.is_finished { - return SearchStep::Done; - } - // empty needle rejects every char and matches every empty string between them - let is_match = searcher.is_match_fw; - searcher.is_match_fw = !searcher.is_match_fw; - let pos = searcher.position; - match self.haystack[pos..].chars().next() { - _ if is_match => SearchStep::Match(pos, pos), - None => { - searcher.is_finished = true; - SearchStep::Done - } - Some(ch) => { - searcher.position += ch.len_utf8(); - SearchStep::Reject(pos, searcher.position) - } - } + searcher.next_fwd(|range| Self::fwd_char(self.haystack, range.start)) } StrSearcherImpl::TwoWay(ref mut searcher) => { // TwoWaySearcher produces valid *Match* indices that split at char boundaries @@ -875,11 +852,7 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { return SearchStep::Done; } let is_long = searcher.memory == usize::MAX; - match searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - is_long, - ) { + match searcher.next(self.haystack.as_bytes(), self.needle.as_bytes(), is_long) { SearchStep::Reject(a, mut b) => { // skip to next char boundary while !self.haystack.is_char_boundary(b) { @@ -897,29 +870,23 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { match self.searcher { - StrSearcherImpl::Empty(..) => loop { - match self.next() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - SearchStep::Reject(..) => {} - } - }, + StrSearcherImpl::Empty(ref mut searcher) => { + searcher + .next_fwd::(|range| Self::fwd_char(self.haystack, range.start)) + .0 + } StrSearcherImpl::TwoWay(ref mut searcher) => { let is_long = searcher.memory == usize::MAX; // write out `true` and `false` cases to encourage the compiler // to specialize the two cases separately. if is_long { - searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - true, - ) + searcher + .next::(self.haystack.as_bytes(), self.needle.as_bytes(), true) + .0 } else { - searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - false, - ) + searcher + .next::(self.haystack.as_bytes(), self.needle.as_bytes(), false) + .0 } } } @@ -931,34 +898,15 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { fn next_back(&mut self) -> SearchStep { match self.searcher { StrSearcherImpl::Empty(ref mut searcher) => { - if searcher.is_finished { - return SearchStep::Done; - } - let is_match = searcher.is_match_bw; - searcher.is_match_bw = !searcher.is_match_bw; - let end = searcher.end; - match self.haystack[..end].chars().next_back() { - _ if is_match => SearchStep::Match(end, end), - None => { - searcher.is_finished = true; - SearchStep::Done - } - Some(ch) => { - searcher.end -= ch.len_utf8(); - SearchStep::Reject(searcher.end, end) - } - } + searcher.next_bwd(|range| Self::bwd_char(self.haystack, range.end)) } StrSearcherImpl::TwoWay(ref mut searcher) => { if searcher.end == 0 { return SearchStep::Done; } let is_long = searcher.memory == usize::MAX; - match searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - is_long, - ) { + match searcher.next_back(self.haystack.as_bytes(), self.needle.as_bytes(), is_long) + { SearchStep::Reject(mut a, b) => { // skip to next char boundary while !self.haystack.is_char_boundary(a) { @@ -976,28 +924,30 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn next_match_back(&mut self) -> Option<(usize, usize)> { match self.searcher { - StrSearcherImpl::Empty(..) => loop { - match self.next_back() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - SearchStep::Reject(..) => {} - } - }, + StrSearcherImpl::Empty(ref mut searcher) => { + searcher + .next_bwd::(|range| Self::bwd_char(self.haystack, range.end)) + .0 + } StrSearcherImpl::TwoWay(ref mut searcher) => { let is_long = searcher.memory == usize::MAX; // write out `true` and `false`, like `next_match` if is_long { - searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - true, - ) + searcher + .next_back::( + self.haystack.as_bytes(), + self.needle.as_bytes(), + true, + ) + .0 } else { - searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - false, - ) + searcher + .next_back::( + self.haystack.as_bytes(), + self.needle.as_bytes(), + false, + ) + .0 } } } @@ -1185,10 +1135,7 @@ impl TwoWaySearcher { // How far we can jump when we encounter a mismatch is all based on the fact // that (u, v) is a critical factorization for the needle. #[inline] - fn next(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> S::Output - where - S: TwoWayStrategy, - { + fn next(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> R { // `next()` uses `self.position` as its cursor let old_pos = self.position; let needle_last = needle.len() - 1; @@ -1200,12 +1147,14 @@ impl TwoWaySearcher { Some(&b) => b, None => { self.position = haystack.len(); - return S::rejecting(old_pos, self.position); + return R::rejecting(old_pos, self.position).unwrap_or(R::DONE); } }; - if S::use_early_reject() && old_pos != self.position { - return S::rejecting(old_pos, self.position); + if old_pos != self.position { + if let Some(ret) = R::rejecting(old_pos, self.position) { + return ret; + } } // Quickly skip by large portions unrelated to our substring @@ -1263,7 +1212,7 @@ impl TwoWaySearcher { self.memory = 0; // set to needle.len() - self.period for overlapping matches } - return S::matching(match_pos, match_pos + needle.len()); + return R::matching(match_pos, match_pos + needle.len()).unwrap(); } } @@ -1280,10 +1229,12 @@ impl TwoWaySearcher { // To search in reverse through the haystack, we search forward through // a reversed haystack with a reversed needle, matching first u' and then v'. #[inline] - fn next_back(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> S::Output - where - S: TwoWayStrategy, - { + fn next_back( + &mut self, + haystack: &[u8], + needle: &[u8], + long_period: bool, + ) -> R { // `next_back()` uses `self.end` as its cursor -- so that `next()` and `next_back()` // are independent. let old_end = self.end; @@ -1296,12 +1247,14 @@ impl TwoWaySearcher { Some(&b) => b, None => { self.end = 0; - return S::rejecting(0, old_end); + return R::rejecting(0, old_end).unwrap_or(R::DONE); } }; - if S::use_early_reject() && old_end != self.end { - return S::rejecting(self.end, old_end); + if old_end != self.end { + if let Some(ret) = R::rejecting(self.end, old_end) { + return ret; + } } // Quickly skip by large portions unrelated to our substring @@ -1361,7 +1314,7 @@ impl TwoWaySearcher { self.memory_back = needle.len(); } - return S::matching(match_pos, match_pos + needle.len()); + return R::matching(match_pos, match_pos + needle.len()).unwrap(); } } @@ -1464,55 +1417,6 @@ impl TwoWaySearcher { } } -// TwoWayStrategy allows the algorithm to either skip non-matches as quickly -// as possible, or to work in a mode where it emits Rejects relatively quickly. -trait TwoWayStrategy { - type Output; - fn use_early_reject() -> bool; - fn rejecting(a: usize, b: usize) -> Self::Output; - fn matching(a: usize, b: usize) -> Self::Output; -} - -/// Skip to match intervals as quickly as possible -enum MatchOnly {} - -impl TwoWayStrategy for MatchOnly { - type Output = Option<(usize, usize)>; - - #[inline] - fn use_early_reject() -> bool { - false - } - #[inline] - fn rejecting(_a: usize, _b: usize) -> Self::Output { - None - } - #[inline] - fn matching(a: usize, b: usize) -> Self::Output { - Some((a, b)) - } -} - -/// Emit Rejects regularly -enum RejectAndMatch {} - -impl TwoWayStrategy for RejectAndMatch { - type Output = SearchStep; - - #[inline] - fn use_early_reject() -> bool { - true - } - #[inline] - fn rejecting(a: usize, b: usize) -> Self::Output { - SearchStep::Reject(a, b) - } - #[inline] - fn matching(a: usize, b: usize) -> Self::Output { - SearchStep::Match(a, b) - } -} - /// SIMD search for short needles based on /// Wojciech Muła's "SIMD-friendly algorithms for substring searching"[0] /// From d359f709abd36d5f67f43ae66115d1861ce87a66 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:46:53 -0400 Subject: [PATCH 09/16] core: add try_next_code_point{,_reverse} internal functions @pacak: - made more things const fn - there was a (copy-paste?) error in try_finish_byte_sequence so I added a test that checks try_next_code_point(_reverse) with some values, including invalid ones. - reworded a few comments (passive voice, etc) Also different comments: since former is public and later is private due to historical reasons. > This is different than [`next_code_point`] in that it doesn't assume > This is different than `next_code_point_reverse` in that it doesn't assume --- library/core/src/str/mod.rs | 4 +- library/core/src/str/validations.rs | 231 +++++++++++++++++++--------- library/coretests/tests/pattern.rs | 55 +++++++ 3 files changed, 220 insertions(+), 70 deletions(-) diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 5564043b8ae7b..6c244c10e9745 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -59,7 +59,9 @@ pub use lossy::{Utf8Chunk, Utf8Chunks}; #[stable(feature = "rust1", since = "1.0.0")] pub use traits::FromStr; #[unstable(feature = "str_internals", issue = "none")] -pub use validations::{next_code_point, utf8_char_width}; +pub use validations::{ + next_code_point, try_next_code_point, try_next_code_point_reverse, utf8_char_width, +}; #[inline(never)] #[cold] diff --git a/library/core/src/str/validations.rs b/library/core/src/str/validations.rs index b54d6478e584d..b95d854712ac2 100644 --- a/library/core/src/str/validations.rs +++ b/library/core/src/str/validations.rs @@ -119,6 +119,78 @@ const fn contains_nonascii(x: usize) -> bool { (x & NONASCII_MASK) != 0 } +/// Reads the first code point and its encoded length out of a byte slice +/// validating whether it's valid. +/// +/// This is different than [`next_code_point`] in that it doesn't assume +/// the argument is a well-formed UTF-8-like string. +/// +/// If front of the bytes slice doesn't contain valid UTF-8 bytes sequence (that +/// includes a WTF-8 encoded surrogate) returns `None`. +/// +/// ``` +/// #![feature(str_internals)] +/// # #![allow(internal_features)] +/// use core::str::try_next_code_point; +/// +/// assert_eq!(Some(('f', 1)), try_next_code_point(b"foo".as_ref())); +/// assert_eq!(Some(('Ż', 2)), try_next_code_point("Żółw".as_bytes())); +/// assert_eq!(None, try_next_code_point(b"\xffoo".as_ref())); +/// ``` +#[unstable(feature = "str_internals", issue = "none")] +#[rustc_const_unstable(feature = "str_internals", issue = "none")] +#[inline] +pub const fn try_next_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let first = *bytes.first()?; + let (value, length) = if first < 0x80 { + (first as u32, 1) + } else { + try_finish_byte_sequence(first, bytes, 0).ok()? + }; + // SAFETY: value is a valid Unicode scalar value. + // Either ASCII (first branch) or a valid non-ASCII Unicode character + // as guaranteed by `try_finish_byte_sequence` (second branch). + Some((unsafe { char::from_u32_unchecked(value) }, length)) +} + +/// Reads the last code point and its encoded length out of a byte slice +/// validating whether it's valid. +/// +/// This is different than `next_code_point_reverse` in that it doesn't assume +/// the argument is a well-formed UTF-8-like string. +/// +/// If back of the bytes slice doesn't contain valid UTF-8 bytes sequence (that +/// includes a WTF-8 encoded surrogate) returns `None`. +/// +/// ``` +/// #![feature(str_internals)] +/// # #![allow(internal_features)] +/// use core::str::try_next_code_point_reverse; +/// +/// assert_eq!(Some(('o', 1)), try_next_code_point_reverse(b"foo".as_ref())); +/// assert_eq!(Some(('‽', 3)), try_next_code_point_reverse("Uh‽".as_bytes())); +/// assert_eq!(None, try_next_code_point_reverse(b"foo\xff".as_ref())); +/// ``` +#[unstable(feature = "str_internals", issue = "none")] +#[rustc_const_unstable(feature = "str_internals", issue = "none")] +#[inline] +pub const fn try_next_code_point_reverse(bytes: &[u8]) -> Option<(char, usize)> { + let mut n = 1; + let limit = bytes.len(); + let limit = if limit < 4 { limit } else { 4 }; // not .min(4) because of const + while n <= limit && !bytes[bytes.len() - n].is_utf8_char_boundary() { + n += 1; + } + if n <= limit { + let bytes = &bytes[bytes.len() - n..]; + let (chr, len) = try_next_code_point(bytes)?; + if n == len { + return Some((chr, len)); + } + } + None +} + /// Walks through `v` checking that it's a valid UTF-8 sequence, /// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. #[inline(always)] @@ -143,78 +215,13 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { ); while index < len { - let old_offset = index; - macro_rules! err { - ($error_len: expr) => { - return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len }) - }; - } - - macro_rules! next { - () => {{ - index += 1; - // we needed data, but there was none: error! - if index >= len { - err!(None) - } - v[index] - }}; - } - + let valid_up_to = index; let first = v[index]; if first >= 128 { - let w = utf8_char_width(first); - // 2-byte encoding is for codepoints \u{0080} to \u{07ff} - // first C2 80 last DF BF - // 3-byte encoding is for codepoints \u{0800} to \u{ffff} - // first E0 A0 80 last EF BF BF - // excluding surrogates codepoints \u{d800} to \u{dfff} - // ED A0 80 to ED BF BF - // 4-byte encoding is for codepoints \u{10000} to \u{10ffff} - // first F0 90 80 80 last F4 8F BF BF - // - // Use the UTF-8 syntax from the RFC - // - // https://tools.ietf.org/html/rfc3629 - // UTF8-1 = %x00-7F - // UTF8-2 = %xC2-DF UTF8-tail - // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - // %xF4 %x80-8F 2( UTF8-tail ) - match w { - 2 => { - if next!() as i8 >= -64 { - err!(Some(1)) - } - } - 3 => { - match (first, next!()) { - (0xE0, 0xA0..=0xBF) - | (0xE1..=0xEC, 0x80..=0xBF) - | (0xED, 0x80..=0x9F) - | (0xEE..=0xEF, 0x80..=0xBF) => {} - _ => err!(Some(1)), - } - if next!() as i8 >= -64 { - err!(Some(2)) - } - } - 4 => { - match (first, next!()) { - (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} - _ => err!(Some(1)), - } - if next!() as i8 >= -64 { - err!(Some(2)) - } - if next!() as i8 >= -64 { - err!(Some(3)) - } - } - _ => err!(Some(1)), + match try_finish_byte_sequence(first, v, index) { + Ok((_value, length)) => index += length, + Err(error_len) => return Err(Utf8Error { valid_up_to, error_len }), } - index += 1; } else { // Ascii case, try to skip forward quickly. // When the pointer is aligned, read 2 words of data per iteration @@ -250,6 +257,92 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { Ok(()) } +/// Try to finish an UTF-8 byte sequence. +/// +/// Assumes that `bytes[index] == first` and then `first >= 128`, i.e. that +/// `index` points at the beginning of a non-ASCII UTF-8 sequence in `bytes`. +/// +/// If the byte sequence at the index is correct, returns decoded code point and +/// length of the sequence. If it was invalid returns number of invalid bytes +/// or None if read was cut short. +#[inline(always)] +const fn try_finish_byte_sequence( + first: u8, + bytes: &[u8], + index: usize, +) -> Result<(u32, usize), Option> { + macro_rules! get { + (raw $offset:expr) => { + if index + $offset < bytes.len() { + bytes[index + $offset] + } else { + return Err(None) + } + }; + (cont $offset:expr) => {{ + let byte = get!(raw $offset); + if !utf8_is_cont_byte(byte) { + return Err(Some($offset as u8)) + } + byte + }} + } + + // 2-byte encoding is for codepoints \u{0080} to \u{07ff} + // first C2 80 last DF BF + // 3-byte encoding is for codepoints \u{0800} to \u{ffff} + // first E0 A0 80 last EF BF BF + // excluding surrogates codepoints \u{d800} to \u{dfff} + // ED A0 80 to ED BF BF + // 4-byte encoding is for codepoints \u{10000} to \u{10ffff} + // first F0 90 80 80 last F4 8F BF BF + // + // Use the UTF-8 syntax from the RFC + // + // https://tools.ietf.org/html/rfc3629 + // UTF8-1 = %x00-7F + // UTF8-2 = %xC2-DF UTF8-tail + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + match utf8_char_width(first) { + 2 => { + let second = get!(cont 1); + let value = utf8_first_byte(first, 2); + let value = utf8_acc_cont_byte(value, second); + Ok((value, 2)) + } + 3 => { + let second = get!(raw 1); + match (first, second) { + (0xE0, 0xA0..=0xBF) + | (0xE1..=0xEC, 0x80..=0xBF) + | (0xED, 0x80..=0x9F) + | (0xEE..=0xEF, 0x80..=0xBF) => {} + _ => return Err(Some(1)), + } + let value = utf8_first_byte(first, 3); + let value = utf8_acc_cont_byte(value, second); + let value = utf8_acc_cont_byte(value, get!(cont 2)); + Ok((value, 3)) + } + 4 => { + let second = get!(raw 1); + match (first, second) { + (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} + _ => return Err(Some(1)), + } + let value = utf8_first_byte(first, 4); + let value = utf8_acc_cont_byte(value, second); + let value = utf8_acc_cont_byte(value, get!(cont 2)); + let value = utf8_acc_cont_byte(value, get!(cont 3)); + Ok((value, 4)) + } + _ => Err(Some(1)), + } +} + // https://tools.ietf.org/html/rfc3629 const UTF8_CHAR_WIDTH: &[u8; 256] = &[ // 1 2 3 4 5 6 7 8 9 A B C D E F diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index e042d16b77875..90a0e4717e219 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -651,3 +651,58 @@ fn two_way_next_reject_never_reports_empty_reject() { assert_eq!(searcher.next_reject_back(), Some((3, 6))); assert_eq!(searcher.next_reject_back(), None); } + +#[test] +fn test_try_next_code_point_fwd_and_rev() { + use core::str::{try_next_code_point, try_next_code_point_reverse}; + + #[track_caller] + fn check_fwbw(input: &[u8], expected: Option<(char, usize)>) { + let mut store = Vec::with_capacity(input.len() + 1); + + assert_eq!(try_next_code_point(input), expected, "fw single"); + assert_eq!(try_next_code_point_reverse(input), expected, "bw single"); + + if input.is_empty() { + return; + } + + store.extend_from_slice(input); + store.push(b'a'); + assert_eq!(try_next_code_point(&store), expected, "fw ext"); + + store.clear(); + store.push(b'a'); + store.extend_from_slice(input); + assert_eq!(try_next_code_point_reverse(&store), expected, "bw, ext"); + } + + // edge case + check_fwbw(&[], None); // edge case + + // valid cases + check_fwbw(&[0x41], Some(('A', 1))); // 1 byte + check_fwbw(&[0xc3, 0xa9], Some(('\u{00e9}', 2))); // 2 byte + check_fwbw(&[0xe2, 0x82, 0xac], Some(('\u{20ac}', 3))); // 3 byte + check_fwbw(&[0xf0, 0x9f, 0x98, 0x8a], Some(('\u{1f60a}', 4))); // 4 byte + + // overlong encoding + check_fwbw(&[0xc0, 0xaf], None); // '\u{002f}' AKA '/' + check_fwbw(&[0xc1, 0x81], None); // '\u{0041}' AKA 'A' + check_fwbw(&[0xe0, 0x81, 0x81], None); // '\u{0041}' AKA 'A' + check_fwbw(&[0xe0, 0x83, 0xa9], None); // '\u{00e9}' AKA 'é', should be 2 bytes + + // surrogates + check_fwbw(&[0xed, 0xa0, 0x80], None); + check_fwbw(&[0xed, 0xb0, 0x80], None); + check_fwbw(&[0xed, 0xa0, 0xbd, 0xed, 0xb8, 0x8a], None); + + // unattached continuation + check_fwbw(&[0x80], None); + + // truncated + check_fwbw(&[0xe2, 0x82], None); + + // out of range + check_fwbw(&[0xf5, 0x80, 0x80, 0x80], None); +} From 5705a4330d0a945a2e4ad7c05f9e1c5921d6d213 Mon Sep 17 00:00:00 2001 From: Michael Baikov Date: Mon, 3 Aug 2026 06:58:06 -0400 Subject: [PATCH 10/16] core: add internal core::str_bytes module handling string-like slices Introduce a new core::str_bytes module with types and functions which handle string-like bytes slices. String-like means that they code treats UTF-8 byte sequences as characters within such slices but doesn't assume that the slices are well-formed. A `str` is trivially a bytes sequence that the module can handle but so is OsStr (which is WTF-8 on Windows and unstructured bytes on Unix). Move bunch of code (most notably implementation of the two-way string-matching algorithm) from core::str to core::str_bytes. Note that this likely introduces regression in some of the str function performance (since the new code cannot assume well-formed UTF-8). This is going to be rectified by following commit which will make it again possible for the code to assume bytes format. This is not done in this commit to keep it smaller. @pacak: - Added a few comments - tried to hide internal types from the diagnostic And then there's two different bugs where it would report matched areas as rejected. This broke str::trim_end_matches and who knows what else. Caught it thanks to tests in the previous commit. And one underflow bug on invalid input. --- library/alloctests/tests/str.rs | 33 +- library/core/src/lib.rs | 1 + library/core/src/pattern.rs | 16 + library/core/src/str/pattern.rs | 859 ++--------------- library/core/src/str_bytes.rs | 1412 ++++++++++++++++++++++++++++ library/coretests/tests/pattern.rs | 156 ++- 6 files changed, 1564 insertions(+), 913 deletions(-) create mode 100644 library/core/src/str_bytes.rs diff --git a/library/alloctests/tests/str.rs b/library/alloctests/tests/str.rs index a9255ee705de5..80e7a6d9dffc9 100644 --- a/library/alloctests/tests/str.rs +++ b/library/alloctests/tests/str.rs @@ -836,6 +836,18 @@ fn test_trim_matches() { assert_eq!("123foo1bar123".trim_matches(|c: char| c.is_numeric()), "foo1bar"); } +#[test] +fn test_trim_matches_with_str_pattern() { + assert_eq!("abc".trim_start_matches("ab"), "c"); + assert_eq!("xyzabcxyz".trim_start_matches("xyz"), "abcxyz"); + assert_eq!("abcabc".trim_start_matches("abc"), ""); + assert_eq!("ababab".trim_start_matches("ab"), ""); + + assert_eq!("abcab".trim_end_matches("ab"), "abc"); + assert_eq!("xyzabcxyz".trim_end_matches("xyz"), "xyzabc"); + assert_eq!("abcabc".trim_end_matches("abc"), ""); +} + #[test] fn test_trim_start() { assert_eq!("".trim_start(), ""); @@ -2107,12 +2119,7 @@ mod pattern { Match(7, 7), ] ); - make_test!( - str_searcher_multibyte_haystack, - " ", - "├──", - [Reject(0, 3), Reject(3, 6), Reject(6, 9),] - ); + make_test!(str_searcher_multibyte_haystack, " ", "├──", [Reject(0, 9),]); make_test!( str_searcher_empty_needle_multibyte_haystack, "", @@ -2143,18 +2150,8 @@ mod pattern { Reject(6, 7), ] ); - make_test!( - char_searcher_multibyte_haystack, - ' ', - "├──", - [Reject(0, 3), Reject(3, 6), Reject(6, 9),] - ); - make_test!( - char_searcher_short_haystack, - '\u{1F4A9}', - "* \t", - [Reject(0, 1), Reject(1, 2), Reject(2, 3),] - ); + make_test!(char_searcher_multibyte_haystack, ' ', "├──", [Reject(0, 9),]); + make_test!(char_searcher_short_haystack, '\u{1F4A9}', "* \t", [Reject(0, 3),]); // See #85462 #[test] diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index c5bf8bea6accc..c82e66cad0087 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -321,6 +321,7 @@ pub mod hash; pub mod pattern; pub mod slice; pub mod str; +pub mod str_bytes; pub mod time; pub mod wtf8; diff --git a/library/core/src/pattern.rs b/library/core/src/pattern.rs index b894e9a911b4d..fdc5c8310a25f 100644 --- a/library/core/src/pattern.rs +++ b/library/core/src/pattern.rs @@ -246,6 +246,19 @@ pub trait SearchResult: Sized + sealed::Sealed { /// Value indicating searching has finished. const DONE: Self; + /// Whether search should return reject as soon as possible. + /// + /// For example, if a search can quickly determine that the very next + /// position cannot be where a next match starts, it should return a reject + /// with that position. This is an optimisation which allows the algorithm + /// to not waste time looking for the next match if caller is only + /// interested in the next position of a reject. + /// + /// If this is `true`, [`rejecting()`][Self::rejecting] is guaranteed to + /// return `Some` and if this is `false`, [`matching()`][Self::matching] is + /// guaranteed to return `Some`. + const USE_EARLY_REJECT: bool; + /// Returns value describing a match or `None` if this implementation /// doesn't care about matches. fn matching(start: usize, end: usize) -> Option; @@ -267,6 +280,7 @@ pub struct RejectOnly(pub Option<(usize, usize)>); impl SearchResult for SearchStep { const DONE: Self = SearchStep::Done; + const USE_EARLY_REJECT: bool = false; #[inline(always)] fn matching(s: usize, e: usize) -> Option { @@ -281,6 +295,7 @@ impl SearchResult for SearchStep { impl SearchResult for MatchOnly { const DONE: Self = Self(None); + const USE_EARLY_REJECT: bool = false; #[inline(always)] fn matching(s: usize, e: usize) -> Option { @@ -295,6 +310,7 @@ impl SearchResult for MatchOnly { impl SearchResult for RejectOnly { const DONE: Self = Self(None); + const USE_EARLY_REJECT: bool = true; #[inline(always)] fn matching(_s: usize, _e: usize) -> Option { diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index ba82e8db0201c..76952557235e5 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -50,14 +50,12 @@ )] use crate::cmp::Ordering; -use crate::convert::TryInto as _; use crate::ops::Range; pub use crate::pattern::{ DoubleEndedSearcher, Haystack, MatchOnly, Pattern, ReverseSearcher, SearchResult, SearchStep, Searcher, Utf8Pattern, }; -use crate::slice::memchr; -use crate::{cmp, fmt}; +use crate::{fmt, str_bytes}; ///////////////////////////////////////////////////////////////////////////// // Impl for Haystack @@ -89,189 +87,50 @@ impl<'a> Haystack for &'a str { // Impl for char ///////////////////////////////////////////////////////////////////////////// -/// Associated type for `>::Searcher`. +/// Associated type for `>::Searcher`. #[derive(Clone, Debug)] -pub struct CharSearcher<'a> { - haystack: &'a str, - // safety invariant: `finger`/`finger_back` must be a valid utf8 byte index of `haystack` - // This invariant can be broken *within* next_match and next_match_back, however - // they must exit with fingers on valid code point boundaries. - /// `finger` is the current byte index of the forward search. - /// Imagine that it exists before the byte at its index, i.e. - /// `haystack[finger]` is the first byte of the slice we must inspect during - /// forward searching - finger: usize, - /// `finger_back` is the current byte index of the reverse search. - /// Imagine that it exists after the byte at its index, i.e. - /// haystack[finger_back - 1] is the last byte of the slice we must inspect during - /// forward searching (and thus the first byte to be inspected when calling next_back()). - finger_back: usize, - /// The character being searched for - needle: char, - - // safety invariant: `utf8_size` must be less than 5 - /// The number of bytes `needle` takes up when encoded in utf8. - utf8_size: u8, - /// A utf8 encoded copy of the `needle` - utf8_encoded: [u8; 4], -} +pub struct CharSearcher<'a>(str_bytes::CharSearcher<'a>); -impl CharSearcher<'_> { - fn utf8_size(&self) -> usize { - self.utf8_size.into() +impl<'a> CharSearcher<'a> { + fn new(haystack: &'a str, chr: char) -> Self { + Self(str_bytes::CharSearcher::new(str_bytes::Bytes::from_str(haystack), chr)) } } unsafe impl<'a> Searcher<&'a str> for CharSearcher<'a> { #[inline] fn haystack(&self) -> &'a str { - self.haystack + // SAFETY: self.0’s haystack was created from &str thus it is valid + // UTF-8. + unsafe { super::from_utf8_unchecked(self.0.haystack().as_bytes()) } } #[inline] fn next(&mut self) -> SearchStep { - let old_finger = self.finger; - // SAFETY: 1-4 guarantee safety of `get_unchecked` - // 1. `self.finger` and `self.finger_back` are kept on unicode boundaries - // (this is invariant) - // 2. `self.finger >= 0` since it starts at 0 and only increases - // 3. `self.finger < self.finger_back` because otherwise the char `iter` - // would return `SearchStep::Done` - // 4. `self.finger` comes before the end of the haystack because `self.finger_back` - // starts at the end and only decreases - let slice = unsafe { self.haystack.get_unchecked(old_finger..self.finger_back) }; - let mut iter = slice.chars(); - let old_len = iter.iter.len(); - if let Some(ch) = iter.next() { - // add byte offset of current character - // without re-encoding as utf-8 - self.finger += old_len - iter.iter.len(); - if ch == self.needle { - SearchStep::Match(old_finger, self.finger) - } else { - SearchStep::Reject(old_finger, self.finger) - } - } else { - SearchStep::Done - } + self.0.next() } #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { - loop { - // get the haystack after the last character found - let bytes = self.haystack.as_bytes().get(self.finger..self.finger_back)?; - // the last byte of the utf8 encoded needle - // SAFETY: we have an invariant that `utf8_size < 5` - let last_byte = unsafe { *self.utf8_encoded.get_unchecked(self.utf8_size() - 1) }; - if let Some(index) = memchr::memchr(last_byte, bytes) { - // The new finger is the index of the byte we found, - // plus one, since we memchr'd for the last byte of the character. - // - // Note that this doesn't always give us a finger on a UTF8 boundary. - // If we *didn't* find our character - // we may have indexed to the non-last byte of a 3-byte or 4-byte character. - // We can't just skip to the next valid starting byte because a character like - // ꁁ (U+A041 YI SYLLABLE PA), utf-8 `EA 81 81` will have us always find - // the second byte when searching for the third. - // - // However, this is totally okay. While we have the invariant that - // self.finger is on a UTF8 boundary, this invariant is not relied upon - // within this method (it is relied upon in CharSearcher::next()). - // - // We only exit this method when we reach the end of the string, or if we - // find something. When we find something the `finger` will be set - // to a UTF8 boundary. - self.finger += index + 1; - if self.finger >= self.utf8_size() { - let found_char = self.finger - self.utf8_size(); - if let Some(slice) = self.haystack.as_bytes().get(found_char..self.finger) { - if slice == &self.utf8_encoded[0..self.utf8_size()] { - return Some((found_char, self.finger)); - } - } - } - } else { - // found nothing, exit - self.finger = self.finger_back; - return None; - } - } + self.0.next_match() + } + #[inline] + fn next_reject(&mut self) -> Option<(usize, usize)> { + self.0.next_reject() } - - // let next_reject use the default implementation from the Searcher trait } unsafe impl<'a> ReverseSearcher<&'a str> for CharSearcher<'a> { #[inline] fn next_back(&mut self) -> SearchStep { - let old_finger = self.finger_back; - // SAFETY: see the comment for next() above - let slice = unsafe { self.haystack.get_unchecked(self.finger..old_finger) }; - let mut iter = slice.chars(); - let old_len = iter.iter.len(); - if let Some(ch) = iter.next_back() { - // subtract byte offset of current character - // without re-encoding as utf-8 - self.finger_back -= old_len - iter.iter.len(); - if ch == self.needle { - SearchStep::Match(self.finger_back, old_finger) - } else { - SearchStep::Reject(self.finger_back, old_finger) - } - } else { - SearchStep::Done - } + self.0.next_back() } #[inline] fn next_match_back(&mut self) -> Option<(usize, usize)> { - let haystack = self.haystack.as_bytes(); - loop { - // get the haystack up to but not including the last character searched - let bytes = haystack.get(self.finger..self.finger_back)?; - // the last byte of the utf8 encoded needle - // SAFETY: we have an invariant that `utf8_size < 5` - let last_byte = unsafe { *self.utf8_encoded.get_unchecked(self.utf8_size() - 1) }; - if let Some(index) = memchr::memrchr(last_byte, bytes) { - // we searched a slice that was offset by self.finger, - // add self.finger to recoup the original index - let index = self.finger + index; - // memrchr will return the index of the byte we wish to - // find. In case of an ASCII character, this is indeed - // were we wish our new finger to be ("after" the found - // char in the paradigm of reverse iteration). For - // multibyte chars we need to skip down by the number of more - // bytes they have than ASCII - let shift = self.utf8_size() - 1; - if index >= shift { - let found_char = index - shift; - if let Some(slice) = haystack.get(found_char..(found_char + self.utf8_size())) { - if slice == &self.utf8_encoded[0..self.utf8_size()] { - // move finger to before the character found (i.e., at its start index) - self.finger_back = found_char; - return Some((self.finger_back, self.finger_back + self.utf8_size())); - } - } - } - // We can't use finger_back = index - size + 1 here. If we found the last char - // of a different-sized character (or the middle byte of a different character) - // we need to bump the finger_back down to `index`. This similarly makes - // `finger_back` have the potential to no longer be on a boundary, - // but this is OK since we only exit this function on a boundary - // or when the haystack has been searched completely. - // - // Unlike next_match this does not - // have the problem of repeated bytes in utf-8 because - // we're searching for the last byte, and we can only have - // found the last byte when searching in reverse. - self.finger_back = index; - } else { - self.finger_back = self.finger; - // found nothing, exit - return None; - } - } + self.0.next_match_back() + } + #[inline] + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + self.0.next_reject_back() } - - // let next_reject_back use the default implementation from the Searcher trait } impl<'a> DoubleEndedSearcher<&'a str> for CharSearcher<'a> {} @@ -282,37 +141,19 @@ impl<'a> DoubleEndedSearcher<&'a str> for CharSearcher<'a> {} /// /// ``` /// assert_eq!("Hello world".find('o'), Some(4)); +/// assert_eq!("Hello world".find('x'), None); /// ``` impl<'a> Pattern<&'a str> for char { type Searcher = CharSearcher<'a>; #[inline] fn into_searcher(self, haystack: &'a str) -> Self::Searcher { - let mut utf8_encoded = [0; char::MAX_LEN_UTF8]; - let utf8_size = self - .encode_utf8(&mut utf8_encoded) - .len() - .try_into() - .expect("char len should be less than 255"); - - CharSearcher { - haystack, - finger: 0, - finger_back: haystack.len(), - needle: self, - utf8_size, - utf8_encoded, - } + CharSearcher::new(haystack, self) } #[inline] - fn is_contained_in(self, haystack: &str) -> bool { - if (self as u32) < 128 { - haystack.as_bytes().contains(&(self as u8)) - } else { - let mut buffer = [0u8; 4]; - self.encode_utf8(&mut buffer).is_contained_in(haystack) - } + fn is_contained_in(self, haystack: &'a str) -> bool { + self.encode_utf8(&mut [0u8; 4]).is_contained_in(haystack) } #[inline] @@ -321,24 +162,28 @@ impl<'a> Pattern<&'a str> for char { } #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { - self.encode_utf8(&mut [0u8; 4]).strip_prefix_of(haystack) + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { + self.strip_prefix_of(str_bytes::Bytes::from_str(haystack)).map(|bytes| { + // SAFETY: Bytes were created from &str and Bytes never splits + // inside of UTF-8 bytes sequences thus `bytes` is still valid + // UTF-8. + unsafe { super::from_utf8_unchecked(bytes.as_bytes()) } + }) } #[inline] - fn is_suffix_of(self, haystack: &'a str) -> bool - where - Self::Searcher: ReverseSearcher<&'a str>, - { + fn is_suffix_of(self, haystack: &'a str) -> bool { self.encode_utf8(&mut [0u8; 4]).is_suffix_of(haystack) } #[inline] - fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> - where - Self::Searcher: ReverseSearcher<&'a str>, - { - self.encode_utf8(&mut [0u8; 4]).strip_suffix_of(haystack) + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { + self.strip_suffix_of(str_bytes::Bytes::from_str(haystack)).map(|bytes| { + // SAFETY: Bytes were created from &str and Bytes never splits + // inside of UTF-8 bytes sequences thus `bytes` is still valid + // UTF-8. + unsafe { super::from_utf8_unchecked(bytes.as_bytes()) } + }) } #[inline] @@ -753,19 +598,13 @@ impl<'a, 'b> Pattern<&'a str> for &'b str { /// Checks whether the pattern matches at the back of the haystack. #[inline] - fn is_suffix_of(self, haystack: &'a str) -> bool - where - Self::Searcher: ReverseSearcher<&'a str>, - { + fn is_suffix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().ends_with(self.as_bytes()) } /// Removes the pattern from the back of haystack, if it matches. #[inline] - fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> - where - Self::Searcher: ReverseSearcher<&'a str>, - { + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { if self.is_suffix_of(haystack) { let i = haystack.len() - self.len(); // SAFETY: suffix was just verified to exist. @@ -787,633 +626,51 @@ impl<'a, 'b> Pattern<&'a str> for &'b str { #[derive(Clone, Debug)] /// Associated type for `<&str as Pattern<&'a str>>::Searcher`. -pub struct StrSearcher<'a, 'b> { - haystack: &'a str, - needle: &'b str, - - searcher: StrSearcherImpl, -} - -#[derive(Clone, Debug)] -enum StrSearcherImpl { - Empty(core::pattern::EmptyNeedleSearcher), - TwoWay(TwoWaySearcher), -} +pub struct StrSearcher<'a, 'b>(crate::str_bytes::StrSearcher<'a, 'b>); impl<'a, 'b> StrSearcher<'a, 'b> { fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { - let searcher = if needle.is_empty() { - StrSearcherImpl::Empty(core::pattern::EmptyNeedleSearcher::new(haystack)) - } else { - StrSearcherImpl::TwoWay(TwoWaySearcher::new(needle.as_bytes(), haystack.len())) - }; - StrSearcher { haystack, needle, searcher } - } - - fn fwd_char(haystack: &str, pos: usize) -> usize { - pos + super::utf8_char_width(haystack.as_bytes()[pos]) - } - - fn bwd_char(haystack: &str, pos: usize) -> usize { - // Note: we are guaranteed to operate on valid UTF-8 thus we will never - // need to go further than four bytes back. - let bytes = haystack.as_bytes(); - if bytes[pos - 1].is_utf8_char_boundary() { - pos - 1 - } else if bytes[pos - 2].is_utf8_char_boundary() { - pos - 2 - } else if bytes[pos - 3].is_utf8_char_boundary() { - pos - 3 - } else { - pos - 4 - } + let haystack = crate::str_bytes::Bytes::from_str(haystack); + Self(crate::str_bytes::StrSearcher::new(haystack, needle)) } } unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn haystack(&self) -> &'a str { - self.haystack + let bytes = self.0.haystack().as_bytes(); + // SAFETY: self.0.haystack() was created from a &str. + unsafe { crate::str::from_utf8_unchecked(bytes) } } #[inline] fn next(&mut self) -> SearchStep { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - searcher.next_fwd(|range| Self::fwd_char(self.haystack, range.start)) - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - // TwoWaySearcher produces valid *Match* indices that split at char boundaries - // as long as it does correct matching and that haystack and needle are - // valid UTF-8 - // *Rejects* from the algorithm can fall on any indices, but we will walk them - // manually to the next character boundary, so that they are utf-8 safe. - if searcher.position == self.haystack.len() { - return SearchStep::Done; - } - let is_long = searcher.memory == usize::MAX; - match searcher.next(self.haystack.as_bytes(), self.needle.as_bytes(), is_long) { - SearchStep::Reject(a, mut b) => { - // skip to next char boundary - while !self.haystack.is_char_boundary(b) { - b += 1; - } - searcher.position = cmp::max(b, searcher.position); - SearchStep::Reject(a, b) - } - otherwise => otherwise, - } - } - } + self.0.next() } #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - searcher - .next_fwd::(|range| Self::fwd_char(self.haystack, range.start)) - .0 - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - let is_long = searcher.memory == usize::MAX; - // write out `true` and `false` cases to encourage the compiler - // to specialize the two cases separately. - if is_long { - searcher - .next::(self.haystack.as_bytes(), self.needle.as_bytes(), true) - .0 - } else { - searcher - .next::(self.haystack.as_bytes(), self.needle.as_bytes(), false) - .0 - } - } - } + self.0.next_match() + } + + fn next_reject(&mut self) -> Option<(usize, usize)> { + self.0.next_reject() } } unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn next_back(&mut self) -> SearchStep { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - searcher.next_bwd(|range| Self::bwd_char(self.haystack, range.end)) - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - if searcher.end == 0 { - return SearchStep::Done; - } - let is_long = searcher.memory == usize::MAX; - match searcher.next_back(self.haystack.as_bytes(), self.needle.as_bytes(), is_long) - { - SearchStep::Reject(mut a, b) => { - // skip to next char boundary - while !self.haystack.is_char_boundary(a) { - a -= 1; - } - searcher.end = cmp::min(a, searcher.end); - SearchStep::Reject(a, b) - } - otherwise => otherwise, - } - } - } + self.0.next_back() } #[inline] fn next_match_back(&mut self) -> Option<(usize, usize)> { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - searcher - .next_bwd::(|range| Self::bwd_char(self.haystack, range.end)) - .0 - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - let is_long = searcher.memory == usize::MAX; - // write out `true` and `false`, like `next_match` - if is_long { - searcher - .next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - true, - ) - .0 - } else { - searcher - .next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - false, - ) - .0 - } - } - } - } -} - -/// The internal state of the two-way substring search algorithm. -#[derive(Clone, Debug)] -struct TwoWaySearcher { - // constants - /// critical factorization index - crit_pos: usize, - /// critical factorization index for reversed needle - crit_pos_back: usize, - period: usize, - /// `byteset` is an extension (not part of the two way algorithm); - /// it's a 64-bit "fingerprint" where each set bit `j` corresponds - /// to a (byte & 63) == j present in the needle. - byteset: u64, - - // variables - position: usize, - end: usize, - /// index into needle before which we have already matched - memory: usize, - /// index into needle after which we have already matched - memory_back: usize, -} - -/* - This is the Two-Way search algorithm, which was introduced in the paper: - Crochemore, M., Perrin, D., 1991, Two-way string-matching, Journal of the ACM 38(3):651-675. - - Here's some background information. - - A *word* is a string of symbols. The *length* of a word should be a familiar - notion, and here we denote it for any word x by |x|. - (We also allow for the possibility of the *empty word*, a word of length zero). - - If x is any non-empty word, then an integer p with 0 < p <= |x| is said to be a - *period* for x iff for all i with 0 <= i <= |x| - p - 1, we have x[i] == x[i+p]. - For example, both 1 and 2 are periods for the string "aa". As another example, - the only period of the string "abcd" is 4. - - We denote by period(x) the *smallest* period of x (provided that x is non-empty). - This is always well-defined since every non-empty word x has at least one period, - |x|. We sometimes call this *the period* of x. - - If u, v and x are words such that x = uv, where uv is the concatenation of u and - v, then we say that (u, v) is a *factorization* of x. - - Let (u, v) be a factorization for a word x. Then if w is a non-empty word such - that both of the following hold - - - either w is a suffix of u or u is a suffix of w - - either w is a prefix of v or v is a prefix of w - - then w is said to be a *repetition* for the factorization (u, v). - - Just to unpack this, there are four possibilities here. Let w = "abc". Then we - might have: - - - w is a suffix of u and w is a prefix of v. ex: ("lolabc", "abcde") - - w is a suffix of u and v is a prefix of w. ex: ("lolabc", "ab") - - u is a suffix of w and w is a prefix of v. ex: ("bc", "abchi") - - u is a suffix of w and v is a prefix of w. ex: ("bc", "a") - - Note that the word vu is a repetition for any factorization (u,v) of x = uv, - so every factorization has at least one repetition. - - If x is a string and (u, v) is a factorization for x, then a *local period* for - (u, v) is an integer r such that there is some word w such that |w| = r and w is - a repetition for (u, v). - - We denote by local_period(u, v) the smallest local period of (u, v). We sometimes - call this *the local period* of (u, v). Provided that x = uv is non-empty, this - is well-defined (because each non-empty word has at least one factorization, as - noted above). - - It can be proven that the following is an equivalent definition of a local period - for a factorization (u, v): any positive integer r such that x[i] == x[i+r] for - all i such that |u| - r <= i <= |u| - 1 and such that both x[i] and x[i+r] are - defined. (i.e., i > 0 and i + r < |x|). - - Using the above reformulation, it is easy to prove that - - 1 <= local_period(u, v) <= period(uv) - - A factorization (u, v) of x such that local_period(u,v) = period(x) is called a - *critical factorization*. - - The algorithm hinges on the following theorem, which is stated without proof: - - **Critical Factorization Theorem** Any word x has at least one critical - factorization (u, v) such that |u| < period(x). - - The purpose of maximal_suffix is to find such a critical factorization. - - If the period is short, compute another factorization x = u' v' to use - for reverse search, chosen instead so that |v'| < period(x). - -*/ -impl TwoWaySearcher { - fn new(needle: &[u8], end: usize) -> TwoWaySearcher { - let (crit_pos_false, period_false) = TwoWaySearcher::maximal_suffix(needle, false); - let (crit_pos_true, period_true) = TwoWaySearcher::maximal_suffix(needle, true); - - let (crit_pos, period) = if crit_pos_false > crit_pos_true { - (crit_pos_false, period_false) - } else { - (crit_pos_true, period_true) - }; - - // A particularly readable explanation of what's going on here can be found - // in Crochemore and Rytter's book "Text Algorithms", ch 13. Specifically - // see the code for "Algorithm CP" on p. 323. - // - // What's going on is we have some critical factorization (u, v) of the - // needle, and we want to determine whether u is a suffix of - // &v[..period]. If it is, we use "Algorithm CP1". Otherwise we use - // "Algorithm CP2", which is optimized for when the period of the needle - // is large. - if needle[..crit_pos] == needle[period..period + crit_pos] { - // short period case -- the period is exact - // compute a separate critical factorization for the reversed needle - // x = u' v' where |v'| < period(x). - // - // This is sped up by the period being known already. - // Note that a case like x = "acba" may be factored exactly forwards - // (crit_pos = 1, period = 3) while being factored with approximate - // period in reverse (crit_pos = 2, period = 2). We use the given - // reverse factorization but keep the exact period. - let crit_pos_back = needle.len() - - cmp::max( - TwoWaySearcher::reverse_maximal_suffix(needle, period, false), - TwoWaySearcher::reverse_maximal_suffix(needle, period, true), - ); - - TwoWaySearcher { - crit_pos, - crit_pos_back, - period, - byteset: Self::byteset_create(&needle[..period]), - - position: 0, - end, - memory: 0, - memory_back: needle.len(), - } - } else { - // long period case -- we have an approximation to the actual period, - // and don't use memorization. - // - // Approximate the period by lower bound max(|u|, |v|) + 1. - // The critical factorization is efficient to use for both forward and - // reverse search. - - TwoWaySearcher { - crit_pos, - crit_pos_back: crit_pos, - period: cmp::max(crit_pos, needle.len() - crit_pos) + 1, - byteset: Self::byteset_create(needle), - - position: 0, - end, - memory: usize::MAX, // Dummy value to signify that the period is long - memory_back: usize::MAX, - } - } - } - - #[inline] - fn byteset_create(bytes: &[u8]) -> u64 { - bytes.iter().fold(0, |a, &b| (1 << (b & 0x3f)) | a) - } - - #[inline] - fn byteset_contains(&self, byte: u8) -> bool { - (self.byteset >> ((byte & 0x3f) as usize)) & 1 != 0 - } - - // One of the main ideas of Two-Way is that we factorize the needle into - // two halves, (u, v), and begin trying to find v in the haystack by scanning - // left to right. If v matches, we try to match u by scanning right to left. - // How far we can jump when we encounter a mismatch is all based on the fact - // that (u, v) is a critical factorization for the needle. - #[inline] - fn next(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> R { - // `next()` uses `self.position` as its cursor - let old_pos = self.position; - let needle_last = needle.len() - 1; - 'search: loop { - // Check that we have room to search in - // position + needle_last can not overflow if we assume slices - // are bounded by isize's range. - let tail_byte = match haystack.get(self.position + needle_last) { - Some(&b) => b, - None => { - self.position = haystack.len(); - return R::rejecting(old_pos, self.position).unwrap_or(R::DONE); - } - }; - - if old_pos != self.position { - if let Some(ret) = R::rejecting(old_pos, self.position) { - return ret; - } - } - - // Quickly skip by large portions unrelated to our substring - if !self.byteset_contains(tail_byte) { - self.position += needle.len(); - if !long_period { - self.memory = 0; - } - continue 'search; - } - - // See if the right part of the needle matches - let start = - if long_period { self.crit_pos } else { cmp::max(self.crit_pos, self.memory) }; - for i in start..needle.len() { - // SAFETY: on every iteration of `'search`, the `haystack.get(self.position + needle_last)` - // check returned `Some`, so `self.position + needle_last < haystack.len()`. - // Since `i < needle.len()` implies `i <= needle_last`, we have - // `self.position + i < haystack.len()`. - // Every path that mutates `self.position` below either returns or re-enters `'search`, - // which re-runs the check before reaching the loop again. - if needle[i] != unsafe { *haystack.get_unchecked(self.position + i) } { - self.position += i - self.crit_pos + 1; - if !long_period { - self.memory = 0; - } - continue 'search; - } - } - - // See if the left part of the needle matches - let start = if long_period { 0 } else { self.memory }; - for i in (start..self.crit_pos).rev() { - // SAFETY: on every iteration of `'search`, the `haystack.get(self.position + needle_last)` - // check returned `Some`, so `self.position + needle_last < haystack.len()`. - // Since `i < self.crit_pos <= needle.len()`, we have `i <= needle_last`, and thus - // `self.position + i <= self.position + needle_last < haystack.len()`. - // Every path that mutates `self.position` below either returns or re-enters `'search`, - // which re-runs the check before reaching the loop again. - if needle[i] != unsafe { *haystack.get_unchecked(self.position + i) } { - self.position += self.period; - if !long_period { - self.memory = needle.len() - self.period; - } - continue 'search; - } - } - - // We have found a match! - let match_pos = self.position; - - // Note: add self.period instead of needle.len() to have overlapping matches - self.position += needle.len(); - if !long_period { - self.memory = 0; // set to needle.len() - self.period for overlapping matches - } - - return R::matching(match_pos, match_pos + needle.len()).unwrap(); - } - } - - // Follows the ideas in `next()`. - // - // The definitions are symmetrical, with period(x) = period(reverse(x)) - // and local_period(u, v) = local_period(reverse(v), reverse(u)), so if (u, v) - // is a critical factorization, so is (reverse(v), reverse(u)). - // - // For the reverse case we have computed a critical factorization x = u' v' - // (field `crit_pos_back`). We need |u| < period(x) for the forward case and - // thus |v'| < period(x) for the reverse. - // - // To search in reverse through the haystack, we search forward through - // a reversed haystack with a reversed needle, matching first u' and then v'. - #[inline] - fn next_back( - &mut self, - haystack: &[u8], - needle: &[u8], - long_period: bool, - ) -> R { - // `next_back()` uses `self.end` as its cursor -- so that `next()` and `next_back()` - // are independent. - let old_end = self.end; - 'search: loop { - // Check that we have room to search in - // end - needle.len() will wrap around when there is no more room, - // but due to slice length limits it can never wrap all the way back - // into the length of haystack. - let front_byte = match haystack.get(self.end.wrapping_sub(needle.len())) { - Some(&b) => b, - None => { - self.end = 0; - return R::rejecting(0, old_end).unwrap_or(R::DONE); - } - }; - - if old_end != self.end { - if let Some(ret) = R::rejecting(self.end, old_end) { - return ret; - } - } - - // Quickly skip by large portions unrelated to our substring - if !self.byteset_contains(front_byte) { - self.end -= needle.len(); - if !long_period { - self.memory_back = needle.len(); - } - continue 'search; - } - - // See if the left part of the needle matches - let crit = if long_period { - self.crit_pos_back - } else { - cmp::min(self.crit_pos_back, self.memory_back) - }; - for i in (0..crit).rev() { - // SAFETY: On every iteration of `'search`, `haystack.get(self.end.wrapping_sub(needle.len()))` - // returned `Some`, so `self.end >= needle.len()` and `self.end - needle.len() < haystack.len()`. - // Since `self.end <= haystack.len()` and `i < needle.len()`, we have - // `self.end - needle.len() + i < self.end <= haystack.len()`, so - // `haystack.get_unchecked(self.end - needle.len() + i)` is safe. - // - The path that mutates `self.end` either re-enters `'search`, which re-runs the checks - // before reaching this loop again, or returns on match, so the invariant holds. - if needle[i] != unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } { - self.end -= self.crit_pos_back - i; - if !long_period { - self.memory_back = needle.len(); - } - continue 'search; - } - } - - // See if the right part of the needle matches - let needle_end = if long_period { needle.len() } else { self.memory_back }; - for i in self.crit_pos_back..needle_end { - // SAFETY: The same `self.end - needle.len() + i < haystack.len()` argument as the - // left-part loop applies: the `haystack.get(self.end.wrapping_sub(needle.len()))` - // check at the top of `'search` established the bound for this iteration, and - // every mutation of `self.end` is followed by `continue 'search` (which re-runs - // the check) or a `return` (which exits before any further unsafe access). - if needle[i] != unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } { - self.end -= self.period; - if !long_period { - self.memory_back = self.period; - } - continue 'search; - } - } - - // We have found a match! - let match_pos = self.end - needle.len(); - // Note: sub self.period instead of needle.len() to have overlapping matches - self.end -= needle.len(); - if !long_period { - self.memory_back = needle.len(); - } - - return R::matching(match_pos, match_pos + needle.len()).unwrap(); - } - } - - // Compute the maximal suffix of `arr`. - // - // The maximal suffix is a possible critical factorization (u, v) of `arr`. - // - // Returns (`i`, `p`) where `i` is the starting index of v and `p` is the - // period of v. - // - // `order_greater` determines if lexical order is `<` or `>`. Both - // orders must be computed -- the ordering with the largest `i` gives - // a critical factorization. - // - // For long period cases, the resulting period is not exact (it is too short). - #[inline] - fn maximal_suffix(arr: &[u8], order_greater: bool) -> (usize, usize) { - let mut left = 0; // Corresponds to i in the paper - let mut right = 1; // Corresponds to j in the paper - let mut offset = 0; // Corresponds to k in the paper, but starting at 0 - // to match 0-based indexing. - let mut period = 1; // Corresponds to p in the paper - - while let Some(&a) = arr.get(right + offset) { - // `left` will be inbounds when `right` is. - let b = arr[left + offset]; - if (a < b && !order_greater) || (a > b && order_greater) { - // Suffix is smaller, period is entire prefix so far. - right += offset + 1; - offset = 0; - period = right - left; - } else if a == b { - // Advance through repetition of the current period. - if offset + 1 == period { - right += offset + 1; - offset = 0; - } else { - offset += 1; - } - } else { - // Suffix is larger, start over from current location. - left = right; - right += 1; - offset = 0; - period = 1; - } - } - (left, period) + self.0.next_match_back() } - // Compute the maximal suffix of the reverse of `arr`. - // - // The maximal suffix is a possible critical factorization (u', v') of `arr`. - // - // Returns `i` where `i` is the starting index of v', from the back; - // returns immediately when a period of `known_period` is reached. - // - // `order_greater` determines if lexical order is `<` or `>`. Both - // orders must be computed -- the ordering with the largest `i` gives - // a critical factorization. - // - // For long period cases, the resulting period is not exact (it is too short). - fn reverse_maximal_suffix(arr: &[u8], known_period: usize, order_greater: bool) -> usize { - let mut left = 0; // Corresponds to i in the paper - let mut right = 1; // Corresponds to j in the paper - let mut offset = 0; // Corresponds to k in the paper, but starting at 0 - // to match 0-based indexing. - let mut period = 1; // Corresponds to p in the paper - let n = arr.len(); - - while right + offset < n { - let a = arr[n - (1 + right + offset)]; - let b = arr[n - (1 + left + offset)]; - if (a < b && !order_greater) || (a > b && order_greater) { - // Suffix is smaller, period is entire prefix so far. - right += offset + 1; - offset = 0; - period = right - left; - } else if a == b { - // Advance through repetition of the current period. - if offset + 1 == period { - right += offset + 1; - offset = 0; - } else { - offset += 1; - } - } else { - // Suffix is larger, start over from current location. - left = right; - right += 1; - offset = 0; - period = 1; - } - if period == known_period { - break; - } - } - debug_assert!(period <= known_period); - left + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + self.0.next_reject_back() } } diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs new file mode 100644 index 0000000000000..ceeed5190f39b --- /dev/null +++ b/library/core/src/str_bytes.rs @@ -0,0 +1,1412 @@ +//! Module provides pattern matching features for string-like bytes slice. +//! +//! A ‘string-like bytes slice’ means that types and functions here try to +//! interpret bytes slices as well-formed WTF-8 but don’t assume it is and treat +//! bytes in invalid portions of the slices as characters for the purpose of +//! deciding where character boundaries lie. This can be demonstrated by how +//! empty pattern is matched (since empty patterns match character boundaries): +//! +//! ``` +//! #![feature(pattern, str_internals)] +//! # #![allow(internal_features)] +//! use core::pattern::{Pattern, Searcher}; +//! use core::str_bytes::Bytes; +//! +//! let data = ["Żółw".as_bytes(), &b"\xff\xff\xff"[..], "🕴".as_bytes()].concat(); +//! let mut searcher = "".into_searcher(Bytes::from_bytes(data.as_slice())); +//! let next = move || searcher.next_match().map(|(x, _)| x); +//! let boundaries = core::iter::from_fn(next).collect::>(); +//! assert_eq!(&[0, 2, 4, 6, 7, 8, 9, 10, 14][..], &boundaries[..]); +//! ``` +#![unstable(feature = "str_internals", issue = "none")] + +use crate::mem::take; +use crate::pattern::{Haystack, MatchOnly, RejectOnly, SearchStep, Searcher}; +use crate::str::{try_next_code_point, try_next_code_point_reverse}; +use crate::{cmp, ops, pattern}; + +type OptRange = Option<(usize, usize)>; +type Range = ops::Range; + +//////////////////////////////////////////////////////////////////////////////// +// Bytes wrapper +//////////////////////////////////////////////////////////////////////////////// + +/// A reference to a string-like bytes slice. +/// +/// ‘String-like’ refers to the fact that parts of the data are valid WTF-8 and +/// when we split the slice we don’t want to split well-formed WTF-8 bytes +/// sequences. This is in a sense a generalisation of a `&str` which allows +/// portions of the buffer to be ill-formed while preserving correctness of +/// existing well-formed parts. +#[derive(Copy, Clone, Debug)] +pub struct Bytes<'a>(&'a [u8]); + +impl<'a> Bytes<'a> { + /// Returns a byte slice of this `Bytes`'s contents. + pub fn as_bytes(self) -> &'a [u8] { + self.0 + } + + /// Returns the length of this `Bytes`, in bytes, not chars or graphemes. + pub fn len(self) -> usize { + self.0.len() + } + + /// Returns `true` if this `Bytes` has a length of zero, and `false` otherwise. + pub fn is_empty(self) -> bool { + self.0.is_empty() + } + + /// Adjusts range’s start position forward so it points at a potential valid + /// WTF-8 byte sequence. + /// + /// `range` represents a possibly invalid range within the bytes; + /// furthermore, `range.start` must be non-zero. This method returns a new + /// start index which is a valid split position. If `range` is already + /// a valid, the method simply returns `range.start`. + /// + /// When dealing with ill-formed WTF-8 sequences, this is not guaranteed to + /// advance position byte at a time. If you need to be able to advance + /// position byte at a time use `advance_range_start` instead. + fn adjust_position_fwd(self, range: Range) -> usize { + range.start + + self.as_bytes()[range.clone()] + .iter() + .take_while(|chr| !chr.is_utf8_char_boundary()) + .count() + } + + /// Adjusts position backward so that it points at the closest potential + /// valid WTF-8 sequence. + /// + /// `range` represents a possibly invalid range within the bytes, + /// furthermore `range.end` must be less that bytes’ length. This method + /// returns a new end index which is a valid split position. If `range` is + /// already a valid, the method simply returns `range.end`. + /// + /// When dealing with ill-formed WTF-8 sequences, this is not guaranteed to + /// advance position byte at a time. If you need to be able to advance + /// position character at a time use `advance_range_end` instead. + fn adjust_position_bwd(self, range: Range) -> usize { + let shift = self.as_bytes()[range.start..range.end + 1] + .iter() + .rev() + .take_while(|chr| !chr.is_utf8_char_boundary()) + .count(); + range.end.saturating_sub(shift).max(range.start) + } + + /// Given a valid range update it’s start so it falls on the next character + /// boundary. + /// + /// `range` must be non-empty. If it starts with a valid WTF-8 sequence, + /// this method returns position pass that sequence. Otherwise, it returns + /// `range.start + 1`. In other words, well-formed WTF-8 bytes sequence are + /// skipped in one go while ill-formed sequences are skipped byte-by-byte. + fn advance_range_start(self, range: Range) -> usize { + assert!(!range.is_empty()); + match try_next_code_point(&self.0[range.clone()]) { + Some((_, len)) => range.start + len, + None => range.end.min(range.start + 1), + } + } + + /// Given a valid range update it’s end so it falls on the previous + /// character boundary. + /// + /// `range` must be non-empty. If it ends with a valid WTF-8 sequence, this + /// method returns position of the start of that sequence. Otherwise, it + /// returns `range.end - 1`. In other words, well-formed WTF-8 bytes + /// sequence are skipped in one go while ill-formed sequences are skipped + /// byte-by-byte. + fn advance_range_end(self, range: Range) -> usize { + assert!(!range.is_empty()); + match try_next_code_point_reverse(&self.0[range.clone()]) { + Some((_, len)) => range.end - len, + None => range.end - 1, + } + } + + /// Returns valid UTF-8 character at the front of the slice. + /// + /// If slice doesn’t start with a valid UTF-8 sequence, returns `None`. + /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. + /// WTF-8 sequences which encode surrogates are considered invalid. + fn get_first_code_point(self) -> Option<(char, usize)> { + try_next_code_point(&self.0) + } + + /// Returns valid UTF-8 character at the end of the slice. + /// + /// If slice doesn’t end with a valid UTF-8 sequence, returns `None`. + /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. + /// WTF-8 sequences which encode surrogates are considered invalid. + fn get_last_code_point(&self) -> Option<(char, usize)> { + try_next_code_point_reverse(&self.0) + } + + /// Looks for the next UTF-8-encoded character in the slice. + /// + /// WTF-8 sequences which encode surrogates are considered invalid. + /// + /// Returns position of the match, decoded character and UTF-8 length of + /// that character. + fn find_code_point_fwd(self, range: Range) -> Option<(usize, char, usize)> { + let bytes = &self.as_bytes()[range.clone()]; + (0..bytes.len()) + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((range.start + pos, chr, len)) + }) + .next() + } + + /// Looks backwards for the next UTF-8 encoded character in the slice. + /// + /// WTF-8 sequences which encode surrogates are considered invalid. + /// + /// Returns position of the match, decoded character and UTF-8 length of + /// that character. + fn find_code_point_bwd(&self, range: Range) -> Option<(usize, char, usize)> { + let bytes = &self.as_bytes()[range.clone()]; + (0..bytes.len()) + .rev() + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((range.start + pos, chr, len)) + }) + .next() + } +} + +impl<'a> Bytes<'a> { + /// Wraps `&[u8]` into `Bytes`. + #[inline] + pub fn from_bytes(val: &'a [u8]) -> Self { + Self(val) + } + + /// Wraps `&str` into `Bytes`. + #[inline] + pub fn from_str(val: &'a str) -> Self { + Self(val.as_bytes()) + } +} + +trait SearchResult: crate::pattern::SearchResult { + /// Adjusts reject’s start position backwards to make sure it doesn’t fall + /// within well-formed WTF-8 sequence. + /// + /// Doesn’t move the start position past `begin`. If position was adjusted, + /// updates `*out` as well. + fn adjust_reject_start_bwd(self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self; + + /// Adjusts reject’s end position forwards to make sure it doesn’t fall + /// within well-formed WTF-8 sequence. + /// + /// Doesn’t move the end position past `len`. If position was adjusted, + /// updates `*out` as well. + fn adjust_reject_end_fwd(self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self; +} + +impl SearchResult for SearchStep { + fn adjust_reject_start_bwd(mut self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self { + if let SearchStep::Reject(ref mut start, _) = self { + *start = bytes.adjust_position_bwd(begin..*start); + *out = *start; + } + self + } + fn adjust_reject_end_fwd(mut self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self { + if let SearchStep::Reject(_, ref mut end) = self { + *end = bytes.adjust_position_fwd(*end..len); + *out = *end; + } + self + } +} + +impl SearchResult for MatchOnly { + fn adjust_reject_start_bwd(self, _bytes: Bytes<'_>, _begin: usize, _out: &mut usize) -> Self { + self + } + fn adjust_reject_end_fwd(self, _bytes: Bytes<'_>, _end: usize, _out: &mut usize) -> Self { + self + } +} + +impl SearchResult for RejectOnly { + fn adjust_reject_start_bwd(mut self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self { + if let RejectOnly(Some((ref mut start, _))) = self { + *start = bytes.adjust_position_bwd(begin..*start); + *out = *start; + } + self + } + fn adjust_reject_end_fwd(mut self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self { + if let RejectOnly(Some((_, ref mut end))) = self { + *end = bytes.adjust_position_fwd(*end..len); + *out = *end; + } + self + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl for Haystack +//////////////////////////////////////////////////////////////////////////////// + +impl Haystack for Bytes<'_> { + fn cursor_at_front(self) -> usize { + 0 + } + fn cursor_at_back(self) -> usize { + self.0.len() + } + fn is_empty(self) -> bool { + self.0.is_empty() + } + unsafe fn get_unchecked(self, range: Range) -> Self { + Self(if cfg!(debug_assertions) { + self.0.get(range).unwrap() + } else { + // SAFETY: Caller promises cursor is a valid split position. + unsafe { self.0.get_unchecked(range) } + }) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for char +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs> pattern::Pattern> for char { + type Searcher = CharSearcher<'hs>; + + fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_contained_in(self, haystack: Bytes<'hs>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_contained_in(haystack) + } + + fn is_prefix_of(self, haystack: Bytes<'hs>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_prefix_of(haystack) + } + fn strip_prefix_of(self, haystack: Bytes<'hs>) -> Option> { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).strip_prefix_of(haystack) + } + + fn is_suffix_of(self, haystack: Bytes<'hs>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_suffix_of(haystack) + } + fn strip_suffix_of(self, haystack: Bytes<'hs>) -> Option> { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).strip_suffix_of(haystack) + } +} + +/// Like `chr.encode_utf8(&mut buf)` but casts result to `&str`. +/// +/// This is useful because we have Pattern impl for &str but not for &mut str. +fn encode_utf8(chr: char, buf: &mut [u8; 4]) -> &str { + chr.encode_utf8(buf) +} + +/// Searcher looking for a single character in the haystack. +#[derive(Clone, Debug)] +pub struct CharSearcher<'hs> { + haystack: Bytes<'hs>, + state: CharSearcherState, +} + +#[derive(Clone, Debug)] +struct CharSearcherState { + /// Not yet processed range of the haystack. + range: crate::ops::Range, + /// Needle the searcher is looking for within the haystack. + needle: CharBuffer, + /// If `true` and `range` is non-empty, `haystack[range]` starts with the + /// needle. + is_match_fwd: bool, + /// If `true` and `range` is non-empty, `haystack[range]` ends with the + /// needle. + is_match_bwd: bool, +} + +impl<'hs> CharSearcher<'hs> { + /// Creates a new searcher for the given character. + #[inline] + pub fn new(haystack: Bytes<'hs>, chr: char) -> Self { + Self { haystack, state: CharSearcherState::new(haystack.len(), chr) } + } +} + +unsafe impl<'hs> pattern::Searcher> for CharSearcher<'hs> { + fn haystack(&self) -> Bytes<'hs> { + self.haystack + } + + fn next(&mut self) -> SearchStep { + self.state.next_fwd(self.haystack) + } + fn next_match(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } + fn next_reject(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } +} + +unsafe impl<'hs> pattern::ReverseSearcher> for CharSearcher<'hs> { + fn next_back(&mut self) -> SearchStep { + self.state.next_bwd(self.haystack) + } + fn next_match_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } + fn next_reject_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } +} + +impl<'hs> pattern::DoubleEndedSearcher> for CharSearcher<'hs> {} + +impl CharSearcherState { + fn new(haystack_len: usize, chr: char) -> Self { + Self { + range: 0..haystack_len, + needle: CharBuffer::new(chr), + is_match_fwd: false, + is_match_bwd: false, + } + } + + fn find_match_fwd(&mut self, haystack: Bytes<'_>) -> OptRange { + let start = if take(&mut self.is_match_fwd) { + (!self.range.is_empty()).then_some(self.range.start) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + // SAFETY: self.needle encodes a single character. + unsafe { naive::find_match_fwd(bytes.as_bytes(), self.needle.as_str()) } + .map(|pos| pos + self.range.start) + }?; + Some((start, start + self.needle.len())) + } + + fn next_reject_fwd(&mut self, haystack: Bytes<'_>) -> OptRange { + if take(&mut self.is_match_fwd) { + if self.range.is_empty() { + return None; + } + self.range.start += self.needle.len() + } + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + if let Some(pos) = naive::find_reject_fwd(bytes.as_bytes(), self.needle.as_str()) { + let pos = pos + self.range.start; + let end = haystack.advance_range_start(pos..self.range.end); + self.range.start = end; + Some((pos, end)) + } else { + self.range.start = self.range.end; + None + } + } + + fn next_fwd(&mut self, haystack: Bytes<'_>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_fwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_fwd(haystack) { + if self.range.start < start { + if let Some(res) = R::rejecting(self.range.start, start) { + self.range.start = start; + self.is_match_fwd = true; + return res; + } + } + self.range.start = end; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let start = self.range.start; + self.range.start = self.range.end; + R::rejecting(start, self.range.end).unwrap_or(R::DONE) + } + } + + fn find_match_bwd(&mut self, haystack: Bytes<'_>) -> OptRange { + let start = if take(&mut self.is_match_bwd) { + (!self.range.is_empty()).then(|| self.range.end - self.needle.len()) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + // SAFETY: self.needle encodes a single character. + unsafe { naive::find_match_bwd(bytes.as_bytes(), self.needle.as_str()) } + .map(|pos| pos + self.range.start) + }?; + Some((start, start + self.needle.len())) + } + + fn next_reject_bwd(&mut self, haystack: Bytes<'_>) -> OptRange { + if take(&mut self.is_match_bwd) { + if self.range.is_empty() { + return None; + } + self.range.end -= self.needle.len(); + } + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + if let Some(end) = naive::find_reject_bwd(bytes.as_bytes(), self.needle.as_str()) { + let end = end + self.range.start; + let start = haystack.advance_range_end(self.range.start..end); + self.range.end = start; + Some((start, end)) + } else { + self.range.end = self.range.start; + None + } + } + + fn next_bwd(&mut self, haystack: Bytes<'_>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_bwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_bwd(haystack) { + if end < self.range.end { + if let Some(res) = R::rejecting(end, self.range.end) { + self.range.end = end; + self.is_match_bwd = true; + return res; + } + } + self.range.end = start; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let end = self.range.end; + self.range.end = self.range.start; + R::rejecting(self.range.start, end).unwrap_or(R::DONE) + } + } +} + +#[derive(Clone, Debug)] +struct CharBuffer([u8; 4], crate::num::NonZeroU8); + +impl CharBuffer { + fn new(chr: char) -> Self { + let mut buf = [0; 4]; + let len = chr.encode_utf8(&mut buf).len(); + // SAFETY: `len` is length of a single character UTF-8 sequence. + let len = unsafe { crate::num::NonZeroU8::new_unchecked(len as u8) }; + Self(buf, len) + } + + fn len(&self) -> usize { + usize::from(self.1.get()) + } + + fn as_str(&self) -> &str { + // SAFETY: `self.0` is UTF-8 encoding of a single character and `self.1` + // is its length. See `new` constructor. + unsafe { crate::str::from_utf8_unchecked(self.0.get_unchecked(..self.len())) } + } +} + +mod naive { + use crate::slice::memchr; + + /// Looks forwards for the next position of needle within haystack. + /// + /// Safety: `needle` must consist of a single character. + pub(super) unsafe fn find_match_fwd(haystack: &[u8], needle: &str) -> Option { + debug_assert!(!needle.is_empty()); + // SAFETY: Caller promises needle is non-empty. + let (&last_byte, head) = unsafe { needle.as_bytes().split_last().unwrap_unchecked() }; + let mut start = 0; + while haystack.len() - start > head.len() { + // SAFETY: + // 1. `start` is initialised to `self.start` and only ever increased + // thus `self.start ≤ start`. + // 2. We've checked `start + head.len() < haystack.len()`. + let bytes = unsafe { haystack.get_unchecked(start + head.len()..) }; + if let Some(index) = memchr::memchr(last_byte, bytes) { + // `start + index + head.len()` is the index of the last byte + // thus `start + index` is the index of the first byte. + let pos = start + index; + // SAFETY: Since we’ve started our search with head.len() + // offset, we know we have at least head.len() bytes in buffer. + if unsafe { haystack.get_unchecked(pos..pos + head.len()) } == head { + return Some(pos); + } + start += index + 1; + } else { + break; + } + } + None + } + + /// Looks backwards for the next position of needle within haystack. + /// + /// Safety: `needle` must consist of a single character. + pub(super) unsafe fn find_match_bwd(haystack: &[u8], needle: &str) -> Option { + // SAFETY: Caller promises needle is non-empty. + let (&first_byte, tail) = unsafe { needle.as_bytes().split_first().unwrap_unchecked() }; + let mut end = haystack.len(); + while end > tail.len() { + // SAFETY: + // 1. `end` is initialised to `haystack.len()` and only ever + // decreased thus `end ≤ haystack.len()`. + // 2. We've checked `end > tail.len()`. + let bytes = unsafe { haystack.get_unchecked(..end - tail.len()) }; + if let Some(pos) = memchr::memrchr(first_byte, bytes) { + // SAFETY: Since we’ve stopped our search with tail.len() + // offset, we know we have at least tail.len() bytes in buffer + // after position of the byte we’ve found. + if unsafe { haystack.get_unchecked(pos + 1..pos + 1 + tail.len()) } == tail { + return Some(pos); + } + end = pos; + } else { + break; + } + } + None + } + + /// Looks forwards for the next position where needle stops matching. + /// + /// Returns start of the next reject or `None` if there is no reject. + pub(super) fn find_reject_fwd(haystack: &[u8], needle: &str) -> Option { + let count = + haystack.chunks(needle.len()).take_while(|&slice| slice == needle.as_bytes()).count(); + let start = count * needle.len(); + (start < haystack.len()).then_some(start) + } + + /// Looks backwards for the next position where needle stops matching. + /// + /// Returns end of the next reject or `None` if there is no reject. + pub(super) fn find_reject_bwd(haystack: &[u8], needle: &str) -> Option { + debug_assert!(!needle.is_empty()); + let count = + haystack.rchunks(needle.len()).take_while(|&slice| slice == needle.as_bytes()).count(); + let end = haystack.len() - count * needle.len(); + (end > 0).then_some(end) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for FnMut(char) and FnMut(Result) +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs, F: FnMut(char) -> bool> pattern::Pattern> for F { + type Searcher = PredicateSearcher<'hs, F>; + + fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_prefix_of(mut self, haystack: Bytes<'hs>) -> bool { + haystack.get_first_code_point().map_or(false, |(chr, _)| self(chr)) + } + fn strip_prefix_of(mut self, haystack: Bytes<'hs>) -> Option> { + let (chr, len) = haystack.get_first_code_point()?; + // SAFETY: We’ve just checked slice starts with len-byte long + // well-formed sequence. + self(chr).then(|| unsafe { haystack.get_unchecked(len..haystack.len()) }) + } + + fn is_suffix_of(mut self, haystack: Bytes<'hs>) -> bool { + haystack.get_last_code_point().map_or(false, |(chr, _)| self(chr)) + } + fn strip_suffix_of(mut self, haystack: Bytes<'hs>) -> Option> { + let (chr, len) = haystack.get_last_code_point()?; + let len = haystack.len() - len; + // SAFETY: We’ve just checked slice ends with len-byte long well-formed + // sequence. + self(chr).then(|| unsafe { haystack.get_unchecked(0..len) }) + } +} + +/// Searcher looking for characters matching a predicate. +#[derive(Clone, Debug)] +pub struct PredicateSearcher<'hs, F> { + haystack: Bytes<'hs>, + pred: F, + start: usize, + end: usize, + fwd_match_len: u8, + bwd_match_len: u8, +} + +impl<'hs, F> PredicateSearcher<'hs, F> { + /// Creates a new searcher for the given predicate. + #[inline] + fn new(haystack: Bytes<'hs>, pred: F) -> Self { + Self { haystack, pred, start: 0, end: haystack.len(), fwd_match_len: 0, bwd_match_len: 0 } + } +} + +impl<'hs, F: FnMut(char) -> bool> PredicateSearcher<'hs, F> { + fn find_match_fwd(&mut self) -> Option<(usize, usize)> { + let mut start = self.start; + while start < self.end { + let (idx, chr, len) = self.haystack.find_code_point_fwd(start..self.end)?; + if (self.pred)(chr) { + return Some((idx, len)); + } + start = idx + len; + } + None + } + + fn find_match_bwd(&mut self) -> Option<(usize, usize)> { + let mut end = self.end; + while self.start < end { + let (idx, chr, len) = self.haystack.find_code_point_bwd(self.start..end)?; + if (self.pred)(chr) { + return Some((idx, len)); + } + end = idx; + } + None + } + + fn next_fwd(&mut self) -> R { + while self.start < self.end { + if self.fwd_match_len == 0 { + let (pos, len) = self.find_match_fwd().unwrap_or((self.end, 0)); + self.fwd_match_len = len as u8; + if pos != self.start { + let start = self.start; + self.start = pos; + if let Some(ret) = R::rejecting(start, pos) { + return ret; + } else if pos >= self.end { + break; + } + } + } + + let pos = self.start; + self.start += usize::from(take(&mut self.fwd_match_len)); + if let Some(ret) = R::matching(pos, self.start) { + return ret; + } + } + R::DONE + } + + fn next_bwd(&mut self) -> R { + while self.start < self.end { + if self.bwd_match_len == 0 { + let (pos, len) = self.find_match_bwd().unwrap_or((self.start, 0)); + self.bwd_match_len = len as u8; + let pos = pos + len; + let end = self.end; + if pos != self.end { + self.end = pos; + if let Some(ret) = R::rejecting(pos, end) { + return ret; + } else if self.start >= self.end { + break; + } + } + } + + let end = self.end; + self.end -= usize::from(take(&mut self.bwd_match_len)); + if let Some(ret) = R::matching(self.end, end) { + return ret; + } + } + R::DONE + } +} + +unsafe impl<'hs, F: FnMut(char) -> bool> Searcher> for PredicateSearcher<'hs, F> { + fn haystack(&self) -> Bytes<'hs> { + self.haystack + } + fn next(&mut self) -> SearchStep { + self.next_fwd() + } + fn next_match(&mut self) -> OptRange { + self.next_fwd::().0 + } + fn next_reject(&mut self) -> OptRange { + self.next_fwd::().0 + } +} + +unsafe impl<'hs, F: FnMut(char) -> bool> pattern::ReverseSearcher> + for PredicateSearcher<'hs, F> +{ + fn next_back(&mut self) -> SearchStep { + self.next_bwd() + } + fn next_match_back(&mut self) -> OptRange { + self.next_bwd::().0 + } + fn next_reject_back(&mut self) -> OptRange { + self.next_bwd::().0 + } +} + +impl<'hs, F: FnMut(char) -> bool> pattern::DoubleEndedSearcher> + for PredicateSearcher<'hs, F> +{ +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for &str +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs, 'p> pattern::Pattern> for &'p str { + type Searcher = StrSearcher<'hs, 'p>; + + fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_prefix_of(self, haystack: Bytes<'hs>) -> bool { + haystack.as_bytes().starts_with(self.as_bytes()) + } + fn strip_prefix_of(self, haystack: Bytes<'hs>) -> Option> { + haystack.as_bytes().strip_prefix(self.as_bytes()).map(Bytes) + } + + fn is_suffix_of(self, haystack: Bytes<'hs>) -> bool { + haystack.as_bytes().ends_with(self.as_bytes()) + } + fn strip_suffix_of(self, haystack: Bytes<'hs>) -> Option> { + haystack.as_bytes().strip_suffix(self.as_bytes()).map(Bytes) + } +} + +/// Searcher looking for a substring in the haystack. +#[derive(Clone, Debug)] +pub struct StrSearcher<'hs, 'p> { + haystack: Bytes<'hs>, + state: StrSearcherInner<'p>, +} + +impl<'hs, 'p> StrSearcher<'hs, 'p> { + /// Creates a new searcher for the given substring. + pub fn new(haystack: Bytes<'hs>, needle: &'p str) -> Self { + let state = StrSearcherInner::new(haystack, needle); + Self { haystack, state } + } +} + +unsafe impl<'hs, 'p> Searcher> for StrSearcher<'hs, 'p> { + fn haystack(&self) -> Bytes<'hs> { + self.haystack + } + fn next(&mut self) -> SearchStep { + self.state.next_fwd(self.haystack) + } + fn next_match(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } + fn next_reject(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } +} + +unsafe impl<'hs, 'p> pattern::ReverseSearcher> for StrSearcher<'hs, 'p> { + fn next_back(&mut self) -> SearchStep { + self.state.next_bwd(self.haystack) + } + fn next_match_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } + fn next_reject_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } +} + +#[derive(Clone, Debug)] +enum StrSearcherInner<'p> { + Empty(EmptySearcherState), + Char(CharSearcherState), + Str(StrSearcherState<'p>), +} + +impl<'p> StrSearcherInner<'p> { + fn new(haystack: Bytes<'_>, needle: &'p str) -> Self { + let mut chars = needle.chars(); + let chr = match chars.next() { + Some(chr) => chr, + None => return Self::Empty(EmptySearcherState::new(haystack)), + }; + if chars.next().is_none() { + Self::Char(CharSearcherState::new(haystack.len(), chr)) + } else { + Self::Str(StrSearcherState::new(haystack, needle)) + } + } + + fn next_fwd(&mut self, haystack: Bytes<'_>) -> R { + match self { + Self::Empty(state) => state.next_fwd::(haystack), + Self::Char(state) => state.next_fwd::(haystack), + Self::Str(state) => state.next_fwd::(haystack), + } + } + + fn next_bwd(&mut self, haystack: Bytes<'_>) -> R { + match self { + Self::Empty(state) => state.next_bwd::(haystack), + Self::Char(state) => state.next_bwd::(haystack), + Self::Str(state) => state.next_bwd::(haystack), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Empty needle searching +//////////////////////////////////////////////////////////////////////////////// + +/// Empty needle rejects every character and matches every character boundary. +/// +/// A character is either a well-formed WTF-8 bytes sequence or a single byte +/// whichever is longer. +#[derive(Clone, Debug)] +struct EmptySearcherState(pattern::EmptyNeedleSearcher); + +impl EmptySearcherState { + fn new(haystack: Bytes<'_>) -> Self { + Self(pattern::EmptyNeedleSearcher::new(haystack)) + } + + fn next_fwd(&mut self, bytes: Bytes<'_>) -> R { + self.0.next_fwd(|range| bytes.advance_range_start(range)) + } + + fn next_bwd(&mut self, bytes: Bytes<'_>) -> R { + self.0.next_bwd(|range| bytes.advance_range_end(range)) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Full substring search +//////////////////////////////////////////////////////////////////////////////// + +/// A substring search. +#[derive(Clone, Debug)] +struct StrSearcherState<'p> { + needle: &'p str, + searcher: TwoWaySearcher, +} + +impl<'p> StrSearcherState<'p> { + fn new(haystack: Bytes<'_>, needle: &'p str) -> Self { + let searcher = TwoWaySearcher::new(haystack.len(), needle.as_bytes()); + Self { needle, searcher } + } + + fn next_fwd(&mut self, bytes: Bytes<'_>) -> R { + if self.searcher.position >= bytes.len() { + return R::DONE; + } + if self.searcher.memory == usize::MAX { + self.searcher.next_fwd::(bytes.0, self.needle.as_bytes(), true) + } else { + self.searcher.next_fwd::(bytes.0, self.needle.as_bytes(), false) + } + .adjust_reject_end_fwd(bytes, bytes.len(), &mut self.searcher.position) + } + + fn next_bwd(&mut self, bytes: Bytes<'_>) -> R { + if self.searcher.end == 0 { + return R::DONE; + } + if self.searcher.memory == usize::MAX { + self.searcher.next_bwd::(bytes.0, self.needle.as_bytes(), true) + } else { + self.searcher.next_bwd::(bytes.0, self.needle.as_bytes(), false) + } + .adjust_reject_start_bwd(bytes, 0, &mut self.searcher.end) + } +} + +/// The internal state of the two-way substring search algorithm. +#[derive(Clone, Debug)] +struct TwoWaySearcher { + // constants + /// critical factorization index + crit_pos: usize, + /// critical factorization index for reversed needle + crit_pos_back: usize, + period: usize, + /// `byteset` is an extension (not part of the two way algorithm); + /// it's a 64-bit "fingerprint" where each set bit `j` corresponds + /// to a (byte & 63) == j present in the needle. + byteset: u64, + + // variables + position: usize, + end: usize, + /// index into needle before which we have already matched + memory: usize, + /// index into needle after which we have already matched + memory_back: usize, +} + +/* + This is the Two-Way search algorithm, which was introduced in the paper: + Crochemore, M., Perrin, D., 1991, Two-way string-matching, Journal of the ACM 38(3):651-675. + + Here's some background information. + + A *word* is a string of symbols. The *length* of a word should be a familiar + notion, and here we denote it for any word x by |x|. + (We also allow for the possibility of the *empty word*, a word of length zero). + + If x is any non-empty word, then an integer p with 0 < p <= |x| is said to be a + *period* for x iff for all i with 0 <= i <= |x| - p - 1, we have x[i] == x[i+p]. + For example, both 1 and 2 are periods for the string "aa". As another example, + the only period of the string "abcd" is 4. + + We denote by period(x) the *smallest* period of x (provided that x is non-empty). + This is always well-defined since every non-empty word x has at least one period, + |x|. We sometimes call this *the period* of x. + + If u, v and x are words such that x = uv, where uv is the concatenation of u and + v, then we say that (u, v) is a *factorization* of x. + + Let (u, v) be a factorization for a word x. Then if w is a non-empty word such + that both of the following hold + + - either w is a suffix of u or u is a suffix of w + - either w is a prefix of v or v is a prefix of w + + then w is said to be a *repetition* for the factorization (u, v). + + Just to unpack this, there are four possibilities here. Let w = "abc". Then we + might have: + + - w is a suffix of u and w is a prefix of v. ex: ("lolabc", "abcde") + - w is a suffix of u and v is a prefix of w. ex: ("lolabc", "ab") + - u is a suffix of w and w is a prefix of v. ex: ("bc", "abchi") + - u is a suffix of w and v is a prefix of w. ex: ("bc", "a") + + Note that the word vu is a repetition for any factorization (u,v) of x = uv, + so every factorization has at least one repetition. + + If x is a string and (u, v) is a factorization for x, then a *local period* for + (u, v) is an integer r such that there is some word w such that |w| = r and w is + a repetition for (u, v). + + We denote by local_period(u, v) the smallest local period of (u, v). We sometimes + call this *the local period* of (u, v). Provided that x = uv is non-empty, this + is well-defined (because each non-empty word has at least one factorization, as + noted above). + + It can be proven that the following is an equivalent definition of a local period + for a factorization (u, v): any positive integer r such that x[i] == x[i+r] for + all i such that |u| - r <= i <= |u| - 1 and such that both x[i] and x[i+r] are + defined. (i.e., i > 0 and i + r < |x|). + + Using the above reformulation, it is easy to prove that + + 1 <= local_period(u, v) <= period(uv) + + A factorization (u, v) of x such that local_period(u,v) = period(x) is called a + *critical factorization*. + + The algorithm hinges on the following theorem, which is stated without proof: + + **Critical Factorization Theorem** Any word x has at least one critical + factorization (u, v) such that |u| < period(x). + + The purpose of maximal_suffix is to find such a critical factorization. + + If the period is short, compute another factorization x = u' v' to use + for reverse search, chosen instead so that |v'| < period(x). + +*/ +impl TwoWaySearcher { + fn new(haystack_len: usize, needle: &[u8]) -> TwoWaySearcher { + let (crit_pos_false, period_false) = TwoWaySearcher::maximal_suffix(needle, false); + let (crit_pos_true, period_true) = TwoWaySearcher::maximal_suffix(needle, true); + + let (crit_pos, period) = if crit_pos_false > crit_pos_true { + (crit_pos_false, period_false) + } else { + (crit_pos_true, period_true) + }; + + // A particularly readable explanation of what's going on here can be found + // in Crochemore and Rytter's book "Text Algorithms", ch 13. Specifically + // see the code for "Algorithm CP" on p. 323. + // + // What's going on is we have some critical factorization (u, v) of the + // needle, and we want to determine whether u is a suffix of + // &v[..period]. If it is, we use "Algorithm CP1". Otherwise we use + // "Algorithm CP2", which is optimized for when the period of the needle + // is large. + if needle[..crit_pos] == needle[period..period + crit_pos] { + // short period case -- the period is exact + // compute a separate critical factorization for the reversed needle + // x = u' v' where |v'| < period(x). + // + // This is sped up by the period being known already. + // Note that a case like x = "acba" may be factored exactly forwards + // (crit_pos = 1, period = 3) while being factored with approximate + // period in reverse (crit_pos = 2, period = 2). We use the given + // reverse factorization but keep the exact period. + let crit_pos_back = needle.len() + - cmp::max( + TwoWaySearcher::reverse_maximal_suffix(needle, period, false), + TwoWaySearcher::reverse_maximal_suffix(needle, period, true), + ); + + TwoWaySearcher { + crit_pos, + crit_pos_back, + period, + byteset: Self::byteset_create(&needle[..period]), + + position: 0, + end: haystack_len, + memory: 0, + memory_back: needle.len(), + } + } else { + // long period case -- we have an approximation to the actual period, + // and don't use memorization. + // + // Approximate the period by lower bound max(|u|, |v|) + 1. + // The critical factorization is efficient to use for both forward and + // reverse search. + + TwoWaySearcher { + crit_pos, + crit_pos_back: crit_pos, + period: cmp::max(crit_pos, needle.len() - crit_pos) + 1, + byteset: Self::byteset_create(needle), + + position: 0, + end: haystack_len, + memory: usize::MAX, // Dummy value to signify that the period is long + memory_back: usize::MAX, + } + } + } + + #[inline] + fn byteset_create(bytes: &[u8]) -> u64 { + bytes.iter().fold(0, |a, &b| (1 << (b & 0x3f)) | a) + } + + #[inline] + fn byteset_contains(&self, byte: u8) -> bool { + (self.byteset >> ((byte & 0x3f) as usize)) & 1 != 0 + } + + // One of the main ideas of Two-Way is that we factorize the needle into + // two halves, (u, v), and begin trying to find v in the haystack by scanning + // left to right. If v matches, we try to match u by scanning right to left. + // How far we can jump when we encounter a mismatch is all based on the fact + // that (u, v) is a critical factorization for the needle. + #[inline] + fn next_fwd( + &mut self, + haystack: &[u8], + needle: &[u8], + long_period: bool, + ) -> R { + // `next()` uses `self.position` as its cursor + let mut old_pos = self.position; + let needle_last = needle.len() - 1; + 'search: loop { + // Check that we have room to search in + // position + needle_last can not overflow if we assume slices + // are bounded by isize's range. + let tail_byte = match haystack.get(self.position + needle_last) { + Some(&b) => b, + None => { + self.position = haystack.len(); + if old_pos == self.position { + return R::DONE; + } + return R::rejecting(old_pos, self.position).unwrap_or(R::DONE); + } + }; + + if old_pos != self.position { + if let Some(ret) = R::rejecting(old_pos, self.position) { + return ret; + } + } + + // Quickly skip by large portions unrelated to our substring + if !self.byteset_contains(tail_byte) { + self.position += needle.len(); + if !long_period { + self.memory = 0; + } + continue 'search; + } + + // See if the right part of the needle matches + let start = + if long_period { self.crit_pos } else { cmp::max(self.crit_pos, self.memory) }; + for i in start..needle.len() { + if needle[i] != haystack[self.position + i] { + self.position += i - self.crit_pos + 1; + if !long_period { + self.memory = 0; + } + continue 'search; + } + } + + // See if the left part of the needle matches + let start = if long_period { 0 } else { self.memory }; + for i in (start..self.crit_pos).rev() { + if needle[i] != haystack[self.position + i] { + self.position += self.period; + if !long_period { + self.memory = needle.len() - self.period; + } + continue 'search; + } + } + + // We have found a match! + let match_pos = self.position; + + // Note: add self.period instead of needle.len() to have overlapping matches + self.position += needle.len(); + if !long_period { + self.memory = 0; // set to needle.len() - self.period for overlapping matches + } + + if let Some(ret) = R::matching(match_pos, match_pos + needle.len()) { + return ret; + } + // `matching` returned `None`, exclude the consumed match from the pending + // reject range so rejects. + old_pos = self.position; + } + } + + // Follows the ideas in `next()`. + // + // The definitions are symmetrical, with period(x) = period(reverse(x)) + // and local_period(u, v) = local_period(reverse(v), reverse(u)), so if (u, v) + // is a critical factorization, so is (reverse(v), reverse(u)). + // + // For the reverse case we have computed a critical factorization x = u' v' + // (field `crit_pos_back`). We need |u| < period(x) for the forward case and + // thus |v'| < period(x) for the reverse. + // + // To search in reverse through the haystack, we search forward through + // a reversed haystack with a reversed needle, matching first u' and then v'. + #[inline] + fn next_bwd( + &mut self, + haystack: &[u8], + needle: &[u8], + long_period: bool, + ) -> R { + // `next_back()` uses `self.end` as its cursor -- so that `next()` and `next_back()` + // are independent. + let mut old_end = self.end; + 'search: loop { + // Check that we have room to search in + // end - needle.len() will wrap around when there is no more room, + // but due to slice length limits it can never wrap all the way back + // into the length of haystack. + let front_byte = match haystack.get(self.end.wrapping_sub(needle.len())) { + Some(&b) => b, + None => { + self.end = 0; + if old_end == 0 { + return R::DONE; + } + return R::rejecting(0, old_end).unwrap_or(R::DONE); + } + }; + + if old_end != self.end { + if let Some(ret) = R::rejecting(self.end, old_end) { + return ret; + } + } + + // Quickly skip by large portions unrelated to our substring + if !self.byteset_contains(front_byte) { + self.end -= needle.len(); + if !long_period { + self.memory_back = needle.len(); + } + continue 'search; + } + + // See if the left part of the needle matches + let crit = if long_period { + self.crit_pos_back + } else { + cmp::min(self.crit_pos_back, self.memory_back) + }; + for i in (0..crit).rev() { + if needle[i] != haystack[self.end - needle.len() + i] { + self.end -= self.crit_pos_back - i; + if !long_period { + self.memory_back = needle.len(); + } + continue 'search; + } + } + + // See if the right part of the needle matches + let needle_end = if long_period { needle.len() } else { self.memory_back }; + for i in self.crit_pos_back..needle_end { + if needle[i] != haystack[self.end - needle.len() + i] { + self.end -= self.period; + if !long_period { + self.memory_back = self.period; + } + continue 'search; + } + } + + // We have found a match! + let match_pos = self.end - needle.len(); + // Note: sub self.period instead of needle.len() to have overlapping matches + self.end -= needle.len(); + if !long_period { + self.memory_back = needle.len(); + } + + if let Some(ret) = R::matching(match_pos, match_pos + needle.len()) { + return ret; + } + // `matching` returned `None`, exclude the consumed match from the pending + // reject range so rejects. + old_end = self.end; + } + } + + // Compute the maximal suffix of `arr`. + // + // The maximal suffix is a possible critical factorization (u, v) of `arr`. + // + // Returns (`i`, `p`) where `i` is the starting index of v and `p` is the + // period of v. + // + // `order_greater` determines if lexical order is `<` or `>`. Both + // orders must be computed -- the ordering with the largest `i` gives + // a critical factorization. + // + // For long period cases, the resulting period is not exact (it is too short). + #[inline] + fn maximal_suffix(arr: &[u8], order_greater: bool) -> (usize, usize) { + let mut left = 0; // Corresponds to i in the paper + let mut right = 1; // Corresponds to j in the paper + let mut offset = 0; // Corresponds to k in the paper, but starting at 0 + // to match 0-based indexing. + let mut period = 1; // Corresponds to p in the paper + + while let Some(&a) = arr.get(right + offset) { + // `left` will be inbounds when `right` is. + let b = arr[left + offset]; + if (a < b && !order_greater) || (a > b && order_greater) { + // Suffix is smaller, period is entire prefix so far. + right += offset + 1; + offset = 0; + period = right - left; + } else if a == b { + // Advance through repetition of the current period. + if offset + 1 == period { + right += offset + 1; + offset = 0; + } else { + offset += 1; + } + } else { + // Suffix is larger, start over from current location. + left = right; + right += 1; + offset = 0; + period = 1; + } + } + (left, period) + } + + // Compute the maximal suffix of the reverse of `arr`. + // + // The maximal suffix is a possible critical factorization (u', v') of `arr`. + // + // Returns `i` where `i` is the starting index of v', from the back; + // returns immediately when a period of `known_period` is reached. + // + // `order_greater` determines if lexical order is `<` or `>`. Both + // orders must be computed -- the ordering with the largest `i` gives + // a critical factorization. + // + // For long period cases, the resulting period is not exact (it is too short). + fn reverse_maximal_suffix(arr: &[u8], known_period: usize, order_greater: bool) -> usize { + let mut left = 0; // Corresponds to i in the paper + let mut right = 1; // Corresponds to j in the paper + let mut offset = 0; // Corresponds to k in the paper, but starting at 0 + // to match 0-based indexing. + let mut period = 1; // Corresponds to p in the paper + let n = arr.len(); + + while right + offset < n { + let a = arr[n - (1 + right + offset)]; + let b = arr[n - (1 + left + offset)]; + if (a < b && !order_greater) || (a > b && order_greater) { + // Suffix is smaller, period is entire prefix so far. + right += offset + 1; + offset = 0; + period = right - left; + } else if a == b { + // Advance through repetition of the current period. + if offset + 1 == period { + right += offset + 1; + offset = 0; + } else { + offset += 1; + } + } else { + // Suffix is larger, start over from current location. + left = right; + right += 1; + offset = 0; + period = 1; + } + if period == known_period { + break; + } + } + debug_assert!(period <= known_period); + left + } +} diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index 90a0e4717e219..548c28d689274 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -60,14 +60,9 @@ fn test_simple_iteration() { 'a', "forward iteration for ASCII string", next => Matches(0, 1), - next => Rejects(1, 2), - next => Rejects(2, 3), - next => Rejects(3, 4), - next => Rejects(4, 5), + next => Rejects(1, 5), next => Matches(5, 6), - next => Rejects(6, 7), - next => Rejects(7, 8), - next => Rejects(8, 9), + next => Rejects(6, 9), next => Done ); @@ -75,14 +70,9 @@ fn test_simple_iteration() { "abcdeabcd", 'a', "reverse iteration for ASCII string", - next_back => Rejects(8, 9), - next_back => Rejects(7, 8), - next_back => Rejects(6, 7), + next_back => Rejects(6, 9), next_back => Matches(5, 6), - next_back => Rejects(4, 5), - next_back => Rejects(3, 4), - next_back => Rejects(2, 3), - next_back => Rejects(1, 2), + next_back => Rejects(1, 5), next_back => Matches(0, 1), next_back => Done ); @@ -94,8 +84,7 @@ fn test_simple_iteration() { next => Matches(0, 3), next => Rejects(3, 6), next => Matches(6, 9), - next => Rejects(9, 12), - next => Rejects(12, 15), + next => Rejects(9, 15), next => Done ); @@ -103,14 +92,9 @@ fn test_simple_iteration() { "我的猫说meow", 'm', "forward iteration for mixed string", - next => Rejects(0, 3), - next => Rejects(3, 6), - next => Rejects(6, 9), - next => Rejects(9, 12), + next => Rejects(0, 12), next => Matches(12, 13), - next => Rejects(13, 14), - next => Rejects(14, 15), - next => Rejects(15, 16), + next => Rejects(13, 16), next => Done ); @@ -118,14 +102,9 @@ fn test_simple_iteration() { "我的猫说meow", '猫', "reverse iteration for mixed string", - next_back => Rejects(15, 16), - next_back => Rejects(14, 15), - next_back => Rejects(13, 14), - next_back => Rejects(12, 13), - next_back => Rejects(9, 12), + next_back => Rejects(9, 16), next_back => Matches(6, 9), - next_back => Rejects(3, 6), - next_back => Rejects(0, 3), + next_back => Rejects(0, 6), next_back => Done ); } @@ -231,30 +210,12 @@ fn test_stress_indices() { next => Done ); + // this test should generate something that will run memchr search_asserts!( STRESS, 'x', "Indices of characters in stress test", - next => Rejects(0, 2), // Á - next => Rejects(2, 3), // a - next => Rejects(3, 7), // 🁀 - next => Rejects(7, 8), // b - next => Rejects(8, 10), // Á - next => Rejects(10, 13), // ꁁ - next => Rejects(13, 14), // f - next => Rejects(14, 15), // g - next => Rejects(15, 19), // 😀 - next => Rejects(19, 22), // 각 - next => Rejects(22, 25), // ก - next => Rejects(25, 28), // ᘀ - next => Rejects(28, 31), // 각 - next => Rejects(31, 32), // a - next => Rejects(32, 34), // Á - next => Rejects(34, 37), // 각 - next => Rejects(37, 40), // ꁁ - next => Rejects(40, 43), // ก - next => Rejects(43, 47), // 😀 - next => Rejects(47, 48), // a + next => Rejects(0, 48), // no character matches next => Done ); } @@ -276,11 +237,11 @@ fn test_forward_search_shared_bytes() { 'Á', "Forward search for two-byte Latin character; check if next() still works", next_match => Matches(0, 2), - next => Rejects(2, 3), + next => Rejects(2, 8), next_match => Matches(8, 10), - next => Rejects(10, 13), + next => Rejects(10, 32), next_match => Matches(32, 34), - next => Rejects(34, 37), + next => Rejects(34, 48), next_match => Done ); @@ -289,7 +250,7 @@ fn test_forward_search_shared_bytes() { '각', "Forward search for three-byte Hangul character", next_match => Matches(19, 22), - next => Rejects(22, 25), + next => Rejects(22, 28), next_match => Matches(28, 31), next_match => Matches(34, 37), next_match => Done @@ -300,11 +261,11 @@ fn test_forward_search_shared_bytes() { '각', "Forward search for three-byte Hangul character; check if next() still works", next_match => Matches(19, 22), - next => Rejects(22, 25), + next => Rejects(22, 28), next_match => Matches(28, 31), - next => Rejects(31, 32), + next => Rejects(31, 34), next_match => Matches(34, 37), - next => Rejects(37, 40), + next => Rejects(37, 48), next_match => Done ); @@ -313,9 +274,9 @@ fn test_forward_search_shared_bytes() { 'ก', "Forward search for three-byte Thai character", next_match => Matches(22, 25), - next => Rejects(25, 28), + next => Rejects(25, 40), next_match => Matches(40, 43), - next => Rejects(43, 47), + next => Rejects(43, 48), next_match => Done ); @@ -324,9 +285,9 @@ fn test_forward_search_shared_bytes() { 'ก', "Forward search for three-byte Thai character; check if next() still works", next_match => Matches(22, 25), - next => Rejects(25, 28), + next => Rejects(25, 40), next_match => Matches(40, 43), - next => Rejects(43, 47), + next => Rejects(43, 48), next_match => Done ); @@ -335,7 +296,7 @@ fn test_forward_search_shared_bytes() { '😁', "Forward search for four-byte emoji", next_match => Matches(15, 19), - next => Rejects(19, 22), + next => Rejects(19, 43), next_match => Matches(43, 47), next => Rejects(47, 48), next_match => Done @@ -346,7 +307,7 @@ fn test_forward_search_shared_bytes() { '😁', "Forward search for four-byte emoji; check if next() still works", next_match => Matches(15, 19), - next => Rejects(19, 22), + next => Rejects(19, 43), next_match => Matches(43, 47), next => Rejects(47, 48), next_match => Done @@ -357,9 +318,9 @@ fn test_forward_search_shared_bytes() { 'ꁁ', "Forward search for three-byte Yi character with repeated bytes", next_match => Matches(10, 13), - next => Rejects(13, 14), + next => Rejects(13, 37), next_match => Matches(37, 40), - next => Rejects(40, 43), + next => Rejects(40, 48), next_match => Done ); @@ -368,9 +329,9 @@ fn test_forward_search_shared_bytes() { 'ꁁ', "Forward search for three-byte Yi character with repeated bytes; check if next() still works", next_match => Matches(10, 13), - next => Rejects(13, 14), + next => Rejects(13, 37), next_match => Matches(37, 40), - next => Rejects(40, 43), + next => Rejects(40, 48), next_match => Done ); } @@ -392,9 +353,9 @@ fn test_reverse_search_shared_bytes() { 'Á', "Reverse search for two-byte Latin character; check if next_back() still works", next_match_back => Matches(32, 34), - next_back => Rejects(31, 32), + next_back => Rejects(10, 32), next_match_back => Matches(8, 10), - next_back => Rejects(7, 8), + next_back => Rejects(2, 8), next_match_back => Matches(0, 2), next_back => Done ); @@ -404,7 +365,7 @@ fn test_reverse_search_shared_bytes() { '각', "Reverse search for three-byte Hangul character", next_match_back => Matches(34, 37), - next_back => Rejects(32, 34), + next_back => Rejects(31, 34), next_match_back => Matches(28, 31), next_match_back => Matches(19, 22), next_match_back => Done @@ -415,11 +376,11 @@ fn test_reverse_search_shared_bytes() { '각', "Reverse search for three-byte Hangul character; check if next_back() still works", next_match_back => Matches(34, 37), - next_back => Rejects(32, 34), + next_back => Rejects(31, 34), next_match_back => Matches(28, 31), - next_back => Rejects(25, 28), + next_back => Rejects(22, 28), next_match_back => Matches(19, 22), - next_back => Rejects(15, 19), + next_back => Rejects(0, 19), next_match_back => Done ); @@ -428,9 +389,9 @@ fn test_reverse_search_shared_bytes() { 'ก', "Reverse search for three-byte Thai character", next_match_back => Matches(40, 43), - next_back => Rejects(37, 40), + next_back => Rejects(25, 40), next_match_back => Matches(22, 25), - next_back => Rejects(19, 22), + next_back => Rejects(0, 22), next_match_back => Done ); @@ -439,9 +400,9 @@ fn test_reverse_search_shared_bytes() { 'ก', "Reverse search for three-byte Thai character; check if next_back() still works", next_match_back => Matches(40, 43), - next_back => Rejects(37, 40), + next_back => Rejects(25, 40), next_match_back => Matches(22, 25), - next_back => Rejects(19, 22), + next_back => Rejects(0, 22), next_match_back => Done ); @@ -450,9 +411,9 @@ fn test_reverse_search_shared_bytes() { '😁', "Reverse search for four-byte emoji", next_match_back => Matches(43, 47), - next_back => Rejects(40, 43), + next_back => Rejects(19, 43), next_match_back => Matches(15, 19), - next_back => Rejects(14, 15), + next_back => Rejects(0, 15), next_match_back => Done ); @@ -461,9 +422,9 @@ fn test_reverse_search_shared_bytes() { '😁', "Reverse search for four-byte emoji; check if next_back() still works", next_match_back => Matches(43, 47), - next_back => Rejects(40, 43), + next_back => Rejects(19, 43), next_match_back => Matches(15, 19), - next_back => Rejects(14, 15), + next_back => Rejects(0, 15), next_match_back => Done ); @@ -472,9 +433,9 @@ fn test_reverse_search_shared_bytes() { 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes", next_match_back => Matches(37, 40), - next_back => Rejects(34, 37), + next_back => Rejects(13, 37), next_match_back => Matches(10, 13), - next_back => Rejects(8, 10), + next_back => Rejects(0, 10), next_match_back => Done ); @@ -483,9 +444,9 @@ fn test_reverse_search_shared_bytes() { 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes; check if next_back() still works", next_match_back => Matches(37, 40), - next_back => Rejects(34, 37), + next_back => Rejects(13, 37), next_match_back => Matches(10, 13), - next_back => Rejects(8, 10), + next_back => Rejects(0, 10), next_match_back => Done ); } @@ -535,9 +496,9 @@ fn double_ended_regression_test() { '각', "Reverse double ended search for three-byte Hangul character", next_match_back => Matches(34, 37), - next_back => Rejects(32, 34), + next_back => Rejects(31, 34), next_match => Matches(19, 22), - next => Rejects(22, 25), + next => Rejects(22, 28), next_match_back => Matches(28, 31), next_match => Done ); @@ -546,8 +507,8 @@ fn double_ended_regression_test() { 'ก', "Double ended search for three-byte Thai character", next_match => Matches(22, 25), - next_back => Rejects(47, 48), - next => Rejects(25, 28), + next_back => Rejects(43, 48), + next => Rejects(25, 40), next_match_back => Matches(40, 43), next_match => Done ); @@ -556,9 +517,9 @@ fn double_ended_regression_test() { '😁', "Double ended search for four-byte emoji", next_match_back => Matches(43, 47), - next => Rejects(0, 2), + next => Rejects(0, 15), next_match => Matches(15, 19), - next_back => Rejects(40, 43), + next_back => Rejects(19, 43), next_match => Done ); search_asserts!( @@ -566,10 +527,9 @@ fn double_ended_regression_test() { 'ꁁ', "Double ended search for three-byte Yi character with repeated bytes", next_match => Matches(10, 13), - next => Rejects(13, 14), + next => Rejects(13, 37), next_match_back => Matches(37, 40), - next_back => Rejects(34, 37), - next_match => Done + next_back => Done ); } @@ -706,3 +666,11 @@ fn test_try_next_code_point_fwd_and_rev() { // out of range check_fwbw(&[0xf5, 0x80, 0x80, 0x80], None); } + +#[test] +fn str_searcher_reject_back_preserves_start_on_bad_input() { + let haystack = core::str_bytes::Bytes::from_bytes(&[0x80, b'x', b'y', b'z'][..]); + let mut searcher = "xyz".into_searcher(haystack); + assert_eq!(searcher.next_reject_back(), Some((0, 1))); + assert_eq!(searcher.next_reject_back(), None); +} From 03679abd6efb054db0176dd172d875763b60e075 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Tue, 11 Aug 2026 18:52:47 -0400 Subject: [PATCH 11/16] coretests: Add a few tests for backward multibyte predicate It works right now, but original implementation of the next commit breaks them with none of existing tests catching this regression. --- library/coretests/tests/pattern.rs | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index 548c28d689274..f65216dd2fd80 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -674,3 +674,36 @@ fn str_searcher_reject_back_preserves_start_on_bad_input() { assert_eq!(searcher.next_reject_back(), Some((0, 1))); assert_eq!(searcher.next_reject_back(), None); } + +#[test] +fn backward_search_predicate_pat() { + use core::pattern::{Pattern, ReverseSearcher}; + use core::str_bytes::Bytes; + + let haystack = Bytes::from_str("abc"); + let mut searcher = (|c: char| c == 'a').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((0, 1))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("abcabc"); + let mut searcher = (|c: char| c == 'c').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((5, 6))); + assert_eq!(searcher.next_match_back(), Some((2, 3))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabc"); + let mut searcher = (|c: char| c == 'é').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((0, 2))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabcé"); + let mut searcher = (|c: char| c == 'é').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((5, 7))); + assert_eq!(searcher.next_match_back(), Some((0, 2))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabc"); + let mut searcher = (|c: char| c == 'c').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((4, 5))); + assert_eq!(searcher.next_match_back(), None); +} From 92ff4289229c9adc22333ffff3240e6bceefe81f Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Sun, 9 Aug 2026 20:39:39 -0400 Subject: [PATCH 12/16] core: add concept of Flavour to core::str_bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since core::str_bytes module cannot assume byte slices it deals with are well-formed UTF-8 (or even WTF-8), the code must be defensive and accept invalid sequences. This eliminates optimisations which would be otherwise possible. Introduce a `Flavour` trait which tags `Bytes` type with information about the byte sequence. For example, if a `Bytes` object is created from `&str` it’s tagged with `Utf8` flavour which gives the code freedom to assume data is well-formed UTF-8. This brings back all the optimisations removed in previous commit. @pacak: - removed IS_WTF8 associated constant - unused - fixed a bug related to multibyte reverse matching: `next_code_point_reverse` reads the input via Iterator::next_back, passing `bytes.iter().rev()` reverses it a second time. Not good. --- library/core/src/str/mod.rs | 1 + library/core/src/str/pattern.rs | 26 +- library/core/src/str/validations.rs | 2 +- library/core/src/str_bytes.rs | 620 +++++++++++++++++++++------- 4 files changed, 480 insertions(+), 169 deletions(-) diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 6c244c10e9745..3724e3572fe98 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -58,6 +58,7 @@ pub use iter::{RSplitN, SplitN}; pub use lossy::{Utf8Chunk, Utf8Chunks}; #[stable(feature = "rust1", since = "1.0.0")] pub use traits::FromStr; +pub(crate) use validations::next_code_point_reverse; #[unstable(feature = "str_internals", issue = "none")] pub use validations::{ next_code_point, try_next_code_point, try_next_code_point_reverse, utf8_char_width, diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index 76952557235e5..7a8f7744ac3f6 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -89,7 +89,7 @@ impl<'a> Haystack for &'a str { /// Associated type for `>::Searcher`. #[derive(Clone, Debug)] -pub struct CharSearcher<'a>(str_bytes::CharSearcher<'a>); +pub struct CharSearcher<'a>(str_bytes::CharSearcher<'a, str_bytes::Utf8>); impl<'a> CharSearcher<'a> { fn new(haystack: &'a str, chr: char) -> Self { @@ -100,9 +100,7 @@ impl<'a> CharSearcher<'a> { unsafe impl<'a> Searcher<&'a str> for CharSearcher<'a> { #[inline] fn haystack(&self) -> &'a str { - // SAFETY: self.0’s haystack was created from &str thus it is valid - // UTF-8. - unsafe { super::from_utf8_unchecked(self.0.haystack().as_bytes()) } + self.0.haystack().into_str() } #[inline] fn next(&mut self) -> SearchStep { @@ -163,12 +161,7 @@ impl<'a> Pattern<&'a str> for char { #[inline] fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { - self.strip_prefix_of(str_bytes::Bytes::from_str(haystack)).map(|bytes| { - // SAFETY: Bytes were created from &str and Bytes never splits - // inside of UTF-8 bytes sequences thus `bytes` is still valid - // UTF-8. - unsafe { super::from_utf8_unchecked(bytes.as_bytes()) } - }) + self.encode_utf8(&mut [0u8; 4]).strip_prefix_of(haystack) } #[inline] @@ -178,12 +171,7 @@ impl<'a> Pattern<&'a str> for char { #[inline] fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { - self.strip_suffix_of(str_bytes::Bytes::from_str(haystack)).map(|bytes| { - // SAFETY: Bytes were created from &str and Bytes never splits - // inside of UTF-8 bytes sequences thus `bytes` is still valid - // UTF-8. - unsafe { super::from_utf8_unchecked(bytes.as_bytes()) } - }) + self.encode_utf8(&mut [0u8; 4]).strip_suffix_of(haystack) } #[inline] @@ -626,7 +614,7 @@ impl<'a, 'b> Pattern<&'a str> for &'b str { #[derive(Clone, Debug)] /// Associated type for `<&str as Pattern<&'a str>>::Searcher`. -pub struct StrSearcher<'a, 'b>(crate::str_bytes::StrSearcher<'a, 'b>); +pub struct StrSearcher<'a, 'b>(crate::str_bytes::StrSearcher<'a, 'b, crate::str_bytes::Utf8>); impl<'a, 'b> StrSearcher<'a, 'b> { fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { @@ -638,9 +626,7 @@ impl<'a, 'b> StrSearcher<'a, 'b> { unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn haystack(&self) -> &'a str { - let bytes = self.0.haystack().as_bytes(); - // SAFETY: self.0.haystack() was created from a &str. - unsafe { crate::str::from_utf8_unchecked(bytes) } + self.0.haystack().into_str() } #[inline] diff --git a/library/core/src/str/validations.rs b/library/core/src/str/validations.rs index b95d854712ac2..4783af75fc1f1 100644 --- a/library/core/src/str/validations.rs +++ b/library/core/src/str/validations.rs @@ -75,7 +75,7 @@ pub unsafe fn next_code_point<'a, I: Iterator>(bytes: &mut I) -> /// /// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string #[inline] -pub(super) unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option +pub(crate) unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option where I: DoubleEndedIterator, { diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs index ceeed5190f39b..8408ac84668ee 100644 --- a/library/core/src/str_bytes.rs +++ b/library/core/src/str_bytes.rs @@ -20,9 +20,13 @@ //! ``` #![unstable(feature = "str_internals", issue = "none")] +use crate::marker::PhantomData; use crate::mem::take; -use crate::pattern::{Haystack, MatchOnly, RejectOnly, SearchStep, Searcher}; -use crate::str::{try_next_code_point, try_next_code_point_reverse}; +use crate::pattern::{Haystack, MatchOnly, RejectOnly, SearchStep}; +use crate::str::{ + next_code_point, next_code_point_reverse, try_next_code_point, try_next_code_point_reverse, + utf8_char_width, +}; use crate::{cmp, ops, pattern}; type OptRange = Option<(usize, usize)>; @@ -39,10 +43,29 @@ type Range = ops::Range; /// sequences. This is in a sense a generalisation of a `&str` which allows /// portions of the buffer to be ill-formed while preserving correctness of /// existing well-formed parts. +/// +/// The `F` generic argument tags the slice with a [flavour][Flavour] which +/// specifies structure of the data. #[derive(Copy, Clone, Debug)] -pub struct Bytes<'a>(&'a [u8]); +pub struct Bytes<'a, F>(&'a [u8], PhantomData); + +impl<'a, F: Flavour> Bytes<'a, F> { + /// Creates a new `Bytes` wrapper around bytes slice. + /// + /// # Safety + /// + /// Caller must guarantee that the bytes adhere to the requirements for the + /// flavour `F`. E.g. for [`Wtf8`] flavour, the bytes must be well-formed + /// WTF-8 encoded string. + /// + /// It may be more convenient to use [`Bytes::from_str`] for `&str` and the + /// `From<&[u8]>` implementation for `&[u8]`. + pub unsafe fn new(bytes: &'a [u8]) -> Bytes<'a, F> { + Self(bytes, PhantomData) + } +} -impl<'a> Bytes<'a> { +impl<'a, F: Flavour> Bytes<'a, F> { /// Returns a byte slice of this `Bytes`'s contents. pub fn as_bytes(self) -> &'a [u8] { self.0 @@ -70,11 +93,7 @@ impl<'a> Bytes<'a> { /// advance position byte at a time. If you need to be able to advance /// position byte at a time use `advance_range_start` instead. fn adjust_position_fwd(self, range: Range) -> usize { - range.start - + self.as_bytes()[range.clone()] - .iter() - .take_while(|chr| !chr.is_utf8_char_boundary()) - .count() + F::adjust_position_fwd(self.as_bytes(), range) } /// Adjusts position backward so that it points at the closest potential @@ -89,12 +108,7 @@ impl<'a> Bytes<'a> { /// advance position byte at a time. If you need to be able to advance /// position character at a time use `advance_range_end` instead. fn adjust_position_bwd(self, range: Range) -> usize { - let shift = self.as_bytes()[range.start..range.end + 1] - .iter() - .rev() - .take_while(|chr| !chr.is_utf8_char_boundary()) - .count(); - range.end.saturating_sub(shift).max(range.start) + F::adjust_position_bwd(self.as_bytes(), range) } /// Given a valid range update it’s start so it falls on the next character @@ -105,11 +119,7 @@ impl<'a> Bytes<'a> { /// `range.start + 1`. In other words, well-formed WTF-8 bytes sequence are /// skipped in one go while ill-formed sequences are skipped byte-by-byte. fn advance_range_start(self, range: Range) -> usize { - assert!(!range.is_empty()); - match try_next_code_point(&self.0[range.clone()]) { - Some((_, len)) => range.start + len, - None => range.end.min(range.start + 1), - } + range.start + F::advance_range_start(&self.as_bytes()[range]) } /// Given a valid range update it’s end so it falls on the previous @@ -121,11 +131,7 @@ impl<'a> Bytes<'a> { /// sequence are skipped in one go while ill-formed sequences are skipped /// byte-by-byte. fn advance_range_end(self, range: Range) -> usize { - assert!(!range.is_empty()); - match try_next_code_point_reverse(&self.0[range.clone()]) { - Some((_, len)) => range.end - len, - None => range.end - 1, - } + range.start + F::advance_range_end(&self.as_bytes()[range]) } /// Returns valid UTF-8 character at the front of the slice. @@ -134,7 +140,7 @@ impl<'a> Bytes<'a> { /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. /// WTF-8 sequences which encode surrogates are considered invalid. fn get_first_code_point(self) -> Option<(char, usize)> { - try_next_code_point(&self.0) + F::get_first_code_point(self.as_bytes()) } /// Returns valid UTF-8 character at the end of the slice. @@ -142,8 +148,8 @@ impl<'a> Bytes<'a> { /// If slice doesn’t end with a valid UTF-8 sequence, returns `None`. /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. /// WTF-8 sequences which encode surrogates are considered invalid. - fn get_last_code_point(&self) -> Option<(char, usize)> { - try_next_code_point_reverse(&self.0) + fn get_last_code_point(self) -> Option<(char, usize)> { + F::get_last_code_point(self.as_bytes()) } /// Looks for the next UTF-8-encoded character in the slice. @@ -153,13 +159,8 @@ impl<'a> Bytes<'a> { /// Returns position of the match, decoded character and UTF-8 length of /// that character. fn find_code_point_fwd(self, range: Range) -> Option<(usize, char, usize)> { - let bytes = &self.as_bytes()[range.clone()]; - (0..bytes.len()) - .filter_map(|pos| { - let (chr, len) = try_next_code_point(&bytes[pos..])?; - Some((range.start + pos, chr, len)) - }) - .next() + F::find_code_point_fwd(&self.as_bytes()[range.clone()]) + .map(|(pos, chr, len)| (range.start + pos, chr, len)) } /// Looks backwards for the next UTF-8 encoded character in the slice. @@ -169,28 +170,297 @@ impl<'a> Bytes<'a> { /// Returns position of the match, decoded character and UTF-8 length of /// that character. fn find_code_point_bwd(&self, range: Range) -> Option<(usize, char, usize)> { - let bytes = &self.as_bytes()[range.clone()]; - (0..bytes.len()) - .rev() - .filter_map(|pos| { - let (chr, len) = try_next_code_point(&bytes[pos..])?; - Some((range.start + pos, chr, len)) - }) - .next() + F::find_code_point_bwd(&self.as_bytes()[range.clone()]) + .map(|(pos, chr, len)| (range.start + pos, chr, len)) } } -impl<'a> Bytes<'a> { - /// Wraps `&[u8]` into `Bytes`. +impl<'a> Bytes<'a, Unstructured> { #[inline] + /// Wraps `&[u8]` into `Bytes`. pub fn from_bytes(val: &'a [u8]) -> Self { - Self(val) + Self(val, PhantomData) } +} - /// Wraps `&str` into `Bytes`. +impl<'a> Bytes<'a, Utf8> { #[inline] + /// Wraps `&str` into `Bytes`. pub fn from_str(val: &'a str) -> Self { - Self(val.as_bytes()) + // SAFETY: `str` is guaranteed to be UTF-8 + unsafe { Bytes::new(val.as_bytes()) } + } + + /// Returns the contents of this `Bytes` as a `&str`. + #[inline] + pub fn into_str(self) -> &'a str { + if cfg!(debug_assertions) { + crate::str::from_utf8(self.as_bytes()).unwrap() + } else { + // SAFETY: Bytes with Utf8 flavor are guaranteed to be valid UTF-8 + unsafe { crate::str::from_utf8_unchecked(self.as_bytes()) } + } + } +} + +/// A marker for bytes slice which is not necessarily well-formed. +#[derive(Clone, Copy, Debug)] +pub enum Unstructured {} +/// A marker for well-formed WTF-8 bytes slice. +#[derive(Clone, Copy, Debug)] +pub enum Wtf8 {} +/// A marker for well-formed UTF-8 bytes slice. +#[derive(Clone, Copy, Debug)] +pub enum Utf8 {} + +/// A marker trait indicating ‘flavour’ of data referred by [`Bytes`] type. +/// +/// The trait abstracts away operations related to identifying and decoding +/// ‘characters’ from a bytes slice. A valid WTF-8 byte sequence is always +/// treated as indivisible ‘character’ but depending on the flavour code can +/// make different assumption about contents of the bytes slice: +/// - [`Unstructured`] flavoured bytes slice may contain ill-formed bytes +/// sequences and in those each byte is treated as separate ‘character’, +/// - [`Wtf8`] flavoured bytes slice is a well-formed WTF-8-encoded string (that +/// is some of the byte sequences may encode surrogate code points) and +/// - [`Utf8`] flavoured bytes slice is a well-formed UTF-8-encoded string (that +/// is all byte sequences encode valid Unicode code points). +pub trait Flavour: private::Flavour {} + +impl Flavour for Unstructured {} +impl Flavour for Wtf8 {} +impl Flavour for Utf8 {} + +mod private { + use super::*; + + /// Private methods of the [`super::Flavour`] trait. + pub trait Flavour: Copy + core::fmt::Debug { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize; + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize; + fn advance_range_start(bytes: &[u8]) -> usize; + fn advance_range_end(bytes: &[u8]) -> usize; + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)>; + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)>; + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)>; + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)>; + } + + impl Flavour for super::Unstructured { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + range.start + + bytes[range.clone()].iter().take_while(|chr| !chr.is_utf8_char_boundary()).count() + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + let shift = bytes[range.start..range.end + 1] + .iter() + .rev() + .take_while(|chr| !chr.is_utf8_char_boundary()) + .count(); + + range.end.saturating_sub(shift).max(range.start) + } + + fn advance_range_start(bytes: &[u8]) -> usize { + assert!(!bytes.is_empty()); + try_next_code_point(bytes).map_or(1, |(_, len)| len) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + assert!(!bytes.is_empty()); + bytes.len() - try_next_code_point_reverse(bytes).map_or(1, |(_, len)| len) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + try_next_code_point(bytes) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + try_next_code_point_reverse(bytes) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + (0..bytes.len()) + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((pos, chr, len)) + }) + .next() + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + (0..bytes.len()) + .rev() + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((pos, chr, len)) + }) + .next() + } + } + + impl Flavour for Wtf8 { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + let mut pos = range.start; + // Input is WTF-8 so we will never need to move more than three + // positions. This happens when we’re at pointing at the first + // continuation byte of a four-byte sequence. Unroll the loop. + for _ in 0..3 { + // We’re not checking pos against _end because we know that _end + // == bytes.len() or falls on a character boundary. We can + // therefore compare against bytes.len() and eliminate that + // comparison. + if bytes.get(pos).map_or(true, |b: &u8| b.is_utf8_char_boundary()) { + break; + } + pos += 1; + } + pos + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + let mut pos = range.end; + // Input is WTF-8 so we will never need to move more than three + // positions. This happens when we’re at pointing at the first + // continuation byte of a four-byte sequence. Unroll the loop. + for _ in 0..3 { + // SAFETY: `bytes` is well-formed WTF-8 sequence and at function + // start `pos` is index within `bytes`. Therefore, `bytes[pos]` + // is valid and a) if it’s a character boundary we exit the + // function or b) otherwise we know that `pos > 0` (because + // otherwise `bytes` wouldn’t be well-formed WTF-8). + if unsafe { bytes.get_unchecked(pos) }.is_utf8_char_boundary() { + break; + } + pos -= 1; + } + pos + } + + fn advance_range_start(bytes: &[u8]) -> usize { + // Input is valid WTF-8 so we can just deduce length of next + // sequence to skip from the first byte. + utf8_char_width(*bytes.get(0).unwrap()) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + let end = bytes.len().checked_sub(1).unwrap(); + Self::adjust_position_bwd(bytes, 0..end) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point(&mut bytes.iter())? }; + // WTF-8 might produce surrogate code points so we still need to + // verify that we got a valid character. + char::from_u32(cp).map(|chr| (chr, len_utf8(cp))) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point_reverse(&mut bytes.iter())? }; + // WTF-8 might produce surrogate code points so we still need to + // verify that we got a valid character. + char::from_u32(cp).map(|chr| (chr, len_utf8(cp))) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + let mut iter = bytes.iter(); + let mut pos = 0; + loop { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point(&mut iter)? }; + let len = len_utf8(cp); + if let Some(chr) = char::from_u32(cp) { + return Some((pos, chr, len)); + } + pos += len; + } + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + let mut iter = bytes.iter(); + let mut pos = bytes.len(); + loop { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point_reverse(&mut iter)? }; + let len = len_utf8(cp); + pos -= len; + if let Some(chr) = char::from_u32(cp) { + return Some((pos, chr, len)); + } + } + } + } + + impl Flavour for Utf8 { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + Wtf8::adjust_position_fwd(bytes, range) + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + Wtf8::adjust_position_bwd(bytes, range) + } + + fn advance_range_start(bytes: &[u8]) -> usize { + Wtf8::advance_range_start(bytes) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + Wtf8::advance_range_end(bytes) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let (_, chr, len) = Self::find_code_point_fwd(bytes)?; + Some((chr, len)) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let (_, chr, len) = Self::find_code_point_bwd(bytes)?; + Some((chr, len)) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + // SAFETY: We’re Utf8 flavour. Client promises that bytes are + // well-formed UTF-8. We can not only assume well-formed byte + // sequence but also that produced code points are valid. + let chr = unsafe { char::from_u32_unchecked(next_code_point(&mut bytes.iter())?) }; + let len = chr.len_utf8(); + Some((0, chr, len)) + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + // SAFETY: We’re Utf8 flavour. Client promises that bytes are + // well-formed UTF-8. We can not only assume well-formed byte + // sequence but also that produced code points are valid. + let chr = unsafe { + let code = next_code_point_reverse(&mut bytes.iter())?; + char::from_u32_unchecked(code) + }; + let len = chr.len_utf8(); + Some((bytes.len() - len, chr, len)) + } + } + + // Copied from src/chars/methods.rs. We need it because it’s not public + // there and char::len_utf8 requires us to have a char and we need this to + // work on surrogate code points as well. + #[inline] + const fn len_utf8(code: u32) -> usize { + if code < 0x80 { + 1 + } else if code < 0x800 { + 2 + } else if code < 0x10000 { + 3 + } else { + 4 + } } } @@ -200,25 +470,45 @@ trait SearchResult: crate::pattern::SearchResult { /// /// Doesn’t move the start position past `begin`. If position was adjusted, /// updates `*out` as well. - fn adjust_reject_start_bwd(self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self; + fn adjust_reject_start_bwd( + self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self; /// Adjusts reject’s end position forwards to make sure it doesn’t fall /// within well-formed WTF-8 sequence. /// /// Doesn’t move the end position past `len`. If position was adjusted, /// updates `*out` as well. - fn adjust_reject_end_fwd(self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self; + fn adjust_reject_end_fwd( + self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self; } impl SearchResult for SearchStep { - fn adjust_reject_start_bwd(mut self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self { + fn adjust_reject_start_bwd( + mut self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self { if let SearchStep::Reject(ref mut start, _) = self { *start = bytes.adjust_position_bwd(begin..*start); *out = *start; } self } - fn adjust_reject_end_fwd(mut self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self { + fn adjust_reject_end_fwd( + mut self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self { if let SearchStep::Reject(_, ref mut end) = self { *end = bytes.adjust_position_fwd(*end..len); *out = *end; @@ -228,23 +518,43 @@ impl SearchResult for SearchStep { } impl SearchResult for MatchOnly { - fn adjust_reject_start_bwd(self, _bytes: Bytes<'_>, _begin: usize, _out: &mut usize) -> Self { + fn adjust_reject_start_bwd( + self, + _bytes: Bytes<'_, F>, + _begin: usize, + _out: &mut usize, + ) -> Self { self } - fn adjust_reject_end_fwd(self, _bytes: Bytes<'_>, _end: usize, _out: &mut usize) -> Self { + fn adjust_reject_end_fwd( + self, + _bytes: Bytes<'_, F>, + _end: usize, + _out: &mut usize, + ) -> Self { self } } impl SearchResult for RejectOnly { - fn adjust_reject_start_bwd(mut self, bytes: Bytes<'_>, begin: usize, out: &mut usize) -> Self { + fn adjust_reject_start_bwd( + mut self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self { if let RejectOnly(Some((ref mut start, _))) = self { *start = bytes.adjust_position_bwd(begin..*start); *out = *start; } self } - fn adjust_reject_end_fwd(mut self, bytes: Bytes<'_>, len: usize, out: &mut usize) -> Self { + fn adjust_reject_end_fwd( + mut self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self { if let RejectOnly(Some((_, ref mut end))) = self { *end = bytes.adjust_position_fwd(*end..len); *out = *end; @@ -257,7 +567,7 @@ impl SearchResult for RejectOnly { // Impl for Haystack //////////////////////////////////////////////////////////////////////////////// -impl Haystack for Bytes<'_> { +impl<'hs, F: Flavour> Haystack for Bytes<'hs, F> { fn cursor_at_front(self) -> usize { 0 } @@ -267,13 +577,17 @@ impl Haystack for Bytes<'_> { fn is_empty(self) -> bool { self.0.is_empty() } + unsafe fn get_unchecked(self, range: Range) -> Self { - Self(if cfg!(debug_assertions) { - self.0.get(range).unwrap() - } else { - // SAFETY: Caller promises cursor is a valid split position. - unsafe { self.0.get_unchecked(range) } - }) + Self( + if cfg!(debug_assertions) { + self.0.get(range).unwrap() + } else { + // SAFETY: Caller promises cursor is a valid split position. + unsafe { self.0.get_unchecked(range) } + }, + PhantomData, + ) } } @@ -282,32 +596,32 @@ impl Haystack for Bytes<'_> { //////////////////////////////////////////////////////////////////////////////// #[diagnostic::do_not_recommend] -impl<'hs> pattern::Pattern> for char { - type Searcher = CharSearcher<'hs>; +impl<'hs, F: Flavour> pattern::Pattern> for char { + type Searcher = CharSearcher<'hs, F>; - fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { Self::Searcher::new(haystack, self) } - fn is_contained_in(self, haystack: Bytes<'hs>) -> bool { + fn is_contained_in(self, haystack: Bytes<'hs, F>) -> bool { let mut buf = [0; 4]; encode_utf8(self, &mut buf).is_contained_in(haystack) } - fn is_prefix_of(self, haystack: Bytes<'hs>) -> bool { + fn is_prefix_of(self, haystack: Bytes<'hs, F>) -> bool { let mut buf = [0; 4]; encode_utf8(self, &mut buf).is_prefix_of(haystack) } - fn strip_prefix_of(self, haystack: Bytes<'hs>) -> Option> { + fn strip_prefix_of(self, haystack: Bytes<'hs, F>) -> Option> { let mut buf = [0; 4]; encode_utf8(self, &mut buf).strip_prefix_of(haystack) } - fn is_suffix_of(self, haystack: Bytes<'hs>) -> bool { + fn is_suffix_of(self, haystack: Bytes<'hs, F>) -> bool { let mut buf = [0; 4]; encode_utf8(self, &mut buf).is_suffix_of(haystack) } - fn strip_suffix_of(self, haystack: Bytes<'hs>) -> Option> { + fn strip_suffix_of(self, haystack: Bytes<'hs, F>) -> Option> { let mut buf = [0; 4]; encode_utf8(self, &mut buf).strip_suffix_of(haystack) } @@ -322,8 +636,8 @@ fn encode_utf8(chr: char, buf: &mut [u8; 4]) -> &str { /// Searcher looking for a single character in the haystack. #[derive(Clone, Debug)] -pub struct CharSearcher<'hs> { - haystack: Bytes<'hs>, +pub struct CharSearcher<'hs, F> { + haystack: Bytes<'hs, F>, state: CharSearcherState, } @@ -341,16 +655,16 @@ struct CharSearcherState { is_match_bwd: bool, } -impl<'hs> CharSearcher<'hs> { +impl<'hs, F: Flavour> CharSearcher<'hs, F> { /// Creates a new searcher for the given character. #[inline] - pub fn new(haystack: Bytes<'hs>, chr: char) -> Self { + pub fn new(haystack: Bytes<'hs, F>, chr: char) -> Self { Self { haystack, state: CharSearcherState::new(haystack.len(), chr) } } } -unsafe impl<'hs> pattern::Searcher> for CharSearcher<'hs> { - fn haystack(&self) -> Bytes<'hs> { +unsafe impl<'hs, F: Flavour> pattern::Searcher> for CharSearcher<'hs, F> { + fn haystack(&self) -> Bytes<'hs, F> { self.haystack } @@ -358,26 +672,26 @@ unsafe impl<'hs> pattern::Searcher> for CharSearcher<'hs> { self.state.next_fwd(self.haystack) } fn next_match(&mut self) -> OptRange { - self.state.next_fwd::(self.haystack).0 + self.state.next_fwd::(self.haystack).0 } fn next_reject(&mut self) -> OptRange { - self.state.next_fwd::(self.haystack).0 + self.state.next_fwd::(self.haystack).0 } } -unsafe impl<'hs> pattern::ReverseSearcher> for CharSearcher<'hs> { +unsafe impl<'hs, F: Flavour> pattern::ReverseSearcher> for CharSearcher<'hs, F> { fn next_back(&mut self) -> SearchStep { self.state.next_bwd(self.haystack) } fn next_match_back(&mut self) -> OptRange { - self.state.next_bwd::(self.haystack).0 + self.state.next_bwd::(self.haystack).0 } fn next_reject_back(&mut self) -> OptRange { - self.state.next_bwd::(self.haystack).0 + self.state.next_bwd::(self.haystack).0 } } -impl<'hs> pattern::DoubleEndedSearcher> for CharSearcher<'hs> {} +impl<'hs, F: Flavour> pattern::DoubleEndedSearcher> for CharSearcher<'hs, F> {} impl CharSearcherState { fn new(haystack_len: usize, chr: char) -> Self { @@ -389,7 +703,7 @@ impl CharSearcherState { } } - fn find_match_fwd(&mut self, haystack: Bytes<'_>) -> OptRange { + fn find_match_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { let start = if take(&mut self.is_match_fwd) { (!self.range.is_empty()).then_some(self.range.start) } else { @@ -402,7 +716,7 @@ impl CharSearcherState { Some((start, start + self.needle.len())) } - fn next_reject_fwd(&mut self, haystack: Bytes<'_>) -> OptRange { + fn next_reject_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { if take(&mut self.is_match_fwd) { if self.range.is_empty() { return None; @@ -422,7 +736,7 @@ impl CharSearcherState { } } - fn next_fwd(&mut self, haystack: Bytes<'_>) -> R { + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { if R::USE_EARLY_REJECT { match self.next_reject_fwd(haystack) { Some((start, end)) => R::rejecting(start, end).unwrap(), @@ -447,7 +761,7 @@ impl CharSearcherState { } } - fn find_match_bwd(&mut self, haystack: Bytes<'_>) -> OptRange { + fn find_match_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { let start = if take(&mut self.is_match_bwd) { (!self.range.is_empty()).then(|| self.range.end - self.needle.len()) } else { @@ -460,7 +774,7 @@ impl CharSearcherState { Some((start, start + self.needle.len())) } - fn next_reject_bwd(&mut self, haystack: Bytes<'_>) -> OptRange { + fn next_reject_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { if take(&mut self.is_match_bwd) { if self.range.is_empty() { return None; @@ -480,7 +794,7 @@ impl CharSearcherState { } } - fn next_bwd(&mut self, haystack: Bytes<'_>) -> R { + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { if R::USE_EARLY_REJECT { match self.next_reject_bwd(haystack) { Some((start, end)) => R::rejecting(start, end).unwrap(), @@ -618,27 +932,27 @@ mod naive { //////////////////////////////////////////////////////////////////////////////// #[diagnostic::do_not_recommend] -impl<'hs, F: FnMut(char) -> bool> pattern::Pattern> for F { - type Searcher = PredicateSearcher<'hs, F>; +impl<'hs, F: Flavour, P: FnMut(char) -> bool> pattern::Pattern> for P { + type Searcher = PredicateSearcher<'hs, F, P>; - fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { Self::Searcher::new(haystack, self) } - fn is_prefix_of(mut self, haystack: Bytes<'hs>) -> bool { + fn is_prefix_of(mut self, haystack: Bytes<'hs, F>) -> bool { haystack.get_first_code_point().map_or(false, |(chr, _)| self(chr)) } - fn strip_prefix_of(mut self, haystack: Bytes<'hs>) -> Option> { + fn strip_prefix_of(mut self, haystack: Bytes<'hs, F>) -> Option> { let (chr, len) = haystack.get_first_code_point()?; // SAFETY: We’ve just checked slice starts with len-byte long // well-formed sequence. self(chr).then(|| unsafe { haystack.get_unchecked(len..haystack.len()) }) } - fn is_suffix_of(mut self, haystack: Bytes<'hs>) -> bool { + fn is_suffix_of(mut self, haystack: Bytes<'hs, F>) -> bool { haystack.get_last_code_point().map_or(false, |(chr, _)| self(chr)) } - fn strip_suffix_of(mut self, haystack: Bytes<'hs>) -> Option> { + fn strip_suffix_of(mut self, haystack: Bytes<'hs, F>) -> Option> { let (chr, len) = haystack.get_last_code_point()?; let len = haystack.len() - len; // SAFETY: We’ve just checked slice ends with len-byte long well-formed @@ -649,24 +963,24 @@ impl<'hs, F: FnMut(char) -> bool> pattern::Pattern> for F { /// Searcher looking for characters matching a predicate. #[derive(Clone, Debug)] -pub struct PredicateSearcher<'hs, F> { - haystack: Bytes<'hs>, - pred: F, +pub struct PredicateSearcher<'hs, F, P> { + haystack: Bytes<'hs, F>, + pred: P, start: usize, end: usize, fwd_match_len: u8, bwd_match_len: u8, } -impl<'hs, F> PredicateSearcher<'hs, F> { +impl<'hs, F: Flavour, P> PredicateSearcher<'hs, F, P> { /// Creates a new searcher for the given predicate. #[inline] - fn new(haystack: Bytes<'hs>, pred: F) -> Self { + pub fn new(haystack: Bytes<'hs, F>, pred: P) -> Self { Self { haystack, pred, start: 0, end: haystack.len(), fwd_match_len: 0, bwd_match_len: 0 } } } -impl<'hs, F: FnMut(char) -> bool> PredicateSearcher<'hs, F> { +impl<'hs, F: Flavour, P: FnMut(char) -> bool> PredicateSearcher<'hs, F, P> { fn find_match_fwd(&mut self) -> Option<(usize, usize)> { let mut start = self.start; while start < self.end { @@ -743,8 +1057,12 @@ impl<'hs, F: FnMut(char) -> bool> PredicateSearcher<'hs, F> { } } -unsafe impl<'hs, F: FnMut(char) -> bool> Searcher> for PredicateSearcher<'hs, F> { - fn haystack(&self) -> Bytes<'hs> { +unsafe impl<'hs, F, P> pattern::Searcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, +{ + fn haystack(&self) -> Bytes<'hs, F> { self.haystack } fn next(&mut self) -> SearchStep { @@ -758,8 +1076,10 @@ unsafe impl<'hs, F: FnMut(char) -> bool> Searcher> for PredicateSearc } } -unsafe impl<'hs, F: FnMut(char) -> bool> pattern::ReverseSearcher> - for PredicateSearcher<'hs, F> +unsafe impl<'hs, F, P> pattern::ReverseSearcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, { fn next_back(&mut self) -> SearchStep { self.next_bwd() @@ -772,8 +1092,10 @@ unsafe impl<'hs, F: FnMut(char) -> bool> pattern::ReverseSearcher> } } -impl<'hs, F: FnMut(char) -> bool> pattern::DoubleEndedSearcher> - for PredicateSearcher<'hs, F> +impl<'hs, F, P> pattern::DoubleEndedSearcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, { } @@ -782,67 +1104,69 @@ impl<'hs, F: FnMut(char) -> bool> pattern::DoubleEndedSearcher> //////////////////////////////////////////////////////////////////////////////// #[diagnostic::do_not_recommend] -impl<'hs, 'p> pattern::Pattern> for &'p str { - type Searcher = StrSearcher<'hs, 'p>; +impl<'hs, 'p, F: Flavour> pattern::Pattern> for &'p str { + type Searcher = StrSearcher<'hs, 'p, F>; - fn into_searcher(self, haystack: Bytes<'hs>) -> Self::Searcher { + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { Self::Searcher::new(haystack, self) } - fn is_prefix_of(self, haystack: Bytes<'hs>) -> bool { + fn is_prefix_of(self, haystack: Bytes<'hs, F>) -> bool { haystack.as_bytes().starts_with(self.as_bytes()) } - fn strip_prefix_of(self, haystack: Bytes<'hs>) -> Option> { - haystack.as_bytes().strip_prefix(self.as_bytes()).map(Bytes) + fn strip_prefix_of(self, haystack: Bytes<'hs, F>) -> Option> { + haystack.as_bytes().strip_prefix(self.as_bytes()).map(|bytes| Bytes(bytes, PhantomData)) } - fn is_suffix_of(self, haystack: Bytes<'hs>) -> bool { + fn is_suffix_of(self, haystack: Bytes<'hs, F>) -> bool { haystack.as_bytes().ends_with(self.as_bytes()) } - fn strip_suffix_of(self, haystack: Bytes<'hs>) -> Option> { - haystack.as_bytes().strip_suffix(self.as_bytes()).map(Bytes) + fn strip_suffix_of(self, haystack: Bytes<'hs, F>) -> Option> { + haystack.as_bytes().strip_suffix(self.as_bytes()).map(|bytes| Bytes(bytes, PhantomData)) } } /// Searcher looking for a substring in the haystack. #[derive(Clone, Debug)] -pub struct StrSearcher<'hs, 'p> { - haystack: Bytes<'hs>, - state: StrSearcherInner<'p>, +pub struct StrSearcher<'hs, 'p, F> { + haystack: Bytes<'hs, F>, + inner: StrSearcherInner<'p>, } -impl<'hs, 'p> StrSearcher<'hs, 'p> { +impl<'hs, 'p, F: Flavour> StrSearcher<'hs, 'p, F> { /// Creates a new searcher for the given substring. - pub fn new(haystack: Bytes<'hs>, needle: &'p str) -> Self { - let state = StrSearcherInner::new(haystack, needle); - Self { haystack, state } + pub fn new(haystack: Bytes<'hs, F>, needle: &'p str) -> Self { + let inner = StrSearcherInner::new(haystack, needle); + Self { haystack, inner } } } -unsafe impl<'hs, 'p> Searcher> for StrSearcher<'hs, 'p> { - fn haystack(&self) -> Bytes<'hs> { +unsafe impl<'hs, 'p, F: Flavour> pattern::Searcher> for StrSearcher<'hs, 'p, F> { + fn haystack(&self) -> Bytes<'hs, F> { self.haystack } fn next(&mut self) -> SearchStep { - self.state.next_fwd(self.haystack) + self.inner.next_fwd(self.haystack) } fn next_match(&mut self) -> OptRange { - self.state.next_fwd::(self.haystack).0 + self.inner.next_fwd::(self.haystack).0 } fn next_reject(&mut self) -> OptRange { - self.state.next_fwd::(self.haystack).0 + self.inner.next_fwd::(self.haystack).0 } } -unsafe impl<'hs, 'p> pattern::ReverseSearcher> for StrSearcher<'hs, 'p> { +unsafe impl<'hs, 'p, F: Flavour> pattern::ReverseSearcher> + for StrSearcher<'hs, 'p, F> +{ fn next_back(&mut self) -> SearchStep { - self.state.next_bwd(self.haystack) + self.inner.next_bwd(self.haystack) } fn next_match_back(&mut self) -> OptRange { - self.state.next_bwd::(self.haystack).0 + self.inner.next_bwd::(self.haystack).0 } fn next_reject_back(&mut self) -> OptRange { - self.state.next_bwd::(self.haystack).0 + self.inner.next_bwd::(self.haystack).0 } } @@ -854,7 +1178,7 @@ enum StrSearcherInner<'p> { } impl<'p> StrSearcherInner<'p> { - fn new(haystack: Bytes<'_>, needle: &'p str) -> Self { + fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { let mut chars = needle.chars(); let chr = match chars.next() { Some(chr) => chr, @@ -867,19 +1191,19 @@ impl<'p> StrSearcherInner<'p> { } } - fn next_fwd(&mut self, haystack: Bytes<'_>) -> R { + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { - Self::Empty(state) => state.next_fwd::(haystack), - Self::Char(state) => state.next_fwd::(haystack), - Self::Str(state) => state.next_fwd::(haystack), + Self::Empty(state) => state.next_fwd::(haystack), + Self::Char(state) => state.next_fwd::(haystack), + Self::Str(state) => state.next_fwd::(haystack), } } - fn next_bwd(&mut self, haystack: Bytes<'_>) -> R { + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { - Self::Empty(state) => state.next_bwd::(haystack), - Self::Char(state) => state.next_bwd::(haystack), - Self::Str(state) => state.next_bwd::(haystack), + Self::Empty(state) => state.next_bwd::(haystack), + Self::Char(state) => state.next_bwd::(haystack), + Self::Str(state) => state.next_bwd::(haystack), } } } @@ -896,15 +1220,15 @@ impl<'p> StrSearcherInner<'p> { struct EmptySearcherState(pattern::EmptyNeedleSearcher); impl EmptySearcherState { - fn new(haystack: Bytes<'_>) -> Self { + fn new(haystack: Bytes<'_, F>) -> Self { Self(pattern::EmptyNeedleSearcher::new(haystack)) } - fn next_fwd(&mut self, bytes: Bytes<'_>) -> R { + fn next_fwd(&mut self, bytes: Bytes<'_, F>) -> R { self.0.next_fwd(|range| bytes.advance_range_start(range)) } - fn next_bwd(&mut self, bytes: Bytes<'_>) -> R { + fn next_bwd(&mut self, bytes: Bytes<'_, F>) -> R { self.0.next_bwd(|range| bytes.advance_range_end(range)) } } @@ -921,12 +1245,12 @@ struct StrSearcherState<'p> { } impl<'p> StrSearcherState<'p> { - fn new(haystack: Bytes<'_>, needle: &'p str) -> Self { + fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { let searcher = TwoWaySearcher::new(haystack.len(), needle.as_bytes()); Self { needle, searcher } } - fn next_fwd(&mut self, bytes: Bytes<'_>) -> R { + fn next_fwd(&mut self, bytes: Bytes<'_, F>) -> R { if self.searcher.position >= bytes.len() { return R::DONE; } @@ -938,7 +1262,7 @@ impl<'p> StrSearcherState<'p> { .adjust_reject_end_fwd(bytes, bytes.len(), &mut self.searcher.position) } - fn next_bwd(&mut self, bytes: Bytes<'_>) -> R { + fn next_bwd(&mut self, bytes: Bytes<'_, F>) -> R { if self.searcher.end == 0 { return R::DONE; } From c0d31dd3fd6ce8f9a09570aab8869c1b6e5c4dd7 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 17 Aug 2026 18:51:41 -0400 Subject: [PATCH 13/16] core: add memchr-based single-ascii byte search to str_bytes I reverted `ByteNeedle` change earlier, time to add the same functionality back. `ByteSearcherState` is mostly copied from `CharSearcherState`, does a single ascii byte search. It is possible to do the dispatch inside of a CharSearcherState, but that makes it a bit slower. --- library/core/src/str_bytes.rs | 197 +++++++++++++++++++++++++++++++++- 1 file changed, 194 insertions(+), 3 deletions(-) diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs index 8408ac84668ee..a44ef9bbee6e4 100644 --- a/library/core/src/str_bytes.rs +++ b/library/core/src/str_bytes.rs @@ -23,6 +23,7 @@ use crate::marker::PhantomData; use crate::mem::take; use crate::pattern::{Haystack, MatchOnly, RejectOnly, SearchStep}; +use crate::slice::memchr; use crate::str::{ next_code_point, next_code_point_reverse, try_next_code_point, try_next_code_point_reverse, utf8_char_width, @@ -638,7 +639,44 @@ fn encode_utf8(chr: char, buf: &mut [u8; 4]) -> &str { #[derive(Clone, Debug)] pub struct CharSearcher<'hs, F> { haystack: Bytes<'hs, F>, - state: CharSearcherState, + state: CharSearcherInner, +} + +/// Dispatches searches over two different needle flavors: +/// +/// - [`ByteSearcherState`] works with ASCII `u8` chars and uses faster `memchr`/`memrchr` +/// - [`CharSearcherState`] works with any chars but works a bit slower. +#[derive(Clone, Debug)] +enum CharSearcherInner { + Byte(ByteSearcherState), + Char(CharSearcherState), +} + +impl CharSearcherInner { + #[inline] + fn new(haystack_len: usize, chr: char) -> Self { + if chr.is_ascii() { + Self::Byte(ByteSearcherState::new(haystack_len, chr as u8)) + } else { + Self::Char(CharSearcherState::new(haystack_len, chr)) + } + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Byte(state) => state.next_fwd::(haystack), + Self::Char(state) => state.next_fwd::(haystack), + } + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Byte(state) => state.next_bwd::(haystack), + Self::Char(state) => state.next_bwd::(haystack), + } + } } #[derive(Clone, Debug)] @@ -659,7 +697,7 @@ impl<'hs, F: Flavour> CharSearcher<'hs, F> { /// Creates a new searcher for the given character. #[inline] pub fn new(haystack: Bytes<'hs, F>, chr: char) -> Self { - Self { haystack, state: CharSearcherState::new(haystack.len(), chr) } + Self { haystack, state: CharSearcherInner::new(haystack.len(), chr) } } } @@ -843,6 +881,154 @@ impl CharBuffer { } } +/// Fast searcher for a single-byte needle using [`memchr`]. +/// +/// A single-byte `&str` needle is always an ASCII byte, which can never appear +/// as a continuation byte of a well-formed WTF-8 sequence, so a memchr hit is +/// always on a character boundary and always a full single-byte match. +#[derive(Clone, Debug)] +struct ByteSearcherState { + /// Not yet processed range of the haystack. + range: crate::ops::Range, + /// Needle (ASCII byte) the searcher is looking for within the haystack. + needle: u8, + /// If `true` and `range` is non-empty, `haystack[range]` starts with the + /// needle. + is_match_fwd: bool, + /// If `true` and `range` is non-empty, `haystack[range]` ends with the + /// needle. + is_match_bwd: bool, +} + +impl ByteSearcherState { + #[inline] + fn new(haystack_len: usize, needle: u8) -> Self { + Self { needle, range: 0..haystack_len, is_match_fwd: false, is_match_bwd: false } + } + + #[inline] + fn find_match_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_fwd) { + (!self.range.is_empty()).then_some(self.range.start) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.as_bytes().get_unchecked(self.range.clone()) }; + memchr::memchr(self.needle, bytes).map(|i| self.range.start + i) + }?; + Some((start, start + 1)) + } + + #[inline] + fn next_reject_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_fwd) { + if self.range.is_empty() { + return None; + } + self.range.start += 1; + } + let bytes = haystack.as_bytes(); + while self.range.start < self.range.end { + let start = self.range.start; + if bytes[start] == self.needle { + self.range.start += 1; + } else { + // `start` is always on a character boundary, so this rejects + // exactly the character starting at `start`. + let end = haystack.advance_range_start(self.range.clone()); + self.range.start = end; + return Some((start, end)); + } + } + None + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_fwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_fwd(haystack) { + if self.range.start < start { + if let Some(res) = R::rejecting(self.range.start, start) { + self.range.start = start; + self.is_match_fwd = true; + return res; + } + } + self.range.start = end; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let start = self.range.start; + self.range.start = self.range.end; + R::rejecting(start, self.range.end).unwrap_or(R::DONE) + } + } + + #[inline] + fn find_match_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_bwd) { + (!self.range.is_empty()).then(|| self.range.end - 1) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.as_bytes().get_unchecked(self.range.clone()) }; + memchr::memrchr(self.needle, bytes).map(|i| self.range.start + i) + }?; + Some((start, start + 1)) + } + + #[inline] + fn next_reject_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_bwd) { + if self.range.is_empty() { + return None; + } + self.range.end -= 1; + } + let bytes = haystack.as_bytes(); + while !self.range.is_empty() { + let end = self.range.end; + if bytes[end - 1] == self.needle { + self.range.end -= 1; + } else { + let start = haystack.advance_range_end(self.range.start..end); + self.range.end = start; + return Some((start, end)); + } + } + None + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_bwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_bwd(haystack) { + if end < self.range.end { + if let Some(res) = R::rejecting(end, self.range.end) { + self.range.end = end; + self.is_match_bwd = true; + return res; + } + } + self.range.end = start; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let end = self.range.end; + self.range.end = self.range.start; + R::rejecting(self.range.start, end).unwrap_or(R::DONE) + } + } +} + mod naive { use crate::slice::memchr; @@ -1173,6 +1359,7 @@ unsafe impl<'hs, 'p, F: Flavour> pattern::ReverseSearcher> #[derive(Clone, Debug)] enum StrSearcherInner<'p> { Empty(EmptySearcherState), + Byte(ByteSearcherState), Char(CharSearcherState), Str(StrSearcherState<'p>), } @@ -1184,7 +1371,9 @@ impl<'p> StrSearcherInner<'p> { Some(chr) => chr, None => return Self::Empty(EmptySearcherState::new(haystack)), }; - if chars.next().is_none() { + if let &[b] = needle.as_bytes() { + Self::Byte(ByteSearcherState::new(haystack.len(), b)) + } else if chars.next().is_none() { Self::Char(CharSearcherState::new(haystack.len(), chr)) } else { Self::Str(StrSearcherState::new(haystack, needle)) @@ -1194,6 +1383,7 @@ impl<'p> StrSearcherInner<'p> { fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { Self::Empty(state) => state.next_fwd::(haystack), + Self::Byte(state) => state.next_fwd::(haystack), Self::Char(state) => state.next_fwd::(haystack), Self::Str(state) => state.next_fwd::(haystack), } @@ -1202,6 +1392,7 @@ impl<'p> StrSearcherInner<'p> { fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { Self::Empty(state) => state.next_bwd::(haystack), + Self::Byte(state) => state.next_bwd::(haystack), Self::Char(state) => state.next_bwd::(haystack), Self::Str(state) => state.next_bwd::(haystack), } From 912814046a56b2a7acde02fb880f384aa293d121 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 17 Aug 2026 18:51:41 -0400 Subject: [PATCH 14/16] core/std: inline small functions --- library/core/src/str/pattern.rs | 4 ++++ library/core/src/str_bytes.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index 7a8f7744ac3f6..8d2477d4da2d4 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -92,6 +92,7 @@ impl<'a> Haystack for &'a str { pub struct CharSearcher<'a>(str_bytes::CharSearcher<'a, str_bytes::Utf8>); impl<'a> CharSearcher<'a> { + #[inline] fn new(haystack: &'a str, chr: char) -> Self { Self(str_bytes::CharSearcher::new(str_bytes::Bytes::from_str(haystack), chr)) } @@ -617,6 +618,7 @@ impl<'a, 'b> Pattern<&'a str> for &'b str { pub struct StrSearcher<'a, 'b>(crate::str_bytes::StrSearcher<'a, 'b, crate::str_bytes::Utf8>); impl<'a, 'b> StrSearcher<'a, 'b> { + #[inline] fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { let haystack = crate::str_bytes::Bytes::from_str(haystack); Self(crate::str_bytes::StrSearcher::new(haystack, needle)) @@ -639,6 +641,7 @@ unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { self.0.next_match() } + #[inline] fn next_reject(&mut self) -> Option<(usize, usize)> { self.0.next_reject() } @@ -655,6 +658,7 @@ unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { self.0.next_match_back() } + #[inline] fn next_reject_back(&mut self) -> Option<(usize, usize)> { self.0.next_reject_back() } diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs index a44ef9bbee6e4..f1fff05dd4fc5 100644 --- a/library/core/src/str_bytes.rs +++ b/library/core/src/str_bytes.rs @@ -702,28 +702,35 @@ impl<'hs, F: Flavour> CharSearcher<'hs, F> { } unsafe impl<'hs, F: Flavour> pattern::Searcher> for CharSearcher<'hs, F> { + #[inline] fn haystack(&self) -> Bytes<'hs, F> { self.haystack } + #[inline] fn next(&mut self) -> SearchStep { self.state.next_fwd(self.haystack) } + #[inline] fn next_match(&mut self) -> OptRange { self.state.next_fwd::(self.haystack).0 } + #[inline] fn next_reject(&mut self) -> OptRange { self.state.next_fwd::(self.haystack).0 } } unsafe impl<'hs, F: Flavour> pattern::ReverseSearcher> for CharSearcher<'hs, F> { + #[inline] fn next_back(&mut self) -> SearchStep { self.state.next_bwd(self.haystack) } + #[inline] fn next_match_back(&mut self) -> OptRange { self.state.next_bwd::(self.haystack).0 } + #[inline] fn next_reject_back(&mut self) -> OptRange { self.state.next_bwd::(self.haystack).0 } @@ -732,6 +739,7 @@ unsafe impl<'hs, F: Flavour> pattern::ReverseSearcher> for CharSea impl<'hs, F: Flavour> pattern::DoubleEndedSearcher> for CharSearcher<'hs, F> {} impl CharSearcherState { + #[inline] fn new(haystack_len: usize, chr: char) -> Self { Self { range: 0..haystack_len, @@ -741,6 +749,7 @@ impl CharSearcherState { } } + #[inline] fn find_match_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { let start = if take(&mut self.is_match_fwd) { (!self.range.is_empty()).then_some(self.range.start) @@ -754,6 +763,7 @@ impl CharSearcherState { Some((start, start + self.needle.len())) } + #[inline] fn next_reject_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { if take(&mut self.is_match_fwd) { if self.range.is_empty() { @@ -774,6 +784,7 @@ impl CharSearcherState { } } + #[inline] fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { if R::USE_EARLY_REJECT { match self.next_reject_fwd(haystack) { @@ -799,6 +810,7 @@ impl CharSearcherState { } } + #[inline] fn find_match_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { let start = if take(&mut self.is_match_bwd) { (!self.range.is_empty()).then(|| self.range.end - self.needle.len()) @@ -812,6 +824,7 @@ impl CharSearcherState { Some((start, start + self.needle.len())) } + #[inline] fn next_reject_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { if take(&mut self.is_match_bwd) { if self.range.is_empty() { @@ -832,6 +845,7 @@ impl CharSearcherState { } } + #[inline] fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { if R::USE_EARLY_REJECT { match self.next_reject_bwd(haystack) { @@ -862,6 +876,7 @@ impl CharSearcherState { struct CharBuffer([u8; 4], crate::num::NonZeroU8); impl CharBuffer { + #[inline] fn new(chr: char) -> Self { let mut buf = [0; 4]; let len = chr.encode_utf8(&mut buf).len(); @@ -870,10 +885,12 @@ impl CharBuffer { Self(buf, len) } + #[inline] fn len(&self) -> usize { usize::from(self.1.get()) } + #[inline] fn as_str(&self) -> &str { // SAFETY: `self.0` is UTF-8 encoding of a single character and `self.1` // is its length. See `new` constructor. @@ -1035,6 +1052,7 @@ mod naive { /// Looks forwards for the next position of needle within haystack. /// /// Safety: `needle` must consist of a single character. + #[inline] pub(super) unsafe fn find_match_fwd(haystack: &[u8], needle: &str) -> Option { debug_assert!(!needle.is_empty()); // SAFETY: Caller promises needle is non-empty. @@ -1066,6 +1084,7 @@ mod naive { /// Looks backwards for the next position of needle within haystack. /// /// Safety: `needle` must consist of a single character. + #[inline] pub(super) unsafe fn find_match_bwd(haystack: &[u8], needle: &str) -> Option { // SAFETY: Caller promises needle is non-empty. let (&first_byte, tail) = unsafe { needle.as_bytes().split_first().unwrap_unchecked() }; @@ -1321,6 +1340,7 @@ pub struct StrSearcher<'hs, 'p, F> { impl<'hs, 'p, F: Flavour> StrSearcher<'hs, 'p, F> { /// Creates a new searcher for the given substring. + #[inline] pub fn new(haystack: Bytes<'hs, F>, needle: &'p str) -> Self { let inner = StrSearcherInner::new(haystack, needle); Self { haystack, inner } @@ -1328,15 +1348,19 @@ impl<'hs, 'p, F: Flavour> StrSearcher<'hs, 'p, F> { } unsafe impl<'hs, 'p, F: Flavour> pattern::Searcher> for StrSearcher<'hs, 'p, F> { + #[inline] fn haystack(&self) -> Bytes<'hs, F> { self.haystack } + #[inline] fn next(&mut self) -> SearchStep { self.inner.next_fwd(self.haystack) } + #[inline] fn next_match(&mut self) -> OptRange { self.inner.next_fwd::(self.haystack).0 } + #[inline] fn next_reject(&mut self) -> OptRange { self.inner.next_fwd::(self.haystack).0 } @@ -1345,12 +1369,15 @@ unsafe impl<'hs, 'p, F: Flavour> pattern::Searcher> for StrSearche unsafe impl<'hs, 'p, F: Flavour> pattern::ReverseSearcher> for StrSearcher<'hs, 'p, F> { + #[inline] fn next_back(&mut self) -> SearchStep { self.inner.next_bwd(self.haystack) } + #[inline] fn next_match_back(&mut self) -> OptRange { self.inner.next_bwd::(self.haystack).0 } + #[inline] fn next_reject_back(&mut self) -> OptRange { self.inner.next_bwd::(self.haystack).0 } @@ -1365,6 +1392,7 @@ enum StrSearcherInner<'p> { } impl<'p> StrSearcherInner<'p> { + #[inline] fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { let mut chars = needle.chars(); let chr = match chars.next() { @@ -1380,6 +1408,7 @@ impl<'p> StrSearcherInner<'p> { } } + #[inline] fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { Self::Empty(state) => state.next_fwd::(haystack), @@ -1389,6 +1418,7 @@ impl<'p> StrSearcherInner<'p> { } } + #[inline] fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { match self { Self::Empty(state) => state.next_bwd::(haystack), @@ -1436,11 +1466,13 @@ struct StrSearcherState<'p> { } impl<'p> StrSearcherState<'p> { + #[inline] fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { let searcher = TwoWaySearcher::new(haystack.len(), needle.as_bytes()); Self { needle, searcher } } + #[inline] fn next_fwd(&mut self, bytes: Bytes<'_, F>) -> R { if self.searcher.position >= bytes.len() { return R::DONE; @@ -1453,6 +1485,7 @@ impl<'p> StrSearcherState<'p> { .adjust_reject_end_fwd(bytes, bytes.len(), &mut self.searcher.position) } + #[inline] fn next_bwd(&mut self, bytes: Bytes<'_, F>) -> R { if self.searcher.end == 0 { return R::DONE; From dcf297e8e2d7e00c145f911e10cf44294bab563e Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 17 Aug 2026 18:51:41 -0400 Subject: [PATCH 15/16] core::str_bytes: use unsafe for indexing pattern::find_str 4775.12ns/iter -> 2561.57ns/iter pattern::rfind_str 5621.05ns/iter -> 2492.68ns/iter --- library/core/src/str_bytes.rs | 50 ++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs index f1fff05dd4fc5..13f512de574f6 100644 --- a/library/core/src/str_bytes.rs +++ b/library/core/src/str_bytes.rs @@ -1723,7 +1723,17 @@ impl TwoWaySearcher { let start = if long_period { self.crit_pos } else { cmp::max(self.crit_pos, self.memory) }; for i in start..needle.len() { - if needle[i] != haystack[self.position + i] { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.position + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.position + needle_last` is a valid index in the haystack. Inside the + // loop `self.position` is unchanged and `i <= needle_last`, so + // `self.position + i` is a valid index. If we adjust `self.position` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.position + i) } + }; + if needle[i] != haystack_byte { self.position += i - self.crit_pos + 1; if !long_period { self.memory = 0; @@ -1735,7 +1745,17 @@ impl TwoWaySearcher { // See if the left part of the needle matches let start = if long_period { 0 } else { self.memory }; for i in (start..self.crit_pos).rev() { - if needle[i] != haystack[self.position + i] { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.position + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.position + needle_last` is a valid index in the haystack. Inside the + // loop `self.position` is unchanged and `i <= needle_last`, so + // `self.position + i` is a valid index. If we adjust `self.position` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.position + i) } + }; + if needle[i] != haystack_byte { self.position += self.period; if !long_period { self.memory = needle.len() - self.period; @@ -1822,7 +1842,18 @@ impl TwoWaySearcher { cmp::min(self.crit_pos_back, self.memory_back) }; for i in (0..crit).rev() { - if needle[i] != haystack[self.end - needle.len() + i] { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.end - needle.len() + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.end.wrapping_sub(needle.len())` is a valid index in the haystack. + // Inside the loop `self.end` is unchanged and `i < needle.len()`, so + // `end - needle.len() + i` is a valid index. If we adjust `self.end` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } + }; + + if needle[i] != haystack_byte { self.end -= self.crit_pos_back - i; if !long_period { self.memory_back = needle.len(); @@ -1834,7 +1865,18 @@ impl TwoWaySearcher { // See if the right part of the needle matches let needle_end = if long_period { needle.len() } else { self.memory_back }; for i in self.crit_pos_back..needle_end { - if needle[i] != haystack[self.end - needle.len() + i] { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.end - needle.len() + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.end.wrapping_sub(needle.len())` is a valid index in the haystack. + // Inside the loop `self.end` is unchanged and `i < needle.len()`, so + // `end - needle.len() + i` is a valid index. If we adjust `self.end` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } + }; + + if needle[i] != haystack_byte { self.end -= self.period; if !long_period { self.memory_back = self.period; From 313cc93ec783e97b401ead899317ae58c3e0c202 Mon Sep 17 00:00:00 2001 From: Mikhail Baykov Date: Mon, 17 Aug 2026 18:51:41 -0400 Subject: [PATCH 16/16] core: skip emitting rejects if result doesn't cary it --- library/core/src/pattern.rs | 9 +++++++++ library/core/src/str_bytes.rs | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/library/core/src/pattern.rs b/library/core/src/pattern.rs index fdc5c8310a25f..fcced0ef5cc33 100644 --- a/library/core/src/pattern.rs +++ b/library/core/src/pattern.rs @@ -259,6 +259,12 @@ pub trait SearchResult: Sized + sealed::Sealed { /// guaranteed to return `Some`. const USE_EARLY_REJECT: bool; + /// Whether [`rejecting()`][Self::rejecting] can ever return `Some`. + /// + /// This allows searches to skip emitting reject ranges entirely when the + /// result type can't carry them (e.g. [`MatchOnly`]). + const HAS_REJECTS: bool; + /// Returns value describing a match or `None` if this implementation /// doesn't care about matches. fn matching(start: usize, end: usize) -> Option; @@ -281,6 +287,7 @@ pub struct RejectOnly(pub Option<(usize, usize)>); impl SearchResult for SearchStep { const DONE: Self = SearchStep::Done; const USE_EARLY_REJECT: bool = false; + const HAS_REJECTS: bool = true; #[inline(always)] fn matching(s: usize, e: usize) -> Option { @@ -296,6 +303,7 @@ impl SearchResult for SearchStep { impl SearchResult for MatchOnly { const DONE: Self = Self(None); const USE_EARLY_REJECT: bool = false; + const HAS_REJECTS: bool = false; #[inline(always)] fn matching(s: usize, e: usize) -> Option { @@ -311,6 +319,7 @@ impl SearchResult for MatchOnly { impl SearchResult for RejectOnly { const DONE: Self = Self(None); const USE_EARLY_REJECT: bool = true; + const HAS_REJECTS: bool = true; #[inline(always)] fn matching(_s: usize, _e: usize) -> Option { diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs index 13f512de574f6..195d2baf789ba 100644 --- a/library/core/src/str_bytes.rs +++ b/library/core/src/str_bytes.rs @@ -1705,8 +1705,10 @@ impl TwoWaySearcher { }; if old_pos != self.position { - if let Some(ret) = R::rejecting(old_pos, self.position) { - return ret; + if R::HAS_REJECTS { + if let Some(ret) = R::rejecting(old_pos, self.position) { + return ret; + } } } @@ -1821,8 +1823,10 @@ impl TwoWaySearcher { }; if old_end != self.end { - if let Some(ret) = R::rejecting(self.end, old_end) { - return ret; + if R::HAS_REJECTS { + if let Some(ret) = R::rejecting(self.end, old_end) { + return ret; + } } }