diff --git a/library/alloc/src/str.rs b/library/alloc/src/str.rs index 4c86d7e06ae44..4e8686d720579 100644 --- a/library/alloc/src/str.rs +++ b/library/alloc/src/str.rs @@ -10,6 +10,7 @@ use core::borrow::{Borrow, BorrowMut}; use core::iter::FusedIterator; use core::mem::MaybeUninit; +use core::pattern::{Pattern, Utf8Pattern}; #[stable(feature = "encode_utf16", since = "1.8.0")] pub use core::str::EncodeUtf16; #[stable(feature = "split_ascii_whitespace", since = "1.34.0")] @@ -20,7 +21,6 @@ pub use core::str::SplitInclusive; pub use core::str::SplitWhitespace; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::pattern; -use core::str::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher, Utf8Pattern}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::str::{Bytes, CharIndices, Chars, from_utf8, from_utf8_mut}; #[stable(feature = "str_escape", since = "1.34.0")] @@ -305,7 +305,10 @@ impl str { without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn replace(&self, from: P, to: &str) -> String { + pub fn replace<'a, P>(&'a self, from: P, to: &str) -> String + where + P: Pattern<&'a str>, + { // Fast path for replacing a single ASCII character with another. if let Some(from_byte) = match from.as_utf8_pattern() { Some(Utf8Pattern::StringPattern(s)) => match s.as_bytes() { @@ -363,7 +366,10 @@ impl str { #[must_use = "this returns the replaced string as a new allocation, \ without modifying the original"] #[stable(feature = "str_replacen", since = "1.16.0")] - pub fn replacen(&self, pat: P, to: &str, count: usize) -> String { + pub fn replacen<'a, P>(&'a self, pat: P, to: &str, count: usize) -> String + where + P: Pattern<&'a str>, + { // Hope to reduce the times of re-allocation let mut result = String::with_capacity(32); let mut last_end = 0; diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index b3fb35a086dcd..787c6acb42945 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -53,7 +53,7 @@ use core::ops::Add; #[cfg(not(no_global_oom_handling))] use core::ops::AddAssign; use core::ops::{self, Range, RangeBounds}; -use core::str::pattern::{Pattern, Utf8Pattern}; +use core::pattern::{Pattern, Utf8Pattern}; use core::{fmt, hash, ptr, slice}; #[cfg(not(no_global_oom_handling))] @@ -1593,8 +1593,11 @@ impl String { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_remove_matches", issue = "72826")] - pub fn remove_matches(&mut self, pat: P) { - use core::str::pattern::Searcher; + pub fn remove_matches<'a, P>(&'a mut self, pat: P) + where + P: for<'x> Pattern<&'x str>, + { + use core::pattern::Searcher; let rejections = { let mut searcher = pat.into_searcher(self); @@ -2143,7 +2146,10 @@ impl String { /// [replacen]: ../../std/primitive.str.html#method.replacen #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_replace_in_place", issue = "147949")] - pub fn replace_first(&mut self, from: P, to: &str) { + pub fn replace_first<'a, P>(&'a mut self, from: P, to: &str) + where + P: for<'x> Pattern<&'x str>, + { let range = match self.match_indices(from).next() { Some((start, match_str)) => start..start + match_str.len(), None => return, @@ -2169,9 +2175,9 @@ impl String { /// ``` #[cfg(not(no_global_oom_handling))] #[unstable(feature = "string_replace_in_place", issue = "147949")] - pub fn replace_last(&mut self, from: P, to: &str) + pub fn replace_last<'a, P>(&'a mut self, from: P, to: &str) where - for<'a> P::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + P: for<'x> Pattern<&'x str, Searcher: core::pattern::ReverseSearcher<&'x str>>, { let range = match self.rmatch_indices(from).next() { Some((start, match_str)) => start..start + match_str.len(), @@ -2664,10 +2670,10 @@ impl<'a> Extend<&'a core::ascii::Char> for String { reason = "API not fully fleshed out and ready to be stabilized", issue = "27721" )] -impl<'b> Pattern for &'b String { - type Searcher<'a> = <&'b str as Pattern>::Searcher<'a>; +impl<'a, 'b> Pattern<&'a str> for &'b String { + type Searcher = <&'b str as Pattern<&'a str>>::Searcher; - fn into_searcher(self, haystack: &str) -> <&'b str as Pattern>::Searcher<'_> { + fn into_searcher(self, haystack: &'a str) -> <&'b str as Pattern<&'a str>>::Searcher { self[..].into_searcher(haystack) } @@ -2687,17 +2693,17 @@ impl<'b> Pattern for &'b String { } #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool + fn is_suffix_of(self, haystack: &'a str) -> bool where - Self::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + Self::Searcher: core::pattern::ReverseSearcher<&'a str>, { self[..].is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> where - Self::Searcher<'a>: core::str::pattern::ReverseSearcher<'a>, + Self::Searcher: core::pattern::ReverseSearcher<&'a str>, { self[..].strip_suffix_of(haystack) } diff --git a/library/alloctests/tests/str.rs b/library/alloctests/tests/str.rs index 830f6972f5af5..80e7a6d9dffc9 100644 --- a/library/alloctests/tests/str.rs +++ b/library/alloctests/tests/str.rs @@ -836,6 +836,18 @@ fn test_trim_matches() { assert_eq!("123foo1bar123".trim_matches(|c: char| c.is_numeric()), "foo1bar"); } +#[test] +fn test_trim_matches_with_str_pattern() { + assert_eq!("abc".trim_start_matches("ab"), "c"); + assert_eq!("xyzabcxyz".trim_start_matches("xyz"), "abcxyz"); + assert_eq!("abcabc".trim_start_matches("abc"), ""); + assert_eq!("ababab".trim_start_matches("ab"), ""); + + assert_eq!("abcab".trim_end_matches("ab"), "abc"); + assert_eq!("xyzabcxyz".trim_end_matches("xyz"), "xyzabc"); + assert_eq!("abcabc".trim_end_matches("abc"), ""); +} + #[test] fn test_trim_start() { assert_eq!("".trim_start(), ""); @@ -2009,14 +2021,14 @@ fn test_repeat() { } mod pattern { - use std::str::pattern::SearchStep::{self, Done, Match, Reject}; - use std::str::pattern::{Pattern, ReverseSearcher, Searcher}; + use std::pattern::SearchStep::{self, Done, Match, Reject}; + use std::pattern::{Pattern, ReverseSearcher, Searcher}; macro_rules! make_test { ($name:ident, $p:expr, $h:expr, [$($e:expr,)*]) => { #[allow(unused_imports)] mod $name { - use std::str::pattern::SearchStep::{Match, Reject}; + use std::pattern::SearchStep::{Match, Reject}; use super::{cmp_search_to_vec}; #[test] fn fwd() { @@ -2032,7 +2044,7 @@ mod pattern { fn cmp_search_to_vec

(rev: bool, pat: P, haystack: &str, right: Vec) where - P: for<'a> Pattern: ReverseSearcher<'a>>, + P: for<'a> Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let mut searcher = pat.into_searcher(haystack); let mut v = vec![]; @@ -2107,12 +2119,7 @@ mod pattern { Match(7, 7), ] ); - make_test!( - str_searcher_multibyte_haystack, - " ", - "├──", - [Reject(0, 3), Reject(3, 6), Reject(6, 9),] - ); + make_test!(str_searcher_multibyte_haystack, " ", "├──", [Reject(0, 9),]); make_test!( str_searcher_empty_needle_multibyte_haystack, "", @@ -2143,18 +2150,8 @@ mod pattern { Reject(6, 7), ] ); - make_test!( - char_searcher_multibyte_haystack, - ' ', - "├──", - [Reject(0, 3), Reject(3, 6), Reject(6, 9),] - ); - make_test!( - char_searcher_short_haystack, - '\u{1F4A9}', - "* \t", - [Reject(0, 1), Reject(1, 2), Reject(2, 3),] - ); + make_test!(char_searcher_multibyte_haystack, ' ', "├──", [Reject(0, 9),]); + make_test!(char_searcher_short_haystack, '\u{1F4A9}', "* \t", [Reject(0, 3),]); // See #85462 #[test] @@ -2196,6 +2193,21 @@ mod pattern { assert_eq!(searcher.next_back(), SearchStep::Done); } } + + #[test] + fn str_searcher_empty_needle_interleaved() { + let mut searcher = "".into_searcher("abc"); + + assert_eq!(searcher.next(), SearchStep::Match(0, 0)); + assert_eq!(searcher.next_back(), SearchStep::Match(3, 3)); + assert_eq!(searcher.next(), SearchStep::Reject(0, 1)); + assert_eq!(searcher.next_back(), SearchStep::Reject(2, 3)); + assert_eq!(searcher.next(), SearchStep::Match(1, 1)); + assert_eq!(searcher.next_back(), SearchStep::Match(2, 2)); + assert_eq!(searcher.next(), SearchStep::Reject(1, 2)); + assert_eq!(searcher.next_back(), SearchStep::Done); + assert_eq!(searcher.next(), SearchStep::Done); + } } macro_rules! generate_iterator_test { @@ -2290,11 +2302,11 @@ generate_iterator_test! { #[test] fn different_str_pattern_forwarding_lifetimes() { - use std::str::pattern::Pattern; + use std::pattern::Pattern; - fn foo

(p: P) + fn foo<'a, P>(p: P) where - for<'b> &'b P: Pattern, + for<'b> &'b P: Pattern<&'a str>, { for _ in 0..3 { "asdf".find(&p); diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index f026434acbbc1..c82e66cad0087 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -318,8 +318,10 @@ pub mod unsafe_binder; pub mod fmt; pub mod hash; +pub mod pattern; pub mod slice; pub mod str; +pub mod str_bytes; pub mod time; pub mod wtf8; diff --git a/library/core/src/pattern.rs b/library/core/src/pattern.rs new file mode 100644 index 0000000000000..fcced0ef5cc33 --- /dev/null +++ b/library/core/src/pattern.rs @@ -0,0 +1,895 @@ +//! The Pattern API. +//! +//! The Pattern API provides a generic mechanism for using different pattern +//! types when searching through different objects. +//! +//! For more details, see the traits [`Pattern`], [`Haystack`], [`Searcher`], +//! [`ReverseSearcher`] and [`DoubleEndedSearcher`]. Although this API is +//! unstable, it is exposed via stable methods on corresponding haystack types. +//! +//! # Examples +//! +//! [`Pattern<&str>`] is [implemented][pattern-impls] in the stable API for +//! [`&str`][`str`], [`char`], slices of [`char`], and functions and closures +//! implementing `FnMut(char) -> bool`. +//! +//! ``` +//! let s = "Can you find a needle in a haystack?"; +//! +//! // &str pattern +//! assert_eq!(s.find("you"), Some(4)); +//! // char pattern +//! assert_eq!(s.find('n'), Some(2)); +//! // array of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u']), Some(1)); +//! // slice of chars pattern +//! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u'][..]), Some(1)); +//! // closure pattern +//! assert_eq!(s.find(|c: char| c.is_ascii_punctuation()), Some(35)); +//! ``` +//! +//! [pattern-impls]: Pattern#implementors + +#![unstable( + feature = "pattern", + reason = "API not fully fleshed out and ready to be stabilized", + issue = "27721" +)] + +use crate::fmt; +use crate::mem::{replace, take}; +use crate::ops::Range; + +// Pattern + +/// A pattern which can be matched against a [`Haystack`]. +/// +/// A `Pattern` expresses that the implementing type can be used as +/// a pattern for searching in an `H`. For example, both the character `'a'` +/// and the string `"aa"` are patterns that would match at index `1` in +/// the string `"baaaab"`. +/// +/// The trait itself acts as a builder for an associated [`Searcher`] type, +/// which does the actual work of finding occurrences of the pattern in a haystack. +/// +/// Depending on the type of the pattern, the behavior of methods like +/// [`str::find`] and [`str::contains`] can change. The table below describes +/// some of those behaviors. +/// +/// | Pattern type | Match condition | +/// |--------------------------|-------------------------------------------| +/// | `&str` | is substring | +/// | `char` | is contained in string | +/// | `&[char]` | any char in slice is contained in string | +/// | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | +/// | `&&str` | is substring | +/// | `&String` | is substring | +/// +/// # Examples +/// +/// ``` +/// // &str +/// assert_eq!("abaaa".find("ba"), Some(1)); +/// assert_eq!("abaaa".find("bac"), None); +/// +/// // char +/// assert_eq!("abaaa".find('a'), Some(0)); +/// assert_eq!("abaaa".find('b'), Some(1)); +/// assert_eq!("abaaa".find('c'), None); +/// +/// // &[char; N] +/// assert_eq!("ab".find(&['b', 'a']), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z']), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd']), None); +/// +/// // &[char] +/// assert_eq!("ab".find(&['b', 'a'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['a', 'z'][..]), Some(0)); +/// assert_eq!("abaaa".find(&['c', 'd'][..]), None); +/// +/// // FnMut(char) -> bool +/// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); +/// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); +/// ``` +pub trait Pattern: Sized { + /// Associated searcher for this pattern + type Searcher: Searcher; + + /// Constructs the associated searcher from + /// `self` and the `haystack` to search in. + fn into_searcher(self, haystack: H) -> Self::Searcher; + + /// Checks whether the pattern matches anywhere in the haystack + #[inline] + fn is_contained_in(self, haystack: H) -> bool { + self.into_searcher(haystack).next_match().is_some() + } + + /// Checks whether the pattern matches at the front of the haystack + #[inline] + fn is_prefix_of(self, haystack: H) -> bool { + matches!( + self.into_searcher(haystack).next(), + SearchStep::Match(start, _) if start == haystack.cursor_at_front() + ) + } + + /// Checks whether the pattern matches at the back of the haystack + #[inline] + fn is_suffix_of(self, haystack: H) -> bool + where + Self::Searcher: ReverseSearcher, + { + matches!( + self.into_searcher(haystack).next_back(), + SearchStep::Match(_, end) if end == haystack.cursor_at_back() + ) + } + + /// Removes the pattern from the front of a haystack, if it matches + #[inline] + fn strip_prefix_of(self, haystack: H) -> Option { + if let SearchStep::Match(start, pos) = self.into_searcher(haystack).next() { + debug_assert_eq!( + start, + haystack.cursor_at_front(), + "The first search step from Searcher \ + must include the first character" + ); + let end = haystack.cursor_at_back(); + // SAFETY: `Searcher` is known to return valid indices. + Some(unsafe { haystack.get_unchecked(pos..end) }) + } else { + None + } + } + + /// Removes the pattern from the back of a haystack, if it matches. + #[inline] + fn strip_suffix_of(self, haystack: H) -> Option + where + Self::Searcher: ReverseSearcher, + { + if let SearchStep::Match(pos, end) = self.into_searcher(haystack).next_back() { + debug_assert_eq!( + end, + haystack.cursor_at_back(), + "The first search step from ReverseSearcher \ + must include the last character" + ); + let start = haystack.cursor_at_front(); + // SAFETY: `Searcher` is known to return valid indices. + Some(unsafe { haystack.get_unchecked(start..pos) }) + } else { + None + } + } + + /// Returns the pattern as UTF-8 if possible. + fn as_utf8_pattern(&self) -> Option> { + None + } +} +/// Result of calling [`Pattern::as_utf8_pattern()`]. +/// Can be used for inspecting the contents of a [`Pattern`] in cases +/// where the underlying representation can be represented as UTF-8. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum Utf8Pattern<'a> { + /// Type returned by String and str types. + /// This stores `str` rather than bytes so callers cannot describe + /// non-UTF-8 string patterns through this API. + StringPattern(&'a str), + /// Type returned by char types. + CharPattern(char), +} + +// Haystack + +/// A type which can be searched in using a [`Pattern`]. +/// +/// The trait is used in combination with the [`Pattern`] trait to express a pattern +/// that can be used to search for elements in a given haystack. +pub trait Haystack: Sized + Copy { + /// Returns a cursor pointing at the beginning of the haystack. + fn cursor_at_front(self) -> usize; + + /// Returns a cursor pointing at the end of the haystack. + fn cursor_at_back(self) -> usize; + + /// Returns whether the haystack is empty. + fn is_empty(self) -> bool { + self.cursor_at_front() == self.cursor_at_back() + } + + /// Returns portions of the haystack indicated by the cursor range. + /// + /// # Safety + /// + /// The range's start and end must be valid haystack split positions, + /// and start must not point to a position after end. + /// + /// Valid split positions are: + /// - the front of the haystack (as returned by + /// [`cursor_at_front()`][Self::cursor_at_front]), + /// - the back of the haystack (as returned by + /// [`cursor_at_back()`][Self::cursor_at_back]), or + /// - any cursor returned by a [`Searcher`] or [`ReverseSearcher`]. + unsafe fn get_unchecked(self, range: Range) -> Self; +} + +// Searcher + +/// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub enum SearchStep { + /// Expresses that a match of the pattern has been found at + /// `haystack[a..b]`. + Match(usize, usize), + /// Expresses that `haystack[a..b]` has been rejected as a possible match + /// of the pattern. + /// + /// Note that there might be more than one `Reject` between two `Match`es, + /// there is no requirement for them to be combined into one. + Reject(usize, usize), + /// Expresses that every byte of the haystack has been visited, ending + /// the iteration. + Done, +} + +/// Possible return type of a search. +/// +/// It abstracts the differences between `next`, `next_match` and `next_reject` methods. Depending +/// on return type an implementation for those functions will generate matches and rejects, only +/// matches or only rejects. +#[unstable(feature = "pattern_internals", issue = "none")] +pub trait SearchResult: Sized + sealed::Sealed { + /// Value indicating searching has finished. + const DONE: Self; + + /// Whether search should return reject as soon as possible. + /// + /// For example, if a search can quickly determine that the very next + /// position cannot be where a next match starts, it should return a reject + /// with that position. This is an optimisation which allows the algorithm + /// to not waste time looking for the next match if caller is only + /// interested in the next position of a reject. + /// + /// If this is `true`, [`rejecting()`][Self::rejecting] is guaranteed to + /// return `Some` and if this is `false`, [`matching()`][Self::matching] is + /// guaranteed to return `Some`. + const USE_EARLY_REJECT: bool; + + /// Whether [`rejecting()`][Self::rejecting] can ever return `Some`. + /// + /// This allows searches to skip emitting reject ranges entirely when the + /// result type can't carry them (e.g. [`MatchOnly`]). + const HAS_REJECTS: bool; + + /// Returns value describing a match or `None` if this implementation + /// doesn't care about matches. + fn matching(start: usize, end: usize) -> Option; + + /// Returns value describing a reject or `None` if this implementation + /// doesn't care about rejects. + fn rejecting(start: usize, end: usize) -> Option; +} + +/// A wrapper for result type which only carries information about matches. +#[unstable(feature = "pattern_internals", issue = "none")] +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub struct MatchOnly(pub Option<(usize, usize)>); + +/// A wrapper for result type which only carries information about rejects. +#[unstable(feature = "pattern_internals", issue = "none")] +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +pub struct RejectOnly(pub Option<(usize, usize)>); + +impl SearchResult for SearchStep { + const DONE: Self = SearchStep::Done; + const USE_EARLY_REJECT: bool = false; + const HAS_REJECTS: bool = true; + + #[inline(always)] + fn matching(s: usize, e: usize) -> Option { + Some(SearchStep::Match(s, e)) + } + + #[inline(always)] + fn rejecting(s: usize, e: usize) -> Option { + Some(SearchStep::Reject(s, e)) + } +} + +impl SearchResult for MatchOnly { + const DONE: Self = Self(None); + const USE_EARLY_REJECT: bool = false; + const HAS_REJECTS: bool = false; + + #[inline(always)] + fn matching(s: usize, e: usize) -> Option { + Some(Self(Some((s, e)))) + } + + #[inline(always)] + fn rejecting(_s: usize, _e: usize) -> Option { + None + } +} + +impl SearchResult for RejectOnly { + const DONE: Self = Self(None); + const USE_EARLY_REJECT: bool = true; + const HAS_REJECTS: bool = true; + + #[inline(always)] + fn matching(_s: usize, _e: usize) -> Option { + None + } + + #[inline(always)] + fn rejecting(s: usize, e: usize) -> Option { + Some(Self(Some((s, e)))) + } +} + +mod sealed { + pub trait Sealed {} + impl Sealed for super::SearchStep {} + impl Sealed for super::MatchOnly {} + impl Sealed for super::RejectOnly {} +} + +/// A searcher for a string pattern. +/// +/// This trait provides methods for searching for non-overlapping +/// matches of a pattern starting from the front (left) of a string. +/// +/// It will be implemented by associated `Searcher` +/// types of the [`Pattern`] trait. +/// +/// The trait is marked unsafe because the indices returned by the +/// [`next()`][Searcher::next] methods are required to lie on valid split +/// positions of the haystack (for `&str` haystacks, these are utf8 character +/// boundaries). This enables consumers of this trait to slice the haystack +/// without additional runtime checks. +pub unsafe trait Searcher { + /// Getter for the underlying haystack to be searched in + /// + /// Will always return the same haystack. + fn haystack(&self) -> H; + + /// Performs the next search step starting from the front. + /// + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` matches + /// the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` can + /// not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack has + /// been visited. + /// + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] + /// will contain index ranges that are adjacent, non-overlapping, + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. + /// + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. + /// + /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` + /// might produce the stream + /// `[Reject(0, 1), Reject(1, 2), Match(2, 5), Reject(5, 8)]` + fn next(&mut self) -> SearchStep; + + /// Finds the next [`Match`][SearchStep::Match] result. See [`next()`][Searcher::next]. + /// + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_reject`][Searcher::next_reject] will overlap. This will return + /// `(start_match, end_match)`, where start_match is the index of where + /// the match begins, and end_match is the index after the end of the match. + #[inline] + fn next_match(&mut self) -> Option<(usize, usize)> { + loop { + match self.next() { + SearchStep::Match(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } + + /// Finds the next [`Reject`][SearchStep::Reject] result. See [`next()`][Searcher::next] + /// and [`next_match()`][Searcher::next_match]. + /// + /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges + /// of this and [`next_match`][Searcher::next_match] will overlap. + #[inline] + fn next_reject(&mut self) -> Option<(usize, usize)> { + loop { + match self.next() { + SearchStep::Reject(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } +} + +/// A reverse searcher for a string pattern. +/// +/// This trait provides methods for searching for non-overlapping +/// matches of a pattern starting from the back (right) of a string. +/// +/// It will be implemented by associated [`Searcher`] +/// types of the [`Pattern`] trait if the pattern supports searching +/// for it from the back. +/// +/// The index ranges returned by this trait are not required +/// to exactly match those of the forward search in reverse. +/// +/// For the reason why this trait is marked unsafe, see the +/// parent trait [`Searcher`]. +pub unsafe trait ReverseSearcher: Searcher { + /// Performs the next search step starting from the back. + /// + /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` + /// matches the pattern. + /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` + /// can not match the pattern, even partially. + /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack + /// has been visited + /// + /// The stream of [`Match`][SearchStep::Match] and + /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] + /// will contain index ranges that are adjacent, non-overlapping, + /// covering the whole haystack, and laying on valid split positions of the + /// haystack. + /// + /// A [`Match`][SearchStep::Match] result needs to contain the whole matched + /// pattern, however [`Reject`][SearchStep::Reject] results may be split up + /// into arbitrary many adjacent fragments. Both ranges may have zero length. + /// + /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` + /// might produce the stream + /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]`. + fn next_back(&mut self) -> SearchStep; + + /// Finds the next [`Match`][SearchStep::Match] result. + /// See [`next_back()`][ReverseSearcher::next_back]. + #[inline] + fn next_match_back(&mut self) -> Option<(usize, usize)> { + loop { + match self.next_back() { + SearchStep::Match(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } + + /// Finds the next [`Reject`][SearchStep::Reject] result. + /// See [`next_back()`][ReverseSearcher::next_back]. + #[inline] + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + loop { + match self.next_back() { + SearchStep::Reject(a, b) => return Some((a, b)), + SearchStep::Done => return None, + _ => continue, + } + } + } +} + +/// A marker trait to express that a [`ReverseSearcher`] +/// can be used for a [`DoubleEndedIterator`] implementation. +/// +/// For this, the impl of [`Searcher`] and [`ReverseSearcher`] need +/// to follow these conditions: +/// +/// - All results of `next()` need to be identical +/// to the results of `next_back()` in reverse order. +/// - `next()` and `next_back()` need to behave as +/// the two ends of a range of values, that is they +/// can not "walk past each other". +/// +/// # Examples +/// +/// `char::Searcher` is a `DoubleEndedSearcher` because searching for a +/// [`char`] only requires looking at one at a time, which behaves the same +/// from both ends. +/// +/// `(&str)::Searcher` is not a `DoubleEndedSearcher` because +/// the pattern `"aa"` in the haystack `"aaa"` matches as either +/// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. +pub trait DoubleEndedSearcher: ReverseSearcher {} + +////////////////////////////////////////////////////////////////////////////// +// Internal EmptyNeedleSearcher helper +////////////////////////////////////////////////////////////////////////////// + +/// Helper for implementing searchers looking for empty patterns. +/// +/// An empty pattern matches around every element of a haystack. For example, +/// within a `&str` it matches around every character. (This includes at the +/// beginning and end of the string). +/// +/// This struct helps implement searchers for empty patterns for various +/// haystacks. The only requirement is a function which advances the start +/// position or end position of the haystack range. +/// +/// # Examples +/// +/// ``` +/// #![feature(pattern, pattern_internals)] +/// # #![allow(internal_features)] +/// +/// use core::pattern::{EmptyNeedleSearcher, SearchStep}; +/// +/// let haystack = "fóó"; +/// let mut searcher = EmptyNeedleSearcher::new(haystack); +/// let advance = |range: core::ops::Range| { +/// range.start + haystack[range].chars().next().unwrap().len_utf8() +/// }; +/// let steps = core::iter::from_fn(|| { +/// match searcher.next_fwd(advance) { +/// SearchStep::Done => None, +/// step => Some(step) +/// } +/// }).collect::>(); +/// assert_eq!(&[ +/// SearchStep::Match(0, 0), +/// SearchStep::Reject(0, 1), +/// SearchStep::Match(1, 1), +/// SearchStep::Reject(1, 3), +/// SearchStep::Match(3, 3), +/// SearchStep::Reject(3, 5), +/// SearchStep::Match(5, 5), +/// ], steps.as_slice()); +/// ``` +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[unstable(feature = "pattern_internals", issue = "none")] +pub struct EmptyNeedleSearcher { + start: usize, + end: usize, + is_match_fwd: bool, + is_match_bwd: bool, + // Needed in case of an empty haystack, see #85462 + is_finished: bool, +} + +impl EmptyNeedleSearcher { + /// Creates a new empty needle searcher for given haystack. + /// + /// The haystack is used to initialise the range of valid cursors positions. + pub fn new(haystack: H) -> Self { + Self { + start: haystack.cursor_at_front(), + end: haystack.cursor_at_back(), + is_match_bwd: true, + is_match_fwd: true, + is_finished: false, + } + } + + /// Returns next search result. + /// + /// The callback function is used to advance the **start** of the range the + /// searcher is working on. It is passed the current range of cursor + /// positions that weren't visited yet and it must return the new start + /// cursor position. It's never called with an empty range. For some + /// haystacks the callback may be as simple as a closure returning the start + /// incremented by one; others might require looking for a new valid + /// boundary. + #[inline] + pub fn next_fwd(&mut self, advance_fwd: F) -> R + where + F: FnOnce(crate::ops::Range) -> usize, + { + if self.is_finished { + return R::DONE; + } + if take(&mut self.is_match_fwd) { + if let Some(ret) = R::matching(self.start, self.start) { + return ret; + } + } + if self.start < self.end { + let pos = self.start; + self.start = advance_fwd(self.start..self.end); + if let Some(ret) = R::rejecting(pos, self.start) { + self.is_match_fwd = true; + return ret; + } + return R::matching(self.start, self.start).unwrap(); + } + self.is_finished = true; + R::DONE + } + + /// Returns next search result. + /// + /// The callback function is used to advance the **end** of the range the + /// searcher is working on backwards. It is passed the current range of + /// cursor positions that weren't visited yet and it must return the new end + /// cursor position. It's never called with an empty range. For some + /// haystacks the callback may be as simple as a closure returning the end + /// decremented by one; others might require looking for a new valid + /// boundary. + #[inline] + pub fn next_bwd(&mut self, advance_bwd: F) -> R + where + F: FnOnce(crate::ops::Range) -> usize, + { + if self.is_finished { + return R::DONE; + } + if take(&mut self.is_match_bwd) { + if let Some(ret) = R::matching(self.end, self.end) { + return ret; + } + } + if self.start < self.end { + let pos = self.end; + self.end = advance_bwd(self.start..self.end); + if let Some(ret) = R::rejecting(self.end, pos) { + self.is_match_bwd = true; + return ret; + } + return R::matching(self.end, self.end).unwrap(); + } + self.is_finished = true; + R::DONE + } +} + +////////////////////////////////////////////////////////////////////////////// +// Internal Split and SplitN implementations +////////////////////////////////////////////////////////////////////////////// + +/// Helper type for implementing split iterators. +/// +/// It’s a generic type which works with any [`Haystack`] and [`Searcher`] over +/// that haystack. Intended usage is to create a newtype wrapping this type +/// which implements the [`Iterator`] interface on top of [`next_fwd`][Split::next_fwd] +/// or [`next_bwd`][Split::next_bwd] methods. +/// +/// Note that unless `S` implements [`DoubleEndedSearcher`] trait, it's +/// incorrect to use this type to implement a double ended iterator. +/// +/// For an example of this type in use, see [`core::str::Split`]. +#[unstable(feature = "pattern_internals", issue = "none")] +pub struct Split> { + /// Start of the region of the haystack yet to be examined. + start: usize, + /// End of the region of the haystack yet to be examined. + end: usize, + /// Searcher returning matches of the delimiter pattern. + searcher: S, + /// Whether to return an empty part if there's a delimiter at the end of the + /// haystack. + allow_trailing_empty: bool, + /// Whether splitting has finished. + finished: bool, + ctx: crate::marker::PhantomData, +} + +/// Helper type for implementing split iterators with a split limit. +/// +/// It's like [`Split`] but limits the number of parts the haystack will be split +/// into. +#[unstable(feature = "pattern_internals", issue = "none")] +pub struct SplitN> { + /// Inner split implementation. + inner: Split, + /// Maximum number of parts the haystack can be split into. + limit: usize, +} + +impl + Clone> Clone for Split { + fn clone(&self) -> Self { + Self { searcher: self.searcher.clone(), ..*self } + } +} + +impl + Clone> Clone for SplitN { + fn clone(&self) -> Self { + Self { inner: self.inner.clone(), ..*self } + } +} + +impl fmt::Debug for Split +where + S: Searcher + fmt::Debug, + H: Haystack, +{ + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Split") + .field("start", &self.start) + .field("end", &self.end) + .field("searcher", &self.searcher) + .field("allow_trailing_empty", &self.allow_trailing_empty) + .field("finished", &self.finished) + .finish() + } +} + +impl fmt::Debug for SplitN +where + S: Searcher + fmt::Debug, + H: Haystack, +{ + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("SplitN").field("inner", &self.inner).field("limit", &self.limit).finish() + } +} + +impl> Split { + /// Creates a new object configured without a limit and with + /// `allow_trailing_empty` option disabled. + /// + /// To set `allow_trailing_empty`, use + /// [`with_allow_trailing_empty()`][Self::with_allow_trailing_empty] method. + /// To set split limit, use [`with_limit()`][Self::with_limit] method. + pub fn new(searcher: S) -> Self { + let haystack = searcher.haystack(); + Self { + searcher, + start: haystack.cursor_at_front(), + end: haystack.cursor_at_back(), + allow_trailing_empty: false, + finished: false, + ctx: crate::marker::PhantomData, + } + } + + /// Changes the split limit from unlimited to the given value. + /// + /// The limit specifies maximum number of parts haystack will be split into. + pub fn with_limit(self, limit: usize) -> SplitN { + SplitN { inner: self, limit } + } + + /// Enables `allow_trailing_empty` option. + /// + /// If enabled (which is not the default) and the haystack is empty or + /// terminated by a pattern match, the last haystack part returned will be + /// empty. Otherwise, the last empty split is not returned. + pub fn with_allow_trailing_empty(mut self) -> Self { + self.allow_trailing_empty = true; + self + } +} + +impl> Split { + /// Returns the next part of the haystack or `None` if splitting is done. + /// + /// If `INCLUSIVE` is `true`, the returned value will include the matching + /// pattern. + #[inline] + pub fn next_fwd(&mut self) -> Option { + if self.finished { + return None; + } + let haystack = self.searcher.haystack(); + if let Some((start, end)) = self.searcher.next_match() { + let range = self.start..(if INCLUSIVE { end } else { start }); + self.start = end; + // SAFETY: self.start and self.end come from Haystack or Searcher + // and thus are guaranteed to be valid split positions. + Some(unsafe { haystack.get_unchecked(range) }) + } else { + self.get_end() + } + } + + /// Returns the next part of the haystack looking from the back or + /// `None` if splitting is done. + /// + /// If `INCLUSIVE` is `true`, the returned value will include the matching + /// pattern. + #[inline] + pub fn next_bwd(&mut self) -> Option + where + S: ReverseSearcher, + { + if self.finished { + return None; + } + + if !self.allow_trailing_empty { + self.allow_trailing_empty = true; + if let Some(elt) = self.next_bwd::() { + if !elt.is_empty() { + return Some(elt); + } + } + if self.finished { + return None; + } + } + + let range = if let Some((start, end)) = self.searcher.next_match_back() { + end..replace(&mut self.end, if INCLUSIVE { end } else { start }) + } else { + self.finished = true; + self.start..self.end + }; + // SAFETY: All indices come from Haystack or Searcher. + // They are known to return good indices + Some(unsafe { self.searcher.haystack().get_unchecked(range) }) + } + + /// Returns the remaining part of the haystack that hasn't been processed yet. + #[inline] + pub fn remainder(&self) -> Option { + (!self.finished).then(|| { + // SAFETY: self.start and self.end come from Haystack or Searcher + // and thus are guaranteed to be valid split positions. + unsafe { self.searcher.haystack().get_unchecked(self.start..self.end) } + }) + } + + /// Returns the final haystack part. + /// + /// Sets the `finished` flag so any further calls to this or other methods will + /// return `None`. + #[inline] + fn get_end(&mut self) -> Option { + if !self.finished { + self.finished = true; + if self.allow_trailing_empty || self.start != self.end { + // SAFETY: self.start and self.end come from Haystack or + // Searcher and thus are guaranteed to be valid split positions. + return Some(unsafe { + self.searcher.haystack().get_unchecked(self.start..self.end) + }); + } + } + None + } +} + +impl> SplitN { + /// Returns next part of the haystack or `None` if splitting is done. + /// + /// If `INCLUSIVE` is `true`, the returned value will include the matching + /// pattern. + #[inline] + pub fn next_fwd(&mut self) -> Option { + match self.dec_limit()? { + 0 => self.inner.get_end(), + _ => self.inner.next_fwd::(), + } + } + + /// Returns next looking from back of the haystack part of the haystack or + /// `None` if splitting is done. + /// + /// If `INCLUSIVE` is `true`, the returned value will include the matching + /// pattern. + #[inline] + pub fn next_bwd(&mut self) -> Option + where + S: ReverseSearcher, + { + match self.dec_limit()? { + 0 => self.inner.get_end(), + _ => self.inner.next_bwd::(), + } + } + + /// Returns the remaining part of the haystack that hasn't been processed yet. + #[inline] + pub fn remainder(&self) -> Option { + self.inner.remainder() + } + + /// Decrements limit and returns its new value or None if it's already zero. + #[inline] + fn dec_limit(&mut self) -> Option { + self.limit = self.limit.checked_sub(1)?; + Some(self.limit) + } +} diff --git a/library/core/src/str/iter.rs b/library/core/src/str/iter.rs index 26c48d48d211e..ddedf03a7063e 100644 --- a/library/core/src/str/iter.rs +++ b/library/core/src/str/iter.rs @@ -1,6 +1,5 @@ //! Iterators for `str` methods. -use super::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use super::validations::{next_code_point, next_code_point_reverse}; use super::{ BytesIsNotEmpty, CharEscapeDebugContinue, CharEscapeDefault, CharEscapeUnicode, @@ -13,6 +12,7 @@ use crate::iter::{ }; use crate::num::NonZero; use crate::ops::Try; +use crate::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::slice::{self, Split as SliceSplit}; use crate::{char as char_mod, option}; @@ -416,7 +416,7 @@ macro_rules! derive_pattern_clone { (clone $t:ident with |$s:ident| $e:expr) => { impl<'a, P> Clone for $t<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { let $s = self; @@ -429,7 +429,7 @@ macro_rules! derive_pattern_clone { /// This macro generates two public iterator structs /// wrapping a private internal one that makes use of the `Pattern` API. /// -/// For all patterns `P: Pattern` the following items will be +/// For all patterns `P: Pattern` the following items will be /// generated (generics omitted): /// /// struct $forward_iterator($internal_iterator); @@ -489,12 +489,12 @@ macro_rules! generate_pattern_iterators { } => { $(#[$forward_iterator_attribute])* $(#[$common_stability_attribute])* - pub struct $forward_iterator<'a, P: Pattern>(pub(super) $internal_iterator<'a, P>); + pub struct $forward_iterator<'a, P: Pattern<&'a str>>(pub(super) $internal_iterator<'a, P>); $(#[$common_stability_attribute])* impl<'a, P> fmt::Debug for $forward_iterator<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple(stringify!($forward_iterator)) @@ -504,7 +504,7 @@ macro_rules! generate_pattern_iterators { } $(#[$common_stability_attribute])* - impl<'a, P: Pattern> Iterator for $forward_iterator<'a, P> { + impl<'a, P: Pattern<&'a str>> Iterator for $forward_iterator<'a, P> { type Item = $iterty; #[inline] @@ -516,7 +516,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Clone for $forward_iterator<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { $forward_iterator(self.0.clone()) @@ -525,12 +525,12 @@ macro_rules! generate_pattern_iterators { $(#[$reverse_iterator_attribute])* $(#[$common_stability_attribute])* - pub struct $reverse_iterator<'a, P: Pattern>(pub(super) $internal_iterator<'a, P>); + pub struct $reverse_iterator<'a, P: Pattern<&'a str>>(pub(super) $internal_iterator<'a, P>); $(#[$common_stability_attribute])* impl<'a, P> fmt::Debug for $reverse_iterator<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple(stringify!($reverse_iterator)) @@ -542,7 +542,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Iterator for $reverse_iterator<'a, P> where - P: Pattern: ReverseSearcher<'a>>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { type Item = $iterty; @@ -555,7 +555,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> Clone for $reverse_iterator<'a, P> where - P: Pattern: Clone>, + P: Pattern<&'a str, Searcher: Clone>, { fn clone(&self) -> Self { $reverse_iterator(self.0.clone()) @@ -563,12 +563,12 @@ macro_rules! generate_pattern_iterators { } #[stable(feature = "fused", since = "1.26.0")] - impl<'a, P: Pattern> FusedIterator for $forward_iterator<'a, P> {} + impl<'a, P: Pattern<&'a str>> FusedIterator for $forward_iterator<'a, P> {} #[stable(feature = "fused", since = "1.26.0")] impl<'a, P> FusedIterator for $reverse_iterator<'a, P> where - P: Pattern: ReverseSearcher<'a>>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, {} generate_pattern_iterators!($($t)* with $(#[$common_stability_attribute])*, @@ -583,7 +583,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> DoubleEndedIterator for $forward_iterator<'a, P> where - P: Pattern: DoubleEndedSearcher<'a>>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { #[inline] fn next_back(&mut self) -> Option<$iterty> { @@ -594,7 +594,7 @@ macro_rules! generate_pattern_iterators { $(#[$common_stability_attribute])* impl<'a, P> DoubleEndedIterator for $reverse_iterator<'a, P> where - P: Pattern: DoubleEndedSearcher<'a>>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { #[inline] fn next_back(&mut self) -> Option<$iterty> { @@ -611,175 +611,66 @@ macro_rules! generate_pattern_iterators { derive_pattern_clone! { clone SplitInternal - with |s| SplitInternal { matcher: s.matcher.clone(), ..*s } + with |s| SplitInternal(s.0.clone()) } -pub(super) struct SplitInternal<'a, P: Pattern> { - pub(super) start: usize, - pub(super) end: usize, - pub(super) matcher: P::Searcher<'a>, - pub(super) allow_trailing_empty: bool, - pub(super) finished: bool, +pub(super) struct SplitInternal<'a, P: Pattern<&'a str>>( + core::pattern::Split<&'a str, P::Searcher>, +); + +impl<'a, P: Pattern<&'a str>> SplitInternal<'a, P> { + pub(super) fn new(haystack: &'a str, pattern: P) -> Self { + Self(core::pattern::Split::new(pattern.into_searcher(haystack))) + } + + pub(super) fn with_allow_trailing_empty(self) -> Self { + Self(self.0.with_allow_trailing_empty()) + } + + pub(super) fn with_limit(self, count: usize) -> SplitNInternal<'a, P> { + SplitNInternal(self.0.with_limit(count)) + } } impl<'a, P> fmt::Debug for SplitInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitInternal") - .field("start", &self.start) - .field("end", &self.end) - .field("matcher", &self.matcher) - .field("allow_trailing_empty", &self.allow_trailing_empty) - .field("finished", &self.finished) - .finish() + self.0.fmt(f) } } -impl<'a, P: Pattern> SplitInternal<'a, P> { - #[inline] - fn get_end(&mut self) -> Option<&'a str> { - if !self.finished { - self.finished = true; - - if self.allow_trailing_empty || self.end - self.start > 0 { - // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. - let string = unsafe { self.matcher.haystack().get_unchecked(self.start..self.end) }; - return Some(string); - } - } - - None - } - +impl<'a, P: Pattern<&'a str>> SplitInternal<'a, P> { #[inline] fn next(&mut self) -> Option<&'a str> { - if self.finished { - return None; - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match() { - // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. - Some((a, b)) => unsafe { - let elt = haystack.get_unchecked(self.start..a); - self.start = b; - Some(elt) - }, - None => self.get_end(), - } + self.0.next_fwd::() } #[inline] fn next_inclusive(&mut self) -> Option<&'a str> { - if self.finished { - return None; - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match() { - // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, - // and self.start is either the start of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - Some((_, b)) => unsafe { - let elt = haystack.get_unchecked(self.start..b); - self.start = b; - Some(elt) - }, - None => self.get_end(), - } + self.0.next_fwd::() } #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { - if self.finished { - return None; - } - - if !self.allow_trailing_empty { - self.allow_trailing_empty = true; - match self.next_back() { - Some(elt) if !elt.is_empty() => return Some(elt), - _ => { - if self.finished { - return None; - } - } - } - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match_back() { - // SAFETY: `Searcher` guarantees that `a` and `b` lie on unicode boundaries. - Some((a, b)) => unsafe { - let elt = haystack.get_unchecked(b..self.end); - self.end = a; - Some(elt) - }, - // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. - None => unsafe { - self.finished = true; - Some(haystack.get_unchecked(self.start..self.end)) - }, - } + self.0.next_bwd::() } #[inline] fn next_back_inclusive(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { - if self.finished { - return None; - } - - if !self.allow_trailing_empty { - self.allow_trailing_empty = true; - match self.next_back_inclusive() { - Some(elt) if !elt.is_empty() => return Some(elt), - _ => { - if self.finished { - return None; - } - } - } - } - - let haystack = self.matcher.haystack(); - match self.matcher.next_match_back() { - // SAFETY: `Searcher` guarantees that `b` lies on unicode boundary, - // and self.end is either the end of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - Some((_, b)) => unsafe { - let elt = haystack.get_unchecked(b..self.end); - self.end = b; - Some(elt) - }, - // SAFETY: self.start is either the start of the original string, - // or start of a substring that represents the part of the string that hasn't - // iterated yet. Either way, it is guaranteed to lie on unicode boundary. - // self.end is either the end of the original string, - // or `b` was assigned to it, so it also lies on unicode boundary. - None => unsafe { - self.finished = true; - Some(haystack.get_unchecked(self.start..self.end)) - }, - } + self.0.next_bwd::() } #[inline] fn remainder(&self) -> Option<&'a str> { - // `Self::get_end` doesn't change `self.start` - if self.finished { - return None; - } - - // SAFETY: `self.start` and `self.end` always lie on unicode boundaries. - Some(unsafe { self.matcher.haystack().get_unchecked(self.start..self.end) }) + self.0.remainder() } } @@ -801,7 +692,7 @@ generate_pattern_iterators! { delegate double ended; } -impl<'a, P: Pattern> Split<'a, P> { +impl<'a, P: Pattern<&'a str>> Split<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -824,7 +715,7 @@ impl<'a, P: Pattern> Split<'a, P> { } } -impl<'a, P: Pattern> RSplit<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplit<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -865,7 +756,7 @@ generate_pattern_iterators! { delegate double ended; } -impl<'a, P: Pattern> SplitTerminator<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitTerminator<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -888,7 +779,7 @@ impl<'a, P: Pattern> SplitTerminator<'a, P> { } } -impl<'a, P: Pattern> RSplitTerminator<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplitTerminator<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -913,64 +804,39 @@ impl<'a, P: Pattern> RSplitTerminator<'a, P> { derive_pattern_clone! { clone SplitNInternal - with |s| SplitNInternal { iter: s.iter.clone(), ..*s } + with |s| SplitNInternal(s.0.clone()) } -pub(super) struct SplitNInternal<'a, P: Pattern> { - pub(super) iter: SplitInternal<'a, P>, - /// The number of splits remaining - pub(super) count: usize, -} +pub(super) struct SplitNInternal<'a, P: Pattern<&'a str>>( + core::pattern::SplitN<&'a str, P::Searcher>, +); impl<'a, P> fmt::Debug for SplitNInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SplitNInternal") - .field("iter", &self.iter) - .field("count", &self.count) - .finish() + self.0.fmt(f) } } -impl<'a, P: Pattern> SplitNInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitNInternal<'a, P> { #[inline] fn next(&mut self) -> Option<&'a str> { - match self.count { - 0 => None, - 1 => { - self.count = 0; - self.iter.get_end() - } - _ => { - self.count -= 1; - self.iter.next() - } - } + self.0.next_fwd::() } #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { - match self.count { - 0 => None, - 1 => { - self.count = 0; - self.iter.get_end() - } - _ => { - self.count -= 1; - self.iter.next_back() - } - } + self.0.next_bwd::() } #[inline] fn remainder(&self) -> Option<&'a str> { - self.iter.remainder() + self.0.remainder() } } @@ -992,7 +858,7 @@ generate_pattern_iterators! { delegate single ended; } -impl<'a, P: Pattern> SplitN<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitN<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -1015,7 +881,7 @@ impl<'a, P: Pattern> SplitN<'a, P> { } } -impl<'a, P: Pattern> RSplitN<'a, P> { +impl<'a, P: Pattern<&'a str>> RSplitN<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. @@ -1043,18 +909,18 @@ derive_pattern_clone! { with |s| MatchIndicesInternal(s.0.clone()) } -pub(super) struct MatchIndicesInternal<'a, P: Pattern>(pub(super) P::Searcher<'a>); +pub(super) struct MatchIndicesInternal<'a, P: Pattern<&'a str>>(pub(super) P::Searcher); impl<'a, P> fmt::Debug for MatchIndicesInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("MatchIndicesInternal").field(&self.0).finish() } } -impl<'a, P: Pattern> MatchIndicesInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> MatchIndicesInternal<'a, P> { #[inline] fn next(&mut self) -> Option<(usize, &'a str)> { self.0 @@ -1066,7 +932,7 @@ impl<'a, P: Pattern> MatchIndicesInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<(usize, &'a str)> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { self.0 .next_match_back() @@ -1098,18 +964,18 @@ derive_pattern_clone! { with |s| MatchesInternal(s.0.clone()) } -pub(super) struct MatchesInternal<'a, P: Pattern>(pub(super) P::Searcher<'a>); +pub(super) struct MatchesInternal<'a, P: Pattern<&'a str>>(pub(super) P::Searcher); impl<'a, P> fmt::Debug for MatchesInternal<'a, P> where - P: Pattern: fmt::Debug>, + P: Pattern<&'a str, Searcher: fmt::Debug>, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("MatchesInternal").field(&self.0).finish() } } -impl<'a, P: Pattern> MatchesInternal<'a, P> { +impl<'a, P: Pattern<&'a str>> MatchesInternal<'a, P> { #[inline] fn next(&mut self) -> Option<&'a str> { // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. @@ -1122,7 +988,7 @@ impl<'a, P: Pattern> MatchesInternal<'a, P> { #[inline] fn next_back(&mut self) -> Option<&'a str> where - P::Searcher<'a>: ReverseSearcher<'a>, + P::Searcher: ReverseSearcher<&'a str>, { // SAFETY: `Searcher` guarantees that `start` and `end` lie on unicode boundaries. self.0.next_match_back().map(|(a, b)| unsafe { @@ -1293,7 +1159,7 @@ pub struct SplitAsciiWhitespace<'a> { /// /// [`split_inclusive`]: str::split_inclusive #[stable(feature = "split_inclusive", since = "1.51.0")] -pub struct SplitInclusive<'a, P: Pattern>(pub(super) SplitInternal<'a, P>); +pub struct SplitInclusive<'a, P: Pattern<&'a str>>(pub(super) SplitInternal<'a, P>); #[stable(feature = "split_whitespace", since = "1.1.0")] impl<'a> Iterator for SplitWhitespace<'a> { @@ -1415,7 +1281,7 @@ impl<'a> SplitAsciiWhitespace<'a> { } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern> Iterator for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str>> Iterator for SplitInclusive<'a, P> { type Item = &'a str; #[inline] @@ -1425,7 +1291,7 @@ impl<'a, P: Pattern> Iterator for SplitInclusive<'a, P> { } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str, Searcher: fmt::Debug>> fmt::Debug for SplitInclusive<'a, P> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("SplitInclusive").field("0", &self.0).finish() } @@ -1433,14 +1299,14 @@ impl<'a, P: Pattern: fmt::Debug>> fmt::Debug for SplitInclusive<'a, // FIXME(#26925) Remove in favor of `#[derive(Clone)]` #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: Clone>> Clone for SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str, Searcher: Clone>> Clone for SplitInclusive<'a, P> { fn clone(&self) -> Self { SplitInclusive(self.0.clone()) } } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern: DoubleEndedSearcher<'a>>> DoubleEndedIterator +impl<'a, P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>> DoubleEndedIterator for SplitInclusive<'a, P> { #[inline] @@ -1450,9 +1316,9 @@ impl<'a, P: Pattern: DoubleEndedSearcher<'a>>> DoubleEndedIterator } #[stable(feature = "split_inclusive", since = "1.51.0")] -impl<'a, P: Pattern> FusedIterator for SplitInclusive<'a, P> {} +impl<'a, P: Pattern<&'a str>> FusedIterator for SplitInclusive<'a, P> {} -impl<'a, P: Pattern> SplitInclusive<'a, P> { +impl<'a, P: Pattern<&'a str>> SplitInclusive<'a, P> { /// Returns remainder of the split string. /// /// If the iterator is empty, returns `None`. diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 79f4f29da2d43..3724e3572fe98 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -13,9 +13,9 @@ mod iter; mod traits; mod validations; -use self::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::char::{self, EscapeDebugExtArgs}; use crate::hint::assert_unchecked; +use crate::pattern::{DoubleEndedSearcher, Pattern, ReverseSearcher, Searcher}; use crate::range::Range; use crate::slice::{self, SliceIndex}; use crate::ub_checks::assert_unsafe_precondition; @@ -47,7 +47,7 @@ pub use iter::{Bytes, CharIndices, Chars, Lines, SplitWhitespace}; pub use iter::{EscapeDebug, EscapeDefault, EscapeUnicode}; #[stable(feature = "str_match_indices", since = "1.5.0")] pub use iter::{MatchIndices, RMatchIndices}; -use iter::{MatchIndicesInternal, MatchesInternal, SplitInternal, SplitNInternal}; +use iter::{MatchIndicesInternal, MatchesInternal, SplitInternal}; #[stable(feature = "str_matches", since = "1.2.0")] pub use iter::{Matches, RMatches}; #[stable(feature = "rust1", since = "1.0.0")] @@ -58,8 +58,11 @@ pub use iter::{RSplitN, SplitN}; pub use lossy::{Utf8Chunk, Utf8Chunks}; #[stable(feature = "rust1", since = "1.0.0")] pub use traits::FromStr; +pub(crate) use validations::next_code_point_reverse; #[unstable(feature = "str_internals", issue = "none")] -pub use validations::{next_code_point, utf8_char_width}; +pub use validations::{ + next_code_point, try_next_code_point, try_next_code_point_reverse, utf8_char_width, +}; #[inline(never)] #[cold] @@ -1380,7 +1383,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn contains(&self, pat: P) -> bool { + pub fn contains<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> bool { pat.is_contained_in(self) } @@ -1418,7 +1421,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_diagnostic_item = "str_starts_with"] - pub fn starts_with(&self, pat: P) -> bool { + pub fn starts_with<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> bool { pat.is_prefix_of(self) } @@ -1443,9 +1446,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_diagnostic_item = "str_ends_with"] - pub fn ends_with(&self, pat: P) -> bool + pub fn ends_with<'a, P>(&'a self, pat: P) -> bool where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { pat.is_suffix_of(self) } @@ -1494,7 +1497,7 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn find(&self, pat: P) -> Option { + pub fn find<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Option { pat.into_searcher(self).next_match().map(|(i, _)| i) } @@ -1540,9 +1543,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rfind(&self, pat: P) -> Option + pub fn rfind<'a, P>(&'a self, pat: P) -> Option where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { pat.into_searcher(self).next_match_back().map(|(i, _)| i) } @@ -1668,14 +1671,8 @@ impl str { /// [`split_whitespace`]: str::split_whitespace #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn split(&self, pat: P) -> Split<'_, P> { - Split(SplitInternal { - start: 0, - end: self.len(), - matcher: pat.into_searcher(self), - allow_trailing_empty: true, - finished: false, - }) + pub fn split<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Split<'a, P> { + Split(SplitInternal::new(self, pat).with_allow_trailing_empty()) } /// Returns an iterator over substrings of this string slice, separated by @@ -1709,14 +1706,8 @@ impl str { /// ``` #[stable(feature = "split_inclusive", since = "1.51.0")] #[inline] - pub fn split_inclusive(&self, pat: P) -> SplitInclusive<'_, P> { - SplitInclusive(SplitInternal { - start: 0, - end: self.len(), - matcher: pat.into_searcher(self), - allow_trailing_empty: false, - finished: false, - }) + pub fn split_inclusive<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> SplitInclusive<'a, P> { + SplitInclusive(SplitInternal::new(self, pat)) } /// Returns an iterator over substrings of the given string slice, separated @@ -1764,9 +1755,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplit(&self, pat: P) -> RSplit<'_, P> + pub fn rsplit<'a, P>(&'a self, pat: P) -> RSplit<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplit(self.split(pat).0) } @@ -1813,8 +1804,8 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn split_terminator(&self, pat: P) -> SplitTerminator<'_, P> { - SplitTerminator(SplitInternal { allow_trailing_empty: false, ..self.split(pat).0 }) + pub fn split_terminator<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> SplitTerminator<'a, P> { + SplitTerminator(SplitInternal::new(self, pat)) } /// Returns an iterator over substrings of `self`, separated by characters @@ -1859,9 +1850,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplit_terminator(&self, pat: P) -> RSplitTerminator<'_, P> + pub fn rsplit_terminator<'a, P>(&'a self, pat: P) -> RSplitTerminator<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplitTerminator(self.split_terminator(pat).0) } @@ -1914,8 +1905,8 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn splitn(&self, n: usize, pat: P) -> SplitN<'_, P> { - SplitN(SplitNInternal { iter: self.split(pat).0, count: n }) + pub fn splitn<'a, P: Pattern<&'a str>>(&'a self, n: usize, pat: P) -> SplitN<'a, P> { + SplitN(self.split(pat).0.with_limit(n)) } /// Returns an iterator over substrings of this string slice, separated by a @@ -1963,9 +1954,9 @@ impl str { /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] - pub fn rsplitn(&self, n: usize, pat: P) -> RSplitN<'_, P> + pub fn rsplitn<'a, P>(&'a self, n: usize, pat: P) -> RSplitN<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RSplitN(self.splitn(n, pat).0) } @@ -1983,7 +1974,10 @@ impl str { /// ``` #[stable(feature = "str_split_once", since = "1.52.0")] #[inline] - pub fn split_once(&self, delimiter: P) -> Option<(&'_ str, &'_ str)> { + pub fn split_once<'a, P>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> + where + P: Pattern<&'a str>, + { let (start, end) = delimiter.into_searcher(self).next_match()?; // SAFETY: `Searcher` is known to return valid indices. unsafe { Some((self.get_unchecked(..start), self.get_unchecked(end..))) } @@ -2002,9 +1996,9 @@ impl str { /// ``` #[stable(feature = "str_split_once", since = "1.52.0")] #[inline] - pub fn rsplit_once(&self, delimiter: P) -> Option<(&'_ str, &'_ str)> + pub fn rsplit_once<'a, P>(&'a self, delimiter: P) -> Option<(&'a str, &'a str)> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let (start, end) = delimiter.into_searcher(self).next_match_back()?; // SAFETY: `Searcher` is known to return valid indices. @@ -2042,7 +2036,7 @@ impl str { /// ``` #[stable(feature = "str_matches", since = "1.2.0")] #[inline] - pub fn matches(&self, pat: P) -> Matches<'_, P> { + pub fn matches<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> Matches<'a, P> { Matches(MatchesInternal(pat.into_searcher(self))) } @@ -2076,9 +2070,9 @@ impl str { /// ``` #[stable(feature = "str_matches", since = "1.2.0")] #[inline] - pub fn rmatches(&self, pat: P) -> RMatches<'_, P> + pub fn rmatches<'a, P>(&'a self, pat: P) -> RMatches<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RMatches(self.matches(pat).0) } @@ -2120,7 +2114,10 @@ impl str { /// ``` #[stable(feature = "str_match_indices", since = "1.5.0")] #[inline] - pub fn match_indices(&self, pat: P) -> MatchIndices<'_, P> { + pub fn match_indices<'a, P>(&'a self, pat: P) -> MatchIndices<'a, P> + where + P: Pattern<&'a str>, + { MatchIndices(MatchIndicesInternal(pat.into_searcher(self))) } @@ -2160,9 +2157,9 @@ impl str { /// ``` #[stable(feature = "str_match_indices", since = "1.5.0")] #[inline] - pub fn rmatch_indices(&self, pat: P) -> RMatchIndices<'_, P> + pub fn rmatch_indices<'a, P>(&'a self, pat: P) -> RMatchIndices<'a, P> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { RMatchIndices(self.match_indices(pat).0) } @@ -2375,9 +2372,9 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "rust1", since = "1.0.0")] - pub fn trim_matches(&self, pat: P) -> &str + pub fn trim_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: DoubleEndedSearcher<'a>, + P: Pattern<&'a str, Searcher: DoubleEndedSearcher<&'a str>>, { let mut i = 0; let mut j = 0; @@ -2422,7 +2419,7 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] - pub fn trim_start_matches(&self, pat: P) -> &str { + pub fn trim_start_matches<'a, P: Pattern<&'a str>>(&'a self, pat: P) -> &'a str { let mut i = self.len(); let mut matcher = pat.into_searcher(self); if let Some((a, _)) = matcher.next_reject() { @@ -2456,7 +2453,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "str_strip", since = "1.45.0")] - pub fn strip_prefix(&self, prefix: P) -> Option<&str> { + pub fn strip_prefix<'a, P>(&'a self, prefix: P) -> Option<&'a str> + where + P: Pattern<&'a str>, + { prefix.strip_prefix_of(self) } @@ -2484,9 +2484,9 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "str_strip", since = "1.45.0")] - pub fn strip_suffix(&self, suffix: P) -> Option<&str> + pub fn strip_suffix<'a, P>(&'a self, suffix: P) -> Option<&'a str> where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { suffix.strip_suffix_of(self) } @@ -2521,9 +2521,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[stable(feature = "strip_circumfix", since = "1.98.0")] - pub fn strip_circumfix(&self, prefix: P, suffix: S) -> Option<&str> + pub fn strip_circumfix<'a, P, S>(&'a self, prefix: P, suffix: S) -> Option<&'a str> where - for<'a> S::Searcher<'a>: ReverseSearcher<'a>, + S: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, + P: Pattern<&'a str>, { self.strip_prefix(prefix)?.strip_suffix(suffix) } @@ -2561,7 +2562,10 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[unstable(feature = "trim_prefix_suffix", issue = "142312")] - pub fn trim_prefix(&self, prefix: P) -> &str { + pub fn trim_prefix<'a, P>(&'a self, prefix: P) -> &'a str + where + P: Pattern<&'a str>, + { prefix.strip_prefix_of(self).unwrap_or(self) } @@ -2598,9 +2602,9 @@ impl str { #[must_use = "this returns the remaining substring as a new slice, \ without modifying the original"] #[unstable(feature = "trim_prefix_suffix", issue = "142312")] - pub fn trim_suffix(&self, suffix: P) -> &str + pub fn trim_suffix<'a, P>(&'a self, suffix: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { suffix.strip_suffix_of(self).unwrap_or(self) } @@ -2641,9 +2645,9 @@ impl str { #[must_use = "this returns the trimmed string as a new slice, \ without modifying the original"] #[stable(feature = "trim_direction", since = "1.30.0")] - pub fn trim_end_matches(&self, pat: P) -> &str + pub fn trim_end_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { let mut j = 0; let mut matcher = pat.into_searcher(self); @@ -2685,7 +2689,10 @@ impl str { note = "superseded by `trim_start_matches`", suggestion = "trim_start_matches" )] - pub fn trim_left_matches(&self, pat: P) -> &str { + pub fn trim_left_matches<'a, P>(&'a self, pat: P) -> &'a str + where + P: Pattern<&'a str>, + { self.trim_start_matches(pat) } @@ -2728,9 +2735,9 @@ impl str { note = "superseded by `trim_end_matches`", suggestion = "trim_end_matches" )] - pub fn trim_right_matches(&self, pat: P) -> &str + pub fn trim_right_matches<'a, P>(&'a self, pat: P) -> &'a str where - for<'a> P::Searcher<'a>: ReverseSearcher<'a>, + P: Pattern<&'a str, Searcher: ReverseSearcher<&'a str>>, { self.trim_end_matches(pat) } diff --git a/library/core/src/str/pattern.rs b/library/core/src/str/pattern.rs index af48c2eae8dd6..8d2477d4da2d4 100644 --- a/library/core/src/str/pattern.rs +++ b/library/core/src/str/pattern.rs @@ -1,36 +1,47 @@ -//! The string Pattern API. +//! [The Pattern API] implementation for searching in `&str`. //! -//! The Pattern API provides a generic mechanism for using different pattern -//! types when searching through a string. +//! The implementation provides generic mechanism for using different pattern +//! types when searching through a string. Although this API is unstable, it is +//! exposed via stable APIs on the [`str`] type. //! -//! For more details, see the traits [`Pattern`], [`Searcher`], -//! [`ReverseSearcher`], and [`DoubleEndedSearcher`]. +//! Depending on the type of the pattern, the behaviour of methods like +//! [`str::find`] and [`str::contains`] can change. The table below describes +//! some of those behaviours. //! -//! Although this API is unstable, it is exposed via stable APIs on the -//! [`str`] type. +//! | Pattern type | Match condition | +//! |--------------------------|-------------------------------------------| +//! | `&str` | is substring | +//! | `char` | is contained in string | +//! | `&[char]` | any char in slice is contained in string | +//! | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | +//! | `&&str` | is substring | +//! | `&String` | is substring | //! //! # Examples //! -//! [`Pattern`] is [implemented][pattern-impls] in the stable API for -//! [`&str`][`str`], [`char`], slices of [`char`], and functions and closures -//! implementing `FnMut(char) -> bool`. -//! //! ``` //! let s = "Can you find a needle in a haystack?"; //! //! // &str pattern //! assert_eq!(s.find("you"), Some(4)); +//! assert_eq!(s.find("thou"), None); +//! //! // char pattern //! assert_eq!(s.find('n'), Some(2)); -//! // array of chars pattern +//! assert_eq!(s.find('N'), None); +//! +//! // Array of chars pattern and slices thereof //! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u']), Some(1)); -//! // slice of chars pattern //! assert_eq!(s.find(&['a', 'e', 'i', 'o', 'u'][..]), Some(1)); -//! // closure pattern +//! assert_eq!(s.find(&['q', 'v', 'x']), None); +//! +//! // Predicate closure //! assert_eq!(s.find(|c: char| c.is_ascii_punctuation()), Some(35)); +//! assert_eq!(s.find(|c: char| c.is_lowercase()), Some(1)); +//! assert_eq!(s.find(|c: char| !c.is_ascii()), None); //! ``` //! -//! [pattern-impls]: Pattern#implementors +//! [The Pattern API]: crate::pattern #![unstable( feature = "pattern", @@ -39,518 +50,89 @@ )] use crate::cmp::Ordering; -use crate::convert::TryInto as _; -use crate::slice::memchr; -use crate::{cmp, fmt}; - -// Pattern - -/// A string pattern. -/// -/// A `Pattern` expresses that the implementing type -/// can be used as a string pattern for searching in a [`&str`][str]. -/// -/// For example, both `'a'` and `"aa"` are patterns that -/// would match at index `1` in the string `"baaaab"`. -/// -/// The trait itself acts as a builder for an associated -/// [`Searcher`] type, which does the actual work of finding -/// occurrences of the pattern in a string. -/// -/// Depending on the type of the pattern, the behavior of methods like -/// [`str::find`] and [`str::contains`] can change. The table below describes -/// some of those behaviors. -/// -/// | Pattern type | Match condition | -/// |--------------------------|-------------------------------------------| -/// | `&str` | is substring | -/// | `char` | is contained in string | -/// | `&[char]` | any char in slice is contained in string | -/// | `F: FnMut(char) -> bool` | `F` returns `true` for a char in string | -/// | `&&str` | is substring | -/// | `&String` | is substring | -/// -/// # Examples -/// -/// ``` -/// // &str -/// assert_eq!("abaaa".find("ba"), Some(1)); -/// assert_eq!("abaaa".find("bac"), None); -/// -/// // char -/// assert_eq!("abaaa".find('a'), Some(0)); -/// assert_eq!("abaaa".find('b'), Some(1)); -/// assert_eq!("abaaa".find('c'), None); -/// -/// // &[char; N] -/// assert_eq!("ab".find(&['b', 'a']), Some(0)); -/// assert_eq!("abaaa".find(&['a', 'z']), Some(0)); -/// assert_eq!("abaaa".find(&['c', 'd']), None); -/// -/// // &[char] -/// assert_eq!("ab".find(&['b', 'a'][..]), Some(0)); -/// assert_eq!("abaaa".find(&['a', 'z'][..]), Some(0)); -/// assert_eq!("abaaa".find(&['c', 'd'][..]), None); -/// -/// // FnMut(char) -> bool -/// assert_eq!("abcdef_z".find(|ch| ch > 'd' && ch < 'y'), Some(4)); -/// assert_eq!("abcddd_z".find(|ch| ch > 'd' && ch < 'y'), None); -/// ``` -pub trait Pattern: Sized { - /// Associated searcher for this pattern - type Searcher<'a>: Searcher<'a>; - - /// Constructs the associated searcher from - /// `self` and the `haystack` to search in. - fn into_searcher(self, haystack: &str) -> Self::Searcher<'_>; - - /// Checks whether the pattern matches anywhere in the haystack - #[inline] - fn is_contained_in(self, haystack: &str) -> bool { - self.into_searcher(haystack).next_match().is_some() - } - - /// Checks whether the pattern matches at the front of the haystack - #[inline] - fn is_prefix_of(self, haystack: &str) -> bool { - matches!(self.into_searcher(haystack).next(), SearchStep::Match(0, _)) - } - - /// Checks whether the pattern matches at the back of the haystack - #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { - matches!(self.into_searcher(haystack).next_back(), SearchStep::Match(_, j) if haystack.len() == j) - } - - /// Removes the pattern from the front of haystack, if it matches. - #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { - if let SearchStep::Match(start, len) = self.into_searcher(haystack).next() { - debug_assert_eq!( - start, 0, - "The first search step from Searcher \ - must include the first character" - ); - // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(haystack.get_unchecked(len..)) } - } else { - None - } - } - - /// Removes the pattern from the back of haystack, if it matches. - #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { - if let SearchStep::Match(start, end) = self.into_searcher(haystack).next_back() { - debug_assert_eq!( - end, - haystack.len(), - "The first search step from ReverseSearcher \ - must include the last character" - ); - // SAFETY: `Searcher` is known to return valid indices. - unsafe { Some(haystack.get_unchecked(..start)) } - } else { - None - } - } - - /// Returns the pattern as UTF-8 if possible. - fn as_utf8_pattern(&self) -> Option> { - None - } -} -/// Result of calling [`Pattern::as_utf8_pattern()`]. -/// Can be used for inspecting the contents of a [`Pattern`] in cases -/// where the underlying representation can be represented as UTF-8. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum Utf8Pattern<'a> { - /// Type returned by String and str types. - /// This stores `str` rather than bytes so callers cannot describe - /// non-UTF-8 string patterns through this API. - StringPattern(&'a str), - /// Type returned by char types. - CharPattern(char), -} +use crate::ops::Range; +pub use crate::pattern::{ + DoubleEndedSearcher, Haystack, MatchOnly, Pattern, ReverseSearcher, SearchResult, SearchStep, + Searcher, Utf8Pattern, +}; +use crate::{fmt, str_bytes}; -// Searcher - -/// Result of calling [`Searcher::next()`] or [`ReverseSearcher::next_back()`]. -#[derive(Copy, Clone, Eq, PartialEq, Debug)] -pub enum SearchStep { - /// Expresses that a match of the pattern has been found at - /// `haystack[a..b]`. - Match(usize, usize), - /// Expresses that `haystack[a..b]` has been rejected as a possible match - /// of the pattern. - /// - /// Note that there might be more than one `Reject` between two `Match`es, - /// there is no requirement for them to be combined into one. - Reject(usize, usize), - /// Expresses that every byte of the haystack has been visited, ending - /// the iteration. - Done, -} +///////////////////////////////////////////////////////////////////////////// +// Impl for Haystack +///////////////////////////////////////////////////////////////////////////// -/// A searcher for a string pattern. -/// -/// This trait provides methods for searching for non-overlapping -/// matches of a pattern starting from the front (left) of a string. -/// -/// It will be implemented by associated `Searcher` -/// types of the [`Pattern`] trait. -/// -/// The trait is marked unsafe because the indices returned by the -/// [`next()`][Searcher::next] methods are required to lie on valid utf8 -/// boundaries in the haystack. This enables consumers of this trait to -/// slice the haystack without additional runtime checks. -pub unsafe trait Searcher<'a> { - /// Getter for the underlying string to be searched in - /// - /// Will always return the same [`&str`][str]. - fn haystack(&self) -> &'a str; - - /// Performs the next search step starting from the front. - /// - /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` matches - /// the pattern. - /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` can - /// not match the pattern, even partially. - /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack has - /// been visited. - /// - /// The stream of [`Match`][SearchStep::Match] and - /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] - /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on utf8 boundaries. - /// - /// A [`Match`][SearchStep::Match] result needs to contain the whole matched - /// pattern, however [`Reject`][SearchStep::Reject] results may be split up - /// into arbitrary many adjacent fragments. Both ranges may have zero length. - /// - /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` - /// might produce the stream - /// `[Reject(0, 1), Reject(1, 2), Match(2, 5), Reject(5, 8)]` - fn next(&mut self) -> SearchStep; - - /// Finds the next [`Match`][SearchStep::Match] result. See [`next()`][Searcher::next]. - /// - /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges - /// of this and [`next_reject`][Searcher::next_reject] will overlap. This will return - /// `(start_match, end_match)`, where start_match is the index of where - /// the match begins, and end_match is the index after the end of the match. - #[inline] - fn next_match(&mut self) -> Option<(usize, usize)> { - loop { - match self.next() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } +impl<'a> Haystack for &'a str { + #[inline(always)] + fn cursor_at_front(self) -> usize { + 0 } - - /// Finds the next [`Reject`][SearchStep::Reject] result. See [`next()`][Searcher::next] - /// and [`next_match()`][Searcher::next_match]. - /// - /// Unlike [`next()`][Searcher::next], there is no guarantee that the returned ranges - /// of this and [`next_match`][Searcher::next_match] will overlap. - #[inline] - fn next_reject(&mut self) -> Option<(usize, usize)> { - loop { - match self.next() { - SearchStep::Reject(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } + #[inline(always)] + fn cursor_at_back(self) -> usize { + self.len() } -} -/// A reverse searcher for a string pattern. -/// -/// This trait provides methods for searching for non-overlapping -/// matches of a pattern starting from the back (right) of a string. -/// -/// It will be implemented by associated [`Searcher`] -/// types of the [`Pattern`] trait if the pattern supports searching -/// for it from the back. -/// -/// The index ranges returned by this trait are not required -/// to exactly match those of the forward search in reverse. -/// -/// For the reason why this trait is marked unsafe, see the -/// parent trait [`Searcher`]. -pub unsafe trait ReverseSearcher<'a>: Searcher<'a> { - /// Performs the next search step starting from the back. - /// - /// - Returns [`Match(a, b)`][SearchStep::Match] if `haystack[a..b]` - /// matches the pattern. - /// - Returns [`Reject(a, b)`][SearchStep::Reject] if `haystack[a..b]` - /// can not match the pattern, even partially. - /// - Returns [`Done`][SearchStep::Done] if every byte of the haystack - /// has been visited - /// - /// The stream of [`Match`][SearchStep::Match] and - /// [`Reject`][SearchStep::Reject] values up to a [`Done`][SearchStep::Done] - /// will contain index ranges that are adjacent, non-overlapping, - /// covering the whole haystack, and laying on utf8 boundaries. - /// - /// A [`Match`][SearchStep::Match] result needs to contain the whole matched - /// pattern, however [`Reject`][SearchStep::Reject] results may be split up - /// into arbitrary many adjacent fragments. Both ranges may have zero length. - /// - /// As an example, the pattern `"aaa"` and the haystack `"cbaaaaab"` - /// might produce the stream - /// `[Reject(7, 8), Match(4, 7), Reject(1, 4), Reject(0, 1)]`. - fn next_back(&mut self) -> SearchStep; - - /// Finds the next [`Match`][SearchStep::Match] result. - /// See [`next_back()`][ReverseSearcher::next_back]. - #[inline] - fn next_match_back(&mut self) -> Option<(usize, usize)> { - loop { - match self.next_back() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } + #[inline(always)] + fn is_empty(self) -> bool { + self.is_empty() } - /// Finds the next [`Reject`][SearchStep::Reject] result. - /// See [`next_back()`][ReverseSearcher::next_back]. - #[inline] - fn next_reject_back(&mut self) -> Option<(usize, usize)> { - loop { - match self.next_back() { - SearchStep::Reject(a, b) => return Some((a, b)), - SearchStep::Done => return None, - _ => continue, - } - } + #[inline(always)] + unsafe fn get_unchecked(self, range: Range) -> Self { + // SAFETY: Caller promises position is a character boundary. + unsafe { self.get_unchecked(range) } } } -/// A marker trait to express that a [`ReverseSearcher`] -/// can be used for a [`DoubleEndedIterator`] implementation. -/// -/// For this, the impl of [`Searcher`] and [`ReverseSearcher`] need -/// to follow these conditions: -/// -/// - All results of `next()` need to be identical -/// to the results of `next_back()` in reverse order. -/// - `next()` and `next_back()` need to behave as -/// the two ends of a range of values, that is they -/// can not "walk past each other". -/// -/// # Examples -/// -/// `char::Searcher` is a `DoubleEndedSearcher` because searching for a -/// [`char`] only requires looking at one at a time, which behaves the same -/// from both ends. -/// -/// `(&str)::Searcher` is not a `DoubleEndedSearcher` because -/// the pattern `"aa"` in the haystack `"aaa"` matches as either -/// `"[aa]a"` or `"a[aa]"`, depending on which side it is searched. -pub trait DoubleEndedSearcher<'a>: ReverseSearcher<'a> {} - ///////////////////////////////////////////////////////////////////////////// // Impl for char ///////////////////////////////////////////////////////////////////////////// -/// Associated type for `::Searcher<'a>`. +/// Associated type for `>::Searcher`. #[derive(Clone, Debug)] -pub struct CharSearcher<'a> { - haystack: &'a str, - // safety invariant: `finger`/`finger_back` must be a valid utf8 byte index of `haystack` - // This invariant can be broken *within* next_match and next_match_back, however - // they must exit with fingers on valid code point boundaries. - /// `finger` is the current byte index of the forward search. - /// Imagine that it exists before the byte at its index, i.e. - /// `haystack[finger]` is the first byte of the slice we must inspect during - /// forward searching - finger: usize, - /// `finger_back` is the current byte index of the reverse search. - /// Imagine that it exists after the byte at its index, i.e. - /// haystack[finger_back - 1] is the last byte of the slice we must inspect during - /// forward searching (and thus the first byte to be inspected when calling next_back()). - finger_back: usize, - /// The character being searched for - needle: char, - - // safety invariant: `utf8_size` must be less than 5 - /// The number of bytes `needle` takes up when encoded in utf8. - utf8_size: u8, - /// A utf8 encoded copy of the `needle` - utf8_encoded: [u8; 4], -} +pub struct CharSearcher<'a>(str_bytes::CharSearcher<'a, str_bytes::Utf8>); -impl CharSearcher<'_> { - fn utf8_size(&self) -> usize { - self.utf8_size.into() +impl<'a> CharSearcher<'a> { + #[inline] + fn new(haystack: &'a str, chr: char) -> Self { + Self(str_bytes::CharSearcher::new(str_bytes::Bytes::from_str(haystack), chr)) } } -unsafe impl<'a> Searcher<'a> for CharSearcher<'a> { +unsafe impl<'a> Searcher<&'a str> for CharSearcher<'a> { #[inline] fn haystack(&self) -> &'a str { - self.haystack + self.0.haystack().into_str() } #[inline] fn next(&mut self) -> SearchStep { - let old_finger = self.finger; - // SAFETY: 1-4 guarantee safety of `get_unchecked` - // 1. `self.finger` and `self.finger_back` are kept on unicode boundaries - // (this is invariant) - // 2. `self.finger >= 0` since it starts at 0 and only increases - // 3. `self.finger < self.finger_back` because otherwise the char `iter` - // would return `SearchStep::Done` - // 4. `self.finger` comes before the end of the haystack because `self.finger_back` - // starts at the end and only decreases - let slice = unsafe { self.haystack.get_unchecked(old_finger..self.finger_back) }; - let mut iter = slice.chars(); - let old_len = iter.iter.len(); - if let Some(ch) = iter.next() { - // add byte offset of current character - // without re-encoding as utf-8 - self.finger += old_len - iter.iter.len(); - if ch == self.needle { - SearchStep::Match(old_finger, self.finger) - } else { - SearchStep::Reject(old_finger, self.finger) - } - } else { - SearchStep::Done - } + self.0.next() } #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { - loop { - // get the haystack after the last character found - let bytes = self.haystack.as_bytes().get(self.finger..self.finger_back)?; - // the last byte of the utf8 encoded needle - // SAFETY: we have an invariant that `utf8_size < 5` - let last_byte = unsafe { *self.utf8_encoded.get_unchecked(self.utf8_size() - 1) }; - if let Some(index) = memchr::memchr(last_byte, bytes) { - // The new finger is the index of the byte we found, - // plus one, since we memchr'd for the last byte of the character. - // - // Note that this doesn't always give us a finger on a UTF8 boundary. - // If we *didn't* find our character - // we may have indexed to the non-last byte of a 3-byte or 4-byte character. - // We can't just skip to the next valid starting byte because a character like - // ꁁ (U+A041 YI SYLLABLE PA), utf-8 `EA 81 81` will have us always find - // the second byte when searching for the third. - // - // However, this is totally okay. While we have the invariant that - // self.finger is on a UTF8 boundary, this invariant is not relied upon - // within this method (it is relied upon in CharSearcher::next()). - // - // We only exit this method when we reach the end of the string, or if we - // find something. When we find something the `finger` will be set - // to a UTF8 boundary. - self.finger += index + 1; - if self.finger >= self.utf8_size() { - let found_char = self.finger - self.utf8_size(); - if let Some(slice) = self.haystack.as_bytes().get(found_char..self.finger) { - if slice == &self.utf8_encoded[0..self.utf8_size()] { - return Some((found_char, self.finger)); - } - } - } - } else { - // found nothing, exit - self.finger = self.finger_back; - return None; - } - } + self.0.next_match() + } + #[inline] + fn next_reject(&mut self) -> Option<(usize, usize)> { + self.0.next_reject() } - - // let next_reject use the default implementation from the Searcher trait } -unsafe impl<'a> ReverseSearcher<'a> for CharSearcher<'a> { +unsafe impl<'a> ReverseSearcher<&'a str> for CharSearcher<'a> { #[inline] fn next_back(&mut self) -> SearchStep { - let old_finger = self.finger_back; - // SAFETY: see the comment for next() above - let slice = unsafe { self.haystack.get_unchecked(self.finger..old_finger) }; - let mut iter = slice.chars(); - let old_len = iter.iter.len(); - if let Some(ch) = iter.next_back() { - // subtract byte offset of current character - // without re-encoding as utf-8 - self.finger_back -= old_len - iter.iter.len(); - if ch == self.needle { - SearchStep::Match(self.finger_back, old_finger) - } else { - SearchStep::Reject(self.finger_back, old_finger) - } - } else { - SearchStep::Done - } + self.0.next_back() } #[inline] fn next_match_back(&mut self) -> Option<(usize, usize)> { - let haystack = self.haystack.as_bytes(); - loop { - // get the haystack up to but not including the last character searched - let bytes = haystack.get(self.finger..self.finger_back)?; - // the last byte of the utf8 encoded needle - // SAFETY: we have an invariant that `utf8_size < 5` - let last_byte = unsafe { *self.utf8_encoded.get_unchecked(self.utf8_size() - 1) }; - if let Some(index) = memchr::memrchr(last_byte, bytes) { - // we searched a slice that was offset by self.finger, - // add self.finger to recoup the original index - let index = self.finger + index; - // memrchr will return the index of the byte we wish to - // find. In case of an ASCII character, this is indeed - // were we wish our new finger to be ("after" the found - // char in the paradigm of reverse iteration). For - // multibyte chars we need to skip down by the number of more - // bytes they have than ASCII - let shift = self.utf8_size() - 1; - if index >= shift { - let found_char = index - shift; - if let Some(slice) = haystack.get(found_char..(found_char + self.utf8_size())) { - if slice == &self.utf8_encoded[0..self.utf8_size()] { - // move finger to before the character found (i.e., at its start index) - self.finger_back = found_char; - return Some((self.finger_back, self.finger_back + self.utf8_size())); - } - } - } - // We can't use finger_back = index - size + 1 here. If we found the last char - // of a different-sized character (or the middle byte of a different character) - // we need to bump the finger_back down to `index`. This similarly makes - // `finger_back` have the potential to no longer be on a boundary, - // but this is OK since we only exit this function on a boundary - // or when the haystack has been searched completely. - // - // Unlike next_match this does not - // have the problem of repeated bytes in utf-8 because - // we're searching for the last byte, and we can only have - // found the last byte when searching in reverse. - self.finger_back = index; - } else { - self.finger_back = self.finger; - // found nothing, exit - return None; - } - } + self.0.next_match_back() + } + #[inline] + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + self.0.next_reject_back() } - - // let next_reject_back use the default implementation from the Searcher trait } -impl<'a> DoubleEndedSearcher<'a> for CharSearcher<'a> {} +impl<'a> DoubleEndedSearcher<&'a str> for CharSearcher<'a> {} /// Searches for chars that are equal to a given [`char`]. /// @@ -558,37 +140,19 @@ impl<'a> DoubleEndedSearcher<'a> for CharSearcher<'a> {} /// /// ``` /// assert_eq!("Hello world".find('o'), Some(4)); +/// assert_eq!("Hello world".find('x'), None); /// ``` -impl Pattern for char { - type Searcher<'a> = CharSearcher<'a>; +impl<'a> Pattern<&'a str> for char { + type Searcher = CharSearcher<'a>; #[inline] - fn into_searcher<'a>(self, haystack: &'a str) -> Self::Searcher<'a> { - let mut utf8_encoded = [0; char::MAX_LEN_UTF8]; - let utf8_size = self - .encode_utf8(&mut utf8_encoded) - .len() - .try_into() - .expect("char len should be less than 255"); - - CharSearcher { - haystack, - finger: 0, - finger_back: haystack.len(), - needle: self, - utf8_size, - utf8_encoded, - } + fn into_searcher(self, haystack: &'a str) -> Self::Searcher { + CharSearcher::new(haystack, self) } #[inline] - fn is_contained_in(self, haystack: &str) -> bool { - if (self as u32) < 128 { - haystack.as_bytes().contains(&(self as u8)) - } else { - let mut buffer = [0u8; 4]; - self.encode_utf8(&mut buffer).is_contained_in(haystack) - } + fn is_contained_in(self, haystack: &'a str) -> bool { + self.encode_utf8(&mut [0u8; 4]).is_contained_in(haystack) } #[inline] @@ -597,23 +161,17 @@ impl Pattern for char { } #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { self.encode_utf8(&mut [0u8; 4]).strip_prefix_of(haystack) } #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { + fn is_suffix_of(self, haystack: &'a str) -> bool { self.encode_utf8(&mut [0u8; 4]).is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { self.encode_utf8(&mut [0u8; 4]).strip_suffix_of(haystack) } @@ -672,16 +230,16 @@ struct MultiCharEqSearcher<'a, C: MultiCharEq> { char_indices: super::CharIndices<'a>, } -impl Pattern for MultiCharEqPattern { - type Searcher<'a> = MultiCharEqSearcher<'a, C>; +impl<'a, C: MultiCharEq> Pattern<&'a str> for MultiCharEqPattern { + type Searcher = MultiCharEqSearcher<'a, C>; #[inline] - fn into_searcher(self, haystack: &str) -> MultiCharEqSearcher<'_, C> { + fn into_searcher(self, haystack: &'a str) -> MultiCharEqSearcher<'a, C> { MultiCharEqSearcher { haystack, char_eq: self.0, char_indices: haystack.char_indices() } } } -unsafe impl<'a, C: MultiCharEq> Searcher<'a> for MultiCharEqSearcher<'a, C> { +unsafe impl<'a, C: MultiCharEq> Searcher<&'a str> for MultiCharEqSearcher<'a, C> { #[inline] fn haystack(&self) -> &'a str { self.haystack @@ -706,7 +264,7 @@ unsafe impl<'a, C: MultiCharEq> Searcher<'a> for MultiCharEqSearcher<'a, C> { } } -unsafe impl<'a, C: MultiCharEq> ReverseSearcher<'a> for MultiCharEqSearcher<'a, C> { +unsafe impl<'a, C: MultiCharEq> ReverseSearcher<&'a str> for MultiCharEqSearcher<'a, C> { #[inline] fn next_back(&mut self) -> SearchStep { let s = &mut self.char_indices; @@ -726,46 +284,46 @@ unsafe impl<'a, C: MultiCharEq> ReverseSearcher<'a> for MultiCharEqSearcher<'a, } } -impl<'a, C: MultiCharEq> DoubleEndedSearcher<'a> for MultiCharEqSearcher<'a, C> {} +impl<'a, C: MultiCharEq> DoubleEndedSearcher<&'a str> for MultiCharEqSearcher<'a, C> {} ///////////////////////////////////////////////////////////////////////////// macro_rules! pattern_methods { ($a:lifetime, $t:ty, $pmap:expr, $smap:expr) => { - type Searcher<$a> = $t; + type Searcher = $t; #[inline] - fn into_searcher<$a>(self, haystack: &$a str) -> $t { + fn into_searcher(self, haystack: &$a str) -> $t { ($smap)(($pmap)(self).into_searcher(haystack)) } #[inline] - fn is_contained_in<$a>(self, haystack: &$a str) -> bool { + fn is_contained_in(self, haystack: &$a str) -> bool { ($pmap)(self).is_contained_in(haystack) } #[inline] - fn is_prefix_of<$a>(self, haystack: &$a str) -> bool { + fn is_prefix_of(self, haystack: &$a str) -> bool { ($pmap)(self).is_prefix_of(haystack) } #[inline] - fn strip_prefix_of<$a>(self, haystack: &$a str) -> Option<&$a str> { + fn strip_prefix_of(self, haystack: &$a str) -> Option<&$a str> { ($pmap)(self).strip_prefix_of(haystack) } #[inline] - fn is_suffix_of<$a>(self, haystack: &$a str) -> bool + fn is_suffix_of(self, haystack: &$a str) -> bool where - $t: ReverseSearcher<$a>, + $t: ReverseSearcher<&$a str>, { ($pmap)(self).is_suffix_of(haystack) } #[inline] - fn strip_suffix_of<$a>(self, haystack: &$a str) -> Option<&$a str> + fn strip_suffix_of(self, haystack: &$a str) -> Option<&$a str> where - $t: ReverseSearcher<$a>, + $t: ReverseSearcher<&$a str>, { ($pmap)(self).strip_suffix_of(haystack) } @@ -807,16 +365,16 @@ macro_rules! searcher_methods { }; } -/// Associated type for `<[char; N] as Pattern>::Searcher<'a>`. +/// Associated type for `<[char; N] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] pub struct CharArraySearcher<'a, const N: usize>( - as Pattern>::Searcher<'a>, + as Pattern<&'a str>>::Searcher, ); -/// Associated type for `<&[char; N] as Pattern>::Searcher<'a>`. +/// Associated type for `<&[char; N] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] pub struct CharArrayRefSearcher<'a, 'b, const N: usize>( - as Pattern>::Searcher<'a>, + as Pattern<&'a str>>::Searcher, ); /// Searches for chars that are equal to any of the [`char`]s in the array. @@ -827,19 +385,19 @@ pub struct CharArrayRefSearcher<'a, 'b, const N: usize>( /// assert_eq!("Hello world".find(['o', 'l']), Some(2)); /// assert_eq!("Hello world".find(['h', 'w']), Some(6)); /// ``` -impl Pattern for [char; N] { +impl<'a, const N: usize> Pattern<&'a str> for [char; N] { pattern_methods!('a, CharArraySearcher<'a, N>, MultiCharEqPattern, CharArraySearcher); } -unsafe impl<'a, const N: usize> Searcher<'a> for CharArraySearcher<'a, N> { +unsafe impl<'a, const N: usize> Searcher<&'a str> for CharArraySearcher<'a, N> { searcher_methods!(forward); } -unsafe impl<'a, const N: usize> ReverseSearcher<'a> for CharArraySearcher<'a, N> { +unsafe impl<'a, const N: usize> ReverseSearcher<&'a str> for CharArraySearcher<'a, N> { searcher_methods!(reverse); } -impl<'a, const N: usize> DoubleEndedSearcher<'a> for CharArraySearcher<'a, N> {} +impl<'a, const N: usize> DoubleEndedSearcher<&'a str> for CharArraySearcher<'a, N> {} /// Searches for chars that are equal to any of the [`char`]s in the array. /// @@ -849,19 +407,19 @@ impl<'a, const N: usize> DoubleEndedSearcher<'a> for CharArraySearcher<'a, N> {} /// assert_eq!("Hello world".find(&['o', 'l']), Some(2)); /// assert_eq!("Hello world".find(&['h', 'w']), Some(6)); /// ``` -impl<'b, const N: usize> Pattern for &'b [char; N] { +impl<'a, 'b, const N: usize> Pattern<&'a str> for &'b [char; N] { pattern_methods!('a, CharArrayRefSearcher<'a, 'b, N>, MultiCharEqPattern, CharArrayRefSearcher); } -unsafe impl<'a, 'b, const N: usize> Searcher<'a> for CharArrayRefSearcher<'a, 'b, N> { +unsafe impl<'a, 'b, const N: usize> Searcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> { searcher_methods!(forward); } -unsafe impl<'a, 'b, const N: usize> ReverseSearcher<'a> for CharArrayRefSearcher<'a, 'b, N> { +unsafe impl<'a, 'b, const N: usize> ReverseSearcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> { searcher_methods!(reverse); } -impl<'a, 'b, const N: usize> DoubleEndedSearcher<'a> for CharArrayRefSearcher<'a, 'b, N> {} +impl<'a, 'b, const N: usize> DoubleEndedSearcher<&'a str> for CharArrayRefSearcher<'a, 'b, N> {} ///////////////////////////////////////////////////////////////////////////// // Impl for &[char] @@ -869,19 +427,21 @@ impl<'a, 'b, const N: usize> DoubleEndedSearcher<'a> for CharArrayRefSearcher<'a // Todo: Change / Remove due to ambiguity in meaning. -/// Associated type for `<&[char] as Pattern>::Searcher<'a>`. +/// Associated type for `<&[char] as Pattern<&'a str>>::Searcher`. #[derive(Clone, Debug)] -pub struct CharSliceSearcher<'a, 'b>( as Pattern>::Searcher<'a>); +pub struct CharSliceSearcher<'a, 'b>( + as Pattern<&'a str>>::Searcher, +); -unsafe impl<'a, 'b> Searcher<'a> for CharSliceSearcher<'a, 'b> { +unsafe impl<'a, 'b> Searcher<&'a str> for CharSliceSearcher<'a, 'b> { searcher_methods!(forward); } -unsafe impl<'a, 'b> ReverseSearcher<'a> for CharSliceSearcher<'a, 'b> { +unsafe impl<'a, 'b> ReverseSearcher<&'a str> for CharSliceSearcher<'a, 'b> { searcher_methods!(reverse); } -impl<'a, 'b> DoubleEndedSearcher<'a> for CharSliceSearcher<'a, 'b> {} +impl<'a, 'b> DoubleEndedSearcher<&'a str> for CharSliceSearcher<'a, 'b> {} /// Searches for chars that are equal to any of the [`char`]s in the slice. /// @@ -891,7 +451,7 @@ impl<'a, 'b> DoubleEndedSearcher<'a> for CharSliceSearcher<'a, 'b> {} /// assert_eq!("Hello world".find(&['o', 'l'][..]), Some(2)); /// assert_eq!("Hello world".find(&['h', 'w'][..]), Some(6)); /// ``` -impl<'b> Pattern for &'b [char] { +impl<'a, 'b> Pattern<&'a str> for &'b [char] { pattern_methods!('a, CharSliceSearcher<'a, 'b>, MultiCharEqPattern, CharSliceSearcher); } @@ -899,9 +459,9 @@ impl<'b> Pattern for &'b [char] { // Impl for F: FnMut(char) -> bool ///////////////////////////////////////////////////////////////////////////// -/// Associated type for `::Searcher<'a>`. +/// Associated type for `>::Searcher`. #[derive(Clone)] -pub struct CharPredicateSearcher<'a, F>( as Pattern>::Searcher<'a>) +pub struct CharPredicateSearcher<'a, F>( as Pattern<&'a str>>::Searcher) where F: FnMut(char) -> bool; @@ -916,21 +476,21 @@ where .finish() } } -unsafe impl<'a, F> Searcher<'a> for CharPredicateSearcher<'a, F> +unsafe impl<'a, F> Searcher<&'a str> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool, { searcher_methods!(forward); } -unsafe impl<'a, F> ReverseSearcher<'a> for CharPredicateSearcher<'a, F> +unsafe impl<'a, F> ReverseSearcher<&'a str> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool, { searcher_methods!(reverse); } -impl<'a, F> DoubleEndedSearcher<'a> for CharPredicateSearcher<'a, F> where F: FnMut(char) -> bool {} +impl<'a, F: FnMut(char) -> bool> DoubleEndedSearcher<&'a str> for CharPredicateSearcher<'a, F> {} /// Searches for [`char`]s that match the given predicate. /// @@ -940,7 +500,7 @@ impl<'a, F> DoubleEndedSearcher<'a> for CharPredicateSearcher<'a, F> where F: Fn /// assert_eq!("Hello world".find(char::is_uppercase), Some(0)); /// assert_eq!("Hello world".find(|c| "aeiou".contains(c)), Some(1)); /// ``` -impl Pattern for F +impl<'a, F> Pattern<&'a str> for F where F: FnMut(char) -> bool, { @@ -952,7 +512,7 @@ where ///////////////////////////////////////////////////////////////////////////// /// Delegates to the `&str` impl. -impl<'b, 'c> Pattern for &'c &'b str { +impl<'a, 'b, 'c> Pattern<&'a str> for &'c &'b str { pattern_methods!('a, StrSearcher<'a, 'b>, |&s| s, |s| s); } @@ -970,23 +530,23 @@ impl<'b, 'c> Pattern for &'c &'b str { /// ``` /// assert_eq!("Hello world".find("world"), Some(6)); /// ``` -impl<'b> Pattern for &'b str { - type Searcher<'a> = StrSearcher<'a, 'b>; +impl<'a, 'b> Pattern<&'a str> for &'b str { + type Searcher = StrSearcher<'a, 'b>; #[inline] - fn into_searcher(self, haystack: &str) -> StrSearcher<'_, 'b> { + fn into_searcher(self, haystack: &'a str) -> StrSearcher<'a, 'b> { StrSearcher::new(haystack, self) } /// Checks whether the pattern matches at the front of the haystack. #[inline] - fn is_prefix_of(self, haystack: &str) -> bool { + fn is_prefix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().starts_with(self.as_bytes()) } /// Checks whether the pattern matches anywhere in the haystack #[inline] - fn is_contained_in(self, haystack: &str) -> bool { + fn is_contained_in(self, haystack: &'a str) -> bool { if self.is_empty() { return true; } @@ -1016,7 +576,7 @@ impl<'b> Pattern for &'b str { /// Removes the pattern from the front of haystack, if it matches. #[inline] - fn strip_prefix_of(self, haystack: &str) -> Option<&str> { + fn strip_prefix_of(self, haystack: &'a str) -> Option<&'a str> { if self.is_prefix_of(haystack) { // SAFETY: prefix was just verified to exist. unsafe { Some(haystack.get_unchecked(self.len()..)) } @@ -1027,19 +587,13 @@ impl<'b> Pattern for &'b str { /// Checks whether the pattern matches at the back of the haystack. #[inline] - fn is_suffix_of<'a>(self, haystack: &'a str) -> bool - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { + fn is_suffix_of(self, haystack: &'a str) -> bool { haystack.as_bytes().ends_with(self.as_bytes()) } /// Removes the pattern from the back of haystack, if it matches. #[inline] - fn strip_suffix_of<'a>(self, haystack: &'a str) -> Option<&'a str> - where - Self::Searcher<'a>: ReverseSearcher<'a>, - { + fn strip_suffix_of(self, haystack: &'a str) -> Option<&'a str> { if self.is_suffix_of(haystack) { let i = haystack.len() - self.len(); // SAFETY: suffix was just verified to exist. @@ -1060,809 +614,53 @@ impl<'b> Pattern for &'b str { ///////////////////////////////////////////////////////////////////////////// #[derive(Clone, Debug)] -/// Associated type for `<&str as Pattern>::Searcher<'a>`. -pub struct StrSearcher<'a, 'b> { - haystack: &'a str, - needle: &'b str, - - searcher: StrSearcherImpl, -} - -#[derive(Clone, Debug)] -enum StrSearcherImpl { - Empty(EmptyNeedle), - Byte(ByteNeedle), - TwoWay(TwoWaySearcher), -} - -#[derive(Clone, Debug)] -struct EmptyNeedle { - position: usize, - end: usize, - is_match_fw: bool, - is_match_bw: bool, - // Needed in case of an empty haystack, see #85462 - is_finished: bool, -} - -/// Fast searcher for a single-byte needle using `memchr`/`memrchr`. -#[derive(Clone, Debug)] -struct ByteNeedle { - b: u8, - /// Forward cursor: `haystack[..position]` has already been reported. - position: usize, - /// Backward cursor: `haystack[end..]` has already been reported. - end: usize, -} +/// Associated type for `<&str as Pattern<&'a str>>::Searcher`. +pub struct StrSearcher<'a, 'b>(crate::str_bytes::StrSearcher<'a, 'b, crate::str_bytes::Utf8>); impl<'a, 'b> StrSearcher<'a, 'b> { + #[inline] fn new(haystack: &'a str, needle: &'b str) -> StrSearcher<'a, 'b> { - if needle.is_empty() { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::Empty(EmptyNeedle { - position: 0, - end: haystack.len(), - is_match_fw: true, - is_match_bw: true, - is_finished: false, - }), - } - } else if let &[b] = needle.as_bytes() { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::Byte(ByteNeedle { b, position: 0, end: haystack.len() }), - } - } else { - StrSearcher { - haystack, - needle, - searcher: StrSearcherImpl::TwoWay(TwoWaySearcher::new( - needle.as_bytes(), - haystack.len(), - )), - } - } + let haystack = crate::str_bytes::Bytes::from_str(haystack); + Self(crate::str_bytes::StrSearcher::new(haystack, needle)) } } -unsafe impl<'a, 'b> Searcher<'a> for StrSearcher<'a, 'b> { +unsafe impl<'a, 'b> Searcher<&'a str> for StrSearcher<'a, 'b> { #[inline] fn haystack(&self) -> &'a str { - self.haystack + self.0.haystack().into_str() } #[inline] fn next(&mut self) -> SearchStep { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - if searcher.is_finished { - return SearchStep::Done; - } - // empty needle rejects every char and matches every empty string between them - let is_match = searcher.is_match_fw; - searcher.is_match_fw = !searcher.is_match_fw; - let pos = searcher.position; - match self.haystack[pos..].chars().next() { - _ if is_match => SearchStep::Match(pos, pos), - None => { - searcher.is_finished = true; - SearchStep::Done - } - Some(ch) => { - searcher.position += ch.len_utf8(); - SearchStep::Reject(pos, searcher.position) - } - } - } - StrSearcherImpl::Byte(ref mut searcher) => { - let bytes = self.haystack.as_bytes(); - let pos = searcher.position; - if pos >= bytes.len() { - return SearchStep::Done; - } - if bytes[pos] == searcher.b { - searcher.position = pos + 1; - SearchStep::Match(pos, pos + 1) - } else { - // `pos` is always on a char boundary, so this rejects - // exactly the char starting at `pos`. - let end = self.haystack.ceil_char_boundary(pos + 1); - searcher.position = end; - SearchStep::Reject(pos, end) - } - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - // TwoWaySearcher produces valid *Match* indices that split at char boundaries - // as long as it does correct matching and that haystack and needle are - // valid UTF-8 - // *Rejects* from the algorithm can fall on any indices, but we will walk them - // manually to the next character boundary, so that they are utf-8 safe. - if searcher.position == self.haystack.len() { - return SearchStep::Done; - } - let is_long = searcher.memory == usize::MAX; - match searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - is_long, - ) { - SearchStep::Reject(a, b) => { - // skip to next char boundary - let b = self.haystack.ceil_char_boundary(b); - searcher.position = cmp::max(b, searcher.position); - SearchStep::Reject(a, b) - } - otherwise => otherwise, - } - } - } + self.0.next() } #[inline] fn next_match(&mut self) -> Option<(usize, usize)> { - match self.searcher { - StrSearcherImpl::Empty(..) => loop { - match self.next() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - SearchStep::Reject(..) => {} - } - }, - StrSearcherImpl::Byte(ref mut searcher) => { - let bytes = self.haystack.as_bytes(); - if searcher.position >= bytes.len() { - return None; - } - match memchr::memchr(searcher.b, &bytes[searcher.position..]) { - Some(i) => { - let pos = searcher.position + i; - searcher.position = pos + 1; - Some((pos, pos + 1)) - } - None => { - searcher.position = bytes.len(); - None - } - } - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - let is_long = searcher.memory == usize::MAX; - // write out `true` and `false` cases to encourage the compiler - // to specialize the two cases separately. - if is_long { - searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - true, - ) - } else { - searcher.next::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - false, - ) - } - } - } - } -} - -unsafe impl<'a, 'b> ReverseSearcher<'a> for StrSearcher<'a, 'b> { - #[inline] - fn next_back(&mut self) -> SearchStep { - match self.searcher { - StrSearcherImpl::Empty(ref mut searcher) => { - if searcher.is_finished { - return SearchStep::Done; - } - let is_match = searcher.is_match_bw; - searcher.is_match_bw = !searcher.is_match_bw; - let end = searcher.end; - match self.haystack[..end].chars().next_back() { - _ if is_match => SearchStep::Match(end, end), - None => { - searcher.is_finished = true; - SearchStep::Done - } - Some(ch) => { - searcher.end -= ch.len_utf8(); - SearchStep::Reject(searcher.end, end) - } - } - } - StrSearcherImpl::Byte(ref mut searcher) => { - let end = searcher.end; - if end == 0 { - return SearchStep::Done; - } - let bytes = self.haystack.as_bytes(); - if bytes[end - 1] == searcher.b { - searcher.end = end - 1; - SearchStep::Match(end - 1, end) - } else { - let start = self.haystack.floor_char_boundary(end - 1); - searcher.end = start; - SearchStep::Reject(start, end) - } - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - if searcher.end == 0 { - return SearchStep::Done; - } - let is_long = searcher.memory == usize::MAX; - match searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - is_long, - ) { - SearchStep::Reject(a, b) => { - // skip to previous char boundary - let a = self.haystack.floor_char_boundary(a); - searcher.end = cmp::min(a, searcher.end); - SearchStep::Reject(a, b) - } - otherwise => otherwise, - } - } - } + self.0.next_match() } #[inline] - fn next_match_back(&mut self) -> Option<(usize, usize)> { - match self.searcher { - StrSearcherImpl::Empty(..) => loop { - match self.next_back() { - SearchStep::Match(a, b) => return Some((a, b)), - SearchStep::Done => return None, - SearchStep::Reject(..) => {} - } - }, - StrSearcherImpl::Byte(ref mut searcher) => { - if searcher.end == 0 { - return None; - } - let bytes = self.haystack.as_bytes(); - match memchr::memrchr(searcher.b, &bytes[..searcher.end]) { - Some(i) => { - searcher.end = i; - Some((i, i + 1)) - } - None => { - searcher.end = 0; - None - } - } - } - StrSearcherImpl::TwoWay(ref mut searcher) => { - let is_long = searcher.memory == usize::MAX; - // write out `true` and `false`, like `next_match` - if is_long { - searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - true, - ) - } else { - searcher.next_back::( - self.haystack.as_bytes(), - self.needle.as_bytes(), - false, - ) - } - } - } + fn next_reject(&mut self) -> Option<(usize, usize)> { + self.0.next_reject() } } -/// The internal state of the two-way substring search algorithm. -#[derive(Clone, Debug)] -struct TwoWaySearcher { - // constants - /// critical factorization index - crit_pos: usize, - /// critical factorization index for reversed needle - crit_pos_back: usize, - period: usize, - /// `byteset` is an extension (not part of the two way algorithm); - /// it's a 64-bit "fingerprint" where each set bit `j` corresponds - /// to a (byte & 63) == j present in the needle. - byteset: u64, - - // variables - position: usize, - end: usize, - /// index into needle before which we have already matched - memory: usize, - /// index into needle after which we have already matched - memory_back: usize, -} - -/* - This is the Two-Way search algorithm, which was introduced in the paper: - Crochemore, M., Perrin, D., 1991, Two-way string-matching, Journal of the ACM 38(3):651-675. - - Here's some background information. - - A *word* is a string of symbols. The *length* of a word should be a familiar - notion, and here we denote it for any word x by |x|. - (We also allow for the possibility of the *empty word*, a word of length zero). - - If x is any non-empty word, then an integer p with 0 < p <= |x| is said to be a - *period* for x iff for all i with 0 <= i <= |x| - p - 1, we have x[i] == x[i+p]. - For example, both 1 and 2 are periods for the string "aa". As another example, - the only period of the string "abcd" is 4. - - We denote by period(x) the *smallest* period of x (provided that x is non-empty). - This is always well-defined since every non-empty word x has at least one period, - |x|. We sometimes call this *the period* of x. - - If u, v and x are words such that x = uv, where uv is the concatenation of u and - v, then we say that (u, v) is a *factorization* of x. - - Let (u, v) be a factorization for a word x. Then if w is a non-empty word such - that both of the following hold - - - either w is a suffix of u or u is a suffix of w - - either w is a prefix of v or v is a prefix of w - - then w is said to be a *repetition* for the factorization (u, v). - - Just to unpack this, there are four possibilities here. Let w = "abc". Then we - might have: - - - w is a suffix of u and w is a prefix of v. ex: ("lolabc", "abcde") - - w is a suffix of u and v is a prefix of w. ex: ("lolabc", "ab") - - u is a suffix of w and w is a prefix of v. ex: ("bc", "abchi") - - u is a suffix of w and v is a prefix of w. ex: ("bc", "a") - - Note that the word vu is a repetition for any factorization (u,v) of x = uv, - so every factorization has at least one repetition. - - If x is a string and (u, v) is a factorization for x, then a *local period* for - (u, v) is an integer r such that there is some word w such that |w| = r and w is - a repetition for (u, v). - - We denote by local_period(u, v) the smallest local period of (u, v). We sometimes - call this *the local period* of (u, v). Provided that x = uv is non-empty, this - is well-defined (because each non-empty word has at least one factorization, as - noted above). - - It can be proven that the following is an equivalent definition of a local period - for a factorization (u, v): any positive integer r such that x[i] == x[i+r] for - all i such that |u| - r <= i <= |u| - 1 and such that both x[i] and x[i+r] are - defined. (i.e., i > 0 and i + r < |x|). - - Using the above reformulation, it is easy to prove that - - 1 <= local_period(u, v) <= period(uv) - - A factorization (u, v) of x such that local_period(u,v) = period(x) is called a - *critical factorization*. - - The algorithm hinges on the following theorem, which is stated without proof: - - **Critical Factorization Theorem** Any word x has at least one critical - factorization (u, v) such that |u| < period(x). - - The purpose of maximal_suffix is to find such a critical factorization. - - If the period is short, compute another factorization x = u' v' to use - for reverse search, chosen instead so that |v'| < period(x). - -*/ -impl TwoWaySearcher { - fn new(needle: &[u8], end: usize) -> TwoWaySearcher { - let (crit_pos_false, period_false) = TwoWaySearcher::maximal_suffix(needle, false); - let (crit_pos_true, period_true) = TwoWaySearcher::maximal_suffix(needle, true); - - let (crit_pos, period) = if crit_pos_false > crit_pos_true { - (crit_pos_false, period_false) - } else { - (crit_pos_true, period_true) - }; - - // A particularly readable explanation of what's going on here can be found - // in Crochemore and Rytter's book "Text Algorithms", ch 13. Specifically - // see the code for "Algorithm CP" on p. 323. - // - // What's going on is we have some critical factorization (u, v) of the - // needle, and we want to determine whether u is a suffix of - // &v[..period]. If it is, we use "Algorithm CP1". Otherwise we use - // "Algorithm CP2", which is optimized for when the period of the needle - // is large. - if needle[..crit_pos] == needle[period..period + crit_pos] { - // short period case -- the period is exact - // compute a separate critical factorization for the reversed needle - // x = u' v' where |v'| < period(x). - // - // This is sped up by the period being known already. - // Note that a case like x = "acba" may be factored exactly forwards - // (crit_pos = 1, period = 3) while being factored with approximate - // period in reverse (crit_pos = 2, period = 2). We use the given - // reverse factorization but keep the exact period. - let crit_pos_back = needle.len() - - cmp::max( - TwoWaySearcher::reverse_maximal_suffix(needle, period, false), - TwoWaySearcher::reverse_maximal_suffix(needle, period, true), - ); - - TwoWaySearcher { - crit_pos, - crit_pos_back, - period, - byteset: Self::byteset_create(&needle[..period]), - - position: 0, - end, - memory: 0, - memory_back: needle.len(), - } - } else { - // long period case -- we have an approximation to the actual period, - // and don't use memorization. - // - // Approximate the period by lower bound max(|u|, |v|) + 1. - // The critical factorization is efficient to use for both forward and - // reverse search. - - TwoWaySearcher { - crit_pos, - crit_pos_back: crit_pos, - period: cmp::max(crit_pos, needle.len() - crit_pos) + 1, - byteset: Self::byteset_create(needle), - - position: 0, - end, - memory: usize::MAX, // Dummy value to signify that the period is long - memory_back: usize::MAX, - } - } - } - - #[inline] - fn byteset_create(bytes: &[u8]) -> u64 { - bytes.iter().fold(0, |a, &b| (1 << (b & 0x3f)) | a) - } - +unsafe impl<'a, 'b> ReverseSearcher<&'a str> for StrSearcher<'a, 'b> { #[inline] - fn byteset_contains(&self, byte: u8) -> bool { - (self.byteset >> ((byte & 0x3f) as usize)) & 1 != 0 - } - - // One of the main ideas of Two-Way is that we factorize the needle into - // two halves, (u, v), and begin trying to find v in the haystack by scanning - // left to right. If v matches, we try to match u by scanning right to left. - // How far we can jump when we encounter a mismatch is all based on the fact - // that (u, v) is a critical factorization for the needle. - #[inline] - fn next(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> S::Output - where - S: TwoWayStrategy, - { - // `next()` uses `self.position` as its cursor - let old_pos = self.position; - let needle_last = needle.len() - 1; - 'search: loop { - // Check that we have room to search in - // position + needle_last can not overflow if we assume slices - // are bounded by isize's range. - let tail_byte = match haystack.get(self.position + needle_last) { - Some(&b) => b, - None => { - self.position = haystack.len(); - return S::rejecting(old_pos, self.position); - } - }; - - if S::use_early_reject() && old_pos != self.position { - return S::rejecting(old_pos, self.position); - } - - // Quickly skip by large portions unrelated to our substring - if !self.byteset_contains(tail_byte) { - self.position += needle.len(); - if !long_period { - self.memory = 0; - } - continue 'search; - } - - // See if the right part of the needle matches - let start = - if long_period { self.crit_pos } else { cmp::max(self.crit_pos, self.memory) }; - for i in start..needle.len() { - // SAFETY: on every iteration of `'search`, the `haystack.get(self.position + needle_last)` - // check returned `Some`, so `self.position + needle_last < haystack.len()`. - // Since `i < needle.len()` implies `i <= needle_last`, we have - // `self.position + i < haystack.len()`. - // Every path that mutates `self.position` below either returns or re-enters `'search`, - // which re-runs the check before reaching the loop again. - if needle[i] != unsafe { *haystack.get_unchecked(self.position + i) } { - self.position += i - self.crit_pos + 1; - if !long_period { - self.memory = 0; - } - continue 'search; - } - } - - // See if the left part of the needle matches - let start = if long_period { 0 } else { self.memory }; - for i in (start..self.crit_pos).rev() { - // SAFETY: on every iteration of `'search`, the `haystack.get(self.position + needle_last)` - // check returned `Some`, so `self.position + needle_last < haystack.len()`. - // Since `i < self.crit_pos <= needle.len()`, we have `i <= needle_last`, and thus - // `self.position + i <= self.position + needle_last < haystack.len()`. - // Every path that mutates `self.position` below either returns or re-enters `'search`, - // which re-runs the check before reaching the loop again. - if needle[i] != unsafe { *haystack.get_unchecked(self.position + i) } { - self.position += self.period; - if !long_period { - self.memory = needle.len() - self.period; - } - continue 'search; - } - } - - // We have found a match! - let match_pos = self.position; - - // Note: add self.period instead of needle.len() to have overlapping matches - self.position += needle.len(); - if !long_period { - self.memory = 0; // set to needle.len() - self.period for overlapping matches - } - - return S::matching(match_pos, match_pos + needle.len()); - } - } - - // Follows the ideas in `next()`. - // - // The definitions are symmetrical, with period(x) = period(reverse(x)) - // and local_period(u, v) = local_period(reverse(v), reverse(u)), so if (u, v) - // is a critical factorization, so is (reverse(v), reverse(u)). - // - // For the reverse case we have computed a critical factorization x = u' v' - // (field `crit_pos_back`). We need |u| < period(x) for the forward case and - // thus |v'| < period(x) for the reverse. - // - // To search in reverse through the haystack, we search forward through - // a reversed haystack with a reversed needle, matching first u' and then v'. - #[inline] - fn next_back(&mut self, haystack: &[u8], needle: &[u8], long_period: bool) -> S::Output - where - S: TwoWayStrategy, - { - // `next_back()` uses `self.end` as its cursor -- so that `next()` and `next_back()` - // are independent. - let old_end = self.end; - 'search: loop { - // Check that we have room to search in - // end - needle.len() will wrap around when there is no more room, - // but due to slice length limits it can never wrap all the way back - // into the length of haystack. - let front_byte = match haystack.get(self.end.wrapping_sub(needle.len())) { - Some(&b) => b, - None => { - self.end = 0; - return S::rejecting(0, old_end); - } - }; - - if S::use_early_reject() && old_end != self.end { - return S::rejecting(self.end, old_end); - } - - // Quickly skip by large portions unrelated to our substring - if !self.byteset_contains(front_byte) { - self.end -= needle.len(); - if !long_period { - self.memory_back = needle.len(); - } - continue 'search; - } - - // See if the left part of the needle matches - let crit = if long_period { - self.crit_pos_back - } else { - cmp::min(self.crit_pos_back, self.memory_back) - }; - for i in (0..crit).rev() { - // SAFETY: On every iteration of `'search`, `haystack.get(self.end.wrapping_sub(needle.len()))` - // returned `Some`, so `self.end >= needle.len()` and `self.end - needle.len() < haystack.len()`. - // Since `self.end <= haystack.len()` and `i < needle.len()`, we have - // `self.end - needle.len() + i < self.end <= haystack.len()`, so - // `haystack.get_unchecked(self.end - needle.len() + i)` is safe. - // - The path that mutates `self.end` either re-enters `'search`, which re-runs the checks - // before reaching this loop again, or returns on match, so the invariant holds. - if needle[i] != unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } { - self.end -= self.crit_pos_back - i; - if !long_period { - self.memory_back = needle.len(); - } - continue 'search; - } - } - - // See if the right part of the needle matches - let needle_end = if long_period { needle.len() } else { self.memory_back }; - for i in self.crit_pos_back..needle_end { - // SAFETY: The same `self.end - needle.len() + i < haystack.len()` argument as the - // left-part loop applies: the `haystack.get(self.end.wrapping_sub(needle.len()))` - // check at the top of `'search` established the bound for this iteration, and - // every mutation of `self.end` is followed by `continue 'search` (which re-runs - // the check) or a `return` (which exits before any further unsafe access). - if needle[i] != unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } { - self.end -= self.period; - if !long_period { - self.memory_back = self.period; - } - continue 'search; - } - } - - // We have found a match! - let match_pos = self.end - needle.len(); - // Note: sub self.period instead of needle.len() to have overlapping matches - self.end -= needle.len(); - if !long_period { - self.memory_back = needle.len(); - } - - return S::matching(match_pos, match_pos + needle.len()); - } - } - - // Compute the maximal suffix of `arr`. - // - // The maximal suffix is a possible critical factorization (u, v) of `arr`. - // - // Returns (`i`, `p`) where `i` is the starting index of v and `p` is the - // period of v. - // - // `order_greater` determines if lexical order is `<` or `>`. Both - // orders must be computed -- the ordering with the largest `i` gives - // a critical factorization. - // - // For long period cases, the resulting period is not exact (it is too short). - #[inline] - fn maximal_suffix(arr: &[u8], order_greater: bool) -> (usize, usize) { - let mut left = 0; // Corresponds to i in the paper - let mut right = 1; // Corresponds to j in the paper - let mut offset = 0; // Corresponds to k in the paper, but starting at 0 - // to match 0-based indexing. - let mut period = 1; // Corresponds to p in the paper - - while let Some(&a) = arr.get(right + offset) { - // `left` will be inbounds when `right` is. - let b = arr[left + offset]; - if (a < b && !order_greater) || (a > b && order_greater) { - // Suffix is smaller, period is entire prefix so far. - right += offset + 1; - offset = 0; - period = right - left; - } else if a == b { - // Advance through repetition of the current period. - if offset + 1 == period { - right += offset + 1; - offset = 0; - } else { - offset += 1; - } - } else { - // Suffix is larger, start over from current location. - left = right; - right += 1; - offset = 0; - period = 1; - } - } - (left, period) + fn next_back(&mut self) -> SearchStep { + self.0.next_back() } - // Compute the maximal suffix of the reverse of `arr`. - // - // The maximal suffix is a possible critical factorization (u', v') of `arr`. - // - // Returns `i` where `i` is the starting index of v', from the back; - // returns immediately when a period of `known_period` is reached. - // - // `order_greater` determines if lexical order is `<` or `>`. Both - // orders must be computed -- the ordering with the largest `i` gives - // a critical factorization. - // - // For long period cases, the resulting period is not exact (it is too short). - fn reverse_maximal_suffix(arr: &[u8], known_period: usize, order_greater: bool) -> usize { - let mut left = 0; // Corresponds to i in the paper - let mut right = 1; // Corresponds to j in the paper - let mut offset = 0; // Corresponds to k in the paper, but starting at 0 - // to match 0-based indexing. - let mut period = 1; // Corresponds to p in the paper - let n = arr.len(); - - while right + offset < n { - let a = arr[n - (1 + right + offset)]; - let b = arr[n - (1 + left + offset)]; - if (a < b && !order_greater) || (a > b && order_greater) { - // Suffix is smaller, period is entire prefix so far. - right += offset + 1; - offset = 0; - period = right - left; - } else if a == b { - // Advance through repetition of the current period. - if offset + 1 == period { - right += offset + 1; - offset = 0; - } else { - offset += 1; - } - } else { - // Suffix is larger, start over from current location. - left = right; - right += 1; - offset = 0; - period = 1; - } - if period == known_period { - break; - } - } - debug_assert!(period <= known_period); - left - } -} - -// TwoWayStrategy allows the algorithm to either skip non-matches as quickly -// as possible, or to work in a mode where it emits Rejects relatively quickly. -trait TwoWayStrategy { - type Output; - fn use_early_reject() -> bool; - fn rejecting(a: usize, b: usize) -> Self::Output; - fn matching(a: usize, b: usize) -> Self::Output; -} - -/// Skip to match intervals as quickly as possible -enum MatchOnly {} - -impl TwoWayStrategy for MatchOnly { - type Output = Option<(usize, usize)>; - - #[inline] - fn use_early_reject() -> bool { - false - } - #[inline] - fn rejecting(_a: usize, _b: usize) -> Self::Output { - None - } #[inline] - fn matching(a: usize, b: usize) -> Self::Output { - Some((a, b)) + fn next_match_back(&mut self) -> Option<(usize, usize)> { + self.0.next_match_back() } -} - -/// Emit Rejects regularly -enum RejectAndMatch {} - -impl TwoWayStrategy for RejectAndMatch { - type Output = SearchStep; #[inline] - fn use_early_reject() -> bool { - true - } - #[inline] - fn rejecting(a: usize, b: usize) -> Self::Output { - SearchStep::Reject(a, b) - } - #[inline] - fn matching(a: usize, b: usize) -> Self::Output { - SearchStep::Match(a, b) + fn next_reject_back(&mut self) -> Option<(usize, usize)> { + self.0.next_reject_back() } } diff --git a/library/core/src/str/validations.rs b/library/core/src/str/validations.rs index b54d6478e584d..4783af75fc1f1 100644 --- a/library/core/src/str/validations.rs +++ b/library/core/src/str/validations.rs @@ -75,7 +75,7 @@ pub unsafe fn next_code_point<'a, I: Iterator>(bytes: &mut I) -> /// /// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string #[inline] -pub(super) unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option +pub(crate) unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option where I: DoubleEndedIterator, { @@ -119,6 +119,78 @@ const fn contains_nonascii(x: usize) -> bool { (x & NONASCII_MASK) != 0 } +/// Reads the first code point and its encoded length out of a byte slice +/// validating whether it's valid. +/// +/// This is different than [`next_code_point`] in that it doesn't assume +/// the argument is a well-formed UTF-8-like string. +/// +/// If front of the bytes slice doesn't contain valid UTF-8 bytes sequence (that +/// includes a WTF-8 encoded surrogate) returns `None`. +/// +/// ``` +/// #![feature(str_internals)] +/// # #![allow(internal_features)] +/// use core::str::try_next_code_point; +/// +/// assert_eq!(Some(('f', 1)), try_next_code_point(b"foo".as_ref())); +/// assert_eq!(Some(('Ż', 2)), try_next_code_point("Żółw".as_bytes())); +/// assert_eq!(None, try_next_code_point(b"\xffoo".as_ref())); +/// ``` +#[unstable(feature = "str_internals", issue = "none")] +#[rustc_const_unstable(feature = "str_internals", issue = "none")] +#[inline] +pub const fn try_next_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let first = *bytes.first()?; + let (value, length) = if first < 0x80 { + (first as u32, 1) + } else { + try_finish_byte_sequence(first, bytes, 0).ok()? + }; + // SAFETY: value is a valid Unicode scalar value. + // Either ASCII (first branch) or a valid non-ASCII Unicode character + // as guaranteed by `try_finish_byte_sequence` (second branch). + Some((unsafe { char::from_u32_unchecked(value) }, length)) +} + +/// Reads the last code point and its encoded length out of a byte slice +/// validating whether it's valid. +/// +/// This is different than `next_code_point_reverse` in that it doesn't assume +/// the argument is a well-formed UTF-8-like string. +/// +/// If back of the bytes slice doesn't contain valid UTF-8 bytes sequence (that +/// includes a WTF-8 encoded surrogate) returns `None`. +/// +/// ``` +/// #![feature(str_internals)] +/// # #![allow(internal_features)] +/// use core::str::try_next_code_point_reverse; +/// +/// assert_eq!(Some(('o', 1)), try_next_code_point_reverse(b"foo".as_ref())); +/// assert_eq!(Some(('‽', 3)), try_next_code_point_reverse("Uh‽".as_bytes())); +/// assert_eq!(None, try_next_code_point_reverse(b"foo\xff".as_ref())); +/// ``` +#[unstable(feature = "str_internals", issue = "none")] +#[rustc_const_unstable(feature = "str_internals", issue = "none")] +#[inline] +pub const fn try_next_code_point_reverse(bytes: &[u8]) -> Option<(char, usize)> { + let mut n = 1; + let limit = bytes.len(); + let limit = if limit < 4 { limit } else { 4 }; // not .min(4) because of const + while n <= limit && !bytes[bytes.len() - n].is_utf8_char_boundary() { + n += 1; + } + if n <= limit { + let bytes = &bytes[bytes.len() - n..]; + let (chr, len) = try_next_code_point(bytes)?; + if n == len { + return Some((chr, len)); + } + } + None +} + /// Walks through `v` checking that it's a valid UTF-8 sequence, /// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. #[inline(always)] @@ -143,78 +215,13 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { ); while index < len { - let old_offset = index; - macro_rules! err { - ($error_len: expr) => { - return Err(Utf8Error { valid_up_to: old_offset, error_len: $error_len }) - }; - } - - macro_rules! next { - () => {{ - index += 1; - // we needed data, but there was none: error! - if index >= len { - err!(None) - } - v[index] - }}; - } - + let valid_up_to = index; let first = v[index]; if first >= 128 { - let w = utf8_char_width(first); - // 2-byte encoding is for codepoints \u{0080} to \u{07ff} - // first C2 80 last DF BF - // 3-byte encoding is for codepoints \u{0800} to \u{ffff} - // first E0 A0 80 last EF BF BF - // excluding surrogates codepoints \u{d800} to \u{dfff} - // ED A0 80 to ED BF BF - // 4-byte encoding is for codepoints \u{10000} to \u{10ffff} - // first F0 90 80 80 last F4 8F BF BF - // - // Use the UTF-8 syntax from the RFC - // - // https://tools.ietf.org/html/rfc3629 - // UTF8-1 = %x00-7F - // UTF8-2 = %xC2-DF UTF8-tail - // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / - // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) - // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / - // %xF4 %x80-8F 2( UTF8-tail ) - match w { - 2 => { - if next!() as i8 >= -64 { - err!(Some(1)) - } - } - 3 => { - match (first, next!()) { - (0xE0, 0xA0..=0xBF) - | (0xE1..=0xEC, 0x80..=0xBF) - | (0xED, 0x80..=0x9F) - | (0xEE..=0xEF, 0x80..=0xBF) => {} - _ => err!(Some(1)), - } - if next!() as i8 >= -64 { - err!(Some(2)) - } - } - 4 => { - match (first, next!()) { - (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} - _ => err!(Some(1)), - } - if next!() as i8 >= -64 { - err!(Some(2)) - } - if next!() as i8 >= -64 { - err!(Some(3)) - } - } - _ => err!(Some(1)), + match try_finish_byte_sequence(first, v, index) { + Ok((_value, length)) => index += length, + Err(error_len) => return Err(Utf8Error { valid_up_to, error_len }), } - index += 1; } else { // Ascii case, try to skip forward quickly. // When the pointer is aligned, read 2 words of data per iteration @@ -250,6 +257,92 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { Ok(()) } +/// Try to finish an UTF-8 byte sequence. +/// +/// Assumes that `bytes[index] == first` and then `first >= 128`, i.e. that +/// `index` points at the beginning of a non-ASCII UTF-8 sequence in `bytes`. +/// +/// If the byte sequence at the index is correct, returns decoded code point and +/// length of the sequence. If it was invalid returns number of invalid bytes +/// or None if read was cut short. +#[inline(always)] +const fn try_finish_byte_sequence( + first: u8, + bytes: &[u8], + index: usize, +) -> Result<(u32, usize), Option> { + macro_rules! get { + (raw $offset:expr) => { + if index + $offset < bytes.len() { + bytes[index + $offset] + } else { + return Err(None) + } + }; + (cont $offset:expr) => {{ + let byte = get!(raw $offset); + if !utf8_is_cont_byte(byte) { + return Err(Some($offset as u8)) + } + byte + }} + } + + // 2-byte encoding is for codepoints \u{0080} to \u{07ff} + // first C2 80 last DF BF + // 3-byte encoding is for codepoints \u{0800} to \u{ffff} + // first E0 A0 80 last EF BF BF + // excluding surrogates codepoints \u{d800} to \u{dfff} + // ED A0 80 to ED BF BF + // 4-byte encoding is for codepoints \u{10000} to \u{10ffff} + // first F0 90 80 80 last F4 8F BF BF + // + // Use the UTF-8 syntax from the RFC + // + // https://tools.ietf.org/html/rfc3629 + // UTF8-1 = %x00-7F + // UTF8-2 = %xC2-DF UTF8-tail + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + match utf8_char_width(first) { + 2 => { + let second = get!(cont 1); + let value = utf8_first_byte(first, 2); + let value = utf8_acc_cont_byte(value, second); + Ok((value, 2)) + } + 3 => { + let second = get!(raw 1); + match (first, second) { + (0xE0, 0xA0..=0xBF) + | (0xE1..=0xEC, 0x80..=0xBF) + | (0xED, 0x80..=0x9F) + | (0xEE..=0xEF, 0x80..=0xBF) => {} + _ => return Err(Some(1)), + } + let value = utf8_first_byte(first, 3); + let value = utf8_acc_cont_byte(value, second); + let value = utf8_acc_cont_byte(value, get!(cont 2)); + Ok((value, 3)) + } + 4 => { + let second = get!(raw 1); + match (first, second) { + (0xF0, 0x90..=0xBF) | (0xF1..=0xF3, 0x80..=0xBF) | (0xF4, 0x80..=0x8F) => {} + _ => return Err(Some(1)), + } + let value = utf8_first_byte(first, 4); + let value = utf8_acc_cont_byte(value, second); + let value = utf8_acc_cont_byte(value, get!(cont 2)); + let value = utf8_acc_cont_byte(value, get!(cont 3)); + Ok((value, 4)) + } + _ => Err(Some(1)), + } +} + // https://tools.ietf.org/html/rfc3629 const UTF8_CHAR_WIDTH: &[u8; 256] = &[ // 1 2 3 4 5 6 7 8 9 A B C D E F diff --git a/library/core/src/str_bytes.rs b/library/core/src/str_bytes.rs new file mode 100644 index 0000000000000..195d2baf789ba --- /dev/null +++ b/library/core/src/str_bytes.rs @@ -0,0 +1,2006 @@ +//! Module provides pattern matching features for string-like bytes slice. +//! +//! A ‘string-like bytes slice’ means that types and functions here try to +//! interpret bytes slices as well-formed WTF-8 but don’t assume it is and treat +//! bytes in invalid portions of the slices as characters for the purpose of +//! deciding where character boundaries lie. This can be demonstrated by how +//! empty pattern is matched (since empty patterns match character boundaries): +//! +//! ``` +//! #![feature(pattern, str_internals)] +//! # #![allow(internal_features)] +//! use core::pattern::{Pattern, Searcher}; +//! use core::str_bytes::Bytes; +//! +//! let data = ["Żółw".as_bytes(), &b"\xff\xff\xff"[..], "🕴".as_bytes()].concat(); +//! let mut searcher = "".into_searcher(Bytes::from_bytes(data.as_slice())); +//! let next = move || searcher.next_match().map(|(x, _)| x); +//! let boundaries = core::iter::from_fn(next).collect::>(); +//! assert_eq!(&[0, 2, 4, 6, 7, 8, 9, 10, 14][..], &boundaries[..]); +//! ``` +#![unstable(feature = "str_internals", issue = "none")] + +use crate::marker::PhantomData; +use crate::mem::take; +use crate::pattern::{Haystack, MatchOnly, RejectOnly, SearchStep}; +use crate::slice::memchr; +use crate::str::{ + next_code_point, next_code_point_reverse, try_next_code_point, try_next_code_point_reverse, + utf8_char_width, +}; +use crate::{cmp, ops, pattern}; + +type OptRange = Option<(usize, usize)>; +type Range = ops::Range; + +//////////////////////////////////////////////////////////////////////////////// +// Bytes wrapper +//////////////////////////////////////////////////////////////////////////////// + +/// A reference to a string-like bytes slice. +/// +/// ‘String-like’ refers to the fact that parts of the data are valid WTF-8 and +/// when we split the slice we don’t want to split well-formed WTF-8 bytes +/// sequences. This is in a sense a generalisation of a `&str` which allows +/// portions of the buffer to be ill-formed while preserving correctness of +/// existing well-formed parts. +/// +/// The `F` generic argument tags the slice with a [flavour][Flavour] which +/// specifies structure of the data. +#[derive(Copy, Clone, Debug)] +pub struct Bytes<'a, F>(&'a [u8], PhantomData); + +impl<'a, F: Flavour> Bytes<'a, F> { + /// Creates a new `Bytes` wrapper around bytes slice. + /// + /// # Safety + /// + /// Caller must guarantee that the bytes adhere to the requirements for the + /// flavour `F`. E.g. for [`Wtf8`] flavour, the bytes must be well-formed + /// WTF-8 encoded string. + /// + /// It may be more convenient to use [`Bytes::from_str`] for `&str` and the + /// `From<&[u8]>` implementation for `&[u8]`. + pub unsafe fn new(bytes: &'a [u8]) -> Bytes<'a, F> { + Self(bytes, PhantomData) + } +} + +impl<'a, F: Flavour> Bytes<'a, F> { + /// Returns a byte slice of this `Bytes`'s contents. + pub fn as_bytes(self) -> &'a [u8] { + self.0 + } + + /// Returns the length of this `Bytes`, in bytes, not chars or graphemes. + pub fn len(self) -> usize { + self.0.len() + } + + /// Returns `true` if this `Bytes` has a length of zero, and `false` otherwise. + pub fn is_empty(self) -> bool { + self.0.is_empty() + } + + /// Adjusts range’s start position forward so it points at a potential valid + /// WTF-8 byte sequence. + /// + /// `range` represents a possibly invalid range within the bytes; + /// furthermore, `range.start` must be non-zero. This method returns a new + /// start index which is a valid split position. If `range` is already + /// a valid, the method simply returns `range.start`. + /// + /// When dealing with ill-formed WTF-8 sequences, this is not guaranteed to + /// advance position byte at a time. If you need to be able to advance + /// position byte at a time use `advance_range_start` instead. + fn adjust_position_fwd(self, range: Range) -> usize { + F::adjust_position_fwd(self.as_bytes(), range) + } + + /// Adjusts position backward so that it points at the closest potential + /// valid WTF-8 sequence. + /// + /// `range` represents a possibly invalid range within the bytes, + /// furthermore `range.end` must be less that bytes’ length. This method + /// returns a new end index which is a valid split position. If `range` is + /// already a valid, the method simply returns `range.end`. + /// + /// When dealing with ill-formed WTF-8 sequences, this is not guaranteed to + /// advance position byte at a time. If you need to be able to advance + /// position character at a time use `advance_range_end` instead. + fn adjust_position_bwd(self, range: Range) -> usize { + F::adjust_position_bwd(self.as_bytes(), range) + } + + /// Given a valid range update it’s start so it falls on the next character + /// boundary. + /// + /// `range` must be non-empty. If it starts with a valid WTF-8 sequence, + /// this method returns position pass that sequence. Otherwise, it returns + /// `range.start + 1`. In other words, well-formed WTF-8 bytes sequence are + /// skipped in one go while ill-formed sequences are skipped byte-by-byte. + fn advance_range_start(self, range: Range) -> usize { + range.start + F::advance_range_start(&self.as_bytes()[range]) + } + + /// Given a valid range update it’s end so it falls on the previous + /// character boundary. + /// + /// `range` must be non-empty. If it ends with a valid WTF-8 sequence, this + /// method returns position of the start of that sequence. Otherwise, it + /// returns `range.end - 1`. In other words, well-formed WTF-8 bytes + /// sequence are skipped in one go while ill-formed sequences are skipped + /// byte-by-byte. + fn advance_range_end(self, range: Range) -> usize { + range.start + F::advance_range_end(&self.as_bytes()[range]) + } + + /// Returns valid UTF-8 character at the front of the slice. + /// + /// If slice doesn’t start with a valid UTF-8 sequence, returns `None`. + /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. + /// WTF-8 sequences which encode surrogates are considered invalid. + fn get_first_code_point(self) -> Option<(char, usize)> { + F::get_first_code_point(self.as_bytes()) + } + + /// Returns valid UTF-8 character at the end of the slice. + /// + /// If slice doesn’t end with a valid UTF-8 sequence, returns `None`. + /// Otherwise returns decoded character and it’s UTF-8 encoding’s length. + /// WTF-8 sequences which encode surrogates are considered invalid. + fn get_last_code_point(self) -> Option<(char, usize)> { + F::get_last_code_point(self.as_bytes()) + } + + /// Looks for the next UTF-8-encoded character in the slice. + /// + /// WTF-8 sequences which encode surrogates are considered invalid. + /// + /// Returns position of the match, decoded character and UTF-8 length of + /// that character. + fn find_code_point_fwd(self, range: Range) -> Option<(usize, char, usize)> { + F::find_code_point_fwd(&self.as_bytes()[range.clone()]) + .map(|(pos, chr, len)| (range.start + pos, chr, len)) + } + + /// Looks backwards for the next UTF-8 encoded character in the slice. + /// + /// WTF-8 sequences which encode surrogates are considered invalid. + /// + /// Returns position of the match, decoded character and UTF-8 length of + /// that character. + fn find_code_point_bwd(&self, range: Range) -> Option<(usize, char, usize)> { + F::find_code_point_bwd(&self.as_bytes()[range.clone()]) + .map(|(pos, chr, len)| (range.start + pos, chr, len)) + } +} + +impl<'a> Bytes<'a, Unstructured> { + #[inline] + /// Wraps `&[u8]` into `Bytes`. + pub fn from_bytes(val: &'a [u8]) -> Self { + Self(val, PhantomData) + } +} + +impl<'a> Bytes<'a, Utf8> { + #[inline] + /// Wraps `&str` into `Bytes`. + pub fn from_str(val: &'a str) -> Self { + // SAFETY: `str` is guaranteed to be UTF-8 + unsafe { Bytes::new(val.as_bytes()) } + } + + /// Returns the contents of this `Bytes` as a `&str`. + #[inline] + pub fn into_str(self) -> &'a str { + if cfg!(debug_assertions) { + crate::str::from_utf8(self.as_bytes()).unwrap() + } else { + // SAFETY: Bytes with Utf8 flavor are guaranteed to be valid UTF-8 + unsafe { crate::str::from_utf8_unchecked(self.as_bytes()) } + } + } +} + +/// A marker for bytes slice which is not necessarily well-formed. +#[derive(Clone, Copy, Debug)] +pub enum Unstructured {} +/// A marker for well-formed WTF-8 bytes slice. +#[derive(Clone, Copy, Debug)] +pub enum Wtf8 {} +/// A marker for well-formed UTF-8 bytes slice. +#[derive(Clone, Copy, Debug)] +pub enum Utf8 {} + +/// A marker trait indicating ‘flavour’ of data referred by [`Bytes`] type. +/// +/// The trait abstracts away operations related to identifying and decoding +/// ‘characters’ from a bytes slice. A valid WTF-8 byte sequence is always +/// treated as indivisible ‘character’ but depending on the flavour code can +/// make different assumption about contents of the bytes slice: +/// - [`Unstructured`] flavoured bytes slice may contain ill-formed bytes +/// sequences and in those each byte is treated as separate ‘character’, +/// - [`Wtf8`] flavoured bytes slice is a well-formed WTF-8-encoded string (that +/// is some of the byte sequences may encode surrogate code points) and +/// - [`Utf8`] flavoured bytes slice is a well-formed UTF-8-encoded string (that +/// is all byte sequences encode valid Unicode code points). +pub trait Flavour: private::Flavour {} + +impl Flavour for Unstructured {} +impl Flavour for Wtf8 {} +impl Flavour for Utf8 {} + +mod private { + use super::*; + + /// Private methods of the [`super::Flavour`] trait. + pub trait Flavour: Copy + core::fmt::Debug { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize; + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize; + fn advance_range_start(bytes: &[u8]) -> usize; + fn advance_range_end(bytes: &[u8]) -> usize; + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)>; + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)>; + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)>; + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)>; + } + + impl Flavour for super::Unstructured { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + range.start + + bytes[range.clone()].iter().take_while(|chr| !chr.is_utf8_char_boundary()).count() + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + let shift = bytes[range.start..range.end + 1] + .iter() + .rev() + .take_while(|chr| !chr.is_utf8_char_boundary()) + .count(); + + range.end.saturating_sub(shift).max(range.start) + } + + fn advance_range_start(bytes: &[u8]) -> usize { + assert!(!bytes.is_empty()); + try_next_code_point(bytes).map_or(1, |(_, len)| len) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + assert!(!bytes.is_empty()); + bytes.len() - try_next_code_point_reverse(bytes).map_or(1, |(_, len)| len) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + try_next_code_point(bytes) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + try_next_code_point_reverse(bytes) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + (0..bytes.len()) + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((pos, chr, len)) + }) + .next() + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + (0..bytes.len()) + .rev() + .filter_map(|pos| { + let (chr, len) = try_next_code_point(&bytes[pos..])?; + Some((pos, chr, len)) + }) + .next() + } + } + + impl Flavour for Wtf8 { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + let mut pos = range.start; + // Input is WTF-8 so we will never need to move more than three + // positions. This happens when we’re at pointing at the first + // continuation byte of a four-byte sequence. Unroll the loop. + for _ in 0..3 { + // We’re not checking pos against _end because we know that _end + // == bytes.len() or falls on a character boundary. We can + // therefore compare against bytes.len() and eliminate that + // comparison. + if bytes.get(pos).map_or(true, |b: &u8| b.is_utf8_char_boundary()) { + break; + } + pos += 1; + } + pos + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + let mut pos = range.end; + // Input is WTF-8 so we will never need to move more than three + // positions. This happens when we’re at pointing at the first + // continuation byte of a four-byte sequence. Unroll the loop. + for _ in 0..3 { + // SAFETY: `bytes` is well-formed WTF-8 sequence and at function + // start `pos` is index within `bytes`. Therefore, `bytes[pos]` + // is valid and a) if it’s a character boundary we exit the + // function or b) otherwise we know that `pos > 0` (because + // otherwise `bytes` wouldn’t be well-formed WTF-8). + if unsafe { bytes.get_unchecked(pos) }.is_utf8_char_boundary() { + break; + } + pos -= 1; + } + pos + } + + fn advance_range_start(bytes: &[u8]) -> usize { + // Input is valid WTF-8 so we can just deduce length of next + // sequence to skip from the first byte. + utf8_char_width(*bytes.get(0).unwrap()) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + let end = bytes.len().checked_sub(1).unwrap(); + Self::adjust_position_bwd(bytes, 0..end) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point(&mut bytes.iter())? }; + // WTF-8 might produce surrogate code points so we still need to + // verify that we got a valid character. + char::from_u32(cp).map(|chr| (chr, len_utf8(cp))) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point_reverse(&mut bytes.iter())? }; + // WTF-8 might produce surrogate code points so we still need to + // verify that we got a valid character. + char::from_u32(cp).map(|chr| (chr, len_utf8(cp))) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + let mut iter = bytes.iter(); + let mut pos = 0; + loop { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point(&mut iter)? }; + let len = len_utf8(cp); + if let Some(chr) = char::from_u32(cp) { + return Some((pos, chr, len)); + } + pos += len; + } + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + let mut iter = bytes.iter(); + let mut pos = bytes.len(); + loop { + // SAFETY: We’re Wtf8 flavour. Client promises that bytes are + // well-formed WTF-8. + let cp = unsafe { next_code_point_reverse(&mut iter)? }; + let len = len_utf8(cp); + pos -= len; + if let Some(chr) = char::from_u32(cp) { + return Some((pos, chr, len)); + } + } + } + } + + impl Flavour for Utf8 { + fn adjust_position_fwd(bytes: &[u8], range: Range) -> usize { + Wtf8::adjust_position_fwd(bytes, range) + } + + fn adjust_position_bwd(bytes: &[u8], range: Range) -> usize { + Wtf8::adjust_position_bwd(bytes, range) + } + + fn advance_range_start(bytes: &[u8]) -> usize { + Wtf8::advance_range_start(bytes) + } + + fn advance_range_end(bytes: &[u8]) -> usize { + Wtf8::advance_range_end(bytes) + } + + fn get_first_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let (_, chr, len) = Self::find_code_point_fwd(bytes)?; + Some((chr, len)) + } + + fn get_last_code_point(bytes: &[u8]) -> Option<(char, usize)> { + let (_, chr, len) = Self::find_code_point_bwd(bytes)?; + Some((chr, len)) + } + + fn find_code_point_fwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + // SAFETY: We’re Utf8 flavour. Client promises that bytes are + // well-formed UTF-8. We can not only assume well-formed byte + // sequence but also that produced code points are valid. + let chr = unsafe { char::from_u32_unchecked(next_code_point(&mut bytes.iter())?) }; + let len = chr.len_utf8(); + Some((0, chr, len)) + } + + fn find_code_point_bwd(bytes: &[u8]) -> Option<(usize, char, usize)> { + // SAFETY: We’re Utf8 flavour. Client promises that bytes are + // well-formed UTF-8. We can not only assume well-formed byte + // sequence but also that produced code points are valid. + let chr = unsafe { + let code = next_code_point_reverse(&mut bytes.iter())?; + char::from_u32_unchecked(code) + }; + let len = chr.len_utf8(); + Some((bytes.len() - len, chr, len)) + } + } + + // Copied from src/chars/methods.rs. We need it because it’s not public + // there and char::len_utf8 requires us to have a char and we need this to + // work on surrogate code points as well. + #[inline] + const fn len_utf8(code: u32) -> usize { + if code < 0x80 { + 1 + } else if code < 0x800 { + 2 + } else if code < 0x10000 { + 3 + } else { + 4 + } + } +} + +trait SearchResult: crate::pattern::SearchResult { + /// Adjusts reject’s start position backwards to make sure it doesn’t fall + /// within well-formed WTF-8 sequence. + /// + /// Doesn’t move the start position past `begin`. If position was adjusted, + /// updates `*out` as well. + fn adjust_reject_start_bwd( + self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self; + + /// Adjusts reject’s end position forwards to make sure it doesn’t fall + /// within well-formed WTF-8 sequence. + /// + /// Doesn’t move the end position past `len`. If position was adjusted, + /// updates `*out` as well. + fn adjust_reject_end_fwd( + self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self; +} + +impl SearchResult for SearchStep { + fn adjust_reject_start_bwd( + mut self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self { + if let SearchStep::Reject(ref mut start, _) = self { + *start = bytes.adjust_position_bwd(begin..*start); + *out = *start; + } + self + } + fn adjust_reject_end_fwd( + mut self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self { + if let SearchStep::Reject(_, ref mut end) = self { + *end = bytes.adjust_position_fwd(*end..len); + *out = *end; + } + self + } +} + +impl SearchResult for MatchOnly { + fn adjust_reject_start_bwd( + self, + _bytes: Bytes<'_, F>, + _begin: usize, + _out: &mut usize, + ) -> Self { + self + } + fn adjust_reject_end_fwd( + self, + _bytes: Bytes<'_, F>, + _end: usize, + _out: &mut usize, + ) -> Self { + self + } +} + +impl SearchResult for RejectOnly { + fn adjust_reject_start_bwd( + mut self, + bytes: Bytes<'_, F>, + begin: usize, + out: &mut usize, + ) -> Self { + if let RejectOnly(Some((ref mut start, _))) = self { + *start = bytes.adjust_position_bwd(begin..*start); + *out = *start; + } + self + } + fn adjust_reject_end_fwd( + mut self, + bytes: Bytes<'_, F>, + len: usize, + out: &mut usize, + ) -> Self { + if let RejectOnly(Some((_, ref mut end))) = self { + *end = bytes.adjust_position_fwd(*end..len); + *out = *end; + } + self + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl for Haystack +//////////////////////////////////////////////////////////////////////////////// + +impl<'hs, F: Flavour> Haystack for Bytes<'hs, F> { + fn cursor_at_front(self) -> usize { + 0 + } + fn cursor_at_back(self) -> usize { + self.0.len() + } + fn is_empty(self) -> bool { + self.0.is_empty() + } + + unsafe fn get_unchecked(self, range: Range) -> Self { + Self( + if cfg!(debug_assertions) { + self.0.get(range).unwrap() + } else { + // SAFETY: Caller promises cursor is a valid split position. + unsafe { self.0.get_unchecked(range) } + }, + PhantomData, + ) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for char +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs, F: Flavour> pattern::Pattern> for char { + type Searcher = CharSearcher<'hs, F>; + + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_contained_in(self, haystack: Bytes<'hs, F>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_contained_in(haystack) + } + + fn is_prefix_of(self, haystack: Bytes<'hs, F>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_prefix_of(haystack) + } + fn strip_prefix_of(self, haystack: Bytes<'hs, F>) -> Option> { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).strip_prefix_of(haystack) + } + + fn is_suffix_of(self, haystack: Bytes<'hs, F>) -> bool { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).is_suffix_of(haystack) + } + fn strip_suffix_of(self, haystack: Bytes<'hs, F>) -> Option> { + let mut buf = [0; 4]; + encode_utf8(self, &mut buf).strip_suffix_of(haystack) + } +} + +/// Like `chr.encode_utf8(&mut buf)` but casts result to `&str`. +/// +/// This is useful because we have Pattern impl for &str but not for &mut str. +fn encode_utf8(chr: char, buf: &mut [u8; 4]) -> &str { + chr.encode_utf8(buf) +} + +/// Searcher looking for a single character in the haystack. +#[derive(Clone, Debug)] +pub struct CharSearcher<'hs, F> { + haystack: Bytes<'hs, F>, + state: CharSearcherInner, +} + +/// Dispatches searches over two different needle flavors: +/// +/// - [`ByteSearcherState`] works with ASCII `u8` chars and uses faster `memchr`/`memrchr` +/// - [`CharSearcherState`] works with any chars but works a bit slower. +#[derive(Clone, Debug)] +enum CharSearcherInner { + Byte(ByteSearcherState), + Char(CharSearcherState), +} + +impl CharSearcherInner { + #[inline] + fn new(haystack_len: usize, chr: char) -> Self { + if chr.is_ascii() { + Self::Byte(ByteSearcherState::new(haystack_len, chr as u8)) + } else { + Self::Char(CharSearcherState::new(haystack_len, chr)) + } + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Byte(state) => state.next_fwd::(haystack), + Self::Char(state) => state.next_fwd::(haystack), + } + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Byte(state) => state.next_bwd::(haystack), + Self::Char(state) => state.next_bwd::(haystack), + } + } +} + +#[derive(Clone, Debug)] +struct CharSearcherState { + /// Not yet processed range of the haystack. + range: crate::ops::Range, + /// Needle the searcher is looking for within the haystack. + needle: CharBuffer, + /// If `true` and `range` is non-empty, `haystack[range]` starts with the + /// needle. + is_match_fwd: bool, + /// If `true` and `range` is non-empty, `haystack[range]` ends with the + /// needle. + is_match_bwd: bool, +} + +impl<'hs, F: Flavour> CharSearcher<'hs, F> { + /// Creates a new searcher for the given character. + #[inline] + pub fn new(haystack: Bytes<'hs, F>, chr: char) -> Self { + Self { haystack, state: CharSearcherInner::new(haystack.len(), chr) } + } +} + +unsafe impl<'hs, F: Flavour> pattern::Searcher> for CharSearcher<'hs, F> { + #[inline] + fn haystack(&self) -> Bytes<'hs, F> { + self.haystack + } + + #[inline] + fn next(&mut self) -> SearchStep { + self.state.next_fwd(self.haystack) + } + #[inline] + fn next_match(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } + #[inline] + fn next_reject(&mut self) -> OptRange { + self.state.next_fwd::(self.haystack).0 + } +} + +unsafe impl<'hs, F: Flavour> pattern::ReverseSearcher> for CharSearcher<'hs, F> { + #[inline] + fn next_back(&mut self) -> SearchStep { + self.state.next_bwd(self.haystack) + } + #[inline] + fn next_match_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } + #[inline] + fn next_reject_back(&mut self) -> OptRange { + self.state.next_bwd::(self.haystack).0 + } +} + +impl<'hs, F: Flavour> pattern::DoubleEndedSearcher> for CharSearcher<'hs, F> {} + +impl CharSearcherState { + #[inline] + fn new(haystack_len: usize, chr: char) -> Self { + Self { + range: 0..haystack_len, + needle: CharBuffer::new(chr), + is_match_fwd: false, + is_match_bwd: false, + } + } + + #[inline] + fn find_match_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_fwd) { + (!self.range.is_empty()).then_some(self.range.start) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + // SAFETY: self.needle encodes a single character. + unsafe { naive::find_match_fwd(bytes.as_bytes(), self.needle.as_str()) } + .map(|pos| pos + self.range.start) + }?; + Some((start, start + self.needle.len())) + } + + #[inline] + fn next_reject_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_fwd) { + if self.range.is_empty() { + return None; + } + self.range.start += self.needle.len() + } + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + if let Some(pos) = naive::find_reject_fwd(bytes.as_bytes(), self.needle.as_str()) { + let pos = pos + self.range.start; + let end = haystack.advance_range_start(pos..self.range.end); + self.range.start = end; + Some((pos, end)) + } else { + self.range.start = self.range.end; + None + } + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_fwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_fwd(haystack) { + if self.range.start < start { + if let Some(res) = R::rejecting(self.range.start, start) { + self.range.start = start; + self.is_match_fwd = true; + return res; + } + } + self.range.start = end; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let start = self.range.start; + self.range.start = self.range.end; + R::rejecting(start, self.range.end).unwrap_or(R::DONE) + } + } + + #[inline] + fn find_match_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_bwd) { + (!self.range.is_empty()).then(|| self.range.end - self.needle.len()) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + // SAFETY: self.needle encodes a single character. + unsafe { naive::find_match_bwd(bytes.as_bytes(), self.needle.as_str()) } + .map(|pos| pos + self.range.start) + }?; + Some((start, start + self.needle.len())) + } + + #[inline] + fn next_reject_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_bwd) { + if self.range.is_empty() { + return None; + } + self.range.end -= self.needle.len(); + } + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.get_unchecked(self.range.clone()) }; + if let Some(end) = naive::find_reject_bwd(bytes.as_bytes(), self.needle.as_str()) { + let end = end + self.range.start; + let start = haystack.advance_range_end(self.range.start..end); + self.range.end = start; + Some((start, end)) + } else { + self.range.end = self.range.start; + None + } + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_bwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_bwd(haystack) { + if end < self.range.end { + if let Some(res) = R::rejecting(end, self.range.end) { + self.range.end = end; + self.is_match_bwd = true; + return res; + } + } + self.range.end = start; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let end = self.range.end; + self.range.end = self.range.start; + R::rejecting(self.range.start, end).unwrap_or(R::DONE) + } + } +} + +#[derive(Clone, Debug)] +struct CharBuffer([u8; 4], crate::num::NonZeroU8); + +impl CharBuffer { + #[inline] + fn new(chr: char) -> Self { + let mut buf = [0; 4]; + let len = chr.encode_utf8(&mut buf).len(); + // SAFETY: `len` is length of a single character UTF-8 sequence. + let len = unsafe { crate::num::NonZeroU8::new_unchecked(len as u8) }; + Self(buf, len) + } + + #[inline] + fn len(&self) -> usize { + usize::from(self.1.get()) + } + + #[inline] + fn as_str(&self) -> &str { + // SAFETY: `self.0` is UTF-8 encoding of a single character and `self.1` + // is its length. See `new` constructor. + unsafe { crate::str::from_utf8_unchecked(self.0.get_unchecked(..self.len())) } + } +} + +/// Fast searcher for a single-byte needle using [`memchr`]. +/// +/// A single-byte `&str` needle is always an ASCII byte, which can never appear +/// as a continuation byte of a well-formed WTF-8 sequence, so a memchr hit is +/// always on a character boundary and always a full single-byte match. +#[derive(Clone, Debug)] +struct ByteSearcherState { + /// Not yet processed range of the haystack. + range: crate::ops::Range, + /// Needle (ASCII byte) the searcher is looking for within the haystack. + needle: u8, + /// If `true` and `range` is non-empty, `haystack[range]` starts with the + /// needle. + is_match_fwd: bool, + /// If `true` and `range` is non-empty, `haystack[range]` ends with the + /// needle. + is_match_bwd: bool, +} + +impl ByteSearcherState { + #[inline] + fn new(haystack_len: usize, needle: u8) -> Self { + Self { needle, range: 0..haystack_len, is_match_fwd: false, is_match_bwd: false } + } + + #[inline] + fn find_match_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_fwd) { + (!self.range.is_empty()).then_some(self.range.start) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.as_bytes().get_unchecked(self.range.clone()) }; + memchr::memchr(self.needle, bytes).map(|i| self.range.start + i) + }?; + Some((start, start + 1)) + } + + #[inline] + fn next_reject_fwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_fwd) { + if self.range.is_empty() { + return None; + } + self.range.start += 1; + } + let bytes = haystack.as_bytes(); + while self.range.start < self.range.end { + let start = self.range.start; + if bytes[start] == self.needle { + self.range.start += 1; + } else { + // `start` is always on a character boundary, so this rejects + // exactly the character starting at `start`. + let end = haystack.advance_range_start(self.range.clone()); + self.range.start = end; + return Some((start, end)); + } + } + None + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_fwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_fwd(haystack) { + if self.range.start < start { + if let Some(res) = R::rejecting(self.range.start, start) { + self.range.start = start; + self.is_match_fwd = true; + return res; + } + } + self.range.start = end; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let start = self.range.start; + self.range.start = self.range.end; + R::rejecting(start, self.range.end).unwrap_or(R::DONE) + } + } + + #[inline] + fn find_match_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + let start = if take(&mut self.is_match_bwd) { + (!self.range.is_empty()).then(|| self.range.end - 1) + } else { + // SAFETY: self.range is valid range of haystack. + let bytes = unsafe { haystack.as_bytes().get_unchecked(self.range.clone()) }; + memchr::memrchr(self.needle, bytes).map(|i| self.range.start + i) + }?; + Some((start, start + 1)) + } + + #[inline] + fn next_reject_bwd(&mut self, haystack: Bytes<'_, F>) -> OptRange { + if take(&mut self.is_match_bwd) { + if self.range.is_empty() { + return None; + } + self.range.end -= 1; + } + let bytes = haystack.as_bytes(); + while !self.range.is_empty() { + let end = self.range.end; + if bytes[end - 1] == self.needle { + self.range.end -= 1; + } else { + let start = haystack.advance_range_end(self.range.start..end); + self.range.end = start; + return Some((start, end)); + } + } + None + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + if R::USE_EARLY_REJECT { + match self.next_reject_bwd(haystack) { + Some((start, end)) => R::rejecting(start, end).unwrap(), + None => R::DONE, + } + } else if let Some((start, end)) = self.find_match_bwd(haystack) { + if end < self.range.end { + if let Some(res) = R::rejecting(end, self.range.end) { + self.range.end = end; + self.is_match_bwd = true; + return res; + } + } + self.range.end = start; + R::matching(start, end).unwrap() + } else if self.range.is_empty() { + R::DONE + } else { + let end = self.range.end; + self.range.end = self.range.start; + R::rejecting(self.range.start, end).unwrap_or(R::DONE) + } + } +} + +mod naive { + use crate::slice::memchr; + + /// Looks forwards for the next position of needle within haystack. + /// + /// Safety: `needle` must consist of a single character. + #[inline] + pub(super) unsafe fn find_match_fwd(haystack: &[u8], needle: &str) -> Option { + debug_assert!(!needle.is_empty()); + // SAFETY: Caller promises needle is non-empty. + let (&last_byte, head) = unsafe { needle.as_bytes().split_last().unwrap_unchecked() }; + let mut start = 0; + while haystack.len() - start > head.len() { + // SAFETY: + // 1. `start` is initialised to `self.start` and only ever increased + // thus `self.start ≤ start`. + // 2. We've checked `start + head.len() < haystack.len()`. + let bytes = unsafe { haystack.get_unchecked(start + head.len()..) }; + if let Some(index) = memchr::memchr(last_byte, bytes) { + // `start + index + head.len()` is the index of the last byte + // thus `start + index` is the index of the first byte. + let pos = start + index; + // SAFETY: Since we’ve started our search with head.len() + // offset, we know we have at least head.len() bytes in buffer. + if unsafe { haystack.get_unchecked(pos..pos + head.len()) } == head { + return Some(pos); + } + start += index + 1; + } else { + break; + } + } + None + } + + /// Looks backwards for the next position of needle within haystack. + /// + /// Safety: `needle` must consist of a single character. + #[inline] + pub(super) unsafe fn find_match_bwd(haystack: &[u8], needle: &str) -> Option { + // SAFETY: Caller promises needle is non-empty. + let (&first_byte, tail) = unsafe { needle.as_bytes().split_first().unwrap_unchecked() }; + let mut end = haystack.len(); + while end > tail.len() { + // SAFETY: + // 1. `end` is initialised to `haystack.len()` and only ever + // decreased thus `end ≤ haystack.len()`. + // 2. We've checked `end > tail.len()`. + let bytes = unsafe { haystack.get_unchecked(..end - tail.len()) }; + if let Some(pos) = memchr::memrchr(first_byte, bytes) { + // SAFETY: Since we’ve stopped our search with tail.len() + // offset, we know we have at least tail.len() bytes in buffer + // after position of the byte we’ve found. + if unsafe { haystack.get_unchecked(pos + 1..pos + 1 + tail.len()) } == tail { + return Some(pos); + } + end = pos; + } else { + break; + } + } + None + } + + /// Looks forwards for the next position where needle stops matching. + /// + /// Returns start of the next reject or `None` if there is no reject. + pub(super) fn find_reject_fwd(haystack: &[u8], needle: &str) -> Option { + let count = + haystack.chunks(needle.len()).take_while(|&slice| slice == needle.as_bytes()).count(); + let start = count * needle.len(); + (start < haystack.len()).then_some(start) + } + + /// Looks backwards for the next position where needle stops matching. + /// + /// Returns end of the next reject or `None` if there is no reject. + pub(super) fn find_reject_bwd(haystack: &[u8], needle: &str) -> Option { + debug_assert!(!needle.is_empty()); + let count = + haystack.rchunks(needle.len()).take_while(|&slice| slice == needle.as_bytes()).count(); + let end = haystack.len() - count * needle.len(); + (end > 0).then_some(end) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for FnMut(char) and FnMut(Result) +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs, F: Flavour, P: FnMut(char) -> bool> pattern::Pattern> for P { + type Searcher = PredicateSearcher<'hs, F, P>; + + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_prefix_of(mut self, haystack: Bytes<'hs, F>) -> bool { + haystack.get_first_code_point().map_or(false, |(chr, _)| self(chr)) + } + fn strip_prefix_of(mut self, haystack: Bytes<'hs, F>) -> Option> { + let (chr, len) = haystack.get_first_code_point()?; + // SAFETY: We’ve just checked slice starts with len-byte long + // well-formed sequence. + self(chr).then(|| unsafe { haystack.get_unchecked(len..haystack.len()) }) + } + + fn is_suffix_of(mut self, haystack: Bytes<'hs, F>) -> bool { + haystack.get_last_code_point().map_or(false, |(chr, _)| self(chr)) + } + fn strip_suffix_of(mut self, haystack: Bytes<'hs, F>) -> Option> { + let (chr, len) = haystack.get_last_code_point()?; + let len = haystack.len() - len; + // SAFETY: We’ve just checked slice ends with len-byte long well-formed + // sequence. + self(chr).then(|| unsafe { haystack.get_unchecked(0..len) }) + } +} + +/// Searcher looking for characters matching a predicate. +#[derive(Clone, Debug)] +pub struct PredicateSearcher<'hs, F, P> { + haystack: Bytes<'hs, F>, + pred: P, + start: usize, + end: usize, + fwd_match_len: u8, + bwd_match_len: u8, +} + +impl<'hs, F: Flavour, P> PredicateSearcher<'hs, F, P> { + /// Creates a new searcher for the given predicate. + #[inline] + pub fn new(haystack: Bytes<'hs, F>, pred: P) -> Self { + Self { haystack, pred, start: 0, end: haystack.len(), fwd_match_len: 0, bwd_match_len: 0 } + } +} + +impl<'hs, F: Flavour, P: FnMut(char) -> bool> PredicateSearcher<'hs, F, P> { + fn find_match_fwd(&mut self) -> Option<(usize, usize)> { + let mut start = self.start; + while start < self.end { + let (idx, chr, len) = self.haystack.find_code_point_fwd(start..self.end)?; + if (self.pred)(chr) { + return Some((idx, len)); + } + start = idx + len; + } + None + } + + fn find_match_bwd(&mut self) -> Option<(usize, usize)> { + let mut end = self.end; + while self.start < end { + let (idx, chr, len) = self.haystack.find_code_point_bwd(self.start..end)?; + if (self.pred)(chr) { + return Some((idx, len)); + } + end = idx; + } + None + } + + fn next_fwd(&mut self) -> R { + while self.start < self.end { + if self.fwd_match_len == 0 { + let (pos, len) = self.find_match_fwd().unwrap_or((self.end, 0)); + self.fwd_match_len = len as u8; + if pos != self.start { + let start = self.start; + self.start = pos; + if let Some(ret) = R::rejecting(start, pos) { + return ret; + } else if pos >= self.end { + break; + } + } + } + + let pos = self.start; + self.start += usize::from(take(&mut self.fwd_match_len)); + if let Some(ret) = R::matching(pos, self.start) { + return ret; + } + } + R::DONE + } + + fn next_bwd(&mut self) -> R { + while self.start < self.end { + if self.bwd_match_len == 0 { + let (pos, len) = self.find_match_bwd().unwrap_or((self.start, 0)); + self.bwd_match_len = len as u8; + let pos = pos + len; + let end = self.end; + if pos != self.end { + self.end = pos; + if let Some(ret) = R::rejecting(pos, end) { + return ret; + } else if self.start >= self.end { + break; + } + } + } + + let end = self.end; + self.end -= usize::from(take(&mut self.bwd_match_len)); + if let Some(ret) = R::matching(self.end, end) { + return ret; + } + } + R::DONE + } +} + +unsafe impl<'hs, F, P> pattern::Searcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, +{ + fn haystack(&self) -> Bytes<'hs, F> { + self.haystack + } + fn next(&mut self) -> SearchStep { + self.next_fwd() + } + fn next_match(&mut self) -> OptRange { + self.next_fwd::().0 + } + fn next_reject(&mut self) -> OptRange { + self.next_fwd::().0 + } +} + +unsafe impl<'hs, F, P> pattern::ReverseSearcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, +{ + fn next_back(&mut self) -> SearchStep { + self.next_bwd() + } + fn next_match_back(&mut self) -> OptRange { + self.next_bwd::().0 + } + fn next_reject_back(&mut self) -> OptRange { + self.next_bwd::().0 + } +} + +impl<'hs, F, P> pattern::DoubleEndedSearcher> for PredicateSearcher<'hs, F, P> +where + F: Flavour, + P: FnMut(char) -> bool, +{ +} + +//////////////////////////////////////////////////////////////////////////////// +// Impl Pattern for &str +//////////////////////////////////////////////////////////////////////////////// + +#[diagnostic::do_not_recommend] +impl<'hs, 'p, F: Flavour> pattern::Pattern> for &'p str { + type Searcher = StrSearcher<'hs, 'p, F>; + + fn into_searcher(self, haystack: Bytes<'hs, F>) -> Self::Searcher { + Self::Searcher::new(haystack, self) + } + + fn is_prefix_of(self, haystack: Bytes<'hs, F>) -> bool { + haystack.as_bytes().starts_with(self.as_bytes()) + } + fn strip_prefix_of(self, haystack: Bytes<'hs, F>) -> Option> { + haystack.as_bytes().strip_prefix(self.as_bytes()).map(|bytes| Bytes(bytes, PhantomData)) + } + + fn is_suffix_of(self, haystack: Bytes<'hs, F>) -> bool { + haystack.as_bytes().ends_with(self.as_bytes()) + } + fn strip_suffix_of(self, haystack: Bytes<'hs, F>) -> Option> { + haystack.as_bytes().strip_suffix(self.as_bytes()).map(|bytes| Bytes(bytes, PhantomData)) + } +} + +/// Searcher looking for a substring in the haystack. +#[derive(Clone, Debug)] +pub struct StrSearcher<'hs, 'p, F> { + haystack: Bytes<'hs, F>, + inner: StrSearcherInner<'p>, +} + +impl<'hs, 'p, F: Flavour> StrSearcher<'hs, 'p, F> { + /// Creates a new searcher for the given substring. + #[inline] + pub fn new(haystack: Bytes<'hs, F>, needle: &'p str) -> Self { + let inner = StrSearcherInner::new(haystack, needle); + Self { haystack, inner } + } +} + +unsafe impl<'hs, 'p, F: Flavour> pattern::Searcher> for StrSearcher<'hs, 'p, F> { + #[inline] + fn haystack(&self) -> Bytes<'hs, F> { + self.haystack + } + #[inline] + fn next(&mut self) -> SearchStep { + self.inner.next_fwd(self.haystack) + } + #[inline] + fn next_match(&mut self) -> OptRange { + self.inner.next_fwd::(self.haystack).0 + } + #[inline] + fn next_reject(&mut self) -> OptRange { + self.inner.next_fwd::(self.haystack).0 + } +} + +unsafe impl<'hs, 'p, F: Flavour> pattern::ReverseSearcher> + for StrSearcher<'hs, 'p, F> +{ + #[inline] + fn next_back(&mut self) -> SearchStep { + self.inner.next_bwd(self.haystack) + } + #[inline] + fn next_match_back(&mut self) -> OptRange { + self.inner.next_bwd::(self.haystack).0 + } + #[inline] + fn next_reject_back(&mut self) -> OptRange { + self.inner.next_bwd::(self.haystack).0 + } +} + +#[derive(Clone, Debug)] +enum StrSearcherInner<'p> { + Empty(EmptySearcherState), + Byte(ByteSearcherState), + Char(CharSearcherState), + Str(StrSearcherState<'p>), +} + +impl<'p> StrSearcherInner<'p> { + #[inline] + fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { + let mut chars = needle.chars(); + let chr = match chars.next() { + Some(chr) => chr, + None => return Self::Empty(EmptySearcherState::new(haystack)), + }; + if let &[b] = needle.as_bytes() { + Self::Byte(ByteSearcherState::new(haystack.len(), b)) + } else if chars.next().is_none() { + Self::Char(CharSearcherState::new(haystack.len(), chr)) + } else { + Self::Str(StrSearcherState::new(haystack, needle)) + } + } + + #[inline] + fn next_fwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Empty(state) => state.next_fwd::(haystack), + Self::Byte(state) => state.next_fwd::(haystack), + Self::Char(state) => state.next_fwd::(haystack), + Self::Str(state) => state.next_fwd::(haystack), + } + } + + #[inline] + fn next_bwd(&mut self, haystack: Bytes<'_, F>) -> R { + match self { + Self::Empty(state) => state.next_bwd::(haystack), + Self::Byte(state) => state.next_bwd::(haystack), + Self::Char(state) => state.next_bwd::(haystack), + Self::Str(state) => state.next_bwd::(haystack), + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Empty needle searching +//////////////////////////////////////////////////////////////////////////////// + +/// Empty needle rejects every character and matches every character boundary. +/// +/// A character is either a well-formed WTF-8 bytes sequence or a single byte +/// whichever is longer. +#[derive(Clone, Debug)] +struct EmptySearcherState(pattern::EmptyNeedleSearcher); + +impl EmptySearcherState { + fn new(haystack: Bytes<'_, F>) -> Self { + Self(pattern::EmptyNeedleSearcher::new(haystack)) + } + + fn next_fwd(&mut self, bytes: Bytes<'_, F>) -> R { + self.0.next_fwd(|range| bytes.advance_range_start(range)) + } + + fn next_bwd(&mut self, bytes: Bytes<'_, F>) -> R { + self.0.next_bwd(|range| bytes.advance_range_end(range)) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Full substring search +//////////////////////////////////////////////////////////////////////////////// + +/// A substring search. +#[derive(Clone, Debug)] +struct StrSearcherState<'p> { + needle: &'p str, + searcher: TwoWaySearcher, +} + +impl<'p> StrSearcherState<'p> { + #[inline] + fn new(haystack: Bytes<'_, F>, needle: &'p str) -> Self { + let searcher = TwoWaySearcher::new(haystack.len(), needle.as_bytes()); + Self { needle, searcher } + } + + #[inline] + fn next_fwd(&mut self, bytes: Bytes<'_, F>) -> R { + if self.searcher.position >= bytes.len() { + return R::DONE; + } + if self.searcher.memory == usize::MAX { + self.searcher.next_fwd::(bytes.0, self.needle.as_bytes(), true) + } else { + self.searcher.next_fwd::(bytes.0, self.needle.as_bytes(), false) + } + .adjust_reject_end_fwd(bytes, bytes.len(), &mut self.searcher.position) + } + + #[inline] + fn next_bwd(&mut self, bytes: Bytes<'_, F>) -> R { + if self.searcher.end == 0 { + return R::DONE; + } + if self.searcher.memory == usize::MAX { + self.searcher.next_bwd::(bytes.0, self.needle.as_bytes(), true) + } else { + self.searcher.next_bwd::(bytes.0, self.needle.as_bytes(), false) + } + .adjust_reject_start_bwd(bytes, 0, &mut self.searcher.end) + } +} + +/// The internal state of the two-way substring search algorithm. +#[derive(Clone, Debug)] +struct TwoWaySearcher { + // constants + /// critical factorization index + crit_pos: usize, + /// critical factorization index for reversed needle + crit_pos_back: usize, + period: usize, + /// `byteset` is an extension (not part of the two way algorithm); + /// it's a 64-bit "fingerprint" where each set bit `j` corresponds + /// to a (byte & 63) == j present in the needle. + byteset: u64, + + // variables + position: usize, + end: usize, + /// index into needle before which we have already matched + memory: usize, + /// index into needle after which we have already matched + memory_back: usize, +} + +/* + This is the Two-Way search algorithm, which was introduced in the paper: + Crochemore, M., Perrin, D., 1991, Two-way string-matching, Journal of the ACM 38(3):651-675. + + Here's some background information. + + A *word* is a string of symbols. The *length* of a word should be a familiar + notion, and here we denote it for any word x by |x|. + (We also allow for the possibility of the *empty word*, a word of length zero). + + If x is any non-empty word, then an integer p with 0 < p <= |x| is said to be a + *period* for x iff for all i with 0 <= i <= |x| - p - 1, we have x[i] == x[i+p]. + For example, both 1 and 2 are periods for the string "aa". As another example, + the only period of the string "abcd" is 4. + + We denote by period(x) the *smallest* period of x (provided that x is non-empty). + This is always well-defined since every non-empty word x has at least one period, + |x|. We sometimes call this *the period* of x. + + If u, v and x are words such that x = uv, where uv is the concatenation of u and + v, then we say that (u, v) is a *factorization* of x. + + Let (u, v) be a factorization for a word x. Then if w is a non-empty word such + that both of the following hold + + - either w is a suffix of u or u is a suffix of w + - either w is a prefix of v or v is a prefix of w + + then w is said to be a *repetition* for the factorization (u, v). + + Just to unpack this, there are four possibilities here. Let w = "abc". Then we + might have: + + - w is a suffix of u and w is a prefix of v. ex: ("lolabc", "abcde") + - w is a suffix of u and v is a prefix of w. ex: ("lolabc", "ab") + - u is a suffix of w and w is a prefix of v. ex: ("bc", "abchi") + - u is a suffix of w and v is a prefix of w. ex: ("bc", "a") + + Note that the word vu is a repetition for any factorization (u,v) of x = uv, + so every factorization has at least one repetition. + + If x is a string and (u, v) is a factorization for x, then a *local period* for + (u, v) is an integer r such that there is some word w such that |w| = r and w is + a repetition for (u, v). + + We denote by local_period(u, v) the smallest local period of (u, v). We sometimes + call this *the local period* of (u, v). Provided that x = uv is non-empty, this + is well-defined (because each non-empty word has at least one factorization, as + noted above). + + It can be proven that the following is an equivalent definition of a local period + for a factorization (u, v): any positive integer r such that x[i] == x[i+r] for + all i such that |u| - r <= i <= |u| - 1 and such that both x[i] and x[i+r] are + defined. (i.e., i > 0 and i + r < |x|). + + Using the above reformulation, it is easy to prove that + + 1 <= local_period(u, v) <= period(uv) + + A factorization (u, v) of x such that local_period(u,v) = period(x) is called a + *critical factorization*. + + The algorithm hinges on the following theorem, which is stated without proof: + + **Critical Factorization Theorem** Any word x has at least one critical + factorization (u, v) such that |u| < period(x). + + The purpose of maximal_suffix is to find such a critical factorization. + + If the period is short, compute another factorization x = u' v' to use + for reverse search, chosen instead so that |v'| < period(x). + +*/ +impl TwoWaySearcher { + fn new(haystack_len: usize, needle: &[u8]) -> TwoWaySearcher { + let (crit_pos_false, period_false) = TwoWaySearcher::maximal_suffix(needle, false); + let (crit_pos_true, period_true) = TwoWaySearcher::maximal_suffix(needle, true); + + let (crit_pos, period) = if crit_pos_false > crit_pos_true { + (crit_pos_false, period_false) + } else { + (crit_pos_true, period_true) + }; + + // A particularly readable explanation of what's going on here can be found + // in Crochemore and Rytter's book "Text Algorithms", ch 13. Specifically + // see the code for "Algorithm CP" on p. 323. + // + // What's going on is we have some critical factorization (u, v) of the + // needle, and we want to determine whether u is a suffix of + // &v[..period]. If it is, we use "Algorithm CP1". Otherwise we use + // "Algorithm CP2", which is optimized for when the period of the needle + // is large. + if needle[..crit_pos] == needle[period..period + crit_pos] { + // short period case -- the period is exact + // compute a separate critical factorization for the reversed needle + // x = u' v' where |v'| < period(x). + // + // This is sped up by the period being known already. + // Note that a case like x = "acba" may be factored exactly forwards + // (crit_pos = 1, period = 3) while being factored with approximate + // period in reverse (crit_pos = 2, period = 2). We use the given + // reverse factorization but keep the exact period. + let crit_pos_back = needle.len() + - cmp::max( + TwoWaySearcher::reverse_maximal_suffix(needle, period, false), + TwoWaySearcher::reverse_maximal_suffix(needle, period, true), + ); + + TwoWaySearcher { + crit_pos, + crit_pos_back, + period, + byteset: Self::byteset_create(&needle[..period]), + + position: 0, + end: haystack_len, + memory: 0, + memory_back: needle.len(), + } + } else { + // long period case -- we have an approximation to the actual period, + // and don't use memorization. + // + // Approximate the period by lower bound max(|u|, |v|) + 1. + // The critical factorization is efficient to use for both forward and + // reverse search. + + TwoWaySearcher { + crit_pos, + crit_pos_back: crit_pos, + period: cmp::max(crit_pos, needle.len() - crit_pos) + 1, + byteset: Self::byteset_create(needle), + + position: 0, + end: haystack_len, + memory: usize::MAX, // Dummy value to signify that the period is long + memory_back: usize::MAX, + } + } + } + + #[inline] + fn byteset_create(bytes: &[u8]) -> u64 { + bytes.iter().fold(0, |a, &b| (1 << (b & 0x3f)) | a) + } + + #[inline] + fn byteset_contains(&self, byte: u8) -> bool { + (self.byteset >> ((byte & 0x3f) as usize)) & 1 != 0 + } + + // One of the main ideas of Two-Way is that we factorize the needle into + // two halves, (u, v), and begin trying to find v in the haystack by scanning + // left to right. If v matches, we try to match u by scanning right to left. + // How far we can jump when we encounter a mismatch is all based on the fact + // that (u, v) is a critical factorization for the needle. + #[inline] + fn next_fwd( + &mut self, + haystack: &[u8], + needle: &[u8], + long_period: bool, + ) -> R { + // `next()` uses `self.position` as its cursor + let mut old_pos = self.position; + let needle_last = needle.len() - 1; + 'search: loop { + // Check that we have room to search in + // position + needle_last can not overflow if we assume slices + // are bounded by isize's range. + let tail_byte = match haystack.get(self.position + needle_last) { + Some(&b) => b, + None => { + self.position = haystack.len(); + if old_pos == self.position { + return R::DONE; + } + return R::rejecting(old_pos, self.position).unwrap_or(R::DONE); + } + }; + + if old_pos != self.position { + if R::HAS_REJECTS { + if let Some(ret) = R::rejecting(old_pos, self.position) { + return ret; + } + } + } + + // Quickly skip by large portions unrelated to our substring + if !self.byteset_contains(tail_byte) { + self.position += needle.len(); + if !long_period { + self.memory = 0; + } + continue 'search; + } + + // See if the right part of the needle matches + let start = + if long_period { self.crit_pos } else { cmp::max(self.crit_pos, self.memory) }; + for i in start..needle.len() { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.position + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.position + needle_last` is a valid index in the haystack. Inside the + // loop `self.position` is unchanged and `i <= needle_last`, so + // `self.position + i` is a valid index. If we adjust `self.position` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.position + i) } + }; + if needle[i] != haystack_byte { + self.position += i - self.crit_pos + 1; + if !long_period { + self.memory = 0; + } + continue 'search; + } + } + + // See if the left part of the needle matches + let start = if long_period { 0 } else { self.memory }; + for i in (start..self.crit_pos).rev() { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.position + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.position + needle_last` is a valid index in the haystack. Inside the + // loop `self.position` is unchanged and `i <= needle_last`, so + // `self.position + i` is a valid index. If we adjust `self.position` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.position + i) } + }; + if needle[i] != haystack_byte { + self.position += self.period; + if !long_period { + self.memory = needle.len() - self.period; + } + continue 'search; + } + } + + // We have found a match! + let match_pos = self.position; + + // Note: add self.period instead of needle.len() to have overlapping matches + self.position += needle.len(); + if !long_period { + self.memory = 0; // set to needle.len() - self.period for overlapping matches + } + + if let Some(ret) = R::matching(match_pos, match_pos + needle.len()) { + return ret; + } + // `matching` returned `None`, exclude the consumed match from the pending + // reject range so rejects. + old_pos = self.position; + } + } + + // Follows the ideas in `next()`. + // + // The definitions are symmetrical, with period(x) = period(reverse(x)) + // and local_period(u, v) = local_period(reverse(v), reverse(u)), so if (u, v) + // is a critical factorization, so is (reverse(v), reverse(u)). + // + // For the reverse case we have computed a critical factorization x = u' v' + // (field `crit_pos_back`). We need |u| < period(x) for the forward case and + // thus |v'| < period(x) for the reverse. + // + // To search in reverse through the haystack, we search forward through + // a reversed haystack with a reversed needle, matching first u' and then v'. + #[inline] + fn next_bwd( + &mut self, + haystack: &[u8], + needle: &[u8], + long_period: bool, + ) -> R { + // `next_back()` uses `self.end` as its cursor -- so that `next()` and `next_back()` + // are independent. + let mut old_end = self.end; + 'search: loop { + // Check that we have room to search in + // end - needle.len() will wrap around when there is no more room, + // but due to slice length limits it can never wrap all the way back + // into the length of haystack. + let front_byte = match haystack.get(self.end.wrapping_sub(needle.len())) { + Some(&b) => b, + None => { + self.end = 0; + if old_end == 0 { + return R::DONE; + } + return R::rejecting(0, old_end).unwrap_or(R::DONE); + } + }; + + if old_end != self.end { + if R::HAS_REJECTS { + if let Some(ret) = R::rejecting(self.end, old_end) { + return ret; + } + } + } + + // Quickly skip by large portions unrelated to our substring + if !self.byteset_contains(front_byte) { + self.end -= needle.len(); + if !long_period { + self.memory_back = needle.len(); + } + continue 'search; + } + + // See if the left part of the needle matches + let crit = if long_period { + self.crit_pos_back + } else { + cmp::min(self.crit_pos_back, self.memory_back) + }; + for i in (0..crit).rev() { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.end - needle.len() + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.end.wrapping_sub(needle.len())` is a valid index in the haystack. + // Inside the loop `self.end` is unchanged and `i < needle.len()`, so + // `end - needle.len() + i` is a valid index. If we adjust `self.end` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } + }; + + if needle[i] != haystack_byte { + self.end -= self.crit_pos_back - i; + if !long_period { + self.memory_back = needle.len(); + } + continue 'search; + } + } + + // See if the right part of the needle matches + let needle_end = if long_period { needle.len() } else { self.memory_back }; + for i in self.crit_pos_back..needle_end { + let haystack_byte = if cfg!(debug_assertions) { + haystack[self.end - needle.len() + i] + } else { + // SAFETY: each outer iteration, before entering this loop, we check that + // `self.end.wrapping_sub(needle.len())` is a valid index in the haystack. + // Inside the loop `self.end` is unchanged and `i < needle.len()`, so + // `end - needle.len() + i` is a valid index. If we adjust `self.end` we redo + // the check via `continue 'search`. + unsafe { *haystack.get_unchecked(self.end - needle.len() + i) } + }; + + if needle[i] != haystack_byte { + self.end -= self.period; + if !long_period { + self.memory_back = self.period; + } + continue 'search; + } + } + + // We have found a match! + let match_pos = self.end - needle.len(); + // Note: sub self.period instead of needle.len() to have overlapping matches + self.end -= needle.len(); + if !long_period { + self.memory_back = needle.len(); + } + + if let Some(ret) = R::matching(match_pos, match_pos + needle.len()) { + return ret; + } + // `matching` returned `None`, exclude the consumed match from the pending + // reject range so rejects. + old_end = self.end; + } + } + + // Compute the maximal suffix of `arr`. + // + // The maximal suffix is a possible critical factorization (u, v) of `arr`. + // + // Returns (`i`, `p`) where `i` is the starting index of v and `p` is the + // period of v. + // + // `order_greater` determines if lexical order is `<` or `>`. Both + // orders must be computed -- the ordering with the largest `i` gives + // a critical factorization. + // + // For long period cases, the resulting period is not exact (it is too short). + #[inline] + fn maximal_suffix(arr: &[u8], order_greater: bool) -> (usize, usize) { + let mut left = 0; // Corresponds to i in the paper + let mut right = 1; // Corresponds to j in the paper + let mut offset = 0; // Corresponds to k in the paper, but starting at 0 + // to match 0-based indexing. + let mut period = 1; // Corresponds to p in the paper + + while let Some(&a) = arr.get(right + offset) { + // `left` will be inbounds when `right` is. + let b = arr[left + offset]; + if (a < b && !order_greater) || (a > b && order_greater) { + // Suffix is smaller, period is entire prefix so far. + right += offset + 1; + offset = 0; + period = right - left; + } else if a == b { + // Advance through repetition of the current period. + if offset + 1 == period { + right += offset + 1; + offset = 0; + } else { + offset += 1; + } + } else { + // Suffix is larger, start over from current location. + left = right; + right += 1; + offset = 0; + period = 1; + } + } + (left, period) + } + + // Compute the maximal suffix of the reverse of `arr`. + // + // The maximal suffix is a possible critical factorization (u', v') of `arr`. + // + // Returns `i` where `i` is the starting index of v', from the back; + // returns immediately when a period of `known_period` is reached. + // + // `order_greater` determines if lexical order is `<` or `>`. Both + // orders must be computed -- the ordering with the largest `i` gives + // a critical factorization. + // + // For long period cases, the resulting period is not exact (it is too short). + fn reverse_maximal_suffix(arr: &[u8], known_period: usize, order_greater: bool) -> usize { + let mut left = 0; // Corresponds to i in the paper + let mut right = 1; // Corresponds to j in the paper + let mut offset = 0; // Corresponds to k in the paper, but starting at 0 + // to match 0-based indexing. + let mut period = 1; // Corresponds to p in the paper + let n = arr.len(); + + while right + offset < n { + let a = arr[n - (1 + right + offset)]; + let b = arr[n - (1 + left + offset)]; + if (a < b && !order_greater) || (a > b && order_greater) { + // Suffix is smaller, period is entire prefix so far. + right += offset + 1; + offset = 0; + period = right - left; + } else if a == b { + // Advance through repetition of the current period. + if offset + 1 == period { + right += offset + 1; + offset = 0; + } else { + offset += 1; + } + } else { + // Suffix is larger, start over from current location. + left = right; + right += 1; + offset = 0; + period = 1; + } + if period == known_period { + break; + } + } + debug_assert!(period <= known_period); + left + } +} diff --git a/library/coretests/tests/pattern.rs b/library/coretests/tests/pattern.rs index d4bec996d89a1..f65216dd2fd80 100644 --- a/library/coretests/tests/pattern.rs +++ b/library/coretests/tests/pattern.rs @@ -1,12 +1,12 @@ -use std::str::pattern::*; +use std::pattern::*; // This macro makes it easier to write // tests that do a series of iterations macro_rules! search_asserts { - ($haystack:expr, $needle:expr, $testname:expr, [$($func:ident),*], $result:expr) => { + ($haystack:expr, $needle:expr, $testname:literal, $($func:ident => $result:expr),*) => { let mut searcher = $needle.into_searcher($haystack); - let arr = [$( Step::from(searcher.$func()) ),*]; - assert_eq!(&arr[..], &$result, $testname); + let arr = [$( searcher.$func().into_step(stringify!($func)) ),*]; + assert_eq!(&arr[..], &[$($result),*], $testname); } } @@ -17,26 +17,31 @@ enum Step { // be the same length for easy alignment Matches(usize, usize), Rejects(usize, usize), - InRange(usize, usize), Done, } -use self::Step::*; +use Step::*; -impl From for Step { - fn from(x: SearchStep) -> Self { - match x { - SearchStep::Match(a, b) => Matches(a, b), - SearchStep::Reject(a, b) => Rejects(a, b), +trait IntoStep { + fn into_step(self, method_name: &str) -> Step; +} + +impl IntoStep for SearchStep { + fn into_step(self, _name: &str) -> Step { + match self { + SearchStep::Match(s, e) => Matches(s, e), + SearchStep::Reject(s, e) => Rejects(s, e), SearchStep::Done => Done, } } } -impl From> for Step { - fn from(x: Option<(usize, usize)>) -> Self { - match x { - Some((a, b)) => InRange(a, b), +impl IntoStep for Option<(usize, usize)> { + fn into_step(self, method_name: &str) -> Step { + let is_reject = method_name.starts_with("next_reject"); + match self { + Some((s, e)) if is_reject => Rejects(s, e), + Some((s, e)) => Matches(s, e), None => Done, } } @@ -54,142 +59,113 @@ fn test_simple_iteration() { "abcdeabcd", 'a', "forward iteration for ASCII string", - // a b c d e a b c d EOF - [next, next, next, next, next, next, next, next, next, next], - [ - Matches(0, 1), - Rejects(1, 2), - Rejects(2, 3), - Rejects(3, 4), - Rejects(4, 5), - Matches(5, 6), - Rejects(6, 7), - Rejects(7, 8), - Rejects(8, 9), - Done - ] + next => Matches(0, 1), + next => Rejects(1, 5), + next => Matches(5, 6), + next => Rejects(6, 9), + next => Done ); search_asserts!( "abcdeabcd", 'a', "reverse iteration for ASCII string", - // d c b a e d c b a EOF - [ - next_back, next_back, next_back, next_back, next_back, next_back, next_back, next_back, - next_back, next_back - ], - [ - Rejects(8, 9), - Rejects(7, 8), - Rejects(6, 7), - Matches(5, 6), - Rejects(4, 5), - Rejects(3, 4), - Rejects(2, 3), - Rejects(1, 2), - Matches(0, 1), - Done - ] + next_back => Rejects(6, 9), + next_back => Matches(5, 6), + next_back => Rejects(1, 5), + next_back => Matches(0, 1), + next_back => Done ); search_asserts!( "我爱我的猫", '我', "forward iteration for Chinese string", - // 我 愛 我 的 貓 EOF - [next, next, next, next, next, next], - [Matches(0, 3), Rejects(3, 6), Matches(6, 9), Rejects(9, 12), Rejects(12, 15), Done] + next => Matches(0, 3), + next => Rejects(3, 6), + next => Matches(6, 9), + next => Rejects(9, 15), + next => Done ); search_asserts!( "我的猫说meow", 'm', "forward iteration for mixed string", - // 我 的 猫 说 m e o w EOF - [next, next, next, next, next, next, next, next, next], - [ - Rejects(0, 3), - Rejects(3, 6), - Rejects(6, 9), - Rejects(9, 12), - Matches(12, 13), - Rejects(13, 14), - Rejects(14, 15), - Rejects(15, 16), - Done - ] + next => Rejects(0, 12), + next => Matches(12, 13), + next => Rejects(13, 16), + next => Done ); search_asserts!( "我的猫说meow", '猫', "reverse iteration for mixed string", - // w o e m 说 猫 的 我 EOF - [ - next_back, next_back, next_back, next_back, next_back, next_back, next_back, next_back, - next_back - ], - [ - Rejects(15, 16), - Rejects(14, 15), - Rejects(13, 14), - Rejects(12, 13), - Rejects(9, 12), - Matches(6, 9), - Rejects(3, 6), - Rejects(0, 3), - Done - ] + next_back => Rejects(9, 16), + next_back => Matches(6, 9), + next_back => Rejects(0, 6), + next_back => Done ); } +#[test] +fn backward_search_predicate() { + assert_eq!("abc".rfind(|c| c == 'a'), Some(0)); + assert_eq!("abcabc".rfind(|c| c == 'c'), Some(5)); + assert_eq!("éabc".rfind(['é']), Some(0)); + assert_eq!("éabc".rfind(|c| c == 'é'), Some(0)); + assert_eq!("éabcé".rfind(|c| c == 'é'), Some(5)); + assert_eq!("€abc".rfind(|c| c == '€'), Some(0)); + assert_eq!("😀abc".rfind(|c| c == '😀'), Some(0)); + + assert_eq!("abc".strip_suffix(|c| c == 'c'), Some("ab")); + assert_eq!("éabc".strip_suffix(|c| c == 'é'), None); +} + #[test] fn test_simple_search() { search_asserts!( "abcdeabcdeabcde", 'a', "next_match for ASCII string", - [next_match, next_match, next_match, next_match], - [InRange(0, 1), InRange(5, 6), InRange(10, 11), Done] + next_match => Matches(0, 1), + next_match => Matches(5, 6), + next_match => Matches(10, 11), + next_match => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "next_match_back for ASCII string", - [next_match_back, next_match_back, next_match_back, next_match_back], - [InRange(10, 11), InRange(5, 6), InRange(0, 1), Done] + next_match_back => Matches(10, 11), + next_match_back => Matches(5, 6), + next_match_back => Matches(0, 1), + next_match_back => Done ); search_asserts!( "abcdeab", 'a', "next_reject for ASCII string", - [next_reject, next_reject, next_match, next_reject, next_reject], - [InRange(1, 2), InRange(2, 3), InRange(5, 6), InRange(6, 7), Done] + next_reject => Rejects(1, 2), + next_reject => Rejects(2, 3), + next_match => Matches(5, 6), + next_reject => Rejects(6, 7), + next_reject => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "next_reject_back for ASCII string", - [ - next_reject_back, - next_reject_back, - next_match_back, - next_reject_back, - next_reject_back, - next_reject_back - ], - [ - InRange(14, 15), - InRange(13, 14), - InRange(10, 11), - InRange(9, 10), - InRange(8, 9), - InRange(7, 8) - ] + next_reject_back => Rejects(14, 15), + next_reject_back => Rejects(13, 14), + next_match_back => Matches(10, 11), + next_reject_back => Rejects(9, 10), + next_reject_back => Rejects(8, 9), + next_reject_back => Rejects(7, 8) ); } @@ -207,38 +183,40 @@ const STRESS: &str = "Áa🁀bÁꁁfg😁각กᘀ각aÁ각ꁁก😁a"; #[test] fn test_stress_indices() { // this isn't really a test, more of documentation on the indices of each character in the stresstest string - + search_asserts!( + STRESS, + |_| true, + "Indices of characters in stress test", + next => Matches(0, 2), // Á + next => Matches(2, 3), // a + next => Matches(3, 7), // 🁀 + next => Matches(7, 8), // b + next => Matches(8, 10), // Á + next => Matches(10, 13), // ꁁ + next => Matches(13, 14), // f + next => Matches(14, 15), // g + next => Matches(15, 19), // 😀 + next => Matches(19, 22), // 각 + next => Matches(22, 25), // ก + next => Matches(25, 28), // ᘀ + next => Matches(28, 31), // 각 + next => Matches(31, 32), // a + next => Matches(32, 34), // Á + next => Matches(34, 37), // 각 + next => Matches(37, 40), // ꁁ + next => Matches(40, 43), // ก + next => Matches(43, 47), // 😀 + next => Matches(47, 48), // a + next => Done + ); + + // this test should generate something that will run memchr search_asserts!( STRESS, 'x', "Indices of characters in stress test", - [ - next, next, next, next, next, next, next, next, next, next, next, next, next, next, - next, next, next, next, next, next, next - ], - [ - Rejects(0, 2), // Á - Rejects(2, 3), // a - Rejects(3, 7), // 🁀 - Rejects(7, 8), // b - Rejects(8, 10), // Á - Rejects(10, 13), // ꁁ - Rejects(13, 14), // f - Rejects(14, 15), // g - Rejects(15, 19), // 😀 - Rejects(19, 22), // 각 - Rejects(22, 25), // ก - Rejects(25, 28), // ᘀ - Rejects(28, 31), // 각 - Rejects(31, 32), // a - Rejects(32, 34), // Á - Rejects(34, 37), // 각 - Rejects(37, 40), // ꁁ - Rejects(40, 43), // ก - Rejects(43, 47), // 😀 - Rejects(47, 48), // a - Done - ] + next => Rejects(0, 48), // no character matches + next => Done ); } @@ -248,96 +226,113 @@ fn test_forward_search_shared_bytes() { STRESS, 'Á', "Forward search for two-byte Latin character", - [next_match, next_match, next_match, next_match], - [InRange(0, 2), InRange(8, 10), InRange(32, 34), Done] + next_match => Matches(0, 2), + next_match => Matches(8, 10), + next_match => Matches(32, 34), + next_match => Done ); search_asserts!( STRESS, 'Á', "Forward search for two-byte Latin character; check if next() still works", - [next_match, next, next_match, next, next_match, next, next_match], - [ - InRange(0, 2), - Rejects(2, 3), - InRange(8, 10), - Rejects(10, 13), - InRange(32, 34), - Rejects(34, 37), - Done - ] + next_match => Matches(0, 2), + next => Rejects(2, 8), + next_match => Matches(8, 10), + next => Rejects(10, 32), + next_match => Matches(32, 34), + next => Rejects(34, 48), + next_match => Done ); search_asserts!( STRESS, '각', "Forward search for three-byte Hangul character", - [next_match, next, next_match, next_match, next_match], - [InRange(19, 22), Rejects(22, 25), InRange(28, 31), InRange(34, 37), Done] + next_match => Matches(19, 22), + next => Rejects(22, 28), + next_match => Matches(28, 31), + next_match => Matches(34, 37), + next_match => Done ); search_asserts!( STRESS, '각', "Forward search for three-byte Hangul character; check if next() still works", - [next_match, next, next_match, next, next_match, next, next_match], - [ - InRange(19, 22), - Rejects(22, 25), - InRange(28, 31), - Rejects(31, 32), - InRange(34, 37), - Rejects(37, 40), - Done - ] + next_match => Matches(19, 22), + next => Rejects(22, 28), + next_match => Matches(28, 31), + next => Rejects(31, 34), + next_match => Matches(34, 37), + next => Rejects(37, 48), + next_match => Done ); search_asserts!( STRESS, 'ก', "Forward search for three-byte Thai character", - [next_match, next, next_match, next, next_match], - [InRange(22, 25), Rejects(25, 28), InRange(40, 43), Rejects(43, 47), Done] + next_match => Matches(22, 25), + next => Rejects(25, 40), + next_match => Matches(40, 43), + next => Rejects(43, 48), + next_match => Done ); search_asserts!( STRESS, 'ก', "Forward search for three-byte Thai character; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(22, 25), Rejects(25, 28), InRange(40, 43), Rejects(43, 47), Done] + next_match => Matches(22, 25), + next => Rejects(25, 40), + next_match => Matches(40, 43), + next => Rejects(43, 48), + next_match => Done ); search_asserts!( STRESS, '😁', "Forward search for four-byte emoji", - [next_match, next, next_match, next, next_match], - [InRange(15, 19), Rejects(19, 22), InRange(43, 47), Rejects(47, 48), Done] + next_match => Matches(15, 19), + next => Rejects(19, 43), + next_match => Matches(43, 47), + next => Rejects(47, 48), + next_match => Done ); search_asserts!( STRESS, '😁', "Forward search for four-byte emoji; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(15, 19), Rejects(19, 22), InRange(43, 47), Rejects(47, 48), Done] + next_match => Matches(15, 19), + next => Rejects(19, 43), + next_match => Matches(43, 47), + next => Rejects(47, 48), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Forward search for three-byte Yi character with repeated bytes", - [next_match, next, next_match, next, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(40, 43), Done] + next_match => Matches(10, 13), + next => Rejects(13, 37), + next_match => Matches(37, 40), + next => Rejects(40, 48), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Forward search for three-byte Yi character with repeated bytes; check if next() still works", - [next_match, next, next_match, next, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(40, 43), Done] + next_match => Matches(10, 13), + next => Rejects(13, 37), + next_match => Matches(37, 40), + next => Rejects(40, 48), + next_match => Done ); } @@ -347,96 +342,112 @@ fn test_reverse_search_shared_bytes() { STRESS, 'Á', "Reverse search for two-byte Latin character", - [next_match_back, next_match_back, next_match_back, next_match_back], - [InRange(32, 34), InRange(8, 10), InRange(0, 2), Done] + next_match_back => Matches(32, 34), + next_match_back => Matches(8, 10), + next_match_back => Matches(0, 2), + next_match_back => Done ); search_asserts!( STRESS, 'Á', "Reverse search for two-byte Latin character; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back, next_back], - [InRange(32, 34), Rejects(31, 32), InRange(8, 10), Rejects(7, 8), InRange(0, 2), Done] + next_match_back => Matches(32, 34), + next_back => Rejects(10, 32), + next_match_back => Matches(8, 10), + next_back => Rejects(2, 8), + next_match_back => Matches(0, 2), + next_back => Done ); search_asserts!( STRESS, '각', "Reverse search for three-byte Hangul character", - [next_match_back, next_back, next_match_back, next_match_back, next_match_back], - [InRange(34, 37), Rejects(32, 34), InRange(28, 31), InRange(19, 22), Done] + next_match_back => Matches(34, 37), + next_back => Rejects(31, 34), + next_match_back => Matches(28, 31), + next_match_back => Matches(19, 22), + next_match_back => Done ); search_asserts!( STRESS, '각', "Reverse search for three-byte Hangul character; check if next_back() still works", - [ - next_match_back, - next_back, - next_match_back, - next_back, - next_match_back, - next_back, - next_match_back - ], - [ - InRange(34, 37), - Rejects(32, 34), - InRange(28, 31), - Rejects(25, 28), - InRange(19, 22), - Rejects(15, 19), - Done - ] + next_match_back => Matches(34, 37), + next_back => Rejects(31, 34), + next_match_back => Matches(28, 31), + next_back => Rejects(22, 28), + next_match_back => Matches(19, 22), + next_back => Rejects(0, 19), + next_match_back => Done ); search_asserts!( STRESS, 'ก', "Reverse search for three-byte Thai character", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(40, 43), Rejects(37, 40), InRange(22, 25), Rejects(19, 22), Done] + next_match_back => Matches(40, 43), + next_back => Rejects(25, 40), + next_match_back => Matches(22, 25), + next_back => Rejects(0, 22), + next_match_back => Done ); search_asserts!( STRESS, 'ก', "Reverse search for three-byte Thai character; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(40, 43), Rejects(37, 40), InRange(22, 25), Rejects(19, 22), Done] + next_match_back => Matches(40, 43), + next_back => Rejects(25, 40), + next_match_back => Matches(22, 25), + next_back => Rejects(0, 22), + next_match_back => Done ); search_asserts!( STRESS, '😁', "Reverse search for four-byte emoji", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(43, 47), Rejects(40, 43), InRange(15, 19), Rejects(14, 15), Done] + next_match_back => Matches(43, 47), + next_back => Rejects(19, 43), + next_match_back => Matches(15, 19), + next_back => Rejects(0, 15), + next_match_back => Done ); search_asserts!( STRESS, '😁', "Reverse search for four-byte emoji; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(43, 47), Rejects(40, 43), InRange(15, 19), Rejects(14, 15), Done] + next_match_back => Matches(43, 47), + next_back => Rejects(19, 43), + next_match_back => Matches(15, 19), + next_back => Rejects(0, 15), + next_match_back => Done ); search_asserts!( STRESS, 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(37, 40), Rejects(34, 37), InRange(10, 13), Rejects(8, 10), Done] + next_match_back => Matches(37, 40), + next_back => Rejects(13, 37), + next_match_back => Matches(10, 13), + next_back => Rejects(0, 10), + next_match_back => Done ); search_asserts!( STRESS, 'ꁁ', "Reverse search for three-byte Yi character with repeated bytes; check if next_back() still works", - [next_match_back, next_back, next_match_back, next_back, next_match_back], - [InRange(37, 40), Rejects(34, 37), InRange(10, 13), Rejects(8, 10), Done] + next_match_back => Matches(37, 40), + next_back => Rejects(13, 37), + next_match_back => Matches(10, 13), + next_back => Rejects(0, 10), + next_match_back => Done ); } @@ -448,56 +459,251 @@ fn double_ended_regression_test() { "abcdeabcdeabcde", 'a', "alternating double ended search", - [next_match, next_match_back, next_match, next_match_back], - [InRange(0, 1), InRange(10, 11), InRange(5, 6), Done] + next_match => Matches(0, 1), + next_match_back => Matches(10, 11), + next_match => Matches(5, 6), + next_match_back => Done ); search_asserts!( "abcdeabcdeabcde", 'a', "triple double ended search for a", - [next_match, next_match_back, next_match_back, next_match_back], - [InRange(0, 1), InRange(10, 11), InRange(5, 6), Done] + next_match => Matches(0, 1), + next_match_back => Matches(10, 11), + next_match_back => Matches(5, 6), + next_match_back => Done ); search_asserts!( "abcdeabcdeabcde", 'd', "triple double ended search for d", - [next_match, next_match_back, next_match_back, next_match_back], - [InRange(3, 4), InRange(13, 14), InRange(8, 9), Done] + next_match => Matches(3, 4), + next_match_back => Matches(13, 14), + next_match_back => Matches(8, 9), + next_match_back => Done ); search_asserts!( STRESS, 'Á', "Double ended search for two-byte Latin character", - [next_match, next_match_back, next_match, next_match_back], - [InRange(0, 2), InRange(32, 34), InRange(8, 10), Done] + next_match => Matches(0, 2), + next_match_back => Matches(32, 34), + next_match => Matches(8, 10), + next_match_back => Done ); search_asserts!( STRESS, '각', "Reverse double ended search for three-byte Hangul character", - [next_match_back, next_back, next_match, next, next_match_back, next_match], - [InRange(34, 37), Rejects(32, 34), InRange(19, 22), Rejects(22, 25), InRange(28, 31), Done] + next_match_back => Matches(34, 37), + next_back => Rejects(31, 34), + next_match => Matches(19, 22), + next => Rejects(22, 28), + next_match_back => Matches(28, 31), + next_match => Done ); search_asserts!( STRESS, 'ก', "Double ended search for three-byte Thai character", - [next_match, next_back, next, next_match_back, next_match], - [InRange(22, 25), Rejects(47, 48), Rejects(25, 28), InRange(40, 43), Done] + next_match => Matches(22, 25), + next_back => Rejects(43, 48), + next => Rejects(25, 40), + next_match_back => Matches(40, 43), + next_match => Done ); search_asserts!( STRESS, '😁', "Double ended search for four-byte emoji", - [next_match_back, next, next_match, next_back, next_match], - [InRange(43, 47), Rejects(0, 2), InRange(15, 19), Rejects(40, 43), Done] + next_match_back => Matches(43, 47), + next => Rejects(0, 15), + next_match => Matches(15, 19), + next_back => Rejects(19, 43), + next_match => Done ); search_asserts!( STRESS, 'ꁁ', "Double ended search for three-byte Yi character with repeated bytes", - [next_match, next, next_match_back, next_back, next_match], - [InRange(10, 13), Rejects(13, 14), InRange(37, 40), Rejects(34, 37), Done] + next_match => Matches(10, 13), + next => Rejects(13, 37), + next_match_back => Matches(37, 40), + next_back => Done ); } + +#[test] +fn two_way_next_reject_skips_matches() { + // `next_reject` must not report the matched regions as rejects + + // plen - pattern len + // ulen - unmatch len + // haystack is always a concatenation of [Match, Reject, Match] + #[track_caller] + fn check_fw_bw(haystack: H, pat: T, plen: usize, ulen: usize) + where + H: Haystack, + T: Pattern + Copy, + T::Searcher: ReverseSearcher, + { + // fragments + let f1 = (0, plen); + let f2 = (f1.1, f1.1 + ulen); + let f3 = (f2.1, f2.1 + plen); + + let mut searcher = pat.into_searcher(haystack); + let (start, end) = searcher.next_reject().expect("fw reject"); + assert_eq!(start, f2.0, "fw reject start"); + assert!(start < end && end <= f2.1, "fw reject must be a non-empty part of f2"); + assert_eq!(searcher.next_match(), Some(f3), "fw match"); + assert_eq!(searcher.next_match(), None, "fw end"); + + let mut searcher = pat.into_searcher(haystack); + let (start, end) = searcher.next_reject_back().expect("bw reject"); + assert_eq!(end, f2.1, "bw reject end"); + assert!(f2.0 <= start && start < end, "bw reject must be a non-empty part of f2"); + assert_eq!(searcher.next_match_back(), Some(f1), "bw match"); + assert_eq!(searcher.next_match_back(), None, "bw end"); + } + check_fw_bw("XYZabcXYZ", "XYZ", 3, 3); + + // single char 1..4 byte str + check_fw_bw("XabcX", "X", 1, 3); + check_fw_bw("\u{00e9}abc\u{00e9}", "\u{00e9}", 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", "\u{20ac}", 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", "\u{1f60a}", 4, 3); + + // single char 2..4 byte + check_fw_bw("\u{00e9}abc\u{00e9}", '\u{00e9}', 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", '\u{20ac}', 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", '\u{1f60a}', 4, 3); + + check_fw_bw("\u{00e9}abc\u{00e9}", ['\u{00e9}'], 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", ['\u{20ac}'], 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", ['\u{1f60a}'], 4, 3); + + check_fw_bw("\u{00e9}abc\u{00e9}", |c| c == '\u{00e9}', 2, 3); + check_fw_bw("\u{20ac}abc\u{20ac}", |c| c == '\u{20ac}', 3, 3); + check_fw_bw("\u{1f60a}abc\u{1f60a}", |c| c == '\u{1f60a}', 4, 3); +} + +#[test] +fn two_way_next_reject_never_reports_empty_reject() { + // A reject must never be an empty range + + // Haystack fully covered by the pattern: there are no rejects at all. + let mut searcher = "XYZ".into_searcher("XYZ"); + assert_eq!(searcher.next_reject(), None); + assert_eq!(searcher.next_reject(), None); + + let mut searcher = "XYZ".into_searcher("XYZ"); + assert_eq!(searcher.next_reject_back(), None); + assert_eq!(searcher.next_reject_back(), None); + + // Haystack ending with a match: the last reject is followed by `None`, + // not by an empty reject after the final match. + let mut searcher = "XYZ".into_searcher("XYZabcXYZ"); + assert_eq!(searcher.next_reject(), Some((3, 6))); + assert_eq!(searcher.next_reject(), None); + + let mut searcher = "XYZ".into_searcher("XYZabcXYZ"); + assert_eq!(searcher.next_reject_back(), Some((3, 6))); + assert_eq!(searcher.next_reject_back(), None); +} + +#[test] +fn test_try_next_code_point_fwd_and_rev() { + use core::str::{try_next_code_point, try_next_code_point_reverse}; + + #[track_caller] + fn check_fwbw(input: &[u8], expected: Option<(char, usize)>) { + let mut store = Vec::with_capacity(input.len() + 1); + + assert_eq!(try_next_code_point(input), expected, "fw single"); + assert_eq!(try_next_code_point_reverse(input), expected, "bw single"); + + if input.is_empty() { + return; + } + + store.extend_from_slice(input); + store.push(b'a'); + assert_eq!(try_next_code_point(&store), expected, "fw ext"); + + store.clear(); + store.push(b'a'); + store.extend_from_slice(input); + assert_eq!(try_next_code_point_reverse(&store), expected, "bw, ext"); + } + + // edge case + check_fwbw(&[], None); // edge case + + // valid cases + check_fwbw(&[0x41], Some(('A', 1))); // 1 byte + check_fwbw(&[0xc3, 0xa9], Some(('\u{00e9}', 2))); // 2 byte + check_fwbw(&[0xe2, 0x82, 0xac], Some(('\u{20ac}', 3))); // 3 byte + check_fwbw(&[0xf0, 0x9f, 0x98, 0x8a], Some(('\u{1f60a}', 4))); // 4 byte + + // overlong encoding + check_fwbw(&[0xc0, 0xaf], None); // '\u{002f}' AKA '/' + check_fwbw(&[0xc1, 0x81], None); // '\u{0041}' AKA 'A' + check_fwbw(&[0xe0, 0x81, 0x81], None); // '\u{0041}' AKA 'A' + check_fwbw(&[0xe0, 0x83, 0xa9], None); // '\u{00e9}' AKA 'é', should be 2 bytes + + // surrogates + check_fwbw(&[0xed, 0xa0, 0x80], None); + check_fwbw(&[0xed, 0xb0, 0x80], None); + check_fwbw(&[0xed, 0xa0, 0xbd, 0xed, 0xb8, 0x8a], None); + + // unattached continuation + check_fwbw(&[0x80], None); + + // truncated + check_fwbw(&[0xe2, 0x82], None); + + // out of range + check_fwbw(&[0xf5, 0x80, 0x80, 0x80], None); +} + +#[test] +fn str_searcher_reject_back_preserves_start_on_bad_input() { + let haystack = core::str_bytes::Bytes::from_bytes(&[0x80, b'x', b'y', b'z'][..]); + let mut searcher = "xyz".into_searcher(haystack); + assert_eq!(searcher.next_reject_back(), Some((0, 1))); + assert_eq!(searcher.next_reject_back(), None); +} + +#[test] +fn backward_search_predicate_pat() { + use core::pattern::{Pattern, ReverseSearcher}; + use core::str_bytes::Bytes; + + let haystack = Bytes::from_str("abc"); + let mut searcher = (|c: char| c == 'a').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((0, 1))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("abcabc"); + let mut searcher = (|c: char| c == 'c').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((5, 6))); + assert_eq!(searcher.next_match_back(), Some((2, 3))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabc"); + let mut searcher = (|c: char| c == 'é').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((0, 2))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabcé"); + let mut searcher = (|c: char| c == 'é').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((5, 7))); + assert_eq!(searcher.next_match_back(), Some((0, 2))); + assert_eq!(searcher.next_match_back(), None); + + let haystack = Bytes::from_str("éabc"); + let mut searcher = (|c: char| c == 'c').into_searcher(haystack); + assert_eq!(searcher.next_match_back(), Some((4, 5))); + assert_eq!(searcher.next_match_back(), None); +} diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index 980ec4416f04a..56ff07a001e86 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -568,6 +568,8 @@ pub use core::mem; pub use core::ops; #[stable(feature = "rust1", since = "1.0.0")] pub use core::option; +#[unstable(feature = "pattern", issue = "27721")] +pub use core::pattern; #[stable(feature = "pin", since = "1.33.0")] pub use core::pin; #[stable(feature = "rust1", since = "1.0.0")] diff --git a/library/std/src/sys/os_str/bytes.rs b/library/std/src/sys/os_str/bytes.rs index a57da01a5d85d..048e5def1369b 100644 --- a/library/std/src/sys/os_str/bytes.rs +++ b/library/std/src/sys/os_str/bytes.rs @@ -16,12 +16,12 @@ mod tests; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: Vec, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: [u8], } diff --git a/library/std/src/sys/os_str/mod.rs b/library/std/src/sys/os_str/mod.rs index f7007cbf18b4c..dd37b558a096e 100644 --- a/library/std/src/sys/os_str/mod.rs +++ b/library/std/src/sys/os_str/mod.rs @@ -3,14 +3,14 @@ cfg_select! { any(target_os = "windows", target_os = "uefi") => { mod wtf8; - pub use wtf8::{Buf, Slice}; + pub(crate) use wtf8::{Buf, Slice}; } any(target_os = "motor") => { mod utf8; - pub use utf8::{Buf, Slice}; + pub(crate) use utf8::{Buf, Slice}; } _ => { mod bytes; - pub use bytes::{Buf, Slice}; + pub(crate) use bytes::{Buf, Slice}; } } diff --git a/library/std/src/sys/os_str/utf8.rs b/library/std/src/sys/os_str/utf8.rs index 289f58aa480f7..3648962da5eeb 100644 --- a/library/std/src/sys/os_str/utf8.rs +++ b/library/std/src/sys/os_str/utf8.rs @@ -11,12 +11,12 @@ use crate::{fmt, mem}; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: String, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: str, } diff --git a/library/std/src/sys/os_str/wtf8.rs b/library/std/src/sys/os_str/wtf8.rs index 9a32ab3f3ea12..5f9d64dc5d09c 100644 --- a/library/std/src/sys/os_str/wtf8.rs +++ b/library/std/src/sys/os_str/wtf8.rs @@ -13,12 +13,12 @@ use crate::{fmt, mem}; #[derive(Hash)] #[repr(transparent)] -pub struct Buf { +pub(crate) struct Buf { pub inner: Wtf8Buf, } #[repr(transparent)] -pub struct Slice { +pub(crate) struct Slice { pub inner: Wtf8, } diff --git a/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs index 4f8cc67f5e019..5ff45e5ae7d53 100644 --- a/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs +++ b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt, quote}; use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::str::pattern::Pattern; +use std::pattern::Pattern; use std::{fmt, str::FromStr}; use crate::context::LocalContext; @@ -68,7 +68,7 @@ impl WildString { pub fn replace

(&self, from: P, to: &str) -> WildString where - P: Pattern + Copy, + P: for<'a> Pattern<&'a str> + Copy, { WildString( self.0 diff --git a/tests/ui/suggestions/issue-104961.fixed b/tests/ui/suggestions/issue-104961.fixed index 3019242880f82..7c76b254578d4 100644 --- a/tests/ui/suggestions/issue-104961.fixed +++ b/tests/ui/suggestions/issue-104961.fixed @@ -2,12 +2,12 @@ fn foo(x: &str) -> bool { x.starts_with(&("hi".to_string() + " you")) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn foo2(x: &str) -> bool { x.starts_with(&"hi".to_string()) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn main() { diff --git a/tests/ui/suggestions/issue-104961.rs b/tests/ui/suggestions/issue-104961.rs index b315e9bab0d1a..8d796f96d73f7 100644 --- a/tests/ui/suggestions/issue-104961.rs +++ b/tests/ui/suggestions/issue-104961.rs @@ -2,12 +2,12 @@ fn foo(x: &str) -> bool { x.starts_with("hi".to_string() + " you") - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn foo2(x: &str) -> bool { x.starts_with("hi".to_string()) - //~^ ERROR the trait bound `String: Pattern` is not satisfied [E0277] + //~^ ERROR the trait bound `String: Pattern<&str>` is not satisfied [E0277] } fn main() { diff --git a/tests/ui/suggestions/issue-104961.stderr b/tests/ui/suggestions/issue-104961.stderr index 0d229e6dada5f..35f166868d3aa 100644 --- a/tests/ui/suggestions/issue-104961.stderr +++ b/tests/ui/suggestions/issue-104961.stderr @@ -1,12 +1,12 @@ -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-104961.rs:4:19 | LL | x.starts_with("hi".to_string() + " you") - | ----------- ^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Pattern` is not implemented for `String` + | ----------- ^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::starts_with` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here @@ -14,15 +14,15 @@ help: consider borrowing here LL | x.starts_with(&("hi".to_string() + " you")) | ++ + -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-104961.rs:9:19 | LL | x.starts_with("hi".to_string()) - | ----------- ^^^^^^^^^^^^^^^^ the trait `Pattern` is not implemented for `String` + | ----------- ^^^^^^^^^^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::starts_with` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here diff --git a/tests/ui/suggestions/issue-62843.stderr b/tests/ui/suggestions/issue-62843.stderr index c3c0360b3a9d1..9a558d06b21cf 100644 --- a/tests/ui/suggestions/issue-62843.stderr +++ b/tests/ui/suggestions/issue-62843.stderr @@ -1,12 +1,12 @@ -error[E0277]: the trait bound `String: Pattern` is not satisfied +error[E0277]: the trait bound `String: Pattern<&str>` is not satisfied --> $DIR/issue-62843.rs:4:32 | LL | println!("{:?}", line.find(pattern)); - | ---- ^^^^^^^ the trait `Pattern` is not implemented for `String` + | ---- ^^^^^^^ the trait `Pattern<&str>` is not implemented for `String` | | | required by a bound introduced by this call | - = note: required for `String` to implement `Pattern` + = note: required for `String` to implement `Pattern<&str>` note: required by a bound in `core::str::::find` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider borrowing here diff --git a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs index 667d283bea3e4..22126705a6840 100644 --- a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs +++ b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.rs @@ -1,10 +1,10 @@ fn strip_lf(s: &str) -> &str { s.strip_suffix(b'\n').unwrap_or(s) - //~^ ERROR the trait bound `u8: Pattern` is not satisfied + //~^ ERROR the trait bound `u8: Pattern<&str>` is not satisfied //~| NOTE required by a bound introduced by this call //~| NOTE the trait `FnMut(char)` is not implemented for `u8` - //~| HELP the following other types implement trait `Pattern`: - //~| NOTE required for `u8` to implement `Pattern` + //~| HELP the following other types implement trait `Pattern`: + //~| NOTE required for `u8` to implement `Pattern<&str>` //~| NOTE required by a bound in `core::str::::strip_suffix` } diff --git a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr index 1cd62d2cbdbaa..ec33e71dfe683 100644 --- a/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr +++ b/tests/ui/traits/bound/assoc-fn-bound-root-obligation.stderr @@ -1,4 +1,4 @@ -error[E0277]: the trait bound `u8: Pattern` is not satisfied +error[E0277]: the trait bound `u8: Pattern<&str>` is not satisfied --> $DIR/assoc-fn-bound-root-obligation.rs:2:20 | LL | s.strip_suffix(b'\n').unwrap_or(s) @@ -6,7 +6,7 @@ LL | s.strip_suffix(b'\n').unwrap_or(s) | | | required by a bound introduced by this call | - = help: the following other types implement trait `Pattern`: + = help: the following other types implement trait `Pattern`: &'b String &'b [char; N] &'b [char] @@ -14,7 +14,7 @@ LL | s.strip_suffix(b'\n').unwrap_or(s) &'c &'b str [char; N] char - = note: required for `u8` to implement `Pattern` + = note: required for `u8` to implement `Pattern<&str>` note: required by a bound in `core::str::::strip_suffix` --> $SRC_DIR/core/src/str/mod.rs:LL:COL diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.fixed b/tests/ui/traits/suggest-dereferences/root-obligation.fixed index ad0f184dc9a42..a6edc29244e46 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.fixed +++ b/tests/ui/traits/suggest-dereferences/root-obligation.fixed @@ -4,7 +4,7 @@ fn get_vowel_count(string: &str) -> usize { string .chars() .filter(|c| "aeiou".contains(*c)) - //~^ ERROR the trait bound `&char: Pattern` is not satisfied + //~^ ERROR the trait bound `&char: Pattern<&str>` is not satisfied .count() } diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.rs b/tests/ui/traits/suggest-dereferences/root-obligation.rs index a31a9955d313a..d14e58f4af01a 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.rs +++ b/tests/ui/traits/suggest-dereferences/root-obligation.rs @@ -4,7 +4,7 @@ fn get_vowel_count(string: &str) -> usize { string .chars() .filter(|c| "aeiou".contains(c)) - //~^ ERROR the trait bound `&char: Pattern` is not satisfied + //~^ ERROR the trait bound `&char: Pattern<&str>` is not satisfied .count() } diff --git a/tests/ui/traits/suggest-dereferences/root-obligation.stderr b/tests/ui/traits/suggest-dereferences/root-obligation.stderr index dafd3469b6f03..fa4cf22e463c6 100644 --- a/tests/ui/traits/suggest-dereferences/root-obligation.stderr +++ b/tests/ui/traits/suggest-dereferences/root-obligation.stderr @@ -1,4 +1,4 @@ -error[E0277]: the trait bound `&char: Pattern` is not satisfied +error[E0277]: the trait bound `&char: Pattern<&str>` is not satisfied --> $DIR/root-obligation.rs:6:38 | LL | .filter(|c| "aeiou".contains(c)) @@ -7,7 +7,7 @@ LL | .filter(|c| "aeiou".contains(c)) | required by a bound introduced by this call | = note: required for `&char` to implement `FnOnce(char)` - = note: required for `&char` to implement `Pattern` + = note: required for `&char` to implement `Pattern<&str>` note: required by a bound in `core::str::::contains` --> $SRC_DIR/core/src/str/mod.rs:LL:COL help: consider dereferencing here