From dc0b429a1fd980ac1fa037a94751b186fca31d5c Mon Sep 17 00:00:00 2001 From: MsfPablo Date: Tue, 18 Aug 2026 13:03:55 +0200 Subject: [PATCH 1/3] grep: support POSIX equivalence classes in brackets --- src/matcher.rs | 171 ++++++++++++++++++++++++++++++++++++++++++++- tests/test_grep.rs | 38 ++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) diff --git a/src/matcher.rs b/src/matcher.rs index dac786b..ee902b6 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -5,6 +5,7 @@ use crate::{Config, RegexMode}; use memchr::memmem; +use std::borrow::Cow; use onig::{RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior, SyntaxOperator}; use onig_sys::{ ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS, OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8, @@ -287,6 +288,13 @@ impl CompiledPattern { )); } + let pattern = if config.regex_mode == RegexMode::Fixed { + Cow::Borrowed(pattern) + } else { + rewrite_equivalence_classes(pattern) + }; + let pattern: &str = &pattern; + let mut normalized_pattern = None; let pattern = if config.regex_mode == RegexMode::Extended { if let Some((op, rest)) = strip_leading_repeat_operator(pattern) { @@ -608,6 +616,127 @@ fn scan_bracket(pattern: &[u8], start: usize) -> (bool, usize) { (false, pattern.len()) } +/// Rewrite POSIX equivalence classes (`[=c=]`) to their bare member `c`. +/// +/// A POSIX bracket expression may contain an equivalence class `[=c=]` that +/// matches every character collating equal to `c`. In the single-byte C locale +/// that set is just `c` itself, and oniguruma has no syntax for equivalence +/// classes at all, so a pattern containing one cannot be compiled as-is. This +/// function rewrites the classes oniguruma cannot express into the nearest +/// equivalent it can, leaving everything else in the pattern untouched: +/// +/// * `[[=a=]]` becomes `[a]` — the class collapses to its sole member. +/// * `[[=a=]b]` becomes `[ab]` — members compose with other entries. +/// * `[[=a=][=b=]]` becomes `[ab]` — several classes in one bracket. +/// * `x[[=a=]]y` becomes `x[a]y` — text outside the bracket is preserved. +/// +/// Returns a borrowed `Cow` when there is nothing to rewrite (the common case, +/// so no allocation is needed) and an owned one otherwise. +/// +/// A class whose body is not exactly one character is left in place, as is one +/// used as a range endpoint (an error in GNU grep, so it must not be quietly +/// turned into a range) or one holding `]`, `^`, `-` or `\`, whose meaning +/// inside a bracket expression depends on where it sits. +fn rewrite_equivalence_classes(pattern: &str) -> Cow<'_, str> { + let bytes = pattern.as_bytes(); + let mut out: Option = None; + let mut copied = 0; + let mut i = 0; + + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'[' => { + let (rewritten, next) = scan_equivalence_bracket(pattern, i); + if let Some(body) = rewritten { + let out = out.get_or_insert_with(String::new); + out.push_str(&pattern[copied..=i]); + out.push_str(&body); + copied = next; + } + i = next; + } + _ => i += 1, + } + } + + match out { + None => Cow::Borrowed(pattern), + Some(mut s) => { + s.push_str(&pattern[copied..]); + Cow::Owned(s) + } + } +} + +/// Scan a single bracket expression whose opening `[` is at `open` and rewrite +/// every rewritable `[=c=]` equivalence class in its body to the bare member +/// `c`. Returns the rewritten body (including the closing `]`) together with +/// the index just past that `]`; the body is `None` when the bracket contains +/// no rewritable equivalence class, so the caller can leave the span alone. +/// Mirrors `scan_bracket`, which does the equivalent job for +/// `has_confusing_bracket`, so the two stay structurally identical. +fn scan_equivalence_bracket(pattern: &str, open: usize) -> (Option, usize) { + let bytes = pattern.as_bytes(); + let mut j = open + 1; + if bytes.get(j) == Some(&b'^') { + j += 1; + } + let body_start = j; + let mut out: Option = None; + let mut copied = open + 1; + + while j < bytes.len() { + // A `]` at the very start of the body is an ordinary character; any + // other `]` closes the bracket. + if bytes[j] == b']' && j != body_start { + if let Some(out) = out.as_mut() { + out.push_str(&pattern[copied..=j]); + } + return (out, j + 1); + } + // `[:`, `[.` and `[=` subexpressions inside the bracket. + if bytes[j] == b'[' && matches!(bytes.get(j + 1), Some(b':' | b'.' | b'=')) { + let delimiter = bytes[j + 1]; + if let Some(end) = find_bracket_subexpr_end(bytes, j + 2, delimiter) { + if delimiter == b'=' + && is_rewritable_equivalence( + &pattern[j + 2..end - 2], + bytes.get(end).copied(), + if j > 0 { Some(bytes[j - 1]) } else { None }, + ) + { + let body = &pattern[j + 2..end - 2]; + let out = out.get_or_insert_with(String::new); + out.push_str(&pattern[copied..j]); + out.push_str(body); + copied = end; + } + j = end; + continue; + } + } + j += 1; + } + // Unterminated bracket: leave it for the regex engine to report, but still + // flush whatever rewrite was already written into `out`. + if let Some(out) = out.as_mut() { + out.push_str(&pattern[copied..]); + } + (out, pattern.len()) +} + +/// True when a `[=...=]` equivalence class whose body is `body` can be replaced +/// by that single character. GNU grep rejects equivalence classes used as range +/// endpoints, so a class directly preceded or followed by `-` (`prev`/`next`) +/// is left in place rather than silently producing a range. +fn is_rewritable_equivalence(body: &str, next: Option, prev: Option) -> bool { + let in_range = next == Some(b'-') || prev == Some(b'-'); + body.chars().count() == 1 + && !body.starts_with([']', '^', '-', '\\']) + && !in_range +} + /// Index just past the `:]`, `.]` or `=]` closing a `[: [. [=` subexpression /// whose body starts at `start`. fn find_bracket_subexpr_end(pattern: &[u8], start: usize, delimiter: u8) -> Option { @@ -618,8 +747,48 @@ fn find_bracket_subexpr_end(pattern: &[u8], start: usize, delimiter: u8) -> Opti #[cfg(test)] mod tests { - use super::{has_confusing_bracket, plain_literal}; + use super::{has_confusing_bracket, plain_literal, rewrite_equivalence_classes}; use crate::RegexMode; + use std::borrow::Cow; + + fn r(p: &str) -> Cow<'_, str> { + rewrite_equivalence_classes(p) + } + + #[test] + fn equivalence_classes_reduce_to_their_member() { + // Rewrites allocate an owned string. + assert_eq!(&*r("[[=a=]]"), "[a]"); + assert_eq!(&*r("[[=a=]b]"), "[ab]"); + assert_eq!(&*r("[b[=a=]]"), "[ba]"); + assert_eq!(&*r("[[=a=][=b=]]"), "[ab]"); + assert_eq!(&*r("[^[=a=]]"), "[^a]"); + assert_eq!(&*r("x[[=a=]]y"), "x[a]y"); + assert_eq!(&*r("[[:alpha:][=a=]]"), "[[:alpha:]a]"); + // The common case borrows the input unchanged: no allocation. + assert!(matches!(r("abc"), Cow::Borrowed("abc"))); + } + + #[test] + fn equivalence_class_rewrite_leaves_other_patterns_alone() { + // Nothing to do. + assert!(matches!(r("abc"), Cow::Borrowed(_))); + assert!(matches!(r("[abc]"), Cow::Borrowed(_))); + assert!(matches!(r("[[:alpha:]]"), Cow::Borrowed(_))); + assert!(matches!(r("[[.a.]]"), Cow::Borrowed(_))); + // Not a bracket expression: `[=a=]` outside `[...]` is literal in GNU. + assert!(matches!(r("\\[[=a=]"), Cow::Borrowed(_))); + // A range endpoint is an error in GNU; don't invent a valid range. + assert!(matches!(r("[[=a=]-c]"), Cow::Borrowed(_))); + assert!(matches!(r("[a-[=c=]]"), Cow::Borrowed(_))); + // Only single-character classes have an obvious C-locale member. + assert!(matches!(r("[[=ab=]]"), Cow::Borrowed(_))); + assert!(matches!(r("[[==]]"), Cow::Borrowed(_))); + // Members whose meaning depends on position inside the bracket. + assert!(matches!(r("[[=]=]]"), Cow::Borrowed(_))); + assert!(matches!(r("[[=^=]]"), Cow::Borrowed(_))); + assert!(matches!(r("[[=-=]]"), Cow::Borrowed(_))); + } fn lit(p: &str, ic: bool, mode: RegexMode) -> Option> { plain_literal(p, ic, mode) diff --git a/tests/test_grep.rs b/tests/test_grep.rs index b477329..23d9d61 100644 --- a/tests/test_grep.rs +++ b/tests/test_grep.rs @@ -100,6 +100,44 @@ fn gnu_buffer_anchors() { .stdout_only("cat\ntar\n"); } +#[test] +fn posix_equivalence_classes() { + // In the C locale `[[=a=]]` is equivalent to `[a]`. + let input = "a\nb\nc\n"; + + for args in [vec!["[[=a=]]"], vec!["-E", "[[=a=]]"], vec!["a[[=a=]]*"]] { + let (_s, mut c) = ucmd(); + c.args(&args).pipe_in(input).succeeds().stdout_only("a\n"); + } + + // The class must not poison the rest of the bracket expression. + for args in [ + vec!["[[=a=]b]"], + vec!["[b[=a=]]"], + vec!["[[=a=][=b=]]"], + vec!["-E", "[[=a=]b]"], + ] { + let (_s, mut c) = ucmd(); + c.args(&args) + .pipe_in(input) + .succeeds() + .stdout_only("a\nb\n"); + } + + let (_s, mut c) = ucmd(); + c.args(&["[^[=a=]]"]) + .pipe_in(input) + .succeeds() + .stdout_only("b\nc\n"); + + // `-F` takes the whole thing literally. + let (_s, mut c) = ucmd(); + c.args(&["-F", "[[=a=]]"]) + .pipe_in("[[=a=]]\na\n") + .succeeds() + .stdout_only("[[=a=]]\n"); +} + #[test] fn ere_metacharacters() { let cases: &[(&[&str], &str, &str)] = &[ From 139d420ee5443668846e7f347fbd38887a01711f Mon Sep 17 00:00:00 2001 From: Pablo Garcia Date: Wed, 19 Aug 2026 17:22:33 +0200 Subject: [PATCH 2/3] style: apply rustfmt (import order, single-line predicate) --- src/matcher.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/matcher.rs b/src/matcher.rs index ee902b6..069d1bb 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -5,11 +5,11 @@ use crate::{Config, RegexMode}; use memchr::memmem; -use std::borrow::Cow; use onig::{RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior, SyntaxOperator}; use onig_sys::{ ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS, OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8, }; +use std::borrow::Cow; use std::ptr::{null, null_mut}; use std::sync::Mutex; use uucore::error::{UResult, USimpleError}; @@ -732,9 +732,7 @@ fn scan_equivalence_bracket(pattern: &str, open: usize) -> (Option, usiz /// is left in place rather than silently producing a range. fn is_rewritable_equivalence(body: &str, next: Option, prev: Option) -> bool { let in_range = next == Some(b'-') || prev == Some(b'-'); - body.chars().count() == 1 - && !body.starts_with([']', '^', '-', '\\']) - && !in_range + body.chars().count() == 1 && !body.starts_with([']', '^', '-', '\\']) && !in_range } /// Index just past the `:]`, `.]` or `=]` closing a `[: [. [=` subexpression From 18525e7aabab6f24f04da42ecf3d5aa4d18132fc Mon Sep 17 00:00:00 2001 From: MsfPablo Date: Mon, 24 Aug 2026 10:30:57 +0200 Subject: [PATCH 3/3] grep: find bracket openings with memchr instead of a byte loop Both has_confusing_bracket and rewrite_equivalence_classes walked the pattern one byte at a time looking for '['. Only '[' is significant between bracket expressions, so share a next_unescaped_bracket helper that jumps to the next one and decides escaping from the parity of the backslash run in front of it. Keeps the two functions structurally identical, as before. --- src/matcher.rs | 89 +++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/src/matcher.rs b/src/matcher.rs index 069d1bb..856598f 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -4,7 +4,7 @@ // file that was distributed with this source code. use crate::{Config, RegexMode}; -use memchr::memmem; +use memchr::{memchr, memmem}; use onig::{RegexOptions, Region, SearchOptions, Syntax, SyntaxBehavior, SyntaxOperator}; use onig_sys::{ ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS, OnigEncCtype_ONIGENC_CTYPE_WORD, OnigEncodingUTF8, @@ -549,22 +549,39 @@ fn strip_leading_interval_repeat(pattern: &str) -> Option<&str> { /// class or collating element. fn has_confusing_bracket(pattern: &[u8]) -> bool { let mut i = 0; - while i < pattern.len() { - match pattern[i] { - b'\\' => i += 2, - b'[' => { - let (confusing, next) = scan_bracket(pattern, i + 1); - if confusing { - return true; - } - i = next; - } - _ => i += 1, + while let Some(open) = next_unescaped_bracket(pattern, i) { + let (confusing, next) = scan_bracket(pattern, open + 1); + if confusing { + return true; } + i = next; } false } +/// Index of the first `[` at or after `from` that is not escaped by a +/// backslash, or `None` if the pattern holds no such bracket. +/// +/// Only `[` is significant between bracket expressions, so jump straight to +/// the next one instead of walking the pattern a byte at a time. A backslash +/// run directly in front of the match decides whether it opens a bracket: an +/// odd count escapes it, an even one leaves the `[` itself unescaped (`\\[`). +fn next_unescaped_bracket(pattern: &[u8], from: usize) -> Option { + let mut search = from; + while let Some(offset) = memchr(b'[', &pattern[search..]) { + let at = search + offset; + let mut backslashes = 0; + while at - backslashes > from && pattern[at - backslashes - 1] == b'\\' { + backslashes += 1; + } + if backslashes % 2 == 0 { + return Some(at); + } + search = at + 1; + } + None +} + /// Scan the body of a bracket expression starting at `start` (just past the /// `[`). Returns whether it is a misspelled character class and the index just /// past its closing `]`. @@ -643,21 +660,15 @@ fn rewrite_equivalence_classes(pattern: &str) -> Cow<'_, str> { let mut copied = 0; let mut i = 0; - while i < bytes.len() { - match bytes[i] { - b'\\' => i += 2, - b'[' => { - let (rewritten, next) = scan_equivalence_bracket(pattern, i); - if let Some(body) = rewritten { - let out = out.get_or_insert_with(String::new); - out.push_str(&pattern[copied..=i]); - out.push_str(&body); - copied = next; - } - i = next; - } - _ => i += 1, + while let Some(open) = next_unescaped_bracket(bytes, i) { + let (rewritten, next) = scan_equivalence_bracket(pattern, open); + if let Some(body) = rewritten { + let out = out.get_or_insert_with(String::new); + out.push_str(&pattern[copied..=open]); + out.push_str(&body); + copied = next; } + i = next; } match out { @@ -837,6 +848,7 @@ mod tests { "[:notaclass:]", "[:x:]", "ab[:blank:]", + "\\\\[:blank:]", // the backslash is escaped, so the bracket is not ] { assert!(has_confusing_bracket(p.as_bytes()), "pattern {p:?}"); } @@ -845,18 +857,19 @@ mod tests { #[test] fn accepts_bracket_expressions_that_are_not_confusing() { for p in [ - "[[:digit:]]", // the correct spelling - "[::]", // no character besides the colons - "[:digit]", // does not end with a colon - "[:digit:qrs]", // ends with an ordinary character - "[:dig-it:]", // holds a range - "[:x[:digit:]:]", // holds a character class - "[:x[.,.]:]", // holds a collating element - "[:x[=e=]:]", // holds an equivalence class - "\\[:digit:]", // the bracket is escaped - "[]:digit:]", // starts with a literal ']' - "[:digit:", // unterminated - "[a-z]+[0-9]", // no colons at all + "[[:digit:]]", // the correct spelling + "[::]", // no character besides the colons + "[:digit]", // does not end with a colon + "[:digit:qrs]", // ends with an ordinary character + "[:dig-it:]", // holds a range + "[:x[:digit:]:]", // holds a character class + "[:x[.,.]:]", // holds a collating element + "[:x[=e=]:]", // holds an equivalence class + "\\[:digit:]", // the bracket is escaped + "\\\\\\[:digit:]", // and still escaped after an escaped backslash + "[]:digit:]", // starts with a literal ']' + "[:digit:", // unterminated + "[a-z]+[0-9]", // no colons at all ] { assert!(!has_confusing_bracket(p.as_bytes()), "pattern {p:?}"); }