From 87e059be1f087b3708ba52802979957d2d205ab3 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Fri, 11 Sep 2026 01:49:27 -0300 Subject: [PATCH 1/2] Added Java regex \R linebreak matcher. --- .../evomaster/core/parser/RegexJavaLexer.g4 | 5 ++ .../evomaster/core/parser/RegexJavaParser.g4 | 1 + .../core/parser/GeneRegexJavaVisitor.kt | 68 ++++++++++++++----- .../core/parser/GeneRegexJavaVisitorTest.kt | 9 +++ 4 files changed, 66 insertions(+), 17 deletions(-) diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 index c26bd66cae..bd1fc7800c 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 @@ -136,6 +136,11 @@ NamedBackReference : SLASH 'k<' [a-zA-Z] [a-zA-Z0-9]* '>' ; +// \R, equivalent to \r\n|[\n-\r\u0085\u2028-\u2029] +LinebreakMatcher + : SLASH 'R' + ; + // Special for Java: \Q...\E literal quoting. // Matching '\Q' switches the lexer into QUOTE_MODE (see below). QUOTE_OPEN diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 index 0b0f7d2834..ac4dfc1405 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 @@ -180,4 +180,5 @@ atomEscape | CharacterEscape | BackReference | NamedBackReference + | LinebreakMatcher // \R ; diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index bbd5ef7cbe..4806b5687b 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -12,24 +12,37 @@ import org.evomaster.core.utils.RegexFlags */ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor(){ - private val hexEscapePrefixes = setOf('x', 'u') - - /** - * Mappings of various escapes to their matching characters. - */ - private val escapeMap = mapOf( - 'a' to "\u0007", - 'e' to "\u001B", - 'f' to "\u000C", - 'n' to "\u000A", - 'r' to "\u000D", - 't' to "\u0009" - ) + companion object { + /** + * Prefixes for hexadecimal escape sequences like \x00 or \u0000, etc. + */ + private val hexEscapePrefixes = setOf('x', 'u') + + /** + * Mappings of various escapes to their matching characters. + */ + private val escapeMap = mapOf( + 'a' to "\u0007", + 'e' to "\u001B", + 'f' to "\u000C", + 'n' to "\u000A", + 'r' to "\u000D", + 't' to "\u0009" + ) - /** - * None of these can be escaped to be treated as literals. Some may be part of legal escape sequences. - */ - private val notIdentityEscapes = ('a'..'z').toList() + ('A'..'Z').toList() + ('0'..'9').toList() + /** + * None of these can be escaped to be treated as literals. Some may be part of legal escape sequences. + */ + private val notIdentityEscapes = ('a'..'z').toList() + ('A'..'Z').toList() + ('0'..'9').toList() + + /** + * All single character line break characters, part of \R linebreak matcher. + */ + private val linebreakCharRanges = MultiCharacterRange( + false, + listOf(CharacterRange('\n', '\r'), CharacterRange('\u0085'), CharacterRange('\u2028', '\u2029')) + ) + } /** * Capture groups in order of appearance (1-based index -> list index 0). @@ -133,6 +146,23 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege else -> txt.substring(1) // identity escape } + /** + * Builds gene for linebreak matcher `\R`, which is equivalent to `\r\n|[\n-\r\u0085\u2028-\u2029]`. + */ + private fun buildLinebreakMatcherGene(): DisjunctionListRxGene { + val crlfBranch = DisjunctionRxGene( + "linebreak_crlf", + listOf(PatternCharacterBlockGene("crlf", "\r\n")), + matchStart = true, matchEnd = true + ) + val singleCharBranch = DisjunctionRxGene( + "linebreak_char", + listOf(CharacterRangeRxGene(linebreakCharRanges, RegexFlags())), + matchStart = true, matchEnd = true + ) + return DisjunctionListRxGene(listOf(crlfBranch, singleCharBranch)) + } + override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult { val res = ctx.disjunction().accept(this) @@ -688,6 +718,10 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege return VisitResult(BackReferenceRxGene(groupIndex, group)) } + if (ctx.LinebreakMatcher() != null){ + return VisitResult(buildLinebreakMatcherGene()) + } + if (ctx.CharacterClassEscape() != null) { return VisitResult(CharacterClassEscapeRxGene(ctx.CharacterClassEscape().text.substring(1), currentFlags)) } diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index f81cead7c1..dd2bcf95f8 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -611,4 +611,13 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { checkSameAsJava("\\w*\\Bfoo") assertThrows { checkSameAsJava("\\b\\Bfoo") } } + + @Test + fun testLinebreakMatcher() { + checkSameAsJava("^\\R\\z") + checkSameAsJava("^(?d)\\R\\z") + checkCanSample("^\\R\\z", listOf("\n", "\u000B", "\u000C", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 100) + // \R is not affected by unixLines flag + checkCanSample("^(?d)\\R\\z", listOf("\n", "\u000B", "\u000C", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 100) + } } \ No newline at end of file From 02db13b5d20aac6b2d01ab2ae20fb3188f061452 Mon Sep 17 00:00:00 2001 From: lmasroca Date: Fri, 11 Sep 2026 15:56:58 -0300 Subject: [PATCH 2/2] Added Java regex \Z assertions. --- .../evomaster/core/parser/RegexJavaLexer.g4 | 1 + .../evomaster/core/parser/RegexJavaParser.g4 | 1 + .../core/parser/GeneRegexJavaVisitor.kt | 1 + .../core/search/gene/regex/AssertionRxGene.kt | 3 +- .../search/gene/regex/DisjunctionRxGene.kt | 73 +++++++++++++++++++ .../search/gene/regex/QuantifierRxGene.kt | 41 +++++++++++ .../core/parser/GeneRegexJavaVisitorTest.kt | 13 ++++ 7 files changed, 132 insertions(+), 1 deletion(-) diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 index bd1fc7800c..182125007b 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 @@ -75,6 +75,7 @@ CharacterClassEscape StartOfInputAssertion: SLASH 'A'; EndOfInputAssertion: SLASH 'z'; +EndOfInputOrFinalLineTerminatorAssertion: SLASH 'Z'; WordBoundaryAssertion: SLASH 'b'; NonWordBoundaryAssertion: SLASH 'B'; diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 index ac4dfc1405..0a8b3d7a1e 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 @@ -54,6 +54,7 @@ assertion //// | '(' '?' '!' disjunction ')' | StartOfInputAssertion // \A | EndOfInputAssertion // \z + | EndOfInputOrFinalLineTerminatorAssertion // \Z | WordBoundaryAssertion // \b | NonWordBoundaryAssertion // \B ; diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index 4806b5687b..ec75ab884b 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -290,6 +290,7 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege assertionCtx.NonWordBoundaryAssertion() != null -> AssertionType.NON_WORD_BOUNDARY assertionCtx.StartOfInputAssertion() != null -> AssertionType.START_OF_INPUT assertionCtx.EndOfInputAssertion() != null -> AssertionType.END_OF_INPUT + assertionCtx.EndOfInputOrFinalLineTerminatorAssertion() != null -> AssertionType.END_OF_INPUT_OR_FINAL_LINE_TERMINATOR assertionCtx.CARET() != null -> AssertionType.CARET assertionCtx.DOLLAR() != null -> AssertionType.DOLLAR assertionCtx.LESS_THAN() != null -> AssertionType.LOOKBEHIND diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt index 1549a24823..1a2dd5ab39 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt @@ -22,7 +22,8 @@ enum class AssertionType(val usesInnerGene: Boolean) { CARET(usesInnerGene = false), DOLLAR(usesInnerGene = false), WORD_BOUNDARY(usesInnerGene = false), - NON_WORD_BOUNDARY(usesInnerGene = false) + NON_WORD_BOUNDARY(usesInnerGene = false), + END_OF_INPUT_OR_FINAL_LINE_TERMINATOR(usesInnerGene = false) } /** diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt index 82c861f02d..7336503d60 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt @@ -403,6 +403,8 @@ class DisjunctionRxGene( repairBidirectional(assertion, wordBoundaryBranches, idx, randomness) AssertionType.NON_WORD_BOUNDARY -> repairBidirectional(assertion, nonWordBoundaryBranches, idx, randomness) + AssertionType.END_OF_INPUT_OR_FINAL_LINE_TERMINATOR -> + repairZAssertion(assertion, genesAfter(idx), randomness) } /** @@ -559,6 +561,77 @@ class DisjunctionRxGene( ) } + /** + * Repairs `\Z` assertion, which is lookahead for end of input or + * final line terminator (line terminator followed by end of input). + */ + private fun repairZAssertion(assertion: AssertionRxGene, target: List, randomness: Randomness): AssertionRepairResult { + return tryInRandomOrder( + { repairStrictBoundaryAssertion(target, false) }, + { repairTerminatorThenForceRestEmpty(assertion, target, randomness) }, + randomness + ) + } + + /** + * Repairs `\Z` assertion's final line terminator branch (line terminator and then end of string). + * In Java, valid line terminators depend on [org.evomaster.core.utils.RegexFlags.unixLines], since if + * that flag is on only `\n` is a line terminator otherwise `\r\n|[\n\r\u0085\u2028\u2029]` are terminators. + */ + private fun repairTerminatorThenForceRestEmpty(assertion: AssertionRxGene, target: List, randomness: Randomness): AssertionRepairResult { + repeat(MAX_LOCAL_ASSERTION_ATTEMPTS) { + val tryCrlf = !assertion.flags.unixLines && randomness.nextBoolean() + val candidate = if (tryCrlf) { + "\r\n" + } else { + assertion.flags.lineTerminatorRanges.sample(randomness).toString() + } + if (forceExactMatch(target, candidate)) { + return AssertionRepairResult.SUCCESS + } + } + return AssertionRepairResult.FAILURE + } + + /** + * Forces [terms]' value to match [value] exactly, one of the genes must absorb that value + * and the rest must be zero width. + * + * @return whether [terms] could be forced to render exactly [value]. + */ + private fun forceExactMatch(terms: List, value: String): Boolean { + + // find the first gene that can absorb the whole value + val absorbedAt = terms.indexOfFirst { (it as RxAbsorbable).absorbableCount(value) == value.length } + if (absorbedAt == -1) return false + + // verify all other genes can collapse to zero width + val allOthersZeroWidth = terms.indices.all { i -> + i == absorbedAt || (terms[i] as RxAbsorbable).canBeZeroWidth + } + if (!allOthersZeroWidth) return false + + // force the genes' values + terms.forEachIndexed { i, gene -> + val g = gene as RxAbsorbable + if (i == absorbedAt) g.tryForce(value) else g.forceZeroWidth() + } + + // check the gene produces exactly that char (and nothing else) + val gene = terms[absorbedAt] + if (gene.getValueAsRawString() == value) { + return true + } + + // some genes could have trailing characters which we could still trim to make the repair successful + return when (gene) { + is QuantifierRxGene -> gene.trimToExactValue(value) + is DisjunctionListRxGene -> forceExactMatch(gene.disjunctions[gene.activeDisjunction].terms, value) + is DisjunctionRxGene -> forceExactMatch(gene.terms, value) + else -> false + } + } + /** * The genes in [terms] lying before index [idx], excluding other assertions. This is the forcing * target for a [AssertionType.LOOKBEHIND] assertion (or an outward requirement) sitting at [idx]. diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt index fd7ca0eef7..46586a59b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt @@ -297,4 +297,45 @@ class QuantifierRxGene( atoms.forEach { (it as RxAbsorbable).forceZeroWidth() } } } + + /** + * Attempts to trim [atoms] so that this gene's value becomes exactly [value] (no trailing characters). + * Used during repairs for `\Z` assertions. + * @see DisjunctionRxGene.repairZAssertion + * @see DisjunctionRxGene.forceExactMatch + */ + fun trimToExactValue(value: String): Boolean { + var consumed = 0 + var neededAtoms = 0 + for (atom in atoms) { + if (consumed >= value.length) break + consumed += atom.getValueAsRawString().length + neededAtoms++ + } + if (consumed != value.length) return false + if (getValueAsRawString() == value) return true + + val minimumAtomsToKeep = maxOf(neededAtoms, min) + + // Remove or zero-width trailing atoms so the gene matches the target value exactly. + for (i in atoms.size - 1 downTo minimumAtomsToKeep) { + val atom = atoms[i] + if (atom is RxAbsorbable && atom.canBeZeroWidth) { + atom.forceZeroWidth() + } else { + killChildByIndex(i) + } + } + + // Zero-width (if possible) atoms required by min but not needed to produce the target value. + for (i in neededAtoms until min) { + val atom = atoms[i] + if (atom !is RxAbsorbable || !atom.canBeZeroWidth) { + return false + } + atom.forceZeroWidth() + } + + return getValueAsRawString() == value + } } diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index dd2bcf95f8..635936dc7c 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -620,4 +620,17 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { // \R is not affected by unixLines flag checkCanSample("^(?d)\\R\\z", listOf("\n", "\u000B", "\u000C", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 100) } + + @Test + fun testZAssertion(){ + checkSameAsJava("^\\Z\\R?") + checkSameAsJava("(?s)^\\Z.{0,1}") + checkSameAsJava("(?s)^\\Z(.*)") + checkSameAsJava("(?sd)^\\Z.*") + checkCanSample("(?sd)^\\Z.*", listOf("", "\n"), 1000) + assertThrows { checkCanSample("(?sd)^\\Z.*", listOf("\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000) } + checkSameAsJava("(?s)^\\Z.*") + checkCanSample("(?s)^\\Z.*", listOf("", "\n", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000) + checkCanSample("(?s)^\\Z(.?){5}", listOf("", "\n", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000) + } } \ No newline at end of file