Java Regex Z Assertion

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4

@@ -75,6 +75,7 @@ CharacterClassEscape
 
 StartOfInputAssertion: SLASH 'A';
 EndOfInputAssertion: SLASH 'z';
+EndOfInputOrFinalLineTerminatorAssertion: SLASH 'Z';
 
 WordBoundaryAssertion: SLASH 'b';
 NonWordBoundaryAssertion: SLASH 'B';
@@ -136,6 +137,11 @@ NamedBackReference
  : SLASH 'k<' [a-zA-Z] [a-zA-Z0-9]* '>'
  ;
 
+// \R, equivalent to \r\n|[\n-\r\u0085\u2028-\u2029]
+LinebreakMatcher
+ : SLASH 'R'
+ ;
+
 // Special for Java: \Q...\E literal quoting.
 // Matching '\Q' switches the lexer into QUOTE_MODE (see below).
 QUOTE_OPEN

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4

@@ -54,6 +54,7 @@ assertion
 //// | '(' '?' '!' disjunction ')'
  | StartOfInputAssertion // \A
  | EndOfInputAssertion // \z
+ | EndOfInputOrFinalLineTerminatorAssertion // \Z
  | WordBoundaryAssertion // \b
  | NonWordBoundaryAssertion // \B
  ;
@@ -180,4 +181,5 @@ atomEscape
  | CharacterEscape
  | BackReference
  | NamedBackReference
+ | LinebreakMatcher // \R
  ;

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -12,24 +12,37 @@ import org.evomaster.core.utils.RegexFlags
  */
 class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor<VisitResult>(){
 
-    private val hexEscapePrefixes = setOf('x', 'u')
-
-    /**
-     * Mappings of various escapes to their matching characters.
-     */
-    private val escapeMap = mapOf(
-        'a' to "\u0007",
-        'e' to "\u001B",
-        'f' to "\u000C",
-        'n' to "\u000A",
-        'r' to "\u000D",
-        't' to "\u0009"
-    )
+    companion object {
+        /**
+         * Prefixes for hexadecimal escape sequences like \x00 or \u0000, etc.
+         */
+        private val hexEscapePrefixes = setOf('x', 'u')
+
+        /**
+         * Mappings of various escapes to their matching characters.
+         */
+        private val escapeMap = mapOf(
+            'a' to "\u0007",
+            'e' to "\u001B",
+            'f' to "\u000C",
+            'n' to "\u000A",
+            'r' to "\u000D",
+            't' to "\u0009"
+        )
 
-    /**
-     * None of these can be escaped to be treated as literals. Some may be part of legal escape sequences.
-     */
-    private val notIdentityEscapes = ('a'..'z').toList() + ('A'..'Z').toList() + ('0'..'9').toList()
+        /**
+         * None of these can be escaped to be treated as literals. Some may be part of legal escape sequences.
+         */
+        private val notIdentityEscapes = ('a'..'z').toList() + ('A'..'Z').toList() + ('0'..'9').toList()
+
+        /**
+         * All single character line break characters, part of \R linebreak matcher.
+         */
+        private val linebreakCharRanges = MultiCharacterRange(
+            false,
+            listOf(CharacterRange('\n', '\r'), CharacterRange('\u0085'), CharacterRange('\u2028', '\u2029'))
+        )
+    }
 
     /**
      * Capture groups in order of appearance (1-based index -> list index 0).
@@ -133,6 +146,23 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
         else -> txt.substring(1) // identity escape
     }
 
+    /**
+     * Builds gene for linebreak matcher `\R`, which is equivalent to `\r\n|[\n-\r\u0085\u2028-\u2029]`.
+     */
+    private fun buildLinebreakMatcherGene(): DisjunctionListRxGene {
+        val crlfBranch = DisjunctionRxGene(
+            "linebreak_crlf",
+            listOf(PatternCharacterBlockGene("crlf", "\r\n")),
+            matchStart = true, matchEnd = true
+        )
+        val singleCharBranch = DisjunctionRxGene(
+            "linebreak_char",
+            listOf(CharacterRangeRxGene(linebreakCharRanges, RegexFlags())),
+            matchStart = true, matchEnd = true
+        )
+        return DisjunctionListRxGene(listOf(crlfBranch, singleCharBranch))
+    }
+
     override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult {
 
         val res = ctx.disjunction().accept(this)
@@ -260,6 +290,7 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
                 assertionCtx.NonWordBoundaryAssertion() != null -> AssertionType.NON_WORD_BOUNDARY
                 assertionCtx.StartOfInputAssertion() != null -> AssertionType.START_OF_INPUT
                 assertionCtx.EndOfInputAssertion() != null -> AssertionType.END_OF_INPUT
+                assertionCtx.EndOfInputOrFinalLineTerminatorAssertion() != null -> AssertionType.END_OF_INPUT_OR_FINAL_LINE_TERMINATOR
                 assertionCtx.CARET() != null -> AssertionType.CARET
                 assertionCtx.DOLLAR() != null -> AssertionType.DOLLAR
                 assertionCtx.LESS_THAN() != null -> AssertionType.LOOKBEHIND
@@ -688,6 +719,10 @@ class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: Rege
             return VisitResult(BackReferenceRxGene(groupIndex, group))
         }
 
+        if (ctx.LinebreakMatcher() != null){
+            return VisitResult(buildLinebreakMatcherGene())
+        }
+
         if (ctx.CharacterClassEscape() != null) {
             return VisitResult(CharacterClassEscapeRxGene(ctx.CharacterClassEscape().text.substring(1), currentFlags))
         }

core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt

@@ -22,7 +22,8 @@ enum class AssertionType(val usesInnerGene: Boolean) {
     CARET(usesInnerGene = false),
     DOLLAR(usesInnerGene = false),
     WORD_BOUNDARY(usesInnerGene = false),
-    NON_WORD_BOUNDARY(usesInnerGene = false)
+    NON_WORD_BOUNDARY(usesInnerGene = false),
+    END_OF_INPUT_OR_FINAL_LINE_TERMINATOR(usesInnerGene = false)
 }
 
 /**

core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt

@@ -403,6 +403,8 @@ class DisjunctionRxGene(
                 repairBidirectional(assertion, wordBoundaryBranches, idx, randomness)
             AssertionType.NON_WORD_BOUNDARY ->
                 repairBidirectional(assertion, nonWordBoundaryBranches, idx, randomness)
+            AssertionType.END_OF_INPUT_OR_FINAL_LINE_TERMINATOR ->
+                repairZAssertion(assertion, genesAfter(idx), randomness)
         }
 
     /**
@@ -559,6 +561,77 @@ class DisjunctionRxGene(
             )
         }
 
+    /**
+     * Repairs `\Z` assertion, which is lookahead for end of input or
+     * final line terminator (line terminator followed by end of input).
+     */
+    private fun repairZAssertion(assertion: AssertionRxGene, target: List<Gene>, randomness: Randomness): AssertionRepairResult {
+        return tryInRandomOrder(
+            { repairStrictBoundaryAssertion(target, false) },
+            { repairTerminatorThenForceRestEmpty(assertion, target, randomness) },
+            randomness
+        )
+    }
+
+    /**
+     * Repairs `\Z` assertion's final line terminator branch (line terminator and then end of string).
+     * In Java, valid line terminators depend on [org.evomaster.core.utils.RegexFlags.unixLines], since if
+     * that flag is on only `\n` is a line terminator otherwise `\r\n|[\n\r\u0085\u2028\u2029]` are terminators.
+     */
+    private fun repairTerminatorThenForceRestEmpty(assertion: AssertionRxGene, target: List<Gene>, randomness: Randomness): AssertionRepairResult {
+        repeat(MAX_LOCAL_ASSERTION_ATTEMPTS) {
+            val tryCrlf = !assertion.flags.unixLines && randomness.nextBoolean()
+            val candidate = if (tryCrlf) {
+                "\r\n"
+            } else {
+                assertion.flags.lineTerminatorRanges.sample(randomness).toString()
+            }
+            if (forceExactMatch(target, candidate)) {
+                return AssertionRepairResult.SUCCESS
+            }
+        }
+        return AssertionRepairResult.FAILURE
+    }
+
+    /**
+     * Forces [terms]' value to match [value] exactly, one of the genes must absorb that value
+     * and the rest must be zero width.
+     *
+     * @return whether [terms] could be forced to render exactly [value].
+     */
+    private fun forceExactMatch(terms: List<Gene>, value: String): Boolean {
+
+        // find the first gene that can absorb the whole value
+        val absorbedAt = terms.indexOfFirst { (it as RxAbsorbable).absorbableCount(value) == value.length }
+        if (absorbedAt == -1) return false
+
+        // verify all other genes can collapse to zero width
+        val allOthersZeroWidth = terms.indices.all { i ->
+            i == absorbedAt || (terms[i] as RxAbsorbable).canBeZeroWidth
+        }
+        if (!allOthersZeroWidth) return false
+
+        // force the genes' values
+        terms.forEachIndexed { i, gene ->
+            val g = gene as RxAbsorbable
+            if (i == absorbedAt) g.tryForce(value) else g.forceZeroWidth()
+        }
+
+        // check the gene produces exactly that char (and nothing else)
+        val gene = terms[absorbedAt]
+        if (gene.getValueAsRawString() == value) {
+            return true
+        }
+
+        // some genes could have trailing characters which we could still trim to make the repair successful
+        return when (gene) {
+            is QuantifierRxGene -> gene.trimToExactValue(value)
+            is DisjunctionListRxGene -> forceExactMatch(gene.disjunctions[gene.activeDisjunction].terms, value)
+            is DisjunctionRxGene -> forceExactMatch(gene.terms, value)
+            else -> false
+        }
+    }
+
     /**
      * The genes in [terms] lying before index [idx], excluding other assertions. This is the forcing
      * target for a [AssertionType.LOOKBEHIND] assertion (or an outward requirement) sitting at [idx].

core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt

@@ -297,4 +297,45 @@ class QuantifierRxGene(
             atoms.forEach { (it as RxAbsorbable).forceZeroWidth() }
         }
     }
+
+    /**
+     * Attempts to trim [atoms] so that this gene's value becomes exactly [value] (no trailing characters).
+     * Used during repairs for `\Z` assertions.
+     * @see DisjunctionRxGene.repairZAssertion
+     * @see DisjunctionRxGene.forceExactMatch
+     */
+    fun trimToExactValue(value: String): Boolean {
+        var consumed = 0
+        var neededAtoms = 0
+        for (atom in atoms) {
+            if (consumed >= value.length) break
+            consumed += atom.getValueAsRawString().length
+            neededAtoms++
+        }
+        if (consumed != value.length) return false
+        if (getValueAsRawString() == value) return true
+
+        val minimumAtomsToKeep = maxOf(neededAtoms, min)
+
+        // Remove or zero-width trailing atoms so the gene matches the target value exactly.
+        for (i in atoms.size - 1 downTo minimumAtomsToKeep) {
+            val atom = atoms[i]
+            if (atom is RxAbsorbable && atom.canBeZeroWidth) {
+                atom.forceZeroWidth()
+            } else {
+                killChildByIndex(i)
+            }
+        }
+
+        // Zero-width (if possible) atoms required by min but not needed to produce the target value.
+        for (i in neededAtoms until min) {
+            val atom = atoms[i]
+            if (atom !is RxAbsorbable || !atom.canBeZeroWidth) {
+                return false
+            }
+            atom.forceZeroWidth()
+        }
+
+        return getValueAsRawString() == value
+    }
 }

core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt

@@ -611,4 +611,26 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() {
         checkSameAsJava("\\w*\\Bfoo")
         assertThrows<AssertionError> { checkSameAsJava("\\b\\Bfoo") }
     }
+
+    @Test
+    fun testLinebreakMatcher() {
+        checkSameAsJava("^\\R\\z")
+        checkSameAsJava("^(?d)\\R\\z")
+        checkCanSample("^\\R\\z", listOf("\n", "\u000B", "\u000C", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 100)
+        // \R is not affected by unixLines flag
+        checkCanSample("^(?d)\\R\\z", listOf("\n", "\u000B", "\u000C", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 100)
+    }
+
+    @Test
+    fun testZAssertion(){
+        checkSameAsJava("^\\Z\\R?")
+        checkSameAsJava("(?s)^\\Z.{0,1}")
+        checkSameAsJava("(?s)^\\Z(.*)")
+        checkSameAsJava("(?sd)^\\Z.*")
+        checkCanSample("(?sd)^\\Z.*", listOf("", "\n"), 1000)
+        assertThrows<AssertionError> { checkCanSample("(?sd)^\\Z.*", listOf("\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000) }
+        checkSameAsJava("(?s)^\\Z.*")
+        checkCanSample("(?s)^\\Z.*", listOf("", "\n", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000)
+        checkCanSample("(?s)^\\Z(.?){5}", listOf("", "\n", "\r", "\u0085", "\u2028", "\u2029", "\r\n"), 1000)
+    }
 }