Merge pull request #1361 from WebFuzzing/external-pr-lmasroca

Regex support for octal, control and long hex escapes

Author: arcuri82

core/src/main/antlr4/org/evomaster/core/parser/RegexEcma262.g4

@@ -96,24 +96,22 @@ atom
 
 
 //TODO
-CharacterEscape
-// : ControlEscape
-// | 'c' ControlLetter
- : HexEscapeSequence
+fragment CharacterEscape
+ : ControlEscape
+ | 'c' ControlLetter
+ | HexEscapeSequence
  | UnicodeEscapeSequence
  //| IdentityEscape
  ;
 
-//TODO
-//ControlEscape
-// //one of f n r t v
-// : [fnrtv]
-// ;
+fragment ControlEscape
+ //one of f n r t v
+ : [fnrtv]
+ ;
 
-//TODO
-//ControlLetter
-// : [a-zA-Z]
-// ;
+fragment ControlLetter
+ : [a-zA-Z]
+ ;
 
 
 //TODO
@@ -238,11 +236,11 @@ BaseChar
  : ~[0-9,^$\\.*+?()[\]{}|-]
  ;
 
-UnicodeEscapeSequence
+fragment UnicodeEscapeSequence
  : 'u' HexDigit HexDigit HexDigit HexDigit
  ;
 
-HexEscapeSequence
+fragment HexEscapeSequence
  : 'x' HexDigit HexDigit
  ;
 

core/src/main/antlr4/org/evomaster/core/parser/RegexJava.g4

@@ -119,24 +119,23 @@ quoteChar
 ;
 
 //TODO
-CharacterEscape
-// : ControlEscape
-// | 'c' ControlLetter
- : HexEscapeSequence
+fragment CharacterEscape
+ : ControlEscape
+ | 'c' ControlLetter
+ | HexEscapeSequence
  | UnicodeEscapeSequence
+ | OctalEscapeSequence
  //| IdentityEscape
  ;
 
-//TODO
-//ControlEscape
-// //one of f n r t v
-// : [fnrtv]
-// ;
+fragment ControlEscape
+ //one of f n r t v
+ : [aefnrt]
+ ;
 
-//TODO
-//ControlLetter
-// : [a-zA-Z]
-// ;
+fragment ControlLetter
+ : [?-_a-z]
+ ;
 
 
 //TODO
@@ -267,18 +266,29 @@ BaseChar
  : ~[0-9,^$\\.*+?()[\]{}|-]
  ;
 
-UnicodeEscapeSequence:
+fragment OctalEscapeSequence
+ : '0' OctalDigit
+ | '0' OctalDigit OctalDigit
+ | '0' [0-3] OctalDigit OctalDigit
+;
+
+fragment UnicodeEscapeSequence:
  'u' HexDigit HexDigit HexDigit HexDigit
 ;
 
-HexEscapeSequence
+fragment HexEscapeSequence
  : 'x' HexDigit HexDigit
+ | 'x' BRACE_open HexDigit+ BRACE_close
  ;
 
 fragment HexDigit:
  [a-fA-F0-9]
  ;
 
+fragment OctalDigit:
+ [0-7]
+ ;
+
 //TODO
 //DecimalIntegerLiteral
 // : '0'

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexEcma262Visitor.kt

@@ -171,9 +171,23 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
         if(ctx.AtomEscape() != null) {
             val txt = ctx.AtomEscape().text
             when {
+                txt[1]== 'c' -> {
+                    val controlLetterValue = txt[2].uppercaseChar().code.xor(0x40)
+                    return VisitResult(PatternCharacterBlockGene(txt, controlLetterValue.toChar().toString()))
+                }
+                txt[1] in "fnrtv" -> {
+                    val escape = when {
+                        txt[1] == 'n' -> "\u000A"
+                        txt[1] == 'v' -> "\u000B"
+                        txt[1] == 'f' -> "\u000C"
+                        txt[1] == 'r' -> "\u000D"
+                        else -> "\u0009"
+                    }
+                    return VisitResult(PatternCharacterBlockGene(txt, escape))
+                }
                 txt[1] == 'x' || txt[1] == 'u' -> {
                     val hexValue =
-                        txt.subSequence(2, txt.length).toString().toInt(16)
+                        txt.substring(2).toInt(16)
                     return VisitResult(
                         PatternCharacterBlockGene(
                             txt,

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -183,13 +183,47 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
         if(ctx.AtomEscape() != null){
             val txt = ctx.AtomEscape().text
             when {
+                txt[1] == '0' -> {
+                    val octalValue = txt.substring(2).toInt(8)
+                    return VisitResult(
+                        PatternCharacterBlockGene(
+                            txt,
+                            String(Character.toChars(octalValue))
+                        )
+                    )
+                }
+                txt[1]== 'c' -> {
+                    val controlLetterValue = if (txt[2].isLowerCase()){
+                        txt[2].uppercaseChar().code.xor(0x60)
+                    } else {
+                        txt[2].code.xor(0x40)
+                    }
+                    return VisitResult(PatternCharacterBlockGene(txt, controlLetterValue.toChar().toString()))
+                }
+                txt[1] in "aefnrt" -> {
+                    val escape = when {
+                        txt[1] == 'a' -> "\u0007"
+                        txt[1] == 'e' -> "\u001B"
+                        txt[1] == 'f' -> "\u000C"
+                        txt[1] == 'n' -> "\u000A"
+                        txt[1] == 'r' -> "\u000D"
+                        else -> "\u0009"
+                    }
+                    return VisitResult(PatternCharacterBlockGene(txt, escape))
+                }
                 txt[1] == 'x' || txt[1] == 'u' -> {
-                    val hexValue =
-                        txt.subSequence(2, txt.length).toString().toInt(16)
+                    val hexValue = when {
+                        txt[1] == 'x' && txt.length > 4 && txt[2] == '{' && txt[txt.length - 1] == '}'
+                            -> txt.substring(3, txt.length - 1).toInt(16)
+                        else -> txt.substring(2).toInt(16)
+                    }
+                    if(hexValue !in Character.MIN_CODE_POINT..Character.MAX_CODE_POINT){
+                        throw IllegalArgumentException("Hexadecimal escape out of range: ${ctx.text}")
+                    }
                     return VisitResult(
                         PatternCharacterBlockGene(
                             txt,
-                            hexValue.toChar().toString()
+                            String(Character.toChars(hexValue))
                         )
                     )
                 }

core/src/test/kotlin/org/evomaster/core/parser/GeneRegexEcma262VisitorTest.kt

@@ -329,11 +329,23 @@ open class GeneRegexEcma262VisitorTest : RegexTestTemplate(){
 
     @Test
     fun testHexEscape(){
-        checkSameAsJava("""\x00\x0a\xba\xFF""")
+        checkSameAsJava("""x00\x00\x0a\xba\xFF""")
     }
 
     @Test
     fun testUnicodeEscape(){
-        checkSameAsJava("""\u0000\u0a0b\uffff""")
+        checkSameAsJava("""u0000\u0000\u0a0b\uffff""")
+    }
+
+    @Test
+    open fun testControlEscape(){
+        checkSameAsJava("""ftnrv\f\t\n\r\v""")
+    }
+
+    @Test
+    open fun testControlLetterEscape(){
+        checkSameAsJava("""cac!\cA\cG\cZ""")
+        // The following escape sequences behave differently in Java and JavaScript.
+        checkCanSample("""\ca\cg\cz""","\u0001\u0007\u001A",10_000)
     }
 }

core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt

@@ -72,4 +72,24 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() {
         //checkSameAsJava("[9-1]") //not valid in Java
         checkCanSample("[9-1]", listOf("1","5","9"),200)
     }
+
+    @Test
+    fun testJavaHexEscape(){
+        checkSameAsJava("""x{3}\x{0}\x{FFFf}\x{0FFFf}\x{01FFFf}\x{10FFFf}""")
+    }
+
+    @Test
+    fun testJavaOctalEscape(){
+        checkSameAsJava("""00\00\07\077\0377\0378\0400""")
+    }
+
+    @Test
+    override fun testControlEscape(){
+        checkSameAsJava("""aefnrt\a\e\f\n\r\t""")
+    }
+
+    @Test
+    override fun testControlLetterEscape() {
+        checkSameAsJava("""cac!\ca\cg\cz\cA\cG\cZ\c@\c[\c\\c]\c^\c\c_\c?""")
+    }
 }

core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt

@@ -125,11 +125,21 @@ internal class RegexHandlerTest{
     fun testCreateGeneForJVMInvalidRegex() {
 
         assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\xR") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\ugggg") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\x{}") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\x{") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\x}") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\x[h}") }
+        assertThrows(IllegalArgumentException::class.java) { RegexHandler.createGeneForJVM("\\x{110000}") }
+        assertThrows(IllegalArgumentException::class.java) { RegexHandler.createGeneForJVM("\\x{ffffff}") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\0") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\09") }
     }
 
     @Test
     fun testCreateGeneForEcma262InvalidRegex() {
 
         assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForEcma262("\\xR") }
+        assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\ugggg") }
     }
 }