RegEx-Engine/Source/Examples/Simple_lexer.pb at main · SicroAtGit/RegEx-Engine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

EnableExplicit

IncludePath ".."
IncludeFile "RegExEngine.pbi"

Enumeration TokenTypes
  #TokenType_Operator_EqualSign
  #TokenType_Operator_Plus
  #TokenType_Number
  #TokenType_String
  #TokenType_Variable
  #TokenType_Whitespace
EndEnumeration

Procedure$ GetTokenTypeName(tokenType)
  Select tokenType
    Case #TokenType_Operator_EqualSign
      ProcedureReturn "Operator_EqualSign"
    Case #TokenType_Operator_Plus
      ProcedureReturn "Operator_Plus"
    Case #TokenType_Number
      ProcedureReturn "Number"
    Case #TokenType_String
      ProcedureReturn "String"
    Case #TokenType_Variable
      ProcedureReturn "Variable"
    Case #TokenType_Whitespace
      ProcedureReturn "Whitespace"
  EndSelect
EndProcedure

Define *lexer = RegEx::Init()
If *lexer = 0
  Debug "Error: Lexer could not be created!"
  End
EndIf

RegEx::AddNfa(*lexer, "=", #TokenType_Operator_EqualSign)
RegEx::AddNfa(*lexer, "\+", #TokenType_Operator_Plus)
RegEx::AddNfa(*lexer, "\d+", #TokenType_Number)
RegEx::AddNfa(*lexer, "'[^']*'", #TokenType_String)
RegEx::AddNfa(*lexer, "[A-Za-z]+", #TokenType_Variable)
RegEx::AddNfa(*lexer, "\s", #TokenType_Whitespace)
; RegEx::CreateDfa(*lexer)

Define code$ = "sum = 100 + 5" + #CRLF$ +
               "string = 'Example text'"

Define *stringPointer = @code$
Define matchLength
Define tokenType

Repeat
  matchLength = RegEx::Match(*lexer, *stringPointer, @tokenType)
  If matchLength
    If tokenType <> #TokenType_Whitespace
      Debug "[" + GetTokenTypeName(tokenType) + "]: " + RegEx::GetString(*stringPointer, matchLength)
      Debug ""
    EndIf
    *stringPointer + matchLength
  EndIf
Until matchLength = 0

RegEx::Free(*lexer)