WebEngine/parser/css/tokenizer.hpp at master · usadson/WebEngine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#pragma once

/**
 * Copyright (C) 2020 Tristan. All Rights Reserved.
 * This file is licensed under the BSD 2-Clause license.
 * See the COPYING file for licensing information.
 */

#include "data/text/unicode.hpp"
#include "data/text/ustring.hpp"
#include "parser/css/context.hpp"
#include "parser/css/parse_error.hpp"
#include "parser/css/token.hpp"
#include "parser/css/tokenizer_stream.hpp"

#ifndef PRIVATE_VISIBILITY
#define PRIVATE_VISIBILITY private
#endif

namespace CSS {
	class Tokenizer {
	  public:
		inline
		Tokenizer(CSS::Context &context, const Unicode::UString &input) :
			context(context), inputString(input), stream(&inputString) {
		}

		[[nodiscard]] bool
		Run() noexcept;

	  PRIVATE_VISIBILITY:
		CSS::Context &context;
		const Unicode::UString &inputString;
		TokenizerStream stream;
		Unicode::UString string;
		std::vector<Token> tokens{};

		// The 'consume comments' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.2.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-comment
		[[nodiscard]] bool
		ConsumeComments() noexcept;

		// The 'consume an escaped code point' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.7.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-escaped-code-point
		[[nodiscard]] Unicode::CodePoint
		ConsumeEscapedCodePoint() noexcept;

		// The 'consume an ident-linke token' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.4.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-an-ident-like-token
		[[nodiscard]] bool
		ConsumeIdentLikeToken() noexcept;

		// The 'consume a name' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.11.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-a-name
		bool
		ConsumeName(std::vector<Unicode::CodePoint> &) noexcept;

		// The 'consume a number' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.12.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-a-number
		[[nodiscard]] std::variant<std::monostate, CSS::IntegerType, CSS::NumberType>
		ConsumeNumber() noexcept;

		// The 'consume a numeric token' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.3.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-a-numeric-token
		[[nodiscard]] bool
		ConsumeNumericToken() noexcept;

		// The 'consume a string token' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.5.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-a-string-token
		[[nodiscard]] bool
		ConsumeStringToken(Unicode::CodePoint endingCodePoint) noexcept;

		// The 'consume the remnants of a bad url' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.14.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-remnants-of-bad-url
		void
		ConsumeRemnantsOfBadURL() noexcept;

		// The 'consume a token' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.1.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-token
		[[nodiscard]] bool
		ConsumeToken(Unicode::CodePoint character) noexcept;

		// The 'convert a string to a number' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.13.
		// URL: https://www.w3.org/TR/css-syntax-3/#convert-string-to-number
		[[nodiscard]] std::variant<std::monostate, CSS::IntegerType, CSS::NumberType>
		ConvertStringToNumber(const std::vector<char> &) noexcept;

		// The 'convert a[n] url token' tokenizer algorithm.
		//
		// [Spec]
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.6.
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-a-url-token
		[[nodiscard]] bool
		ConsumeURLToken() noexcept;

		// Will try to run the algorithm of ConsumeToken when the code point is
		// NUMBER SIGN. Will return false if the if-statement is failing,
		// otherwise the algorithm will be executed.
		//
		// [Spec] (Subsection)
		// Title: CSS Syntax Module Level 3
		// Section: 4.3.1.
		// Case: U+0023 NUMBER SIGN (#)
		// URL: https://www.w3.org/TR/css-syntax-3/#consume-token
		[[nodiscard]] bool
		TryParseHashTokenName() noexcept;

		// --- // --- // Helper Functions // --- // --- //
		[[nodiscard]] bool
		DoesStreamStartWithNumber() const noexcept;

		[[nodiscard]] bool
		ConsumeTokenHelperHyphenMinus(Unicode::CodePoint) noexcept;

		[[nodiscard]] bool
		SkipWhitespace() noexcept;

		friend class Parser;
	};
} // namespace CSS