Skip to content

[Pratt Parser] Cache source content size in Lexer to avoid repeated variant dispatch in character loops. #2274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Sep 18, 2026
Merged

[Pratt Parser] Cache source content size in Lexer to avoid repeated variant dispatch in character loops. #2274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 21 additions & 22 deletions parser/internal/lexer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ std::string_view TokenTypeToString(TokenType type) {

Token Lexer::Lex() {
int32_t start = GetPosition();
if (ABSL_PREDICT_FALSE(position_ >= content_.size())) {
if (ABSL_PREDICT_FALSE(position_ >= content_size_)) {
return MakeToken(TokenType::kEnd, start, start);
}
char32_t c = content_.at(position_);
Expand All @@ -182,8 +182,7 @@ Token Lexer::Lex() {
return MakeToken(TokenType::kWhitespace, start, GetPosition());
}
case '.': {
if (position_ + 1 < content_.size() &&
content_.at(position_ + 1) <= 0x7f &&
if (position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f &&
absl::ascii_isdigit(static_cast<char>(content_.at(position_ + 1)))) {
return ConsumeNumericLiteral();
}
Expand Down Expand Up @@ -342,7 +341,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) {
ABSL_DCHECK_NE(c, '\r');
int32_t pos = position_;
bool escaped = false;
while (pos < content_.size()) {
while (pos < content_size_) {
char32_t cc = content_.at(pos);
if (cc == '\n' || cc == '\r') {
AdvanceProcessingNewLines(pos);
Expand All @@ -359,7 +358,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) {
}
++pos;
}
AdvanceProcessingNewLines(content_.size());
AdvanceProcessingNewLines(content_size_);
return false;
}

Expand All @@ -370,7 +369,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) {
bool Lexer::ConsumeUntilAfterString(std::u32string_view s) {
ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos);
int32_t pos = position_;
while (pos + static_cast<int32_t>(s.size()) <= content_.size()) {
while (pos + static_cast<int32_t>(s.size()) <= content_size_) {
bool match = true;
for (size_t i = 0; i < s.size(); ++i) {
if (content_.at(pos + static_cast<int32_t>(i)) != s[i]) {
Expand All @@ -384,7 +383,7 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) {
}
++pos;
}
AdvanceProcessingNewLines(content_.size());
AdvanceProcessingNewLines(content_size_);
return false;
}

Expand All @@ -396,12 +395,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) {
ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos);
int32_t pos = position_;
bool escaped = false;
while (pos < content_.size()) {
while (pos < content_size_) {
char32_t cc = content_.at(pos);
if (cc == '\\') {
escaped = !escaped;
} else {
if (!escaped && pos + static_cast<int32_t>(s.size()) <= content_.size()) {
if (!escaped && pos + static_cast<int32_t>(s.size()) <= content_size_) {
bool match = true;
for (size_t j = 0; j < s.size(); ++j) {
if (content_.at(pos + static_cast<int32_t>(j)) != s[j]) {
Expand All @@ -418,12 +417,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) {
}
++pos;
}
AdvanceProcessingNewLines(content_.size());
AdvanceProcessingNewLines(content_size_);
return false;
}

bool Lexer::MatchString(std::u32string_view s) const {
if (position_ + static_cast<int32_t>(s.size()) > content_.size()) {
if (position_ + static_cast<int32_t>(s.size()) > content_size_) {
return false;
}
for (size_t i = 0; i < s.size(); ++i) {
Expand All @@ -436,7 +435,7 @@ bool Lexer::MatchString(std::u32string_view s) const {

std::optional<char32_t> Lexer::MatchIf(
absl::FunctionRef<bool(char32_t)> predicate) const {
if (position_ < content_.size()) {
if (position_ < content_size_) {
char32_t cp = content_.at(position_);
if (predicate(cp)) {
return cp;
Expand All @@ -446,7 +445,7 @@ std::optional<char32_t> Lexer::MatchIf(
}

void Lexer::ConsumeLine() {
while (position_ < content_.size()) {
while (position_ < content_size_) {
if (content_.at(position_) == '\n') {
Advance(1);
return;
Expand All @@ -456,7 +455,7 @@ void Lexer::ConsumeLine() {
}

void Lexer::ConsumeWhitespace() {
while (position_ < content_.size()) {
while (position_ < content_size_) {
char32_t c = content_.at(position_);
switch (c) {
case '\f':
Expand Down Expand Up @@ -517,7 +516,7 @@ std::optional<char32_t> Lexer::ConsumeIf(

bool Lexer::ConsumeDigits() {
bool advanced = false;
while (position_ < content_.size()) {
while (position_ < content_size_) {
char32_t c = content_.at(position_);
if (c > 0x7f || !absl::ascii_isdigit(static_cast<char>(c))) {
break;
Expand All @@ -530,7 +529,7 @@ bool Lexer::ConsumeDigits() {

bool Lexer::ConsumeHexDigits() {
bool advanced = false;
while (position_ < content_.size()) {
while (position_ < content_size_) {
char32_t c = content_.at(position_);
if (c > 0x7f || !absl::ascii_isxdigit(static_cast<char>(c))) {
break;
Expand Down Expand Up @@ -588,12 +587,12 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes,
// rb"""...""", rb'''...'''
std::optional<Token> Lexer::ConsumePrefixedStringLiteral() {
int32_t start = GetPosition();
if (position_ >= content_.size()) return std::nullopt;
if (position_ >= content_size_) return std::nullopt;
char32_t c = content_.at(position_);
bool is_bytes = (c == 'b' || c == 'B');
bool is_raw = (c == 'r' || c == 'R');
size_t lookahead = 1;
if (position_ + 1 < content_.size()) {
if (position_ + 1 < content_size_) {
char32_t c2 = content_.at(position_ + 1);
if ((is_bytes && (c2 == 'r' || c2 == 'R')) ||
(!is_bytes && (c2 == 'b' || c2 == 'B'))) {
Expand All @@ -602,7 +601,7 @@ std::optional<Token> Lexer::ConsumePrefixedStringLiteral() {
lookahead = 2;
}
}
if (position_ + static_cast<int32_t>(lookahead) < content_.size()) {
if (position_ + static_cast<int32_t>(lookahead) < content_size_) {
char32_t quote = content_.at(position_ + static_cast<int32_t>(lookahead));
if (quote == '"' || quote == '\'') {
Advance(lookahead);
Expand Down Expand Up @@ -649,8 +648,8 @@ Token Lexer::ConsumeNumericLiteral() {
}
}
static_cast<void>(ConsumeDigits());
if (position_ < content_.size() && content_.at(position_) == '.' &&
position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f &&
if (position_ < content_size_ && content_.at(position_) == '.' &&
position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f &&
absl::ascii_isdigit(static_cast<char>(content_.at(position_ + 1)))) {
floating_point = true;
Advance(1);
Expand Down Expand Up @@ -679,7 +678,7 @@ Token Lexer::ConsumeNumericLiteral() {

Token Lexer::ConsumeIdent() {
int32_t start = GetPosition();
while (position_ < content_.size()) {
while (position_ < content_size_) {
char32_t c = content_.at(position_);
if (!IsIdentTrailing(c)) {
break;
Expand Down
15 changes: 9 additions & 6 deletions parser/internal/lexer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,9 @@ struct LexerError final {
class Lexer final {
public:
explicit Lexer(const cel::Source& source)
: content_(source.content()), position_(0) {
: content_(source.content()),
content_size_(static_cast<int32_t>(content_.size())),
position_(0) {
ABSL_DCHECK_LE(content_.size(), static_cast<SourcePosition>(
std::numeric_limits<int32_t>::max()));
}
Expand All @@ -162,31 +164,31 @@ class Lexer final {

void RestorePosition(int32_t position) {
ABSL_DCHECK_GE(position, 0);
ABSL_DCHECK_LE(position, static_cast<int32_t>(content_.size()));
ABSL_DCHECK_LE(position, content_size_);
position_ = position;
error_ = LexerError{};
}

private:
[[nodiscard]] bool Match(char32_t c) const {
return position_ < content_.size() && content_.at(position_) == c;
return position_ < content_size_ && content_.at(position_) == c;
}

[[nodiscard]] bool MatchIgnoreCase(char32_t c) const {
if (position_ >= content_.size()) return false;
if (position_ >= content_size_) return false;
char32_t cp = content_.at(position_);
return cp <= 0x7f && c <= 0x7f &&
absl::ascii_tolower(static_cast<char>(cp)) ==
absl::ascii_tolower(static_cast<char>(c));
}

void Advance(size_t n) {
ABSL_DCHECK_LE(n, static_cast<size_t>(content_.size() - position_));
ABSL_DCHECK_LE(n, static_cast<size_t>(content_size_ - position_));
position_ += static_cast<int32_t>(n);
}

void AdvanceProcessingNewLines(int32_t end_position) {
ABSL_DCHECK_LE(end_position, content_.size());
ABSL_DCHECK_LE(end_position, content_size_);
ABSL_DCHECK_GE(end_position, position_);
Advance(static_cast<size_t>(end_position - position_));
}
Expand Down Expand Up @@ -275,6 +277,7 @@ class Lexer final {
[[nodiscard]] Token ConsumeIdent();

cel::SourceContentView content_;
int32_t content_size_ = 0;
int32_t position_ = 0;
LexerError error_;
};
Expand Down
Loading