Skip to content

[textinput] Support UTF-8 at the prompt #23085

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
guitargeek wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[textinput] Support UTF-8 at the prompt #23085

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/textinput/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ target_sources(Core PRIVATE
src/textinput/TerminalDisplayWin.cpp
src/textinput/TextInputContext.cpp
src/textinput/TextInput.cpp
src/textinput/UTF8.cpp
)

target_include_directories(Core
Expand Down
13 changes: 8 additions & 5 deletions core/textinput/src/Getline.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,10 @@ namespace {
EditorRange& r /*out*/,
std::vector<std::string>& displayCompletions /*out*/) override {
strlcpy(fLineBuf, line.GetText().c_str(), fgLineBufSize);
int cursorInt = (int) cursor;
// TTabCom edits the UTF-8 buffer, so it speaks byte offsets; the
// editor's cursor and ranges count characters. Convert on the way in
// and back out.
int cursorInt = (int) line.GetByteOffset(cursor);
std::stringstream sstr;
size_t posFirstChange = gApplication->TabCompletionHook(fLineBuf, &cursorInt, sstr);
if (posFirstChange == (size_t) -1) {
Expand All @@ -73,12 +76,12 @@ namespace {
r.fEdit.Extend(Range::AllText());
r.fDisplay.Extend(Range::AllText());
} else {
r.fEdit.Extend(Range(posFirstChange, Range::End()));
r.fDisplay.Extend(Range(posFirstChange, Range::End()));
size_t charFirstChange = line.GetCharIndex(posFirstChange);
r.fEdit.Extend(Range(charFirstChange, Range::End()));
r.fDisplay.Extend(Range(charFirstChange, Range::End()));
}
}
cursor = (size_t)cursorInt;
line.GetColors().resize(lenLineBuf);
cursor = line.GetCharIndex((size_t)cursorInt);
return true;
}
private:
Expand Down
32 changes: 21 additions & 11 deletions core/textinput/src/Getline_color.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

#include "Getline_color.h"

#include <cstring>
#include <stack>
#include <string>

Expand All @@ -21,6 +22,7 @@
#include "TROOT.h"
#include "textinput/Range.h"
#include "textinput/Text.h"
#include "textinput/UTF8.h"

using std::stack;
using namespace textinput;
Expand Down Expand Up @@ -109,8 +111,13 @@ namespace {
return Color();
} // ColorFromName()

bool IsAlnum_(char c) { return c == '_' || isalnum(c); }
bool IsAlpha_(char c) { return c == '_' || isalpha(c); }
// The text is UTF-32; the classification functions from <cctype> only
// accept values that fit in an unsigned char. Nothing above ASCII can be
// part of a C++ type name, so those characters are simply "not a word".
bool IsAlnum_(char32_t c) { return c == U'_' || (c <= 0x7F && isalnum((int)c)); }
bool IsAlpha_(char32_t c) { return c == U'_' || (c <= 0x7F && isalpha((int)c)); }
bool IsDigit_(char32_t c) { return c <= 0x7F && isdigit((int)c); }
bool IsSpace_(char32_t c) { return c <= 0x7F && isspace((int)c); }
} // unnamed namespace


Expand Down Expand Up @@ -185,7 +192,10 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification,
Text& input) {
// The text has changed; look for word that are types.

const std::string& text = input.GetText();
// Index characters, not bytes: the ranges and the color vector are per
// character, so indexing input.GetText() would drift apart from them as
// soon as the line contains anything outside ASCII.
const std::u32string& text = input.GetChars();

size_t modStart = Modification.fEdit.fStart;
size_t inputLength = input.length();
Expand All @@ -209,14 +219,14 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification,
while (modStart && IsAlnum_(text[modStart])) --modStart;

// Ignore spaces
while (modStart < modEnd && isspace(text[modStart]))
while (modStart < modEnd && IsSpace_(text[modStart]))
++modStart;
while (modEnd > modStart && isspace(text[modEnd]))
--modStart;
while (modEnd > modStart && IsSpace_(text[modEnd - 1]))
--modEnd;

for (size_t i = modStart; i < modEnd;) {
// i points to beginning of word here.
if (isdigit(text[i])) {
if (IsDigit_(text[i])) {
// "12", or "12ull". Default color.
ExtendRangeAndSetColor(input, i, 0, Modification.fDisplay);
++i;
Expand All @@ -230,7 +240,7 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification,
while (i + wordLen < modEnd && IsAlnum_(text[i + wordLen])) {
++wordLen;
}
std::string word = text.substr(i, wordLen);
std::string word = UTF32ToUTF8(text.substr(i, wordLen));
char color = kColorNone;
if (gClassTable->GetDict(word.c_str())
|| gInterpreter->GetClassSharedLibs(word.c_str())
Expand Down Expand Up @@ -258,7 +268,7 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification,
}

// skip trailing whitespace.
while (i < modEnd && isspace(text[i])) {
while (i < modEnd && IsSpace_(text[i])) {
ExtendRangeAndSetColor(input, i, kColorNone, Modification.fDisplay);
++i;
}
Expand All @@ -280,7 +290,7 @@ void ROOT::TextInputColorizer::ProcessCursorChange(size_t Cursor,
// if so, check for its closing one and color them green.

static const int numBrackets = 3;
static const char bTypes[numBrackets][3] = {"()", "{}", "[]"};
static const char32_t bTypes[numBrackets][3] = {U"()", U"{}", U"[]"};

if (input.empty()) return;

Expand All @@ -307,7 +317,7 @@ void ROOT::TextInputColorizer::ProcessCursorChange(size_t Cursor,
stack<size_t> locBrackets;
int foundParenIdx = -1;
int parenType = 0;
const std::string& text = input.GetText();
const std::u32string& text = input.GetChars();

if (Cursor < input.length()) {
// check against each bracket type
Expand Down
2 changes: 2 additions & 0 deletions core/textinput/src/textinput/Display.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ namespace textinput {

bool operator==(const Pos& O) const {
return fCol == O.fCol && fLine == O.fLine; }
bool operator<(const Pos& O) const {
return fLine != O.fLine ? fLine < O.fLine : fCol < O.fCol; }

size_t fCol;
size_t fLine;
Expand Down
92 changes: 57 additions & 35 deletions core/textinput/src/textinput/Editor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,52 @@

namespace textinput {

namespace {
// isalnum() and friends are only defined for values that fit in an
// unsigned char, so they cannot be asked about a character outside ASCII.
// Treat everything above ASCII as part of a word: a word is what the user
// steps over with Alt-F, and stopping inside "Amp猫re" would be surprising.
bool IsWordChar(char32_t c) {
if (c > 0x7F) return true;
return c == U'_' || isalnum(static_cast<int>(c));
}

// Case conversion, ASCII only. Mapping the rest correctly needs the full
// Unicode case tables, and getting it half right (missing the special
// cases where a character's upper case is two characters, say) would be
// worse than leaving those characters alone.
char32_t ToUpper(char32_t c) {
return c <= 0x7F ? static_cast<char32_t>(toupper(static_cast<int>(c))) : c;
}
char32_t ToLower(char32_t c) {
return c <= 0x7F ? static_cast<char32_t>(tolower(static_cast<int>(c))) : c;
}
}

// Functions to find first/last non alphanumeric ("word-boundaries")
size_t find_first_non_alnum(const std::string &str,
std::string::size_type index = 0) {
size_t find_first_non_alnum(const std::u32string &str,
std::u32string::size_type index = 0) {
bool atleast_one_alnum = false;
std::string::size_type len = str.length();
std::u32string::size_type len = str.length();
for(; index < len; ++index) {
const char c = str[index];
bool is_alpha = isalnum(c) || c == '_';
bool is_alpha = IsWordChar(str[index]);
if (is_alpha) atleast_one_alnum = true;
else if (atleast_one_alnum) return index;
}
return std::string::npos;
return std::u32string::npos;
}

size_t find_last_non_alnum(const std::string &str,
std::string::size_type index = std::string::npos) {
std::string::size_type len = str.length();
if (index == std::string::npos) index = len - 1;
size_t find_last_non_alnum(const std::u32string &str,
std::u32string::size_type index = std::u32string::npos) {
std::u32string::size_type len = str.length();
if (index == std::u32string::npos) index = len - 1;
bool atleast_one_alnum = false;
for(; index != std::string::npos; --index) {
const char c = str[index];
bool is_alpha = isalnum(c) || c == '_';
for(; index != std::u32string::npos; --index) {
bool is_alpha = IsWordChar(str[index]);
if (is_alpha) atleast_one_alnum = true;
else if (atleast_one_alnum) return index;
}
return std::string::npos;
return std::u32string::npos;
}

Editor::EProcessResult
Expand Down Expand Up @@ -103,7 +123,7 @@ namespace textinput {
Editor::SetHistSearchModePrompt(Range& RDisplay) {
assert(fMode == kHistFwdSearchMode || fMode == kHistRevSearchMode);
const std::string direction(fMode == kHistFwdSearchMode ? "fwd" : "bkw");
SetEditorPrompt(Text("[" + direction + "'" + fSearch + "'] "));
SetEditorPrompt(Text("[" + direction + "'" + UTF32ToUTF8(fSearch) + "'] "));
RDisplay.ExtendPromptUpdate(Range::kUpdateEditorPrompt);
}

Expand All @@ -114,6 +134,8 @@ namespace textinput {
std::ptrdiff_t NewHistEntry = -1;
if (fSearch.empty())
return true;
// History is kept as UTF-8; search in the same encoding.
const std::string SearchUTF8 = UTF32ToUTF8(fSearch);
std::ptrdiff_t startAt = fCurHistEntry;
if (startAt == -1) {
startAt = 0;
Expand All @@ -122,7 +144,7 @@ namespace textinput {
? -1 : static_cast<std::ptrdiff_t>(Hist->GetSize());
const std::ptrdiff_t step = (fMode == kHistFwdSearchMode ? -1 : 1);
for (std::ptrdiff_t i = startAt; i != stopAt; i += step) {
if (Hist->GetLine(i).find(fSearch) != std::string::npos) {
if (Hist->GetLine(i).find(SearchUTF8) != std::string::npos) {
NewHistEntry = i;
break;
}
Expand Down Expand Up @@ -168,7 +190,7 @@ namespace textinput {
}

Editor::EProcessResult
Editor::ProcessChar(char C, EditorRange& R) {
Editor::ProcessChar(char32_t C, EditorRange& R) {
if (C < 32) return kPRError;

if (fMode == kHistRevSearchMode || fMode == kHistFwdSearchMode) {
Expand All @@ -186,7 +208,7 @@ namespace textinput {

if (fOverwrite) {
if (Cursor < Line.length()) {
Line[Cursor] = C;
Line.SetChar(Cursor, C);
} else {
Line += C;
}
Expand Down Expand Up @@ -321,15 +343,15 @@ namespace textinput {
R.fDisplay.Extend(Range(Cursor, Range::End()));
return kPRSuccess;
case kCmdCutToEnd:
AddToPasteBuf(1, Line.GetText().c_str() + Cursor);
AddToPasteBuf(1, Line.substr(Cursor));
Line.erase(Cursor, Line.length() - Cursor);
R.fEdit.Extend(Range(Cursor));
R.fDisplay.Extend(Range(Cursor, Range::End()));
return kPRSuccess;
case kCmdCutNextWord:
{
size_t posWord = FindWordBoundary(1);
AddToPasteBuf(1, Line.GetText().substr(Cursor, posWord - Cursor));
AddToPasteBuf(1, Line.substr(Cursor, posWord - Cursor));
R.fEdit.Extend(Range(Cursor, posWord));
R.fDisplay.Extend(Range(Cursor, Range::End()));
Line.erase(Cursor, posWord - Cursor);
Expand All @@ -338,7 +360,7 @@ namespace textinput {
case kCmdCutPrevWord:
{
size_t posWord = FindWordBoundary(-1);
AddToPasteBuf(-1, Line.GetText().substr(posWord, Cursor - posWord));
AddToPasteBuf(-1, Line.substr(posWord, Cursor - posWord));
R.fEdit.Extend(Range(posWord, Cursor));
R.fDisplay.Extend(Range(posWord, Range::End()));
Line.erase(posWord, Cursor - posWord);
Expand All @@ -357,7 +379,7 @@ namespace textinput {
case kCmdCutToFront:
R.fEdit.Extend(Range(0, Cursor));
R.fDisplay.Extend(Range::AllText());
AddToPasteBuf(-1, Line.GetText().substr(0, Cursor));
AddToPasteBuf(-1, Line.substr(0, Cursor));
Line.erase(0, Cursor);
fContext->SetCursor(0);
return kPRSuccess;
Expand All @@ -377,16 +399,16 @@ namespace textinput {
size_t posSwap = Cursor < Line.length() ? Cursor : Line.length() - 1;
R.fEdit.Extend(Range(posSwap - 1, posSwap));
R.fDisplay.Extend(Range(posSwap - 1, Range::End()));
char tmp = Line.GetText()[posSwap];
Line[posSwap] = Line[posSwap - 1];
Line[posSwap - 1] = tmp;
char32_t tmp = Line[posSwap];
Line.SetChar(posSwap, Line[posSwap - 1]);
Line.SetChar(posSwap - 1, tmp);
ProcessMove(kMoveRight, R);
return kPRSuccess;
}
case kCmdToUpperMoveNextWord:
{
if (Cursor >= Line.length()) return kPRError;
Line[Cursor] = toupper(Line[Cursor]);
Line.SetChar(Cursor, ToUpper(Line[Cursor]));
R.fEdit.Extend(Range(Cursor));
R.fDisplay.Extend(Range(Cursor));
ProcessMove(kMoveNextWord, R);
Expand All @@ -398,11 +420,11 @@ namespace textinput {
size_t posWord = FindWordBoundary(1);
if (M == kCmdWordToUpper) {
for (size_t i = Cursor; i < posWord; ++i) {
Line[i] = toupper(Line[i]);
Line.SetChar(i, ToUpper(Line[i]));
}
} else {
for (size_t i = Cursor; i < posWord; ++i) {
Line[i] = tolower(Line[i]);
Line.SetChar(i, ToLower(Line[i]));
}
}
R.fEdit.Extend(Range(Cursor, posWord));
Expand Down Expand Up @@ -519,26 +541,26 @@ namespace textinput {
if (Direction < 0 && Cursor < 2) return 0;

size_t ret = Direction > 0 ?
find_first_non_alnum(Line.GetText(), Cursor + 1)
: find_last_non_alnum(Line.GetText(), Cursor - 2);
find_first_non_alnum(Line.GetChars(), Cursor + 1)
: find_last_non_alnum(Line.GetChars(), Cursor - 2);

if (ret == std::string::npos) {
if (ret == std::u32string::npos) {
if (Direction > 0) return Line.length();
else return 0;
}

if (Direction < 0)
ret += 1;

if (ret == std::string::npos) {
if (ret == std::u32string::npos) {
if (Direction > 0) return Line.length();
else return 0;
}
return ret;
}

void
Editor::AddToPasteBuf(int Dir, std::string const &T) {
Editor::AddToPasteBuf(int Dir, std::u32string const &T) {
if (fCutDirection == Dir) {
if (Dir < 0) {
fPasteBuf = T + fPasteBuf;
Expand All @@ -552,10 +574,10 @@ namespace textinput {
}

void
Editor::AddToPasteBuf(int Dir, char T) {
Editor::AddToPasteBuf(int Dir, char32_t T) {
if (fCutDirection == Dir) {
if (Dir < 0) {
fPasteBuf = std::string(1, T) + fPasteBuf;
fPasteBuf = std::u32string(1, T) + fPasteBuf;
} else {
fPasteBuf += T;
}
Expand Down
Loading
Loading