diff --git a/core/textinput/CMakeLists.txt b/core/textinput/CMakeLists.txt index a06f61d5a0b96..e1e8f487e070f 100644 --- a/core/textinput/CMakeLists.txt +++ b/core/textinput/CMakeLists.txt @@ -27,6 +27,7 @@ target_sources(Core PRIVATE src/textinput/TerminalDisplayWin.cpp src/textinput/TextInputContext.cpp src/textinput/TextInput.cpp + src/textinput/UTF8.cpp ) target_include_directories(Core diff --git a/core/textinput/src/Getline.cxx b/core/textinput/src/Getline.cxx index 75e102c9bf565..5ca39897fbfb0 100644 --- a/core/textinput/src/Getline.cxx +++ b/core/textinput/src/Getline.cxx @@ -48,7 +48,10 @@ namespace { EditorRange& r /*out*/, std::vector& displayCompletions /*out*/) override { strlcpy(fLineBuf, line.GetText().c_str(), fgLineBufSize); - int cursorInt = (int) cursor; + // TTabCom edits the UTF-8 buffer, so it speaks byte offsets; the + // editor's cursor and ranges count characters. Convert on the way in + // and back out. + int cursorInt = (int) line.GetByteOffset(cursor); std::stringstream sstr; size_t posFirstChange = gApplication->TabCompletionHook(fLineBuf, &cursorInt, sstr); if (posFirstChange == (size_t) -1) { @@ -73,12 +76,12 @@ namespace { r.fEdit.Extend(Range::AllText()); r.fDisplay.Extend(Range::AllText()); } else { - r.fEdit.Extend(Range(posFirstChange, Range::End())); - r.fDisplay.Extend(Range(posFirstChange, Range::End())); + size_t charFirstChange = line.GetCharIndex(posFirstChange); + r.fEdit.Extend(Range(charFirstChange, Range::End())); + r.fDisplay.Extend(Range(charFirstChange, Range::End())); } } - cursor = (size_t)cursorInt; - line.GetColors().resize(lenLineBuf); + cursor = line.GetCharIndex((size_t)cursorInt); return true; } private: diff --git a/core/textinput/src/Getline_color.cxx b/core/textinput/src/Getline_color.cxx index 0c6506cc9b47e..1c73171d88bd6 100644 --- a/core/textinput/src/Getline_color.cxx +++ b/core/textinput/src/Getline_color.cxx @@ -11,6 +11,7 @@ #include "Getline_color.h" +#include #include #include @@ -21,6 +22,7 @@ #include "TROOT.h" #include "textinput/Range.h" #include "textinput/Text.h" +#include "textinput/UTF8.h" using std::stack; using namespace textinput; @@ -109,8 +111,13 @@ namespace { return Color(); } // ColorFromName() - bool IsAlnum_(char c) { return c == '_' || isalnum(c); } - bool IsAlpha_(char c) { return c == '_' || isalpha(c); } + // The text is UTF-32; the classification functions from only + // accept values that fit in an unsigned char. Nothing above ASCII can be + // part of a C++ type name, so those characters are simply "not a word". + bool IsAlnum_(char32_t c) { return c == U'_' || (c <= 0x7F && isalnum((int)c)); } + bool IsAlpha_(char32_t c) { return c == U'_' || (c <= 0x7F && isalpha((int)c)); } + bool IsDigit_(char32_t c) { return c <= 0x7F && isdigit((int)c); } + bool IsSpace_(char32_t c) { return c <= 0x7F && isspace((int)c); } } // unnamed namespace @@ -185,7 +192,10 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification, Text& input) { // The text has changed; look for word that are types. - const std::string& text = input.GetText(); + // Index characters, not bytes: the ranges and the color vector are per + // character, so indexing input.GetText() would drift apart from them as + // soon as the line contains anything outside ASCII. + const std::u32string& text = input.GetChars(); size_t modStart = Modification.fEdit.fStart; size_t inputLength = input.length(); @@ -209,14 +219,14 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification, while (modStart && IsAlnum_(text[modStart])) --modStart; // Ignore spaces - while (modStart < modEnd && isspace(text[modStart])) + while (modStart < modEnd && IsSpace_(text[modStart])) ++modStart; - while (modEnd > modStart && isspace(text[modEnd])) - --modStart; + while (modEnd > modStart && IsSpace_(text[modEnd - 1])) + --modEnd; for (size_t i = modStart; i < modEnd;) { // i points to beginning of word here. - if (isdigit(text[i])) { + if (IsDigit_(text[i])) { // "12", or "12ull". Default color. ExtendRangeAndSetColor(input, i, 0, Modification.fDisplay); ++i; @@ -230,7 +240,7 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification, while (i + wordLen < modEnd && IsAlnum_(text[i + wordLen])) { ++wordLen; } - std::string word = text.substr(i, wordLen); + std::string word = UTF32ToUTF8(text.substr(i, wordLen)); char color = kColorNone; if (gClassTable->GetDict(word.c_str()) || gInterpreter->GetClassSharedLibs(word.c_str()) @@ -258,7 +268,7 @@ void ROOT::TextInputColorizer::ProcessTextChange(EditorRange& Modification, } // skip trailing whitespace. - while (i < modEnd && isspace(text[i])) { + while (i < modEnd && IsSpace_(text[i])) { ExtendRangeAndSetColor(input, i, kColorNone, Modification.fDisplay); ++i; } @@ -280,7 +290,7 @@ void ROOT::TextInputColorizer::ProcessCursorChange(size_t Cursor, // if so, check for its closing one and color them green. static const int numBrackets = 3; - static const char bTypes[numBrackets][3] = {"()", "{}", "[]"}; + static const char32_t bTypes[numBrackets][3] = {U"()", U"{}", U"[]"}; if (input.empty()) return; @@ -307,7 +317,7 @@ void ROOT::TextInputColorizer::ProcessCursorChange(size_t Cursor, stack locBrackets; int foundParenIdx = -1; int parenType = 0; - const std::string& text = input.GetText(); + const std::u32string& text = input.GetChars(); if (Cursor < input.length()) { // check against each bracket type diff --git a/core/textinput/src/textinput/Display.h b/core/textinput/src/textinput/Display.h index 0d3689bacf8f5..dfe200751dc4e 100644 --- a/core/textinput/src/textinput/Display.h +++ b/core/textinput/src/textinput/Display.h @@ -35,6 +35,8 @@ namespace textinput { bool operator==(const Pos& O) const { return fCol == O.fCol && fLine == O.fLine; } + bool operator<(const Pos& O) const { + return fLine != O.fLine ? fLine < O.fLine : fCol < O.fCol; } size_t fCol; size_t fLine; diff --git a/core/textinput/src/textinput/Editor.cpp b/core/textinput/src/textinput/Editor.cpp index 505fb5f2966b6..9c7c62f83916a 100644 --- a/core/textinput/src/textinput/Editor.cpp +++ b/core/textinput/src/textinput/Editor.cpp @@ -29,32 +29,52 @@ namespace textinput { + namespace { + // isalnum() and friends are only defined for values that fit in an + // unsigned char, so they cannot be asked about a character outside ASCII. + // Treat everything above ASCII as part of a word: a word is what the user + // steps over with Alt-F, and stopping inside "Ampère" would be surprising. + bool IsWordChar(char32_t c) { + if (c > 0x7F) return true; + return c == U'_' || isalnum(static_cast(c)); + } + + // Case conversion, ASCII only. Mapping the rest correctly needs the full + // Unicode case tables, and getting it half right (missing the special + // cases where a character's upper case is two characters, say) would be + // worse than leaving those characters alone. + char32_t ToUpper(char32_t c) { + return c <= 0x7F ? static_cast(toupper(static_cast(c))) : c; + } + char32_t ToLower(char32_t c) { + return c <= 0x7F ? static_cast(tolower(static_cast(c))) : c; + } + } + // Functions to find first/last non alphanumeric ("word-boundaries") - size_t find_first_non_alnum(const std::string &str, - std::string::size_type index = 0) { + size_t find_first_non_alnum(const std::u32string &str, + std::u32string::size_type index = 0) { bool atleast_one_alnum = false; - std::string::size_type len = str.length(); + std::u32string::size_type len = str.length(); for(; index < len; ++index) { - const char c = str[index]; - bool is_alpha = isalnum(c) || c == '_'; + bool is_alpha = IsWordChar(str[index]); if (is_alpha) atleast_one_alnum = true; else if (atleast_one_alnum) return index; } - return std::string::npos; + return std::u32string::npos; } - size_t find_last_non_alnum(const std::string &str, - std::string::size_type index = std::string::npos) { - std::string::size_type len = str.length(); - if (index == std::string::npos) index = len - 1; + size_t find_last_non_alnum(const std::u32string &str, + std::u32string::size_type index = std::u32string::npos) { + std::u32string::size_type len = str.length(); + if (index == std::u32string::npos) index = len - 1; bool atleast_one_alnum = false; - for(; index != std::string::npos; --index) { - const char c = str[index]; - bool is_alpha = isalnum(c) || c == '_'; + for(; index != std::u32string::npos; --index) { + bool is_alpha = IsWordChar(str[index]); if (is_alpha) atleast_one_alnum = true; else if (atleast_one_alnum) return index; } - return std::string::npos; + return std::u32string::npos; } Editor::EProcessResult @@ -103,7 +123,7 @@ namespace textinput { Editor::SetHistSearchModePrompt(Range& RDisplay) { assert(fMode == kHistFwdSearchMode || fMode == kHistRevSearchMode); const std::string direction(fMode == kHistFwdSearchMode ? "fwd" : "bkw"); - SetEditorPrompt(Text("[" + direction + "'" + fSearch + "'] ")); + SetEditorPrompt(Text("[" + direction + "'" + UTF32ToUTF8(fSearch) + "'] ")); RDisplay.ExtendPromptUpdate(Range::kUpdateEditorPrompt); } @@ -114,6 +134,8 @@ namespace textinput { std::ptrdiff_t NewHistEntry = -1; if (fSearch.empty()) return true; + // History is kept as UTF-8; search in the same encoding. + const std::string SearchUTF8 = UTF32ToUTF8(fSearch); std::ptrdiff_t startAt = fCurHistEntry; if (startAt == -1) { startAt = 0; @@ -122,7 +144,7 @@ namespace textinput { ? -1 : static_cast(Hist->GetSize()); const std::ptrdiff_t step = (fMode == kHistFwdSearchMode ? -1 : 1); for (std::ptrdiff_t i = startAt; i != stopAt; i += step) { - if (Hist->GetLine(i).find(fSearch) != std::string::npos) { + if (Hist->GetLine(i).find(SearchUTF8) != std::string::npos) { NewHistEntry = i; break; } @@ -168,7 +190,7 @@ namespace textinput { } Editor::EProcessResult - Editor::ProcessChar(char C, EditorRange& R) { + Editor::ProcessChar(char32_t C, EditorRange& R) { if (C < 32) return kPRError; if (fMode == kHistRevSearchMode || fMode == kHistFwdSearchMode) { @@ -186,7 +208,7 @@ namespace textinput { if (fOverwrite) { if (Cursor < Line.length()) { - Line[Cursor] = C; + Line.SetChar(Cursor, C); } else { Line += C; } @@ -321,7 +343,7 @@ namespace textinput { R.fDisplay.Extend(Range(Cursor, Range::End())); return kPRSuccess; case kCmdCutToEnd: - AddToPasteBuf(1, Line.GetText().c_str() + Cursor); + AddToPasteBuf(1, Line.substr(Cursor)); Line.erase(Cursor, Line.length() - Cursor); R.fEdit.Extend(Range(Cursor)); R.fDisplay.Extend(Range(Cursor, Range::End())); @@ -329,7 +351,7 @@ namespace textinput { case kCmdCutNextWord: { size_t posWord = FindWordBoundary(1); - AddToPasteBuf(1, Line.GetText().substr(Cursor, posWord - Cursor)); + AddToPasteBuf(1, Line.substr(Cursor, posWord - Cursor)); R.fEdit.Extend(Range(Cursor, posWord)); R.fDisplay.Extend(Range(Cursor, Range::End())); Line.erase(Cursor, posWord - Cursor); @@ -338,7 +360,7 @@ namespace textinput { case kCmdCutPrevWord: { size_t posWord = FindWordBoundary(-1); - AddToPasteBuf(-1, Line.GetText().substr(posWord, Cursor - posWord)); + AddToPasteBuf(-1, Line.substr(posWord, Cursor - posWord)); R.fEdit.Extend(Range(posWord, Cursor)); R.fDisplay.Extend(Range(posWord, Range::End())); Line.erase(posWord, Cursor - posWord); @@ -357,7 +379,7 @@ namespace textinput { case kCmdCutToFront: R.fEdit.Extend(Range(0, Cursor)); R.fDisplay.Extend(Range::AllText()); - AddToPasteBuf(-1, Line.GetText().substr(0, Cursor)); + AddToPasteBuf(-1, Line.substr(0, Cursor)); Line.erase(0, Cursor); fContext->SetCursor(0); return kPRSuccess; @@ -377,16 +399,16 @@ namespace textinput { size_t posSwap = Cursor < Line.length() ? Cursor : Line.length() - 1; R.fEdit.Extend(Range(posSwap - 1, posSwap)); R.fDisplay.Extend(Range(posSwap - 1, Range::End())); - char tmp = Line.GetText()[posSwap]; - Line[posSwap] = Line[posSwap - 1]; - Line[posSwap - 1] = tmp; + char32_t tmp = Line[posSwap]; + Line.SetChar(posSwap, Line[posSwap - 1]); + Line.SetChar(posSwap - 1, tmp); ProcessMove(kMoveRight, R); return kPRSuccess; } case kCmdToUpperMoveNextWord: { if (Cursor >= Line.length()) return kPRError; - Line[Cursor] = toupper(Line[Cursor]); + Line.SetChar(Cursor, ToUpper(Line[Cursor])); R.fEdit.Extend(Range(Cursor)); R.fDisplay.Extend(Range(Cursor)); ProcessMove(kMoveNextWord, R); @@ -398,11 +420,11 @@ namespace textinput { size_t posWord = FindWordBoundary(1); if (M == kCmdWordToUpper) { for (size_t i = Cursor; i < posWord; ++i) { - Line[i] = toupper(Line[i]); + Line.SetChar(i, ToUpper(Line[i])); } } else { for (size_t i = Cursor; i < posWord; ++i) { - Line[i] = tolower(Line[i]); + Line.SetChar(i, ToLower(Line[i])); } } R.fEdit.Extend(Range(Cursor, posWord)); @@ -519,10 +541,10 @@ namespace textinput { if (Direction < 0 && Cursor < 2) return 0; size_t ret = Direction > 0 ? - find_first_non_alnum(Line.GetText(), Cursor + 1) - : find_last_non_alnum(Line.GetText(), Cursor - 2); + find_first_non_alnum(Line.GetChars(), Cursor + 1) + : find_last_non_alnum(Line.GetChars(), Cursor - 2); - if (ret == std::string::npos) { + if (ret == std::u32string::npos) { if (Direction > 0) return Line.length(); else return 0; } @@ -530,7 +552,7 @@ namespace textinput { if (Direction < 0) ret += 1; - if (ret == std::string::npos) { + if (ret == std::u32string::npos) { if (Direction > 0) return Line.length(); else return 0; } @@ -538,7 +560,7 @@ namespace textinput { } void - Editor::AddToPasteBuf(int Dir, std::string const &T) { + Editor::AddToPasteBuf(int Dir, std::u32string const &T) { if (fCutDirection == Dir) { if (Dir < 0) { fPasteBuf = T + fPasteBuf; @@ -552,10 +574,10 @@ namespace textinput { } void - Editor::AddToPasteBuf(int Dir, char T) { + Editor::AddToPasteBuf(int Dir, char32_t T) { if (fCutDirection == Dir) { if (Dir < 0) { - fPasteBuf = std::string(1, T) + fPasteBuf; + fPasteBuf = std::u32string(1, T) + fPasteBuf; } else { fPasteBuf += T; } diff --git a/core/textinput/src/textinput/Editor.h b/core/textinput/src/textinput/Editor.h index c5eff62db8c1a..ce1e1fbd91736 100644 --- a/core/textinput/src/textinput/Editor.h +++ b/core/textinput/src/textinput/Editor.h @@ -103,13 +103,13 @@ namespace textinput { public: Command(ECommandID C): fKind(kCKCommand), fCmd(C) {} Command(EMoveID M): fKind(kCKMove), fMove(M) {} - Command(char C, ECommandKind k = kCKChar): fKind(k), fChar(C) {} + Command(char32_t C, ECommandKind k = kCKChar): fKind(k), fChar(C) {} ECommandKind GetKind() const { return fKind; } ECommandID GetCommandID() const { return fCmd;} EMoveID GetMoveID() const { return fMove;} - char GetChar() const { return fChar;} + char32_t GetChar() const { return fChar;} bool isCtrlD() const { return fKind == kCKControl && (fChar == 'd'-0x60); } @@ -118,7 +118,7 @@ namespace textinput { union { ECommandID fCmd; // editor command value EMoveID fMove; // move value - char fChar; // character input value + char32_t fChar; // character input value }; }; @@ -136,14 +136,14 @@ namespace textinput { void CancelAndRevertSpecialInputMode(EditorRange& R); private: - EProcessResult ProcessChar(char C, EditorRange& R); + EProcessResult ProcessChar(char32_t C, EditorRange& R); EProcessResult ProcessMove(EMoveID M, EditorRange& R); EProcessResult ProcessCommand(ECommandID M, EditorRange& R); size_t FindWordBoundary(int Direction); void PushUndo(); - void AddToPasteBuf(int Dir, const std::string& T); - void AddToPasteBuf(int Dir, char T); + void AddToPasteBuf(int Dir, const std::u32string& T); + void AddToPasteBuf(int Dir, char32_t T); void ClearPasteBuf() { fCutDirection = 0; } void SetHistSearchModePrompt(Range& RDisplay); bool UpdateHistSearch(EditorRange& R); @@ -159,9 +159,9 @@ namespace textinput { TextInputContext* fContext; // Context object Text fEditorPrompt; // for special modes, e.g. reverse search - std::string fLineNotInHist; // current input line, not pushed to hist yet - std::string fPasteBuf; // cut strings that can be pasted - std::string fSearch; // for forward / backward hist search + std::string fLineNotInHist; // current input line (UTF-8), not pushed to hist yet + std::u32string fPasteBuf; // cut strings that can be pasted + std::u32string fSearch; // for forward / backward hist search size_t fCurHistEntry; // the current line stems from a hist entry, -1 if not size_t fReplayHistEntry; // set next line to this hist entry, kCmdHistReplay EEditMode fMode; // current input mode diff --git a/core/textinput/src/textinput/InputData.h b/core/textinput/src/textinput/InputData.h index 5cfe815d10cf2..2cbedb388031b 100644 --- a/core/textinput/src/textinput/InputData.h +++ b/core/textinput/src/textinput/InputData.h @@ -69,21 +69,23 @@ namespace textinput { }; InputData(): fExt(kEIUninitialized), fMod(0) {} - InputData(int ch, char mod = 0): fRaw(ch), fMod(mod | kIsRaw) {} + InputData(char32_t ch, char mod = 0): fRaw(ch), fMod(mod | kIsRaw) {} bool IsRaw() const { return (fMod & kIsRaw) != 0; } - int GetRaw() const { return fRaw; } + char32_t GetRaw() const { return fRaw; } EExtendedInput GetExtendedInput() const { return fExt; } unsigned char GetModifier() const { return fMod & ~kIsRaw; } - void SetRaw(char R) { fRaw = R; fMod |= kIsRaw; } + void SetRaw(char32_t R) { fRaw = R; fMod |= kIsRaw; } void SetExtended(EExtendedInput E) { fExt = E; fMod &= ~kIsRaw; } void SetModifier(char M) { fMod = M | (fMod & kIsRaw); } private: union { - char fRaw; // raw input character, if kIsRaw & fMod + // A whole character, not a byte: the readers assemble multi-byte input + // (UTF-8 on Unix, UTF-16 surrogate pairs on Windows) before getting here. + char32_t fRaw; // raw input character, if kIsRaw & fMod EExtendedInput fExt; // non-character input }; unsigned char fMod; // Modifiers, also stores union descriminator (kIsRaw) diff --git a/core/textinput/src/textinput/KeyBinding.cpp b/core/textinput/src/textinput/KeyBinding.cpp index 4871bd3a845b1..088f599ea34e4 100644 --- a/core/textinput/src/textinput/KeyBinding.cpp +++ b/core/textinput/src/textinput/KeyBinding.cpp @@ -42,7 +42,7 @@ namespace textinput { } Editor::Command - KeyBinding::ToCommandCtrl(char In, + KeyBinding::ToCommandCtrl(char32_t In, bool HadEscPending) { // Control was pressed and In was hit. Convert to command. typedef Editor::Command C; @@ -96,10 +96,12 @@ namespace textinput { } Editor::Command - KeyBinding::ToCommandEsc(char In) { + KeyBinding::ToCommandEsc(char32_t In) { // ESC was entered, followed by In. Convert to command. + // The Esc-prefixed bindings are all ASCII; anything else falls through to + // the error case below. typedef Editor::Command C; - switch (toupper(In)) { + switch (In <= 0x7F ? toupper(static_cast(In)) : static_cast(In)) { case 'B': return C(Editor::kMovePrevWord); case 'C': return C(Editor::kCmdToUpperMoveNextWord); case 'D': return C(Editor::kCmdCutNextWord); diff --git a/core/textinput/src/textinput/KeyBinding.h b/core/textinput/src/textinput/KeyBinding.h index 916177eecab2f..9e4c230628b27 100644 --- a/core/textinput/src/textinput/KeyBinding.h +++ b/core/textinput/src/textinput/KeyBinding.h @@ -39,8 +39,8 @@ namespace textinput { bool IsEscPending() const { return fEscPending; } private: - Editor::Command ToCommandCtrl(char In, bool HadEscPending); - Editor::Command ToCommandEsc(char In); + Editor::Command ToCommandCtrl(char32_t In, bool HadEscPending); + Editor::Command ToCommandEsc(char32_t In); Editor::Command ToCommandExtended(InputData::EExtendedInput EI, unsigned char modifier, bool HadEscPending); diff --git a/core/textinput/src/textinput/StreamReaderUnix.cpp b/core/textinput/src/textinput/StreamReaderUnix.cpp index 05c53d6655680..7d082d05c7624 100644 --- a/core/textinput/src/textinput/StreamReaderUnix.cpp +++ b/core/textinput/src/textinput/StreamReaderUnix.cpp @@ -30,12 +30,14 @@ #include #include #include +#include #include #include "textinput/InputData.h" #include "textinput/KeyBinding.h" #include "textinput/TerminalConfigUnix.h" #include "textinput/TextInputContext.h" +#include "textinput/UTF8.h" namespace { using namespace textinput; @@ -43,23 +45,23 @@ namespace { class Rewind { public: - Rewind(std::queue& rab, InputData::EExtendedInput& ret): + Rewind(std::deque& rab, InputData::EExtendedInput& ret): RAB(rab), Ret(ret) {} ~Rewind() { if (Ret != InputData::kEIUninitialized) return; // RAB.push(0x1b); already handled by ProcessCSI returning false. while (!Q.empty()) { - RAB.push(Q.front()); + RAB.push_back(Q.front()); Q.pop(); } } - void push(char C) { Q.push(C); } + void push(unsigned char C) { Q.push(C); } private: - std::queue Q; - std::queue& RAB; + std::queue Q; + std::deque& RAB; InputData::EExtendedInput& Ret; }; @@ -250,9 +252,15 @@ namespace textinput { mod = EKM->getMod(); EKM = nullptr; } else { - char c1 = ReadRawCharacter(); - rwd.push(c1); - EKM = EKM->find(c1); + int c1 = ReadRawByte(); + if (c1 == -1) { + EKM = nullptr; + break; + } + rwd.push(static_cast(c1)); + // The escape sequences are ASCII; a byte above it cannot match and + // gets pushed back by Rewind. + EKM = c1 < 0x80 ? EKM->find(static_cast(c1)) : nullptr; } } in.SetExtended(ret); @@ -261,15 +269,49 @@ namespace textinput { } //////////////////////////////////////////////////////////////////////////////// - /// Read one char from stdin. Converts the read char to InputData + /// Read the continuation bytes of the UTF-8 sequence started by Lead and + /// store the resulting character in in. + /// + /// A byte that is not a continuation byte is pushed back: it may well be the + /// start of the next character, or an ESC introducing a key sequence, and + /// swallowing it would turn one mistyped character into a lost keypress. + /// + /// \param[in] Lead the lead byte, already read + /// \param[in] in input data to be filled out + void + StreamReaderUnix::ReadUTF8Rest(unsigned char Lead, InputData& in) { + UTF8Decoder Dec; + char32_t Ch = 0; + bool Reprocess = false; + UTF8Decoder::EResult Res = Dec.Push(Lead, Ch, Reprocess); + while (Res == UTF8Decoder::kNeedMore) { + int b = ReadRawByte(); + if (b == -1) { // stream ended mid-character + Ch = kInvalidChar; + break; + } + Res = Dec.Push(static_cast(b), Ch, Reprocess); + if (Res == UTF8Decoder::kInvalid && Reprocess) { + fReadAheadBuffer.push_front(static_cast(b)); + break; + } + } + in.SetRaw(Ch); + } + + //////////////////////////////////////////////////////////////////////////////// + /// Read one character from stdin. Converts the read character to InputData. + /// + /// One character can be several bytes: a UTF-8 sequence is assembled here, so + /// that everything above sees whole characters and nRead counts characters. /// /// \param[in] nRead number of already read characters. Increment after reading /// \param[in] in input char / data to be filled out bool StreamReaderUnix::ReadInput(size_t& nRead, InputData& in) { - int c = ReadRawCharacter(); + int c = ReadRawByte(); in.SetModifier(InputData::kModNone); - if (c == -1) { // non-character value, EOF negative + if (c == -1) { // EOF in.SetExtended(InputData::kEIEOF); } else if (c == 0x1b) { // ESC // Only try to process CSI if Esc does not have a meaning by itself. @@ -278,9 +320,11 @@ namespace textinput { || !ProcessCSI(in)) { in.SetExtended(InputData::kEIEsc); } + } else if (c >= 0x80) { // start of a multi-byte UTF-8 sequence + ReadUTF8Rest(static_cast(c), in); } else if (isprint(c)) { // c >= 0x20(32) && c < 0x7f(127) in.SetRaw(c); - } else if (c < 32 || c == (char)127 /* ^?, DEL on MacOS */) { // non-printable + } else if (c < 32 || c == 127 /* ^?, DEL on MacOS */) { // non-printable if (c == 13) { // 0x0d CR (INLCR - NL converted to CR) in.SetExtended(InputData::kEIEnter); } else { // mark CTRL pressed if other non-print char @@ -296,13 +340,13 @@ namespace textinput { } //////////////////////////////////////////////////////////////////////////////// - /// Read one character from stdin. Block if not available. + /// Read one byte from stdin. Block if not available. Returns -1 on EOF. int - StreamReaderUnix::ReadRawCharacter() { - char buf; + StreamReaderUnix::ReadRawByte() { + unsigned char buf; if (!fReadAheadBuffer.empty()) { buf = fReadAheadBuffer.front(); - fReadAheadBuffer.pop(); + fReadAheadBuffer.pop_front(); } else { ssize_t ret = read(fileno(stdin), &buf, 1); #ifdef __APPLE__ diff --git a/core/textinput/src/textinput/StreamReaderUnix.h b/core/textinput/src/textinput/StreamReaderUnix.h index ff0afddf0d464..f24e6d7e05cb4 100644 --- a/core/textinput/src/textinput/StreamReaderUnix.h +++ b/core/textinput/src/textinput/StreamReaderUnix.h @@ -17,8 +17,9 @@ #define TEXTINPUT_STREAMREADERUNIX_H #include "textinput/StreamReader.h" +#include "textinput/UTF8.h" #include -#include +#include namespace textinput { class InputData; @@ -38,12 +39,17 @@ namespace textinput { bool IsFromTTY() override { return fIsTTY; } private: - int ReadRawCharacter(); + // Read one byte; -1 on EOF. Bytes are returned unsigned (0..255) so that + // a 0xFF byte of a UTF-8 sequence cannot be mistaken for EOF. + int ReadRawByte(); bool ProcessCSI(InputData& in); + // Read the continuation bytes of a UTF-8 sequence started by Lead and + // store the character in in. + void ReadUTF8Rest(unsigned char Lead, InputData& in); bool fHaveInputFocus; // whether we configured the tty bool fIsTTY; // whether input FD is a tty - std::queue fReadAheadBuffer; // input chars we read too much (CSI) + std::deque fReadAheadBuffer; // input bytes we read too much (CSI) }; } diff --git a/core/textinput/src/textinput/StreamReaderWin.cpp b/core/textinput/src/textinput/StreamReaderWin.cpp index fd6d6d5a7477d..9df974b86e3af 100644 --- a/core/textinput/src/textinput/StreamReaderWin.cpp +++ b/core/textinput/src/textinput/StreamReaderWin.cpp @@ -41,9 +41,17 @@ #endif // End MSVC 7.1 quirks +// winnls.h only defines these for WINVER >= 0x0600. +#ifndef IS_HIGH_SURROGATE +# define IS_HIGH_SURROGATE(wch) (((wch) >= 0xD800) && ((wch) <= 0xDBFF)) +#endif +#ifndef IS_LOW_SURROGATE +# define IS_LOW_SURROGATE(wch) (((wch) >= 0xDC00) && ((wch) <= 0xDFFF)) +#endif + namespace textinput { StreamReaderWin::StreamReaderWin(): fHaveInputFocus(false), fIsConsole(true), - fOldMode(0), fMyMode(0) { + fOldMode(0), fMyMode(0), fPendingSurrogate(0) { fIn = ::GetStdHandle(STD_INPUT_HANDLE); bool fIsConsole = ::GetConsoleMode(fIn, &fOldMode) != 0; if (fIsConsole) { @@ -99,10 +107,12 @@ namespace textinput { StreamReaderWin::ReadInput(size_t& nRead, InputData& in) { DWORD NRead = 0; in.SetModifier(InputData::kModNone); - char C; + char32_t C = 0; if (fIsConsole) { INPUT_RECORD buf; - if (!::ReadConsoleInput(fIn, &buf, 1, &NRead)) { + // Read the wide variant: uChar.AsciiChar loses everything the console's + // code page cannot represent, which is most of Unicode. + if (!::ReadConsoleInputW(fIn, &buf, 1, &NRead)) { HandleError("reading console input"); return false; } @@ -113,6 +123,7 @@ namespace textinput { if (!buf.Event.KeyEvent.bKeyDown) return false; WORD Key = buf.Event.KeyEvent.wVirtualKeyCode; + const wchar_t Unicode = buf.Event.KeyEvent.uChar.UnicodeChar; if (buf.Event.KeyEvent.dwControlKeyState & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED)) { if (buf.Event.KeyEvent.dwControlKeyState @@ -128,7 +139,8 @@ namespace textinput { || (Key >= VK_NUMPAD0 && Key <= VK_DIVIDE) || (Key >= VK_OEM_1 && Key <= VK_OEM_102) || Key == VK_SPACE) { - C = buf.Event.KeyEvent.uChar.AsciiChar; + // Half a surrogate pair is not yet a character; wait for the rest. + if (!DecodeUTF16(Unicode, C)) return false; if (buf.Event.KeyEvent.dwControlKeyState & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED)) { // C is already 1.. @@ -161,7 +173,19 @@ namespace textinput { case VK_F10: in.SetExtended(InputData::kEIF10); break; case VK_F11: in.SetExtended(InputData::kEIF11); break; case VK_F12: in.SetExtended(InputData::kEIF12); break; - default: in.SetExtended(InputData::kEIUninitialized); return false; + default: + // No virtual key code of its own, but it still produced a + // character: IME composition, a dead key resolving, or an AltGr + // combination on a non-US layout. Those are exactly the keys + // that type the non-ASCII characters we are here for. + if (Unicode >= 0x20 || IS_HIGH_SURROGATE(Unicode) + || IS_LOW_SURROGATE(Unicode)) { + if (!DecodeUTF16(Unicode, C)) return false; + HandleKeyEvent(C, in); + ++nRead; + return true; + } + in.SetExtended(InputData::kEIUninitialized); return false; } return true; } @@ -176,9 +200,35 @@ namespace textinput { return false; } } else { + if (!ReadPipeChar(C)) { + in.SetExtended(InputData::kEIEOF); + return true; + } + } + HandleKeyEvent(C, in); + ++nRead; + return true; + } + + //////////////////////////////////////////////////////////////////////////////// + /// Read one character from redirected input, which is a byte stream and is + /// taken to be UTF-8 - the same encoding the rest of ROOT uses for text. + /// + /// \param[out] Out the character read + /// \return false at end of input + bool + StreamReaderWin::ReadPipeChar(char32_t& Out) { + UTF8Decoder Dec; + bool Reprocess = false; + UTF8Decoder::EResult Res = UTF8Decoder::kNeedMore; + bool AnyByteRead = false; + + while (Res == UTF8Decoder::kNeedMore) { + unsigned char Byte = 0; + DWORD NRead = 0; // Testing for the End of a File // https://msdn.microsoft.com/en-us/library/windows/desktop/aa365690(v=vs.85).aspx - if (!::ReadFile(fIn, &C, 1, &NRead, NULL)) { + if (!::ReadFile(fIn, &Byte, 1, &NRead, NULL)) { if (NRead != 0) { switch (::GetLastError()) { default: @@ -192,12 +242,49 @@ namespace textinput { } } if (NRead == 0) { - in.SetExtended(InputData::kEIEOF); + // End of input. If it arrived in the middle of a character, report + // the truncated character rather than losing it silently. + if (!AnyByteRead) return false; + Out = kInvalidChar; return true; } + AnyByteRead = true; + Res = Dec.Push(Byte, Out, Reprocess); } - HandleKeyEvent(C, in); - ++nRead; + return true; + } + + //////////////////////////////////////////////////////////////////////////////// + /// Combine the UTF-16 code units the console hands us into a code point. + /// + /// wchar_t is 16 bits on Windows, so anything above the basic multilingual + /// plane - emoji, most notably - arrives as two events that have to be put + /// back together. + /// + /// \param[in] U the code unit just read + /// \param[out] Out the character, when one is complete + /// \return false if this was a high surrogate and the low half is still to come + bool + StreamReaderWin::DecodeUTF16(wchar_t U, char32_t& Out) { + if (fPendingSurrogate) { + const wchar_t High = fPendingSurrogate; + fPendingSurrogate = 0; + if (IS_LOW_SURROGATE(U)) { + Out = 0x10000 + ((static_cast(High) - 0xD800) << 10) + + (static_cast(U) - 0xDC00); + return true; + } + // The high surrogate was never completed; drop it and carry on with U. + } + if (IS_HIGH_SURROGATE(U)) { + fPendingSurrogate = U; + return false; + } + if (IS_LOW_SURROGATE(U)) { // unpaired + Out = kInvalidChar; + return true; + } + Out = U; return true; } @@ -214,14 +301,14 @@ namespace textinput { } void - StreamReaderWin::HandleKeyEvent(unsigned char C, InputData& in) { - if (isprint(C)) { + StreamReaderWin::HandleKeyEvent(char32_t C, InputData& in) { + if (C < 0x80 && isprint(static_cast(C))) { in.SetRaw(C); } else if (C < 32) { in.SetRaw(C); in.SetModifier(InputData::kModCtrl); } else { - // woohoo, what's that?! + // Everything else, including every character outside ASCII. in.SetRaw(C); } } diff --git a/core/textinput/src/textinput/StreamReaderWin.h b/core/textinput/src/textinput/StreamReaderWin.h index 074a52d799c41..5a0b4d8784622 100644 --- a/core/textinput/src/textinput/StreamReaderWin.h +++ b/core/textinput/src/textinput/StreamReaderWin.h @@ -16,6 +16,7 @@ #define TEXTINPUT_STREAMREADERWIN_H #include "textinput/StreamReader.h" +#include "textinput/UTF8.h" #include namespace textinput { @@ -35,13 +36,21 @@ namespace textinput { private: void HandleError(const char* Where) const; - void HandleKeyEvent(unsigned char C, InputData& in); + void HandleKeyEvent(char32_t C, InputData& in); + // Turn a UTF-16 code unit from the console into a code point. Returns + // false while waiting for the second half of a surrogate pair, i.e. when + // there is no character to report yet. + bool DecodeUTF16(wchar_t U, char32_t& Out); + // Read one character's worth of UTF-8 from a redirected (non-console) + // input. Returns false on EOF. + bool ReadPipeChar(char32_t& Out); bool fHaveInputFocus; // whether the console is configured bool fIsConsole; // whether the input is a console or file HANDLE fIn; // input handle DWORD fOldMode; // configuration before grabbing input device DWORD fMyMode; // configuration while active + wchar_t fPendingSurrogate; // high surrogate awaiting its low half }; } diff --git a/core/textinput/src/textinput/TerminalDisplay.cpp b/core/textinput/src/textinput/TerminalDisplay.cpp index 45505b765fa57..0f9269b6f9cbf 100644 --- a/core/textinput/src/textinput/TerminalDisplay.cpp +++ b/core/textinput/src/textinput/TerminalDisplay.cpp @@ -46,11 +46,41 @@ namespace textinput { TerminalDisplay::NotifyTextChange(Range r) { if (!IsTTY()) return; Attach(); + ExtendRangeForCombiningChars(r); WriteWrapped(r.fPromptUpdate, GetContext()->GetTextInput()->IsInputMasked(), r.fStart, r.fLength); Move(GetCursor()); } + //////////////////////////////////////////////////////////////////////////////// + /// Move the start of r back to the character whose terminal cell the change + /// really affects. + /// + /// A zero-width character - a combining accent, say - is not drawn into a + /// cell of its own but into the cell of the character it follows. So + /// redrawing from the mark would leave the base character behind, and + /// redrawing from just past a mark that was deleted would leave the mark + /// itself on the screen. Both are fixed by starting one character earlier + /// and then skipping back over any further marks. + /// + /// \param[in,out] r range to redraw, in characters of the input line + void + TerminalDisplay::ExtendRangeForCombiningChars(Range& r) const { + if (r.fStart == 0) return; + const Text& Line = GetContext()->GetLine(); + if (r.fStart > Line.length()) return; + + size_t Start = r.fStart; + do { + --Start; + } while (Start > 0 && Line.GetWidthOfChar(Start) == 0); + + if (r.fLength != Range::End()) { + r.fLength += r.fStart - Start; + } + r.fStart = Start; + } + //////////////////////////////////////////////////////////////////////////////// /// Notify the display that the cursor has been changed. Move to the cursor. void @@ -68,7 +98,7 @@ namespace textinput { if (IsTTY()) { WriteRawString("\n", 1); } - fWriteLen = 0; + fWriteEnd = Pos(); fWritePos = Pos(); } @@ -95,7 +125,9 @@ namespace textinput { WriteRawString("\n", 1); for (size_t i = 0, n = Options.size(); i < n; ++i) { Text t(Options[i], infoColIdx); - WriteWrappedTextPart(t, 0, 0, (size_t) -1); + // Each option starts on a line of its own. + fWritePos.fCol = 0; + WriteWrappedTextPart(t, 0, (size_t) -1); WriteRawString("\n", 1); } // Reset position @@ -109,7 +141,7 @@ namespace textinput { void TerminalDisplay::Detach() { fWritePos = Pos(); - fWriteLen = 0; + fWriteEnd = Pos(); if (GetContext()->GetColorizer()) { Color DefaultColor; GetContext()->GetColorizer()->GetColor(0, DefaultColor); @@ -119,16 +151,61 @@ namespace textinput { } } + //////////////////////////////////////////////////////////////////////////////// + /// Width in terminal columns of character idx of the concatenation of the + /// prompt, the editor prompt and the input line. + size_t + TerminalDisplay::WidthOfDisplayChar(size_t idx) const { + const Text& Prompt = GetContext()->GetPrompt(); + if (idx < Prompt.length()) return Prompt.GetWidthOfChar(idx); + idx -= Prompt.length(); + + const Text& EditPrompt = GetContext()->GetEditor()->GetEditorPrompt(); + if (idx < EditPrompt.length()) return EditPrompt.GetWidthOfChar(idx); + idx -= EditPrompt.length(); + + const Text& Line = GetContext()->GetLine(); + if (idx < Line.length()) { + // Masked input is echoed as '*', whatever was actually typed. + if (GetContext()->GetTextInput()->IsInputMasked()) return 1; + return Line.GetWidthOfChar(idx); + } + return 1; // past the end of the text: the cursor itself + } + + //////////////////////////////////////////////////////////////////////////////// + /// Where the idx'th character of the displayed text ends up on the terminal. + Display::Pos + TerminalDisplay::IndexToPos(size_t idx) const { + Pos P; + for (size_t i = 0; i < idx; ++i) { + AdvancePos(P, WidthOfDisplayChar(i)); + } + return P; + } + + //////////////////////////////////////////////////////////////////////////////// + /// Write n spaces. + void + TerminalDisplay::WriteBlanks(size_t n) { + if (!n || !IsTTY()) return; + const std::string Blanks(n, ' '); + WriteRawString(Blanks.c_str(), Blanks.length()); + } + //////////////////////////////////////////////////////////////////////////////// /// Write out wrapped text to the display. Used in WriteWrapped and DisplayInfo /// + /// Writing starts at fWritePos, which the caller must have moved to the right + /// place; the position is advanced by the width of what is written, so that + /// it stays in step with where the terminal's cursor actually is. + /// /// \param[in] text text to write out /// \param[in] TextOffset where to begin writing out text from - /// \param[in] WriteOffset where to begin writing out text at the display /// \param[in] NumRequested number of text characters requested for output size_t TerminalDisplay::WriteWrappedTextPart(const Text &text, size_t TextOffset, - size_t WriteOffset, size_t NumRequested) { + size_t NumRequested) { size_t Start = TextOffset; size_t NumRemaining = NumRequested; // optimistic @@ -145,22 +222,38 @@ namespace textinput { } while (NumRemaining > 0) { - // How much can this line hold? - size_t numToEOL = GetWidth() - ((Start + WriteOffset) % GetWidth()); - if (numToEOL == 0) { // we are at EOL, move down - MoveDown(); - ++fWritePos.fLine; - MoveFront(); - fWritePos.fCol = 0; - numToEOL = GetWidth(); + // How many columns can this line still hold? + size_t numToEOL = GetWidth() - fWritePos.fCol; + + // How many characters fit into them? Not the same number, as soon as + // the text contains anything but plain single-width characters. + size_t numThisLine = 0; + size_t colsThisLine = 0; + while (numThisLine < NumRemaining) { + size_t w = text.GetWidthOfChar(Start + numThisLine); + if (colsThisLine + w > numToEOL) break; + colsThisLine += w; + ++numThisLine; } - // How much of our text can we fit in this line? - size_t numThisLine; - if (NumRemaining > numToEOL) { - numThisLine = numToEOL; - } else { - numThisLine = NumRemaining; + if (numThisLine == 0) { + if (fWritePos.fCol == 0) { + // The character is wider than the whole terminal. Nothing is + // gained by wrapping again, and looping would never end, so write + // it and let the terminal cope. + numThisLine = 1; + colsThisLine = numToEOL; + } else { + // A double-width character does not fit into what is left of this + // line. Blank those columns out and put the character at the start + // of the next line - splitting it across the margin would leave + // the terminal and us disagreeing about where the cursor is. + WriteBlanks(numToEOL); + ActOnEOL(); + fWritePos.fCol = 0; + ++fWritePos.fLine; + continue; + } } // If there is a Colorizer, we only write same-colored chunks. @@ -172,7 +265,10 @@ namespace textinput { while (numSameColor < numThisLine && ThisColor == Colors[Start + numSameColor]) ++numSameColor; - numThisLine = numSameColor; + if (numSameColor < numThisLine) { + numThisLine = numSameColor; + colsThisLine = text.GetWidth(Start, Start + numThisLine); + } if (ThisColor != fPrevColor) { Color C; @@ -182,11 +278,16 @@ namespace textinput { } } - // Write out the line and update the write position - WriteRawString(text.GetText().c_str() + Start, numThisLine); - fWritePos = IndexToPos(PosToIndex(fWritePos) + numThisLine); - if (numThisLine == numToEOL) { // If we hit EOL, wrap around + // Write out the characters and update the write position. The terminal + // wants bytes, so translate the character range into a byte range. + const size_t ByteStart = text.GetByteOffset(Start); + const size_t ByteEnd = text.GetByteOffset(Start + numThisLine); + WriteRawString(text.GetText().c_str() + ByteStart, ByteEnd - ByteStart); + fWritePos.fCol += colsThisLine; + if (fWritePos.fCol >= GetWidth()) { // If we hit EOL, wrap around ActOnEOL(); + fWritePos.fCol = 0; + ++fWritePos.fLine; } Start += numThisLine; @@ -196,13 +297,11 @@ namespace textinput { // If we have processed the characters we have requested if (NumRequested == NumAvailable) { - size_t NumPrevLines = fWriteLen / GetWidth(); - size_t LenWrote = WriteOffset + TextOffset + NumRequested; - size_t NumWroteLines = LenWrote / GetWidth(); - size_t NumToEOL = GetWidth() - (LenWrote % GetWidth()); - if (LenWrote < fWriteLen && NumToEOL > 0) { - // If we wrote less than previously and not at EOL - // Erase the rest of the current line + const size_t NumWroteLines = fWritePos.fLine; + const size_t NumPrevLines = fWriteEnd.fLine; + if (fWritePos < fWriteEnd) { + // If we wrote less than previously, + // erase the rest of the current line EraseToRight(); } if (NumWroteLines < NumPrevLines) { @@ -242,14 +341,14 @@ namespace textinput { if (PromptUpdate & Range::kUpdatePrompt) { // Writing from front means we write the prompt, too Move(Pos()); - WriteWrappedTextPart(Prompt, 0, 0, PromptLen); + WriteWrappedTextPart(Prompt, 0, PromptLen); } // If updating any prompt if (PromptUpdate != Range::kNoPromptUpdate) { // Any prompt update means we'll have to re-write the editor prompt Move(IndexToPos(PromptLen)); if (EditorPromptLen) { - WriteWrappedTextPart(EditPrompt, 0, PromptLen, EditorPromptLen); + WriteWrappedTextPart(EditPrompt, 0, EditorPromptLen); } // Any prompt update means we'll have to re-write the text Offset = 0; @@ -259,14 +358,13 @@ namespace textinput { size_t avail = 0; if (masked) { - Text mask(std::string(GetContext()->GetLine().length(), '*'), 0); - avail = WriteWrappedTextPart(mask, Offset, - PromptLen + EditorPromptLen, Requested); + Text mask(std::u32string(GetContext()->GetLine().length(), U'*'), 0); + avail = WriteWrappedTextPart(mask, Offset, Requested); } else { - avail = WriteWrappedTextPart(GetContext()->GetLine(), Offset, - PromptLen + EditorPromptLen, Requested); + avail = WriteWrappedTextPart(GetContext()->GetLine(), Offset, Requested); } - fWriteLen = PromptLen + EditorPromptLen + GetContext()->GetLine().length(); + fWriteEnd = IndexToPos(PromptLen + EditorPromptLen + + GetContext()->GetLine().length()); return avail; } diff --git a/core/textinput/src/textinput/TerminalDisplay.h b/core/textinput/src/textinput/TerminalDisplay.h index 1b23cb51bb24f..c0f08ed017dc0 100644 --- a/core/textinput/src/textinput/TerminalDisplay.h +++ b/core/textinput/src/textinput/TerminalDisplay.h @@ -43,7 +43,7 @@ namespace textinput { protected: TerminalDisplay(bool isTTY): - fIsTTY(isTTY), fWidth(80), fWriteLen(0), fPrevColor(-1) {} + fIsTTY(isTTY), fWidth(80), fPrevColor(-1) {} void SetIsTTY(bool isTTY) { fIsTTY = isTTY; } Pos GetCursor() const { // Collect the different prompts and the text cursor to calculate @@ -53,11 +53,38 @@ namespace textinput { idx += GetContext()->GetEditor()->GetEditorPrompt().length(); return IndexToPos(idx); } - Pos IndexToPos(size_t idx) const { return Pos(idx % fWidth, idx / fWidth); } - size_t PosToIndex(const Pos& pos) const { - // Convert a x|y position to an index. - return pos.fCol + pos.fLine * fWidth; + + // Lay out the first idx characters of what is displayed - the prompt, the + // editor prompt and the input line, concatenated - and return where the + // next character goes. + // + // This cannot be index arithmetic: a character is not a column. Combining + // marks take no column of their own and CJK characters and emoji take two, + // so the mapping has to walk the text and add up the widths. + Pos IndexToPos(size_t idx) const; + + // Width, in columns, of character idx of that same concatenation. + size_t WidthOfDisplayChar(size_t idx) const; + + // Grow r at the front so that it starts on a character that owns a cell, + // not on a combining mark that shares the previous one. + void ExtendRangeForCombiningChars(Range& r) const; + + // Place a character of width w and move on, wrapping at the right margin. + // Every column computation goes through here, so that the cursor position + // we compute and the position the terminal actually reaches agree. + void AdvancePos(Pos& p, size_t w) const { + if (p.fCol + w > fWidth) { // does not fit, starts on the next line + p.fCol = 0; + ++p.fLine; + } + p.fCol += w; + if (p.fCol >= fWidth) { // filled the line exactly + p.fCol = 0; + ++p.fLine; + } } + size_t GetWidth() const { return fWidth; } void SetWidth(size_t width) { fWidth = width; } @@ -70,7 +97,10 @@ namespace textinput { size_t WriteWrapped(Range::EPromptUpdate PromptUpdate, bool masked, size_t offset, size_t len = (size_t)-1); size_t WriteWrappedTextPart(const Text &text, size_t TextOffset, - size_t WriteOffset, size_t Requested); + size_t Requested); + // Write n spaces, to fill the columns a double-width character could not + // fit into before it wraps to the next line. + void WriteBlanks(size_t n); virtual void SetColor(char CIdx, const Color& C) = 0; virtual void WriteRawString(const char* text, size_t len) = 0; virtual void ActOnEOL() {} @@ -80,7 +110,7 @@ namespace textinput { protected: bool fIsTTY; // whether this is a terminal or redirected size_t fWidth; // Width of the terminal in character columns - size_t fWriteLen; // Length of output written. + Pos fWriteEnd; // Position just past the end of the output written. Pos fWritePos; // Current position of writing (temporarily != cursor) char fPrevColor; // currently configured color }; diff --git a/core/textinput/src/textinput/TerminalDisplayUnix.cpp b/core/textinput/src/textinput/TerminalDisplayUnix.cpp index 448a8fef5f65e..c2c2c5f2b91e3 100644 --- a/core/textinput/src/textinput/TerminalDisplayUnix.cpp +++ b/core/textinput/src/textinput/TerminalDisplayUnix.cpp @@ -310,7 +310,7 @@ namespace textinput { SYNC_OUT(STDOUT_FILENO); TerminalConfigUnix::Get().Attach(); fWritePos = Pos(); - fWriteLen = 0; + fWriteEnd = Pos(); fIsAttached = true; } diff --git a/core/textinput/src/textinput/TerminalDisplayWin.cpp b/core/textinput/src/textinput/TerminalDisplayWin.cpp index a3fc5cc811b89..289b41f820fb9 100644 --- a/core/textinput/src/textinput/TerminalDisplayWin.cpp +++ b/core/textinput/src/textinput/TerminalDisplayWin.cpp @@ -16,8 +16,32 @@ #ifdef _WIN32 #include "textinput/TerminalDisplayWin.h" #include "textinput/Color.h" +#include "textinput/UTF8.h" #include +#include + +namespace { + // The console API speaks UTF-16; textinput hands out UTF-8. Converting here + // and writing with WriteConsoleW keeps the output correct whatever code page + // the console happens to be set to - which is why SetConsoleOutputCP(65001) + // is not needed (it used to break line wrapping on Windows 10). + std::wstring ToUTF16(const char* Text, size_t Len) { + const std::u32string U32 = textinput::UTF8ToUTF32(Text, Len); + std::wstring Ret; + Ret.reserve(U32.length()); + for (char32_t C : U32) { + if (C < 0x10000) { + Ret += static_cast(C); + } else { + const char32_t V = C - 0x10000; + Ret += static_cast(0xD800 + (V >> 10)); + Ret += static_cast(0xDC00 + (V & 0x3FF)); + } + } + return Ret; + } +} #ifdef UNICODE #define filename L"CONOUT$" @@ -186,12 +210,18 @@ namespace textinput { TerminalDisplayWin::WriteRawString(const char *text, size_t len) { DWORD NumWritten = 0; if (IsTTY()) { - WriteConsole(fOut, text, (DWORD) len, &NumWritten, NULL); + const std::wstring WText = ToUTF16(text, len); + WriteConsoleW(fOut, WText.c_str(), (DWORD) WText.length(), &NumWritten, + NULL); + if (NumWritten != WText.length()) { + ShowError("writing to output"); + } } else { + // Redirected: pass the UTF-8 bytes through unchanged. WriteFile(fOut, text, (DWORD) len, &NumWritten, NULL); - } - if (NumWritten != len) { - ShowError("writing to output"); + if (NumWritten != len) { + ShowError("writing to output"); + } } } diff --git a/core/textinput/src/textinput/Text.h b/core/textinput/src/textinput/Text.h index 5a03e33f09046..1f64678f21723 100644 --- a/core/textinput/src/textinput/Text.h +++ b/core/textinput/src/textinput/Text.h @@ -15,46 +15,98 @@ #ifndef TEXTINPUT_TEXT_H #define TEXTINPUT_TEXT_H +#include #include -#include #include #include #include "textinput/Range.h" +#include "textinput/UTF8.h" namespace textinput { class Colorizer; - using std::strlen; // A colored string. + // + // Stored as UTF-32 so that one element is exactly one character: every index + // in this class - and thus the cursor, the ranges handed to the display and + // the color vector - counts characters, never bytes. The UTF-8 form, which + // is what everything outside textinput speaks, is produced by GetText() and + // cached until the text is modified. + // + // Note that one character is not one terminal column: see GetWidthOfChar(). class Text { public: Text() {} - Text(const char* S): fString(S), fColor(strlen(S)) {} - Text(const std::string& S, char C = 0): fString(S), fColor(S.length(), C) {} + Text(const char* S): fString(UTF8ToUTF32(S, std::char_traits::length(S))), + fColor(fString.length()) {} + Text(const std::string& S, char C = 0): fString(UTF8ToUTF32(S)), + fColor(fString.length(), C) {} + Text(const std::u32string& S, char C = 0): fString(S), + fColor(S.length(), C) {} + + // The text as UTF-8. Cached; invalidated by every mutation. + const std::string& GetText() const { + UpdateUTF8(); + return fUTF8; + } + const std::u32string& GetChars() const { return fString; } - const std::string& GetText() const { return fString; } const std::vector& GetColors() const { return fColor; } std::vector& GetColors() { return fColor; } char GetColor(size_t i) const { return fColor[i]; } + + // Number of characters, not bytes and not columns. size_t length() const { return fString.length(); } - bool empty() const { return fColor.empty(); } + bool empty() const { return fString.empty(); } + + // Number of terminal columns taken up by character i, and by the + // characters in [from, to). + // Not called GetCharWidth(): windows.h #defines that to GetCharWidthA. + size_t GetWidthOfChar(size_t i) const { return CharWidth(fString[i]); } + size_t GetWidth(size_t from, size_t to) const { + if (to > length()) to = length(); + size_t W = 0; + for (size_t i = from; i < to; ++i) W += CharWidth(fString[i]); + return W; + } + + // Byte offset of character i in GetText(); i may be length(), giving the + // total byte count. Used to hand whole characters to the terminal. + size_t GetByteOffset(size_t i) const { + UpdateUTF8(); + return fByteOffset[i < fByteOffset.size() ? i : fByteOffset.size() - 1]; + } + + // Inverse of GetByteOffset(): index of the character containing the byte + // at Offset in GetText(). An offset at or past the end gives length(). + size_t GetCharIndex(size_t Offset) const { + UpdateUTF8(); + return std::upper_bound(fByteOffset.begin(), fByteOffset.end(), Offset) + - fByteOffset.begin() - 1; + } + + std::u32string substr(size_t pos, size_t len = std::u32string::npos) const { + return fString.substr(pos, len); + } - void insert(size_t pos, char C) { + void insert(size_t pos, char32_t C) { // Insert C at pos, set to default color. fString.insert(pos, 1, C); fColor.insert(fColor.begin() + pos, 0); + fUTF8Valid = false; } - void insert(size_t pos, const std::string& S) { - // Inset S at pos, set to default color. - size_t len = S.length(); - fColor.insert(fColor.begin() + pos, len, 0); + void insert(size_t pos, const std::u32string& S) { + // Insert S at pos, set to default color. + fColor.insert(fColor.begin() + pos, S.length(), 0); fString.insert(pos, S); + fUTF8Valid = false; } void erase(size_t pos, size_t len = 1) { // Erase len characters starting at pos. fString.erase(pos, len); fColor.erase(fColor.begin() + pos, fColor.begin() + pos + len); + fUTF8Valid = false; } - void clear() { fString.clear(); fColor.clear(); } + void clear() { fString.clear(); fColor.clear(); fUTF8Valid = false; } void SetColor(const Range &R, char C) { @@ -66,20 +118,47 @@ namespace textinput { std::fill_n(fColor.begin() + R.fStart, len, C); } - char operator[](size_t i) const { return fString[i]; } - char& operator[](size_t i) { return fString[i]; } + char32_t operator[](size_t i) const { return fString[i]; } + // No non-const operator[]: handing out a reference would let a caller + // change the text behind the back of the UTF-8 cache. + void SetChar(size_t i, char32_t C) { fString[i] = C; fUTF8Valid = false; } - Text& operator+=(char C) { insert(length(), C); return *this; } + Text& operator+=(char32_t C) { insert(length(), C); return *this; } Text& operator=(const std::string& S) { - // Assign string S to this, initialize with default colors. - fColor.clear(); - fColor.resize(S.length()); + // Assign UTF-8 string S to this, initialize with default colors. + fString = UTF8ToUTF32(S); + fColor.assign(fString.length(), 0); + fUTF8Valid = false; + return *this; + } + Text& operator=(const std::u32string& S) { fString = S; + fColor.assign(S.length(), 0); + fUTF8Valid = false; return *this; } private: - std::string fString; // actual text + // Rebuild the UTF-8 form and the character -> byte offset table, if the + // text has changed since they were last built. + void UpdateUTF8() const { + if (fUTF8Valid) return; + fUTF8.clear(); + fUTF8.reserve(fString.length()); + fByteOffset.clear(); + fByteOffset.reserve(fString.length() + 1); + for (char32_t C : fString) { + fByteOffset.push_back(fUTF8.length()); + AppendUTF8(fUTF8, C); + } + fByteOffset.push_back(fUTF8.length()); // one past the end + fUTF8Valid = true; + } + + std::u32string fString; // actual text, one element per character std::vector fColor; // color index of chars; Colorizer converts to RGB + mutable std::string fUTF8; // cache of fString as UTF-8 + mutable std::vector fByteOffset; // offset of each char in fUTF8 + mutable bool fUTF8Valid = false; // whether the two caches are up to date }; } -#endif // TEXTINPUT_COLOR_H +#endif // TEXTINPUT_TEXT_H diff --git a/core/textinput/src/textinput/TextInput.cpp b/core/textinput/src/textinput/TextInput.cpp index ead12076e1c3a..03922bf16dc35 100644 --- a/core/textinput/src/textinput/TextInput.cpp +++ b/core/textinput/src/textinput/TextInput.cpp @@ -236,10 +236,12 @@ namespace textinput { } void - TextInput::HandleControl(char C, EditorRange& R) { + TextInput::HandleControl(char32_t C, EditorRange& R) { if (C == 3) { // Control+C std::string input = fContext->GetLine().GetText(); - size_t length = input.size(); + // Ranges count characters, so this must not be input.size(), which + // counts the bytes of the UTF-8 encoding. + size_t length = fContext->GetLine().length(); fContext->SetLine(input + "^C"); UpdateDisplay(EditorRange(Range(length), Range::AllText())); TakeInput(input, true); diff --git a/core/textinput/src/textinput/TextInput.h b/core/textinput/src/textinput/TextInput.h index 251939e244cd1..023af0784587d 100644 --- a/core/textinput/src/textinput/TextInput.h +++ b/core/textinput/src/textinput/TextInput.h @@ -67,7 +67,9 @@ namespace textinput { // Read interface EReadResult ReadInput(); EReadResult GetReadState() const { return fLastReadResult; } - char GetLastKey() const { return fLastKey; } + // The most recent key as a character, not a byte: a multi-byte UTF-8 + // sequence is reported once, as the code point it encodes. + char32_t GetLastKey() const { return fLastKey; } const std::string& GetInput(); void TakeInput(std::string& input, bool force = false); // Take and reset input bool AtEOL() const { return fLastReadResult == kRRReadEOLDelimiter || AtEOF(); } @@ -91,13 +93,13 @@ namespace textinput { void AddHistoryLine(const char* line); private: - void HandleControl(char c, EditorRange& r); + void HandleControl(char32_t c, EditorRange& r); void ProcessNewInput(const InputData& in, EditorRange& r); void DisplayNewInput(EditorRange& r, size_t& oldCursorPos); bool fMasked; // whether input should be shown bool fAutoHistAdd; // whether input should be added to history - char fLastKey; // most recently read key + char32_t fLastKey; // most recently read key size_t fMaxChars; // Num chars to read; 0 for blocking, -1 for all available EReadResult fLastReadResult; // current input state TextInputContext* fContext; // context object diff --git a/core/textinput/src/textinput/UTF8.cpp b/core/textinput/src/textinput/UTF8.cpp new file mode 100644 index 0000000000000..5dc4825d442a3 --- /dev/null +++ b/core/textinput/src/textinput/UTF8.cpp @@ -0,0 +1,327 @@ +//===--- UTF8.cpp - UTF-8 Conversion And Display Width ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "textinput/UTF8.h" + +namespace { + struct Range { + char32_t fFirst; + char32_t fLast; + }; + + // Characters that take up no column of their own because they combine with + // the preceding one: Unicode general categories Mn and Me, plus the format + // characters that terminals do not advance the cursor for. + // + // This covers the ranges that actually turn up at an interactive prompt + // (Latin/Greek/Cyrillic accents, Hebrew and Arabic marks, Indic matras, + // variation selectors, the zero-width joiners). It is deliberately not a + // generated copy of the full Unicode database: a wrong answer here costs a + // mispositioned cursor, not a crash, and the table has to stay reviewable. + // clang-format off + const Range kZeroWidth[] = { + {0x00AD, 0x00AD}, // soft hyphen + {0x0300, 0x036F}, // combining diacritical marks + {0x0483, 0x0489}, + {0x0591, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2}, + {0x05C4, 0x05C5}, {0x05C7, 0x05C7}, + {0x0610, 0x061A}, {0x064B, 0x065F}, {0x0670, 0x0670}, + {0x06D6, 0x06DC}, {0x06DF, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED}, + {0x0711, 0x0711}, {0x0730, 0x074A}, {0x07A6, 0x07B0}, {0x07EB, 0x07F3}, + {0x0816, 0x0819}, {0x081B, 0x0823}, {0x0825, 0x0827}, {0x0829, 0x082D}, + {0x0859, 0x085B}, + {0x08E3, 0x0903}, {0x093A, 0x093C}, {0x093E, 0x094F}, {0x0951, 0x0957}, + {0x0962, 0x0963}, + {0x0981, 0x0983}, {0x09BC, 0x09BC}, {0x09BE, 0x09C4}, {0x09C7, 0x09C8}, + {0x09CB, 0x09CD}, {0x09D7, 0x09D7}, {0x09E2, 0x09E3}, + {0x0A01, 0x0A03}, {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, + {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, {0x0A70, 0x0A71}, {0x0A75, 0x0A75}, + {0x0A81, 0x0A83}, {0x0ABC, 0x0ABC}, {0x0ABE, 0x0AC5}, {0x0AC7, 0x0AC9}, + {0x0ACB, 0x0ACD}, {0x0AE2, 0x0AE3}, + {0x0B01, 0x0B03}, {0x0B3C, 0x0B3C}, {0x0B3E, 0x0B44}, {0x0B47, 0x0B48}, + {0x0B4B, 0x0B4D}, {0x0B56, 0x0B57}, {0x0B62, 0x0B63}, {0x0B82, 0x0B82}, + {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD}, {0x0BD7, 0x0BD7}, + {0x0C00, 0x0C03}, {0x0C3E, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, + {0x0C55, 0x0C56}, {0x0C62, 0x0C63}, + {0x0C81, 0x0C83}, {0x0CBC, 0x0CBC}, {0x0CBE, 0x0CC4}, {0x0CC6, 0x0CC8}, + {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, {0x0CE2, 0x0CE3}, + {0x0D01, 0x0D03}, {0x0D3E, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4D}, + {0x0D57, 0x0D57}, {0x0D62, 0x0D63}, + {0x0D82, 0x0D83}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, + {0x0DD8, 0x0DDF}, {0x0DF2, 0x0DF3}, + {0x0E31, 0x0E31}, {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, + {0x0EB1, 0x0EB1}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD}, + {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37}, {0x0F39, 0x0F39}, + {0x0F71, 0x0F84}, {0x0F86, 0x0F87}, {0x0F8D, 0x0F97}, {0x0F99, 0x0FBC}, + {0x0FC6, 0x0FC6}, + {0x102B, 0x103E}, {0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, + {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, {0x108F, 0x108F}, + {0x109A, 0x109D}, + {0x135D, 0x135F}, {0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753}, + {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, + {0x180B, 0x180E}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, + {0x1920, 0x192B}, {0x1930, 0x193B}, + {0x1A17, 0x1A1B}, {0x1A55, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, + {0x1AB0, 0x1ABE}, + {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, + {0x1BA1, 0x1BAD}, {0x1BE6, 0x1BF3}, + {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2}, {0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, + {0x1CF2, 0x1CF4}, {0x1CF8, 0x1CF9}, + {0x1DC0, 0x1DFF}, // combining diacritical marks supplement + {0x200B, 0x200F}, // zero width space .. RTL mark (includes ZWNJ, ZWJ) + {0x202A, 0x202E}, // bidi embedding controls + {0x2060, 0x2064}, // word joiner, invisible operators + {0x206A, 0x206F}, + {0x20D0, 0x20F0}, // combining marks for symbols + {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, {0x2DE0, 0x2DFF}, + {0x302A, 0x302D}, // ideographic tone marks (the 302E/302F are wide) + {0x3099, 0x309A}, // combining kana voiced sound marks + {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, + {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, {0xA80B, 0xA80B}, + {0xA823, 0xA827}, {0xA880, 0xA881}, {0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, + {0xA926, 0xA92D}, {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, + {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, {0xAA4C, 0xAA4D}, + {0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, + {0xAABE, 0xAABF}, {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, + {0xABE3, 0xABEA}, {0xABEC, 0xABED}, + {0xFB1E, 0xFB1E}, {0xFE00, 0xFE0F}, // variation selectors + {0xFE20, 0xFE2F}, // combining half marks + {0xFEFF, 0xFEFF}, // zero width no-break space / BOM + {0xFFF9, 0xFFFB}, // interlinear annotation + {0x101FD, 0x101FD}, {0x102E0, 0x102E0}, {0x10376, 0x1037A}, + {0x10A01, 0x10A0F}, {0x10A38, 0x10A3F}, {0x10AE5, 0x10AE6}, + {0x11000, 0x11002}, {0x11038, 0x11046}, {0x1107F, 0x11082}, + {0x110B0, 0x110BA}, {0x11100, 0x11102}, {0x11127, 0x11134}, + {0x11173, 0x11173}, {0x11180, 0x11182}, {0x111B3, 0x111C0}, + {0x1122C, 0x11237}, {0x112DF, 0x112EA}, {0x11300, 0x11303}, + {0x1133C, 0x1133C}, {0x1133E, 0x11344}, {0x11347, 0x11348}, + {0x1134B, 0x1134D}, {0x11362, 0x11363}, {0x11366, 0x11374}, + {0x114B0, 0x114C3}, {0x115AF, 0x115C0}, {0x11630, 0x11640}, + {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, {0x16F51, 0x16F7E}, {0x16F8F, 0x16F92}, + {0x1BC9D, 0x1BC9E}, {0x1BCA0, 0x1BCA3}, + {0x1D165, 0x1D169}, {0x1D16D, 0x1D182}, {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, + {0x1E8D0, 0x1E8D6}, + {0xE0001, 0xE0001}, {0xE0020, 0xE007F}, // language tags + {0xE0100, 0xE01EF} // variation selectors supplement + }; + + // Characters that take up two columns: East Asian Wide (W) and Fullwidth (F), + // plus the emoji blocks that terminals render double-width. + const Range kDoubleWidth[] = { + {0x1100, 0x115F}, // Hangul Jamo initial consonants + {0x231A, 0x231B}, // watch, hourglass + {0x2329, 0x232A}, // angle brackets + {0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3}, + {0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653}, + {0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1}, + {0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, + {0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA}, + {0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA}, + {0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B}, + {0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E}, + {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, + {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, + {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, + {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3}, // CJK radicals + {0x2F00, 0x2FD5}, // Kangxi radicals + {0x2FF0, 0x2FFB}, // ideographic description + {0x3000, 0x303E}, // CJK symbols and punctuation (303F is narrow) + {0x3041, 0x3096}, // Hiragana + {0x3099, 0x30FF}, // Katakana (the combining marks are caught earlier) + {0x3105, 0x312D}, // Bopomofo + {0x3131, 0x318E}, // Hangul compatibility Jamo + {0x3190, 0x31BA}, {0x31C0, 0x31E3}, {0x31F0, 0x321E}, + {0x3220, 0x3247}, {0x3250, 0x32FE}, + {0x3300, 0x4DBF}, // CJK compatibility, extension A + {0x4E00, 0xA48C}, // CJK unified ideographs, Yi + {0xA490, 0xA4C6}, + {0xA960, 0xA97C}, // Hangul Jamo extended-A + {0xAC00, 0xD7A3}, // Hangul syllables + {0xF900, 0xFAFF}, // CJK compatibility ideographs + {0xFE10, 0xFE19}, // vertical forms + {0xFE30, 0xFE6B}, // CJK compatibility forms, small form variants + {0xFF00, 0xFF60}, // fullwidth forms + {0xFFE0, 0xFFE6}, // fullwidth signs + {0x16FE0, 0x16FE1}, {0x17000, 0x187F1}, // Tangut + {0x18800, 0x18AF2}, + {0x1B000, 0x1B11E}, {0x1B170, 0x1B2FB}, + {0x1F004, 0x1F004}, {0x1F0CF, 0x1F0CF}, + {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, + {0x1F200, 0x1F320}, {0x1F32D, 0x1F335}, {0x1F337, 0x1F37C}, + {0x1F37E, 0x1F393}, {0x1F3A0, 0x1F3CA}, {0x1F3CF, 0x1F3D3}, + {0x1F3E0, 0x1F3F0}, {0x1F3F4, 0x1F3F4}, {0x1F3F8, 0x1F43E}, + {0x1F440, 0x1F440}, {0x1F442, 0x1F4FC}, {0x1F4FF, 0x1F53D}, + {0x1F54B, 0x1F54E}, {0x1F550, 0x1F567}, {0x1F57A, 0x1F57A}, + {0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A4}, {0x1F5FB, 0x1F64F}, + {0x1F680, 0x1F6C5}, {0x1F6CC, 0x1F6CC}, {0x1F6D0, 0x1F6D2}, + {0x1F6EB, 0x1F6EC}, {0x1F6F4, 0x1F6F9}, + {0x1F910, 0x1F93E}, {0x1F940, 0x1F970}, {0x1F973, 0x1F976}, + {0x1F97A, 0x1F9A2}, {0x1F9B0, 0x1F9B9}, {0x1F9C0, 0x1F9C2}, + {0x1F9D0, 0x1F9FF}, + {0x20000, 0x2FFFD}, // CJK extension B and beyond + {0x30000, 0x3FFFD} + }; + // clang-format on + + template + bool InRanges(const Range (&Ranges)[N], char32_t C) { + size_t Lo = 0; + size_t Hi = N; + while (Lo < Hi) { + size_t Mid = Lo + (Hi - Lo) / 2; + if (C < Ranges[Mid].fFirst) { + Hi = Mid; + } else if (C > Ranges[Mid].fLast) { + Lo = Mid + 1; + } else { + return true; + } + } + return false; + } +} // unnamed namespace + +namespace textinput { + + void AppendUTF8(std::string& Out, char32_t C) { + if (!IsValidCodePoint(C)) { + C = kInvalidChar; + } + if (C < 0x80) { + Out += static_cast(C); + } else if (C < 0x800) { + Out += static_cast(0xC0 | (C >> 6)); + Out += static_cast(0x80 | (C & 0x3F)); + } else if (C < 0x10000) { + Out += static_cast(0xE0 | (C >> 12)); + Out += static_cast(0x80 | ((C >> 6) & 0x3F)); + Out += static_cast(0x80 | (C & 0x3F)); + } else { + Out += static_cast(0xF0 | (C >> 18)); + Out += static_cast(0x80 | ((C >> 12) & 0x3F)); + Out += static_cast(0x80 | ((C >> 6) & 0x3F)); + Out += static_cast(0x80 | (C & 0x3F)); + } + } + + std::u32string UTF8ToUTF32(const char* S, size_t Len) { + std::u32string Ret; + Ret.reserve(Len); + size_t I = 0; + while (I < Len) { + const unsigned char Lead = static_cast(S[I]); + const size_t SeqLen = UTF8SequenceLength(Lead); + if (SeqLen == 0 || I + SeqLen > Len) { + // Not a lead byte, or the string ends mid-sequence. + Ret += kInvalidChar; + ++I; + continue; + } + if (SeqLen == 1) { + Ret += static_cast(Lead); + ++I; + continue; + } + static const unsigned char LeadMask[5] = {0, 0, 0x1F, 0x0F, 0x07}; + char32_t Value = Lead & LeadMask[SeqLen]; + size_t NumCont = 0; + for (; NumCont < SeqLen - 1; ++NumCont) { + const unsigned char Cont = static_cast(S[I + 1 + NumCont]); + if (!IsUTF8Continuation(Cont)) break; + Value = (Value << 6) | (Cont & 0x3F); + } + if (NumCont != SeqLen - 1 || !IsValidCodePoint(Value)) { + // Truncated or surrogate/out-of-range: consume only what we validated + // so that a following lead byte still starts a fresh character. + Ret += kInvalidChar; + I += 1 + NumCont; + continue; + } + Ret += Value; + I += SeqLen; + } + return Ret; + } + + std::string UTF32ToUTF8(const char32_t* S, size_t Len) { + std::string Ret; + Ret.reserve(Len); + for (size_t I = 0; I < Len; ++I) { + AppendUTF8(Ret, S[I]); + } + return Ret; + } + + size_t CharWidth(char32_t C) { + if (C < 0x7F) { + // Fast path for ASCII. Control characters never reach the line buffer - + // the editor rejects them - but the prompt may contain a stray one, and + // claiming a column for it keeps the cursor arithmetic honest. + return C < 0x20 ? 0 : 1; + } + if (C < 0xA0) return 0; // C1 controls + if (InRanges(kZeroWidth, C)) return 0; + if (InRanges(kDoubleWidth, C)) return 2; + return 1; + } + + UTF8Decoder::EResult + UTF8Decoder::Push(unsigned char C, char32_t& Out, bool& Reprocess) { + Reprocess = false; + if (fPending == 0) { + const size_t SeqLen = UTF8SequenceLength(C); + if (SeqLen == 0) { + // A continuation byte with nothing to continue, or a byte that is + // never valid UTF-8. + Out = kInvalidChar; + return kInvalid; + } + if (SeqLen == 1) { + Out = C; + return kComplete; + } + static const unsigned char LeadMask[5] = {0, 0, 0x1F, 0x0F, 0x07}; + fValue = C & LeadMask[SeqLen]; + fLength = SeqLen; + fPending = SeqLen - 1; + return kNeedMore; + } + + if (!IsUTF8Continuation(C)) { + // The sequence was cut short. Report it and let the caller re-feed this + // byte, which may well be the start of the next character (or an ESC + // that we must not swallow). + Reset(); + Out = kInvalidChar; + Reprocess = true; + return kInvalid; + } + + fValue = (fValue << 6) | (C & 0x3F); + if (--fPending != 0) { + return kNeedMore; + } + + const char32_t Value = fValue; + const size_t Length = fLength; + Reset(); + // Reject surrogates, out-of-range values and overlong encodings. + static const char32_t MinForLength[5] = {0, 0, 0x80, 0x800, 0x10000}; + if (!IsValidCodePoint(Value) || Value < MinForLength[Length]) { + Out = kInvalidChar; + return kInvalid; + } + Out = Value; + return kComplete; + } +} diff --git a/core/textinput/src/textinput/UTF8.h b/core/textinput/src/textinput/UTF8.h new file mode 100644 index 0000000000000..ec52d2e77c52b --- /dev/null +++ b/core/textinput/src/textinput/UTF8.h @@ -0,0 +1,99 @@ +//===--- UTF8.h - UTF-8 Conversion And Display Width ------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the conversion between UTF-8 (used by everything outside +// textinput: the interpreter, the history file, Getline's C interface) and +// UTF-32 (used inside textinput, where one buffer element must be exactly one +// character), plus the number of terminal columns a character occupies. +// +//===----------------------------------------------------------------------===// + +#ifndef TEXTINPUT_UTF8_H +#define TEXTINPUT_UTF8_H + +#include +#include + +namespace textinput { + + // The character substituted for malformed input, U+FFFD REPLACEMENT + // CHARACTER. Decoding never fails; it produces this instead, so that a stray + // byte from a mistyped paste cannot desynchronize the line buffer. + const char32_t kInvalidChar = 0xFFFD; + + // Number of bytes in the UTF-8 sequence introduced by Lead, or 0 if Lead is + // not a valid lead byte (i.e. it is a continuation byte or is never valid). + inline size_t UTF8SequenceLength(unsigned char Lead) { + if (Lead < 0x80) return 1; + if (Lead < 0xC2) return 0; // continuation byte, or overlong lead C0/C1 + if (Lead < 0xE0) return 2; + if (Lead < 0xF0) return 3; + if (Lead < 0xF5) return 4; // F5..FF encode beyond U+10FFFF + return 0; + } + + inline bool IsUTF8Continuation(unsigned char C) { + return (C & 0xC0) == 0x80; + } + + // Whether C is a valid Unicode scalar value, i.e. neither beyond the + // Unicode range nor one half of a surrogate pair. + inline bool IsValidCodePoint(char32_t C) { + return C <= 0x10FFFF && (C < 0xD800 || C > 0xDFFF); + } + + // Append C to Out as UTF-8. Invalid code points are replaced. + void AppendUTF8(std::string& Out, char32_t C); + + // Decode a UTF-8 string. Malformed sequences become kInvalidChar. + std::u32string UTF8ToUTF32(const char* S, size_t Len); + inline std::u32string UTF8ToUTF32(const std::string& S) { + return UTF8ToUTF32(S.data(), S.length()); + } + + // Encode as UTF-8. + std::string UTF32ToUTF8(const char32_t* S, size_t Len); + inline std::string UTF32ToUTF8(const std::u32string& S) { + return UTF32ToUTF8(S.data(), S.length()); + } + + // The number of terminal columns taken up by C: 0 for combining marks and + // other zero-width characters, 2 for East Asian Wide / Fullwidth characters + // and emoji, 1 for everything else. + size_t CharWidth(char32_t C); + + // Accumulates the bytes of one UTF-8 sequence as they arrive from a stream. + // The readers cannot decode a whole buffer at once: they hand textinput one + // character at a time and must not block waiting for a character that the + // user has not typed yet. + class UTF8Decoder { + public: + // What Push() did with the byte it was given. + enum EResult { + kNeedMore, // byte consumed, sequence still incomplete + kComplete, // byte consumed, Out holds the decoded character + kInvalid // sequence is malformed; Out is kInvalidChar and, if + // Reprocess is set, the byte was not consumed + }; + + // Feed one byte. On kInvalid with Reprocess == true, the caller must feed + // C again (to a now-reset decoder), because it starts a new sequence. + EResult Push(unsigned char C, char32_t& Out, bool& Reprocess); + + void Reset() { fPending = 0; } + bool IsPending() const { return fPending != 0; } + + private: + char32_t fValue = 0; // code point assembled so far + size_t fPending = 0; // continuation bytes still expected + size_t fLength = 0; // total length of the sequence being decoded + }; +} + +#endif // TEXTINPUT_UTF8_H