Add general category and case folding queries to the Unicode module

src/core/unicode/CMakeLists.txt

@@ -27,6 +27,7 @@ add_custom_command(
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/UnicodeData.txt"
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/CompositionExclusions.txt"
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/DerivedCoreProperties.txt"
+    "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/CaseFolding.txt"
   DEPENDS
     "${SOURCEMETA_CORE_UNICODE_CODEGEN_TARGET}"
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/PropertyValueAliases.txt"
@@ -39,6 +40,7 @@ add_custom_command(
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/UnicodeData.txt"
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/CompositionExclusions.txt"
     "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/DerivedCoreProperties.txt"
+    "${SOURCEMETA_CORE_UNICODE_UCD_DIR}/CaseFolding.txt"
   COMMENT "Generating Unicode property tables"
   VERBATIM)
 

src/core/unicode/include/sourcemeta/core/unicode.h

@@ -13,7 +13,7 @@
 #include <optional>    // std::optional
 #include <ostream>     // std::ostream
 #include <string>      // std::string, std::u32string, std::wstring
-#include <string_view> // std::string_view, std::wstring_view
+#include <string_view> // std::string_view, std::u32string_view, std::wstring_view
 #include <utility>     // std::pair, std::make_pair
 
 /// @defgroup unicode Unicode
@@ -587,6 +587,26 @@ auto bidi_class(const char32_t codepoint) noexcept -> BidiClass;
 SOURCEMETA_CORE_UNICODE_EXPORT
 auto script(const char32_t codepoint) noexcept -> UnicodeScript;
 
+/// @ingroup unicode
+/// Return the general category of a Unicode codepoint. Codepoints beyond
+/// U+10FFFF are reported as unassigned. See
+/// https://www.unicode.org/reports/tr44/ for the property's definition.
+/// For example:
+///
+/// ```cpp
+/// #include <sourcemeta/core/unicode.h>
+/// #include <cassert>
+///
+/// assert(sourcemeta::core::general_category(U'A') ==
+///        sourcemeta::core::GeneralCategory::UppercaseLetter);
+/// assert(sourcemeta::core::general_category(U'\u0301') ==
+///        sourcemeta::core::GeneralCategory::NonspacingMark);
+/// assert(sourcemeta::core::general_category(U' ') ==
+///        sourcemeta::core::GeneralCategory::SpaceSeparator);
+/// ```
+SOURCEMETA_CORE_UNICODE_EXPORT
+auto general_category(const char32_t codepoint) noexcept -> GeneralCategory;
+
 /// @ingroup unicode
 /// Return whether a Unicode codepoint is a combining mark, in the sense
 /// of UAX #44 general category Mn (Nonspacing_Mark), Mc (Spacing_Mark),
@@ -730,6 +750,40 @@ auto nfc(const std::u32string_view input) -> std::u32string;
 SOURCEMETA_CORE_UNICODE_EXPORT
 auto is_nfc(const std::u32string_view input) -> bool;
 
+/// @ingroup unicode
+/// Return the full case folding of a Unicode codepoint per Section 3.13 of the
+/// Unicode Standard, which takes the common and full mappings of the Unicode
+/// Character Database and never the simple or Turkic ones. The view points
+/// into static data and remains valid for the program's lifetime. An empty
+/// view means the codepoint folds to itself, which is also the case for every
+/// codepoint beyond U+10FFFF. For example:
+///
+/// ```cpp
+/// #include <sourcemeta/core/unicode.h>
+/// #include <cassert>
+///
+/// assert(sourcemeta::core::case_fold(U'a').empty());
+/// assert(sourcemeta::core::case_fold(U'A') == std::u32string_view{U"a"});
+/// assert(sourcemeta::core::case_fold(U'\u00DF') ==
+///        std::u32string_view{U"ss"});
+/// ```
+SOURCEMETA_CORE_UNICODE_EXPORT
+auto case_fold(const char32_t codepoint) noexcept -> std::u32string_view;
+
+/// @ingroup unicode
+/// Return the full case folding of `input` per Section 3.13 of the Unicode
+/// Standard, folding every codepoint in turn. For example:
+///
+/// ```cpp
+/// #include <sourcemeta/core/unicode.h>
+/// #include <cassert>
+///
+/// assert(sourcemeta::core::case_fold(U"Stra\u00DFe") == U"strasse");
+/// assert(sourcemeta::core::case_fold(U"\u0130") == U"i\u0307");
+/// ```
+SOURCEMETA_CORE_UNICODE_EXPORT
+auto case_fold(const std::u32string_view input) -> std::u32string;
+
 /// @ingroup unicode
 /// Determine the byte length of the valid UTF-8 codepoint starting at the
 /// given position within the input. Returns 1 for an ASCII byte, 2/3/4 for a

src/core/unicode/include/sourcemeta/core/unicode_ucd.h

@@ -257,6 +257,51 @@ enum class UnicodeScript : std::uint8_t {
 #endif
 };
 
+/// @ingroup unicode
+/// Each entry maps a `GeneralCategory` enum name to its UCD short alias.
+#define SOURCEMETA_CORE_GENERAL_CATEGORY_LIST(X)                               \
+  X(UppercaseLetter, "Lu")                                                     \
+  X(LowercaseLetter, "Ll")                                                     \
+  X(TitlecaseLetter, "Lt")                                                     \
+  X(ModifierLetter, "Lm")                                                      \
+  X(OtherLetter, "Lo")                                                         \
+  X(NonspacingMark, "Mn")                                                      \
+  X(SpacingMark, "Mc")                                                         \
+  X(EnclosingMark, "Me")                                                       \
+  X(DecimalNumber, "Nd")                                                       \
+  X(LetterNumber, "Nl")                                                        \
+  X(OtherNumber, "No")                                                         \
+  X(ConnectorPunctuation, "Pc")                                                \
+  X(DashPunctuation, "Pd")                                                     \
+  X(OpenPunctuation, "Ps")                                                     \
+  X(ClosePunctuation, "Pe")                                                    \
+  X(InitialPunctuation, "Pi")                                                  \
+  X(FinalPunctuation, "Pf")                                                    \
+  X(OtherPunctuation, "Po")                                                    \
+  X(MathSymbol, "Sm")                                                          \
+  X(CurrencySymbol, "Sc")                                                      \
+  X(ModifierSymbol, "Sk")                                                      \
+  X(OtherSymbol, "So")                                                         \
+  X(SpaceSeparator, "Zs")                                                      \
+  X(LineSeparator, "Zl")                                                       \
+  X(ParagraphSeparator, "Zp")                                                  \
+  X(Control, "Cc")                                                             \
+  X(Format, "Cf")                                                              \
+  X(Surrogate, "Cs")                                                           \
+  X(PrivateUse, "Co")                                                          \
+  X(Unassigned, "Cn")
+
+/// @ingroup unicode
+/// The general category of a Unicode codepoint per UAX #44. See
+/// https://www.unicode.org/reports/tr44/ for the property's definition.
+enum class GeneralCategory : std::uint8_t {
+#if !defined(DOXYGEN)
+#define SOURCEMETA_CORE_UCD_ENUM_ENTRY(name, alias) name,
+  SOURCEMETA_CORE_GENERAL_CATEGORY_LIST(SOURCEMETA_CORE_UCD_ENUM_ENTRY)
+#undef SOURCEMETA_CORE_UCD_ENUM_ENTRY
+#endif
+};
+
 /// @ingroup unicode
 /// Each entry maps an `NFCQuickCheck` enum name to its UCD short alias.
 #define SOURCEMETA_CORE_NFC_QUICK_CHECK_LIST(X)                                \

src/core/unicode/unicode.cc

@@ -5,8 +5,8 @@
 #include <cstddef>     // std::size_t
 #include <cstdint>     // std::uint8_t
 #include <optional>    // std::optional, std::nullopt
-#include <string>      // std::string, std::wstring
-#include <string_view> // std::string_view, std::wstring_view
+#include <string>      // std::string, std::u32string, std::wstring
+#include <string_view> // std::string_view, std::u32string_view, std::wstring_view
 
 #if defined(_WIN32) || defined(__CYGWIN__)
 #define WIN32_LEAN_AND_MEAN
@@ -368,6 +368,15 @@ auto script(const char32_t codepoint) noexcept -> UnicodeScript {
       UNICODE_SCRIPT_STAGE2[(page << 10U) | (codepoint & 0x3FFU)]);
 }
 
+auto general_category(const char32_t codepoint) noexcept -> GeneralCategory {
+  if (codepoint > 0x10FFFF) {
+    return GeneralCategory::Unassigned;
+  }
+  const std::size_t page{GENERAL_CATEGORY_STAGE1[codepoint >> 10U]};
+  return static_cast<GeneralCategory>(
+      GENERAL_CATEGORY_STAGE2[(page << 10U) | (codepoint & 0x3FFU)]);
+}
+
 auto is_combining_mark(const char32_t codepoint) noexcept -> bool {
   if (codepoint > 0x10FFFF) {
     return false;
@@ -439,4 +448,30 @@ auto canonical_composition(const char32_t starter,
   return std::nullopt;
 }
 
+auto case_fold(const char32_t codepoint) noexcept -> std::u32string_view {
+  if (codepoint > 0x10FFFF) {
+    return {};
+  }
+  const std::size_t page{CASE_FOLDING_STAGE1[codepoint >> 10U]};
+  const std::uint16_t packed{
+      CASE_FOLDING_STAGE2[(page << 10U) | (codepoint & 0x3FFU)]};
+  const auto length{static_cast<std::size_t>(packed >> 14U)};
+  const auto offset{static_cast<std::size_t>(packed & 0x3FFFU)};
+  return std::u32string_view{CASE_FOLDING_BLOB + offset, length};
+}
+
+auto case_fold(const std::u32string_view input) -> std::u32string {
+  std::u32string result;
+  result.reserve(input.size());
+  for (const auto codepoint : input) {
+    const auto folding{case_fold(codepoint)};
+    if (folding.empty()) {
+      result.push_back(codepoint);
+    } else {
+      result.append(folding);
+    }
+  }
+  return result;
+}
+
 } // namespace sourcemeta::core

test/unicode/CMakeLists.txt

@@ -20,6 +20,7 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME unicode
     unicode_joining_type_test.cc
     unicode_bidi_class_test.cc
     unicode_script_test.cc
+    unicode_general_category_test.cc
     unicode_is_combining_mark_test.cc
     unicode_is_id_continue_test.cc
     unicode_is_id_start_test.cc
@@ -28,6 +29,7 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME unicode
     unicode_canonical_composition_test.cc
     unicode_nfc_test.cc
     unicode_is_nfc_test.cc
+    unicode_case_fold_test.cc
     unicode_is_ucschar_test.cc
     unicode_is_iprivate_test.cc)
 

test/unicode/unicode_case_fold_test.cc

@@ -0,0 +1,134 @@
+#include <sourcemeta/core/test.h>
+#include <sourcemeta/core/unicode.h>
+
+#include <string_view> // std::u32string_view
+
+TEST(ascii_uppercase_letter) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'A'), std::u32string_view{U"a"});
+}
+
+TEST(ascii_uppercase_letter_z) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'Z'), std::u32string_view{U"z"});
+}
+
+TEST(ascii_lowercase_letter_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'a').empty());
+}
+
+TEST(ascii_digit_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'0').empty());
+}
+
+TEST(null_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(0x0000).empty());
+}
+
+TEST(latin_small_sharp_s) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u00DF'), std::u32string_view{U"ss"});
+}
+
+TEST(latin_capital_i_with_dot_above) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u0130'),
+            std::u32string_view{U"i\u0307"});
+}
+
+TEST(latin_capital_sharp_s_prefers_full_over_simple) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u1E9E'), std::u32string_view{U"ss"});
+}
+
+TEST(latin_small_ligature_ffi) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\uFB03'),
+            std::u32string_view{U"ffi"});
+}
+
+TEST(greek_small_iota_with_dialytika_and_tonos) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u0390'),
+            std::u32string_view{U"\u03B9\u0308\u0301"});
+}
+
+TEST(latin_capital_i_ignores_turkic) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'I'), std::u32string_view{U"i"});
+}
+
+TEST(latin_small_dotless_i_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'\u0131').empty());
+}
+
+TEST(greek_small_final_sigma) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u03C2'),
+            std::u32string_view{U"\u03C3"});
+}
+
+TEST(greek_capital_sigma) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u03A3'),
+            std::u32string_view{U"\u03C3"});
+}
+
+TEST(greek_small_sigma_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'\u03C3').empty());
+}
+
+TEST(micro_sign) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u00B5'),
+            std::u32string_view{U"\u03BC"});
+}
+
+TEST(kelvin_sign) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u212A'), std::u32string_view{U"k"});
+}
+
+TEST(cherokee_small_letter_a) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\uAB70'),
+            std::u32string_view{U"\u13A0"});
+}
+
+TEST(cherokee_small_letter_ye) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\u13F8'),
+            std::u32string_view{U"\u13F0"});
+}
+
+TEST(cherokee_capital_letter_a_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'\u13A0').empty());
+}
+
+TEST(deseret_capital_letter_long_i) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\U00010400'),
+            std::u32string_view{U"\U00010428"});
+}
+
+TEST(deseret_small_letter_long_i_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(U'\U00010428').empty());
+}
+
+TEST(adlam_capital_letter_sha) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U'\U0001E921'),
+            std::u32string_view{U"\U0001E943"});
+}
+
+TEST(max_codepoint_folds_to_itself) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(0x10FFFF).empty());
+}
+
+TEST(above_max_codepoint) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(0x110000).empty());
+}
+
+TEST(above_max_codepoint_high) {
+  EXPECT_TRUE(sourcemeta::core::case_fold(0xFFFFFFFF).empty());
+}
+
+TEST(string_empty) { EXPECT_EQ(sourcemeta::core::case_fold(U""), U""); }
+
+TEST(string_ascii) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U"Hello World"), U"hello world");
+}
+
+TEST(string_already_folded) {
+  EXPECT_EQ(sourcemeta::core::case_fold(U"strasse"), U"strasse");
+}
+
+TEST(string_mixed) {
+  EXPECT_EQ(sourcemeta::core::case_fold(
+                U"Stra\u00DFe \u03A3\u03C2 \u0130I \uFB03 \U00010400"),
+            U"strasse \u03C3\u03C3 i\u0307i ffi \U00010428");
+}