From c1d1a498187d6873c1760d9022a9b8d125b9bf72 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Mon, 14 Sep 2026 20:46:52 -0300 Subject: [PATCH] Replace invalid Markdown UTF-8 per maximal subpart and test nested footnotes Signed-off-by: Juan Cruz Viotti --- config.cmake.in | 1 + src/core/markdown/CMakeLists.txt | 3 ++ src/core/markdown/markdown.cc | 30 +++++++++++--- test/markdown/markdown_characters_test.cc | 9 +++-- test/markdown/markdown_footnotes_test.cc | 49 +++++++++++++++++++++++ 5 files changed, 83 insertions(+), 9 deletions(-) diff --git a/config.cmake.in b/config.cmake.in index e7a9d88bfd..189f0c87d6 100644 --- a/config.cmake.in +++ b/config.cmake.in @@ -352,6 +352,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_css.cmake") elseif(component STREQUAL "markdown") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_markdown.cmake") elseif(component STREQUAL "diff") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") diff --git a/src/core/markdown/CMakeLists.txt b/src/core/markdown/CMakeLists.txt index be72c09dba..048612f09a 100644 --- a/src/core/markdown/CMakeLists.txt +++ b/src/core/markdown/CMakeLists.txt @@ -6,3 +6,6 @@ if(SOURCEMETA_CORE_INSTALL) endif() target_link_libraries(sourcemeta_core_markdown PRIVATE CMarkGFM::cmark_gfm) + +target_link_libraries(sourcemeta_core_markdown PRIVATE + sourcemeta::core::unicode) diff --git a/src/core/markdown/markdown.cc b/src/core/markdown/markdown.cc index 367dc2d8d8..a35ffc5993 100644 --- a/src/core/markdown/markdown.cc +++ b/src/core/markdown/markdown.cc @@ -1,13 +1,16 @@ #include +#include #include // cmark_gfm_core_extensions_ensure_registered #include // cmark_find_syntax_extension, cmark_parser_attach_syntax_extension, cmark_parser_get_syntax_extensions #include // cmark_parser_new, cmark_parser_feed, cmark_parser_finish, cmark_parser_free, cmark_render_html, cmark_node_free -#include // std::array -#include // std::free -#include // std::mutex, std::scoped_lock -#include // std::string +#include // std::array +#include // std::size_t +#include // std::free +#include // std::mutex, std::scoped_lock +#include // std::string +#include // std::string_view namespace sourcemeta::core { @@ -16,6 +19,23 @@ auto markdown_to_html(const std::string_view input, const bool safe) [[maybe_unused]] static const bool CMARK_INITIALIZED{ (cmark_gfm_core_extensions_ensure_registered(), true)}; + // Byte sequences that are not UTF-8 become one replacement character per + // maximal subpart, as the Unicode Standard recommends, rather than the one + // replacement character per sequence that the parser would produce + std::size_t valid_length{0}; + while (valid_length < input.size()) { + const auto length{utf8_codepoint_length(input, valid_length)}; + if (length == 0) { + break; + } + + valid_length += length; + } + + const auto is_valid{valid_length == input.size()}; + const auto repaired{is_valid ? std::string{} : to_valid_utf8(input)}; + const std::string_view source{is_valid ? input : std::string_view{repaired}}; + // cmark-gfm toggles process-global special-character tables when syntax // extensions are attached and detached, so parser construction through // teardown cannot run concurrently @@ -40,7 +60,7 @@ auto markdown_to_html(const std::string_view input, const bool safe) } } - cmark_parser_feed(parser, input.data(), input.size()); + cmark_parser_feed(parser, source.data(), source.size()); auto *document{cmark_parser_finish(parser)}; auto *result{cmark_render_html(document, options, cmark_parser_get_syntax_extensions(parser))}; diff --git a/test/markdown/markdown_characters_test.cc b/test/markdown/markdown_characters_test.cc index 45f9059805..22ea5653d0 100644 --- a/test/markdown/markdown_characters_test.cc +++ b/test/markdown/markdown_characters_test.cc @@ -182,23 +182,24 @@ TEST(invalid_utf8_consecutive_continuation_bytes) { TEST(invalid_utf8_overlong_encoded_slash) { const auto result{sourcemeta::core::markdown_to_html("path\xC0\xAF" "etc")}; - EXPECT_EQ(result, "

path\xef\xbf\xbd" + EXPECT_EQ(result, "

path\xef\xbf\xbd\xef\xbf\xbd" "etc

\n"); } TEST(invalid_utf8_overlong_encoded_nul) { const auto result{sourcemeta::core::markdown_to_html("x\xE0\x80\x80y")}; - EXPECT_EQ(result, "

x\xef\xbf\xbdy

\n"); + EXPECT_EQ(result, "

x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy

\n"); } TEST(invalid_utf8_encoded_surrogate) { const auto result{sourcemeta::core::markdown_to_html("x\xED\xA0\x80y")}; - EXPECT_EQ(result, "

x\xef\xbf\xbdy

\n"); + EXPECT_EQ(result, "

x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy

\n"); } TEST(invalid_utf8_code_point_above_unicode_range) { const auto result{sourcemeta::core::markdown_to_html("x\xF4\x90\x80\x80y")}; - EXPECT_EQ(result, "

x\xef\xbf\xbdy

\n"); + EXPECT_EQ(result, + "

x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy

\n"); } TEST(invalid_utf8_five_byte_sequence) { diff --git a/test/markdown/markdown_footnotes_test.cc b/test/markdown/markdown_footnotes_test.cc index f253a34a26..73f28a89e1 100644 --- a/test/markdown/markdown_footnotes_test.cc +++ b/test/markdown/markdown_footnotes_test.cc @@ -239,3 +239,52 @@ TEST(footnote_label_with_space_is_not_footnote) { "body[^two words]\n\n[^two words]: text")}; EXPECT_EQ(result, "

body^two words

\n"); } + +TEST(footnote_definition_nested_inside_another_definition) { + const auto result{sourcemeta::core::markdown_to_html( + "[^a]\n\n[^a]: outer\n [^b]: inner\n\n[^b]")}; + EXPECT_EQ(result, + "

1

\n" + "

2

\n" + "
\n
    \n" + "
  1. \n" + "

    outer \xe2\x86\xa9

    \n" + "
    2. \n" + "
  2. \n" + "

    inner \xe2\x86\xa9

    \n" + "
    3. \n
\n
\n"); +} + +TEST(footnote_nested_definition_referenced_without_its_parent) { + const auto result{sourcemeta::core::markdown_to_html( + "[^b]\n\n[^a]: outer\n [^b]: inner")}; + EXPECT_EQ(result, + "

1

\n" + "
\n
    \n" + "
  1. \n" + "

    inner \xe2\x86\xa9

    \n" + "
    2. \n
\n
\n"); +} + +TEST(footnote_nested_definition_with_the_same_label_takes_precedence) { + const auto result{ + sourcemeta::core::markdown_to_html("[^x]\n\n[^x]: one\n [^x]: two")}; + EXPECT_EQ(result, + "

1

\n" + "
\n
    \n" + "
  1. \n" + "

    two \xe2\x86\xa9

    \n" + "
    2. \n
\n
\n"); +}