Replace invalid Markdown UTF-8 per maximal subpart and test nested footnotes

config.cmake.in

@@ -352,6 +352,7 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS})
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake")
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_css.cmake")
   elseif(component STREQUAL "markdown")
+    include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake")
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_markdown.cmake")
   elseif(component STREQUAL "diff")
     include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake")

src/core/markdown/CMakeLists.txt

@@ -6,3 +6,6 @@ if(SOURCEMETA_CORE_INSTALL)
 endif()
 
 target_link_libraries(sourcemeta_core_markdown PRIVATE CMarkGFM::cmark_gfm)
+
+target_link_libraries(sourcemeta_core_markdown PRIVATE
+  sourcemeta::core::unicode)

src/core/markdown/markdown.cc

@@ -1,13 +1,16 @@
 #include <sourcemeta/core/markdown.h>
+#include <sourcemeta/core/unicode.h>
 
 #include <cmark-gfm-core-extensions.h> // cmark_gfm_core_extensions_ensure_registered
 #include <cmark-gfm-extension_api.h> // cmark_find_syntax_extension, cmark_parser_attach_syntax_extension, cmark_parser_get_syntax_extensions
 #include <cmark-gfm.h> // cmark_parser_new, cmark_parser_feed, cmark_parser_finish, cmark_parser_free, cmark_render_html, cmark_node_free
 
-#include <array>   // std::array
-#include <cstdlib> // std::free
-#include <mutex>   // std::mutex, std::scoped_lock
-#include <string>  // std::string
+#include <array>       // std::array
+#include <cstddef>     // std::size_t
+#include <cstdlib>     // std::free
+#include <mutex>       // std::mutex, std::scoped_lock
+#include <string>      // std::string
+#include <string_view> // std::string_view
 
 namespace sourcemeta::core {
 
@@ -16,6 +19,23 @@ auto markdown_to_html(const std::string_view input, const bool safe)
   [[maybe_unused]] static const bool CMARK_INITIALIZED{
       (cmark_gfm_core_extensions_ensure_registered(), true)};
 
+  // Byte sequences that are not UTF-8 become one replacement character per
+  // maximal subpart, as the Unicode Standard recommends, rather than the one
+  // replacement character per sequence that the parser would produce
+  std::size_t valid_length{0};
+  while (valid_length < input.size()) {
+    const auto length{utf8_codepoint_length(input, valid_length)};
+    if (length == 0) {
+      break;
+    }
+
+    valid_length += length;
+  }
+
+  const auto is_valid{valid_length == input.size()};
+  const auto repaired{is_valid ? std::string{} : to_valid_utf8(input)};
+  const std::string_view source{is_valid ? input : std::string_view{repaired}};
+
   // cmark-gfm toggles process-global special-character tables when syntax
   // extensions are attached and detached, so parser construction through
   // teardown cannot run concurrently
@@ -40,7 +60,7 @@ auto markdown_to_html(const std::string_view input, const bool safe)
     }
   }
 
-  cmark_parser_feed(parser, input.data(), input.size());
+  cmark_parser_feed(parser, source.data(), source.size());
   auto *document{cmark_parser_finish(parser)};
   auto *result{cmark_render_html(document, options,
                                  cmark_parser_get_syntax_extensions(parser))};

test/markdown/markdown_characters_test.cc

@@ -182,23 +182,24 @@ TEST(invalid_utf8_consecutive_continuation_bytes) {
 TEST(invalid_utf8_overlong_encoded_slash) {
   const auto result{sourcemeta::core::markdown_to_html("path\xC0\xAF"
                                                        "etc")};
-  EXPECT_EQ(result, "<p>path\xef\xbf\xbd"
+  EXPECT_EQ(result, "<p>path\xef\xbf\xbd\xef\xbf\xbd"
                     "etc</p>\n");
 }
 
 TEST(invalid_utf8_overlong_encoded_nul) {
   const auto result{sourcemeta::core::markdown_to_html("x\xE0\x80\x80y")};
-  EXPECT_EQ(result, "<p>x\xef\xbf\xbdy</p>\n");
+  EXPECT_EQ(result, "<p>x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy</p>\n");
 }
 
 TEST(invalid_utf8_encoded_surrogate) {
   const auto result{sourcemeta::core::markdown_to_html("x\xED\xA0\x80y")};
-  EXPECT_EQ(result, "<p>x\xef\xbf\xbdy</p>\n");
+  EXPECT_EQ(result, "<p>x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy</p>\n");
 }
 
 TEST(invalid_utf8_code_point_above_unicode_range) {
   const auto result{sourcemeta::core::markdown_to_html("x\xF4\x90\x80\x80y")};
-  EXPECT_EQ(result, "<p>x\xef\xbf\xbdy</p>\n");
+  EXPECT_EQ(result,
+            "<p>x\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbd\xef\xbf\xbdy</p>\n");
 }
 
 TEST(invalid_utf8_five_byte_sequence) {

test/markdown/markdown_footnotes_test.cc

@@ -239,3 +239,52 @@ TEST(footnote_label_with_space_is_not_footnote) {
       "body[^two words]\n\n[^two words]: text")};
   EXPECT_EQ(result, "<p>body<a href=\"text\">^two words</a></p>\n");
 }
+
+TEST(footnote_definition_nested_inside_another_definition) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      "[^a]\n\n[^a]: outer\n    [^b]: inner\n\n[^b]")};
+  EXPECT_EQ(result,
+            "<p><sup class=\"footnote-ref\"><a href=\"#fn-a\" id=\"fnref-a\" "
+            "data-footnote-ref>1</a></sup></p>\n"
+            "<p><sup class=\"footnote-ref\"><a href=\"#fn-b\" id=\"fnref-b\" "
+            "data-footnote-ref>2</a></sup></p>\n"
+            "<section class=\"footnotes\" data-footnotes>\n<ol>\n"
+            "<li id=\"fn-a\">\n"
+            "<p>outer <a href=\"#fnref-a\" class=\"footnote-backref\" "
+            "data-footnote-backref data-footnote-backref-idx=\"1\" "
+            "aria-label=\"Back to reference 1\">\xe2\x86\xa9</a></p>\n"
+            "</li>\n"
+            "<li id=\"fn-b\">\n"
+            "<p>inner <a href=\"#fnref-b\" class=\"footnote-backref\" "
+            "data-footnote-backref data-footnote-backref-idx=\"2\" "
+            "aria-label=\"Back to reference 2\">\xe2\x86\xa9</a></p>\n"
+            "</li>\n</ol>\n</section>\n");
+}
+
+TEST(footnote_nested_definition_referenced_without_its_parent) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      "[^b]\n\n[^a]: outer\n    [^b]: inner")};
+  EXPECT_EQ(result,
+            "<p><sup class=\"footnote-ref\"><a href=\"#fn-b\" id=\"fnref-b\" "
+            "data-footnote-ref>1</a></sup></p>\n"
+            "<section class=\"footnotes\" data-footnotes>\n<ol>\n"
+            "<li id=\"fn-b\">\n"
+            "<p>inner <a href=\"#fnref-b\" class=\"footnote-backref\" "
+            "data-footnote-backref data-footnote-backref-idx=\"1\" "
+            "aria-label=\"Back to reference 1\">\xe2\x86\xa9</a></p>\n"
+            "</li>\n</ol>\n</section>\n");
+}
+
+TEST(footnote_nested_definition_with_the_same_label_takes_precedence) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("[^x]\n\n[^x]: one\n    [^x]: two")};
+  EXPECT_EQ(result,
+            "<p><sup class=\"footnote-ref\"><a href=\"#fn-x\" id=\"fnref-x\" "
+            "data-footnote-ref>1</a></sup></p>\n"
+            "<section class=\"footnotes\" data-footnotes>\n<ol>\n"
+            "<li id=\"fn-x\">\n"
+            "<p>two <a href=\"#fnref-x\" class=\"footnote-backref\" "
+            "data-footnote-backref data-footnote-backref-idx=\"1\" "
+            "aria-label=\"Back to reference 1\">\xe2\x86\xa9</a></p>\n"
+            "</li>\n</ol>\n</section>\n");
+}