diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f94b3db1a..1c57ef021b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,6 @@ if(SOURCEMETA_CORE_CSS) endif() if(SOURCEMETA_CORE_MARKDOWN) - find_package(CMarkGFM REQUIRED) add_subdirectory(src/core/markdown) endif() diff --git a/DEPENDENCIES b/DEPENDENCIES index a5810af50f..c16c2bda05 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -2,7 +2,6 @@ vendorpull https://github.com/sourcemeta/vendorpull 89f348a97842e05aeab45d338d41 mimalloc https://github.com/microsoft/mimalloc v3.4.4 jsontestsuite https://github.com/nst/JSONTestSuite d64aefb55228d9584d3e5b2433f720ea8fd00c82 yaml-test-suite https://github.com/yaml/yaml-test-suite data-2022-01-17 -cmark-gfm https://github.com/github/cmark-gfm 587a12bb54d95ac37241377e6ddc93ea0e45439b uritemplate-test https://github.com/uri-templates/uritemplate-test 1eb27ab4462b9e5819dc47db99044f5fd1fa9bc7 pyca-cryptography https://github.com/pyca/cryptography 9747d06e83764e7f1ea4c04daf134cb8f861700b wycheproof https://github.com/C2SP/wycheproof 6d7cccd0fcb1917368579adeeac10fe802f1b521 diff --git a/cmake/FindCMarkGFM.cmake b/cmake/FindCMarkGFM.cmake deleted file mode 100644 index 0155c9839d..0000000000 --- a/cmake/FindCMarkGFM.cmake +++ /dev/null @@ -1,121 +0,0 @@ -if(NOT CMarkGFM_FOUND) - set(CMARK_GFM_DIR "${PROJECT_SOURCE_DIR}/vendor/cmark-gfm") - set(CMARK_GFM_SOURCE_DIR "${CMARK_GFM_DIR}/src") - set(CMARK_GFM_EXTENSIONS_DIR "${CMARK_GFM_DIR}/extensions") - set(CMARK_GFM_BINARY_DIR "${PROJECT_BINARY_DIR}/cmark-gfm") - - file(MAKE_DIRECTORY "${CMARK_GFM_BINARY_DIR}/include") - - set(CMARK_GFM_VERSION_MAJOR 0) - set(CMARK_GFM_VERSION_MINOR 0) - set(CMARK_GFM_VERSION_PATCH 0) - set(CMARK_GFM_VERSION_GFM 0) - - include(CheckIncludeFile) - include(CheckSymbolExists) - check_include_file(stdbool.h HAVE_STDBOOL_H) - check_symbol_exists(__builtin_expect "" HAVE___BUILTIN_EXPECT) - - if(SOURCEMETA_COMPILER_LLVM OR SOURCEMETA_COMPILER_GCC) - set(HAVE___ATTRIBUTE__ 1) - endif() - - configure_file( - "${CMARK_GFM_SOURCE_DIR}/config.h.in" - "${CMARK_GFM_BINARY_DIR}/include/config.h") - - set(_SAVED_PROJECT_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) - set(_SAVED_PROJECT_VERSION_MINOR ${PROJECT_VERSION_MINOR}) - set(_SAVED_PROJECT_VERSION_PATCH ${PROJECT_VERSION_PATCH}) - set(PROJECT_VERSION_MAJOR ${CMARK_GFM_VERSION_MAJOR}) - set(PROJECT_VERSION_MINOR ${CMARK_GFM_VERSION_MINOR}) - set(PROJECT_VERSION_PATCH ${CMARK_GFM_VERSION_PATCH}) - set(PROJECT_VERSION_GFM ${CMARK_GFM_VERSION_GFM}) - configure_file( - "${CMARK_GFM_SOURCE_DIR}/cmark-gfm_version.h.in" - "${CMARK_GFM_BINARY_DIR}/include/cmark-gfm_version.h") - set(PROJECT_VERSION_MAJOR ${_SAVED_PROJECT_VERSION_MAJOR}) - set(PROJECT_VERSION_MINOR ${_SAVED_PROJECT_VERSION_MINOR}) - set(PROJECT_VERSION_PATCH ${_SAVED_PROJECT_VERSION_PATCH}) - unset(_SAVED_PROJECT_VERSION_MAJOR) - unset(_SAVED_PROJECT_VERSION_MINOR) - unset(_SAVED_PROJECT_VERSION_PATCH) - - set(CMARK_GFM_CORE_SOURCES - "${CMARK_GFM_SOURCE_DIR}/arena.c" - "${CMARK_GFM_SOURCE_DIR}/blocks.c" - "${CMARK_GFM_SOURCE_DIR}/buffer.c" - "${CMARK_GFM_SOURCE_DIR}/cmark.c" - "${CMARK_GFM_SOURCE_DIR}/cmark_ctype.c" - "${CMARK_GFM_SOURCE_DIR}/commonmark.c" - "${CMARK_GFM_SOURCE_DIR}/footnotes.c" - "${CMARK_GFM_SOURCE_DIR}/houdini_href_e.c" - "${CMARK_GFM_SOURCE_DIR}/houdini_html_e.c" - "${CMARK_GFM_SOURCE_DIR}/houdini_html_u.c" - "${CMARK_GFM_SOURCE_DIR}/html.c" - "${CMARK_GFM_SOURCE_DIR}/inlines.c" - "${CMARK_GFM_SOURCE_DIR}/iterator.c" - "${CMARK_GFM_SOURCE_DIR}/latex.c" - "${CMARK_GFM_SOURCE_DIR}/linked_list.c" - "${CMARK_GFM_SOURCE_DIR}/man.c" - "${CMARK_GFM_SOURCE_DIR}/map.c" - "${CMARK_GFM_SOURCE_DIR}/node.c" - "${CMARK_GFM_SOURCE_DIR}/plaintext.c" - "${CMARK_GFM_SOURCE_DIR}/plugin.c" - "${CMARK_GFM_SOURCE_DIR}/references.c" - "${CMARK_GFM_SOURCE_DIR}/registry.c" - "${CMARK_GFM_SOURCE_DIR}/render.c" - "${CMARK_GFM_SOURCE_DIR}/scanners.c" - "${CMARK_GFM_SOURCE_DIR}/syntax_extension.c" - "${CMARK_GFM_SOURCE_DIR}/utf8.c" - "${CMARK_GFM_SOURCE_DIR}/xml.c") - - set(CMARK_GFM_EXTENSION_SOURCES - "${CMARK_GFM_EXTENSIONS_DIR}/autolink.c" - "${CMARK_GFM_EXTENSIONS_DIR}/core-extensions.c" - "${CMARK_GFM_EXTENSIONS_DIR}/ext_scanners.c" - "${CMARK_GFM_EXTENSIONS_DIR}/strikethrough.c" - "${CMARK_GFM_EXTENSIONS_DIR}/table.c" - "${CMARK_GFM_EXTENSIONS_DIR}/tagfilter.c" - "${CMARK_GFM_EXTENSIONS_DIR}/tasklist.c") - - # Merged into the library that uses it, so that no archive, no header and - # no CMake package of our own build of it reaches an installed consumer - add_library(cmark_gfm OBJECT - ${CMARK_GFM_CORE_SOURCES} ${CMARK_GFM_EXTENSION_SOURCES}) - sourcemeta_add_default_options(PRIVATE cmark_gfm) - - if(SOURCEMETA_COMPILER_LLVM OR SOURCEMETA_COMPILER_GCC) - target_compile_options(cmark_gfm PRIVATE -Wno-sign-conversion) - target_compile_options(cmark_gfm PRIVATE -Wno-unused-parameter) - endif() - - if(SOURCEMETA_COMPILER_MSVC) - target_compile_options(cmark_gfm PRIVATE /wd4100) - target_compile_definitions(cmark_gfm PRIVATE _CRT_SECURE_NO_WARNINGS) - endif() - - target_include_directories(cmark_gfm PRIVATE - "${CMARK_GFM_BINARY_DIR}/include" - "${CMARK_GFM_SOURCE_DIR}" - "${CMARK_GFM_EXTENSIONS_DIR}") - - target_include_directories(cmark_gfm PUBLIC - "$" - "$" - "$") - - target_compile_definitions(cmark_gfm PRIVATE HAVE_CONFIG_H) - # Marking every entry point as visible would publish this library from the - # one it is merged into, where it can collide with a real installation of it - target_compile_definitions(cmark_gfm PUBLIC CMARK_GFM_STATIC_DEFINE) - - include(GenerateExportHeader) - generate_export_header(cmark_gfm - BASE_NAME CMARK_GFM - EXPORT_FILE_NAME "${CMARK_GFM_BINARY_DIR}/include/cmark-gfm_export.h") - - add_library(CMarkGFM::cmark_gfm ALIAS cmark_gfm) - - set(CMarkGFM_FOUND ON) -endif() diff --git a/config.cmake.in b/config.cmake.in index 189f0c87d6..468d75ab4a 100644 --- a/config.cmake.in +++ b/config.cmake.in @@ -352,7 +352,17 @@ foreach(component ${SOURCEMETA_CORE_COMPONENTS}) include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_css.cmake") elseif(component STREQUAL "markdown") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_io.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_unicode.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_punycode.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_idna.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_dns.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_ip.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_uri.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_email.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_preprocessor.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_html.cmake") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_markdown.cmake") elseif(component STREQUAL "diff") include("${CMAKE_CURRENT_LIST_DIR}/sourcemeta_core_text.cmake") diff --git a/src/core/markdown/CMakeLists.txt b/src/core/markdown/CMakeLists.txt index 048612f09a..db0aa8c116 100644 --- a/src/core/markdown/CMakeLists.txt +++ b/src/core/markdown/CMakeLists.txt @@ -1,11 +1,14 @@ sourcemeta_library(NAMESPACE sourcemeta PROJECT core NAME markdown + PRIVATE_HEADERS error.h SOURCES markdown.cc) if(SOURCEMETA_CORE_INSTALL) sourcemeta_library_install(NAMESPACE sourcemeta PROJECT core NAME markdown) endif() -target_link_libraries(sourcemeta_core_markdown PRIVATE CMarkGFM::cmark_gfm) - target_link_libraries(sourcemeta_core_markdown PRIVATE - sourcemeta::core::unicode) + sourcemeta::core::email + sourcemeta::core::html + sourcemeta::core::text + sourcemeta::core::unicode + sourcemeta::core::uri) diff --git a/src/core/markdown/blocks.h b/src/core/markdown/blocks.h new file mode 100644 index 0000000000..1866754ad2 --- /dev/null +++ b/src/core/markdown/blocks.h @@ -0,0 +1,1336 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_BLOCKS_H_ +#define SOURCEMETA_CORE_MARKDOWN_BLOCKS_H_ + +#include +#include + +#include "characters.h" +#include "document.h" +#include "references.h" +#include "scanners.h" + +#include // std::min, std::max +#include // std::size_t, std::ptrdiff_t +#include // std::int32_t, std::int64_t, std::uint8_t, std::uint32_t +#include // std::memchr +#include // std::string +#include // std::string_view +#include // std::vector + +namespace sourcemeta::core::markdown { + +constexpr std::ptrdiff_t TAB_STOP{4}; +constexpr std::ptrdiff_t CODE_INDENT{4}; +// GFM section 4.10: "If there are a number of cells fewer than the number of +// cells in the header row, empty cells are inserted", which lets a small input +// turn into a huge table, so the conversion throws once a table inserts more +// empty cells than this +constexpr std::int64_t MAXIMUM_AUTOCOMPLETED_CELLS{0x80000}; +// The destinations and titles that rendered link references expand to can take +// up to this many times the size of the input or the minimum below, whichever +// is larger, before the conversion throws +constexpr std::size_t REFERENCE_SIZE_FACTOR{16}; +constexpr std::size_t MINIMUM_REFERENCE_SIZE_LIMIT{1048576}; + +struct TableCellSpan { + std::size_t offset; + std::size_t length; +}; + +inline auto is_block_type(const NodeType type) noexcept -> bool { + return type <= NodeType::TableCell; +} + +inline auto is_inline_type(const NodeType type) noexcept -> bool { + return type > NodeType::TableCell; +} + +inline auto accepts_lines(const NodeType type) noexcept -> bool { + return type == NodeType::Paragraph || type == NodeType::Heading || + type == NodeType::CodeBlock; +} + +// Whether the content is blank up to its first line ending +inline auto is_blank_until_line_end(const std::string_view content) noexcept + -> bool { + for (const auto character : content) { + if (character == '\r' || character == '\n') { + return true; + } + + if (character != ' ' && character != '\t') { + return false; + } + } + + return true; +} + +// The position of the next line feed, or the size of the input if there is +// none +inline auto find_line_feed(const std::string_view input, + const std::size_t position) noexcept -> std::size_t { + const auto *const found{static_cast( + std::memchr(input.data() + position, '\n', input.size() - position))}; + return found == nullptr ? input.size() + : static_cast(found - input.data()); +} + +// A row of the GFM table extension, where a string of several lines resolves +// to the row on its last line, whose offset is reported +inline auto parse_table_row(const std::string_view string, + std::vector &cells, + std::size_t &last_line_offset) -> bool { + cells.clear(); + last_line_offset = 0; + const auto length{string.size()}; + auto offset{scan_table_cell_end(string, 0)}; + bool expect_more_cells{true}; + while (offset < length && expect_more_cells) { + const auto cell_length{scan_table_cell(string, offset)}; + const auto pipe_length{scan_table_cell_end(string, offset + cell_length)}; + if (cell_length > 0 || pipe_length > 0) { + cells.push_back({.offset = offset, .length = cell_length}); + } + + offset += cell_length + pipe_length; + if (pipe_length > 0) { + continue; + } + + const auto row_end{scan_table_row_end(string, offset)}; + offset += row_end; + if (row_end > 0 && offset != length) { + last_line_offset = offset; + cells.clear(); + offset += scan_table_cell_end(string, offset); + } else { + expect_more_cells = false; + } + } + + return offset == length && !cells.empty(); +} + +// The block structure of GFM section 4 and 5 and of the table and task list +// extensions, following the parsing strategy of the specification appendix +class BlockParser { +public: + explicit BlockParser(Document &document) : document_{document} {} + + auto parse(const std::string_view input, const std::size_t total_size) + -> void { + this->current_ = ROOT_NODE; + this->line_number_ = 0; + this->blank_line_matched_everything_ = false; + this->skip_blank_continuations_ = false; + this->open_footnote_definitions_ = 0; + this->last_blank_container_ = NO_NODE; + const auto size{input.size()}; + const auto has_carriage_return{input.find('\r') != std::string_view::npos}; + std::size_t position{0}; + while (position < size) { + auto end{size}; + if (has_carriage_return) { + end = position; + while (end < size && input[end] != '\n' && input[end] != '\r') { + ++end; + } + } else { + end = find_line_feed(input, position); + } + + if (end < size && input[end] == '\n') { + this->process_line(input.substr(position, end + 1 - position)); + position = end + 1; + continue; + } + + this->line_buffer_.assign(input.substr(position, end - position)); + this->line_buffer_.push_back('\n'); + this->process_line(this->line_buffer_); + position = end; + if (position < size) { + ++position; + if (position < size && input[position] == '\n') { + ++position; + } + } + } + + while (this->current_ != ROOT_NODE) { + this->current_ = this->finalize(this->current_); + } + + this->finalize(ROOT_NODE); + this->document_.reference_size_limit = std::max( + total_size * REFERENCE_SIZE_FACTOR, MINIMUM_REFERENCE_SIZE_LIMIT); + } + +private: + auto node(const std::uint32_t index) -> Node & { + return this->document_.nodes[index]; + } + + [[nodiscard]] auto peek(const std::ptrdiff_t index) const noexcept -> char { + return index >= 0 && static_cast(index) < this->line_.size() + ? this->line_[static_cast(index)] + : '\0'; + } + + [[nodiscard]] auto line_size() const noexcept -> std::ptrdiff_t { + return static_cast(this->line_.size()); + } + + auto process_line(const std::string_view line) -> void { + this->line_ = line; + this->line_end_ = this->line_size(); + this->offset_ = 0; + this->column_ = 0; + this->first_nonspace_ = 0; + this->first_nonspace_column_ = 0; + this->thematic_break_kill_position_ = 0; + this->indent_ = 0; + this->blank_ = false; + this->partially_consumed_tab_ = false; + if (this->line_number_ == 0 && line.starts_with("\xEF\xBB\xBF")) { + this->offset_ += 3; + } + + ++this->line_number_; + const auto starting_tip{this->current_}; + const auto starting_nodes{this->document_.nodes.size()}; + const auto starting_offset{static_cast(this->offset_)}; + // An open footnote definition only continues on an unindented blank line + // when the line is empty, and an item without blocks does not continue on + // a blank line at all + this->skip_blank_continuations_ = + this->blank_line_matched_everything_ && + is_blank_until_line_end(line.substr(starting_offset)) && + (this->open_footnote_definitions_ == 0 || this->peek(0) == '\n' || + (this->peek(0) == '\r' && this->peek(1) == '\n')) && + !(this->node(starting_tip).type == NodeType::Item && + this->node(starting_tip).first_child == NO_NODE); + this->blank_line_matched_everything_ = false; + bool all_matched{true}; + const auto last_matched{this->check_open_blocks(all_matched)}; + if (last_matched == NO_NODE) { + return; + } + + auto container{last_matched}; + this->open_new_blocks(container, all_matched); + this->add_text_to_container(container, last_matched); + this->blank_line_matched_everything_ = + all_matched && last_matched == starting_tip && + this->current_ == starting_tip && + this->document_.nodes.size() == starting_nodes && this->blank_ && + is_blank_until_line_end(line.substr(starting_offset)); + } + + auto find_first_nonspace() -> void { + auto chars_to_tab{TAB_STOP - (this->column_ % TAB_STOP)}; + if (this->first_nonspace_ <= this->offset_) { + this->first_nonspace_ = this->offset_; + this->first_nonspace_column_ = this->column_; + while (true) { + const auto character{this->peek(this->first_nonspace_)}; + if (character == ' ') { + ++this->first_nonspace_; + ++this->first_nonspace_column_; + --chars_to_tab; + if (chars_to_tab == 0) { + chars_to_tab = TAB_STOP; + } + } else if (character == '\t') { + ++this->first_nonspace_; + this->first_nonspace_column_ += chars_to_tab; + chars_to_tab = TAB_STOP; + } else { + break; + } + } + } + + this->indent_ = this->first_nonspace_column_ - this->column_; + this->blank_ = is_line_end(this->peek(this->first_nonspace_)); + } + + // Advance by a number of bytes, or by a number of columns where a tab might + // only be partially consumed + auto advance_offset(std::ptrdiff_t count, const bool columns) -> void { + // Without tabs, advancing by bytes also advances by the same columns + if (!columns && count > 0 && this->offset_ + count <= this->line_size() && + std::memchr(this->line_.data() + this->offset_, '\t', + static_cast(count)) == nullptr) { + this->partially_consumed_tab_ = false; + this->offset_ += count; + this->column_ += count; + return; + } + + while (count > 0) { + const auto character{this->peek(this->offset_)}; + if (character == '\0') { + break; + } + + if (character != '\t') { + this->partially_consumed_tab_ = false; + ++this->offset_; + ++this->column_; + --count; + continue; + } + + const auto chars_to_tab{TAB_STOP - (this->column_ % TAB_STOP)}; + if (columns) { + this->partially_consumed_tab_ = chars_to_tab > count; + const auto chars_to_advance{std::min(count, chars_to_tab)}; + this->column_ += chars_to_advance; + this->offset_ += this->partially_consumed_tab_ ? 0 : 1; + count -= chars_to_advance; + } else { + this->partially_consumed_tab_ = false; + this->column_ += chars_to_tab; + ++this->offset_; + --count; + } + } + } + + auto advance_to_line_end() -> void { + this->advance_offset(this->line_size() - 1 - this->offset_, false); + } + + auto advance_to_first_nonspace() -> void { + this->advance_offset(this->first_nonspace_ - this->offset_, false); + } + + auto last_child_is_open(const std::uint32_t index) -> bool { + const auto last_child{this->node(index).last_child}; + return last_child != NO_NODE && has_flag(this->node(last_child), FLAG_OPEN); + } + + auto can_contain(const std::uint32_t parent, const NodeType child) -> bool { + const auto &parent_node{this->node(parent)}; + switch (parent_node.type) { + case NodeType::Document: + case NodeType::BlockQuote: + case NodeType::FootnoteDefinition: + return is_block_type(child) && child != NodeType::Item; + case NodeType::Item: + return has_flag(parent_node, FLAG_TASK) || + (is_block_type(child) && child != NodeType::Item); + case NodeType::List: + return child == NodeType::Item; + case NodeType::Table: + return child == NodeType::TableRow; + case NodeType::TableRow: + return child == NodeType::TableCell; + case NodeType::Paragraph: + case NodeType::Heading: + return is_inline_type(child); + case NodeType::CodeBlock: + case NodeType::HTMLBlock: + case NodeType::ThematicBreak: + case NodeType::TableCell: + case NodeType::Text: + case NodeType::SoftBreak: + case NodeType::LineBreak: + case NodeType::Code: + case NodeType::HTMLInline: + case NodeType::Emphasis: + case NodeType::Strong: + case NodeType::Link: + case NodeType::Image: + case NodeType::FootnoteReference: + case NodeType::Strikethrough: + return false; + } + + return false; + } + + auto add_child(std::uint32_t parent, const NodeType type) -> std::uint32_t { + while (!this->can_contain(parent, type)) { + parent = this->finalize(parent); + } + + const auto child{this->document_.create(type)}; + this->node(child).flags = FLAG_OPEN; + this->document_.append_child(parent, child); + if (type == NodeType::FootnoteDefinition) { + ++this->open_footnote_definitions_; + } + + return child; + } + + auto add_line(const std::uint32_t index) -> void { + auto &content{this->document_.content}; + auto &target{this->node(index)}; + if (target.content_length == 0) { + target.content_offset = static_cast(content.size()); + } else if (target.content_offset + target.content_length != + content.size()) { + this->relocation_buffer_.assign(this->document_.content_of(target)); + target.content_offset = static_cast(content.size()); + content.append(this->relocation_buffer_); + } + + const auto before{content.size()}; + if (this->partially_consumed_tab_) { + ++this->offset_; + const auto chars_to_tab{TAB_STOP - (this->column_ % TAB_STOP)}; + content.append(static_cast(chars_to_tab), ' '); + } + + if (this->line_end_ > this->offset_) { + content.append(this->line_.substr( + static_cast(this->offset_), + static_cast(this->line_end_ - this->offset_))); + } + + target.content_length += + static_cast(content.size() - before); + } + + auto resolve_reference_definitions(const std::uint32_t index) -> bool { + const auto content{this->document_.content_of(this->node(index))}; + std::size_t position{0}; + while (position < content.size() && content[position] == '[') { + const auto consumed{ + parse_reference_definition(this->document_, content.substr(position), + this->label_buffer_, this->value_buffer_)}; + if (consumed == 0) { + break; + } + + position += consumed; + } + + auto &target{this->node(index)}; + target.content_offset += static_cast(position); + target.content_length -= static_cast(position); + return !is_blank_until_line_end(content.substr(position)); + } + + auto ends_with_blank_line(std::uint32_t index) -> bool { + while (true) { + auto ¤t{this->node(index)}; + if (has_flag(current, FLAG_LAST_LINE_CHECKED)) { + return has_flag(current, FLAG_LAST_LINE_BLANK); + } + + set_flag(current, FLAG_LAST_LINE_CHECKED, true); + if ((current.type == NodeType::List || current.type == NodeType::Item) && + current.last_child != NO_NODE) { + index = current.last_child; + continue; + } + + return has_flag(current, FLAG_LAST_LINE_BLANK); + } + } + + auto finalize_list(const std::uint32_t index) -> void { + bool tight{true}; + for (auto item{this->node(index).first_child}; item != NO_NODE && tight; + item = this->node(item).next) { + if (has_flag(this->node(item), FLAG_LAST_LINE_BLANK) && + this->node(item).next != NO_NODE) { + tight = false; + break; + } + + for (auto child{this->node(item).first_child}; child != NO_NODE; + child = this->node(child).next) { + if ((this->node(item).next != NO_NODE || + this->node(child).next != NO_NODE) && + this->ends_with_blank_line(child)) { + tight = false; + break; + } + } + } + + set_flag(this->node(index), FLAG_TIGHT, tight); + } + + auto finalize_code_block(const std::uint32_t index) -> void { + auto &target{this->node(index)}; + const auto content{this->document_.content_of(target)}; + if (!has_flag(target, FLAG_FENCED)) { + auto end{content.size()}; + while (end > 0 && (content[end - 1] == ' ' || content[end - 1] == '\t' || + is_line_end(content[end - 1]))) { + --end; + } + + if (end == 0) { + target.content_length = 0; + target.literal = "\n"; + return; + } + + const auto line_end{content.find_first_of("\r\n", end - 1)}; + if (line_end == std::string_view::npos) { + this->value_buffer_.assign(content); + this->value_buffer_.push_back('\n'); + target.literal = this->document_.strings.store(this->value_buffer_); + return; + } + + if (content[line_end] == '\n') { + target.content_length = static_cast(line_end + 1); + return; + } + + this->value_buffer_.assign(content.substr(0, line_end)); + this->value_buffer_.push_back('\n'); + target.literal = this->document_.strings.store(this->value_buffer_); + + return; + } + + auto position{content.find_first_of("\r\n")}; + if (position == std::string_view::npos) { + position = content.size(); + } + + this->value_buffer_.clear(); + decode_escapes_and_references(this->value_buffer_, + content.substr(0, position)); + target.title = this->document_.strings.store( + sourcemeta::core::trim(this->value_buffer_, is_space)); + if (character_at(content, position) == '\r') { + ++position; + } + + if (character_at(content, position) == '\n') { + ++position; + } + + position = std::min(position, content.size()); + target.content_offset += static_cast(position); + target.content_length -= static_cast(position); + } + + auto finalize(const std::uint32_t index) -> std::uint32_t { + const auto parent{this->node(index).parent}; + set_flag(this->node(index), FLAG_OPEN, false); + const auto type{this->node(index).type}; + if (type == NodeType::FootnoteDefinition) { + --this->open_footnote_definitions_; + } + + if (type == NodeType::Paragraph) { + if (!this->resolve_reference_definitions(index)) { + this->document_.unlink(index); + } + } else if (type == NodeType::CodeBlock) { + this->finalize_code_block(index); + } else if (type == NodeType::List) { + this->finalize_list(index); + } + + return parent; + } + + auto parse_block_quote_prefix() -> bool { + if (this->indent_ > 3 || this->peek(this->first_nonspace_) != '>') { + return false; + } + + this->advance_offset(this->indent_ + 1, true); + if (is_space_or_tab(this->peek(this->offset_))) { + this->advance_offset(1, true); + } + + return true; + } + + auto parse_footnote_definition_prefix() -> bool { + if (this->indent_ >= CODE_INDENT) { + this->advance_offset(CODE_INDENT, true); + return true; + } + + return this->peek(0) == '\n' || + (this->peek(0) == '\r' && this->peek(1) == '\n'); + } + + auto parse_item_prefix(const std::uint32_t index) -> bool { + const auto &data{this->document_.lists[this->node(index).data]}; + const auto width{static_cast(data.marker_offset) + + static_cast(data.padding)}; + if (this->indent_ >= width) { + this->advance_offset(width, true); + return true; + } + + if (this->blank_ && this->node(index).first_child != NO_NODE) { + this->advance_to_first_nonspace(); + return true; + } + + return false; + } + + auto parse_code_block_prefix(const std::uint32_t index, bool &should_continue) + -> bool { + if (!has_flag(this->node(index), FLAG_FENCED)) { + if (this->indent_ >= CODE_INDENT) { + this->advance_offset(CODE_INDENT, true); + return true; + } + + if (this->blank_) { + this->advance_to_first_nonspace(); + return true; + } + + return false; + } + + std::size_t matched{0}; + if (this->indent_ <= 3 && + static_cast(this->peek(this->first_nonspace_)) == + this->node(index).data) { + matched = scan_close_code_fence( + this->line_, static_cast(this->first_nonspace_)); + } + + if (matched > 0 && matched >= this->node(index).level) { + should_continue = false; + this->advance_offset(static_cast(matched), false); + this->current_ = this->finalize(index); + return false; + } + + auto remaining{static_cast(this->node(index).extra)}; + while (remaining > 0 && is_space_or_tab(this->peek(this->offset_))) { + this->advance_offset(1, true); + --remaining; + } + + return true; + } + + auto table_row_matches() -> bool { + std::size_t last_line_offset{0}; + this->cached_row_valid_ = parse_table_row( + this->line_.substr(static_cast(this->first_nonspace_)), + this->cells_, last_line_offset); + this->cached_row_line_ = this->line_number_; + this->cached_row_offset_ = this->first_nonspace_; + return this->cached_row_valid_; + } + + auto continues_container(const std::uint32_t index, bool &should_continue) + -> bool { + switch (this->node(index).type) { + case NodeType::BlockQuote: + return this->parse_block_quote_prefix(); + case NodeType::Item: + return this->parse_item_prefix(index); + case NodeType::CodeBlock: + return this->parse_code_block_prefix(index, should_continue); + case NodeType::HTMLBlock: + return this->node(index).level <= 5 || !this->blank_; + case NodeType::Paragraph: + return !this->blank_; + case NodeType::FootnoteDefinition: + return this->parse_footnote_definition_prefix(); + case NodeType::Table: + return this->table_row_matches(); + // GFM section 4.1 makes a thematic break out of a single line, so it does + // not continue, and a blank line after it reaches the blocks around it + case NodeType::Heading: + case NodeType::ThematicBreak: + case NodeType::TableRow: + case NodeType::TableCell: + return false; + case NodeType::Document: + case NodeType::List: + case NodeType::Text: + case NodeType::SoftBreak: + case NodeType::LineBreak: + case NodeType::Code: + case NodeType::HTMLInline: + case NodeType::Emphasis: + case NodeType::Strong: + case NodeType::Link: + case NodeType::Image: + case NodeType::FootnoteReference: + case NodeType::Strikethrough: + return true; + } + + return true; + } + + auto check_open_blocks(bool &all_matched) -> std::uint32_t { + bool should_continue{true}; + all_matched = true; + auto container{ROOT_NODE}; + while (this->last_child_is_open(container)) { + container = this->node(container).last_child; + this->find_first_nonspace(); + // Once a blank line has no indentation left, every remaining open block + // continues without consuming anything if the previous line was blank, + // matched every open block, and changed nothing, which spares walking + // through deeply nested blocks for every blank line + if (this->skip_blank_continuations_ && this->blank_ && + this->indent_ == 0) { + return this->current_; + } + + if (!this->continues_container(container, should_continue)) { + all_matched = false; + break; + } + } + + if (!all_matched) { + container = this->node(container).parent; + } + + return should_continue ? container : NO_NODE; + } + + auto scan_thematic_break(const std::ptrdiff_t position) -> std::ptrdiff_t { + const auto marker{this->peek(position)}; + if (marker != '*' && marker != '_' && marker != '-') { + this->thematic_break_kill_position_ = position; + return 0; + } + + std::ptrdiff_t count{1}; + auto index{position}; + char next{'\0'}; + while (true) { + ++index; + next = this->peek(index); + if (next == '\0') { + break; + } + + if (next == marker) { + ++count; + } else if (next != ' ' && next != '\t') { + break; + } + } + + if (count >= 3 && (next == '\r' || next == '\n')) { + return index - position + 1; + } + + this->thematic_break_kill_position_ = index; + return 0; + } + + auto parse_list_marker(const std::ptrdiff_t position, + const bool interrupts_paragraph, ListData &data) + -> std::ptrdiff_t { + auto cursor{position}; + const auto marker{this->peek(cursor)}; + if (marker == '*' || marker == '-' || marker == '+') { + ++cursor; + if (!is_space(this->peek(cursor))) { + return 0; + } + + if (interrupts_paragraph) { + auto index{cursor}; + while (is_space_or_tab(this->peek(index))) { + ++index; + } + + if (this->peek(index) == '\n') { + return 0; + } + } + + data = ListData{}; + data.bullet_character = marker; + return cursor - position; + } + + if (!sourcemeta::core::is_digit(marker)) { + return 0; + } + + std::int32_t start{0}; + std::int32_t digits{0}; + while (true) { + start = (10 * start) + (this->peek(cursor) - '0'); + ++cursor; + ++digits; + if (digits >= 9 || !sourcemeta::core::is_digit(this->peek(cursor))) { + break; + } + } + + if (interrupts_paragraph && start != 1) { + return 0; + } + + const auto delimiter{this->peek(cursor)}; + if (delimiter != '.' && delimiter != ')') { + return 0; + } + + ++cursor; + if (!is_space(this->peek(cursor))) { + return 0; + } + + if (interrupts_paragraph) { + auto index{cursor}; + while (is_space_or_tab(this->peek(index))) { + ++index; + } + + if (is_line_end(this->peek(index))) { + return 0; + } + } + + data = ListData{}; + data.start = start; + data.delimiter = delimiter; + data.ordered = true; + return cursor - position; + } + + auto append_table_cell_content(const std::uint32_t cell, + const std::string_view raw) -> void { + const auto text{sourcemeta::core::trim(raw, is_space)}; + auto &content{this->document_.content}; + auto &target{this->node(cell)}; + target.content_offset = static_cast(content.size()); + std::size_t run_start{0}; + for (std::size_t index{0}; index + 1 < text.size(); ++index) { + if (text[index] == '\\' && text[index + 1] == '|') { + content.append(text.substr(run_start, index - run_start)); + run_start = index + 1; + } + } + + content.append(text.substr(run_start)); + + target.content_length = + static_cast(content.size() - target.content_offset); + } + + auto try_opening_table_header(const std::uint32_t paragraph) + -> std::uint32_t { + this->cached_row_line_ = -1; + if (has_flag(this->node(paragraph), FLAG_TABLE_VISITED)) { + return paragraph; + } + + const auto nonspace{static_cast(this->first_nonspace_)}; + std::size_t last_line_offset{0}; + if (scan_table_start(this->line_, nonspace) == 0 || + !parse_table_row(this->line_.substr(nonspace), this->cells_, + last_line_offset)) { + return paragraph; + } + + this->header_buffer_.assign( + this->document_.content_of(this->node(paragraph))); + if (!parse_table_row(this->header_buffer_, this->header_cells_, + last_line_offset) || + this->header_cells_.size() != this->cells_.size()) { + set_flag(this->node(paragraph), FLAG_TABLE_VISITED, true); + return paragraph; + } + + if (last_line_offset > 0) { + const auto preceding{this->document_.create(NodeType::Paragraph)}; + this->append_table_cell_content( + preceding, + std::string_view{this->header_buffer_}.substr(0, last_line_offset)); + this->document_.insert_before(paragraph, preceding); + } + + const auto table_index{ + static_cast(this->document_.tables.size())}; + this->document_.tables.push_back( + {.alignments_offset = + static_cast(this->document_.alignments.size()), + .columns = static_cast(this->header_cells_.size()), + .rows = 1, + .nonempty_cells = + static_cast(this->header_cells_.size())}); + for (const auto &cell : this->cells_) { + const auto marker{sourcemeta::core::trim( + this->line_.substr(nonspace + cell.offset, cell.length), is_space)}; + const auto left{!marker.empty() && marker.front() == ':'}; + const auto right{!marker.empty() && marker.back() == ':'}; + std::uint8_t alignment{0}; + if (left && right) { + alignment = 'c'; + } else if (left) { + alignment = 'l'; + } else if (right) { + alignment = 'r'; + } + + this->document_.alignments.push_back(alignment); + } + + auto &table{this->node(paragraph)}; + table.type = NodeType::Table; + table.data = table_index; + const auto row{this->add_child(paragraph, NodeType::TableRow)}; + set_flag(this->node(row), FLAG_HEADER, true); + for (std::size_t index{0}; index < this->header_cells_.size(); ++index) { + const auto cell{this->add_child(row, NodeType::TableCell)}; + this->node(cell).data = static_cast(index); + this->append_table_cell_content( + cell, std::string_view{this->header_buffer_}.substr( + this->header_cells_[index].offset, + this->header_cells_[index].length)); + } + + this->advance_to_line_end(); + return paragraph; + } + + auto try_opening_table_row(const std::uint32_t table) -> std::uint32_t { + if (this->blank_) { + return NO_NODE; + } + + const auto table_index{this->node(table).data}; + + const auto row{this->add_child(table, NodeType::TableRow)}; + const auto nonspace{static_cast(this->first_nonspace_)}; + std::size_t last_line_offset{0}; + const auto cached{this->cached_row_line_ == this->line_number_ && + this->cached_row_offset_ == this->first_nonspace_}; + if (!(cached ? this->cached_row_valid_ + : parse_table_row(this->line_.substr(nonspace), this->cells_, + last_line_offset))) { + this->document_.unlink(row); + return NO_NODE; + } + + const auto columns{this->document_.tables[table_index].columns}; + std::size_t index{0}; + for (; index < this->cells_.size() && index < columns; ++index) { + const auto cell{this->add_child(row, NodeType::TableCell)}; + this->node(cell).data = static_cast(index); + this->append_table_cell_content( + cell, this->line_.substr(nonspace + this->cells_[index].offset, + this->cells_[index].length)); + } + + auto &data{this->document_.tables[table_index]}; + data.rows += 1; + data.nonempty_cells += static_cast(index); + if ((static_cast(data.columns) * data.rows) - + data.nonempty_cells > + MAXIMUM_AUTOCOMPLETED_CELLS) { + throw sourcemeta::core::MarkdownError{ + "The table inserts more empty cells than its bound"}; + } + for (; index < columns; ++index) { + const auto cell{this->add_child(row, NodeType::TableCell)}; + this->node(cell).data = static_cast(index); + } + + this->advance_to_line_end(); + return row; + } + + // A list item whose first block is a paragraph that starts with a task list + // item marker of GFM section 5.3, which the item can only get while it has + // no block yet + auto open_task_list_item(const std::uint32_t item) -> void { + if (this->node(item).type != NodeType::Item || + this->node(item).first_child != NO_NODE || + !scan_task_list_marker( + this->line_, static_cast(this->first_nonspace_))) { + return; + } + + set_flag(this->node(item), FLAG_TASK, true); + // GFM section 5.3: "If the character between the brackets is a whitespace + // character, the checkbox is unchecked. Otherwise, the checkbox is checked" + const auto state{this->peek(this->first_nonspace_ + 1)}; + set_flag(this->node(item), FLAG_CHECKED, state == 'x' || state == 'X'); + this->advance_offset(this->first_nonspace_ + 3 - this->offset_, false); + } + + auto register_footnote_definition(const std::uint32_t definition) -> void { + for (auto ancestor{this->node(definition).parent}; ancestor != NO_NODE; + ancestor = this->node(ancestor).parent) { + if (this->node(ancestor).type == NodeType::FootnoteDefinition) { + this->document_.nested_footnote_definitions = true; + break; + } + } + + this->document_.footnote_definition_nodes.push_back(definition); + } + + auto open_block(std::uint32_t &container, const NodeType container_type, + const bool all_matched, const bool maybe_lazy) -> bool { + const auto indented{this->indent_ >= CODE_INDENT}; + const auto nonspace{static_cast(this->first_nonspace_)}; + const auto character{this->peek(this->first_nonspace_)}; + if (!indented && this->peek(this->first_nonspace_) == '>') { + this->advance_offset(this->first_nonspace_ + 1 - this->offset_, false); + if (is_space_or_tab(this->peek(this->offset_))) { + this->advance_offset(1, true); + } + + container = this->add_child(container, NodeType::BlockQuote); + return true; + } + + if (!indented && character == '#') { + const auto matched{scan_atx_heading_start(this->line_, nonspace)}; + if (matched > 0) { + this->advance_offset(this->first_nonspace_ + + static_cast(matched) - + this->offset_, + false); + container = this->add_child(container, NodeType::Heading); + auto hash{this->line_.find('#', nonspace)}; + std::uint8_t level{0}; + while (hash < this->line_.size() && this->line_[hash] == '#') { + ++level; + ++hash; + } + + this->node(container).level = level; + return true; + } + } + + if (!indented && (character == '`' || character == '~')) { + const auto matched{scan_open_code_fence(this->line_, nonspace)}; + if (matched > 0) { + container = this->add_child(container, NodeType::CodeBlock); + auto &code{this->node(container)}; + set_flag(code, FLAG_FENCED, true); + code.data = + static_cast(this->peek(this->first_nonspace_)); + code.level = + static_cast(std::min(matched, 255)); + code.extra = + static_cast(this->first_nonspace_ - this->offset_); + this->advance_offset(this->first_nonspace_ + + static_cast(matched) - + this->offset_, + false); + return true; + } + } + + if (!indented && character == '<') { + auto condition{scan_html_block_start(this->line_, nonspace)}; + if (condition == 0 && container_type != NodeType::Paragraph && + scan_html_block_start_7(this->line_, nonspace)) { + condition = 7; + } + + if (condition > 0) { + container = this->add_child(container, NodeType::HTMLBlock); + this->node(container).level = static_cast(condition); + return true; + } + } + + if (!indented && container_type == NodeType::Paragraph && + (character == '=' || character == '-')) { + const auto level{scan_setext_heading_line(this->line_, nonspace)}; + if (level > 0) { + if (this->resolve_reference_definitions(container)) { + auto &heading{this->node(container)}; + heading.type = NodeType::Heading; + heading.level = level; + set_flag(heading, FLAG_SETEXT, true); + this->advance_to_line_end(); + } + + return true; + } + } + + if (!indented && + (character == '*' || character == '_' || character == '-') && + !(container_type == NodeType::Paragraph && !all_matched) && + this->thematic_break_kill_position_ <= this->first_nonspace_ && + this->scan_thematic_break(this->first_nonspace_) > 0) { + container = this->add_child(container, NodeType::ThematicBreak); + this->advance_to_line_end(); + return true; + } + + if (!indented && character == '[') { + const auto matched{scan_footnote_definition(this->line_, nonspace)}; + if (matched > 0) { + auto label{this->line_.substr(nonspace + 2, matched - 2)}; + while (!label.empty() && label.back() != ']') { + label.remove_suffix(1); + } + + if (!label.empty()) { + label.remove_suffix(1); + } + + const auto stored{this->document_.strings.store(label)}; + this->advance_offset(this->first_nonspace_ + + static_cast(matched) - + this->offset_, + false); + container = this->add_child(container, NodeType::FootnoteDefinition); + this->register_footnote_definition(container); + this->node(container).literal = stored; + return true; + } + } + + if ((character == '*' || character == '-' || character == '+' || + sourcemeta::core::is_digit(character)) && + (!indented || container_type == NodeType::List) && + this->indent_ < CODE_INDENT) { + ListData data{}; + const auto matched{this->parse_list_marker( + this->first_nonspace_, container_type == NodeType::Paragraph, data)}; + if (matched > 0) { + this->advance_offset(this->first_nonspace_ + matched - this->offset_, + false); + const auto saved_partially_consumed_tab{this->partially_consumed_tab_}; + const auto saved_offset{this->offset_}; + const auto saved_column{this->column_}; + while (this->column_ - saved_column <= 5 && + is_space_or_tab(this->peek(this->offset_))) { + this->advance_offset(1, true); + } + + const auto spaces{this->column_ - saved_column}; + if (spaces >= 5 || spaces < 1 || + is_line_end(this->peek(this->offset_))) { + data.padding = static_cast(matched + 1); + this->offset_ = saved_offset; + this->column_ = saved_column; + this->partially_consumed_tab_ = saved_partially_consumed_tab; + if (spaces > 0) { + this->advance_offset(1, true); + } + } else { + data.padding = static_cast(matched + spaces); + } + + data.marker_offset = static_cast(this->indent_); + auto &lists{this->document_.lists}; + if (container_type != NodeType::List || + !lists_match(lists[this->node(container).data], data)) { + container = this->add_child(container, NodeType::List); + this->node(container).data = static_cast(lists.size()); + lists.push_back(data); + } + + data.start_line = this->line_number_; + container = this->add_child(container, NodeType::Item); + this->node(container).data = static_cast(lists.size()); + lists.push_back(data); + return true; + } + } + + if (indented && !maybe_lazy && !this->blank_) { + this->advance_offset(CODE_INDENT, true); + container = this->add_child(container, NodeType::CodeBlock); + return true; + } + + if (!indented && container_type == NodeType::Paragraph) { + if (character == '|' || character == ':' || character == '-' || + character == '\v' || character == '\f') { + container = this->try_opening_table_header(container); + } + return true; + } + + if (!indented && container_type == NodeType::Table) { + const auto row{this->try_opening_table_row(container)}; + if (row != NO_NODE) { + container = row; + return true; + } + } + + this->open_task_list_item(container); + return false; + } + + static auto lists_match(const ListData &list, const ListData &item) noexcept + -> bool { + return list.ordered == item.ordered && list.delimiter == item.delimiter && + list.bullet_character == item.bullet_character; + } + + auto open_new_blocks(std::uint32_t &container, const bool all_matched) + -> void { + bool maybe_lazy{this->node(this->current_).type == NodeType::Paragraph}; + auto container_type{this->node(container).type}; + while (container_type != NodeType::CodeBlock && + container_type != NodeType::HTMLBlock) { + this->find_first_nonspace(); + if (!this->open_block(container, container_type, all_matched, + maybe_lazy)) { + break; + } + + container_type = this->node(container).type; + if (accepts_lines(container_type)) { + break; + } + + maybe_lazy = false; + } + } + + auto chop_trailing_hashes() -> void { + auto length{this->line_end_}; + while (length > 0 && is_space(this->peek(length - 1))) { + --length; + } + + auto position{length - 1}; + while (position >= 0 && this->peek(position) == '#') { + --position; + } + + if (position != length - 1 && position >= 0 && + is_space_or_tab(this->peek(position))) { + length = position; + while (length > 0 && is_space(this->peek(length - 1))) { + --length; + } + } + + this->line_end_ = length; + } + + auto add_text_to_container(std::uint32_t container, + const std::uint32_t last_matched) -> void { + this->find_first_nonspace(); + if (this->blank_ && this->node(container).last_child != NO_NODE) { + set_flag(this->node(this->node(container).last_child), + FLAG_LAST_LINE_BLANK, true); + } + + const auto container_type{this->node(container).type}; + // A table only contains the lines of its rows, as GFM section 4.10 says + // that "The table is broken at the first empty line", so it never ends with + // a blank line, even when its delimiter row consumed the rest of its line + const auto last_line_blank{ + this->blank_ && container_type != NodeType::BlockQuote && + container_type != NodeType::Heading && + container_type != NodeType::ThematicBreak && + container_type != NodeType::Table && + container_type != NodeType::TableRow && + !(container_type == NodeType::CodeBlock && + has_flag(this->node(container), FLAG_FENCED)) && + !(container_type == NodeType::Item && + this->node(container).first_child == NO_NODE && + this->document_.lists[this->node(container).data].start_line == + this->line_number_)}; + // The ancestors of the container do not end with a blank line. The only + // open block that can is the container of the previous line, which is the + // deepest open block when this line starts, so it is an ancestor exactly + // when this line matched it and opened blocks under it + const auto previous_blank{this->last_blank_container_}; + if (previous_blank != NO_NODE && previous_blank == last_matched && + previous_blank != container && + has_flag(this->node(previous_blank), FLAG_OPEN)) { + set_flag(this->node(previous_blank), FLAG_LAST_LINE_BLANK, false); + } + + set_flag(this->node(container), FLAG_LAST_LINE_BLANK, last_line_blank); + this->last_blank_container_ = last_line_blank ? container : NO_NODE; + + if (this->current_ != last_matched && container == last_matched && + !this->blank_ && + this->node(this->current_).type == NodeType::Paragraph) { + this->add_line(this->current_); + return; + } + + while (this->current_ != last_matched) { + this->current_ = this->finalize(this->current_); + } + + if (container_type == NodeType::CodeBlock) { + this->add_line(container); + } else if (container_type == NodeType::HTMLBlock) { + this->add_line(container); + const auto condition{this->node(container).level}; + if (condition <= 5 && + scan_html_block_end(this->line_, + static_cast(this->first_nonspace_), + condition)) { + container = this->finalize(container); + } + } else if (this->blank_) { + // Blank lines only separate blocks + } else if (accepts_lines(container_type)) { + if (container_type == NodeType::Heading && + !has_flag(this->node(container), FLAG_SETEXT)) { + this->chop_trailing_hashes(); + } + + this->advance_to_first_nonspace(); + this->add_line(container); + } else { + container = this->add_child(container, NodeType::Paragraph); + this->advance_to_first_nonspace(); + this->add_line(container); + } + + this->current_ = container; + } + + Document &document_; + std::string_view line_; + std::ptrdiff_t line_end_{0}; + std::string line_buffer_; + std::string relocation_buffer_; + std::string label_buffer_; + std::string value_buffer_; + std::string header_buffer_; + std::vector cells_; + std::vector header_cells_; + std::int32_t cached_row_line_{-1}; + std::ptrdiff_t cached_row_offset_{0}; + bool cached_row_valid_{false}; + std::uint32_t current_{ROOT_NODE}; + std::int32_t line_number_{0}; + std::ptrdiff_t offset_{0}; + std::ptrdiff_t column_{0}; + std::ptrdiff_t first_nonspace_{0}; + std::ptrdiff_t first_nonspace_column_{0}; + std::ptrdiff_t thematic_break_kill_position_{0}; + std::ptrdiff_t indent_{0}; + bool blank_{false}; + bool partially_consumed_tab_{false}; + bool blank_line_matched_everything_{false}; + bool skip_blank_continuations_{false}; + std::size_t open_footnote_definitions_{0}; + std::uint32_t last_blank_container_{NO_NODE}; +}; + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/characters.h b/src/core/markdown/characters.h new file mode 100644 index 0000000000..e1ed0fcd48 --- /dev/null +++ b/src/core/markdown/characters.h @@ -0,0 +1,270 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_CHARACTERS_H_ +#define SOURCEMETA_CORE_MARKDOWN_CHARACTERS_H_ + +#include +#include +#include + +#include // std::min +#include // std::size_t +#include // std::string +#include // std::string_view + +namespace sourcemeta::core::markdown { + +// The ASCII whitespace that the parser trims is only the tab, the line feed, +// the carriage return, and the space +inline auto is_space(const char character) noexcept -> bool { + return character == ' ' || character == '\t' || character == '\n' || + character == '\r'; +} + +// GFM section 2.1: "A whitespace character is a space (U+0020), tab (U+0009), +// newline (U+000A), line tabulation (U+000B), form feed (U+000C), or carriage +// return (U+000D)" +inline auto is_whitespace_character(const char character) noexcept -> bool { + return character == ' ' || character == '\t' || character == '\n' || + character == '\v' || character == '\f' || character == '\r'; +} + +// GFM section 6.6: a link destination that is not enclosed in angle brackets +// "does not include ASCII space or control characters" +inline auto is_space_or_control(const char character) noexcept -> bool { + const auto byte{static_cast(character)}; + return byte <= 0x20 || byte == 0x7F; +} + +inline auto is_space_or_tab(const char character) noexcept -> bool { + return character == ' ' || character == '\t'; +} + +inline auto is_line_end(const char character) noexcept -> bool { + return character == '\n' || character == '\r'; +} + +// The Unicode whitespace of GFM section 2.1 +inline auto is_unicode_whitespace(const char32_t codepoint) noexcept -> bool { + if (codepoint < 0x80) { + return codepoint == U'\t' || codepoint == U'\n' || codepoint == U'\f' || + codepoint == U'\r' || codepoint == U' '; + } + + return sourcemeta::core::general_category(codepoint) == + sourcemeta::core::GeneralCategory::SpaceSeparator; +} + +// The punctuation of GFM section 2.1 +inline auto is_unicode_punctuation(const char32_t codepoint) noexcept -> bool { + if (codepoint < 0x80) { + return sourcemeta::core::is_punctuation(static_cast(codepoint)); + } + + const auto category{sourcemeta::core::general_category(codepoint)}; + return category == sourcemeta::core::GeneralCategory::ConnectorPunctuation || + category == sourcemeta::core::GeneralCategory::DashPunctuation || + category == sourcemeta::core::GeneralCategory::OpenPunctuation || + category == sourcemeta::core::GeneralCategory::ClosePunctuation || + category == sourcemeta::core::GeneralCategory::InitialPunctuation || + category == sourcemeta::core::GeneralCategory::FinalPunctuation || + category == sourcemeta::core::GeneralCategory::OtherPunctuation; +} + +// Decode the character reference that follows an ampersand as GFM section 6.2 +// describes, appending its characters and returning the number of bytes it +// takes after the ampersand, or zero if the input does not start with one +inline auto decode_character_reference(std::string &output, + const std::string_view input) + -> std::size_t { + const auto size{input.size()}; + if (size >= 3 && input[0] == '#') { + char32_t codepoint{0}; + std::size_t index{0}; + std::size_t digits{0}; + // GFM section 6.2: "Decimal numeric character references consist of &# + + // a string of 1-7 arabic digits" followed by a semicolon, and "Hexadecimal + // numeric character references consist of &# + either X or x + a string + // of 1-6 hexadecimal digits" followed by a semicolon + std::size_t maximum_digits{0}; + if (sourcemeta::core::is_digit(input[1])) { + for (index = 1; index < size && sourcemeta::core::is_digit(input[index]); + ++index) { + codepoint = + (codepoint * 10) + static_cast(input[index] - '0'); + codepoint = std::min(codepoint, char32_t{0x110000}); + } + + digits = index - 1; + maximum_digits = 7; + } else if (input[1] == 'x' || input[1] == 'X') { + for (index = 2; index < size; ++index) { + const auto value{sourcemeta::core::hex_digit_value(input[index])}; + if (value < 0) { + break; + } + + codepoint = (codepoint * 16) + static_cast(value); + codepoint = std::min(codepoint, char32_t{0x110000}); + } + + digits = index - 2; + maximum_digits = 6; + } + + if (digits >= 1 && digits <= maximum_digits && index < size && + input[index] == ';') { + if (codepoint == 0 || !sourcemeta::core::is_valid_codepoint(codepoint)) { + codepoint = 0xFFFD; + } + + sourcemeta::core::codepoint_to_utf8(codepoint, output); + return index + 1; + } + + return 0; + } + + // The longest name is 31 characters long, so there is no point in looking + // for a semicolon any further + const auto limit{size > 32 ? std::size_t{32} : size}; + for (std::size_t index{2}; index < limit; ++index) { + if (input[index] == ' ') { + break; + } + + if (input[index] == ';') { + const auto characters{ + sourcemeta::core::html_entity(input.substr(0, index + 1))}; + if (!characters.empty()) { + output.append(characters); + return index + 1; + } + + break; + } + } + + return 0; +} + +// Replace every character reference of the input, returning whether the input +// had any ampersand at all, in which case the output holds the result +inline auto decode_character_references(std::string &output, + const std::string_view input) -> bool { + auto ampersand{input.find('&')}; + if (ampersand == std::string_view::npos) { + return false; + } + + std::size_t index{0}; + while (ampersand != std::string_view::npos) { + output.append(input.substr(index, ampersand - index)); + index = ampersand + 1; + const auto consumed{ + decode_character_reference(output, input.substr(index))}; + if (consumed == 0) { + output.push_back('&'); + } else { + index += consumed; + } + + ampersand = input.find('&', index); + } + + output.append(input.substr(index)); + return true; +} + +// Resolve the backslash escapes of GFM section 2.4 and the character +// references of GFM section 6.5 in one pass from left to right, so that an +// escaped ampersand stays literal rather than opening a character reference, +// as GFM section 2.4 example 310 requires +inline auto decode_escapes_and_references(std::string &output, + const std::string_view input) + -> void { + const auto size{input.size()}; + std::size_t index{0}; + auto next{input.find_first_of("&\\")}; + while (next != std::string_view::npos) { + output.append(input.substr(index, next - index)); + index = next; + if (input[index] == '\\') { + if (index + 1 < size && + sourcemeta::core::is_punctuation(input[index + 1])) { + output.push_back(input[index + 1]); + index += 2; + } else { + output.push_back('\\'); + index += 1; + } + } else { + const auto consumed{ + decode_character_reference(output, input.substr(index + 1))}; + if (consumed == 0) { + output.push_back('&'); + index += 1; + } else { + index += consumed + 1; + } + } + + next = input.find_first_of("&\\", index); + } + + output.append(input.substr(index)); +} + +// Normalise a link label as GFM section 4.7 describes, folding its case, +// trimming it, and collapsing every run of whitespace into a single space +inline auto normalize_label(std::string &output, const std::string_view label) + -> void { + output.clear(); + bool pending_space{false}; + std::size_t index{0}; + while (index < label.size()) { + const auto byte{static_cast(label[index])}; + if (byte < 0x80) { + if (is_whitespace_character(label[index])) { + pending_space = !output.empty(); + ++index; + continue; + } + + if (pending_space) { + output.push_back(' '); + pending_space = false; + } + + output.push_back(sourcemeta::core::to_lowercase(label[index])); + ++index; + continue; + } + + if (pending_space) { + output.push_back(' '); + pending_space = false; + } + + const auto decoded{sourcemeta::core::utf8_decode(label, index)}; + if (!decoded.has_value()) { + sourcemeta::core::codepoint_to_utf8(0xFFFD, output); + ++index; + continue; + } + + const auto [codepoint, length]{decoded.value()}; + const auto folded{sourcemeta::core::case_fold(codepoint)}; + if (folded.empty()) { + output.append(label.substr(index, length)); + } else { + for (const auto character : folded) { + sourcemeta::core::codepoint_to_utf8(character, output); + } + } + + index += length; + } +} + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/document.h b/src/core/markdown/document.h new file mode 100644 index 0000000000..89aa6228ea --- /dev/null +++ b/src/core/markdown/document.h @@ -0,0 +1,278 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_DOCUMENT_H_ +#define SOURCEMETA_CORE_MARKDOWN_DOCUMENT_H_ + +#include // std::max +#include // std::size_t +#include // std::int32_t, std::uint8_t, std::uint16_t, std::uint32_t +#include // std::memcpy +#include // std::string +#include // std::string_view +#include // std::unordered_map +#include // std::vector + +namespace sourcemeta::core::markdown { + +enum class NodeType : std::uint8_t { + Document, + BlockQuote, + List, + Item, + CodeBlock, + HTMLBlock, + Paragraph, + Heading, + ThematicBreak, + FootnoteDefinition, + Table, + TableRow, + TableCell, + Text, + SoftBreak, + LineBreak, + Code, + HTMLInline, + Emphasis, + Strong, + Link, + Image, + FootnoteReference, + Strikethrough +}; + +constexpr std::uint32_t NO_NODE{0xFFFFFFFF}; +constexpr std::uint32_t ROOT_NODE{0}; + +constexpr std::uint16_t FLAG_OPEN{1U << 0U}; +constexpr std::uint16_t FLAG_LAST_LINE_BLANK{1U << 1U}; +constexpr std::uint16_t FLAG_LAST_LINE_CHECKED{1U << 2U}; +constexpr std::uint16_t FLAG_TABLE_VISITED{1U << 3U}; +constexpr std::uint16_t FLAG_TASK{1U << 4U}; +constexpr std::uint16_t FLAG_CHECKED{1U << 5U}; +constexpr std::uint16_t FLAG_HEADER{1U << 6U}; +constexpr std::uint16_t FLAG_FENCED{1U << 7U}; +constexpr std::uint16_t FLAG_SETEXT{1U << 8U}; +constexpr std::uint16_t FLAG_TIGHT{1U << 9U}; +constexpr std::uint16_t FLAG_REFERENCED{1U << 10U}; +constexpr std::uint16_t FLAG_ATTACHED{1U << 11U}; +constexpr std::uint16_t FLAG_DETACHED{1U << 12U}; +constexpr std::uint16_t FLAG_REFERENCE_LINK{1U << 13U}; + +// A block or inline element, whose generic fields mean different things for +// different types of nodes, such as the fence of a code block or the numbering +// of a footnote +struct Node { + std::string_view literal; + std::string_view title; + std::uint32_t parent{NO_NODE}; + std::uint32_t first_child{NO_NODE}; + std::uint32_t last_child{NO_NODE}; + std::uint32_t previous{NO_NODE}; + std::uint32_t next{NO_NODE}; + std::uint32_t content_offset{0}; + std::uint32_t content_length{0}; + std::uint32_t data{0}; + std::uint32_t extra{0}; + std::uint16_t flags{0}; + NodeType type{NodeType::Document}; + std::uint8_t level{0}; +}; + +inline auto has_flag(const Node &node, const std::uint16_t flag) noexcept + -> bool { + return (node.flags & flag) != 0; +} + +inline auto set_flag(Node &node, const std::uint16_t flag, + const bool value) noexcept -> void { + node.flags = value ? static_cast(node.flags | flag) + : static_cast(node.flags & ~flag); +} + +struct ListData { + std::int32_t start{0}; + std::int32_t marker_offset{0}; + std::int32_t padding{0}; + std::int32_t start_line{0}; + char bullet_character{0}; + char delimiter{0}; + bool ordered{false}; +}; + +struct TableData { + std::uint32_t alignments_offset{0}; + std::uint32_t columns{0}; + std::int64_t rows{0}; + std::int64_t nonempty_cells{0}; +}; + +struct Reference { + std::string_view url; + std::string_view title; +}; + +// Strings that stay at the same address until the arena is cleared +class StringArena { +public: + auto clear() -> void { + if (this->blocks_.empty()) { + return; + } + + if (this->blocks_.size() > 1) { + this->blocks_.erase(this->blocks_.begin() + 1, this->blocks_.end()); + } + + this->cursor_ = this->blocks_.front().data(); + this->remaining_ = this->blocks_.front().size(); + } + + auto allocate(const std::size_t size) -> char * { + if (this->remaining_ < size) [[unlikely]] { + const auto capacity{std::max(size, BLOCK_SIZE)}; + this->blocks_.emplace_back(capacity, '\0'); + this->cursor_ = this->blocks_.back().data(); + this->remaining_ = capacity; + } + + auto *const result{this->cursor_}; + this->cursor_ += size; + this->remaining_ -= size; + return result; + } + + auto store(const std::string_view value) -> std::string_view { + if (value.empty()) { + return {}; + } + + auto *const target{this->allocate(value.size())}; + std::memcpy(target, value.data(), value.size()); + return {target, value.size()}; + } + +private: + static constexpr std::size_t BLOCK_SIZE{16384}; + std::vector blocks_; + char *cursor_{nullptr}; + std::size_t remaining_{0}; +}; + +struct Document { + auto clear() -> void { + this->nodes.clear(); + this->content.clear(); + this->strings.clear(); + this->lists.clear(); + this->tables.clear(); + this->alignments.clear(); + this->references.clear(); + this->reference_size_limit = 0; + this->footnote_definition_nodes.clear(); + this->footnote_reference_nodes.clear(); + this->nested_footnote_definitions = false; + this->nodes.emplace_back().flags = FLAG_OPEN; + } + + auto create(const NodeType type) -> std::uint32_t { + const auto index{static_cast(this->nodes.size())}; + this->nodes.emplace_back().type = type; + return index; + } + + [[nodiscard]] auto content_of(const Node &node) const -> std::string_view { + return std::string_view{this->content}.substr(node.content_offset, + node.content_length); + } + + auto append_child(const std::uint32_t parent, const std::uint32_t child) + -> void { + auto &parent_node{this->nodes[parent]}; + auto &child_node{this->nodes[child]}; + child_node.parent = parent; + child_node.next = NO_NODE; + child_node.previous = parent_node.last_child; + if (parent_node.last_child == NO_NODE) { + parent_node.first_child = child; + } else { + this->nodes[parent_node.last_child].next = child; + } + + parent_node.last_child = child; + } + + auto unlink(const std::uint32_t index) -> void { + auto &node{this->nodes[index]}; + if (node.previous != NO_NODE) { + this->nodes[node.previous].next = node.next; + } + + if (node.next != NO_NODE) { + this->nodes[node.next].previous = node.previous; + } + + if (node.parent != NO_NODE) { + auto &parent{this->nodes[node.parent]}; + if (parent.first_child == index) { + parent.first_child = node.next; + } + + if (parent.last_child == index) { + parent.last_child = node.previous; + } + } + + node.parent = NO_NODE; + node.previous = NO_NODE; + node.next = NO_NODE; + } + + auto insert_before(const std::uint32_t index, const std::uint32_t sibling) + -> void { + this->unlink(sibling); + auto &node{this->nodes[index]}; + auto &sibling_node{this->nodes[sibling]}; + sibling_node.parent = node.parent; + sibling_node.next = index; + sibling_node.previous = node.previous; + if (node.previous != NO_NODE) { + this->nodes[node.previous].next = sibling; + } else if (node.parent != NO_NODE) { + this->nodes[node.parent].first_child = sibling; + } + + node.previous = sibling; + } + + auto insert_after(const std::uint32_t index, const std::uint32_t sibling) + -> void { + this->unlink(sibling); + auto &node{this->nodes[index]}; + auto &sibling_node{this->nodes[sibling]}; + sibling_node.parent = node.parent; + sibling_node.previous = index; + sibling_node.next = node.next; + if (node.next != NO_NODE) { + this->nodes[node.next].previous = sibling; + } else if (node.parent != NO_NODE) { + this->nodes[node.parent].last_child = sibling; + } + + node.next = sibling; + } + + std::vector nodes; + std::string content; + StringArena strings; + std::vector lists; + std::vector tables; + std::vector alignments; + std::unordered_map references; + std::size_t reference_size_limit{0}; + std::vector footnote_definition_nodes; + std::vector footnote_reference_nodes; + bool nested_footnote_definitions{false}; +}; + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/include/sourcemeta/core/markdown.h b/src/core/markdown/include/sourcemeta/core/markdown.h index 910b207d2c..6901d12eb7 100644 --- a/src/core/markdown/include/sourcemeta/core/markdown.h +++ b/src/core/markdown/include/sourcemeta/core/markdown.h @@ -5,6 +5,10 @@ #include #endif +// NOLINTBEGIN(misc-include-cleaner) +#include +// NOLINTEND(misc-include-cleaner) + #include // std::string #include // std::string_view @@ -27,7 +31,11 @@ namespace sourcemeta::core { /// links are suppressed by default, so the result is safe to render from /// untrusted input. Passing a false safe argument lets raw HTML and unsafe /// links pass through unchanged, which must only be done for trusted input. -/// For example: +/// The specification lets link references and short table rows expand to an +/// output far larger than the input, so the conversion throws `MarkdownError` +/// once the rendered link references expand past the larger of 16 times the +/// input size and 1,048,576 bytes, or once a table inserts more than 524,288 +/// empty cells. For example: /// /// ```cpp /// #include diff --git a/src/core/markdown/include/sourcemeta/core/markdown_error.h b/src/core/markdown/include/sourcemeta/core/markdown_error.h new file mode 100644 index 0000000000..9d6b2dd786 --- /dev/null +++ b/src/core/markdown/include/sourcemeta/core/markdown_error.h @@ -0,0 +1,47 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_ERROR_H_ +#define SOURCEMETA_CORE_MARKDOWN_ERROR_H_ + +#ifndef SOURCEMETA_CORE_MARKDOWN_EXPORT +#include +#endif + +#include // std::exception +#include // std::string +#include // std::string_view + +namespace sourcemeta::core { + +// Exporting symbols that depends on the standard C++ library is considered +// safe. +// https://learn.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-2-c4275?view=msvc-170&redirectedfrom=MSDN +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4251 4275) +#endif + +/// @ingroup markdown +/// An error that represents a Markdown conversion that would exceed one of the +/// bounds on the size of its output +class SOURCEMETA_CORE_MARKDOWN_EXPORT MarkdownError : public std::exception { +public: + /// Construct an error with the given message + MarkdownError(const char *message) : message_{message} {} + MarkdownError(std::string message) = delete; + MarkdownError(std::string &&message) = delete; + MarkdownError(std::string_view message) = delete; + + [[nodiscard]] auto what() const noexcept -> const char * override { + return this->message_; + } + +private: + const char *message_; +}; + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +} // namespace sourcemeta::core + +#endif diff --git a/src/core/markdown/inlines.h b/src/core/markdown/inlines.h new file mode 100644 index 0000000000..4b893b3ca3 --- /dev/null +++ b/src/core/markdown/inlines.h @@ -0,0 +1,1244 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_INLINES_H_ +#define SOURCEMETA_CORE_MARKDOWN_INLINES_H_ + +#include +#include + +#include "characters.h" +#include "document.h" +#include "references.h" +#include "scanners.h" + +#include // std::min +#include // std::array +#include // std::size_t, std::ptrdiff_t +#include // std::uint8_t, std::uint32_t +#include // std::memchr +#include // std::less +#include // std::string +#include // std::string_view +#include // std::vector + +namespace sourcemeta::core::markdown { + +constexpr std::uint32_t NO_DELIMITER{0xFFFFFFFF}; + +constexpr std::uint8_t SKIP_HTML_CDATA{1U << 0U}; +constexpr std::uint8_t SKIP_HTML_DECLARATION{1U << 1U}; +constexpr std::uint8_t SKIP_HTML_PROCESSING_INSTRUCTION{1U << 2U}; + +// The bytes that may start an inline other than plain text, which are the +// line endings, the backslash, the backtick, the ampersand, the angle bracket, +// the emphasis and strikethrough delimiters, the brackets and the exclamation +// mark, and the colon and the letter that may start an extended autolink +constexpr std::array INLINE_SPECIAL{{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, // 0x00 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x10 + 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, // 0x20 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, // 0x30 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x40 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, // 0x50 + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x60 + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, // 0x70 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x80 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x90 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xA0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xB0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xC0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xD0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0xE0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 // 0xF0 +}}; + +struct Delimiter { + std::uint32_t previous{NO_DELIMITER}; + std::uint32_t next{NO_DELIMITER}; + std::uint32_t text{NO_NODE}; + std::size_t position{0}; + std::size_t length{0}; + char character{0}; + bool can_open{false}; + bool can_close{false}; +}; + +struct Bracket { + std::uint32_t text{NO_NODE}; + std::size_t position{0}; + std::ptrdiff_t start_column{0}; + bool image{false}; + bool bracket_after{false}; +}; + +// A character of a segment of a valid domain of GFM section 6.9, which +// "consists of segments of alphanumeric characters, underscores (_) and hyphens +// (-) separated by periods (.)", where alphanumeric characters are ASCII as +// the whitespace characters of GFM section 2.1 are, which the specification +// only extends to Unicode when it says so +inline auto is_domain_character(const char character) noexcept -> bool { + return sourcemeta::core::is_alphanum(character) || character == '_' || + character == '-'; +} + +// GFM section 6.9: "All such recognized autolinks can only come at the +// beginning of a line, after whitespace, or any of the delimiting characters +// *, _, ~, and (" +inline auto may_precede_extended_autolink(const char character) noexcept + -> bool { + return is_whitespace_character(character) || character == '*' || + character == '_' || character == '~' || character == '('; +} + +// The end of an extended autolink once its trailing punctuation, unbalanced +// parentheses, and entity references are removed as GFM section 6.9 describes +inline auto trim_autolink_end(const std::string_view data, + std::size_t link_end) noexcept -> std::size_t { + std::size_t opening{0}; + std::size_t closing{0}; + for (std::size_t index{0}; index < link_end; ++index) { + const auto character{data[index]}; + if (character == '<') { + link_end = index; + break; + } + + if (character == '(') { + ++opening; + } else if (character == ')') { + ++closing; + } + } + + while (link_end > 0) { + switch (data[link_end - 1]) { + case ')': + if (closing <= opening) { + return link_end; + } + + --closing; + --link_end; + break; + // GFM section 6.9: "Trailing punctuation (specifically, ?, !, ., ,, :, + // *, _, and ~) will not be considered part of the autolink" + case '?': + case '!': + case '.': + case ',': + case ':': + case '*': + case '_': + case '~': + --link_end; + break; + case ';': { + if (link_end < 2) { + --link_end; + break; + } + + // GFM section 6.9 only excludes a semicolon from an autolink along + // with an entity reference lookalike, which is "& followed by one or + // more alphanumeric characters" + auto entity_start{link_end - 2}; + while (entity_start > 0 && + sourcemeta::core::is_alphanum(data[entity_start])) { + --entity_start; + } + + if (entity_start < link_end - 2 && data[entity_start] == '&') { + link_end = entity_start; + break; + } + + return link_end; + } + + default: + return link_end; + } + } + + return link_end; +} + +// Whether the lengths of the delimiter runs of an opener and a closer of the +// same character let them match. GFM section 6.5 wraps strikethrough text in +// "a matching pair of one or two tildes", while GFM section 6.4 says that "If +// one of the delimiters can both open and close emphasis, then the sum of the +// lengths of the delimiter runs containing the opening and closing delimiters +// must not be a multiple of 3 unless both lengths are multiples of 3" +inline auto delimiter_lengths_match(const Delimiter &opener, + const Delimiter &closer) noexcept -> bool { + if (closer.character == '~') { + return opener.length == closer.length; + } + + return !(closer.can_open || opener.can_close) || closer.length % 3 == 0 || + (opener.length + closer.length) % 3 != 0; +} + +// The inlines of GFM section 6 and of the strikethrough and autolink +// extensions +class InlineParser { +public: + explicit InlineParser(Document &document) : document_{document} {} + + auto parse(const std::uint32_t parent) -> void { + this->parent_ = parent; + this->input_ = sourcemeta::core::strip_right( + this->document_.content_of(this->document_.nodes[parent]), is_space); + this->position_ = 0; + this->column_offset_ = 0; + this->last_delimiter_ = NO_DELIMITER; + this->delimiters_.clear(); + this->brackets_.clear(); + this->backticks_.clear(); + this->scanned_for_backticks_ = false; + this->rejected_domain_start_ = 0; + this->rejected_domain_limit_ = 0; + this->last_link_opener_position_ = 0; + this->flags_ = 0; + while (this->position_ < this->input_.size()) { + this->parse_inline(); + } + + this->process_emphasis(0); + } + +private: + auto node(const std::uint32_t index) -> Node & { + return this->document_.nodes[index]; + } + + [[nodiscard]] auto peek() const noexcept -> char { + return character_at(this->input_, this->position_); + } + + auto append(const NodeType type, const std::string_view literal) + -> std::uint32_t { + const auto index{this->document_.create(type)}; + this->node(index).literal = literal; + this->document_.append_child(this->parent_, index); + return index; + } + + auto append_text_child(const std::uint32_t parent, + const std::string_view literal) -> void { + const auto index{this->document_.create(NodeType::Text)}; + this->node(index).literal = literal; + this->document_.append_child(parent, index); + } + + auto parse_inline() -> void { + const auto character{this->input_[this->position_]}; + switch (character) { + case '\n': + case '\r': + this->handle_newline(); + return; + case '`': + this->handle_backticks(); + return; + case '\\': + this->handle_backslash(); + return; + case '&': + this->handle_entity(); + return; + case '<': + this->handle_pointy_brace(); + return; + case '*': + case '_': + this->handle_delimiter(character); + return; + case '\'': + case '"': + case '-': + case '.': + this->append(NodeType::Text, this->input_.substr(this->position_, 1)); + ++this->position_; + return; + case '[': + ++this->position_; + this->push_bracket( + false, this->append(NodeType::Text, + this->input_.substr(this->position_ - 1, 1))); + return; + case ']': + this->handle_close_bracket(); + return; + case '!': + this->handle_bang(); + return; + case '~': + this->handle_tilde(); + return; + default: + break; + } + + if (character == ':' && this->match_url_autolink()) { + return; + } + + if (character == 'w' && this->match_www_autolink()) { + return; + } + + const auto end{this->find_special_character(this->position_ + 1)}; + auto contents{this->input_.substr(this->position_, end - this->position_)}; + this->position_ = end; + if (end < this->input_.size() && is_line_end(this->input_[end])) { + contents = sourcemeta::core::strip_right(contents, is_space); + } + + this->append(NodeType::Text, contents); + } + + // Extended autolinks do not start inside brackets that may turn into link + // text. GFM section 6.6 says that autolinks "bind more tightly than the + // brackets in link text", but the path of an extended autolink of GFM + // section 6.9 takes "zero or more non-space non-< characters", so it would + // take the rest of a link such as [www.example.com](/x) along with it + [[nodiscard]] auto + may_start_www_autolink(const std::size_t index) const noexcept -> bool { + if (!this->brackets_.empty()) { + return false; + } + + if (index > 0 && !may_precede_extended_autolink(this->input_[index - 1])) { + return false; + } + + return this->input_.substr(index, 4) == "www."; + } + + [[nodiscard]] auto + may_start_url_autolink(const std::size_t index) const noexcept -> bool { + return this->brackets_.empty() && index + 3 < this->input_.size() && + this->input_[index + 1] == '/' && this->input_[index + 2] == '/'; + } + + [[nodiscard]] auto + find_special_character(const std::size_t from) const noexcept -> std::size_t { + const auto size{this->input_.size()}; + for (auto index{from}; index < size; ++index) { + const auto character{this->input_[index]}; + if (INLINE_SPECIAL[static_cast(character)] == 0) { + continue; + } + + if ((character == 'w' && !this->may_start_www_autolink(index)) || + (character == ':' && !this->may_start_url_autolink(index))) { + continue; + } + + return index; + } + + return size; + } + + auto handle_newline() -> void { + const auto newline_position{this->position_}; + if (this->input_[this->position_] == '\r') { + ++this->position_; + } + + if (this->peek() == '\n') { + ++this->position_; + } + + this->column_offset_ = -static_cast(this->position_); + skip_spaces(this->input_, this->position_); + if (newline_position > 1 && this->input_[newline_position - 1] == ' ' && + this->input_[newline_position - 2] == ' ') { + this->append(NodeType::LineBreak, {}); + } else { + this->append(NodeType::SoftBreak, {}); + } + } + + auto scan_to_closing_backticks(const std::size_t opening_length) + -> std::size_t { + // GFM section 6.3 puts no bound on the length of a backtick string. Once + // the rest of the input was scanned, the last position of every backtick + // string length is known, so an opening backtick string without a closing + // one of the same length after it is rejected without scanning again + if (this->scanned_for_backticks_ && + (opening_length >= this->backticks_.size() || + this->backticks_[opening_length] <= this->position_)) { + return 0; + } + + const auto size{this->input_.size()}; + while (this->position_ < size) { + const auto *const found{static_cast(std::memchr( + this->input_.data() + this->position_, '`', size - this->position_))}; + if (found == nullptr) { + this->position_ = size; + break; + } + + this->position_ = static_cast(found - this->input_.data()); + std::size_t count{0}; + while (this->position_ < size && this->input_[this->position_] == '`') { + ++this->position_; + ++count; + } + + if (count >= this->backticks_.size()) { + this->backticks_.resize(count + 1, 0); + } + + // A scan that starts after a complete one must not replace the last + // position of a backtick string length with an earlier one + this->backticks_[count] = + std::max(this->backticks_[count], this->position_ - count); + + if (count == opening_length) { + return this->position_; + } + } + + this->scanned_for_backticks_ = true; + return 0; + } + + // The content of a code span of GFM section 6.3: "First, line endings are + // converted to spaces. If the resulting string both begins and ends with a + // space character, but does not consist entirely of space characters, a + // single space character is removed from the front and back" + auto normalize_code(const std::string_view raw) -> std::string_view { + if (raw.find_first_of("\r\n") == std::string_view::npos) { + if (raw.size() >= 2 && raw.front() == ' ' && raw.back() == ' ' && + raw.find_first_not_of(' ') != std::string_view::npos) { + return raw.substr(1, raw.size() - 2); + } + + return raw; + } + + auto &buffer{this->buffer_}; + buffer.assign(raw); + const auto size{buffer.size()}; + std::size_t write{0}; + bool contains_nonspace{false}; + for (std::size_t read{0}; read < size; ++read) { + const auto character{buffer[read]}; + if (character == '\r') { + if (read + 1 >= size || buffer[read + 1] != '\n') { + buffer[write] = ' '; + ++write; + } + } else if (character == '\n') { + buffer[write] = ' '; + ++write; + } else { + buffer[write] = character; + ++write; + if (character != ' ') { + contains_nonspace = true; + } + } + } + + if (contains_nonspace && write >= 2 && buffer[0] == ' ' && + buffer[write - 1] == ' ') { + return this->document_.strings.store( + std::string_view{buffer}.substr(1, write - 2)); + } + + return this->document_.strings.store( + std::string_view{buffer}.substr(0, write)); + } + + auto handle_backticks() -> void { + const auto start{this->position_}; + while (this->peek() == '`') { + ++this->position_; + } + + const auto opening_length{this->position_ - start}; + const auto content_start{this->position_}; + const auto end{this->scan_to_closing_backticks(opening_length)}; + if (end == 0) { + this->position_ = content_start; + this->append(NodeType::Text, this->input_.substr(start, opening_length)); + return; + } + + this->append(NodeType::Code, + this->normalize_code(this->input_.substr( + content_start, end - content_start - opening_length))); + } + + auto handle_backslash() -> void { + ++this->position_; + const auto next{this->peek()}; + if (sourcemeta::core::is_punctuation(next)) { + ++this->position_; + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + return; + } + + if (next == '\r' || next == '\n') { + skip_line_end(this->input_, this->position_); + this->append(NodeType::LineBreak, {}); + return; + } + + this->append(NodeType::Text, "\\"); + } + + auto handle_entity() -> void { + ++this->position_; + this->buffer_.clear(); + const auto consumed{decode_character_reference( + this->buffer_, this->input_.substr(this->position_))}; + if (consumed == 0) { + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + return; + } + + this->position_ += consumed; + this->append(NodeType::Text, this->document_.strings.store(this->buffer_)); + } + + auto make_autolink(const std::string_view contents, const bool is_email) + -> void { + const auto trimmed{sourcemeta::core::trim(contents, is_space)}; + const auto link{this->append(NodeType::Link, {})}; + if (!trimmed.empty()) { + this->buffer_.clear(); + if (is_email) { + this->buffer_.append("mailto:"); + } + + if (!decode_character_references(this->buffer_, trimmed)) { + this->buffer_.append(trimmed); + } + + this->node(link).literal = this->document_.strings.store(this->buffer_); + } + + this->buffer_.clear(); + this->append_text_child(link, + decode_character_references(this->buffer_, trimmed) + ? this->document_.strings.store(this->buffer_) + : trimmed); + } + + auto scan_raw_html() -> std::size_t { + const auto size{this->input_.size()}; + const auto position{this->position_}; + if (position + 2 > size) { + return 0; + } + + const auto character{this->input_[position]}; + if (character == '!') { + const auto next{this->input_[position + 1]}; + if (next == '-' && character_at(this->input_, position + 2) == '-') { + const auto length{scan_html_comment(this->input_, position + 1)}; + return length > 0 ? length + 1 : 0; + } + + if (next == '[') { + if ((this->flags_ & SKIP_HTML_CDATA) != 0) { + return 0; + } + + const auto length{scan_html_cdata(this->input_, position + 2)}; + if (length == 0) { + return 0; + } + + if (position + length + 5 > size) { + this->flags_ |= SKIP_HTML_CDATA; + return 0; + } + + return length + 5; + } + + if ((this->flags_ & SKIP_HTML_DECLARATION) != 0) { + return 0; + } + + const auto length{scan_html_declaration(this->input_, position + 1)}; + if (length == 0) { + return 0; + } + + if (position + length + 2 > size) { + this->flags_ |= SKIP_HTML_DECLARATION; + return 0; + } + + return length + 2; + } + + if (character == '?') { + if ((this->flags_ & SKIP_HTML_PROCESSING_INSTRUCTION) != 0) { + return 0; + } + + const auto length{ + scan_html_processing_instruction(this->input_, position + 1) + 3}; + if (position + length > size) { + this->flags_ |= SKIP_HTML_PROCESSING_INSTRUCTION; + return 0; + } + + return length; + } + + return scan_html_tag(this->input_, position); + } + + auto handle_pointy_brace() -> void { + ++this->position_; + auto length{scan_autolink_uri(this->input_, this->position_)}; + if (length > 0) { + const auto contents{this->input_.substr(this->position_, length - 1)}; + this->position_ += length; + this->make_autolink(contents, false); + return; + } + + length = scan_autolink_email(this->input_, this->position_); + if (length > 0) { + const auto contents{this->input_.substr(this->position_, length - 1)}; + this->position_ += length; + this->make_autolink(contents, true); + return; + } + + length = this->scan_raw_html(); + if (length > 0) { + const auto contents{this->input_.substr(this->position_ - 1, length + 1)}; + this->position_ += length; + this->append(NodeType::HTMLInline, contents); + return; + } + + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + } + + // The characters around a delimiter run of GFM section 6.4, where "the + // beginning and the end of the line count as Unicode whitespace" + [[nodiscard]] auto character_before_delimiters() const noexcept -> char32_t { + if (this->position_ == 0) { + return U'\n'; + } + + auto index{this->position_ - 1}; + while (index > 0 && sourcemeta::core::is_utf8_continuation( + static_cast(this->input_[index]))) { + --index; + } + + const auto preceding{sourcemeta::core::utf8_decode( + this->input_.substr(index, this->position_ - index), 0)}; + return preceding.has_value() ? preceding->first : U'\n'; + } + + [[nodiscard]] auto character_after_delimiters() const noexcept -> char32_t { + const auto following{ + sourcemeta::core::utf8_decode(this->input_, this->position_)}; + return following.has_value() ? following->first : U'\n'; + } + + auto handle_delimiter(const char character) -> void { + const auto before{this->character_before_delimiters()}; + const auto start{this->position_}; + while (this->peek() == character) { + ++this->position_; + } + + const auto count{this->position_ - start}; + const auto after{this->character_after_delimiters()}; + const auto space_before{is_unicode_whitespace(before)}; + const auto space_after{is_unicode_whitespace(after)}; + const auto punctuation_before{is_unicode_punctuation(before)}; + const auto punctuation_after{is_unicode_punctuation(after)}; + const auto left_flanking{ + !space_after && + (!punctuation_after || space_before || punctuation_before)}; + const auto right_flanking{ + !space_before && + (!punctuation_before || space_after || punctuation_after)}; + bool can_open{left_flanking}; + bool can_close{right_flanking}; + if (character == '_') { + can_open = left_flanking && (!right_flanking || punctuation_before); + can_close = right_flanking && (!left_flanking || punctuation_after); + } + + const auto text{ + this->append(NodeType::Text, this->input_.substr(start, count))}; + if (can_open || can_close) { + this->push_delimiter(character, can_open, can_close, text); + } + } + + // GFM section 6.5: "Strikethrough text is any text wrapped in a matching pair + // of one or two tildes (~)", and "Three or more tildes do not create a + // strikethrough" + auto handle_tilde() -> void { + const auto before{this->character_before_delimiters()}; + const auto start{this->position_}; + while (this->peek() == '~') { + ++this->position_; + } + + const auto count{this->position_ - start}; + const auto after{this->character_after_delimiters()}; + const auto space_before{is_unicode_whitespace(before)}; + const auto space_after{is_unicode_whitespace(after)}; + const auto punctuation_before{is_unicode_punctuation(before)}; + const auto punctuation_after{is_unicode_punctuation(after)}; + const auto left_flanking{ + !space_after && + (!punctuation_after || space_before || punctuation_before)}; + const auto right_flanking{ + !space_before && + (!punctuation_before || space_after || punctuation_after)}; + const auto text{ + this->append(NodeType::Text, this->input_.substr(start, count))}; + if ((left_flanking || right_flanking) && count <= 2) { + this->push_delimiter('~', left_flanking, right_flanking, text); + } + } + + auto handle_bang() -> void { + ++this->position_; + if (this->peek() == '[' && + character_at(this->input_, this->position_ + 1) != '^') { + ++this->position_; + this->push_bracket( + true, this->append(NodeType::Text, + this->input_.substr(this->position_ - 2, 2))); + return; + } + + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + } + + auto push_delimiter(const char character, const bool can_open, + const bool can_close, const std::uint32_t text) -> void { + const auto index{static_cast(this->delimiters_.size())}; + this->delimiters_.push_back({.previous = this->last_delimiter_, + .next = NO_DELIMITER, + .text = text, + .position = this->position_, + .length = this->node(text).literal.size(), + .character = character, + .can_open = can_open, + .can_close = can_close}); + if (this->last_delimiter_ != NO_DELIMITER) { + this->delimiters_[this->last_delimiter_].next = index; + } + + this->last_delimiter_ = index; + } + + auto remove_delimiter(const std::uint32_t index) -> void { + const auto &delimiter{this->delimiters_[index]}; + if (delimiter.next == NO_DELIMITER) { + this->last_delimiter_ = delimiter.previous; + } else { + this->delimiters_[delimiter.next].previous = delimiter.previous; + } + + if (delimiter.previous != NO_DELIMITER) { + this->delimiters_[delimiter.previous].next = delimiter.next; + } + } + + auto push_bracket(const bool image, const std::uint32_t text) -> void { + if (!this->brackets_.empty()) { + this->brackets_.back().bracket_after = true; + } + + const auto position{static_cast(this->position_)}; + this->brackets_.push_back( + {.text = text, + .position = this->position_, + .start_column = + (image ? position - 1 : position) + this->column_offset_, + .image = image, + .bracket_after = false}); + } + + static auto delimiter_index(const char character) noexcept -> std::size_t { + if (character == '*') { + return 0; + } + + return character == '_' ? 1 : 2; + } + + // Move the inlines between two siblings into a new parent that takes the + // place right after the first sibling + auto wrap_between(const std::uint32_t first, const std::uint32_t last, + const std::uint32_t wrapper) -> void { + auto current{this->node(first).next}; + while (current != NO_NODE && current != last) { + const auto next{this->node(current).next}; + this->document_.unlink(current); + this->document_.append_child(wrapper, current); + current = next; + } + } + + auto insert_emphasis(const std::uint32_t opener, std::uint32_t closer) + -> std::uint32_t { + const auto opener_text{this->delimiters_[opener].text}; + const auto closer_text{this->delimiters_[closer].text}; + auto opener_characters{this->node(opener_text).literal.size()}; + auto closer_characters{this->node(closer_text).literal.size()}; + const std::size_t used{ + closer_characters >= 2 && opener_characters >= 2 ? 2U : 1U}; + opener_characters -= used; + closer_characters -= used; + this->node(opener_text).literal = + this->node(opener_text).literal.substr(0, opener_characters); + this->node(closer_text).literal = + this->node(closer_text).literal.substr(0, closer_characters); + + auto between{this->delimiters_[closer].previous}; + while (between != NO_DELIMITER && between != opener) { + const auto previous{this->delimiters_[between].previous}; + this->remove_delimiter(between); + between = previous; + } + + const auto emphasis{this->document_.create(used == 1 ? NodeType::Emphasis + : NodeType::Strong)}; + this->wrap_between(opener_text, closer_text, emphasis); + this->document_.insert_after(opener_text, emphasis); + if (opener_characters == 0) { + this->document_.unlink(opener_text); + this->remove_delimiter(opener); + } + + if (closer_characters == 0) { + this->document_.unlink(closer_text); + const auto next{this->delimiters_[closer].next}; + this->remove_delimiter(closer); + closer = next; + } + + return closer; + } + + auto insert_strikethrough(const std::uint32_t opener, + const std::uint32_t closer) -> std::uint32_t { + const auto result{this->delimiters_[closer].next}; + const auto opener_text{this->delimiters_[opener].text}; + const auto closer_text{this->delimiters_[closer].text}; + this->node(opener_text).type = NodeType::Strikethrough; + this->node(opener_text).literal = {}; + this->wrap_between(opener_text, closer_text, opener_text); + this->document_.unlink(closer_text); + + auto current{closer}; + while (current != NO_DELIMITER && current != opener) { + const auto previous{this->delimiters_[current].previous}; + this->remove_delimiter(current); + current = previous; + } + + this->remove_delimiter(opener); + return result; + } + + auto process_emphasis(const std::size_t stack_bottom) -> void { + std::array, 3> openers_bottom{}; + for (auto &row : openers_bottom) { + row.fill(stack_bottom); + } + + auto closer{NO_DELIMITER}; + auto candidate{this->last_delimiter_}; + while (candidate != NO_DELIMITER && + this->delimiters_[candidate].position >= stack_bottom) { + closer = candidate; + candidate = this->delimiters_[candidate].previous; + } + + while (closer != NO_DELIMITER) { + const auto closer_delimiter{this->delimiters_[closer]}; + if (!closer_delimiter.can_close) { + closer = closer_delimiter.next; + continue; + } + + const auto kind{delimiter_index(closer_delimiter.character)}; + const auto remainder{closer_delimiter.length % 3}; + auto opener{closer_delimiter.previous}; + bool opener_found{false}; + while (opener != NO_DELIMITER && + this->delimiters_[opener].position >= stack_bottom && + this->delimiters_[opener].position >= + openers_bottom[remainder][kind]) { + const auto &opener_delimiter{this->delimiters_[opener]}; + if (opener_delimiter.can_open && + opener_delimiter.character == closer_delimiter.character && + delimiter_lengths_match(opener_delimiter, closer_delimiter)) { + opener_found = true; + break; + } + + opener = opener_delimiter.previous; + } + + if (opener_found) { + closer = closer_delimiter.character == '~' + ? this->insert_strikethrough(opener, closer) + : this->insert_emphasis(opener, closer); + continue; + } + + openers_bottom[remainder][kind] = closer_delimiter.position; + const auto next{closer_delimiter.next}; + if (!closer_delimiter.can_open) { + this->remove_delimiter(closer); + } + + closer = next; + } + + while (this->last_delimiter_ != NO_DELIMITER && + this->delimiters_[this->last_delimiter_].position >= stack_bottom) { + this->remove_delimiter(this->last_delimiter_); + } + } + + auto handle_close_bracket() -> void { + ++this->position_; + const auto initial_position{this->position_}; + if (this->brackets_.empty()) { + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + return; + } + + const auto opener{this->brackets_.back()}; + // GFM section 6.6: "Links may not contain other links, at any level of + // nesting", so a bracket that opens before the latest link cannot close + // another one + if (!opener.image && opener.position < this->last_link_opener_position_) { + this->brackets_.pop_back(); + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + return; + } + + std::string_view url; + std::string_view title; + bool matched{false}; + bool from_reference{false}; + if (this->peek() == '(') { + const auto spaces{scan_spacechars(this->input_, this->position_ + 1)}; + std::string_view destination; + const auto destination_length{scan_link_destination( + this->input_, this->position_ + 1 + spaces, destination)}; + if (destination_length >= 0) { + const auto url_end{this->position_ + 1 + spaces + + static_cast(destination_length)}; + const auto title_start{url_end + + scan_spacechars(this->input_, url_end)}; + const auto title_end{ + title_start == url_end + ? title_start + : title_start + scan_link_title(this->input_, title_start)}; + const auto end{title_end + scan_spacechars(this->input_, title_end)}; + if (character_at(this->input_, end) == ')') { + this->position_ = end + 1; + url = clean_url(this->document_, destination, this->buffer_, true); + title = clean_title( + this->document_, + this->input_.substr(title_start, title_end - title_start), + this->buffer_, true); + matched = true; + } + } + } + + if (!matched) { + this->position_ = initial_position; + std::string_view label; + auto found_label{scan_link_label(this->input_, this->position_, label)}; + if ((!found_label || label.empty()) && !opener.bracket_after) { + label = this->input_.substr(opener.position, + initial_position - opener.position - 1); + found_label = true; + } + + const auto *const reference{ + found_label + ? find_reference(this->document_, label, this->label_buffer_) + : nullptr}; + if (reference != nullptr) { + url = reference->url; + title = reference->title; + matched = true; + from_reference = true; + } + } + + if (!matched) { + this->handle_unmatched_bracket(opener, initial_position); + return; + } + + const auto link{this->document_.create(opener.image ? NodeType::Image + : NodeType::Link)}; + this->node(link).literal = url; + this->node(link).title = title; + set_flag(this->node(link), FLAG_REFERENCE_LINK, from_reference); + this->document_.insert_before(opener.text, link); + this->wrap_between(opener.text, NO_NODE, link); + this->document_.unlink(opener.text); + this->process_emphasis(opener.position); + this->brackets_.pop_back(); + if (!opener.image) { + this->last_link_opener_position_ = + std::max(this->last_link_opener_position_, opener.position); + } + } + + // A bracket that is neither a link nor an image might still be a footnote + // reference, whose label is the text between its brackets + auto handle_unmatched_bracket(const Bracket &opener, + const std::size_t initial_position) -> void { + const auto after_opener{this->node(opener.text).next}; + if (after_opener != NO_NODE && + this->node(after_opener).type == NodeType::Text) { + const auto literal{this->node(after_opener).literal}; + if (!literal.empty() && literal.front() == '^' && + (literal.size() > 1 || this->node(after_opener).next != NO_NODE)) { + this->position_ = initial_position; + const auto end_column{static_cast(this->position_) + + this->column_offset_}; + const auto length{ + opener.start_column + 2 <= end_column + ? static_cast(end_column - opener.start_column - 2) + : std::size_t{0}}; + const auto *const label_start{literal.data() + 1}; + const auto *const input_start{this->input_.data()}; + const auto *const input_end{this->input_.data() + this->input_.size()}; + const std::less before{}; + const auto available{ + !before(label_start, input_start) && !before(input_end, label_start) + ? static_cast(input_end - label_start) + : literal.size() - 1}; + const auto reference{ + this->document_.create(NodeType::FootnoteReference)}; + this->document_.footnote_reference_nodes.push_back(reference); + this->node(reference).literal = + std::string_view{label_start, std::min(length, available)}; + this->document_.insert_before(opener.text, reference); + this->process_emphasis(opener.position); + auto current{this->node(opener.text).next}; + while (current != NO_NODE) { + const auto next{this->node(current).next}; + this->document_.unlink(current); + current = next; + } + + this->document_.unlink(opener.text); + this->brackets_.pop_back(); + return; + } + } + + this->brackets_.pop_back(); + this->position_ = initial_position; + this->append(NodeType::Text, this->input_.substr(this->position_ - 1, 1)); + } + + // GFM section 6.9: "An extended url autolink will be recognised when one of + // the schemes http://, or https://, followed by a valid domain", where RFC + // 3986 Section 3.1 says that "schemes are case-insensitive" + static auto + starts_with_extended_url_scheme(const std::string_view link) noexcept + -> bool { + for (const auto scheme : + std::array{{"http://", "https://"}}) { + if (link.size() > scheme.size() && + sourcemeta::core::starts_with_ignore_case(link, scheme) && + is_domain_character(link[scheme.size()])) { + return true; + } + } + + return false; + } + + // The length of the valid domain of GFM section 6.9 at a position of the + // input, or zero if there is none. "There must be at least one period, and + // no underscores may be present in the last two segments of the domain" + auto scan_autolink_domain(const std::size_t start) -> std::size_t { + // A later start within a domain that was rejected for its underscores has + // the same last two segments until the second to last period, so it is + // rejected without scanning the same characters again + if (start > this->rejected_domain_start_ && + start < this->rejected_domain_limit_) { + return 0; + } + + const auto data{this->input_.substr(start)}; + std::size_t index{0}; + std::size_t periods{0}; + std::size_t last_period{0}; + std::size_t second_to_last_period{0}; + std::size_t underscores_before_last_period{0}; + std::size_t underscores_after_last_period{0}; + for (; index < data.size(); ++index) { + const auto character{data[index]}; + if (character == '.') { + // The segments that periods separate have at least one character + if (index + 1 >= data.size() || !is_domain_character(data[index + 1])) { + break; + } + + underscores_before_last_period = underscores_after_last_period; + underscores_after_last_period = 0; + second_to_last_period = last_period; + last_period = index; + ++periods; + } else if (character == '_') { + ++underscores_after_last_period; + } else if (!is_domain_character(character)) { + break; + } + } + + if (underscores_before_last_period > 0 || + underscores_after_last_period > 0) { + if (periods >= 2) { + this->rejected_domain_start_ = start; + this->rejected_domain_limit_ = start + second_to_last_period; + } + + return 0; + } + + return periods > 0 ? index : 0; + } + + auto match_www_autolink() -> bool { + if (!this->may_start_www_autolink(this->position_)) { + return false; + } + + const auto data{this->input_.substr(this->position_)}; + auto link_end{this->scan_autolink_domain(this->position_)}; + if (link_end == 0) { + return false; + } + + while (link_end < data.size() && !is_space(data[link_end]) && + data[link_end] != '<') { + ++link_end; + } + + link_end = trim_autolink_end(data, link_end); + if (link_end == 0) { + return false; + } + + this->position_ += link_end; + this->buffer_.assign("http://"); + this->buffer_.append(data.substr(0, link_end)); + const auto link{this->append(NodeType::Link, + this->document_.strings.store(this->buffer_))}; + this->append_text_child(link, data.substr(0, link_end)); + return true; + } + + auto remove_trailing_text(std::size_t count) -> void { + auto index{this->node(this->parent_).last_child}; + while (count > 0 && index != NO_NODE && + this->node(index).type == NodeType::Text) { + auto &literal{this->node(index).literal}; + if (literal.size() < count) { + count -= literal.size(); + literal = literal.substr(0, 0); + } else { + literal = literal.substr(0, literal.size() - count); + count = 0; + } + + index = this->node(index).previous; + } + } + + auto match_url_autolink() -> bool { + if (!this->may_start_url_autolink(this->position_)) { + return false; + } + + const auto data{this->input_.substr(this->position_)}; + std::size_t rewind{0}; + while (rewind < this->position_ && + sourcemeta::core::is_alpha( + this->input_[this->position_ - rewind - 1])) { + ++rewind; + } + + const auto start{this->position_ - rewind}; + if ((start > 0 && + !may_precede_extended_autolink(this->input_[start - 1])) || + !starts_with_extended_url_scheme(this->input_.substr(start))) { + return false; + } + + const auto domain_length{this->scan_autolink_domain(this->position_ + 3)}; + if (domain_length == 0) { + return false; + } + + auto link_end{3 + domain_length}; + while (link_end < data.size() && !is_space(data[link_end]) && + data[link_end] != '<') { + ++link_end; + } + + link_end = trim_autolink_end(data, link_end); + if (link_end == 0) { + return false; + } + + this->position_ += link_end; + this->remove_trailing_text(rewind); + const auto url{this->input_.substr(start, link_end + rewind)}; + const auto link{this->append(NodeType::Link, url)}; + this->append_text_child(link, url); + return true; + } + + Document &document_; + std::uint32_t parent_{NO_NODE}; + std::string_view input_; + std::size_t position_{0}; + std::ptrdiff_t column_offset_{0}; + std::uint32_t last_delimiter_{NO_DELIMITER}; + std::vector delimiters_; + std::vector brackets_; + std::vector backticks_; + bool scanned_for_backticks_{false}; + std::size_t rejected_domain_start_{0}; + std::size_t rejected_domain_limit_{0}; + std::uint8_t flags_{0}; + std::string buffer_; + std::string label_buffer_; + std::size_t last_link_opener_position_{0}; +}; + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/markdown.cc b/src/core/markdown/markdown.cc index a35ffc5993..ae2584cb74 100644 --- a/src/core/markdown/markdown.cc +++ b/src/core/markdown/markdown.cc @@ -1,75 +1,95 @@ #include +#include #include -#include // cmark_gfm_core_extensions_ensure_registered -#include // cmark_find_syntax_extension, cmark_parser_attach_syntax_extension, cmark_parser_get_syntax_extensions -#include // cmark_parser_new, cmark_parser_feed, cmark_parser_finish, cmark_parser_free, cmark_render_html, cmark_node_free +#include "blocks.h" +#include "characters.h" +#include "document.h" +#include "inlines.h" +#include "postprocess.h" +#include "render.h" -#include // std::array #include // std::size_t -#include // std::free -#include // std::mutex, std::scoped_lock +#include // std::uint32_t +#include // std::numeric_limits +#include // std::optional #include // std::string #include // std::string_view -namespace sourcemeta::core { +namespace { -auto markdown_to_html(const std::string_view input, const bool safe) - -> std::string { - [[maybe_unused]] static const bool CMARK_INITIALIZED{ - (cmark_gfm_core_extensions_ensure_registered(), true)}; +// An input past this size gives up the state of its thread once it is done, +// so that an unusually large conversion does not hold on to the memory it +// needed for the rest of the life of the thread. Inputs this large are far +// past what documentation is made of, so paying for their buffers again is +// better than keeping them around +constexpr std::size_t MAXIMUM_RETAINED_INPUT{0x800000}; - // Byte sequences that are not UTF-8 become one replacement character per - // maximal subpart, as the Unicode Standard recommends, rather than the one - // replacement character per sequence that the parser would produce - std::size_t valid_length{0}; - while (valid_length < input.size()) { - const auto length{utf8_codepoint_length(input, valid_length)}; - if (length == 0) { - break; - } +// The state of every thread is kept across calls, so that rendering many +// small inputs does not allocate the same buffers over and over +struct MarkdownConverter { + sourcemeta::core::markdown::Document document; + sourcemeta::core::markdown::BlockParser blocks{document}; + sourcemeta::core::markdown::InlineParser inlines{document}; + sourcemeta::core::markdown::PostProcessor postprocessor{document}; + sourcemeta::core::markdown::HTMLRenderer renderer{document}; + std::string repaired; +}; - valid_length += length; +// Give up the state of the thread even when the conversion throws +struct ConverterRelease { + std::optional &converter; + bool oversized; + ~ConverterRelease() { + if (this->oversized) { + this->converter.reset(); + } } +}; - const auto is_valid{valid_length == input.size()}; - const auto repaired{is_valid ? std::string{} : to_valid_utf8(input)}; - const std::string_view source{is_valid ? input : std::string_view{repaired}}; +auto replace_invalid_characters(const std::string_view input, + std::string &output) -> void { + output = sourcemeta::core::to_valid_utf8(input); + if (output.find('\0') != std::string::npos) { + output = sourcemeta::core::replace(output, std::string_view{"\0", 1}, + "\xEF\xBF\xBD"); + } +} - // cmark-gfm toggles process-global special-character tables when syntax - // extensions are attached and detached, so parser construction through - // teardown cannot run concurrently - static std::mutex cmark_mutex; - const std::scoped_lock lock{cmark_mutex}; +} // namespace - static constexpr auto BASE_OPTIONS{ - CMARK_OPT_VALIDATE_UTF8 | CMARK_OPT_FOOTNOTES | - CMARK_OPT_STRIKETHROUGH_DOUBLE_TILDE | CMARK_OPT_GITHUB_PRE_LANG}; - // This cmark-gfm suppresses raw HTML and unsafe links by default, so raw - // output requires explicitly opting out through CMARK_OPT_UNSAFE - const int options{safe ? BASE_OPTIONS : (BASE_OPTIONS | CMARK_OPT_UNSAFE)}; +namespace sourcemeta::core { - auto *parser{cmark_parser_new(options)}; +auto markdown_to_html(const std::string_view input, const bool safe) + -> std::string { + thread_local std::optional state; + if (!state.has_value()) { + state.emplace(); + } - static constexpr std::array EXTENSION_NAMES{ - {"table", "autolink", "strikethrough", "tagfilter", "tasklist"}}; - for (const auto *name : EXTENSION_NAMES) { - auto *extension{cmark_find_syntax_extension(name)}; - if (extension != nullptr) { - cmark_parser_attach_syntax_extension(parser, extension); - } + const ConverterRelease release{ + .converter = state, .oversized = input.size() > MAXIMUM_RETAINED_INPUT}; + auto &converter{state.value()}; + converter.document.clear(); + auto source{input}; + // GFM section 2.3 requires replacing the NUL character, and byte sequences + // that are not well-formed UTF-8 are replaced too + if (input.find('\0') != std::string_view::npos || + !sourcemeta::core::is_valid_utf8(input)) { + replace_invalid_characters(input, converter.repaired); + source = converter.repaired; } - cmark_parser_feed(parser, source.data(), source.size()); - auto *document{cmark_parser_finish(parser)}; - auto *result{cmark_render_html(document, options, - cmark_parser_get_syntax_extensions(parser))}; + // The parser addresses the input through 32-bit offsets + if (source.size() > std::numeric_limits::max()) { + throw MarkdownError{"The input is larger than the supported size bound"}; + } - std::string output{result}; - std::free(result); - cmark_node_free(document); - cmark_parser_free(parser); - return output; + converter.blocks.parse(source, input.size()); + converter.postprocessor.parse_inlines(converter.inlines); + converter.postprocessor.process_footnotes(); + return converter.renderer.render(!safe, + source.size() + (source.size() / 2) + 64); } } // namespace sourcemeta::core diff --git a/src/core/markdown/postprocess.h b/src/core/markdown/postprocess.h new file mode 100644 index 0000000000..a06b0088d4 --- /dev/null +++ b/src/core/markdown/postprocess.h @@ -0,0 +1,524 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_POSTPROCESS_H_ +#define SOURCEMETA_CORE_MARKDOWN_POSTPROCESS_H_ + +#include +#include + +#include "characters.h" +#include "document.h" +#include "inlines.h" +#include "references.h" + +#include // std::sort +#include // std::array +#include // std::size_t +#include // std::uint32_t +#include // std::memchr +#include // std::less +#include // std::string +#include // std::string_view +#include // std::unordered_map +#include // std::vector + +namespace sourcemeta::core::markdown { + +inline auto is_leaf_type(const NodeType type) noexcept -> bool { + return type == NodeType::HTMLBlock || type == NodeType::ThematicBreak || + type == NodeType::CodeBlock || type == NodeType::Text || + type == NodeType::SoftBreak || type == NodeType::LineBreak || + type == NodeType::Code || type == NodeType::HTMLInline; +} + +// The node after the given one in document order when skipping its +// descendants, without leaving the subtree of the given root, or no node at +// the end of that subtree +inline auto next_skipping_descendants(const Document &document, + std::uint32_t index, + const std::uint32_t root) noexcept + -> std::uint32_t { + while (index != root && index != NO_NODE) { + const auto ¤t{document.nodes[index]}; + if (current.next != NO_NODE) { + return current.next; + } + + index = current.parent; + } + + return NO_NODE; +} + +// A character of the part of an email address before the at sign, which GFM +// section 6.9 makes of characters "which are alphanumeric, or ., -, _, or +" +inline auto is_email_local_character(const char character) noexcept -> bool { + return sourcemeta::core::is_alphanum(character) || character == '.' || + character == '-' || character == '_' || character == '+'; +} + +// The passes that run on the block structure once it is complete +class PostProcessor { +public: + explicit PostProcessor(Document &document) : document_{document} {} + + // Parse the inlines of every leaf block in document order, which is the + // order in which references count towards their expansion limit. Blocks + // without footnote references are finished right away, while their nodes + // are still in the cache, and the rest wait for footnotes to resolve + auto parse_inlines(InlineParser &parser) -> void { + this->deferred_.clear(); + auto current{this->document_.nodes[ROOT_NODE].first_child}; + while (current != NO_NODE) { + const auto type{this->document_.nodes[current].type}; + if (type == NodeType::Paragraph || type == NodeType::Heading || + type == NodeType::TableCell) { + const auto references{this->document_.footnote_reference_nodes.size()}; + parser.parse(current); + if (this->document_.footnote_reference_nodes.size() == references) { + this->finish_inlines(current); + } else { + this->deferred_.push_back(current); + } + + current = + next_skipping_descendants(this->document_, current, ROOT_NODE); + } else if (this->document_.nodes[current].first_child != NO_NODE) { + current = this->document_.nodes[current].first_child; + } else { + current = + next_skipping_descendants(this->document_, current, ROOT_NODE); + } + } + } + + // Number footnote definitions in the order in which they are first + // referenced and move them to the end of the document, dropping those that + // are never referenced or that repeat a label + auto process_footnotes() -> void { + const auto &definitions{this->document_.footnote_definition_nodes}; + const auto &references{this->document_.footnote_reference_nodes}; + if (definitions.empty() && references.empty()) { + return; + } + + auto &nodes{this->document_.nodes}; + this->definitions_.clear(); + this->definition_nodes_.clear(); + // Without nesting, the order in which definitions were opened matches the + // order in which they end, which is the order in which labels claim them + if (this->document_.nested_footnote_definitions) { + this->register_definitions_in_post_order(); + } else { + for (const auto definition : definitions) { + this->register_definition(definition); + } + } + + std::uint32_t last_index{0}; + for (const auto reference : references) { + if (this->is_attached(reference)) { + this->resolve_reference(reference, last_index); + } + } + + this->ordered_.clear(); + for (const auto &entry : this->definitions_) { + this->ordered_.push_back(entry.second); + } + + std::sort(this->ordered_.begin(), this->ordered_.end(), + [&nodes](const std::uint32_t left, const std::uint32_t right) { + return nodes[left].extra < nodes[right].extra; + }); + for (const auto definition : this->ordered_) { + this->document_.unlink(definition); + if (nodes[definition].extra > 0) { + set_flag(nodes[definition], FLAG_REFERENCED, true); + this->document_.append_child(ROOT_NODE, definition); + } + } + + for (const auto definition : this->definition_nodes_) { + if (!has_flag(nodes[definition], FLAG_REFERENCED)) { + this->document_.unlink(definition); + } + } + + for (const auto block : this->deferred_) { + this->finish_inlines(block); + } + } + +private: + // Merge the adjacent text nodes of a leaf block and turn the email + // addresses of GFM section 6.9 that appear outside of links into links + auto finish_inlines(const std::uint32_t block) -> void { + auto current{this->document_.nodes[block].first_child}; + while (current != NO_NODE) { + if (this->document_.nodes[current].type == NodeType::Text) { + this->merge_following_text(current); + } + + if (this->document_.nodes[current].first_child != NO_NODE) { + current = this->document_.nodes[current].first_child; + } else { + current = next_skipping_descendants(this->document_, current, block); + } + } + + // Only text with an at sign, which a character reference may produce, can + // have email addresses + const auto content{ + this->document_.content_of(this->document_.nodes[block])}; + if (content.empty() || + (std::memchr(content.data(), '@', content.size()) == nullptr && + std::memchr(content.data(), '&', content.size()) == nullptr)) { + return; + } + + current = this->document_.nodes[block].first_child; + while (current != NO_NODE) { + const auto type{this->document_.nodes[current].type}; + if (type == NodeType::Text) { + const auto next{ + next_skipping_descendants(this->document_, current, block)}; + this->link_emails_in_text(current); + current = next; + } else if (type != NodeType::Link && + this->document_.nodes[current].first_child != NO_NODE) { + current = this->document_.nodes[current].first_child; + } else { + current = next_skipping_descendants(this->document_, current, block); + } + } + } + + auto register_definitions_in_post_order() -> void { + const auto &nodes{this->document_.nodes}; + auto current{ROOT_NODE}; + bool entering{true}; + while (true) { + if (entering && !is_leaf_type(nodes[current].type) && + nodes[current].first_child != NO_NODE) { + current = nodes[current].first_child; + continue; + } + + if (nodes[current].type == NodeType::FootnoteDefinition) { + this->register_definition(current); + } + + if (current == ROOT_NODE) { + break; + } + + entering = nodes[current].next != NO_NODE; + current = entering ? nodes[current].next : nodes[current].parent; + } + } + + auto register_definition(const std::uint32_t index) -> void { + normalize_label(this->label_buffer_, this->document_.nodes[index].literal); + if (this->label_buffer_.empty()) { + return; + } + + this->definition_nodes_.push_back(index); + if (!this->definitions_.contains(std::string_view{this->label_buffer_})) { + this->definitions_.emplace( + this->document_.strings.store(this->label_buffer_), index); + } + } + + auto resolve_reference(const std::uint32_t index, std::uint32_t &last_index) + -> void { + auto &nodes{this->document_.nodes}; + const auto label{nodes[index].literal}; + auto definition{NO_NODE}; + if (sourcemeta::core::utf8_codepoint_within(label, 1, + MAXIMUM_LINK_LABEL_LENGTH)) { + normalize_label(this->label_buffer_, label); + const auto match{ + this->definitions_.find(std::string_view{this->label_buffer_})}; + if (match != this->definitions_.end()) { + definition = match->second; + } + } + + if (definition == NO_NODE) { + this->buffer_.assign("[^"); + this->buffer_.append(label); + this->buffer_.push_back(']'); + const auto text{this->document_.create(NodeType::Text)}; + nodes[text].literal = this->document_.strings.store(this->buffer_); + this->document_.insert_after(index, text); + this->document_.unlink(index); + return; + } + + if (nodes[definition].extra == 0) { + ++last_index; + nodes[definition].extra = last_index; + } + + ++nodes[definition].data; + nodes[index].data = definition; + nodes[index].extra = nodes[definition].data; + sourcemeta::core::DigitsBuffer digits; + nodes[index].literal = this->document_.strings.store( + sourcemeta::core::digits_view(nodes[definition].extra, digits)); + } + + [[nodiscard]] auto is_in_content(const std::string_view value) const noexcept + -> bool { + const std::less before{}; + const auto *const begin{this->document_.content.data()}; + const auto *const end{begin + this->document_.content.size()}; + return !before(value.data(), begin) && + !before(end, value.data() + value.size()); + } + + auto merge_following_text(const std::uint32_t index) -> void { + auto &nodes{this->document_.nodes}; + auto next{nodes[index].next}; + if (next == NO_NODE || nodes[next].type != NodeType::Text) { + return; + } + + auto literal{nodes[index].literal}; + bool buffered{false}; + while (next != NO_NODE && nodes[next].type == NodeType::Text) { + const auto piece{nodes[next].literal}; + if (buffered) { + this->buffer_.append(piece); + } else if (literal.empty()) { + literal = piece; + } else if (!piece.empty()) { + if (literal.data() + literal.size() == piece.data() && + this->is_in_content(literal) && this->is_in_content(piece)) { + literal = + std::string_view{literal.data(), literal.size() + piece.size()}; + } else { + this->buffer_.assign(literal); + this->buffer_.append(piece); + buffered = true; + } + } + + const auto following{nodes[next].next}; + this->document_.unlink(next); + next = following; + } + + nodes[index].literal = + buffered ? this->document_.strings.store(this->buffer_) : literal; + } + + // Whether a node is still part of the document, remembering the answer for + // every ancestor on the way, so that the references of a deeply nested block + // do not walk the same ancestors over and over + auto is_attached(const std::uint32_t index) -> bool { + auto &nodes{this->document_.nodes}; + this->ancestors_.clear(); + auto current{index}; + while (current != ROOT_NODE && current != NO_NODE && + !has_flag(nodes[current], FLAG_ATTACHED) && + !has_flag(nodes[current], FLAG_DETACHED)) { + this->ancestors_.push_back(current); + current = nodes[current].parent; + } + + const auto attached{ + current == ROOT_NODE || + (current != NO_NODE && has_flag(nodes[current], FLAG_ATTACHED))}; + for (const auto ancestor : this->ancestors_) { + set_flag(nodes[ancestor], attached ? FLAG_ATTACHED : FLAG_DETACHED, true); + } + + return attached; + } + + // Whether an extended autolink may start at the beginning of a text node, + // where GFM section 6.9 says that such autolinks "can only come at the + // beginning of a line, after whitespace, or any of the delimiting characters + // *, _, ~, and (", which are also the characters around emphasis, strong + // emphasis, and strikethrough + [[nodiscard]] auto may_start_autolink_at(std::uint32_t text) const noexcept + -> bool { + const auto &nodes{this->document_.nodes}; + while (true) { + const auto previous{nodes[text].previous}; + if (previous == NO_NODE) { + return nodes[nodes[text].parent].type != NodeType::Image; + } + + const auto &node{nodes[previous]}; + switch (node.type) { + case NodeType::Text: + if (!node.literal.empty()) { + return may_precede_extended_autolink(node.literal.back()); + } + + text = previous; + break; + case NodeType::SoftBreak: + case NodeType::LineBreak: + case NodeType::Emphasis: + case NodeType::Strong: + case NodeType::Strikethrough: + return true; + case NodeType::Document: + case NodeType::BlockQuote: + case NodeType::List: + case NodeType::Item: + case NodeType::CodeBlock: + case NodeType::HTMLBlock: + case NodeType::Paragraph: + case NodeType::Heading: + case NodeType::ThematicBreak: + case NodeType::FootnoteDefinition: + case NodeType::Table: + case NodeType::TableRow: + case NodeType::TableCell: + case NodeType::Code: + case NodeType::HTMLInline: + case NodeType::Link: + case NodeType::Image: + case NodeType::FootnoteReference: + return false; + } + } + } + + // Turn the email addresses of GFM section 6.9 in a text node into links, + // along with the mailto and xmpp protocols that may come before them + auto link_emails_in_text(std::uint32_t text) -> void { + auto &nodes{this->document_.nodes}; + const auto data{nodes[text].literal}; + const auto first_text{text}; + // The position of the data at which the current text node starts + std::size_t consumed{0}; + std::size_t search{0}; + while (search < data.size()) { + const auto *const found{static_cast( + std::memchr(data.data() + search, '@', data.size() - search))}; + if (found == nullptr) { + break; + } + + const auto separator{static_cast(found - data.data())}; + search = separator + 1; + auto link_start{separator}; + while (link_start > consumed && + is_email_local_character(data[link_start - 1])) { + --link_start; + } + + if (link_start == separator) { + continue; + } + + // GFM section 6.9: "One or more characters which are alphanumeric, or - + // or _, separated by periods (.). There must be at least one period. The + // last character must not be one of - or _", where "only . may occur at + // the end of the email address, in which case it will not be considered + // part of the address" + auto end{separator + 1}; + std::size_t periods{0}; + while (end < data.size()) { + if (is_domain_character(data[end])) { + ++end; + } else if (data[end] == '.' && end > separator + 1 && + end + 1 < data.size() && + is_domain_character(data[end + 1])) { + ++periods; + ++end; + } else { + break; + } + } + + if (periods == 0 || data[end - 1] == '-' || data[end - 1] == '_') { + continue; + } + + // GFM section 6.9: "An extended protocol autolink will be recognised + // when a protocol is recognised within any text node", where the valid + // protocols are mailto and xmpp + bool has_protocol{false}; + bool xmpp{false}; + for (const auto protocol : + std::array{{"mailto:", "xmpp:"}}) { + if (link_start - consumed >= protocol.size() && + sourcemeta::core::starts_with_ignore_case( + data.substr(link_start - protocol.size(), protocol.size()), + protocol)) { + link_start -= protocol.size(); + has_protocol = true; + xmpp = protocol == "xmpp:"; + break; + } + } + + // GFM section 6.9: xmpp "offers an optional / followed by a resource. + // The resource can contain all alphanumeric characters, as well as @ and + // .", while "Further / characters are not considered part of the domain" + if (xmpp && end + 1 < data.size() && data[end] == '/') { + auto resource_end{end + 1}; + while (resource_end < data.size() && + (sourcemeta::core::is_alphanum(data[resource_end]) || + data[resource_end] == '@' || data[resource_end] == '.')) { + ++resource_end; + } + + while (resource_end > end + 1 && data[resource_end - 1] == '.') { + --resource_end; + } + + if (resource_end > end + 1) { + end = resource_end; + } + } + + if (link_start == consumed + ? consumed > 0 || !this->may_start_autolink_at(first_text) + : !may_precede_extended_autolink(data[link_start - 1])) { + continue; + } + + const auto address{data.substr(link_start, end - link_start)}; + const auto link{this->document_.create(NodeType::Link)}; + if (has_protocol) { + nodes[link].literal = address; + } else { + this->buffer_.assign("mailto:"); + this->buffer_.append(address); + nodes[link].literal = this->document_.strings.store(this->buffer_); + } + + const auto link_text{this->document_.create(NodeType::Text)}; + nodes[link_text].literal = address; + this->document_.append_child(link, link_text); + this->document_.insert_after(text, link); + const auto after{this->document_.create(NodeType::Text)}; + nodes[after].literal = data.substr(end); + this->document_.insert_after(link, after); + nodes[text].literal = data.substr(consumed, link_start - consumed); + text = after; + consumed = end; + search = end; + } + } + + Document &document_; + std::unordered_map definitions_; + std::vector definition_nodes_; + std::vector ordered_; + std::vector deferred_; + std::vector ancestors_; + std::string label_buffer_; + std::string buffer_; +}; + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/references.h b/src/core/markdown/references.h new file mode 100644 index 0000000000..6fcc5fc21f --- /dev/null +++ b/src/core/markdown/references.h @@ -0,0 +1,310 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_REFERENCES_H_ +#define SOURCEMETA_CORE_MARKDOWN_REFERENCES_H_ + +#include + +#include "characters.h" +#include "document.h" +#include "scanners.h" + +#include // std::size_t, std::ptrdiff_t +#include // std::string +#include // std::string_view + +namespace sourcemeta::core::markdown { + +// GFM section 6.6: "A link label can have at most 999 characters inside the +// square brackets", where GFM section 2.1 says that "A character is a Unicode +// code point" +constexpr std::size_t MAXIMUM_LINK_LABEL_LENGTH{999}; + +// GFM section 6.6: "Implementations may impose limits on parentheses nesting to +// avoid performance issues, but at least three levels of nesting should be +// supported" +constexpr std::size_t MAXIMUM_DESTINATION_PARENTHESES{32}; + +// A link label of GFM section 4.7, moving the position past it on success +inline auto scan_link_label(const std::string_view input, std::size_t &position, + std::string_view &label) noexcept -> bool { + const auto start{position}; + if (character_at(input, start) != '[') { + return false; + } + + auto cursor{start + 1}; + std::size_t length{0}; + while (cursor < input.size()) { + const auto character{input[cursor]}; + if (character == '[' || character == ']') { + break; + } + + ++cursor; + if (!sourcemeta::core::is_utf8_continuation( + static_cast(character))) { + ++length; + } + + if (character == '\\' && + sourcemeta::core::is_punctuation(character_at(input, cursor))) { + ++cursor; + ++length; + } + + if (length > MAXIMUM_LINK_LABEL_LENGTH) { + return false; + } + } + + if (character_at(input, cursor) != ']') { + return false; + } + + label = sourcemeta::core::trim(input.substr(start + 1, cursor - start - 1), + is_space); + position = cursor + 1; + return true; +} + +// A link destination of GFM section 6.3, returning the length it takes +// including any angle brackets, or minus one if there is none +inline auto scan_link_destination(const std::string_view input, + const std::size_t offset, + std::string_view &destination) noexcept + -> std::ptrdiff_t { + const auto size{input.size()}; + auto index{offset}; + if (index < size && input[index] == '<') { + ++index; + while (index < size) { + const auto character{input[index]}; + if (character == '>') { + ++index; + break; + } + + // GFM section 6.3: a link destination can be "a sequence of zero or + // more characters between an opening < and a closing > that contains + // no line breaks or unescaped < or > characters", where GFM section 2.4 + // only lets "Any ASCII punctuation character" be backslash-escaped + if (character == '\\' && index + 1 < size && + sourcemeta::core::is_punctuation(input[index + 1])) { + index += 2; + } else if (character == '\n' || character == '\r' || character == '<') { + return -1; + } else { + ++index; + } + } + + if (index >= size) { + return -1; + } + + destination = input.substr(offset + 1, index - offset - 2); + return static_cast(index - offset); + } + + std::size_t parentheses{0}; + while (index < size) { + const auto character{input[index]}; + if (character == '\\' && index + 1 < size && + sourcemeta::core::is_punctuation(input[index + 1])) { + index += 2; + } else if (character == '(') { + ++parentheses; + ++index; + if (parentheses > MAXIMUM_DESTINATION_PARENTHESES) { + return -1; + } + } else if (character == ')') { + if (parentheses == 0) { + break; + } + + --parentheses; + ++index; + } else if (is_space_or_control(character)) { + if (index == offset) { + return -1; + } + + break; + } else { + ++index; + } + } + + if (index >= size) { + return -1; + } + + destination = input.substr(offset, index - offset); + return static_cast(index - offset); +} + +// Resolve the character references and backslash escapes of a link +// destination, where a destination that needs neither and lives in stable +// storage is returned as it is +inline auto clean_url(Document &document, const std::string_view url, + std::string &buffer, const bool stable) + -> std::string_view { + if (url.empty()) { + return {}; + } + + if (stable && url.find_first_of("&\\") == std::string_view::npos) { + return url; + } + + buffer.clear(); + decode_escapes_and_references(buffer, url); + return document.strings.store(buffer); +} + +// Resolve the character references and backslash escapes of a link title, +// dropping its surrounding delimiters +inline auto clean_title(Document &document, const std::string_view title, + std::string &buffer, const bool stable) + -> std::string_view { + if (title.empty()) { + return {}; + } + + const auto first{title.front()}; + const auto last{title.back()}; + auto inner{title}; + if ((first == '\'' && last == '\'') || (first == '(' && last == ')') || + (first == '"' && last == '"')) { + inner = title.size() < 2 ? std::string_view{} + : title.substr(1, title.size() - 2); + } + + if (stable && inner.find_first_of("&\\") == std::string_view::npos) { + return inner; + } + + buffer.clear(); + decode_escapes_and_references(buffer, inner); + return document.strings.store(buffer); +} + +inline auto skip_spaces(const std::string_view input, + std::size_t &position) noexcept -> void { + while (position < input.size() && + (input[position] == ' ' || input[position] == '\t')) { + ++position; + } +} + +inline auto skip_line_end(const std::string_view input, + std::size_t &position) noexcept -> bool { + bool seen{false}; + if (character_at(input, position) == '\r') { + ++position; + seen = true; + } + + if (character_at(input, position) == '\n') { + ++position; + seen = true; + } + + return seen || position >= input.size(); +} + +// Spaces or tabs, including up to one line ending +inline auto skip_spaces_and_line_end(const std::string_view input, + std::size_t &position) noexcept -> void { + skip_spaces(input, position); + if (skip_line_end(input, position)) { + skip_spaces(input, position); + } +} + +// A link reference definition of GFM section 4.7 at the start of the input, +// which is registered unless a previous definition has the same label, +// returning the length it takes or zero if there is none +inline auto +parse_reference_definition(Document &document, const std::string_view input, + std::string &label_buffer, std::string &value_buffer) + -> std::size_t { + std::size_t position{0}; + std::string_view label; + if (!scan_link_label(input, position, label) || label.empty() || + character_at(input, position) != ':') { + return 0; + } + + ++position; + skip_spaces_and_line_end(input, position); + std::string_view url; + const auto url_length{scan_link_destination(input, position, url)}; + if (url_length < 0) { + return 0; + } + + position += static_cast(url_length); + const auto before_title{position}; + skip_spaces_and_line_end(input, position); + const auto title_length{ + position == before_title ? 0 : scan_link_title(input, position)}; + std::string_view title; + if (title_length > 0) { + title = input.substr(position, title_length); + position += title_length; + } else { + position = before_title; + } + + skip_spaces(input, position); + if (!skip_line_end(input, position)) { + if (title_length == 0) { + return 0; + } + + position = before_title; + skip_spaces(input, position); + if (!skip_line_end(input, position)) { + return 0; + } + + // GFM section 4.7, example 179: a title followed by other text on its line + // leaves "a link reference definition, but it has no title" + title = {}; + } + + normalize_label(label_buffer, label); + if (!label_buffer.empty() && + !document.references.contains(std::string_view{label_buffer})) { + const auto key{document.strings.store(label_buffer)}; + const auto cleaned_url{clean_url(document, url, value_buffer, false)}; + const auto cleaned_title{clean_title(document, title, value_buffer, false)}; + document.references.emplace( + key, Reference{.url = cleaned_url, .title = cleaned_title}); + } + + return position; +} + +// Find the definition of a link label +inline auto find_reference(const Document &document, + const std::string_view label, std::string &buffer) + -> const Reference * { + if (document.references.empty() || !sourcemeta::core::utf8_codepoint_within( + label, 1, MAXIMUM_LINK_LABEL_LENGTH)) { + return nullptr; + } + + normalize_label(buffer, label); + if (buffer.empty()) { + return nullptr; + } + + const auto match{document.references.find(std::string_view{buffer})}; + return match == document.references.end() ? nullptr : &match->second; +} + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/render.h b/src/core/markdown/render.h new file mode 100644 index 0000000000..a176b393da --- /dev/null +++ b/src/core/markdown/render.h @@ -0,0 +1,625 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_RENDER_H_ +#define SOURCEMETA_CORE_MARKDOWN_RENDER_H_ + +#include +#include +#include +#include + +#include "characters.h" +#include "document.h" +#include "postprocess.h" +#include "scanners.h" + +#include // std::array +#include // std::size_t +#include // std::uint8_t, std::uint32_t +#include // std::string +#include // std::string_view + +namespace sourcemeta::core::markdown { + +// The tags that the GFM tag filter of section 6.11 disallows +constexpr std::array FILTERED_TAG_NAMES{ + {"title", "textarea", "style", "xmp", "iframe", "noembed", "noframes", + "script", "plaintext"}}; + +// Whether raw HTML starts with a tag that the tag filter disallows. The tag +// name state of the HTML Standard tokenizer ends a tag name at "U+0009 +// CHARACTER TABULATION (tab) U+000A LINE FEED (LF) U+000C FORM FEED (FF) +// U+0020 SPACE", at "U+002F SOLIDUS (/)", and at "U+003E GREATER-THAN SIGN +// (>)", where a carriage return reaches the tokenizer as a line feed once the +// input stream normalizes newlines +inline auto is_filtered_tag(const std::string_view tag) noexcept -> bool { + if (tag.size() < 3 || tag[0] != '<') { + return false; + } + + for (const auto name : FILTERED_TAG_NAMES) { + std::size_t index{tag[1] == '/' ? 2U : 1U}; + std::size_t name_index{0}; + while (index < tag.size() && name_index < name.size() && + sourcemeta::core::to_lowercase(tag[index]) == name[name_index]) { + ++index; + ++name_index; + } + + if (name_index < name.size() || index == tag.size()) { + continue; + } + + switch (tag[index]) { + case '\t': + case '\n': + case '\f': + case '\r': + case ' ': + case '/': + case '>': + return true; + default: + break; + } + } + + return false; +} + +class HTMLRenderer { +public: + explicit HTMLRenderer(const Document &document) : document_{document} {} + + auto render(const bool unsafe, const std::size_t capacity) -> std::string { + // A conversion that threw while rendering leaves its partial output behind + this->writer_.clear(); + this->writer_.reserve(capacity); + this->reference_size_ = 0; + this->unsafe_ = unsafe; + this->footnote_index_ = 0; + this->written_footnote_index_ = 0; + this->needs_closing_table_body_ = false; + this->in_table_header_ = false; + const auto &nodes{this->document_.nodes}; + auto current{ROOT_NODE}; + bool entering{true}; + while (true) { + this->render_node(current, entering); + const auto &node{nodes[current]}; + // An image renders the text of its descendants as its alternative text + // when entering it, so they are not visited on their own + if (entering && !is_leaf_type(node.type) && + node.type != NodeType::Image) { + if (node.first_child != NO_NODE) { + current = node.first_child; + } else { + entering = false; + } + + continue; + } + + if (current == ROOT_NODE) { + break; + } + + entering = node.next != NO_NODE; + current = entering ? node.next : node.parent; + } + + if (this->footnote_index_ > 0) { + this->writer_.close().raw("\n").close().raw("\n"); + } + + return this->writer_.take(); + } + +private: + // A link destination, which safe mode drops when its scheme can run code or + // read local files + auto destination(const std::string_view url) -> std::string_view { + this->value_.clear(); + if (this->unsafe_ || !is_dangerous_url(url)) { + sourcemeta::core::URI::escape_reference(url, this->value_); + } + + return this->value_.view(); + } + + // The fragment that links to a footnote element, including the number sign + // that introduces it. The label is part of the fragment, where RFC 3986 + // Section 3.5 leaves no room for another number sign + auto footnote_fragment(const std::string_view prefix, + const std::string_view label) -> std::string_view { + this->fragment_.clear(); + this->fragment_.append('#'); + this->fragment_.append(prefix); + this->fragment_.append(label); + this->value_.clear(); + sourcemeta::core::URI::escape_reference(this->fragment_.view(), + this->value_); + return this->value_.view(); + } + + // The alternative text of an image, which is the text of its descendants + // without their markup + auto alternative_text(const std::uint32_t image) -> std::string_view { + this->value_.clear(); + const auto &nodes{this->document_.nodes}; + auto current{nodes[image].first_child}; + while (current != NO_NODE) { + const auto &node{nodes[current]}; + if (node.type == NodeType::Text || node.type == NodeType::Code || + node.type == NodeType::HTMLInline) { + this->value_.append(node.literal); + } else if (node.type == NodeType::SoftBreak || + node.type == NodeType::LineBreak) { + this->value_.append(' '); + } + + current = + node.first_child != NO_NODE + ? node.first_child + : next_skipping_descendants(this->document_, current, image); + } + + return this->value_.view(); + } + + auto append_footnote_backreference(const Node &definition) -> bool { + if (this->written_footnote_index_ >= this->footnote_index_) { + return false; + } + + this->written_footnote_index_ = this->footnote_index_; + auto &writer{this->writer_}; + writer.a().attribute("href", + this->footnote_fragment("fnref-", definition.literal)); + writer.attribute("class", "footnote-backref") + .attribute("data-footnote-backref"); + sourcemeta::core::DigitsBuffer written_digits; + const auto written_index{sourcemeta::core::digits_view( + this->written_footnote_index_, written_digits)}; + writer.attribute("data-footnote-backref-idx", written_index); + this->value_.clear(); + this->value_.append("Back to reference "); + this->value_.append(written_index); + writer.attribute("aria-label", this->value_.view()); + writer.raw("\xE2\x86\xA9").close(); + for (std::uint32_t index{2}; index <= definition.data; ++index) { + sourcemeta::core::DigitsBuffer repeat_digits; + const auto repeat{sourcemeta::core::digits_view(index, repeat_digits)}; + writer.raw(" "); + this->footnote_fragment("fnref-", definition.literal); + this->value_.append('-'); + this->value_.append(repeat); + writer.a().attribute("href", this->value_.view()); + writer.attribute("class", "footnote-backref") + .attribute("data-footnote-backref"); + this->value_.clear(); + this->value_.append(written_index); + this->value_.append('-'); + this->value_.append(repeat); + writer.attribute("data-footnote-backref-idx", this->value_.view()); + this->value_.clear(); + this->value_.append("Back to reference "); + this->value_.append(written_index); + this->value_.append('-'); + this->value_.append(repeat); + writer.attribute("aria-label", this->value_.view()); + writer.raw("\xE2\x86\xA9"); + writer.sup().attribute("class", "footnote-ref").text(repeat); + writer.close().close(); + } + + return true; + } + + [[nodiscard]] auto is_tight_paragraph(const Node ¶graph) const noexcept + -> bool { + const auto &nodes{this->document_.nodes}; + const auto grandparent{nodes[paragraph.parent].parent}; + return grandparent != NO_NODE && + nodes[grandparent].type == NodeType::List && + has_flag(nodes[grandparent], FLAG_TIGHT); + } + + auto open_heading(const std::uint8_t level) -> void { + switch (level) { + case 1: + this->writer_.h1(); + break; + case 2: + this->writer_.h2(); + break; + case 3: + this->writer_.h3(); + break; + case 4: + this->writer_.h4(); + break; + case 5: + this->writer_.h5(); + break; + default: + this->writer_.h6(); + break; + } + } + + // The destinations and titles that rendered link references expand to, which + // the specification lets grow quadratically with the input + auto count_reference_expansion(const Node &node) -> void { + if (!has_flag(node, FLAG_REFERENCE_LINK)) { + return; + } + + this->reference_size_ += node.literal.size() + node.title.size(); + if (this->reference_size_ > this->document_.reference_size_limit) { + throw sourcemeta::core::MarkdownError{ + "The link references expand past the size bound"}; + } + } + + auto render_code_block(const Node &node) -> void { + auto &writer{this->writer_}; + writer.ensure_line_feed(); + writer.pre().code(); + const auto info{node.title}; + if (!info.empty()) { + std::size_t first_tag{0}; + while (first_tag < info.size() && !is_space(info[first_tag])) { + ++first_tag; + } + + // GFM section 4.5: "The first word of the info string is typically used + // to specify the language of the code sample, and rendered in the class + // attribute of the code tag", which is how the HTML Standard marks the + // language of computer code too, "by adding a class prefixed with + // "language-" to the element", as its lang attribute is for the + // language of the text + this->value_.clear(); + this->value_.append("language-"); + this->value_.append(info.substr(0, first_tag)); + writer.attribute("class", this->value_.view()); + } + + writer.text(node.literal.data() != nullptr + ? node.literal + : this->document_.content_of(node)); + writer.close().close().raw("\n"); + } + + auto render_table_cell(const Node &node, const bool entering) -> void { + auto &writer{this->writer_}; + if (!entering) { + writer.close(); + return; + } + + writer.ensure_line_feed(); + if (this->in_table_header_) { + writer.th(); + } else { + writer.td(); + } + + const auto &nodes{this->document_.nodes}; + const auto &table{nodes[nodes[node.parent].parent]}; + const auto &data{this->document_.tables[table.data]}; + switch (this->document_.alignments[data.alignments_offset + node.data]) { + case 'l': + writer.attribute("align", "left"); + break; + case 'c': + writer.attribute("align", "center"); + break; + case 'r': + writer.attribute("align", "right"); + break; + default: + break; + } + } + + auto render_node(const std::uint32_t index, const bool entering) -> void { + const auto &nodes{this->document_.nodes}; + const auto &node{nodes[index]}; + auto &writer{this->writer_}; + switch (node.type) { + case NodeType::Document: + break; + case NodeType::BlockQuote: + writer.ensure_line_feed(); + if (entering) { + writer.blockquote(); + } else { + writer.close(); + } + + writer.raw("\n"); + break; + case NodeType::List: { + if (!entering) { + writer.close().raw("\n"); + break; + } + + const auto &data{this->document_.lists[node.data]}; + writer.ensure_line_feed(); + if (!data.ordered) { + writer.ul(); + } else { + writer.ol(); + if (data.start != 1) { + sourcemeta::core::DigitsBuffer digits; + writer.attribute("start", + sourcemeta::core::digits_view(data.start, digits)); + } + } + + writer.raw("\n"); + break; + } + + case NodeType::Item: + if (!entering) { + writer.close().raw("\n"); + break; + } + + writer.ensure_line_feed(); + writer.li(); + if (has_flag(node, FLAG_TASK)) { + writer.input().attribute("type", "checkbox"); + if (has_flag(node, FLAG_CHECKED)) { + writer.attribute("checked", ""); + } + + writer.attribute("disabled", "").raw(" "); + } + + break; + case NodeType::Heading: + if (entering) { + writer.ensure_line_feed(); + this->open_heading(node.level); + } else { + writer.close().raw("\n"); + } + + break; + case NodeType::CodeBlock: + this->render_code_block(node); + break; + case NodeType::HTMLBlock: + writer.ensure_line_feed(); + if (!this->unsafe_) { + writer.raw(""); + } else { + this->append_filtered_html(this->document_.content_of(node)); + } + + writer.ensure_line_feed(); + break; + case NodeType::ThematicBreak: + writer.ensure_line_feed(); + writer.hr().raw("\n"); + break; + case NodeType::Paragraph: + if (this->is_tight_paragraph(node)) { + break; + } + + if (entering) { + writer.ensure_line_feed(); + writer.p(); + break; + } + + if (nodes[node.parent].type == NodeType::FootnoteDefinition && + node.next == NO_NODE) { + writer.raw(" "); + this->append_footnote_backreference(nodes[node.parent]); + } + + writer.close().raw("\n"); + break; + case NodeType::Text: + writer.text(node.literal); + break; + case NodeType::SoftBreak: + writer.raw("\n"); + break; + case NodeType::LineBreak: + writer.br().raw("\n"); + break; + case NodeType::Code: + writer.code(node.literal); + break; + case NodeType::HTMLInline: + if (!this->unsafe_) { + writer.raw(""); + } else if (is_filtered_tag(node.literal)) { + writer.raw("<").raw(node.literal.substr(1)); + } else { + writer.raw(node.literal); + } + + break; + // GFM section 6.4, example 473: "****foo****" renders as + // "

foo

" + case NodeType::Strong: + if (entering) { + writer.strong(); + } else { + writer.close(); + } + + break; + case NodeType::Emphasis: + if (entering) { + writer.em(); + } else { + writer.close(); + } + + break; + case NodeType::Strikethrough: + if (entering) { + writer.del(); + } else { + writer.close(); + } + + break; + case NodeType::Link: + if (!entering) { + writer.close(); + break; + } + + this->count_reference_expansion(node); + writer.a().attribute("href", this->destination(node.literal)); + if (!node.title.empty()) { + writer.attribute("title", node.title); + } + + break; + case NodeType::Image: + this->count_reference_expansion(node); + writer.img().attribute("src", this->destination(node.literal)); + writer.attribute("alt", this->alternative_text(index)); + if (!node.title.empty()) { + writer.attribute("title", node.title); + } + + break; + case NodeType::FootnoteDefinition: + if (entering) { + if (this->footnote_index_ == 0) { + writer.section() + .attribute("class", "footnotes") + .attribute("data-footnotes") + .raw("\n"); + writer.ol().raw("\n"); + } + + ++this->footnote_index_; + writer.li() + .attribute("id", + this->footnote_fragment("fn-", node.literal).substr(1)) + .raw("\n"); + break; + } + + if (this->append_footnote_backreference(node)) { + writer.raw("\n"); + } + + writer.close().raw("\n"); + break; + case NodeType::FootnoteReference: { + if (!entering) { + break; + } + + const auto &definition{nodes[node.data]}; + writer.sup().attribute("class", "footnote-ref"); + writer.a().attribute( + "href", this->footnote_fragment("fn-", definition.literal)); + this->footnote_fragment("fnref-", definition.literal); + if (node.extra > 1) { + sourcemeta::core::DigitsBuffer extra_digits; + this->value_.append('-'); + this->value_.append( + sourcemeta::core::digits_view(node.extra, extra_digits)); + } + + writer.attribute("id", this->value_.view().substr(1)) + .attribute("data-footnote-ref"); + writer.text(node.literal).close().close(); + break; + } + + case NodeType::Table: + if (entering) { + writer.ensure_line_feed(); + writer.table(); + this->needs_closing_table_body_ = false; + break; + } + + if (this->needs_closing_table_body_) { + writer.ensure_line_feed(); + writer.close(); + writer.ensure_line_feed(); + } + + this->needs_closing_table_body_ = false; + writer.ensure_line_feed(); + writer.close(); + writer.ensure_line_feed(); + break; + case NodeType::TableRow: + writer.ensure_line_feed(); + if (!entering) { + writer.close(); + if (has_flag(node, FLAG_HEADER)) { + writer.ensure_line_feed(); + writer.close(); + this->in_table_header_ = false; + } + + break; + } + + if (has_flag(node, FLAG_HEADER)) { + this->in_table_header_ = true; + writer.thead(); + writer.ensure_line_feed(); + } else if (!this->needs_closing_table_body_) { + writer.tbody(); + writer.ensure_line_feed(); + this->needs_closing_table_body_ = true; + } + + writer.tr(); + break; + case NodeType::TableCell: + this->render_table_cell(node, entering); + break; + } + } + + auto append_filtered_html(std::string_view html) -> void { + auto &writer{this->writer_}; + while (!html.empty()) { + const auto bracket{html.find('<')}; + if (bracket == std::string_view::npos) { + break; + } + + writer.raw(html.substr(0, bracket)); + html.remove_prefix(bracket); + writer.raw(is_filtered_tag(html) ? "<" : "<"); + html.remove_prefix(1); + } + + writer.raw(html); + } + + const Document &document_; + sourcemeta::core::HTMLWriter writer_; + sourcemeta::core::HTMLBuffer value_; + sourcemeta::core::HTMLBuffer fragment_; + std::size_t reference_size_{0}; + bool unsafe_{false}; + std::uint32_t footnote_index_{0}; + std::uint32_t written_footnote_index_{0}; + bool needs_closing_table_body_{false}; + bool in_table_header_{false}; +}; + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/src/core/markdown/scanners.h b/src/core/markdown/scanners.h new file mode 100644 index 0000000000..66caa0b375 --- /dev/null +++ b/src/core/markdown/scanners.h @@ -0,0 +1,857 @@ +#ifndef SOURCEMETA_CORE_MARKDOWN_SCANNERS_H_ +#define SOURCEMETA_CORE_MARKDOWN_SCANNERS_H_ + +#include +#include +#include + +#include "characters.h" + +#include // std::binary_search +#include // std::array +#include // std::size_t +#include // std::uint8_t +#include // std::string_view + +// Every scanner matches at the given offset of the input, never looks past +// its end, and returns the length of the match, where zero means no match + +namespace sourcemeta::core::markdown { + +inline auto character_at(const std::string_view input, + const std::size_t index) noexcept -> char { + return index < input.size() ? input[index] : '\0'; +} + +// The whitespace that raw HTML allows between its parts +inline auto is_html_space(const char character) noexcept -> bool { + return character == ' ' || character == '\t' || character == '\v' || + character == '\f' || character == '\r' || character == '\n'; +} + +// The whitespace that the table and task list extensions allow within a line +inline auto is_line_space(const char character) noexcept -> bool { + return character == ' ' || character == '\t' || character == '\v' || + character == '\f'; +} + +inline auto scan_spacechars(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + auto position{offset}; + while (position < input.size() && is_html_space(input[position])) { + ++position; + } + + return position - offset; +} + +// An absolute URI of GFM section 6.9, right after the opening angle bracket +inline auto scan_autolink_uri(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + auto position{offset}; + if (!sourcemeta::core::is_alpha(character_at(input, position))) { + return 0; + } + + ++position; + std::size_t scheme_length{0}; + while (scheme_length < 31) { + const auto character{character_at(input, position)}; + if (!sourcemeta::core::is_alphanum(character) && character != '.' && + character != '+' && character != '-') { + break; + } + + ++position; + ++scheme_length; + } + + if (scheme_length == 0 || character_at(input, position) != ':') { + return 0; + } + + ++position; + while (position < input.size()) { + const auto character{static_cast(input[position])}; + if (character <= 0x20 || character == '<' || character == '>') { + break; + } + + ++position; + } + + if (character_at(input, position) != '>') { + return 0; + } + + return position + 1 - offset; +} + +// An email address of GFM section 6.9, right after the opening angle bracket, +// which is a valid email address of the HTML Standard up to the closing angle +// bracket +inline auto scan_autolink_email(const std::string_view input, + const std::size_t offset) -> std::size_t { + auto position{offset}; + while (position < input.size()) { + const auto character{static_cast(input[position])}; + if (character <= 0x20 || character == '<' || character == '>') { + break; + } + + ++position; + } + + if (character_at(input, position) != '>' || + !sourcemeta::core::is_html_email( + input.substr(offset, position - offset))) { + return 0; + } + + return position + 1 - offset; +} + +inline auto scan_tag_name(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + if (!sourcemeta::core::is_alpha(character_at(input, offset))) { + return 0; + } + + auto position{offset + 1}; + while (position < input.size() && + (sourcemeta::core::is_alphanum(input[position]) || + input[position] == '-')) { + ++position; + } + + return position - offset; +} + +inline auto is_attribute_name_start(const char character) noexcept -> bool { + return sourcemeta::core::is_alpha(character) || character == '_' || + character == ':'; +} + +inline auto is_attribute_name_character(const char character) noexcept -> bool { + return sourcemeta::core::is_alphanum(character) || character == '_' || + character == ':' || character == '.' || character == '-'; +} + +// The value of an attribute, at the equals sign or the whitespace before it +inline auto +scan_attribute_value_specification(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + auto position{offset + scan_spacechars(input, offset)}; + if (character_at(input, position) != '=') { + return 0; + } + + ++position; + position += scan_spacechars(input, position); + const auto quote{character_at(input, position)}; + if (quote == '"' || quote == '\'') { + const auto closing{input.find(quote, position + 1)}; + if (closing == std::string_view::npos) { + return 0; + } + + return closing + 1 - offset; + } + + const auto value_start{position}; + while (position < input.size()) { + const auto character{input[position]}; + if (character == ' ' || character == '\t' || character == '\r' || + character == '\n' || character == '\v' || character == '\f' || + character == '"' || character == '\'' || character == '=' || + character == '<' || character == '>' || character == '`') { + break; + } + + ++position; + } + + if (position == value_start) { + return 0; + } + + return position - offset; +} + +// An open tag or a closing tag of GFM section 6.6, right after the opening +// angle bracket +inline auto scan_html_tag(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + auto position{offset}; + if (character_at(input, position) == '/') { + ++position; + const auto name_length{scan_tag_name(input, position)}; + if (name_length == 0) { + return 0; + } + + position += name_length; + position += scan_spacechars(input, position); + if (character_at(input, position) != '>') { + return 0; + } + + return position + 1 - offset; + } + + const auto name_length{scan_tag_name(input, position)}; + if (name_length == 0) { + return 0; + } + + position += name_length; + while (true) { + const auto spaces{scan_spacechars(input, position)}; + if (spaces == 0 || + !is_attribute_name_start(character_at(input, position + spaces))) { + break; + } + + auto name_end{position + spaces + 1}; + while (name_end < input.size() && + is_attribute_name_character(input[name_end])) { + ++name_end; + } + + position = name_end + scan_attribute_value_specification(input, name_end); + } + + position += scan_spacechars(input, position); + if (character_at(input, position) == '/') { + ++position; + } + + if (character_at(input, position) != '>') { + return 0; + } + + return position + 1 - offset; +} + +// An HTML comment of GFM section 6.10, at the two dashes that follow the +// exclamation mark: "An HTML comment consists of , where text +// does not start with > or ->, does not end with -, and does not contain --" +inline auto scan_html_comment(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + if (character_at(input, offset) != '-' || + character_at(input, offset + 1) != '-') { + return 0; + } + + const auto text{offset + 2}; + if (character_at(input, text) == '>' || + (character_at(input, text) == '-' && + character_at(input, text + 1) == '>')) { + return 0; + } + + // As the text can neither contain two dashes nor end with one, the first two + // dashes after the opening have to be the start of the closing + const auto closing{input.find("--", text)}; + if (closing == std::string_view::npos || + character_at(input, closing + 2) != '>') { + return 0; + } + + return closing + 3 - offset; +} + +// The content of a processing instruction, right after its question mark, +// without the closing question mark and angle bracket +inline auto scan_html_processing_instruction(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + auto position{offset}; + while (position < input.size()) { + if (input[position] == '?') { + if (position + 1 < input.size() && input[position + 1] != '>') { + ++position; + continue; + } + + break; + } + + ++position; + } + + return position - offset; +} + +// A declaration, right after its exclamation mark, without the closing angle +// bracket +inline auto scan_html_declaration(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + auto position{offset}; + while (position < input.size() && input[position] >= 'A' && + input[position] <= 'Z') { + ++position; + } + + if (position == offset) { + return 0; + } + + const auto spaces{scan_spacechars(input, position)}; + if (spaces == 0) { + return 0; + } + + const auto closing{input.find('>', position + spaces)}; + return (closing == std::string_view::npos ? input.size() : closing) - offset; +} + +// A CDATA section, right after its opening bracket, without the closing +// brackets and angle bracket +inline auto scan_html_cdata(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + if (offset > input.size() || + input.substr(offset, 6) != std::string_view{"CDATA["}) { + return 0; + } + + auto boundary{offset + 6}; + std::size_t brackets{0}; + for (auto position{offset + 6}; position < input.size(); ++position) { + const auto character{input[position]}; + if (brackets == 2) { + if (character == '>') { + break; + } + + brackets = 0; + boundary = position + 1; + } else if (character == ']') { + ++brackets; + } else { + brackets = 0; + boundary = position + 1; + } + } + + return boundary - offset; +} + +// The lowercase names of the HTML block condition 6 of GFM section 4.6 +constexpr std::array BLOCK_TAG_NAMES{ + {"address", "article", "aside", "base", "basefont", "blockquote", + "body", "caption", "center", "col", "colgroup", "dd", + "details", "dialog", "dir", "div", "dl", "dt", + "fieldset", "figcaption", "figure", "footer", "form", "frame", + "frameset", "h1", "h2", "h3", "h4", "h5", + "h6", "head", "header", "hr", "html", "iframe", + "legend", "li", "link", "main", "menu", "menuitem", + "nav", "noframes", "ol", "optgroup", "option", "p", + "param", "section", "source", "summary", "table", "tbody", + "td", "tfoot", "th", "thead", "title", "tr", + "track", "ul"}}; + +// Returns the lowercase alphanumeric run at the given offset, if it is short +// enough to be a tag name that the HTML block conditions care about +inline auto lowercase_tag_name(const std::string_view input, + const std::size_t offset, + std::array &buffer) noexcept + -> std::string_view { + std::size_t length{0}; + while (offset + length < input.size() && + sourcemeta::core::is_alphanum(input[offset + length])) { + if (length == buffer.size()) { + return {}; + } + + buffer[length] = sourcemeta::core::to_lowercase(input[offset + length]); + ++length; + } + + return {buffer.data(), length}; +} + +// The start condition of an HTML block of GFM section 4.6, returning the +// number of the condition from one to six or zero +inline auto scan_html_block_start(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + if (character_at(input, offset) != '<') { + return 0; + } + + const auto next{character_at(input, offset + 1)}; + if (next == '!') { + const auto after{character_at(input, offset + 2)}; + if (after == '-' && character_at(input, offset + 3) == '-') { + return 2; + } + + if (after == '[' && sourcemeta::core::starts_with_ignore_case( + input.substr(offset + 3), "cdata[")) { + return 5; + } + + return after >= 'A' && after <= 'Z' ? 4 : 0; + } + + if (next == '?') { + return 3; + } + + std::array buffer{}; + const auto name{lowercase_tag_name(input, offset + 1, buffer)}; + if (name == "script" || name == "pre" || name == "textarea" || + name == "style") { + const auto after{character_at(input, offset + 1 + name.size())}; + return is_html_space(after) || after == '>' ? 1 : 0; + } + + auto position{offset + 1}; + if (next == '/') { + ++position; + } + + const auto block_name{lowercase_tag_name(input, position, buffer)}; + if (block_name.empty() || + !std::binary_search(BLOCK_TAG_NAMES.cbegin(), BLOCK_TAG_NAMES.cend(), + block_name)) { + return 0; + } + + position += block_name.size(); + const auto after{character_at(input, position)}; + if (is_html_space(after) || after == '>') { + return 6; + } + + return after == '/' && character_at(input, position + 1) == '>' ? 6 : 0; +} + +// The start condition 7 of an HTML block of GFM section 4.6 +inline auto scan_html_block_start_7(const std::string_view input, + const std::size_t offset) noexcept -> bool { + if (character_at(input, offset) != '<') { + return false; + } + + const auto length{scan_html_tag(input, offset + 1)}; + if (length == 0) { + return false; + } + + auto position{offset + 1 + length}; + while (position < input.size() && + (input[position] == '\t' || input[position] == '\f' || + input[position] == ' ')) { + ++position; + } + + const auto character{character_at(input, position)}; + return character == '\n' || character == '\r'; +} + +// The end conditions of the HTML blocks of GFM section 4.6 +inline auto scan_html_block_end(const std::string_view input, + const std::size_t offset, + const std::size_t condition) noexcept -> bool { + if (offset > input.size()) { + return false; + } + + const auto line{input.substr(offset)}; + switch (condition) { + case 1: { + auto position{line.find(" buffer{}; + const auto name{lowercase_tag_name(line, position + 2, buffer)}; + if ((name == "script" || name == "pre" || name == "textarea" || + name == "style") && + character_at(line, position + 2 + name.size()) == '>') { + return true; + } + + position = line.find("") != std::string_view::npos; + case 3: + return line.find("?>") != std::string_view::npos; + case 4: + return line.find('>') != std::string_view::npos; + case 5: + return line.find("]]>") != std::string_view::npos; + default: + return false; + } +} + +// A link title of GFM section 6.3, which "consists of either a sequence of zero +// or more characters between straight double-quote characters ("), including a +// " character only if it is backslash-escaped, or a sequence of zero or more +// characters between straight single-quote characters ('), including a ' +// character only if it is backslash-escaped, or a sequence of zero or more +// characters between matching parentheses ((...)), including a ( or ) +// character only if it is backslash-escaped" +inline auto scan_link_title(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + const auto opening{character_at(input, offset)}; + if (opening != '"' && opening != '\'' && opening != '(') { + return 0; + } + + const auto closing{opening == '(' ? ')' : opening}; + for (auto position{offset + 1}; position < input.size(); ++position) { + const auto character{input[position]}; + // GFM section 2.4: "Any ASCII punctuation character may be + // backslash-escaped" + if (character == '\\' && position + 1 < input.size() && + sourcemeta::core::is_punctuation(input[position + 1])) { + ++position; + continue; + } + + if (character == closing) { + return position + 1 - offset; + } + + if (opening == '(' && character == '(') { + return 0; + } + } + + return 0; +} + +// The start of an ATX heading of GFM section 4.2, including the whitespace +// after its opening sequence +inline auto scan_atx_heading_start(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + auto position{offset}; + while (position < input.size() && input[position] == '#') { + ++position; + } + + const auto hashes{position - offset}; + if (hashes == 0 || hashes > 6) { + return 0; + } + + const auto character{character_at(input, position)}; + if (character == ' ' || character == '\t') { + while (position < input.size() && + (input[position] == ' ' || input[position] == '\t')) { + ++position; + } + + return position - offset; + } + + return character == '\n' || character == '\r' ? position + 1 - offset : 0; +} + +// The underline of a setext heading of GFM section 4.3, returning the level +// of the heading or zero +inline auto scan_setext_heading_line(const std::string_view input, + const std::size_t offset) noexcept + -> std::uint8_t { + const auto marker{character_at(input, offset)}; + if (marker != '=' && marker != '-') { + return 0; + } + + auto position{offset}; + while (position < input.size() && input[position] == marker) { + ++position; + } + + while (position < input.size() && + (input[position] == ' ' || input[position] == '\t')) { + ++position; + } + + const auto character{character_at(input, position)}; + if (character != '\n' && character != '\r') { + return 0; + } + + return marker == '=' ? 1 : 2; +} + +// The opening fence of a fenced code block of GFM section 4.5, returning the +// length of the fence +inline auto scan_open_code_fence(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + const auto marker{character_at(input, offset)}; + if (marker != '`' && marker != '~') { + return 0; + } + + auto position{offset}; + while (position < input.size() && input[position] == marker) { + ++position; + } + + const auto length{position - offset}; + if (length < 3) { + return 0; + } + + while (position < input.size()) { + const auto character{input[position]}; + if (character == '\n' || character == '\r') { + return length; + } + + if (marker == '`' && character == '`') { + return 0; + } + + ++position; + } + + return 0; +} + +// The closing fence of a fenced code block of GFM section 4.5, returning the +// length of the fence +inline auto scan_close_code_fence(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + const auto marker{character_at(input, offset)}; + if (marker != '`' && marker != '~') { + return 0; + } + + auto position{offset}; + while (position < input.size() && input[position] == marker) { + ++position; + } + + const auto length{position - offset}; + if (length < 3) { + return 0; + } + + while (position < input.size() && + (input[position] == ' ' || input[position] == '\t')) { + ++position; + } + + const auto character{character_at(input, position)}; + return character == '\n' || character == '\r' ? length : 0; +} + +// The image media types whose data URLs safe mode keeps +constexpr std::array SAFE_DATA_URL_PREFIXES{ + {"data:image/png", "data:image/gif", "data:image/jpeg", "data:image/webp"}}; + +// Whether a link destination starts with a scheme that can run code or read +// local files, where only a few image data URLs are safe. RFC 2397 Section 3 +// writes a data URL as `dataurl := "data:" [ mediatype ] [ ";base64" ] "," +// data` with `mediatype := [ type "/" subtype ] *( ";" parameter )`, so the +// subtype of an allowed image ends at a semicolon or a comma +inline auto is_dangerous_url(const std::string_view url) noexcept -> bool { + for (const auto prefix : SAFE_DATA_URL_PREFIXES) { + if (sourcemeta::core::starts_with_ignore_case(url, prefix) && + url.size() > prefix.size() && + (url[prefix.size()] == ';' || url[prefix.size()] == ',')) { + return false; + } + } + + return sourcemeta::core::starts_with_ignore_case(url, "javascript:") || + sourcemeta::core::starts_with_ignore_case(url, "vbscript:") || + sourcemeta::core::starts_with_ignore_case(url, "file:") || + sourcemeta::core::starts_with_ignore_case(url, "data:"); +} + +// The start of a footnote definition, including the whitespace after its +// colon +inline auto scan_footnote_definition(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + if (character_at(input, offset) != '[' || + character_at(input, offset + 1) != '^') { + return 0; + } + + auto position{offset + 2}; + while (position < input.size()) { + const auto character{input[position]}; + if (character == ']' || character == ' ' || character == '\r' || + character == '\n' || character == '\t') { + break; + } + + ++position; + } + + if (position == offset + 2 || character_at(input, position) != ']' || + character_at(input, position + 1) != ':') { + return 0; + } + + position += 2; + while (position < input.size() && + (input[position] == ' ' || input[position] == '\t')) { + ++position; + } + + return position - offset; +} + +inline auto scan_table_marker(const std::string_view input, + std::size_t position) noexcept -> std::size_t { + const auto start{position}; + while (position < input.size() && is_line_space(input[position])) { + ++position; + } + + if (character_at(input, position) == ':') { + ++position; + } + + const auto dashes_start{position}; + while (position < input.size() && input[position] == '-') { + ++position; + } + + if (position == dashes_start) { + return 0; + } + + if (character_at(input, position) == ':') { + ++position; + } + + while (position < input.size() && is_line_space(input[position])) { + ++position; + } + + return position - start; +} + +// The end of a table row, which is any whitespace and a line ending +inline auto scan_table_row_end(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + if (offset >= input.size()) { + return 0; + } + + auto position{offset}; + while (position < input.size() && is_line_space(input[position])) { + ++position; + } + + if (character_at(input, position) == '\r') { + ++position; + } + + return character_at(input, position) == '\n' ? position + 1 - offset : 0; +} + +// The delimiter row of a table of GFM section 4.10 +inline auto scan_table_start(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + if (offset >= input.size()) { + return 0; + } + + auto position{offset}; + if (input[position] == '|') { + ++position; + } + + const auto first{scan_table_marker(input, position)}; + if (first == 0) { + return 0; + } + + position += first; + while (character_at(input, position) == '|') { + const auto marker{scan_table_marker(input, position + 1)}; + if (marker == 0) { + ++position; + break; + } + + position += 1 + marker; + } + + const auto end{scan_table_row_end(input, position)}; + return end == 0 ? 0 : position + end - offset; +} + +// The content of a table cell, which ends at a line ending or at a pipe that +// is not preceded by a backslash +inline auto scan_table_cell(const std::string_view input, + const std::size_t offset) noexcept -> std::size_t { + auto position{offset}; + while (position < input.size()) { + const auto character{input[position]}; + if (character == '\r' || character == '\n') { + break; + } + + if (character == '|' && + (position == offset || input[position - 1] != '\\')) { + break; + } + + ++position; + } + + return position - offset; +} + +// A pipe that ends a table cell, including the whitespace after it +inline auto scan_table_cell_end(const std::string_view input, + const std::size_t offset) noexcept + -> std::size_t { + if (character_at(input, offset) != '|') { + return 0; + } + + auto position{offset + 1}; + while (position < input.size() && is_line_space(input[position])) { + ++position; + } + + return position - offset; +} + +// A task list item marker of GFM section 5.3 at the start of the content of a +// list item, which "consists of an optional number of spaces, a left bracket +// ([), either a whitespace character or the letter x in either lowercase or +// uppercase, and then a right bracket (])", where the list item already took +// the spaces, and which needs "at least one whitespace character before any +// other content". A line ending cannot come between the brackets, so the +// whitespace there is one of the other whitespace characters of GFM section 2.1 +inline auto scan_task_list_marker(const std::string_view input, + const std::size_t position) noexcept -> bool { + if (character_at(input, position) != '[' || + character_at(input, position + 2) != ']') { + return false; + } + + const auto state{character_at(input, position + 1)}; + return (is_line_space(state) || state == 'x' || state == 'X') && + is_line_space(character_at(input, position + 3)); +} + +} // namespace sourcemeta::core::markdown + +#endif diff --git a/test/markdown/CMakeLists.txt b/test/markdown/CMakeLists.txt index 96f9fdd72d..1b0ddb701c 100644 --- a/test/markdown/CMakeLists.txt +++ b/test/markdown/CMakeLists.txt @@ -18,6 +18,8 @@ sourcemeta_test(NAMESPACE sourcemeta PROJECT core NAME markdown markdown_paragraphs_test.cc markdown_pathological_test.cc markdown_safety_test.cc + markdown_spec_blocks_test.cc + markdown_spec_inlines_test.cc markdown_strikethrough_test.cc markdown_tables_test.cc markdown_task_lists_test.cc diff --git a/test/markdown/markdown_autolinks_test.cc b/test/markdown/markdown_autolinks_test.cc index bd2ebd78e7..0872e31ac0 100644 --- a/test/markdown/markdown_autolinks_test.cc +++ b/test/markdown/markdown_autolinks_test.cc @@ -93,9 +93,7 @@ TEST(autolink_uri_scheme_too_short_is_not_autolink) { TEST(autolink_uri_with_space_is_not_autolink) { const auto result{ sourcemeta::core::markdown_to_html("")}; - EXPECT_EQ(result, "

<https://sourcemeta.com/" - "a b>

\n"); + EXPECT_EQ(result, "

<https://sourcemeta.com/a b>

\n"); } TEST(autolink_uri_backslash_is_literal) { @@ -126,20 +124,14 @@ TEST(autolink_email_with_dots_plus_and_hyphen) { TEST(autolink_email_with_backslash_is_not_autolink) { const auto result{ sourcemeta::core::markdown_to_html("")}; - EXPECT_EQ( - result, - "

<team@sourcemeta.com>

\n"); + EXPECT_EQ(result, "

<team@sourcemeta.com>

\n"); } TEST(autolink_blank_or_trailing_space_angle_brackets) { const auto result{ sourcemeta::core::markdown_to_html("< >\n\n")}; - EXPECT_EQ( - result, - "

< >

\n" - "

<https://sourcemeta.com " - ">

\n"); + EXPECT_EQ(result, "

< >

\n" + "

<https://sourcemeta.com >

\n"); } TEST(autolink_without_scheme_is_not_autolink) { @@ -278,19 +270,17 @@ TEST(extended_autolink_not_after_word_character) { TEST(extended_autolink_inside_emphasis) { const auto result{ - sourcemeta::core::markdown_to_html("_https://sourcemeta.com_")}; + sourcemeta::core::markdown_to_html("*https://sourcemeta.com*")}; EXPECT_EQ( result, "

https://sourcemeta.com

\n"); } -TEST(extended_autolink_ftp_scheme) { +TEST(extended_autolink_ftp_scheme_is_not_autolink) { const auto result{ sourcemeta::core::markdown_to_html("ftp://files.sourcemeta.com/archive")}; - EXPECT_EQ(result, "

ftp://" - "files.sourcemeta.com/archive

\n"); + EXPECT_EQ(result, "

ftp://files.sourcemeta.com/archive

\n"); } TEST(extended_autolink_inside_link_text) { @@ -304,3 +294,228 @@ TEST(extended_autolink_inside_code_span) { sourcemeta::core::markdown_to_html("`https://sourcemeta.com`")}; EXPECT_EQ(result, "

https://sourcemeta.com

\n"); } + +TEST(extended_autolink_email_after_invalid_candidate) { + const auto result{ + sourcemeta::core::markdown_to_html("a@b.c_@d\n\nmailto:a@b_@c.d")}; + EXPECT_EQ(result, "

a@b.c_@d

\n" + "

mailto:a@b_@c.d

\n"); +} + +TEST(extended_autolink_after_underscore_emphasis_delimiter) { + const auto result{ + sourcemeta::core::markdown_to_html("_https://sourcemeta.com_")}; + EXPECT_EQ(result, "

https://sourcemeta.com

\n"); +} + +TEST(extended_autolink_trailing_periods) { + const auto result{sourcemeta::core::markdown_to_html( + "Visit www.commonmark.org.\n\nVisit www.commonmark.org/a.b.")}; + EXPECT_EQ(result, "

Visit www.commonmark.org.

\n" + "

Visit www.commonmark.org/" + "a.b.

\n"); +} + +TEST(extended_autolink_trailing_parentheses) { + const auto result{sourcemeta::core::markdown_to_html( + "www.google.com/search?q=Markup+(business)\n\n" + "www.google.com/search?q=Markup+(business)))\n\n" + "(www.google.com/search?q=Markup+(business))\n\n" + "(www.google.com/search?q=Markup+(business)")}; + EXPECT_EQ(result, + "

" + "www.google.com/search?q=Markup+(business)

\n" + "

" + "www.google.com/search?q=Markup+(business)))

\n" + "

(" + "www.google.com/search?q=Markup+(business))

\n" + "

(" + "www.google.com/search?q=Markup+(business)

\n"); +} + +TEST(extended_autolink_interior_parentheses) { + const auto result{sourcemeta::core::markdown_to_html( + "www.google.com/search?q=(business))+ok")}; + EXPECT_EQ(result, + "

" + "www.google.com/search?q=(business))+ok

\n"); +} + +TEST(extended_autolink_entity_reference_lookalike) { + const auto result{sourcemeta::core::markdown_to_html( + "www.google.com/search?q=commonmark&hl=en\n\n" + "www.google.com/search?q=commonmark&hl;")}; + EXPECT_EQ( + result, + "

" + "www.google.com/search?q=commonmark&hl=en

\n" + "

" + "www.google.com/search?q=commonmark&hl;

\n"); +} + +TEST(extended_autolink_entity_reference_lookalike_with_digits) { + const auto result{ + sourcemeta::core::markdown_to_html("www.sourcemeta.com/a&b2;")}; + EXPECT_EQ(result, "

" + "www.sourcemeta.com/a&b2;

\n"); +} + +TEST(extended_autolink_trailing_semicolon_is_kept) { + const auto result{ + sourcemeta::core::markdown_to_html("www.sourcemeta.com/a;")}; + EXPECT_EQ(result, "

" + "www.sourcemeta.com/a;

\n"); +} + +TEST(extended_autolink_trailing_quote_is_kept) { + const auto result{ + sourcemeta::core::markdown_to_html("www.sourcemeta.com/a'")}; + EXPECT_EQ(result, "

" + "www.sourcemeta.com/a'

\n"); +} + +TEST(extended_autolink_less_than_ends_link) { + const auto result{ + sourcemeta::core::markdown_to_html("www.commonmark.org/he" + "www.commonmark.org/he<lp

\n"); +} + +TEST(extended_url_autolinks) { + const auto result{sourcemeta::core::markdown_to_html( + "http://commonmark.org\n\n" + "(Visit https://encrypted.google.com/search?q=Markup+(business))")}; + EXPECT_EQ( + result, + "

http://commonmark.org" + "

\n" + "

(Visit " + "https://encrypted.google.com/search?q=Markup+(business))

\n"); +} + +TEST(extended_url_autolink_without_period_is_not_autolink) { + const auto result{ + sourcemeta::core::markdown_to_html("http://localhost:8080/x")}; + EXPECT_EQ(result, "

http://localhost:8080/x

\n"); +} + +TEST(extended_url_autolink_after_other_character_is_not_autolink) { + const auto result{ + sourcemeta::core::markdown_to_html("x.https://sourcemeta.com")}; + EXPECT_EQ(result, "

x.https://sourcemeta.com

\n"); +} + +TEST(extended_autolink_underscore_in_final_segments_of_long_domain) { + const auto result{ + sourcemeta::core::markdown_to_html("www.a.b.c.d.e.f.g.h.i.j.k.l_m")}; + EXPECT_EQ(result, "

www.a.b.c.d.e.f.g.h.i.j.k.l_m

\n"); +} + +TEST(extended_autolink_email_with_plus_before_at_sign_only) { + const auto result{sourcemeta::core::markdown_to_html( + "hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.")}; + EXPECT_EQ(result, "

hello@mail+xyz.example isn't valid, but " + "hello+xyz@mail.example is.

\n"); +} + +TEST(extended_autolink_email_ending_characters) { + const auto result{sourcemeta::core::markdown_to_html( + "a.b-c_d@a.b\n\na.b-c_d@a.b.\n\na.b-c_d@a.b-\n\na.b-c_d@a.b_")}; + EXPECT_EQ(result, "

a.b-c_d@a.b

\n" + "

a.b-c_d@a.b.

\n" + "

a.b-c_d@a.b-

\n" + "

a.b-c_d@a.b_

\n"); +} + +TEST(extended_autolink_email_ending_in_digit) { + const auto result{sourcemeta::core::markdown_to_html("a@b.c1")}; + EXPECT_EQ(result, "

a@b.c1

\n"); +} + +TEST(extended_autolink_email_followed_by_at_sign) { + const auto result{sourcemeta::core::markdown_to_html("a@b.c@d.e")}; + EXPECT_EQ(result, "

a@b.c@d.e

\n"); +} + +TEST(extended_autolink_email_after_other_character_is_not_autolink) { + const auto result{sourcemeta::core::markdown_to_html( + "x:team@sourcemeta.com\n\n`x`team@sourcemeta.com")}; + EXPECT_EQ(result, "

x:team@sourcemeta.com

\n" + "

xteam@sourcemeta.com

\n"); +} + +TEST(extended_autolink_email_after_parenthesis_and_emphasis) { + const auto result{sourcemeta::core::markdown_to_html( + "(team@sourcemeta.com) *x*team@sourcemeta.com")}; + EXPECT_EQ(result, "

(" + "team@sourcemeta.com) x" + "team@sourcemeta.com

\n"); +} + +TEST(extended_autolink_protocols) { + const auto result{sourcemeta::core::markdown_to_html( + "mailto:foo@bar.baz\n\nmailto:a.b-c_d@a.b\n\nmailto:a.b-c_d@a.b.\n\n" + "mailto:a.b-c_d@a.b/\n\nmailto:a.b-c_d@a.b-\n\nmailto:a.b-c_d@a.b_\n\n" + "xmpp:foo@bar.baz\n\nxmpp:foo@bar.baz.")}; + EXPECT_EQ(result, + "

mailto:foo@bar.baz

\n" + "

mailto:a.b-c_d@a.b

\n" + "

mailto:a.b-c_d@a.b.

\n" + "

mailto:a.b-c_d@a.b/

\n" + "

mailto:a.b-c_d@a.b-

\n" + "

mailto:a.b-c_d@a.b_

\n" + "

xmpp:foo@bar.baz

\n" + "

xmpp:foo@bar.baz.

\n"); +} + +TEST(extended_autolink_xmpp_resources) { + const auto result{sourcemeta::core::markdown_to_html( + "xmpp:foo@bar.baz/txt\n\nxmpp:foo@bar.baz/txt@bin\n\n" + "xmpp:foo@bar.baz/txt@bin.com")}; + EXPECT_EQ(result, + "

xmpp:foo@bar.baz/txt

\n" + "

xmpp:foo@bar.baz/txt@bin" + "

\n" + "

" + "xmpp:foo@bar.baz/txt@bin.com

\n"); +} + +TEST(extended_autolink_xmpp_resource_ends_at_slash) { + const auto result{ + sourcemeta::core::markdown_to_html("xmpp:foo@bar.baz/txt/bin")}; + EXPECT_EQ(result, + "

xmpp:foo@bar.baz/txt/bin" + "

\n"); +} + +TEST(extended_autolink_www_without_domain_is_not_autolink) { + const auto result{ + sourcemeta::core::markdown_to_html("Visit www. or www..com")}; + EXPECT_EQ(result, "

Visit www. or www..com

\n"); +} + +TEST(extended_autolinks_after_form_feed_and_line_tabulation) { + const auto result{sourcemeta::core::markdown_to_html( + "a\fwww.commonmark.org a\vfoo@bar.baz")}; + EXPECT_EQ( + result, + "

a\fwww.commonmark.org " + "a\vfoo@bar.baz

\n"); +} + +TEST(extended_url_autolink_domain_starting_with_underscore) { + const auto result{sourcemeta::core::markdown_to_html("http://_a.b")}; + EXPECT_EQ(result, "

http://_a.b

\n"); +} + +TEST(extended_autolink_inside_brackets_that_are_not_a_link) { + const auto result{ + sourcemeta::core::markdown_to_html("[see www.sourcemeta.com]")}; + EXPECT_EQ(result, "

[see www.sourcemeta.com]

\n"); +} diff --git a/test/markdown/markdown_characters_test.cc b/test/markdown/markdown_characters_test.cc index 22ea5653d0..f9eaae406f 100644 --- a/test/markdown/markdown_characters_test.cc +++ b/test/markdown/markdown_characters_test.cc @@ -243,8 +243,8 @@ TEST(invalid_utf8_in_fenced_code) { TEST(invalid_utf8_in_fenced_code_info_string) { const auto result{sourcemeta::core::markdown_to_html("```l\xFF" "ang\ncode\n```")}; - EXPECT_EQ(result, "
code\n"
+  EXPECT_EQ(result, "
code\n"
                     "
\n");
 }
 
@@ -430,7 +430,7 @@ TEST(crlf_after_trailing_spaces_is_hard_break) {
 
 TEST(non_breaking_space_after_list_marker_is_not_list) {
   const auto result{sourcemeta::core::markdown_to_html("*\xC2\xA0item")};
-  EXPECT_EQ(result, "
*\xc2\xa0item
\n");
+  EXPECT_EQ(result, "
* item
\n");
 }
 
 TEST(em_spaces_do_not_indent_code) {
@@ -479,7 +479,7 @@ TEST(non_ascii_link_destination_is_percent_encoded) {
 
 TEST(quotes_in_text_are_escaped) {
   const auto result{sourcemeta::core::markdown_to_html("say \"hi\" and 'bye'")};
-  EXPECT_EQ(result, "
say "hi" and 'bye'
\n");
+  EXPECT_EQ(result, "
say "hi" and 'bye'
\n");
 }
 
 TEST(nul_characters_are_replaced) {
diff --git a/test/markdown/markdown_code_blocks_test.cc b/test/markdown/markdown_code_blocks_test.cc
index b41f7eae28..7c07d326ff 100644
--- a/test/markdown/markdown_code_blocks_test.cc
+++ b/test/markdown/markdown_code_blocks_test.cc
@@ -15,7 +15,8 @@ TEST(fenced_code_block_tildes) {
 TEST(fenced_code_block_with_language) {
   const auto result{
       sourcemeta::core::markdown_to_html("```cpp\nint x = 0;\n```")};
-  EXPECT_EQ(result, "
int x = 0;\n
\n");
+  EXPECT_EQ(result,
+            "
int x = 0;\n
\n");
 }
 
 TEST(fenced_code_block_html_escaped) {
@@ -40,7 +41,8 @@ TEST(fenced_code_block_with_empty_lines) {
 TEST(github_pre_lang_format) {
   const auto result{
       sourcemeta::core::markdown_to_html("```python\nprint()\n```")};
-  EXPECT_EQ(result, "
print()\n
\n");
+  EXPECT_EQ(result,
+            "
print()\n
\n");
 }
 
 TEST(indented_code_chunks_separated_by_blank_lines) {
@@ -241,13 +243,19 @@ TEST(fenced_code_interrupts_paragraph) {
 TEST(fenced_code_info_string_first_word_is_language) {
   const auto result{sourcemeta::core::markdown_to_html(
       "~~~ python extra words here\nprint(1)\n~~~")};
-  EXPECT_EQ(result, "
print(1)\n"
+  EXPECT_EQ(result, "
print(1)\n"
                     "
\n");
 }
 
 TEST(fenced_code_info_string_of_symbols) {
   const auto result{sourcemeta::core::markdown_to_html("```#!\n```")};
-  EXPECT_EQ(result, "
\n");
+  EXPECT_EQ(result, "
\n");
+}
+
+TEST(fenced_code_info_string_escaped_ampersand_is_not_an_entity) {
+  const auto result{sourcemeta::core::markdown_to_html("```a\\&b\n```")};
+  EXPECT_EQ(result,
+            "
\n");
 }
 
 TEST(fenced_code_backtick_fence_info_cannot_contain_backticks) {
@@ -259,7 +267,7 @@ TEST(fenced_code_backtick_fence_info_cannot_contain_backticks) {
 TEST(fenced_code_tilde_fence_info_can_contain_backticks) {
   const auto result{
       sourcemeta::core::markdown_to_html("~~~ `js` ~~~\ncode\n~~~")};
-  EXPECT_EQ(result, "
code\n"
+  EXPECT_EQ(result, "
code\n"
                     "
\n");
 }
 
@@ -273,12 +281,12 @@ TEST(fenced_code_closing_fence_cannot_have_info) {
 TEST(fenced_code_info_string_entities_decoded) {
   const auto result{
       sourcemeta::core::markdown_to_html("```c++\ncode\n```")};
-  EXPECT_EQ(result, "
code\n"
+  EXPECT_EQ(result, "
code\n"
                     "
\n");
 }
 
 TEST(fenced_code_info_string_backslash_escapes) {
   const auto result{sourcemeta::core::markdown_to_html("```c\\#\ncode\n```")};
-  EXPECT_EQ(result, "
code\n"
+  EXPECT_EQ(result, "
code\n"
                     "
\n");
 }
diff --git a/test/markdown/markdown_code_spans_test.cc b/test/markdown/markdown_code_spans_test.cc
index 6d1412d6ff..332e59905b 100644
--- a/test/markdown/markdown_code_spans_test.cc
+++ b/test/markdown/markdown_code_spans_test.cc
@@ -42,7 +42,7 @@ TEST(code_span_space_on_one_side_is_kept) {
 TEST(code_span_non_breaking_space_is_kept) {
   const auto result{sourcemeta::core::markdown_to_html("`\xC2\xA0"
                                                        "x`")};
-  EXPECT_EQ(result, "
\xc2\xa0x
\n");
+  EXPECT_EQ(result, "
 x
\n");
 }
 
 TEST(code_span_of_only_spaces_is_kept) {
@@ -95,3 +95,13 @@ TEST(code_span_takes_precedence_over_autolink) {
       sourcemeta::core::markdown_to_html("``")};
   EXPECT_EQ(result, "
<https://sourcemeta.com/x>`
\n");
 }
+
+TEST(code_span_of_a_space_and_a_crlf_is_kept) {
+  const auto result{sourcemeta::core::markdown_to_html("` \r\n`")};
+  EXPECT_EQ(result, "
  
\n");
+}
+
+TEST(code_span_after_rejected_and_matched_backtick_strings) {
+  const auto result{sourcemeta::core::markdown_to_html("```a ``b`c`` `d`")};
+  EXPECT_EQ(result, "
```a b`c d
\n");
+}
diff --git a/test/markdown/markdown_emphasis_test.cc b/test/markdown/markdown_emphasis_test.cc
index d0a7327967..bef4d40762 100644
--- a/test/markdown/markdown_emphasis_test.cc
+++ b/test/markdown/markdown_emphasis_test.cc
@@ -78,7 +78,7 @@ TEST(emphasis_opening_between_letter_and_punctuation) {
 TEST(emphasis_opening_followed_by_non_breaking_space) {
   const auto result{sourcemeta::core::markdown_to_html("_\xC2\xA0"
                                                        "x\xC2\xA0_")};
-  EXPECT_EQ(result, "
_\xc2\xa0x\xc2\xa0_
\n");
+  EXPECT_EQ(result, "
_ x _
\n");
 }
 
 TEST(emphasis_intraword_with_asterisks) {
@@ -174,7 +174,7 @@ TEST(strong_emphasis_intraword_with_asterisks) {
 
 TEST(strong_emphasis_nested_asterisks) {
   const auto result{sourcemeta::core::markdown_to_html("**x, **y**, z**")};
-  EXPECT_EQ(result, "
x, y, z
\n");
+  EXPECT_EQ(result, "
x, y, z
\n");
 }
 
 TEST(strong_emphasis_underscore_after_punctuation) {
@@ -209,7 +209,7 @@ TEST(emphasis_rule_of_three_intraword) {
 
 TEST(emphasis_rule_of_three_long_runs) {
   const auto result{sourcemeta::core::markdown_to_html("a****b*******c")};
-  EXPECT_EQ(result, "
ab***c
\n");
+  EXPECT_EQ(result, "
ab***c
\n");
 }
 
 TEST(emphasis_containing_link) {
@@ -233,17 +233,17 @@ TEST(emphasis_spanning_lines) {
 
 TEST(strong_emphasis_nested_inside_itself) {
   const auto result{sourcemeta::core::markdown_to_html("**x **y** z**")};
-  EXPECT_EQ(result, "
x y z
\n");
+  EXPECT_EQ(result, "
x y z
\n");
 }
 
 TEST(strong_emphasis_quadruple_delimiters) {
   const auto result{sourcemeta::core::markdown_to_html("****x****")};
-  EXPECT_EQ(result, "
x
\n");
+  EXPECT_EQ(result, "
x
\n");
 }
 
 TEST(strong_emphasis_quintuple_delimiters) {
   const auto result{sourcemeta::core::markdown_to_html("*****x*****")};
-  EXPECT_EQ(result, "
x
\n");
+  EXPECT_EQ(result, "
x
\n");
 }
 
 TEST(emphasis_with_escaped_delimiters) {
@@ -310,3 +310,8 @@ TEST(emphasis_does_not_span_link_text_brackets) {
   const auto result{sourcemeta::core::markdown_to_html("_a [b_ c]")};
   EXPECT_EQ(result, "
a [b c]
\n");
 }
+
+TEST(emphasis_underscore_closer_before_tilde) {
+  const auto result{sourcemeta::core::markdown_to_html("_a_~b")};
+  EXPECT_EQ(result, "
a~b
\n");
+}
diff --git a/test/markdown/markdown_entities_test.cc b/test/markdown/markdown_entities_test.cc
index b8159d413b..ecaeb187b7 100644
--- a/test/markdown/markdown_entities_test.cc
+++ b/test/markdown/markdown_entities_test.cc
@@ -16,7 +16,7 @@ TEST(entity_named_references) {
   const auto result{sourcemeta::core::markdown_to_html(
       "< > " ' € …\n→ "
       "⪡̸ ★")};
-  EXPECT_EQ(result, "
< > " ' \xe2\x82\xac \xe2\x80\xa6\n"
+  EXPECT_EQ(result, "
< > " ' \xe2\x82\xac \xe2\x80\xa6\n"
                     "\xe2\x86\x92 \xe2\xaa\xa1\xcc\xb8 \xe2\x98\x85
\n");
 }
 
@@ -36,8 +36,8 @@ TEST(entity_invalid_references_are_literal) {
   const auto result{sourcemeta::core::markdown_to_html(
       "&euro &y; &#z; &#xz;\n�\n�\n&NotARealEntity;")};
   EXPECT_EQ(result, "
&euro &y; &#z; &#xz;\n"
-                    "\xef\xbf\xbd\n"
-                    "\xef\xbf\xbd\n"
+                    "&#12345678;\n"
+                    "&#x1234567;\n"
                     "&NotARealEntity;
\n");
 }
 
@@ -112,3 +112,18 @@ TEST(entity_quotes_do_not_delimit_link_title) {
       sourcemeta::core::markdown_to_html("[docs](/manual "Title")")};
   EXPECT_EQ(result, "
[docs](/manual "Title")
\n");
 }
+
+TEST(entity_nonentities) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      "  &x; &#; &#x;\n�\n&#abcdef0;\n&ThisIsNotDefined; &hi?;")};
+  EXPECT_EQ(result, "
&nbsp &x; &#; &#x;\n"
+                    "&#87654321;\n"
+                    "&#abcdef0;\n"
+                    "&ThisIsNotDefined; &hi?;
\n");
+}
+
+TEST(entity_seven_decimal_digits_and_six_hexadecimal_digits) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("A A")};
+  EXPECT_EQ(result, "
A A
\n");
+}
diff --git a/test/markdown/markdown_escapes_test.cc b/test/markdown/markdown_escapes_test.cc
index 1ce9327360..2cbe83e38b 100644
--- a/test/markdown/markdown_escapes_test.cc
+++ b/test/markdown/markdown_escapes_test.cc
@@ -28,7 +28,7 @@ TEST(backslash_escapes_every_ascii_punctuation) {
       "a\\~b\\}c\\|d\\{e\\`f\\_g\\^h\\]i\\\\j\\[k\\@l\\?m\\>n\\=o\\a~b}c|d{e`f_g^h]i\\j[k@l?m>n=o<p;q:r/"
-                    "s.t-u,v+w*x)y(z'1&2%3$4#5"6!
\n");
+                    "s.t-u,v+w*x)y(z'1&2%3$4#5"6!
\n");
 }
 
 TEST(backslash_before_other_characters_is_literal) {
diff --git a/test/markdown/markdown_footnotes_test.cc b/test/markdown/markdown_footnotes_test.cc
index 73f28a89e1..27d80ea3fc 100644
--- a/test/markdown/markdown_footnotes_test.cc
+++ b/test/markdown/markdown_footnotes_test.cc
@@ -288,3 +288,20 @@ TEST(footnote_nested_definition_with_the_same_label_takes_precedence) {
             "aria-label=\"Back to reference 1\">\xe2\x86\xa9
\n"
             "\n\n\n");
 }
+
+TEST(footnote_label_with_number_sign) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("body[^a#b]\n\n[^a#b]: text")};
+  EXPECT_EQ(result,
+            "
body1
\n"
+            "
\n"
+            "
    \n"
+            "
  1. \n"
+            "
    text \xe2\x86\xa9
    \n"
+            "
    2. \n"
+            "
\n"
+            "
\n");
+}
diff --git a/test/markdown/markdown_images_test.cc b/test/markdown/markdown_images_test.cc
index bfb60d0864..9be5406f17 100644
--- a/test/markdown/markdown_images_test.cc
+++ b/test/markdown/markdown_images_test.cc
@@ -81,5 +81,5 @@ TEST(image_title_with_escaped_quote) {
   const auto result{
       sourcemeta::core::markdown_to_html("![logo](/l.png 'it\\'s')")};
   EXPECT_EQ(result,
-            "
\"logo\"
\n");
+            "
\"logo\"
\n");
 }
diff --git a/test/markdown/markdown_links_test.cc b/test/markdown/markdown_links_test.cc
index 682a6fa1a7..e4879f0856 100644
--- a/test/markdown/markdown_links_test.cc
+++ b/test/markdown/markdown_links_test.cc
@@ -121,7 +121,8 @@ TEST(link_reference_definition_title_without_separating_space) {
 TEST(link_reference_definition_backslash_escapes) {
   const auto result{sourcemeta::core::markdown_to_html(
       "[docs]: /a\\b\\#c 'it\\'s'\n\n[docs]")};
-  EXPECT_EQ(result, "
docs
\n");
+  EXPECT_EQ(result,
+            "
docs
\n");
 }
 
 TEST(link_reference_definition_after_its_use) {
@@ -158,14 +159,14 @@ TEST(link_reference_definition_alone_renders_nothing) {
 TEST(link_reference_definition_with_trailing_text_is_paragraph) {
   const auto result{
       sourcemeta::core::markdown_to_html("[docs]: /manual 'Manual' trailing")};
-  EXPECT_EQ(result, "
[docs]: /manual 'Manual' trailing
\n");
+  EXPECT_EQ(result, "
[docs]: /manual 'Manual' trailing
\n");
 }
 
 TEST(link_reference_definition_title_with_trailing_text_on_next_line) {
   const auto result{sourcemeta::core::markdown_to_html(
       "[docs]: /manual\n'Manual' trailing\n\n[docs]")};
-  EXPECT_EQ(result, "
'Manual' trailing
\n"
-                    "
docs
\n");
+  EXPECT_EQ(result, "
'Manual' trailing
\n"
+                    "
docs
\n");
 }
 
 TEST(link_reference_definition_indented_four_spaces_is_code) {
@@ -230,6 +231,12 @@ TEST(link_reference_definition_label_whitespace_normalized) {
   EXPECT_EQ(result, "
The long label
\n");
 }
 
+TEST(link_reference_definition_label_line_tabulation_normalized) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      "[the\flong\vlabel]: /manual\n\n[The long label]")};
+  EXPECT_EQ(result, "
The long label
\n");
+}
+
 TEST(link_reference_definition_empty_label_is_invalid) {
   const auto result{
       sourcemeta::core::markdown_to_html("[]: /manual\n\n[] text")};
@@ -259,11 +266,21 @@ TEST(link_destination_angle_brackets_with_spaces) {
   EXPECT_EQ(result, "
docs
\n");
 }
 
+TEST(link_destination_angle_brackets_with_surrounding_spaces) {
+  const auto result{sourcemeta::core::markdown_to_html("[docs](< a >)")};
+  EXPECT_EQ(result, "
docs
\n");
+}
+
 TEST(link_destination_with_space_is_not_link) {
   const auto result{sourcemeta::core::markdown_to_html("[docs](/a b)")};
   EXPECT_EQ(result, "
[docs](/a b)
\n");
 }
 
+TEST(link_destination_with_control_character_is_not_link) {
+  const auto result{sourcemeta::core::markdown_to_html("[docs](/a\x01y)")};
+  EXPECT_EQ(result, "
[docs](/a\x01y)
\n");
+}
+
 TEST(link_destination_with_newline_is_not_link) {
   const auto result{sourcemeta::core::markdown_to_html("[docs](/a\n/b)")};
   EXPECT_EQ(result, "
[docs](/a\n"
@@ -307,6 +324,11 @@ TEST(link_destination_unbalanced_parentheses_is_not_link) {
   EXPECT_EQ(result, "
[docs](a(b(c)d)
\n");
 }
 
+TEST(link_destination_escaped_ampersand_is_not_an_entity) {
+  const auto result{sourcemeta::core::markdown_to_html("[docs](/x\\&y)")};
+  EXPECT_EQ(result, "
docs
\n");
+}
+
 TEST(link_destination_escaped_unbalanced_parentheses) {
   const auto result{sourcemeta::core::markdown_to_html("[docs](a\\(b\\(c)")};
   EXPECT_EQ(result, "
docs
\n");
@@ -343,7 +365,7 @@ TEST(link_destination_percent_encoding_and_entities) {
 
 TEST(link_destination_that_looks_like_title) {
   const auto result{sourcemeta::core::markdown_to_html("[docs]('Title')")};
-  EXPECT_EQ(result, "
docs
\n");
+  EXPECT_EQ(result, "
docs
\n");
 }
 
 TEST(link_title_quote_styles) {
@@ -358,24 +380,30 @@ TEST(link_title_escaped_quotes_and_entities) {
   const auto result{
       sourcemeta::core::markdown_to_html("[docs](/x 'it\\'s & more')")};
   EXPECT_EQ(result,
-            "
docs
\n");
+            "
docs
\n");
+}
+
+TEST(link_title_escaped_ampersand_is_not_an_entity) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("[docs](/x 'a\\&b')")};
+  EXPECT_EQ(result, "
docs
\n");
 }
 
 TEST(link_title_separated_by_non_breaking_space) {
   const auto result{
       sourcemeta::core::markdown_to_html("[docs](/x\xC2\xA0'title')")};
-  EXPECT_EQ(result, "
docs
\n");
+  EXPECT_EQ(result, "
docs
\n");
 }
 
 TEST(link_title_with_unescaped_inner_quote_is_not_link) {
   const auto result{sourcemeta::core::markdown_to_html("[docs](/x 'a'b')")};
-  EXPECT_EQ(result, "
[docs](/x 'a'b')
\n");
+  EXPECT_EQ(result, "
[docs](/x 'a'b')
\n");
 }
 
 TEST(link_title_parentheses_with_quotes_inside) {
   const auto result{sourcemeta::core::markdown_to_html(
       "[docs](/x (with 'single' and \"double\"))")};
-  EXPECT_EQ(result, "
docs
\n");
 }
 
@@ -587,3 +615,43 @@ TEST(inline_link_invalid_falls_back_to_reference) {
       sourcemeta::core::markdown_to_html("[manual](a b)\n\n[manual]: /manual")};
   EXPECT_EQ(result, "
manual(a b)
\n");
 }
+
+TEST(link_destination_angle_brackets_backslash_before_line_ending) {
+  const auto result{sourcemeta::core::markdown_to_html("[x]()")};
+  EXPECT_EQ(result, "
[x](<a
\n"
+                    "b>)
\n");
+}
+
+TEST(link_destination_angle_brackets_with_carriage_return_is_not_link) {
+  const auto result{sourcemeta::core::markdown_to_html("[x]()")};
+  EXPECT_EQ(result, "
[x](</a\n"
+                    "b>)
\n");
+}
+
+TEST(link_title_escaped_backslash_before_closing_quote) {
+  const auto result{sourcemeta::core::markdown_to_html(R"MD([x](/y "a\\"))MD")};
+  EXPECT_EQ(result, "
x
\n");
+}
+
+TEST(link_title_escaped_backslash_does_not_escape_the_quote) {
+  const auto result{
+      sourcemeta::core::markdown_to_html(R"MD([x](/y "a\\"b"))MD")};
+  EXPECT_EQ(result, "
[x](/y "a\\"b")
\n");
+}
+
+TEST(link_title_escaped_backslash_before_opening_parenthesis) {
+  const auto result{sourcemeta::core::markdown_to_html("[x](/y (a\\\\(b))")};
+  EXPECT_EQ(result, "
[x](/y (a\\(b))
\n");
+}
+
+TEST(link_reference_definition_before_title_followed_by_text) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("[foo]: /url\n\"title\" ok")};
+  EXPECT_EQ(result, "
"title" ok
\n");
+}
+
+TEST(link_inside_link_text_with_later_bracket_is_not_allowed) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("[a [b](/c) [d] e](/u)")};
+  EXPECT_EQ(result, "
[a b [d] e](/u)
\n");
+}
diff --git a/test/markdown/markdown_lists_test.cc b/test/markdown/markdown_lists_test.cc
index c7cd696ec6..6ab086835e 100644
--- a/test/markdown/markdown_lists_test.cc
+++ b/test/markdown/markdown_lists_test.cc
@@ -791,3 +791,32 @@ TEST(list_tight_sublist_inside_loose_list) {
                     "\n"
                     "\n");
 }
+
+TEST(list_items_separated_by_blank_line_after_thematic_break_are_loose) {
+  const auto result{sourcemeta::core::markdown_to_html("- ***\n\n- b\n")};
+  EXPECT_EQ(result, "
    \n"
+                    "
  • \n"
+                    "
    \n"
+                    "
    • \n"
+                    "
  • \n"
+                    "
    b
    \n"
+                    "
    • \n"
+                    "
\n");
+}
+
+TEST(list_item_with_table_header_stays_tight) {
+  const auto result{
+      sourcemeta::core::markdown_to_html("- | a |\n  | --- |\n- b\n")};
+  EXPECT_EQ(result, "
    \n"
+                    "
  • \n"
+                    "\n"
+                    "\n"
+                    "\n"
+                    "\n"
+                    "\n"
+                    "\n"
+                    "
    a
    \n"
+                    "
    • \n"
+                    "
  • b
    • \n"
+                    "
\n");
+}
diff --git a/test/markdown/markdown_pathological_test.cc b/test/markdown/markdown_pathological_test.cc
index 045c3e64e3..8b47d39b4c 100644
--- a/test/markdown/markdown_pathological_test.cc
+++ b/test/markdown/markdown_pathological_test.cc
@@ -166,9 +166,15 @@ TEST(code_span_with_eighty_backticks) {
 }
 
 TEST(code_span_with_eighty_one_backticks) {
-  const auto input{std::string(81, '`') + "x" + std::string(81, '`')};
-  const auto result{sourcemeta::core::markdown_to_html(input)};
-  EXPECT_EQ(result, "
" + input + "
\n");
+  const auto result{sourcemeta::core::markdown_to_html(
+      std::string(81, '`') + "x" + std::string(81, '`'))};
+  EXPECT_EQ(result, "
x
\n");
+}
+
+TEST(code_span_with_one_thousand_backticks) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      std::string(1000, '`') + "x" + std::string(1000, '`'))};
+  EXPECT_EQ(result, "
x
\n");
 }
 
 TEST(alternating_blockquote_and_list_nesting_on_one_line) {
@@ -180,14 +186,37 @@ TEST(alternating_blockquote_and_list_nesting_on_one_line) {
                         repeat("\n\n\n", 29));
 }
 
-TEST(alternating_blockquote_and_list_nesting_on_one_line_beyond_depth_limit) {
+TEST(alternating_blockquote_and_list_nesting_on_one_line_of_fifty_levels) {
   const auto result{
       sourcemeta::core::markdown_to_html(repeat("> - ", 50) + "deep")};
   EXPECT_EQ(result, repeat("
\n
    \n
  • \n", 49) +
-                        "
    \n
    - deep
    \n
    \n" +
+                        "
    \n
      \n
    • deep
      • \n
    \n"
+                        "
    \n" +
                         repeat("
    • \n
\n
\n", 49));
 }
 
+TEST(blank_lines_after_deeply_nested_list_items) {
+  const auto result{sourcemeta::core::markdown_to_html(
+      repeat("- ", 20000) + "x\n" + repeat("\n \n", 10000))};
+  EXPECT_EQ(result, repeat("
    \n
  • \n", 19999) +
+                        "
      \n
    • x
      • \n
    \n" +
+                        repeat("
    • \n
\n", 19999));
+}
+
+TEST(footnote_references_inside_deeply_nested_block_quotes) {
+  const auto result{sourcemeta::core::markdown_to_html(repeat("> ", 20000) +
+                                                       repeat("x[^a]", 20000))};
+  EXPECT_EQ(result, repeat("
\n", 20000) + "
" +
+                        repeat("x[^a]", 20000) + "
\n" +
+                        repeat("
\n", 20000));
+}
+
+TEST(www_autolinks_after_underscores_in_one_long_domain) {
+  const auto input{repeat("x_www.", 10000)};
+  const auto result{sourcemeta::core::markdown_to_html(input)};
+  EXPECT_EQ(result, "
" + input + "
\n");
+}
+
 TEST(staircase_list_of_one_hundred_fifty_levels) {
   const auto result{sourcemeta::core::markdown_to_html(staircase_list(150))};
   EXPECT_EQ(result, repeat("