diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 859ebeb487..525cca9465 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -366,6 +366,27 @@ A UBJSON high-precision number could not be parsed. [json.exception.parse_error.115] parse error at byte 5: syntax error while parsing UBJSON high-precision number: invalid number text: 1A ``` +### json.exception.parse_error.116 + +A binary input format (CBOR, MessagePack, UBJSON/BJData, or BSON) contained +arrays/objects nested deeper than the library's internal depth limit (4096). +The binary format parsers track container nesting on an explicit, +heap-allocated stack rather than the native call stack, so *parsing* alone +can no longer cause a stack overflow regardless of this limit. The limit +itself stays conservative rather than being raised as high as that guarantee +would allow, because operations on the resulting value - the copy +constructor, `dump()`, `to_cbor()`/`to_msgpack()`/`to_ubjson()`/`to_bson()`, +and `operator==` - are still recursive, and a successfully parsed value +nested deeper than the low thousands can overflow the native stack in +*those* regardless of how it was produced. Inputs nested beyond the limit +are rejected with this parse error instead. + +!!! failure "Example message" + + ``` + [json.exception.parse_error.116] parse error at byte 4097: syntax error while parsing CBOR value: maximum depth of nested arrays/objects exceeded + ``` + ## Iterator errors This exception is thrown if iterators passed to a library function do not match diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index ad862827d0..0eb67de5c6 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -163,6 +163,29 @@ class binary_reader // BSON // ////////// + /*! + @brief a single pending "resume this document/array" entry used by the + iterative container traversal in @ref parse_bson_internal; see + @ref cbor_container_frame for the general idea. + + BSON documents and arrays share one on-the-wire layout (a size-prefixed, + 0x00-terminated list of typed elements); @ref is_object only controls + whether each element's key is relayed to the SAX consumer via + @c sax->key() (an object) or not (an array, whose "0", "1", ... keys the + SAX array interface has no use for). Every BSON container is + terminator-based (there is no known-element-count form), so unlike CBOR/ + MessagePack/UBJSON this frame needs no length/indefinite bookkeeping at + all. @ref document_start / @ref document_size are recorded so @ref + check_bson_document_size can validate this frame's declared size once its + terminator is reached. + */ + struct bson_container_frame + { + bool is_object; ///< false: array, true: object/document + std::size_t document_start; ///< chars_read before this document/array's size prefix + std::int32_t document_size; ///< this document/array's declared size + }; + /*! @brief Validate a BSON document's declared size against the bytes read. @@ -195,26 +218,192 @@ class binary_reader */ bool parse_bson_internal() { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - get_number(input_format_t::bson, document_size); + std::vector stack; + + // reads the 4-byte document-size prefix and records where it + // started (both are needed by @ref check_bson_document_size once + // this document/array's terminator is reached); deliberately does + // not touch the SAX consumer, so callers can run the max_depth + // check before the corresponding start_object()/start_array() + // event fires + auto read_document_header = [&](std::size_t& document_start, std::int32_t& document_size) + { + document_start = chars_read; + get_number(input_format_t::bson, document_size); + }; + std::size_t document_start{}; + std::int32_t document_size{}; + read_document_header(document_start, document_size); if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) { return false; } + stack.push_back(bson_container_frame{true, document_start, document_size}); - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + string_t key; + while (true) { - return false; - } + const std::size_t top_index = stack.size() - 1; - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + const auto element_type = get(); + if (element_type == 0) + { + // 0x00 terminates the current document/array + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(stack[top_index].document_start, stack[top_index].document_size))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!(stack[top_index].is_object ? sax->end_object() : sax->end_array()))) + { + return false; + } + stack.pop_back(); + if (stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } - return sax->end_object(); + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + if (stack[top_index].is_object && JSON_HEDLEY_UNLIKELY(!sax->key(key))) + { + return false; + } + + switch (element_type) + { + case 0x01: // double + { + double number{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), "")))) + { + return false; + } + continue; + } + + case 0x02: // string + { + std::int32_t len{}; + string_t value; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value)))) + { + return false; + } + continue; + } + + case 0x03: // object + { + std::size_t nested_document_start{}; + std::int32_t nested_document_size{}; + read_document_header(nested_document_start, nested_document_size); + if (JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + return reached_max_depth(input_format_t::bson); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + { + return false; + } + stack.push_back(bson_container_frame{true, nested_document_start, nested_document_size}); + continue; + } + + case 0x04: // array + { + std::size_t nested_document_start{}; + std::int32_t nested_document_size{}; + read_document_header(nested_document_start, nested_document_size); + if (JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + return reached_max_depth(input_format_t::bson); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) + { + return false; + } + stack.push_back(bson_container_frame{false, nested_document_start, nested_document_size}); + continue; + } + + case 0x05: // binary + { + std::int32_t len{}; + binary_t value; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value)))) + { + return false; + } + continue; + } + + case 0x08: // boolean + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(get() != 0))) + { + return false; + } + continue; + + case 0x0A: // null + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + continue; + + case 0x10: // int32 + { + std::int32_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_integer(value)))) + { + return false; + } + continue; + } + + case 0x12: // int64 + { + std::int64_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_integer(value)))) + { + return false; + } + continue; + } + + case 0x11: // uint64 + { + std::uint64_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_unsigned(value)))) + { + return false; + } + continue; + } + + default: // anything else is not supported (yet) + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + const std::string cr_str{cr.data()}; + return sax->parse_error(element_type_parse_position, cr_str, + parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + } + } } /*! @@ -293,164 +482,6 @@ class binary_reader return get_binary(input_format_t::bson, len, result); } - /*! - @brief Read a BSON document element of the given @a element_type. - @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html - @param[in] element_type_parse_position The position in the input stream, - where the `element_type` was read. - @warning Not all BSON element types are supported yet. An unsupported - @a element_type will give rise to a parse_error.114: - Unsupported BSON record type 0x... - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_internal(const char_int_type element_type, - const std::size_t element_type_parse_position) - { - switch (element_type) - { - case 0x01: // double - { - double number{}; - return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); - } - - case 0x02: // string - { - std::int32_t len{}; - string_t value; - return get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value); - } - - case 0x03: // object - { - return parse_bson_internal(); - } - - case 0x04: // array - { - return parse_bson_array(); - } - - case 0x05: // binary - { - std::int32_t len{}; - binary_t value; - return get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value); - } - - case 0x08: // boolean - { - return sax->boolean(get() != 0); - } - - case 0x0A: // null - { - return sax->null(); - } - - case 0x10: // int32 - { - std::int32_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); - } - - case 0x12: // int64 - { - std::int64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); - } - - case 0x11: // uint64 - { - std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_unsigned(value); - } - - default: // anything else is not supported (yet) - { - std::array cr{{}}; - static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - const std::string cr_str{cr.data()}; - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); - } - } - } - - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - get_number(input_format_t::bson, document_size); - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } - ////////// // CBOR // ////////// @@ -482,471 +513,665 @@ class binary_reader return sax->number_integer(static_cast(-1) - static_cast(number)); } + /*! + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_cbor_internal + + Nested CBOR containers are parsed by pushing one of these onto a + heap-allocated std::vector instead of recursing on the native call stack + for every nesting level (see #5104): parsing a scalar never recurses, + and parsing a container just pushes a frame here and loops back around + to parse its first child; when a container finishes, its frame is + popped and parsing resumes where the parent left off. Native call-stack + depth used while parsing is therefore O(1) regardless of how deeply the + input nests. + */ + struct cbor_container_frame + { + bool is_object; ///< false: array, true: object/map + bool indefinite; ///< true: length is unknown; read until a 0xFF break marker + std::size_t remaining; ///< remaining element count; meaningful only if !indefinite + bool awaiting_key; ///< object only: true if the next thing to read is a key, not a value + }; + bool parse_cbor_internal(const bool get_char, const cbor_tag_handler_t tag_handler) { - switch (get_char ? get() : current) + std::vector stack; + + // whether the next value-read below should fetch a fresh byte via + // get() (true) or reuse the byte already sitting in `current` (false) + bool fetch_char = get_char; + + // called whenever a value (scalar, or a container that was just + // completed/immediately-empty) was produced at the position the + // current top-of-stack frame is waiting on; returns true if that + // means the whole parse is finished (the stack is empty, i.e. this + // was the outermost value), false if parsing should continue. + auto produce = [&]() -> bool { - // EOF - case char_traits::eof(): - return unexpect_eof(input_format_t::cbor, "value"); - - // Integer 0x00..0x17 (0..23) - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - return sax->number_unsigned(static_cast(current)); - - case 0x18: // Unsigned integer (one-byte uint8_t follows) + if (stack.empty()) { - std::uint8_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return true; } - - case 0x19: // Unsigned integer (two-byte uint16_t follows) + cbor_container_frame& top = stack.back(); + if (top.is_object) { - std::uint16_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + top.awaiting_key = true; } - - case 0x1A: // Unsigned integer (four-byte uint32_t follows) + if (!top.indefinite) { - std::uint32_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + --top.remaining; } + return false; + }; - case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - { - std::uint64_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); - } - - // Negative integer -1-0x00..-1-0x17 (-1..-24) - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - return sax->number_integer(static_cast(0x20 - 1 - current)); - - case 0x38: // Negative integer (one-byte uint8_t follows) - return get_cbor_negative_integer(); - - case 0x39: // Negative integer -1-n (two-byte uint16_t follows) - return get_cbor_negative_integer(); - - case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) - return get_cbor_negative_integer(); - - case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) - return get_cbor_negative_integer(); - - // Binary data (0x00..0x17 bytes follow) - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: // Binary data (one-byte uint8_t for n follows) - case 0x59: // Binary data (two-byte uint16_t for n follow) - case 0x5A: // Binary data (four-byte uint32_t for n follow) - case 0x5B: // Binary data (eight-byte uint64_t for n follow) - case 0x5F: // Binary data (indefinite length) + // begin parsing a nested array/object of the given length + // (detail::unknown_size() for indefinite length); returns 0 on SAX + // failure or exceeding max_depth (caller should `return false`), 1 + // if a frame was pushed (caller should `continue` the outer loop to + // parse the container's first element), 2 if the container was + // immediately completed because it is empty (caller should treat + // this like any other produced value via `produce()`). + auto enter_container = [&](const bool is_object, const std::size_t len) -> int + { + // len == 0 (definite-length, empty) never pushes a frame, so it + // never counts against max_depth; every other case - definite + // non-zero or indefinite - always does, so the check can run + // before the SAX start event fires + if (len != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) { - binary_t b; - return get_cbor_binary(b) && sax->binary(b); + reached_max_depth(input_format_t::cbor); + return 0; } - - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) - case 0x7F: // UTF-8 string (indefinite length) - { - string_t s; - return get_cbor_string(s) && sax->string(s); - } - - // array (0x00..0x17 data items follow) - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); - - case 0x98: // array (one-byte uint8_t for n follows) + const bool started = is_object ? sax->start_object(len) : sax->start_array(len); + if (JSON_HEDLEY_UNLIKELY(!started)) { - std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return 0; } - - case 0x99: // array (two-byte uint16_t for n follow) + if (len == 0) { - std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + const bool ended = is_object ? sax->end_object() : sax->end_array(); + return ended ? 2 : 0; } + stack.push_back(cbor_container_frame{is_object, len == detail::unknown_size(), len, is_object}); + return 1; + }; - case 0x9A: // array (four-byte uint32_t for n follow) - { - std::uint32_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast(len), tag_handler); - } +#define JSON_CBOR_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_CBOR_ENTER(is_obj, length) \ + switch (enter_container(is_obj, length)) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ + } - case 0x9B: // array (eight-byte uint64_t for n follow) + string_t key; + while (true) + { + if (!stack.empty()) { - std::uint64_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast(len), tag_handler); - } + cbor_container_frame& top = stack.back(); - case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + if (top.is_object && top.awaiting_key) + { + const bool finished = top.indefinite ? (get() == 0xFF) : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } - // map (0x00..0x17 pairs of data items follow) - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + if (!top.indefinite) + { + get(); + } + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + top.awaiting_key = false; + fetch_char = true; + } + else if (!top.is_object) + { + const bool finished = top.indefinite ? (get() == 0xFF) : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + fetch_char = !top.indefinite; + } + // else: top is an object frame with !awaiting_key, i.e. a + // key was just read above in this same iteration (no branch + // above sets awaiting_key back to false except that one); + // fetch_char is already true from that path, so just fall + // through to read the value. + } + +read_value: + switch (fetch_char ? get() : current) + { + // EOF + case char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + JSON_CBOR_VALUE(sax->number_unsigned(static_cast(current))); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xB8: // map (one-byte uint8_t for n follows) - { - std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); - } + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xB9: // map (two-byte uint16_t for n follow) - { - std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); - } + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xBA: // map (four-byte uint32_t for n follow) - { - std::uint32_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast(len), tag_handler); - } + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xBB: // map (eight-byte uint64_t for n follow) - { - std::uint64_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast(len), tag_handler); - } - - case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); - - case 0xC6: // tagged item - case 0xC7: - case 0xC8: - case 0xC9: - case 0xCA: - case 0xCB: - case 0xCC: - case 0xCD: - case 0xCE: - case 0xCF: - case 0xD0: - case 0xD1: - case 0xD2: - case 0xD3: - case 0xD4: - case 0xD8: // tagged item (1 byte follows) - case 0xD9: // tagged item (2 bytes follow) - case 0xDA: // tagged item (4 bytes follow) - case 0xDB: // tagged item (8 bytes follow) - { - switch (tag_handler) - { - case cbor_tag_handler_t::error: + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + JSON_CBOR_VALUE(sax->number_integer(static_cast(0x20 - 1 - current))); + + case 0x38: // Negative integer (one-byte uint8_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: // Binary data (one-byte uint8_t for n follows) + case 0x59: // Binary data (two-byte uint16_t for n follow) + case 0x5A: // Binary data (four-byte uint32_t for n follow) + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + case 0x5F: // Binary data (indefinite length) + { + binary_t b; + JSON_CBOR_VALUE(get_cbor_binary(b) && sax->binary(b)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + JSON_CBOR_VALUE(get_cbor_string(s) && sax->string(s)); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + JSON_CBOR_ENTER(false, conditional_static_cast(static_cast(current) & 0x1Fu)); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return false; } + JSON_CBOR_ENTER(false, static_cast(len)); + } - case cbor_tag_handler_t::ignore: + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - // ignore binary subtype - switch (current) - { - case 0xD8: - { - std::uint8_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xD9: - { - std::uint16_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xDA: - { - std::uint32_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xDB: - { - std::uint64_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - default: - break; - } - return parse_cbor_internal(true, tag_handler); + return false; } + JSON_CBOR_ENTER(false, static_cast(len)); + } - case cbor_tag_handler_t::store: + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - binary_t b; - // use binary subtype and store in a binary container - switch (current) + return false; + } + JSON_CBOR_ENTER(false, conditional_static_cast(len)); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(false, conditional_static_cast(len)); + } + + case 0x9F: // array (indefinite length) + JSON_CBOR_ENTER(false, detail::unknown_size()); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + JSON_CBOR_ENTER(true, conditional_static_cast(static_cast(current) & 0x1Fu)); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, static_cast(len)); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, static_cast(len)); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, conditional_static_cast(len)); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, conditional_static_cast(len)); + } + + case 0xBF: // map (indefinite length) + JSON_CBOR_ENTER(true, detail::unknown_size()); + + case 0xC6: // tagged item + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD8: // tagged item (1 byte follows) + case 0xD9: // tagged item (2 bytes follow) + case 0xDA: // tagged item (4 bytes follow) + case 0xDB: // tagged item (8 bytes follow) + { + switch (tag_handler) + { + case cbor_tag_handler_t::error: { - case 0xD8: - { - std::uint8_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; - } - case 0xD9: - { - std::uint16_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; - } - case 0xDA: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } + + case cbor_tag_handler_t::ignore: + { + // ignore binary subtype + switch (current) { - std::uint32_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; + case 0xD8: + { + std::uint8_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xD9: + { + std::uint16_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDA: + { + std::uint32_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDB: + { + std::uint64_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + default: + break; } - case 0xDB: + fetch_char = true; + goto read_value; + } + + case cbor_tag_handler_t::store: + { + binary_t b; + // use binary subtype and store in a binary container + switch (current) { - std::uint64_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; + case 0xD8: + { + std::uint8_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xD9: + { + std::uint16_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDA: + { + std::uint32_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDB: + { + std::uint64_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + default: + fetch_char = true; + goto read_value; } - default: - return parse_cbor_internal(true, tag_handler); + get(); + JSON_CBOR_VALUE(get_cbor_binary(b) && sax->binary(b)); } - get(); - return get_cbor_binary(b) && sax->binary(b); - } - default: // LCOV_EXCL_LINE - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return false; // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } } - } - case 0xF4: // false - return sax->boolean(false); + case 0xF4: // false + JSON_CBOR_VALUE(sax->boolean(false)); - case 0xF5: // true - return sax->boolean(true); + case 0xF5: // true + JSON_CBOR_VALUE(sax->boolean(true)); - case 0xF6: // null - return sax->null(); + case 0xF6: // null + JSON_CBOR_VALUE(sax->null()); - case 0xF9: // Half-Precision Float (two-byte IEEE 754) - { - const auto byte1_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + case 0xF9: // Half-Precision Float (two-byte IEEE 754) { - return false; - } - const auto byte2_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) - { - return false; - } - - const auto byte1 = static_cast(byte1_raw); - const auto byte2 = static_cast(byte2_raw); - - // Code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const auto half = static_cast((byte1 << 8u) + byte2); - const double val = [&half] - { - const int exp = (half >> 10u) & 0x1Fu; - const unsigned int mant = half & 0x3FFu; - JSON_ASSERT(0 <= exp&& exp <= 32); - JSON_ASSERT(mant <= 1024); - switch (exp) + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; } - }(); - return sax->number_float((half & 0x8000u) != 0 - ? static_cast(-val) - : static_cast(val), ""); - } - case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); - } + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // Code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + JSON_CBOR_VALUE(sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), "")); + } - case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); - } + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), "")); + } - default: // anything else (0xFF is handled inside the other types) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), "")); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } } } + +#undef JSON_CBOR_VALUE +#undef JSON_CBOR_ENTER } /*! @@ -1145,475 +1370,521 @@ class binary_reader } } + ///////////// + // MsgPack // + ///////////// + /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_msgpack_internal; + see @ref cbor_container_frame for the general idea. MessagePack + has no indefinite-length containers, so unlike CBOR this frame + does not need an "indefinite" flag. */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + struct msgpack_container_frame { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + bool is_object; ///< false: array, true: map + std::size_t remaining; ///< remaining element count + bool awaiting_key; ///< map only: true if the next thing to read is a key, not a value + }; + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + std::vector stack; + + auto produce = [&]() -> bool { + if (stack.empty()) + { + return true; + } + msgpack_container_frame& top = stack.back(); + if (top.is_object) + { + top.awaiting_key = true; + } + --top.remaining; return false; - } + }; - if (len != detail::unknown_size()) + auto enter_container = [&](const bool is_object, const std::size_t len) -> int { - for (std::size_t i = 0; i < len; ++i) + // MessagePack containers are always definite-length, so len == 0 + // is the only case that never pushes a frame / never counts + // against max_depth + if (len != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } + reached_max_depth(input_format_t::msgpack); + return 0; } - } - else - { - while (get() != 0xFF) + const bool started = is_object ? sax->start_object(len) : sax->start_array(len); + if (JSON_HEDLEY_UNLIKELY(!started)) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) - { - return false; - } + return 0; } - } + if (len == 0) + { + const bool ended = is_object ? sax->end_object() : sax->end_array(); + return ended ? 2 : 0; + } + stack.push_back(msgpack_container_frame{is_object, len, is_object}); + return 1; + }; - return sax->end_array(); +#define JSON_MSGPACK_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_MSGPACK_ENTER(is_obj, length) \ + switch (enter_container(is_obj, length)) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ } - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) + string_t key; + while (true) { - string_t key; - if (len != detail::unknown_size()) + if (!stack.empty()) { - for (std::size_t i = 0; i < len; ++i) + msgpack_container_frame& top = stack.back(); + + if (top.is_object && top.awaiting_key) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + if (top.remaining == 0) { - return false; + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) { return false; } - key.clear(); + top.awaiting_key = false; } - } - else - { - while (get() != 0xFF) + else if (!top.is_object) { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + if (top.remaining == 0) { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; } - key.clear(); } - } - } - - return sax->end_object(); - } - - ///////////// - // MsgPack // - ///////////// - - /*! - @return whether a valid MessagePack value was passed to the SAX parser - */ - bool parse_msgpack_internal() - { - switch (get()) - { - // EOF - case char_traits::eof(): - return unexpect_eof(input_format_t::msgpack, "value"); - - // positive fixint - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: - case 0x59: - case 0x5A: - case 0x5B: - case 0x5C: - case 0x5D: - case 0x5E: - case 0x5F: - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: - case 0x79: - case 0x7A: - case 0x7B: - case 0x7C: - case 0x7D: - case 0x7E: - case 0x7F: - return sax->number_unsigned(static_cast(current)); - - // fixmap - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); - - // fixarray - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - case 0x98: - case 0x99: - case 0x9A: - case 0x9B: - case 0x9C: - case 0x9D: - case 0x9E: - case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); - - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - case 0xD9: // str 8 - case 0xDA: // str 16 - case 0xDB: // str 32 - { - string_t s; - return get_msgpack_string(s) && sax->string(s); - } - - case 0xC0: // nil - return sax->null(); - - case 0xC2: // false - return sax->boolean(false); + // else: top is a map frame with !awaiting_key, i.e. a key + // was just read above in this same iteration; fall through + // to read that key's value. + } + + switch (get()) + { + // EOF + case char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + JSON_MSGPACK_VALUE(sax->number_unsigned(static_cast(current))); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + JSON_MSGPACK_ENTER(true, conditional_static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + JSON_MSGPACK_ENTER(false, conditional_static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + JSON_MSGPACK_VALUE(get_msgpack_string(s) && sax->string(s)); + } - case 0xC3: // true - return sax->boolean(true); + case 0xC0: // nil + JSON_MSGPACK_VALUE(sax->null()); + + case 0xC2: // false + JSON_MSGPACK_VALUE(sax->boolean(false)); + + case 0xC3: // true + JSON_MSGPACK_VALUE(sax->boolean(true)); + + case 0xC4: // bin 8 + case 0xC5: // bin 16 + case 0xC6: // bin 32 + case 0xC7: // ext 8 + case 0xC8: // ext 16 + case 0xC9: // ext 32 + case 0xD4: // fixext 1 + case 0xD5: // fixext 2 + case 0xD6: // fixext 4 + case 0xD7: // fixext 8 + case 0xD8: // fixext 16 + { + binary_t b; + JSON_MSGPACK_VALUE(get_msgpack_binary(b) && sax->binary(b)); + } - case 0xC4: // bin 8 - case 0xC5: // bin 16 - case 0xC6: // bin 32 - case 0xC7: // ext 8 - case 0xC8: // ext 16 - case 0xC9: // ext 32 - case 0xD4: // fixext 1 - case 0xD5: // fixext 2 - case 0xD6: // fixext 4 - case 0xD7: // fixext 8 - case 0xD8: // fixext 16 - { - binary_t b; - return get_msgpack_binary(b) && sax->binary(b); - } + case 0xCA: // float 32 + { + float number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), "")); + } - case 0xCA: // float 32 - { - float number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); - } + case 0xCB: // float 64 + { + double number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), "")); + } - case 0xCB: // float 64 - { - double number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); - } + case 0xCC: // uint 8 + { + std::uint8_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - case 0xCC: // uint 8 - { - std::uint8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } + case 0xCD: // uint 16 + { + std::uint16_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - case 0xCD: // uint 16 - { - std::uint16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } + case 0xCE: // uint 32 + { + std::uint32_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - case 0xCE: // uint 32 - { - std::uint32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } + case 0xCF: // uint 64 + { + std::uint64_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - case 0xCF: // uint 64 - { - std::uint64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } + case 0xD0: // int 8 + { + std::int8_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - case 0xD0: // int 8 - { - std::int8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } + case 0xD1: // int 16 + { + std::int16_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - case 0xD1: // int 16 - { - std::int16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } + case 0xD2: // int 32 + { + std::int32_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - case 0xD2: // int 32 - { - std::int32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } + case 0xD3: // int 64 + { + std::int64_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - case 0xD3: // int 64 - { - std::int64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } + case 0xDC: // array 16 + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) + { + return false; + } + JSON_MSGPACK_ENTER(false, static_cast(len)); + } - case 0xDC: // array 16 - { - std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); - } + case 0xDD: // array 32 + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) + { + return false; + } + JSON_MSGPACK_ENTER(false, conditional_static_cast(len)); + } - case 0xDD: // array 32 - { - std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); - } + case 0xDE: // map 16 + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) + { + return false; + } + JSON_MSGPACK_ENTER(true, static_cast(len)); + } - case 0xDE: // map 16 - { - std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); - } + case 0xDF: // map 32 + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) + { + return false; + } + JSON_MSGPACK_ENTER(true, conditional_static_cast(len)); + } - case 0xDF: // map 32 - { - std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); - } - - // negative fixint - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - case 0xF0: - case 0xF1: - case 0xF2: - case 0xF3: - case 0xF4: - case 0xF5: - case 0xF6: - case 0xF7: - case 0xF8: - case 0xF9: - case 0xFA: - case 0xFB: - case 0xFC: - case 0xFD: - case 0xFE: - case 0xFF: - return sax->number_integer(static_cast(current)); - - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + JSON_MSGPACK_VALUE(sax->number_integer(static_cast(current))); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } } } + +#undef JSON_MSGPACK_VALUE +#undef JSON_MSGPACK_ENTER } /*! @@ -1816,72 +2087,523 @@ class binary_reader } } + //////////// + // UBJSON // + //////////// + + /*! + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_ubjson_internal; see + @ref cbor_container_frame for the general idea. + + UBJSON/BJData needs two extra bits of state beyond CBOR/MessagePack: + @ref fixed_type captures the optimized `$type#count` container's + homogeneous element type (0 means "heterogeneous": read a fresh type + marker per element, like an ordinary UBJSON array/object); @ref + extra_close_object is set for the synthetic JData-annotated-array object + ("_ArrayType_"/"_ArraySize_"/"_ArrayData_") BJData wraps an ND-array in, + where finishing the `_ArrayData_` array must also close that wrapping + object. + */ + struct ubjson_container_frame + { + bool is_object; ///< false: array, true: object + bool indefinite; ///< true: no known count; read until a ']'/'}' terminator + std::size_t remaining; ///< remaining element count; meaningful only if !indefinite + bool awaiting_key; ///< object only: true if the next thing to read is a key, not a value + char_int_type fixed_type; ///< 0: heterogeneous; else every element has this known type marker + bool extra_close_object; ///< true: end_array() for this frame must be followed by an end_object() + }; + /*! - @param[in] len the length of the array - @return whether array creation completed + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser */ - bool get_msgpack_array(const std::size_t len) + bool parse_ubjson_internal(const bool get_char = true) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + std::vector stack; + const bool fetch_char = get_char; + + auto produce = [&]() -> bool { + if (stack.empty()) + { + return true; + } + ubjson_container_frame& top = stack.back(); + if (top.is_object) + { + top.awaiting_key = true; + } + if (!top.indefinite) + { + --top.remaining; + } + else + { + // indefinite-length containers are terminated by inspecting + // `current`, so advance past the just-produced element/pair + // to whatever byte should be inspected next (mirrors the + // `get_ignore_noop()` at the end of each iteration of the + // original recursive implementation's while-loops) + get_ignore_noop(); + } return false; - } + }; - for (std::size_t i = 0; i < len; ++i) + // returns 0 = failure (caller returns false), 1 = a frame was + // pushed (caller continues the outer loop to parse the first + // element), 2 = a value was produced immediately - either because + // the container turned out to be empty, or (BJData only) because + // the "optimized binary array" special case produced a single + // binary value directly, without ever being a SAX array at all + // (caller should invoke produce() same as for any other value) + auto enter_array = [&]() -> int { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) { - return false; + return 0; } - } - return sax->end_array(); - } + // ND-array: encode as an object in JData annotated array format + // (https://github.com/NeuroJSON/jdata). The wrapping object was + // already start_object()'d (with the "_ArraySize_" key/array + // already emitted) inside get_ubjson_size_type(), so only + // "_ArrayType_" and "_ArrayData_" remain to be added here. + if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + { + size_and_type.second &= ~(static_cast(1) << 8); + auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t) + { + return p.first < t; + }); + string_t key = "_ArrayType_"; + if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) + { + auto last_token = get_token_string(); + sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); + return 0; + } - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + string_t type = it->second; // sax->string() takes a reference + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type))) + { + return 0; + } + + if (size_and_type.second == 'C' || size_and_type.second == 'B') + { + size_and_type.second = 'U'; + } + + if (size_and_type.first != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + + key = "_ArrayData_"; + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first))) + { + return 0; + } + + if (size_and_type.first == 0) + { + return (sax->end_array() && sax->end_object()) ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, false, size_and_type.first, false, size_and_type.second, true}); + return 1; + } + + // BJData type marker 'B': decode as binary (not a SAX array at all) + if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + { + binary_t result; + return (get_binary(input_format, size_and_type.first, result) && sax->binary(result)) ? 2 : 0; + } + + if (size_and_type.first != npos) + { + // a homogeneous type of 'N' (no-op) means no elements are + // actually read, no matter what count was declared + const bool empty = size_and_type.second == 'N' || size_and_type.first == 0; + if (!empty && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + { + return 0; + } + if (empty) + { + return sax->end_array() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, false, size_and_type.first, false, size_and_type.second, false}); + return 1; + } + + // indefinite length + if (current != ']' && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) + { + return 0; + } + if (current == ']') + { + return sax->end_array() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, true, 0, false, 0, false}); + return 1; + }; + + auto enter_object = [&]() -> int { - return false; - } + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return 0; + } + + // do not accept ND-array size in objects in BJData + if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + { + auto last_token = get_token_string(); + sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); + return 0; + } + + if (size_and_type.first != npos) + { + if (size_and_type.first != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + { + return 0; + } + if (size_and_type.first == 0) + { + return sax->end_object() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{true, false, size_and_type.first, true, size_and_type.second, false}); + return 1; + } + + if (current != '}' && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + { + return 0; + } + if (current == '}') + { + return sax->end_object() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{true, true, 0, true, 0, false}); + return 1; + }; + +#define JSON_UBJSON_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_UBJSON_ENTER(fn) \ + switch (fn()) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ + } string_t key; - for (std::size_t i = 0; i < len; ++i) + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!stack.empty()) { - return false; + ubjson_container_frame& top = stack.back(); + + if (top.is_object && top.awaiting_key) + { + const bool finished = top.indefinite ? (current == '}') : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, !top.indefinite) || !sax->key(key))) + { + return false; + } + top.awaiting_key = false; + } + else if (!top.is_object) + { + const bool finished = top.indefinite ? (current == ']') : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + const bool extra_close = top.extra_close_object; + stack.pop_back(); + if (extra_close && JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + if (produce()) + { + return true; + } + continue; + } + } + // else: top is an object frame with !awaiting_key, i.e. a + // key was just read above in this same iteration; fall + // through to read that key's value below. } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + char_int_type prefix; + if (!stack.empty()) { - return false; + const ubjson_container_frame& top = stack.back(); + if (top.fixed_type != 0) + { + prefix = top.fixed_type; + } + else if (!top.is_object && top.indefinite) + { + prefix = current; + } + else + { + prefix = get_ignore_noop(); + } + } + else + { + prefix = fetch_char ? get_ignore_noop() : current; } - key.clear(); - } - return sax->end_object(); - } + switch (prefix) + { + case char_traits::eof(): // EOF + return unexpect_eof(input_format, "value"); - //////////// - // UBJSON // - //////////// + case 'T': // true + JSON_UBJSON_VALUE(sax->boolean(true)); + case 'F': // false + JSON_UBJSON_VALUE(sax->boolean(false)); + + case 'Z': // null + JSON_UBJSON_VALUE(sax->null()); + + case 'B': // byte + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'U': + { + std::uint8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'i': + { + std::int8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'I': + { + std::int16_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'l': + { + std::int32_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'L': + { + std::int64_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'h': + { + if (input_format != input_format_t::bjdata) + { + break; + } + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // Code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte2 << 8u) + byte1); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + JSON_UBJSON_VALUE(sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), "")); + } + + case 'd': + { + float number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_float(static_cast(number), "")); + } + + case 'D': + { + double number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_float(static_cast(number), "")); + } + + case 'H': + { + JSON_UBJSON_VALUE(get_ubjson_high_precision_number()); + } + + case 'C': // char + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + } + string_t s(1, static_cast(current)); + JSON_UBJSON_VALUE(sax->string(s)); + } + + case 'S': // string + { + string_t s; + JSON_UBJSON_VALUE(get_ubjson_string(s) && sax->string(s)); + } + + case '[': // array + JSON_UBJSON_ENTER(enter_array); + + case '{': // object + JSON_UBJSON_ENTER(enter_object); - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead + default: // anything else + break; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); + } - @return whether a valid UBJSON value was passed to the SAX parser - */ - bool parse_ubjson_internal(const bool get_char = true) - { - return get_ubjson_value(get_char ? get_ignore_noop() : current); +#undef JSON_UBJSON_VALUE +#undef JSON_UBJSON_ENTER } /*! @@ -2197,574 +2919,182 @@ class binary_reader return false; } result = conditional_static_cast(number); - return true; - } - - case 'M': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint64_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) - { - return false; - } - if (!value_in_range_of(number)) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); - } - result = detail::conditional_static_cast(number); - return true; - } - - case '[': - { - if (input_format != input_format_t::bjdata) - { - break; - } - if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); - } - std::vector dim; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) - { - return false; - } - if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector - { - result = dim.at(dim.size() - 1); - return true; - } - if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format - { - for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container - { - if ( i == 0 ) - { - result = 0; - return true; - } - } - - string_t key = "_ArraySize_"; - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) - { - return false; - } - result = 1; - for (auto i : dim) - { - // Pre-multiplication overflow check: if i > 0 and result > SIZE_MAX/i, then result*i would overflow. - // This check must happen before multiplication since overflow detection after the fact is unreliable - // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. - if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); - } - result *= i; - // Additional post-multiplication check to catch any edge cases the pre-check might miss - if (result == 0 || result == npos) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); - } - if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) - { - return false; - } - } - is_ndarray = true; - return sax->end_array(); - } - result = 0; - return true; - } - - default: - break; - } - auto last_token = get_token_string(); - std::string message; - - if (input_format != input_format_t::bjdata) - { - message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; - } - else - { - message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; - } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); - } - - /*! - @brief determine the type and size for a container - - In the optimized UBJSON format, a type and a size can be provided to allow - for a more compact representation. - - @param[out] result pair of the size and the type - @param[in] inside_ndarray whether the parser is parsing an ND array dimensional vector - - @return whether pair creation completed - */ - bool get_ubjson_size_type(std::pair& result, bool inside_ndarray = false) - { - result.first = npos; // size - result.second = 0; // type - bool is_ndarray = false; - - get_ignore_noop(); - - if (current == '$') - { - result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); - } - - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) - { - return false; - } - - get_ignore_noop(); - if (JSON_HEDLEY_UNLIKELY(current != '#')) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) - { - return false; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); - } - - const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) - { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } - result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters - } - return is_error; - } - - if (current == '#') - { - const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); - } - return is_error; - } - - return true; - } - - /*! - @param prefix the previously read or set type prefix - @return whether value creation completed - */ - bool get_ubjson_value(const char_int_type prefix) - { - switch (prefix) - { - case char_traits::eof(): // EOF - return unexpect_eof(input_format, "value"); - - case 'T': // true - return sax->boolean(true); - case 'F': // false - return sax->boolean(false); - - case 'Z': // null - return sax->null(); - - case 'B': // byte - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'U': - { - std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'i': - { - std::int8_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'I': - { - std::int16_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'l': - { - std::int32_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'L': - { - std::int64_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'u': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint16_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'm': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint32_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'M': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint64_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); + return true; } - case 'h': + case 'M': { if (input_format != input_format_t::bjdata) { break; } - const auto byte1_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + std::uint64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } - const auto byte2_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + if (!value_in_range_of(number)) { - return false; + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } - - const auto byte1 = static_cast(byte1_raw); - const auto byte2 = static_cast(byte2_raw); - - // Code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const auto half = static_cast((byte2 << 8u) + byte1); - const double val = [&half] - { - const int exp = (half >> 10u) & 0x1Fu; - const unsigned int mant = half & 0x3FFu; - JSON_ASSERT(0 <= exp&& exp <= 32); - JSON_ASSERT(mant <= 1024); - switch (exp) - { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); - } - }(); - return sax->number_float((half & 0x8000u) != 0 - ? static_cast(-val) - : static_cast(val), ""); - } - - case 'd': - { - float number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); - } - - case 'D': - { - double number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); - } - - case 'H': - { - return get_ubjson_high_precision_number(); + result = detail::conditional_static_cast(number); + return true; } - case 'C': // char + case '[': { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) + if (input_format != input_format_t::bjdata) { - return false; + break; } - if (JSON_HEDLEY_UNLIKELY(current > 127)) + if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); } - string_t s(1, static_cast(current)); - return sax->string(s); - } - - case 'S': // string - { - string_t s; - return get_ubjson_string(s) && sax->string(s); - } - - case '[': // array - return get_ubjson_array(); - - case '{': // object - return get_ubjson_object(); - - default: // anything else - break; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); - } - - /*! - @return whether array creation completed - */ - bool get_ubjson_array() - { - std::pair size_and_type; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) - { - return false; - } - - // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): - // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) - { - size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker - auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t) - { - return p.first < t; - }); - string_t key = "_ArrayType_"; - if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); - } - - string_t type = it->second; // sax->string() takes a reference - if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type))) - { - return false; - } - - if (size_and_type.second == 'C' || size_and_type.second == 'B') - { - size_and_type.second = 'U'; - } - - key = "_ArrayData_"; - if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) )) - { - return false; - } - - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + std::vector dim; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) { return false; } - } - - return (sax->end_array() && sax->end_object()); - } - - // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') - { - binary_t result; - return get_binary(input_format, size_and_type.first, result) && sax->binary(result); - } - - if (size_and_type.first != npos) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - if (size_and_type.second != 'N') + if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector { - for (std::size_t i = 0; i < size_and_type.first; ++i) + result = dim.at(dim.size() - 1); + return true; + } + if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format + { + for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + if ( i == 0 ) { - return false; + result = 0; + return true; } } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + + string_t key = "_ArraySize_"; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) { return false; } + result = 1; + for (auto i : dim) + { + // Pre-multiplication overflow check: if i > 0 and result > SIZE_MAX/i, then result*i would overflow. + // This check must happen before multiplication since overflow detection after the fact is unreliable + // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. + if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + } + result *= i; + // Additional post-multiplication check to catch any edge cases the pre-check might miss + if (result == 0 || result == npos) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) + { + return false; + } + } + is_ndarray = true; + return sax->end_array(); } + result = 0; + return true; } + + default: + break; + } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; } else { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; } - - return sax->end_array(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! - @return whether object creation completed + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + @param[in] inside_ndarray whether the parser is parsing an ND array dimensional vector + + @return whether pair creation completed */ - bool get_ubjson_object() + bool get_ubjson_size_type(std::pair& result, bool inside_ndarray = false) { - std::pair size_and_type; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) - { - return false; - } + result.first = npos; // size + result.second = 0; // type + bool is_ndarray = false; - // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); - } + get_ignore_noop(); - string_t key; - if (size_and_type.first != npos) + if (current == '$') { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (input_format == input_format_t::bjdata + && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) { return false; } - if (size_and_type.second != 0) + get_ignore_noop(); + if (JSON_HEDLEY_UNLIKELY(current != '#')) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); + return false; } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } - else + + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); + if (input_format == input_format_t::bjdata && is_ndarray) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (inside_ndarray) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); } + result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } + return is_error; } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - while (current != '}') + if (current == '#') + { + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); + if (input_format == input_format_t::bjdata && is_ndarray) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); } + return is_error; } - return sax->end_object(); + return true; } // Note, no reader for UBJSON binary types is implemented because they do @@ -3121,9 +3451,47 @@ class binary_reader return concat(error_msg, ' ', context, ": ", detail); } + /*! + @brief report a max_depth parse_error(116) for the given format + + Shared by every container entry point in CBOR/MessagePack/UBJSON/BSON so + there is exactly one place that can forget to guard a new one (see the + BSON 0x04/array gap this PR closes, which happened precisely because the + check was duplicated instead of shared). + + @param[in] format the current format + @return whether parsing should continue (always false: this always + reports an error) + */ + bool reached_max_depth(const input_format_t format) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(116, chars_read, + exception_message(format, "maximum depth of nested arrays/objects exceeded", "value"), nullptr)); + } + private: static JSON_INLINE_VARIABLE constexpr std::size_t npos = detail::unknown_size(); + /// maximum allowed nesting depth of arrays/objects for the binary input + /// formats. + /// + /// The container-parsing loops in this file are iterative (see e.g. + /// @ref cbor_container_frame): nesting depth is tracked as the size of a + /// heap-allocated std::vector, not native call-stack recursion, so + /// *parsing* can no longer overflow the stack regardless of this value. + /// However, operations on the resulting value - @ref + /// basic_json(const basic_json&) "the copy constructor", @ref dump(), + /// to_cbor()/to_msgpack()/to_ubjson()/to_bson(), and @ref operator== in + /// particular - are still recursive, and a value nested deeper than the + /// low thousands can overflow the native stack in *those* regardless of + /// how it was produced. This limit therefore stays low enough that a + /// successfully parsed value remains safe to copy/dump/compare even on a + /// constrained (1 MiB) stack, with generous headroom over the library's + /// own deepest legitimate test fixture + /// (tests/data/json_testsuite/sample.json, 458 levels). + static JSON_INLINE_VARIABLE constexpr std::size_t max_depth = 4096; + /// input adapter InputAdapterType ia; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index a460bb29ff..d2ee375f6d 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4461,7 +4461,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4478,7 +4478,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -4504,7 +4504,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in MessagePack format @@ -4519,7 +4519,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4535,7 +4535,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -4559,7 +4559,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in UBJSON format @@ -4574,7 +4574,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4590,7 +4590,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -4614,7 +4614,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BJData format @@ -4629,7 +4629,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4645,7 +4645,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BSON format @@ -4660,7 +4660,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4676,7 +4676,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -4700,7 +4700,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @} diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4859352122..22b973c5d4 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10712,6 +10712,29 @@ class binary_reader // BSON // ////////// + /*! + @brief a single pending "resume this document/array" entry used by the + iterative container traversal in @ref parse_bson_internal; see + @ref cbor_container_frame for the general idea. + + BSON documents and arrays share one on-the-wire layout (a size-prefixed, + 0x00-terminated list of typed elements); @ref is_object only controls + whether each element's key is relayed to the SAX consumer via + @c sax->key() (an object) or not (an array, whose "0", "1", ... keys the + SAX array interface has no use for). Every BSON container is + terminator-based (there is no known-element-count form), so unlike CBOR/ + MessagePack/UBJSON this frame needs no length/indefinite bookkeeping at + all. @ref document_start / @ref document_size are recorded so @ref + check_bson_document_size can validate this frame's declared size once its + terminator is reached. + */ + struct bson_container_frame + { + bool is_object; ///< false: array, true: object/document + std::size_t document_start; ///< chars_read before this document/array's size prefix + std::int32_t document_size; ///< this document/array's declared size + }; + /*! @brief Validate a BSON document's declared size against the bytes read. @@ -10744,26 +10767,192 @@ class binary_reader */ bool parse_bson_internal() { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - get_number(input_format_t::bson, document_size); + std::vector stack; - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + // reads the 4-byte document-size prefix and records where it + // started (both are needed by @ref check_bson_document_size once + // this document/array's terminator is reached); deliberately does + // not touch the SAX consumer, so callers can run the max_depth + // check before the corresponding start_object()/start_array() + // event fires + auto read_document_header = [&](std::size_t& document_start, std::int32_t& document_size) { - return false; - } + document_start = chars_read; + get_number(input_format_t::bson, document_size); + }; - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + std::size_t document_start{}; + std::int32_t document_size{}; + read_document_header(document_start, document_size); + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) { return false; } + stack.push_back(bson_container_frame{true, document_start, document_size}); - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) + string_t key; + while (true) { - return false; - } + const std::size_t top_index = stack.size() - 1; + + const auto element_type = get(); + if (element_type == 0) + { + // 0x00 terminates the current document/array + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(stack[top_index].document_start, stack[top_index].document_size))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!(stack[top_index].is_object ? sax->end_object() : sax->end_array()))) + { + return false; + } + stack.pop_back(); + if (stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + if (stack[top_index].is_object && JSON_HEDLEY_UNLIKELY(!sax->key(key))) + { + return false; + } + + switch (element_type) + { + case 0x01: // double + { + double number{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), "")))) + { + return false; + } + continue; + } + + case 0x02: // string + { + std::int32_t len{}; + string_t value; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value)))) + { + return false; + } + continue; + } + + case 0x03: // object + { + std::size_t nested_document_start{}; + std::int32_t nested_document_size{}; + read_document_header(nested_document_start, nested_document_size); + if (JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + return reached_max_depth(input_format_t::bson); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + { + return false; + } + stack.push_back(bson_container_frame{true, nested_document_start, nested_document_size}); + continue; + } + + case 0x04: // array + { + std::size_t nested_document_start{}; + std::int32_t nested_document_size{}; + read_document_header(nested_document_start, nested_document_size); + if (JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + return reached_max_depth(input_format_t::bson); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) + { + return false; + } + stack.push_back(bson_container_frame{false, nested_document_start, nested_document_size}); + continue; + } + + case 0x05: // binary + { + std::int32_t len{}; + binary_t value; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value)))) + { + return false; + } + continue; + } + + case 0x08: // boolean + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(get() != 0))) + { + return false; + } + continue; + + case 0x0A: // null + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + continue; + + case 0x10: // int32 + { + std::int32_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_integer(value)))) + { + return false; + } + continue; + } + + case 0x12: // int64 + { + std::int64_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_integer(value)))) + { + return false; + } + continue; + } + + case 0x11: // uint64 + { + std::uint64_t value{}; + if (JSON_HEDLEY_UNLIKELY(!(get_number(input_format_t::bson, value) && sax->number_unsigned(value)))) + { + return false; + } + continue; + } - return sax->end_object(); + default: // anything else is not supported (yet) + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + const std::string cr_str{cr.data()}; + return sax->parse_error(element_type_parse_position, cr_str, + parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); + } + } + } } /*! @@ -10842,164 +11031,6 @@ class binary_reader return get_binary(input_format_t::bson, len, result); } - /*! - @brief Read a BSON document element of the given @a element_type. - @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html - @param[in] element_type_parse_position The position in the input stream, - where the `element_type` was read. - @warning Not all BSON element types are supported yet. An unsupported - @a element_type will give rise to a parse_error.114: - Unsupported BSON record type 0x... - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_internal(const char_int_type element_type, - const std::size_t element_type_parse_position) - { - switch (element_type) - { - case 0x01: // double - { - double number{}; - return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); - } - - case 0x02: // string - { - std::int32_t len{}; - string_t value; - return get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value); - } - - case 0x03: // object - { - return parse_bson_internal(); - } - - case 0x04: // array - { - return parse_bson_array(); - } - - case 0x05: // binary - { - std::int32_t len{}; - binary_t value; - return get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value); - } - - case 0x08: // boolean - { - return sax->boolean(get() != 0); - } - - case 0x0A: // null - { - return sax->null(); - } - - case 0x10: // int32 - { - std::int32_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); - } - - case 0x12: // int64 - { - std::int64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_integer(value); - } - - case 0x11: // uint64 - { - std::uint64_t value{}; - return get_number(input_format_t::bson, value) && sax->number_unsigned(value); - } - - default: // anything else is not supported (yet) - { - std::array cr{{}}; - static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - const std::string cr_str{cr.data()}; - return sax->parse_error(element_type_parse_position, cr_str, - parse_error::create(114, element_type_parse_position, concat("Unsupported BSON record type 0x", cr_str), nullptr)); - } - } - } - - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - get_number(input_format_t::bson, document_size); - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } - ////////// // CBOR // ////////// @@ -11031,471 +11062,665 @@ class binary_reader return sax->number_integer(static_cast(-1) - static_cast(number)); } + /*! + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_cbor_internal + + Nested CBOR containers are parsed by pushing one of these onto a + heap-allocated std::vector instead of recursing on the native call stack + for every nesting level (see #5104): parsing a scalar never recurses, + and parsing a container just pushes a frame here and loops back around + to parse its first child; when a container finishes, its frame is + popped and parsing resumes where the parent left off. Native call-stack + depth used while parsing is therefore O(1) regardless of how deeply the + input nests. + */ + struct cbor_container_frame + { + bool is_object; ///< false: array, true: object/map + bool indefinite; ///< true: length is unknown; read until a 0xFF break marker + std::size_t remaining; ///< remaining element count; meaningful only if !indefinite + bool awaiting_key; ///< object only: true if the next thing to read is a key, not a value + }; + bool parse_cbor_internal(const bool get_char, const cbor_tag_handler_t tag_handler) { - switch (get_char ? get() : current) + std::vector stack; + + // whether the next value-read below should fetch a fresh byte via + // get() (true) or reuse the byte already sitting in `current` (false) + bool fetch_char = get_char; + + // called whenever a value (scalar, or a container that was just + // completed/immediately-empty) was produced at the position the + // current top-of-stack frame is waiting on; returns true if that + // means the whole parse is finished (the stack is empty, i.e. this + // was the outermost value), false if parsing should continue. + auto produce = [&]() -> bool { - // EOF - case char_traits::eof(): - return unexpect_eof(input_format_t::cbor, "value"); - - // Integer 0x00..0x17 (0..23) - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - return sax->number_unsigned(static_cast(current)); - - case 0x18: // Unsigned integer (one-byte uint8_t follows) + if (stack.empty()) { - std::uint8_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return true; } - - case 0x19: // Unsigned integer (two-byte uint16_t follows) + cbor_container_frame& top = stack.back(); + if (top.is_object) { - std::uint16_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + top.awaiting_key = true; } + if (!top.indefinite) + { + --top.remaining; + } + return false; + }; - case 0x1A: // Unsigned integer (four-byte uint32_t follows) + // begin parsing a nested array/object of the given length + // (detail::unknown_size() for indefinite length); returns 0 on SAX + // failure or exceeding max_depth (caller should `return false`), 1 + // if a frame was pushed (caller should `continue` the outer loop to + // parse the container's first element), 2 if the container was + // immediately completed because it is empty (caller should treat + // this like any other produced value via `produce()`). + auto enter_container = [&](const bool is_object, const std::size_t len) -> int + { + // len == 0 (definite-length, empty) never pushes a frame, so it + // never counts against max_depth; every other case - definite + // non-zero or indefinite - always does, so the check can run + // before the SAX start event fires + if (len != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format_t::cbor); + return 0; + } + const bool started = is_object ? sax->start_object(len) : sax->start_array(len); + if (JSON_HEDLEY_UNLIKELY(!started)) { - std::uint32_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + return 0; } + if (len == 0) + { + const bool ended = is_object ? sax->end_object() : sax->end_array(); + return ended ? 2 : 0; + } + stack.push_back(cbor_container_frame{is_object, len == detail::unknown_size(), len, is_object}); + return 1; + }; - case 0x1B: // Unsigned integer (eight-byte uint64_t follows) +#define JSON_CBOR_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_CBOR_ENTER(is_obj, length) \ + switch (enter_container(is_obj, length)) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ + } + + string_t key; + while (true) + { + if (!stack.empty()) { - std::uint64_t number{}; - return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); - } - - // Negative integer -1-0x00..-1-0x17 (-1..-24) - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - return sax->number_integer(static_cast(0x20 - 1 - current)); - - case 0x38: // Negative integer (one-byte uint8_t follows) - return get_cbor_negative_integer(); - - case 0x39: // Negative integer -1-n (two-byte uint16_t follows) - return get_cbor_negative_integer(); - - case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) - return get_cbor_negative_integer(); - - case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) - return get_cbor_negative_integer(); + cbor_container_frame& top = stack.back(); - // Binary data (0x00..0x17 bytes follow) - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: // Binary data (one-byte uint8_t for n follows) - case 0x59: // Binary data (two-byte uint16_t for n follow) - case 0x5A: // Binary data (four-byte uint32_t for n follow) - case 0x5B: // Binary data (eight-byte uint64_t for n follow) - case 0x5F: // Binary data (indefinite length) - { - binary_t b; - return get_cbor_binary(b) && sax->binary(b); - } + if (top.is_object && top.awaiting_key) + { + const bool finished = top.indefinite ? (get() == 0xFF) : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) - case 0x7F: // UTF-8 string (indefinite length) - { - string_t s; - return get_cbor_string(s) && sax->string(s); - } - - // array (0x00..0x17 data items follow) - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); - - case 0x98: // array (one-byte uint8_t for n follows) - { - std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + if (!top.indefinite) + { + get(); + } + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + top.awaiting_key = false; + fetch_char = true; + } + else if (!top.is_object) + { + const bool finished = top.indefinite ? (get() == 0xFF) : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + fetch_char = !top.indefinite; + } + // else: top is an object frame with !awaiting_key, i.e. a + // key was just read above in this same iteration (no branch + // above sets awaiting_key back to false except that one); + // fetch_char is already true from that path, so just fall + // through to read the value. } - case 0x99: // array (two-byte uint16_t for n follow) +read_value: + switch (fetch_char ? get() : current) { - std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); - } + // EOF + case char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); - case 0x9A: // array (four-byte uint32_t for n follow) - { - std::uint32_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast(len), tag_handler); - } + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + JSON_CBOR_VALUE(sax->number_unsigned(static_cast(current))); - case 0x9B: // array (eight-byte uint64_t for n follow) - { - std::uint64_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(conditional_static_cast(len), tag_handler); - } + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - // map (0x00..0x17 pairs of data items follow) - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xB8: // map (one-byte uint8_t for n follows) - { - std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); - } + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_unsigned(number)); + } - case 0xB9: // map (two-byte uint16_t for n follow) - { - std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); - } + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + JSON_CBOR_VALUE(sax->number_integer(static_cast(0x20 - 1 - current))); - case 0xBA: // map (four-byte uint32_t for n follow) - { - std::uint32_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast(len), tag_handler); - } + case 0x38: // Negative integer (one-byte uint8_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); - case 0xBB: // map (eight-byte uint64_t for n follow) - { - std::uint64_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(conditional_static_cast(len), tag_handler); - } - - case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); - - case 0xC6: // tagged item - case 0xC7: - case 0xC8: - case 0xC9: - case 0xCA: - case 0xCB: - case 0xCC: - case 0xCD: - case 0xCE: - case 0xCF: - case 0xD0: - case 0xD1: - case 0xD2: - case 0xD3: - case 0xD4: - case 0xD8: // tagged item (1 byte follows) - case 0xD9: // tagged item (2 bytes follow) - case 0xDA: // tagged item (4 bytes follow) - case 0xDB: // tagged item (8 bytes follow) - { - switch (tag_handler) - { - case cbor_tag_handler_t::error: + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + JSON_CBOR_VALUE(get_cbor_negative_integer()); + + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: // Binary data (one-byte uint8_t for n follows) + case 0x59: // Binary data (two-byte uint16_t for n follow) + case 0x5A: // Binary data (four-byte uint32_t for n follow) + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + case 0x5F: // Binary data (indefinite length) + { + binary_t b; + JSON_CBOR_VALUE(get_cbor_binary(b) && sax->binary(b)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + JSON_CBOR_VALUE(get_cbor_string(s) && sax->string(s)); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + JSON_CBOR_ENTER(false, conditional_static_cast(static_cast(current) & 0x1Fu)); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + return false; } + JSON_CBOR_ENTER(false, static_cast(len)); + } - case cbor_tag_handler_t::ignore: + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - // ignore binary subtype - switch (current) - { - case 0xD8: - { - std::uint8_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xD9: - { - std::uint16_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xDA: - { - std::uint32_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - case 0xDB: - { - std::uint64_t subtype_to_ignore{}; - get_number(input_format_t::cbor, subtype_to_ignore); - break; - } - default: - break; - } - return parse_cbor_internal(true, tag_handler); + return false; + } + JSON_CBOR_ENTER(false, static_cast(len)); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; } + JSON_CBOR_ENTER(false, conditional_static_cast(len)); + } - case cbor_tag_handler_t::store: + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) { - binary_t b; - // use binary subtype and store in a binary container - switch (current) + return false; + } + JSON_CBOR_ENTER(false, conditional_static_cast(len)); + } + + case 0x9F: // array (indefinite length) + JSON_CBOR_ENTER(false, detail::unknown_size()); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + JSON_CBOR_ENTER(true, conditional_static_cast(static_cast(current) & 0x1Fu)); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, static_cast(len)); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, static_cast(len)); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, conditional_static_cast(len)); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::cbor, len))) + { + return false; + } + JSON_CBOR_ENTER(true, conditional_static_cast(len)); + } + + case 0xBF: // map (indefinite length) + JSON_CBOR_ENTER(true, detail::unknown_size()); + + case 0xC6: // tagged item + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD8: // tagged item (1 byte follows) + case 0xD9: // tagged item (2 bytes follow) + case 0xDA: // tagged item (4 bytes follow) + case 0xDB: // tagged item (8 bytes follow) + { + switch (tag_handler) + { + case cbor_tag_handler_t::error: { - case 0xD8: - { - std::uint8_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; - } - case 0xD9: - { - std::uint16_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; - } - case 0xDA: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } + + case cbor_tag_handler_t::ignore: + { + // ignore binary subtype + switch (current) { - std::uint32_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; + case 0xD8: + { + std::uint8_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xD9: + { + std::uint16_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDA: + { + std::uint32_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDB: + { + std::uint64_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + default: + break; } - case 0xDB: + fetch_char = true; + goto read_value; + } + + case cbor_tag_handler_t::store: + { + binary_t b; + // use binary subtype and store in a binary container + switch (current) { - std::uint64_t subtype{}; - get_number(input_format_t::cbor, subtype); - b.set_subtype(detail::conditional_static_cast(subtype)); - break; + case 0xD8: + { + std::uint8_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xD9: + { + std::uint16_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDA: + { + std::uint32_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDB: + { + std::uint64_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + default: + fetch_char = true; + goto read_value; } - default: - return parse_cbor_internal(true, tag_handler); + get(); + JSON_CBOR_VALUE(get_cbor_binary(b) && sax->binary(b)); } - get(); - return get_cbor_binary(b) && sax->binary(b); - } - default: // LCOV_EXCL_LINE - JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE - return false; // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } } - } - case 0xF4: // false - return sax->boolean(false); + case 0xF4: // false + JSON_CBOR_VALUE(sax->boolean(false)); + + case 0xF5: // true + JSON_CBOR_VALUE(sax->boolean(true)); - case 0xF5: // true - return sax->boolean(true); + case 0xF6: // null + JSON_CBOR_VALUE(sax->null()); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } - case 0xF6: // null - return sax->null(); + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // Code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + JSON_CBOR_VALUE(sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), "")); + } - case 0xF9: // Half-Precision Float (two-byte IEEE 754) - { - const auto byte1_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + case 0xFA: // Single-Precision Float (four-byte IEEE 754) { - return false; + float number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), "")); } - const auto byte2_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { - return false; + double number{}; + JSON_CBOR_VALUE(get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), "")); } - const auto byte1 = static_cast(byte1_raw); - const auto byte2 = static_cast(byte2_raw); - - // Code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const auto half = static_cast((byte1 << 8u) + byte2); - const double val = [&half] - { - const int exp = (half >> 10u) & 0x1Fu; - const unsigned int mant = half & 0x3FFu; - JSON_ASSERT(0 <= exp&& exp <= 32); - JSON_ASSERT(mant <= 1024); - switch (exp) - { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); - } - }(); - return sax->number_float((half & 0x8000u) != 0 - ? static_cast(-val) - : static_cast(val), ""); - } - - case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); - } - - case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number{}; - return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); - } - - default: // anything else (0xFF is handled inside the other types) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::cbor, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } } } + +#undef JSON_CBOR_VALUE +#undef JSON_CBOR_ENTER } /*! @@ -11682,487 +11907,533 @@ class binary_reader } result.insert(result.end(), chunk.begin(), chunk.end()); } - return true; - } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); + } + } + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_msgpack_internal; + see @ref cbor_container_frame for the general idea. MessagePack + has no indefinite-length containers, so unlike CBOR this frame + does not need an "indefinite" flag. + */ + struct msgpack_container_frame + { + bool is_object; ///< false: array, true: map + std::size_t remaining; ///< remaining element count + bool awaiting_key; ///< map only: true if the next thing to read is a key, not a value + }; + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + std::vector stack; + + auto produce = [&]() -> bool + { + if (stack.empty()) + { + return true; + } + msgpack_container_frame& top = stack.back(); + if (top.is_object) + { + top.awaiting_key = true; + } + --top.remaining; + return false; + }; + + auto enter_container = [&](const bool is_object, const std::size_t len) -> int + { + // MessagePack containers are always definite-length, so len == 0 + // is the only case that never pushes a frame / never counts + // against max_depth + if (len != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format_t::msgpack); + return 0; + } + const bool started = is_object ? sax->start_object(len) : sax->start_array(len); + if (JSON_HEDLEY_UNLIKELY(!started)) + { + return 0; + } + if (len == 0) + { + const bool ended = is_object ? sax->end_object() : sax->end_array(); + return ended ? 2 : 0; + } + stack.push_back(msgpack_container_frame{is_object, len, is_object}); + return 1; + }; + +#define JSON_MSGPACK_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_MSGPACK_ENTER(is_obj, length) \ + switch (enter_container(is_obj, length)) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ + } + + string_t key; + while (true) + { + if (!stack.empty()) + { + msgpack_container_frame& top = stack.back(); + + if (top.is_object && top.awaiting_key) + { + if (top.remaining == 0) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + get(); + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + top.awaiting_key = false; + } + else if (!top.is_object) + { + if (top.remaining == 0) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + } + // else: top is a map frame with !awaiting_key, i.e. a key + // was just read above in this same iteration; fall through + // to read that key's value. + } + + switch (get()) + { + // EOF + case char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + JSON_MSGPACK_VALUE(sax->number_unsigned(static_cast(current))); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + JSON_MSGPACK_ENTER(true, conditional_static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + JSON_MSGPACK_ENTER(false, conditional_static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + JSON_MSGPACK_VALUE(get_msgpack_string(s) && sax->string(s)); + } + + case 0xC0: // nil + JSON_MSGPACK_VALUE(sax->null()); + + case 0xC2: // false + JSON_MSGPACK_VALUE(sax->boolean(false)); + + case 0xC3: // true + JSON_MSGPACK_VALUE(sax->boolean(true)); + + case 0xC4: // bin 8 + case 0xC5: // bin 16 + case 0xC6: // bin 32 + case 0xC7: // ext 8 + case 0xC8: // ext 16 + case 0xC9: // ext 32 + case 0xD4: // fixext 1 + case 0xD5: // fixext 2 + case 0xD6: // fixext 4 + case 0xD7: // fixext 8 + case 0xD8: // fixext 16 + { + binary_t b; + JSON_MSGPACK_VALUE(get_msgpack_binary(b) && sax->binary(b)); + } + + case 0xCA: // float 32 + { + float number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), "")); + } + + case 0xCB: // float 64 + { + double number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), "")); + } + + case 0xCC: // uint 8 + { + std::uint8_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } + + case 0xCD: // uint 16 + { + std::uint16_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::cbor, concat("expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x", last_token), "binary"), nullptr)); - } - } - } + case 0xCE: // uint 32 + { + std::uint32_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed - */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + case 0xCF: // uint 64 + { + std::uint64_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_unsigned(number)); + } - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + case 0xD0: // int 8 { - return false; + std::int8_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + + case 0xD1: // int 16 { - return false; + std::int16_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); } - } - } - return sax->end_array(); - } + case 0xD2: // int 32 + { + std::int32_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } + case 0xD3: // int 64 + { + std::int64_t number{}; + JSON_MSGPACK_VALUE(get_number(input_format_t::msgpack, number) && sax->number_integer(number)); + } - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + case 0xDC: // array 16 { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) { return false; } + JSON_MSGPACK_ENTER(false, static_cast(len)); + } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + case 0xDD: // array 32 + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) { return false; } - key.clear(); + JSON_MSGPACK_ENTER(false, conditional_static_cast(len)); } - } - else - { - while (get() != 0xFF) + + case 0xDE: // map 16 { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + std::uint16_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) { return false; } + JSON_MSGPACK_ENTER(true, static_cast(len)); + } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + case 0xDF: // map 32 + { + std::uint32_t len{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::msgpack, len))) { return false; } - key.clear(); + JSON_MSGPACK_ENTER(true, conditional_static_cast(len)); } - } - } - - return sax->end_object(); - } - - ///////////// - // MsgPack // - ///////////// - - /*! - @return whether a valid MessagePack value was passed to the SAX parser - */ - bool parse_msgpack_internal() - { - switch (get()) - { - // EOF - case char_traits::eof(): - return unexpect_eof(input_format_t::msgpack, "value"); - - // positive fixint - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: - case 0x59: - case 0x5A: - case 0x5B: - case 0x5C: - case 0x5D: - case 0x5E: - case 0x5F: - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: - case 0x79: - case 0x7A: - case 0x7B: - case 0x7C: - case 0x7D: - case 0x7E: - case 0x7F: - return sax->number_unsigned(static_cast(current)); - - // fixmap - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - return get_msgpack_object(conditional_static_cast(static_cast(current) & 0x0Fu)); - - // fixarray - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - case 0x98: - case 0x99: - case 0x9A: - case 0x9B: - case 0x9C: - case 0x9D: - case 0x9E: - case 0x9F: - return get_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); - - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - case 0xD9: // str 8 - case 0xDA: // str 16 - case 0xDB: // str 32 - { - string_t s; - return get_msgpack_string(s) && sax->string(s); - } - - case 0xC0: // nil - return sax->null(); - - case 0xC2: // false - return sax->boolean(false); - - case 0xC3: // true - return sax->boolean(true); - - case 0xC4: // bin 8 - case 0xC5: // bin 16 - case 0xC6: // bin 32 - case 0xC7: // ext 8 - case 0xC8: // ext 16 - case 0xC9: // ext 32 - case 0xD4: // fixext 1 - case 0xD5: // fixext 2 - case 0xD6: // fixext 4 - case 0xD7: // fixext 8 - case 0xD8: // fixext 16 - { - binary_t b; - return get_msgpack_binary(b) && sax->binary(b); - } - - case 0xCA: // float 32 - { - float number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); - } - - case 0xCB: // float 64 - { - double number{}; - return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); - } - - case 0xCC: // uint 8 - { - std::uint8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } - - case 0xCD: // uint 16 - { - std::uint16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } - - case 0xCE: // uint 32 - { - std::uint32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } - - case 0xCF: // uint 64 - { - std::uint64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); - } - - case 0xD0: // int 8 - { - std::int8_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } - - case 0xD1: // int 16 - { - std::int16_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } - - case 0xD2: // int 32 - { - std::int32_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } - - case 0xD3: // int 64 - { - std::int64_t number{}; - return get_number(input_format_t::msgpack, number) && sax->number_integer(number); - } - - case 0xDC: // array 16 - { - std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); - } - - case 0xDD: // array 32 - { - std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_array(conditional_static_cast(len)); - } - - case 0xDE: // map 16 - { - std::uint16_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); - } - case 0xDF: // map 32 - { - std::uint32_t len{}; - return get_number(input_format_t::msgpack, len) && get_msgpack_object(conditional_static_cast(len)); - } - - // negative fixint - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - case 0xF0: - case 0xF1: - case 0xF2: - case 0xF3: - case 0xF4: - case 0xF5: - case 0xF6: - case 0xF7: - case 0xF8: - case 0xF9: - case 0xFA: - case 0xFB: - case 0xFC: - case 0xFD: - case 0xFE: - case 0xFF: - return sax->number_integer(static_cast(current)); - - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + JSON_MSGPACK_VALUE(sax->number_integer(static_cast(current))); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format_t::msgpack, concat("invalid byte: 0x", last_token), "value"), nullptr)); + } } } + +#undef JSON_MSGPACK_VALUE +#undef JSON_MSGPACK_ENTER } /*! @@ -12365,72 +12636,523 @@ class binary_reader } } + //////////// + // UBJSON // + //////////// + + /*! + @brief a single pending "resume this array/object" entry used by the + iterative container traversal in @ref parse_ubjson_internal; see + @ref cbor_container_frame for the general idea. + + UBJSON/BJData needs two extra bits of state beyond CBOR/MessagePack: + @ref fixed_type captures the optimized `$type#count` container's + homogeneous element type (0 means "heterogeneous": read a fresh type + marker per element, like an ordinary UBJSON array/object); @ref + extra_close_object is set for the synthetic JData-annotated-array object + ("_ArrayType_"/"_ArraySize_"/"_ArrayData_") BJData wraps an ND-array in, + where finishing the `_ArrayData_` array must also close that wrapping + object. + */ + struct ubjson_container_frame + { + bool is_object; ///< false: array, true: object + bool indefinite; ///< true: no known count; read until a ']'/'}' terminator + std::size_t remaining; ///< remaining element count; meaningful only if !indefinite + bool awaiting_key; ///< object only: true if the next thing to read is a key, not a value + char_int_type fixed_type; ///< 0: heterogeneous; else every element has this known type marker + bool extra_close_object; ///< true: end_array() for this frame must be followed by an end_object() + }; + /*! - @param[in] len the length of the array - @return whether array creation completed + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser */ - bool get_msgpack_array(const std::size_t len) + bool parse_ubjson_internal(const bool get_char = true) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + std::vector stack; + const bool fetch_char = get_char; + + auto produce = [&]() -> bool { + if (stack.empty()) + { + return true; + } + ubjson_container_frame& top = stack.back(); + if (top.is_object) + { + top.awaiting_key = true; + } + if (!top.indefinite) + { + --top.remaining; + } + else + { + // indefinite-length containers are terminated by inspecting + // `current`, so advance past the just-produced element/pair + // to whatever byte should be inspected next (mirrors the + // `get_ignore_noop()` at the end of each iteration of the + // original recursive implementation's while-loops) + get_ignore_noop(); + } return false; - } + }; - for (std::size_t i = 0; i < len; ++i) + // returns 0 = failure (caller returns false), 1 = a frame was + // pushed (caller continues the outer loop to parse the first + // element), 2 = a value was produced immediately - either because + // the container turned out to be empty, or (BJData only) because + // the "optimized binary array" special case produced a single + // binary value directly, without ever being a SAX array at all + // (caller should invoke produce() same as for any other value) + auto enter_array = [&]() -> int { - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) { - return false; + return 0; } - } - return sax->end_array(); - } + // ND-array: encode as an object in JData annotated array format + // (https://github.com/NeuroJSON/jdata). The wrapping object was + // already start_object()'d (with the "_ArraySize_" key/array + // already emitted) inside get_ubjson_size_type(), so only + // "_ArrayType_" and "_ArrayData_" remain to be added here. + if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + { + size_and_type.second &= ~(static_cast(1) << 8); + auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t) + { + return p.first < t; + }); + string_t key = "_ArrayType_"; + if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) + { + auto last_token = get_token_string(); + sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); + return 0; + } - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + string_t type = it->second; // sax->string() takes a reference + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type))) + { + return 0; + } + + if (size_and_type.second == 'C' || size_and_type.second == 'B') + { + size_and_type.second = 'U'; + } + + if (size_and_type.first != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + + key = "_ArrayData_"; + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first))) + { + return 0; + } + + if (size_and_type.first == 0) + { + return (sax->end_array() && sax->end_object()) ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, false, size_and_type.first, false, size_and_type.second, true}); + return 1; + } + + // BJData type marker 'B': decode as binary (not a SAX array at all) + if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') + { + binary_t result; + return (get_binary(input_format, size_and_type.first, result) && sax->binary(result)) ? 2 : 0; + } + + if (size_and_type.first != npos) + { + // a homogeneous type of 'N' (no-op) means no elements are + // actually read, no matter what count was declared + const bool empty = size_and_type.second == 'N' || size_and_type.first == 0; + if (!empty && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + { + return 0; + } + if (empty) + { + return sax->end_array() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, false, size_and_type.first, false, size_and_type.second, false}); + return 1; + } + + // indefinite length + if (current != ']' && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) + { + return 0; + } + if (current == ']') + { + return sax->end_array() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{false, true, 0, false, 0, false}); + return 1; + }; + + auto enter_object = [&]() -> int { - return false; - } + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return 0; + } + + // do not accept ND-array size in objects in BJData + if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) + { + auto last_token = get_token_string(); + sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); + return 0; + } + + if (size_and_type.first != npos) + { + if (size_and_type.first != 0 && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + { + return 0; + } + if (size_and_type.first == 0) + { + return sax->end_object() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{true, false, size_and_type.first, true, size_and_type.second, false}); + return 1; + } + + if (current != '}' && JSON_HEDLEY_UNLIKELY(stack.size() >= max_depth)) + { + reached_max_depth(input_format); + return 0; + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + { + return 0; + } + if (current == '}') + { + return sax->end_object() ? 2 : 0; + } + stack.push_back(ubjson_container_frame{true, true, 0, true, 0, false}); + return 1; + }; + +#define JSON_UBJSON_VALUE(expr) \ + if (JSON_HEDLEY_UNLIKELY(!(expr))) { return false; } \ + if (produce()) { return true; } \ + continue + +#define JSON_UBJSON_ENTER(fn) \ + switch (fn()) \ + { \ + case 0: return false; \ + case 2: if (produce()) { return true; } continue; \ + default: continue; \ + } string_t key; - for (std::size_t i = 0; i < len; ++i) + while (true) { - get(); - if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + if (!stack.empty()) { - return false; + ubjson_container_frame& top = stack.back(); + + if (top.is_object && top.awaiting_key) + { + const bool finished = top.indefinite ? (current == '}') : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + stack.pop_back(); + if (produce()) + { + return true; + } + continue; + } + + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, !top.indefinite) || !sax->key(key))) + { + return false; + } + top.awaiting_key = false; + } + else if (!top.is_object) + { + const bool finished = top.indefinite ? (current == ']') : (top.remaining == 0); + if (finished) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + const bool extra_close = top.extra_close_object; + stack.pop_back(); + if (extra_close && JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + if (produce()) + { + return true; + } + continue; + } + } + // else: top is an object frame with !awaiting_key, i.e. a + // key was just read above in this same iteration; fall + // through to read that key's value below. } - if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + char_int_type prefix; + if (!stack.empty()) { - return false; + const ubjson_container_frame& top = stack.back(); + if (top.fixed_type != 0) + { + prefix = top.fixed_type; + } + else if (!top.is_object && top.indefinite) + { + prefix = current; + } + else + { + prefix = get_ignore_noop(); + } + } + else + { + prefix = fetch_char ? get_ignore_noop() : current; } - key.clear(); - } - return sax->end_object(); - } + switch (prefix) + { + case char_traits::eof(): // EOF + return unexpect_eof(input_format, "value"); - //////////// - // UBJSON // - //////////// + case 'T': // true + JSON_UBJSON_VALUE(sax->boolean(true)); + case 'F': // false + JSON_UBJSON_VALUE(sax->boolean(false)); + + case 'Z': // null + JSON_UBJSON_VALUE(sax->null()); + + case 'B': // byte + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'U': + { + std::uint8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'i': + { + std::int8_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'I': + { + std::int16_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'l': + { + std::int32_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'L': + { + std::int64_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_integer(number)); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_unsigned(number)); + } + + case 'h': + { + if (input_format != input_format_t::bjdata) + { + break; + } + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // Code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte2 << 8u) + byte1); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + JSON_UBJSON_VALUE(sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), "")); + } + + case 'd': + { + float number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_float(static_cast(number), "")); + } + + case 'D': + { + double number{}; + JSON_UBJSON_VALUE(get_number(input_format, number) && sax->number_float(static_cast(number), "")); + } + + case 'H': + { + JSON_UBJSON_VALUE(get_ubjson_high_precision_number()); + } + + case 'C': // char + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + } + string_t s(1, static_cast(current)); + JSON_UBJSON_VALUE(sax->string(s)); + } + + case 'S': // string + { + string_t s; + JSON_UBJSON_VALUE(get_ubjson_string(s) && sax->string(s)); + } - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead + case '[': // array + JSON_UBJSON_ENTER(enter_array); - @return whether a valid UBJSON value was passed to the SAX parser - */ - bool parse_ubjson_internal(const bool get_char = true) - { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + case '{': // object + JSON_UBJSON_ENTER(enter_object); + + default: // anything else + break; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); + } + +#undef JSON_UBJSON_VALUE +#undef JSON_UBJSON_ENTER } /*! @@ -12755,565 +13477,173 @@ class binary_reader { break; } - std::uint64_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) - { - return false; - } - if (!value_in_range_of(number)) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, - exception_message(input_format, "integer value overflow", "size"), nullptr)); - } - result = detail::conditional_static_cast(number); - return true; - } - - case '[': - { - if (input_format != input_format_t::bjdata) - { - break; - } - if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); - } - std::vector dim; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) - { - return false; - } - if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector - { - result = dim.at(dim.size() - 1); - return true; - } - if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format - { - for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container - { - if ( i == 0 ) - { - result = 0; - return true; - } - } - - string_t key = "_ArraySize_"; - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) - { - return false; - } - result = 1; - for (auto i : dim) - { - // Pre-multiplication overflow check: if i > 0 and result > SIZE_MAX/i, then result*i would overflow. - // This check must happen before multiplication since overflow detection after the fact is unreliable - // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. - if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); - } - result *= i; - // Additional post-multiplication check to catch any edge cases the pre-check might miss - if (result == 0 || result == npos) - { - return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); - } - if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) - { - return false; - } - } - is_ndarray = true; - return sax->end_array(); - } - result = 0; - return true; - } - - default: - break; - } - auto last_token = get_token_string(); - std::string message; - - if (input_format != input_format_t::bjdata) - { - message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; - } - else - { - message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; - } - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); - } - - /*! - @brief determine the type and size for a container - - In the optimized UBJSON format, a type and a size can be provided to allow - for a more compact representation. - - @param[out] result pair of the size and the type - @param[in] inside_ndarray whether the parser is parsing an ND array dimensional vector - - @return whether pair creation completed - */ - bool get_ubjson_size_type(std::pair& result, bool inside_ndarray = false) - { - result.first = npos; // size - result.second = 0; // type - bool is_ndarray = false; - - get_ignore_noop(); - - if (current == '$') - { - result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (input_format == input_format_t::bjdata - && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); - } - - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) - { - return false; - } - - get_ignore_noop(); - if (JSON_HEDLEY_UNLIKELY(current != '#')) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) - { - return false; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); - } - - const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) - { - if (inside_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); - } - result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters - } - return is_error; - } - - if (current == '#') - { - const bool is_error = get_ubjson_size_value(result.first, is_ndarray); - if (input_format == input_format_t::bjdata && is_ndarray) - { - return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, - exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); - } - return is_error; - } - - return true; - } - - /*! - @param prefix the previously read or set type prefix - @return whether value creation completed - */ - bool get_ubjson_value(const char_int_type prefix) - { - switch (prefix) - { - case char_traits::eof(): // EOF - return unexpect_eof(input_format, "value"); - - case 'T': // true - return sax->boolean(true); - case 'F': // false - return sax->boolean(false); - - case 'Z': // null - return sax->null(); - - case 'B': // byte - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'U': - { - std::uint8_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'i': - { - std::int8_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'I': - { - std::int16_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'l': - { - std::int32_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'L': - { - std::int64_t number{}; - return get_number(input_format, number) && sax->number_integer(number); - } - - case 'u': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint16_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'm': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint32_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'M': - { - if (input_format != input_format_t::bjdata) - { - break; - } - std::uint64_t number{}; - return get_number(input_format, number) && sax->number_unsigned(number); - } - - case 'h': - { - if (input_format != input_format_t::bjdata) - { - break; - } - const auto byte1_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + std::uint64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } - const auto byte2_raw = get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + if (!value_in_range_of(number)) { - return false; + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "integer value overflow", "size"), nullptr)); } - - const auto byte1 = static_cast(byte1_raw); - const auto byte2 = static_cast(byte2_raw); - - // Code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const auto half = static_cast((byte2 << 8u) + byte1); - const double val = [&half] - { - const int exp = (half >> 10u) & 0x1Fu; - const unsigned int mant = half & 0x3FFu; - JSON_ASSERT(0 <= exp&& exp <= 32); - JSON_ASSERT(mant <= 1024); - switch (exp) - { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); - } - }(); - return sax->number_float((half & 0x8000u) != 0 - ? static_cast(-val) - : static_cast(val), ""); - } - - case 'd': - { - float number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); - } - - case 'D': - { - double number{}; - return get_number(input_format, number) && sax->number_float(static_cast(number), ""); - } - - case 'H': - { - return get_ubjson_high_precision_number(); + result = detail::conditional_static_cast(number); + return true; } - case 'C': // char + case '[': { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) + if (input_format != input_format_t::bjdata) { - return false; + break; } - if (JSON_HEDLEY_UNLIKELY(current > 127)) + if (is_ndarray) // ndarray dimensional vector can only contain integers and cannot embed another array { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(113, chars_read, exception_message(input_format, "ndarray dimensional vector is not allowed", "size"), nullptr)); } - string_t s(1, static_cast(current)); - return sax->string(s); - } - - case 'S': // string - { - string_t s; - return get_ubjson_string(s) && sax->string(s); - } - - case '[': // array - return get_ubjson_array(); - - case '{': // object - return get_ubjson_object(); - - default: // anything else - break; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); - } - - /*! - @return whether array creation completed - */ - bool get_ubjson_array() - { - std::pair size_and_type; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) - { - return false; - } - - // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): - // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} - - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) - { - size_and_type.second &= ~(static_cast(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker - auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t) - { - return p.first < t; - }); - string_t key = "_ArrayType_"; - if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second)) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr)); - } - - string_t type = it->second; // sax->string() takes a reference - if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type))) - { - return false; - } - - if (size_and_type.second == 'C' || size_and_type.second == 'B') - { - size_and_type.second = 'U'; - } - - key = "_ArrayData_"; - if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) )) - { - return false; - } - - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + std::vector dim; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) { return false; } - } - - return (sax->end_array() && sax->end_object()); - } - - // If BJData type marker is 'B' decode as binary - if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B') - { - binary_t result; - return get_binary(input_format, size_and_type.first, result) && sax->binary(result); - } - - if (size_and_type.first != npos) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - if (size_and_type.second != 'N') + if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector { - for (std::size_t i = 0; i < size_and_type.first; ++i) + result = dim.at(dim.size() - 1); + return true; + } + if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format + { + for (auto i : dim) // test if any dimension in an ndarray is 0, if so, return a 1D empty container { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + if ( i == 0 ) { - return false; + result = 0; + return true; } } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + + string_t key = "_ArraySize_"; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) { return false; } + result = 1; + for (auto i : dim) + { + // Pre-multiplication overflow check: if i > 0 and result > SIZE_MAX/i, then result*i would overflow. + // This check must happen before multiplication since overflow detection after the fact is unreliable + // as modular arithmetic can produce any value, not just 0 or SIZE_MAX. + if (JSON_HEDLEY_UNLIKELY(i > 0 && result > (std::numeric_limits::max)() / i)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + } + result *= i; + // Additional post-multiplication check to catch any edge cases the pre-check might miss + if (result == 0 || result == npos) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, exception_message(input_format, "excessive ndarray size caused overflow", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(static_cast(i)))) + { + return false; + } + } + is_ndarray = true; + return sax->end_array(); } + result = 0; + return true; } + + default: + break; + } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; } else { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; } - - return sax->end_array(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! - @return whether object creation completed + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + @param[in] inside_ndarray whether the parser is parsing an ND array dimensional vector + + @return whether pair creation completed */ - bool get_ubjson_object() + bool get_ubjson_size_type(std::pair& result, bool inside_ndarray = false) { - std::pair size_and_type; - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) - { - return false; - } + result.first = npos; // size + result.second = 0; // type + bool is_ndarray = false; - // do not accept ND-array size in objects in BJData - if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); - } + get_ignore_noop(); - string_t key; - if (size_and_type.first != npos) + if (current == '$') { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (input_format == input_format_t::bjdata + && JSON_HEDLEY_UNLIKELY(std::binary_search(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second))) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr)); + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type"))) { return false; } - if (size_and_type.second != 0) + get_ignore_noop(); + if (JSON_HEDLEY_UNLIKELY(current != '#')) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); + return false; } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } - else + + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); + if (input_format == input_format_t::bjdata && is_ndarray) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (inside_ndarray) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray can not be recursive", "size"), nullptr)); } + result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters } + return is_error; } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - while (current != '}') + if (current == '#') + { + const bool is_error = get_ubjson_size_value(result.first, is_ndarray); + if (input_format == input_format_t::bjdata && is_ndarray) { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); + return sax->parse_error(chars_read, get_token_string(), parse_error::create(112, chars_read, + exception_message(input_format, "ndarray requires both type and size", "size"), nullptr)); } + return is_error; } - return sax->end_object(); + return true; } // Note, no reader for UBJSON binary types is implemented because they do @@ -13670,9 +14000,47 @@ class binary_reader return concat(error_msg, ' ', context, ": ", detail); } + /*! + @brief report a max_depth parse_error(116) for the given format + + Shared by every container entry point in CBOR/MessagePack/UBJSON/BSON so + there is exactly one place that can forget to guard a new one (see the + BSON 0x04/array gap this PR closes, which happened precisely because the + check was duplicated instead of shared). + + @param[in] format the current format + @return whether parsing should continue (always false: this always + reports an error) + */ + bool reached_max_depth(const input_format_t format) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(116, chars_read, + exception_message(format, "maximum depth of nested arrays/objects exceeded", "value"), nullptr)); + } + private: static JSON_INLINE_VARIABLE constexpr std::size_t npos = detail::unknown_size(); + /// maximum allowed nesting depth of arrays/objects for the binary input + /// formats. + /// + /// The container-parsing loops in this file are iterative (see e.g. + /// @ref cbor_container_frame): nesting depth is tracked as the size of a + /// heap-allocated std::vector, not native call-stack recursion, so + /// *parsing* can no longer overflow the stack regardless of this value. + /// However, operations on the resulting value - @ref + /// basic_json(const basic_json&) "the copy constructor", @ref dump(), + /// to_cbor()/to_msgpack()/to_ubjson()/to_bson(), and @ref operator== in + /// particular - are still recursive, and a value nested deeper than the + /// low thousands can overflow the native stack in *those* regardless of + /// how it was produced. This limit therefore stays low enough that a + /// successfully parsed value remains safe to copy/dump/compare even on a + /// constrained (1 MiB) stack, with generous headroom over the library's + /// own deepest legitimate test fixture + /// (tests/data/json_testsuite/sample.json, 458 levels). + static JSON_INLINE_VARIABLE constexpr std::size_t max_depth = 4096; + /// input adapter InputAdapterType ia; @@ -25591,7 +25959,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25608,7 +25976,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -25634,7 +26002,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in MessagePack format @@ -25649,7 +26017,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25665,7 +26033,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -25689,7 +26057,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in UBJSON format @@ -25704,7 +26072,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25720,7 +26088,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -25744,7 +26112,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BJData format @@ -25759,7 +26127,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25775,7 +26143,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BSON format @@ -25790,7 +26158,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25806,7 +26174,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); detail::json_sax_dom_parser sdp(result, allow_exceptions); const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } template @@ -25830,7 +26198,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved] - return res ? result : basic_json(value_t::discarded); + return res ? std::move(result) : basic_json(value_t::discarded); } /// @} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4383b582c3..0d8b0cda88 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -104,6 +104,23 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") json_test_set_test_options("test-cbor;test-msgpack;test-ubjson;test-bjdata;test-binary_formats" LINK_OPTIONS /STACK:4000000) endif() +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND MINGW) + # test-regression2 is already the largest test TU and exercises CBOR/BSON/ + # MessagePack/UBJSON parsing; GNU ld's COFF backend can exhaust addressable + # range for IMAGE_REL_AMD64_REL32 relocations against .rdata once template + # instantiation grows far enough ("relocation truncated to fit"). lld lays + # out COFF sections differently and doesn't hit this ceiling for the same input. + # + # Guarded rather than unconditional: this was originally added as an + # untested hypothesis for a CI linker error, and hard-requiring lld with + # no fallback would break the build for anyone building the tests with + # Clang+MinGW who doesn't happen to have it installed. + find_program(JSON_TEST_LLD_PROGRAM lld) + if (JSON_TEST_LLD_PROGRAM) + json_test_set_test_options(test-regression2 LINK_OPTIONS -fuse-ld=lld) + endif() +endif() + # disable exceptions for test-disabled_exceptions json_test_set_test_options(test-disabled_exceptions COMPILE_DEFINITIONS diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index a60d0a1641..782895cba1 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3712,6 +3712,38 @@ TEST_CASE("Universal Binary JSON Specification Examples 1") } } +TEST_CASE("BJData regressions") +{ + SECTION("stack overflow via deeply nested input (issue #5104)") + { + // BJData shares UBJSON's container-parsing code path (see the + // equivalent test in unit-ubjson.cpp), but also reaches enter_array + // through paths UBJSON doesn't have at all (the ND-array + // "_ArrayData_" frame, the optimized $type#count container) - a + // dedicated depth test here is cheap insurance against a regression + // that's specific to the bjdata format tag. + // + // 4200 nested arrays - comfortably past max_depth (4096, see + // binary_reader.hpp); the (now iterative, heap-stack based) parser + // must reject this with a clean parse_error once the nesting + // exceeds the depth limit, rather than exhausting the native call + // stack. + const int nesting = 4200; + std::string payload(static_cast(nesting), '['); + payload += "Z"; + for (int i = 0; i < nesting; ++i) + { + payload += ']'; + } + std::vector const v(payload.begin(), payload.end()); + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(v), + "[json.exception.parse_error.116] parse error at byte 4098: syntax error while parsing BJData value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + } +} + TEST_CASE("Parse BJData directly from a file using iterator and sentinel") { std::string const filename = TEST_DATA_DIRECTORY "/json_testsuite/sample.json.bjdata"; diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 87344b9b1e..d581b86025 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -854,6 +854,90 @@ TEST_CASE("Unsupported BSON input") CHECK(!json::sax_parse(bson, &scp, json::input_format_t::bson)); } +TEST_CASE("BSON regressions") +{ + // Shared by both sections below: builds a chain of n embedded BSON + // containers of the given record_type (0x03 object / 0x04 array), each + // wrapping the previous one under a 1-byte key - differing from the + // other section only in those two parameters. + // + // Built directly in O(N) rather than by repeatedly wrapping and copying + // the previous document (which would be O(N^2) and much too slow at + // this depth). Layout, front to back: + // [header_N]...[header_1][innermost 5-byte empty doc][term_1]...[term_N] + // where header_i is a 4-byte little-endian size_i followed by + // {record_type, key_char, 0x00}, and size_i = size_{i-1} + 8, size_0 = 5. + // Terminators are all 0x00, already the buffer's default value. + auto make_deeply_nested_bson = [](int n, std::uint8_t record_type, char key_char) -> std::vector + { + const std::size_t header_size = 7; // 4 (size) + 3 (record_type, key_char, 0x00) + const std::size_t inner_size = 5; + const std::size_t total = (static_cast(n) * header_size) + inner_size + static_cast(n); + std::vector inner(total, 0x00); + + std::vector size_at(static_cast(n) + 1); + size_at[0] = static_cast(inner_size); + for (int i = 1; i <= n; ++i) + { + size_at[static_cast(i)] = size_at[static_cast(i) - 1] + 8; + } + + std::size_t pos = 0; + for (int i = n; i >= 1; --i) + { + const auto sz = static_cast(size_at[static_cast(i)]); + inner[pos + 0] = static_cast(sz & 0xFF); + inner[pos + 1] = static_cast((sz >> 8) & 0xFF); + inner[pos + 2] = static_cast((sz >> 16) & 0xFF); + inner[pos + 3] = static_cast((sz >> 24) & 0xFF); + inner[pos + 4] = record_type; + inner[pos + 5] = static_cast(key_char); + inner[pos + 6] = 0x00; + pos += header_size; + } + // innermost empty document: size=5 (little-endian) + terminator 0x00 + inner[pos + 0] = 5; + inner[pos + 1] = 0; + inner[pos + 2] = 0; + inner[pos + 3] = 0; + inner[pos + 4] = 0x00; + // trailing n terminator bytes are already 0x00 from initialization + return inner; + }; + + // n chosen comfortably past max_depth (4096, see binary_reader.hpp) with + // real margin, while staying far smaller than a stress test needs to be + // - unlike a recursive-descent parser, the iterative parser's memory use + // is the only thing scaling with n, so there's no need to pick a value + // "far beyond any depth a recursive parser could survive". + const int n = 4200; + + SECTION("stack overflow via deeply nested input (issue #5104)") + { + const auto inner = make_deeply_nested_bson(n, 0x03, 'a'); + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(inner), + "[json.exception.parse_error.116] parse error at byte 28676: syntax error while parsing BSON value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_bson(inner, true, false).is_discarded()); + } + + SECTION("array nesting depth is also capped (issue #5104)") + { + // The original depth guard only protected nested *objects* (record + // type 0x03); nested *arrays* (record type 0x04) went through a + // separate code path with no depth check of their own at all. The + // iterative rewrite unifies object/array container handling into one + // loop, so this gap is now closed too - verify it here explicitly. + const auto inner = make_deeply_nested_bson(n, 0x04, '0'); + json _; + CHECK_THROWS_WITH_AS(_ = json::from_bson(inner), + "[json.exception.parse_error.116] parse error at byte 28676: syntax error while parsing BSON value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_bson(inner, true, false).is_discarded()); + } +} + TEST_CASE("BSON document size mismatch") { json _; diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index af73642997..316ff57016 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -1996,6 +1996,22 @@ TEST_CASE("CBOR regressions") } } } + + SECTION("stack overflow via deeply nested input (issue #5104)") + { + // 4200 nested CBOR arrays of length 1 (0x81 each) - comfortably past + // max_depth (4096, see binary_reader.hpp); the (now iterative, + // heap-stack based) parser must reject this with a clean parse_error + // once the nesting exceeds the depth limit, rather than exhausting + // the native call stack. + std::vector vec(4200, 0x81); + vec.push_back(0x00); + json _; // NOLINT(readability-identifier-naming) + CHECK_THROWS_WITH_AS(_ = json::from_cbor(vec), + "[json.exception.parse_error.116] parse error at byte 4097: syntax error while parsing CBOR value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_cbor(vec, true, false).is_discarded()); + } } #endif diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 4358cd4646..2f46e464d3 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1570,6 +1570,22 @@ TEST_CASE("MessagePack") CHECK(json::from_msgpack(vec, true, false).is_discarded()); } } + + SECTION("stack overflow via deeply nested input (issue #5104)") + { + // 4200 nested fixarrays of length 1 (0x91 each) - comfortably past + // max_depth (4096, see binary_reader.hpp); the (now iterative, + // heap-stack based) parser must reject this with a clean + // parse_error once the nesting exceeds the depth limit, rather + // than exhausting the native call stack. + std::vector vec(4200, 0x91); + vec.push_back(0x00); + json _; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(vec), + "[json.exception.parse_error.116] parse error at byte 4097: syntax error while parsing MessagePack value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_msgpack(vec, true, false).is_discarded()); + } } SECTION("SAX aborts") diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 6df9acfac6..18c12a67c7 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -1638,6 +1638,28 @@ TEST_CASE("UBJSON") CHECK_THROWS_AS(_ = json::sax_parse(v_ubjson, &scp, json::input_format_t::ubjson), json::out_of_range&); } } + + SECTION("stack overflow via deeply nested input (issue #5104)") + { + // 4200 nested UBJSON arrays - comfortably past max_depth (4096, + // see binary_reader.hpp); the (now iterative, heap-stack based) + // parser must reject this with a clean parse_error once the + // nesting exceeds the depth limit, rather than exhausting the + // native call stack. + const int nesting = 4200; + std::string payload(static_cast(nesting), '['); + payload += "Z"; + for (int i = 0; i < nesting; ++i) + { + payload += ']'; + } + std::vector const v(payload.begin(), payload.end()); + json _; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(v), + "[json.exception.parse_error.116] parse error at byte 4098: syntax error while parsing UBJSON value: maximum depth of nested arrays/objects exceeded", + json::parse_error&); + CHECK(json::from_ubjson(v, true, false).is_discarded()); + } } SECTION("SAX aborts")