diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 2a18ea600..74993401f 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -23,6 +23,7 @@ #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 #endif +#include #include #include #include @@ -204,24 +205,12 @@ class binary_writer } case value_t::number_integer: - { - if (j.m_data.m_value.number_integer >= 0) - { - // CBOR does not differentiate between positive signed - // integers and unsigned integers - write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); - } - else - { - // a negative integer n is encoded as -1 - n - write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); - } - break; - } - case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + // one case for both, so that an array mixing negative + // (number_integer) and non-negative (number_unsigned) values + // does not mispredict the switch + write_cbor_integer(j); break; } @@ -256,13 +245,7 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - write_cbor_head(0x80, j.m_data.m_value.array->size()); - - for (const auto& el : *j.m_data.m_value.array) - { - write_cbor(el, depth + 1); - } + write_cbor_array(j, depth); break; } @@ -1951,6 +1934,127 @@ class binary_writer } } + /*! + @brief encode the head of a CBOR integer to @a out without branching on its + size + @return the number of characters of the head; up to @ref cbor_integer_head_space + characters at @a out are overwritten + + Same output as @ref write_cbor_head. The lengths of strings and containers + are mostly predictable, so their heads keep the branches; the sizes of a + run of integers are not, so an integer's head is assembled without them. + */ + std::size_t encode_cbor_integer_head(CharType* out, const std::uint8_t major_type, const std::uint64_t argument) const noexcept + { + // The size class - 0 if the argument fits the first byte, else 1 to 4 + // for 1, 2, 4, or 8 following bytes - is looked up from the number of + // leading zero bits instead of found by branches: the sizes of a stream + // of numbers are hard to predict, and a mispredicted branch costs more + // than writing the head. + static constexpr std::uint8_t size_class_by_leading_zeros[64] = + { + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + // an argument of at most 23 is in the table's class 1 (one byte) and is + // moved to class 0 by subtracting 1; argument | 1 keeps + // count_leading_zeros() away from 0 + const auto is_immediate = static_cast(argument <= 0x17u); + const unsigned size_class = size_class_by_leading_zeros[count_leading_zeros(argument | 1u)] - is_immediate; + const std::size_t length = (std::size_t{1} << size_class) >> 1u; + + // The first byte - the argument itself if it is immediate, else + // 0x17 + size_class - selected through a mask, then the length + // low-order bytes of the argument in network order, written as one + // 8-byte word whose bytes after the argument are scratch that + // the caller drops. For an immediate argument the word is all + // scratch; it is shifted as if length were 1, avoiding a shift by 64. + const std::uint64_t immediate_mask = std::uint64_t{0} - is_immediate; + out[0] = to_char_type(static_cast(major_type + ((argument & immediate_mask) | ((0x17u + size_class) & ~immediate_mask)))); + const unsigned unused_bits = 64u - 8u * static_cast(length + is_immediate); + const std::uint64_t tail = is_little_endian ? byte_swap(argument) >> unused_bits : argument << unused_bits; + std::memcpy(out + 1, &tail, sizeof(tail)); + return length + 1; + } + + /*! + @brief write an array value (head and elements) as CBOR + + Integer elements, the most common scalar, are written without a call to + @ref write_cbor. The loop is kept out of line: inlined into write_cbor, its + registers would be saved and restored on every call of write_cbor, which + is one call per value nested in an object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_cbor_array(const BasicJsonType& j, const std::size_t depth) + { + write_cbor_head(0x80, j.m_data.m_value.array->size()); + + // Runs of integers are encoded straight into space reserved from the + // sink for a whole run: the write pointer stays a local, where it would + // otherwise be reloaded from the sink for every value, as a store of a + // character may alias it. + constexpr std::size_t max_run = max_reserve_size / cbor_integer_head_space; + auto it = j.m_data.m_value.array->begin(); + const auto end = j.m_data.m_value.array->end(); + while (it != end) + { + if (it->is_number_integer()) + { + CharType* const start = oa.reserve(max_run * cbor_integer_head_space); + CharType* out = start; + for (std::size_t n = 0; n < max_run && it != end && it->is_number_integer(); ++n, ++it) + { + out += encode_cbor_integer(out, *it); + } + oa.commit(static_cast(out - start)); + } + else + { + write_cbor(*it, depth + 1); + ++it; + } + } + } + + /*! + @brief write a number_integer or number_unsigned value as a CBOR integer + + A negative integer n has major type 1 and the argument -1 - n, which is ~n. + The sign and the type select the major type and the argument through a + mask rather than a branch, as they alternate unpredictably in an array of + mixed signs. + */ + void write_cbor_integer(const BasicJsonType& j) + { + oa.commit(encode_cbor_integer(oa.reserve(cbor_integer_head_space), j)); + } + + /// encode a number_integer or number_unsigned value to @a out (see + /// @ref write_cbor_integer) and return the number of characters + std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept + { + const bool is_signed = j.m_data.m_type == value_t::number_integer; + const std::uint64_t raw = is_signed + ? static_cast(j.m_data.m_value.number_integer) + : static_cast(j.m_data.m_value.number_unsigned); + // All ones for a negative integer, else zero. Up to 64 bits, the sign is + // bit 63 of raw (a narrower integer is sign-extended); computing it + // with masks keeps GCC from branching on is_signed. A wider integer + // has to be compared. + const std::uint64_t sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) + ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) + : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + // ~raw is -1 - n in the 64 bits that are written + return encode_cbor_integer_head(out, static_cast(sign_mask & 0x20u), raw ^ sign_mask); + } + + /// characters encode_cbor_integer_head() may overwrite: the first byte and + /// one 8-byte word + static constexpr std::size_t cbor_integer_head_space = 9; + /*! @brief write an object key as a CBOR text string diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index dc2c06587..f9aa76688 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -9,6 +9,7 @@ #pragma once #include // max +#include // array #include // size_t #include // memcpy #include // shared_ptr, make_shared @@ -50,8 +51,8 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; -/// output_vector_sink appends writes of at least this many characters with -/// insert() instead of copying them to the space it keeps ahead +/// the largest number of characters a writer may reserve() from a sink at once; +/// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; /// @brief non-virtual output sink writing into a std::vector @@ -132,6 +133,31 @@ class output_vector_sink } } + /// @brief space for writing up to @a length characters directly + /// + /// Returns a pointer to at least @a length (at most @ref max_reserve_size) + /// writable characters at the write position; @ref commit then keeps the + /// first ones of them. A writer can store whole words there and keep only + /// the bytes it needs, and keep its write pointer in a register across many + /// values instead of updating the sink after each. + JSON_HEDLEY_ALWAYS_INLINE + CharType* reserve(const std::size_t length) + { + JSON_ASSERT(length <= max_reserve_size); + if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + { + grow(length); + } + return v->data() + position; + } + + /// keep the first @a length characters written to the space from reserve() + JSON_HEDLEY_ALWAYS_INLINE + void commit(const std::size_t length) noexcept + { + position += length; + } + private: /// make room for at least @a length more characters after the position; /// kept out of line so that the write functions stay small enough for @@ -276,8 +302,24 @@ class output_adapter_sink oa->write_characters(s, length); } + /// space for writing up to @a length characters (see + /// output_vector_sink::reserve) + CharType* reserve(const std::size_t length) noexcept + { + JSON_ASSERT(length <= max_reserve_size); + static_cast(length); + return reserved.data(); + } + + /// write the first @a length characters written to the space from reserve() + void commit(const std::size_t length) + { + oa->write_characters(reserved.data(), length); + } + private: output_adapter_t oa; + std::array reserved{}; }; /// @brief whether std::basic_string has a non-deprecated std::char_traits diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f83725215..6effa7464 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21619,6 +21619,8 @@ NLOHMANN_JSON_NAMESPACE_END #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 #endif +// #include + // #include // #include @@ -21639,6 +21641,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // max +#include // array #include // size_t #include // memcpy #include // shared_ptr, make_shared @@ -21681,8 +21684,8 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; -/// output_vector_sink appends writes of at least this many characters with -/// insert() instead of copying them to the space it keeps ahead +/// the largest number of characters a writer may reserve() from a sink at once; +/// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; /// @brief non-virtual output sink writing into a std::vector @@ -21763,6 +21766,31 @@ class output_vector_sink } } + /// @brief space for writing up to @a length characters directly + /// + /// Returns a pointer to at least @a length (at most @ref max_reserve_size) + /// writable characters at the write position; @ref commit then keeps the + /// first ones of them. A writer can store whole words there and keep only + /// the bytes it needs, and keep its write pointer in a register across many + /// values instead of updating the sink after each. + JSON_HEDLEY_ALWAYS_INLINE + CharType* reserve(const std::size_t length) + { + JSON_ASSERT(length <= max_reserve_size); + if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + { + grow(length); + } + return v->data() + position; + } + + /// keep the first @a length characters written to the space from reserve() + JSON_HEDLEY_ALWAYS_INLINE + void commit(const std::size_t length) noexcept + { + position += length; + } + private: /// make room for at least @a length more characters after the position; /// kept out of line so that the write functions stay small enough for @@ -21907,8 +21935,24 @@ class output_adapter_sink oa->write_characters(s, length); } + /// space for writing up to @a length characters (see + /// output_vector_sink::reserve) + CharType* reserve(const std::size_t length) noexcept + { + JSON_ASSERT(length <= max_reserve_size); + static_cast(length); + return reserved.data(); + } + + /// write the first @a length characters written to the space from reserve() + void commit(const std::size_t length) + { + oa->write_characters(reserved.data(), length); + } + private: output_adapter_t oa; + std::array reserved{}; }; /// @brief whether std::basic_string has a non-deprecated std::char_traits @@ -22193,24 +22237,12 @@ class binary_writer } case value_t::number_integer: - { - if (j.m_data.m_value.number_integer >= 0) - { - // CBOR does not differentiate between positive signed - // integers and unsigned integers - write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); - } - else - { - // a negative integer n is encoded as -1 - n - write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); - } - break; - } - case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + // one case for both, so that an array mixing negative + // (number_integer) and non-negative (number_unsigned) values + // does not mispredict the switch + write_cbor_integer(j); break; } @@ -22245,13 +22277,7 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - write_cbor_head(0x80, j.m_data.m_value.array->size()); - - for (const auto& el : *j.m_data.m_value.array) - { - write_cbor(el, depth + 1); - } + write_cbor_array(j, depth); break; } @@ -23940,6 +23966,127 @@ class binary_writer } } + /*! + @brief encode the head of a CBOR integer to @a out without branching on its + size + @return the number of characters of the head; up to @ref cbor_integer_head_space + characters at @a out are overwritten + + Same output as @ref write_cbor_head. The lengths of strings and containers + are mostly predictable, so their heads keep the branches; the sizes of a + run of integers are not, so an integer's head is assembled without them. + */ + std::size_t encode_cbor_integer_head(CharType* out, const std::uint8_t major_type, const std::uint64_t argument) const noexcept + { + // The size class - 0 if the argument fits the first byte, else 1 to 4 + // for 1, 2, 4, or 8 following bytes - is looked up from the number of + // leading zero bits instead of found by branches: the sizes of a stream + // of numbers are hard to predict, and a mispredicted branch costs more + // than writing the head. + static constexpr std::uint8_t size_class_by_leading_zeros[64] = + { + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + // an argument of at most 23 is in the table's class 1 (one byte) and is + // moved to class 0 by subtracting 1; argument | 1 keeps + // count_leading_zeros() away from 0 + const auto is_immediate = static_cast(argument <= 0x17u); + const unsigned size_class = size_class_by_leading_zeros[count_leading_zeros(argument | 1u)] - is_immediate; + const std::size_t length = (std::size_t{1} << size_class) >> 1u; + + // The first byte - the argument itself if it is immediate, else + // 0x17 + size_class - selected through a mask, then the length + // low-order bytes of the argument in network order, written as one + // 8-byte word whose bytes after the argument are scratch that + // the caller drops. For an immediate argument the word is all + // scratch; it is shifted as if length were 1, avoiding a shift by 64. + const std::uint64_t immediate_mask = std::uint64_t{0} - is_immediate; + out[0] = to_char_type(static_cast(major_type + ((argument & immediate_mask) | ((0x17u + size_class) & ~immediate_mask)))); + const unsigned unused_bits = 64u - 8u * static_cast(length + is_immediate); + const std::uint64_t tail = is_little_endian ? byte_swap(argument) >> unused_bits : argument << unused_bits; + std::memcpy(out + 1, &tail, sizeof(tail)); + return length + 1; + } + + /*! + @brief write an array value (head and elements) as CBOR + + Integer elements, the most common scalar, are written without a call to + @ref write_cbor. The loop is kept out of line: inlined into write_cbor, its + registers would be saved and restored on every call of write_cbor, which + is one call per value nested in an object. + */ + JSON_HEDLEY_NEVER_INLINE + void write_cbor_array(const BasicJsonType& j, const std::size_t depth) + { + write_cbor_head(0x80, j.m_data.m_value.array->size()); + + // Runs of integers are encoded straight into space reserved from the + // sink for a whole run: the write pointer stays a local, where it would + // otherwise be reloaded from the sink for every value, as a store of a + // character may alias it. + constexpr std::size_t max_run = max_reserve_size / cbor_integer_head_space; + auto it = j.m_data.m_value.array->begin(); + const auto end = j.m_data.m_value.array->end(); + while (it != end) + { + if (it->is_number_integer()) + { + CharType* const start = oa.reserve(max_run * cbor_integer_head_space); + CharType* out = start; + for (std::size_t n = 0; n < max_run && it != end && it->is_number_integer(); ++n, ++it) + { + out += encode_cbor_integer(out, *it); + } + oa.commit(static_cast(out - start)); + } + else + { + write_cbor(*it, depth + 1); + ++it; + } + } + } + + /*! + @brief write a number_integer or number_unsigned value as a CBOR integer + + A negative integer n has major type 1 and the argument -1 - n, which is ~n. + The sign and the type select the major type and the argument through a + mask rather than a branch, as they alternate unpredictably in an array of + mixed signs. + */ + void write_cbor_integer(const BasicJsonType& j) + { + oa.commit(encode_cbor_integer(oa.reserve(cbor_integer_head_space), j)); + } + + /// encode a number_integer or number_unsigned value to @a out (see + /// @ref write_cbor_integer) and return the number of characters + std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept + { + const bool is_signed = j.m_data.m_type == value_t::number_integer; + const std::uint64_t raw = is_signed + ? static_cast(j.m_data.m_value.number_integer) + : static_cast(j.m_data.m_value.number_unsigned); + // All ones for a negative integer, else zero. Up to 64 bits, the sign is + // bit 63 of raw (a narrower integer is sign-extended); computing it + // with masks keeps GCC from branching on is_signed. A wider integer + // has to be compared. + const std::uint64_t sign_mask = sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) + ? (std::uint64_t{0} - static_cast(is_signed)) & (std::uint64_t{0} - (raw >> 63u)) + : std::uint64_t{0} - static_cast(is_signed && j.m_data.m_value.number_integer < 0); + // ~raw is -1 - n in the 64 bits that are written + return encode_cbor_integer_head(out, static_cast(sign_mask & 0x20u), raw ^ sign_mask); + } + + /// characters encode_cbor_integer_head() may overwrite: the first byte and + /// one 8-byte word + static constexpr std::size_t cbor_integer_head_space = 9; + /*! @brief write an object key as a CBOR text string diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index cad77172e..c17bb1ed3 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -13,6 +13,7 @@ using nlohmann::json; #include #include +#include #include #include @@ -182,6 +183,161 @@ TEST_CASE("binary writer output sinks") } } +namespace +{ + +// the CBOR head of an integer, encoded independently of binary_writer +void reference_cbor_head(std::vector& out, const std::uint8_t major_type, const std::uint64_t argument) +{ + if (argument <= 23) + { + out.push_back(static_cast(major_type + argument)); + return; + } + + int length = 8; + std::uint8_t additional = 27; + if (argument <= 0xFF) + { + length = 1; + additional = 24; + } + else if (argument <= 0xFFFF) + { + length = 2; + additional = 25; + } + else if (argument <= 0xFFFFFFFF) + { + length = 4; + additional = 26; + } + + out.push_back(static_cast(major_type + additional)); + for (int i = length - 1; i >= 0; --i) + { + out.push_back(static_cast(argument >> (8 * i))); + } +} + +// the CBOR encoding of an integer value +void reference_cbor_integer(std::vector& out, const json& j) +{ + if (j.is_number_unsigned()) + { + reference_cbor_head(out, 0x00, j.get()); + return; + } + + const auto n = j.get(); + if (n >= 0) + { + reference_cbor_head(out, 0x00, static_cast(n)); + } + else + { + reference_cbor_head(out, 0x20, static_cast(-1 - n)); + } +} + +// integers at every boundary of the CBOR head sizes, signed and unsigned +std::vector boundary_integers() +{ + std::vector result; + const std::uint64_t boundaries[] = + { + 0, 1, 22, 23, 24, 25, 254, 255, 256, 257, 65534, 65535, 65536, 65537, + 4294967294ULL, 4294967295ULL, 4294967296ULL, 4294967297ULL, + 9223372036854775806ULL, 9223372036854775807ULL, 9223372036854775808ULL, + 18446744073709551614ULL, 18446744073709551615ULL + }; + for (const std::uint64_t u : boundaries) + { + result.emplace_back(u); + if (u <= static_cast((std::numeric_limits::max)())) + { + const auto n = static_cast(u); + result.emplace_back(static_cast(n)); + result.emplace_back(static_cast(-n)); + result.emplace_back(static_cast(-n - 1)); + } + } + return result; +} + +} // namespace + +// Arrays of integers are written in runs straight into space reserved from the +// sink, with a branch-free head; check them against an independent encoder for +// runs shorter and longer than one reservation, broken up by other values, and +// through each sink. +TEST_CASE("CBOR integer arrays") +{ + const std::vector integers = boundary_integers(); + + for (const std::size_t length : + { + std::size_t{1}, std::size_t{27}, std::size_t{28}, std::size_t{29}, std::size_t{57}, std::size_t{1000} + }) + { + for (const std::size_t break_every : + { + std::size_t{0}, std::size_t{1}, std::size_t{7}, std::size_t{28} + }) + { + CAPTURE(length) + CAPTURE(break_every) + + json array = json::array(); + std::vector expected; + for (std::size_t i = 0; i < length; ++i) + { + if (break_every != 0 && i % (break_every + 1) == break_every) + { + array.push_back("x"); + expected.push_back(0x61); + expected.push_back('x'); + } + else + { + const json& value = integers[(i * 7) % integers.size()]; + array.push_back(value); + reference_cbor_integer(expected, value); + } + } + + std::vector head; + reference_cbor_head(head, 0x80, length); + expected.insert(expected.begin(), head.begin(), head.end()); + + CHECK(json::to_cbor(array) == expected); + + std::vector through_adapter = {0xAA}; + json::to_cbor(array, through_adapter); + CHECK(std::vector(through_adapter.begin() + 1, through_adapter.end()) == expected); + + std::ostringstream stream; + json::to_cbor(array, stream); + const std::string streamed = stream.str(); + CHECK(std::vector(streamed.begin(), streamed.end()) == expected); + + CHECK(json::from_cbor(expected) == array); + + // the same array as an object member, and as single values + const json object = {{"a", array}, {"b", array}}; + CHECK(json::from_cbor(json::to_cbor(object)) == object); + } + } + + for (const auto& value : integers) + { + CAPTURE(value) + std::vector expected; + reference_cbor_integer(expected, value); + CHECK(json::to_cbor(value) == expected); + } +} + // binary_reserve_hint() is documented as a *lower* bound on the serialized size, // so that reserving it up front can never leave the returned vector holding // capacity beyond what the value actually needs.