Merge branch 'develop' into fix/issue-5993

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -344,8 +344,8 @@ class binary_reader
     nested document could declare a length that disagrees with where its
     terminator actually falls and quietly hand the bytes in between to the
     enclosing document. A well-formed document is at least 5 bytes (the prefix
-    plus the terminator); open_bson_document() rejects a smaller size before
+    plus the terminator); the equality also rejects those impossible sizes,
-    it is stored, so @a document_size is never negative here.
+    since at least 5 bytes are always consumed.
 
     @param[in] document_start  value of chars_read before the size prefix
     @param[in] document_size   the declared document size
@@ -353,7 +353,7 @@ class binary_reader
     */
     bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
     {
-        if (JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(document_size) != chars_read - document_start))
+        if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
         {
             return bson_document_size_error(document_size, chars_read - document_start);
         }
@@ -3092,13 +3092,16 @@ class binary_reader
         {
             if (size_and_type.second != 0)
             {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                if (size_and_type.second != 'N')
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))
+                    for (std::size_t i = 0; i < size_and_type.first; ++i)
                     {
-                        return false;
+                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))
+                        {
+                            return false;
+                        }
+                        dim.push_back(dimlen);
                     }
-                    dim.push_back(dimlen);
                 }
             }
             else
@@ -3300,9 +3303,11 @@ class binary_reader
                     // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
                     if (ndarray_dtype != 0)
                     {
-                        // every optimized type marker is_valid_optimized_type() accepts for BJData has a name
                         const char* type_name = bjd_type_name(ndarray_dtype);
-                        JSON_ASSERT(type_name != nullptr);
+                        if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
+                        {
+                            return unexpected_byte("invalid byte", "type");
+                        }
 
                         string_t type_key = "_ArrayType_";
                         string_t type = type_name; // sax->string() takes a reference
@@ -3385,6 +3390,12 @@ class binary_reader
         if (current == '$')
         {
             result.second = get();  // must not ignore 'N', because 'N' maybe the type
+            if (input_format == input_format_t::bjdata
+                    && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second)))
+            {
+                return optimized_type_error();
+            }
+
             if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type")))
             {
                 return false;
@@ -3412,7 +3423,7 @@ class binary_reader
             // an ndarray was read here only if the flag flipped; when it was
             // seeded true, get_ubjson_size_value() already rejected the nested
             // dimension vector
-            if (is_ndarray && !inside_ndarray)
+            if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
             {
                 result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters
             }
@@ -3422,7 +3433,7 @@ class binary_reader
         if (current == '#')
         {
             const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
-            if (is_ndarray && !inside_ndarray)
+            if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
             {
                 return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size");
             }
@@ -3596,6 +3607,7 @@ class binary_reader
     JSON_HEDLEY_NEVER_INLINE
     bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/)
     {
+        const bool bjdata = input_format == input_format_t::bjdata;
         switch (type)
         {
             case 'U':
@@ -3609,11 +3621,11 @@ class binary_reader
             case 'L':
                 return parse_ubjson_typed_run<std::int64_t>(remaining);
             case 'u':
-                return parse_ubjson_typed_run<std::uint16_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint16_t>(remaining);
             case 'm':
-                return parse_ubjson_typed_run<std::uint32_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint32_t>(remaining);
             case 'M':
-                return parse_ubjson_typed_run<std::uint64_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint64_t>(remaining);
             case 'd':
                 return parse_ubjson_typed_run<float>(remaining);
             case 'D':
@@ -3694,7 +3706,7 @@ class binary_reader
         // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):
         // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]}
 
-        if ((size_and_type.second & (1 << 8)) != 0)
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
         {
             size_and_type.second &= ~(static_cast<char_int_type>(1) << 8);  // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
 
@@ -3753,7 +3765,7 @@ class binary_reader
         }
 
         // If BJData type marker is 'B' decode as binary
-        if (size_and_type.first != npos && size_and_type.second == 'B')
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B')
         {
             binary_t result;
             return get_binary(size_and_type.first, result) && sax->binary(result);
@@ -3802,7 +3814,7 @@ class binary_reader
         }
 
         // do not accept ND-array size in objects in BJData
-        if ((size_and_type.second & (1 << 8)) != 0)
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
         {
             return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object");
         }
@@ -5083,9 +5095,8 @@ class binary_reader
   JSON_PRIVATE_UNLESS_TESTED:
     /*!
     @brief check whether @a marker is a type marker that may follow '$'
-    UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C.
+    UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C
-    The markers BJData excludes (F H N S T Z [ {) are the ones
+    (BJData's excluded markers are rejected separately).
-    binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync.
     */
     bool is_valid_optimized_type(const char_int_type marker) const noexcept
     {
@@ -5120,6 +5131,20 @@ class binary_reader
         }
     }
 
+    /*!
+    @brief whether @a marker is excluded from BJData's optimized ND-array types
+    @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{'
+
+    Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker
+    "is_bjdata_excluded_type_marker()`, which encodes the same list the other
+    way; keep the two in sync.
+    */
+    static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept
+    {
+        return marker == '[' || marker == '{' || marker == 'S' || marker == 'H'
+               || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z';
+    }
+
     /*!
     @brief look up the ND-array element type name for a BJData dtype marker
     @return the type name ("uint8", "int8", ...), or nullptr if @a marker does

include/nlohmann/detail/output/binary_writer.hpp

@@ -3126,19 +3126,22 @@ class binary_writer
                 // negative fixnum
                 write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
             {
                 // int 8
                 oa.write_character(to_char_type(0xD0));
                 write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
             {
                 // int 16
                 oa.write_character(to_char_type(0xD1));
                 write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
             {
                 // int 32
                 oa.write_character(to_char_type(0xD2));
@@ -4054,7 +4057,7 @@ class binary_writer
     Containers, strings, high-precision numbers, booleans and null cannot be
     declared as the single type of an optimized container in BJData; such a
     container is written unoptimized. The reader rejects them with the same
-    list (binary_reader::is_valid_optimized_type()).
+    list (binary_reader::is_bjd_excluded_optimized_type()).
     */
     static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept
     {

include/nlohmann/detail/output/serializer.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -2080,31 +2080,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
+        // look up the locale's decimal point now, matching what
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto first = number_buffer.begin();
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            const auto last = first + len;
-            if (dec_pos != number_buffer.end())
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

single_include/nlohmann/json.hpp

@@ -18359,8 +18359,8 @@ class binary_reader
     nested document could declare a length that disagrees with where its
     terminator actually falls and quietly hand the bytes in between to the
     enclosing document. A well-formed document is at least 5 bytes (the prefix
-    plus the terminator); open_bson_document() rejects a smaller size before
+    plus the terminator); the equality also rejects those impossible sizes,
-    it is stored, so @a document_size is never negative here.
+    since at least 5 bytes are always consumed.
 
     @param[in] document_start  value of chars_read before the size prefix
     @param[in] document_size   the declared document size
@@ -18368,7 +18368,7 @@ class binary_reader
     */
     bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
     {
-        if (JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(document_size) != chars_read - document_start))
+        if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
         {
             return bson_document_size_error(document_size, chars_read - document_start);
         }
@@ -21107,13 +21107,16 @@ class binary_reader
         {
             if (size_and_type.second != 0)
             {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
+                if (size_and_type.second != 'N')
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))
+                    for (std::size_t i = 0; i < size_and_type.first; ++i)
                     {
-                        return false;
+                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, no_ndarray, size_and_type.second)))
+                        {
+                            return false;
+                        }
+                        dim.push_back(dimlen);
                     }
-                    dim.push_back(dimlen);
                 }
             }
             else
@@ -21315,9 +21318,11 @@ class binary_reader
                     // follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
                     if (ndarray_dtype != 0)
                     {
-                        // every optimized type marker is_valid_optimized_type() accepts for BJData has a name
                         const char* type_name = bjd_type_name(ndarray_dtype);
-                        JSON_ASSERT(type_name != nullptr);
+                        if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
+                        {
+                            return unexpected_byte("invalid byte", "type");
+                        }
 
                         string_t type_key = "_ArrayType_";
                         string_t type = type_name; // sax->string() takes a reference
@@ -21400,6 +21405,12 @@ class binary_reader
         if (current == '$')
         {
             result.second = get();  // must not ignore 'N', because 'N' maybe the type
+            if (input_format == input_format_t::bjdata
+                    && JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second)))
+            {
+                return optimized_type_error();
+            }
+
             if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type")))
             {
                 return false;
@@ -21427,7 +21438,7 @@ class binary_reader
             // an ndarray was read here only if the flag flipped; when it was
             // seeded true, get_ubjson_size_value() already rejected the nested
             // dimension vector
-            if (is_ndarray && !inside_ndarray)
+            if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
             {
                 result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters
             }
@@ -21437,7 +21448,7 @@ class binary_reader
         if (current == '#')
         {
             const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
-            if (is_ndarray && !inside_ndarray)
+            if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
             {
                 return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size");
             }
@@ -21611,6 +21622,7 @@ class binary_reader
     JSON_HEDLEY_NEVER_INLINE
     bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/)
     {
+        const bool bjdata = input_format == input_format_t::bjdata;
         switch (type)
         {
             case 'U':
@@ -21624,11 +21636,11 @@ class binary_reader
             case 'L':
                 return parse_ubjson_typed_run<std::int64_t>(remaining);
             case 'u':
-                return parse_ubjson_typed_run<std::uint16_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint16_t>(remaining);
             case 'm':
-                return parse_ubjson_typed_run<std::uint32_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint32_t>(remaining);
             case 'M':
-                return parse_ubjson_typed_run<std::uint64_t>(remaining);
+                return !bjdata || parse_ubjson_typed_run<std::uint64_t>(remaining);
             case 'd':
                 return parse_ubjson_typed_run<float>(remaining);
             case 'D':
@@ -21709,7 +21721,7 @@ class binary_reader
         // if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):
         // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]}
 
-        if ((size_and_type.second & (1 << 8)) != 0)
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
         {
             size_and_type.second &= ~(static_cast<char_int_type>(1) << 8);  // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
 
@@ -21768,7 +21780,7 @@ class binary_reader
         }
 
         // If BJData type marker is 'B' decode as binary
-        if (size_and_type.first != npos && size_and_type.second == 'B')
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B')
         {
             binary_t result;
             return get_binary(size_and_type.first, result) && sax->binary(result);
@@ -21817,7 +21829,7 @@ class binary_reader
         }
 
         // do not accept ND-array size in objects in BJData
-        if ((size_and_type.second & (1 << 8)) != 0)
+        if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
         {
             return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object");
         }
@@ -23098,9 +23110,8 @@ class binary_reader
   JSON_PRIVATE_UNLESS_TESTED:
     /*!
     @brief check whether @a marker is a type marker that may follow '$'
-    UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C.
+    UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C
-    The markers BJData excludes (F H N S T Z [ {) are the ones
+    (BJData's excluded markers are rejected separately).
-    binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync.
     */
     bool is_valid_optimized_type(const char_int_type marker) const noexcept
     {
@@ -23135,6 +23146,20 @@ class binary_reader
         }
     }
 
+    /*!
+    @brief whether @a marker is excluded from BJData's optimized ND-array types
+    @return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{'
+
+    Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker
+    "is_bjdata_excluded_type_marker()`, which encodes the same list the other
+    way; keep the two in sync.
+    */
+    static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept
+    {
+        return marker == '[' || marker == '{' || marker == 'S' || marker == 'H'
+               || marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z';
+    }
+
     /*!
     @brief look up the ND-array element type name for a BJData dtype marker
     @return the type name ("uint8", "int8", ...), or nullptr if @a marker does
@@ -31085,19 +31110,22 @@ class binary_writer
                 // negative fixnum
                 write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
             {
                 // int 8
                 oa.write_character(to_char_type(0xD0));
                 write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
             {
                 // int 16
                 oa.write_character(to_char_type(0xD1));
                 write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
             }
-            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)())
+            else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
+                     j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
             {
                 // int 32
                 oa.write_character(to_char_type(0xD2));
@@ -32013,7 +32041,7 @@ class binary_writer
     Containers, strings, high-precision numbers, booleans and null cannot be
     declared as the single type of an optimized container in BJData; such a
     container is written unoptimized. The reader rejects them with the same
-    list (binary_reader::is_valid_optimized_type()).
+    list (binary_reader::is_bjd_excluded_optimized_type()).
     */
     static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept
     {
@@ -33418,7 +33446,7 @@ NLOHMANN_JSON_NAMESPACE_END
 
 
 
-#include <algorithm> // remove, fill, find, none_of, min
+#include <algorithm> // copy, none_of, min, search
 #include <array> // array
 #include <clocale> // localeconv, lconv
 #include <cmath> // isfinite
@@ -37272,31 +37300,35 @@ class serializer
         // check if the buffer was large enough
         JSON_ASSERT(static_cast<std::size_t>(len) < number_buffer.size());
 
-        // look up the locale's thousands separator and decimal point now,
+        // look up the locale's decimal point now, matching what
-        // matching what snprintf_float() just used (see lexer::get_decimal_point())
+        // snprintf_float() just used (see get_decimal_point() in
+        // number_parse.hpp); it is a string, which can be longer than one
+        // byte (U+066B in ps_AF, #5993).
+        // A thousands separator needs no handling: %g never groups digits.
         const auto* loc = std::localeconv();
         JSON_ASSERT(loc != nullptr);
-        const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep;
+        const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point;
-        const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point;
+        const std::size_t decimal_point_len = std::char_traits<char>::length(decimal_point);
-
-        // erase thousands separators
-        if (thousands_sep != '\0')
-        {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081
-            const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep);
-            std::fill(end, number_buffer.end(), '\0');
-            JSON_ASSERT((end - number_buffer.begin()) <= len);
-            len = (end - number_buffer.begin());
-        }
 
         // convert decimal point to '.'
-        if (decimal_point != '\0' && decimal_point != '.')
+        if ((decimal_point_len > 1) || (decimal_point_len == 1 && decimal_point[0] != '.'))
         {
-            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto first = number_buffer.begin();
-            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            const auto last = first + len;
-            if (dec_pos != number_buffer.end())
+            // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081
+            const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len);
+            if (dec_pos != last)
             {
                 *dec_pos = '.';
+                if (decimal_point_len > 1)
+                {
+                    // move the fraction and exponent behind the '.' (std::copy
+                    // allows the overlap, as the destination is before the source)
+                    // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081
+                    const auto end = std::copy(dec_pos + static_cast<std::ptrdiff_t>(decimal_point_len), last, dec_pos + 1);
+                    *end = '\0';
+                    len = end - first;
+                }
             }
         }
 

tests/src/unit-allocator.cpp

@@ -1038,6 +1038,14 @@ template<class T>
 struct sweep_allocator
 {
     using value_type = T;
+    // GCC 4.8's std::map takes these from the allocator instead of from
+    // std::allocator_traits
+    using pointer = T*;
+    using const_pointer = const T*;
+    using reference = T&;
+    using const_reference = const T&;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
 
     sweep_allocator() = default;
     template<class U> sweep_allocator(const sweep_allocator<U>& /*unused*/) noexcept {}
@@ -1155,9 +1163,14 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
 
     // With iterator debugging (MSVC STL debug builds, also used by clang-cl),
     // containers allocate a debug proxy through the allocator inside their
-    // noexcept move constructors, so failing that allocation terminates the
+    // noexcept default and move constructors, so failing that allocation
-    // program instead of throwing std::bad_alloc. Nothing to check there.
+    // terminates the program instead of throwing std::bad_alloc. Nothing to
+    // check there.
 #if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
+    // Clang 3.4 and 3.5 leak memory when an element constructor throws
+    // inside libstdc++'s containers (a std::map copy, a std::vector
+    // reallocation), without any json value involved.
+#if !(defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ == 3 && __clang_minor__ < 6)
     SECTION("parser callback")
     {
         const auto keep_all = [](int /*depth*/, sweep_json::parse_event_t /*event*/, sweep_json& /*parsed*/) noexcept
@@ -1178,7 +1191,6 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
             return sweep_json::parse(text, drop_d);
         });
     }
-#endif
 
     SECTION("sax_parse into an existing value")
     {
@@ -1192,5 +1204,7 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
             return value;
         });
     }
+#endif
+#endif
 }
 #endif

tests/src/unit-bjdata.cpp

@@ -78,25 +78,23 @@ TEST_CASE("BJData")
 {
     SECTION("binary_reader BJData lookup tables")
     {
-        // bjd_type_name() is static; is_valid_optimized_type() needs a BJData reader
+        // both lookups are static member functions
         std::vector<std::uint8_t> const data;
         using reader_t = nlohmann::detail::binary_reader<json, decltype(nlohmann::detail::input_adapter(data))>;
 
-        // the markers BJData excludes as optimized types must match
+        // the excluded optimized-type markers must match binary_writer's
-        // binary_writer's is_bjdata_excluded_type_marker(), which encodes the
+        // is_bjdata_excluded_type_marker(), which encodes the same 8 markers
-        // same 8 markers
-        reader_t reader(nlohmann::detail::input_adapter(data), json::input_format_t::bjdata);
         for (const char marker :
-                {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z', 'x'
+                {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'
                 })
         {
-            CHECK(!reader.is_valid_optimized_type(static_cast<unsigned char>(marker)));
+            CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
         }
         for (const char marker :
-                {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'h'
+                {'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x'
                 })
         {
-            CHECK(reader.is_valid_optimized_type(static_cast<unsigned char>(marker)));
+            CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
         }
 
         // every dtype marker must round-trip to its ND-array type name

tests/src/unit-class_lexer.cpp

@@ -2222,9 +2222,10 @@ TEST_CASE("serialization of short strings")
     {
         CAPTURE(s)
         const json j = s;
-        CHECK(j.dump() == "\"" + s + "\"");
+        const std::string quoted = "\"" + s + "\""; // NOLINT(performance-inefficient-string-concatenation)
-        CHECK(j.dump(-1, ' ', true) == "\"" + s + "\"");
+        CHECK(j.dump() == quoted);
-        CHECK(json({{s, s}}).dump() == "{\"" + s + "\":\"" + s + "\"}");
+        CHECK(j.dump(-1, ' ', true) == quoted);
+        CHECK(json({{s, s}}).dump() == "{" + quoted + ":" + quoted + "}"); // NOLINT(performance-inefficient-string-concatenation)
     }
 
     // a character that needs an escape

tests/src/unit-custom-object-key-type.cpp

@@ -160,12 +160,12 @@ class strict_key
     }
 
   private:
-    std::string m_value{};
+    std::string m_value;
 };
 
 strict_key make_strict_key(const std::string& key)
 {
-    return strict_key(key.c_str());
+    return {key.c_str()};
 }
 
 template<class Key, class Value, class Compare, class Allocator>
@@ -224,7 +224,9 @@ TEST_CASE("converting a value with keys that cannot be made from a std::string n
         }
     }
 
-    const strict_json converted = value;
+    // direct-initialization: a copy-initialization makes GCC 4.8 instantiate the
+    // copy constructor, which needs a range constructor strict_object lacks
+    const strict_json converted(value);
 
     // at() needs to make a std::string of a key for its error message
     const auto member = [](const strict_json & object, const char* name) -> const strict_json &

tests/src/unit-json_view.cpp

@@ -105,19 +105,53 @@ template<typename T>
 struct failing_allocator
 {
     using value_type = T;
+    // GCC 4.8's std::map takes these from the allocator instead of from
+    // std::allocator_traits
+    using pointer = T*;
+    using const_pointer = const T*;
+    using reference = T&;
+    using const_reference = const T&;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+
+    template<typename U>
+    struct rebind
+    {
+        using other = failing_allocator<U>;
+    };
 
     failing_allocator() = default;
 
     template<typename U>
     explicit failing_allocator(const failing_allocator<U>& /*other*/) noexcept {}
 
-    T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
+    [[noreturn]] T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
     {
         throw std::bad_alloc();
     }
 
     void deallocate(T* /*p*/, std::size_t /*n*/) noexcept {} // NOLINT(readability-convert-member-functions-to-static)
 
+    // GCC 4.8's std::map calls these on the allocator directly instead of
+    // going through std::allocator_traits
+    std::size_t max_size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
+    {
+        return (std::numeric_limits<std::size_t>::max)() / sizeof(T);
+    }
+
+    template<typename U, typename... Args>
+    void construct(U* p, Args&& ... args) // NOLINT(readability-convert-member-functions-to-static)
+    {
+        ::new (static_cast<void*>(p)) U(std::forward<Args>(args)...);
+    }
+
+    template<typename U>
+    void destroy(U* p) noexcept // NOLINT(readability-convert-member-functions-to-static)
+    {
+        static_cast<void>(p); // unused for trivially destructible types
+        p->~U();
+    }
+
     template<typename U>
     bool operator==(const failing_allocator<U>& /*other*/) const noexcept
     {
@@ -270,7 +304,7 @@ TEST_CASE("json_view")
                     "\"", "\"abc", "\"\\x\"", "\"\\u12\"", "\"\\ud800\"", "\"\\udc00\"", "\"\x01\"", "\"\xff\"", "\"\xc3\"", "[1]x", "/", "/*", "[\n  1,\n  x\n]", // NOLINT(modernize-raw-string-literal)
                     "1e400", "-1e400", "[1.7976931348623159e308]", "{\"a\":\n{\"b\": [1, 2,\n 3 x]}}",
                     // (an error after every kind of value: the error is found by replaying the parser)
-                    "[null, true, -1, 1, 1.5, \"s\", {\"a\": 1} 2]", "{\"a\": null, \"b\": -1, \"c\": 1.5, \"d\": \"s\", \"e\": {}, 1}"
+                    R"([null, true, -1, 1, 1.5, "s", {"a": 1} 2])", R"({"a": null, "b": -1, "c": 1.5, "d": "s", "e": {}, 1})"
                 })
         {
             CAPTURE(text)
@@ -1556,7 +1590,7 @@ TEST_CASE("json_view values")
         CHECK_THROWS_WITH_AS(not_an_object.root().get<view_map>(), json_view_test::message("type_error.302", "", 0, 5, "type must be object, but is array").c_str(), json::type_error&);
 #if !defined(JSON_NOEXCEPTION)
         // (and one that cannot allocate its nodes)
-        using failing_map = std::map<std::string, json_view, std::less<std::string>, failing_allocator<std::pair<const std::string, json_view>>>;
+        using failing_map = std::map<std::string, json_view, std::less<std::string>, failing_allocator<std::pair<const std::string, json_view>>>; // NOLINT(modernize-use-transparent-functors)
         const json_document one_member = json_document::parse(R"({"a":1})");
         CHECK_THROWS_AS(one_member.root().get<failing_map>(), std::bad_alloc&);
 #endif

tests/src/unit-locale-cpp.cpp

@@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point")
 
         // a value the locale-independent paths convert is not affected
         CHECK(json::parse("12.5") == 12.5);
+
+        // dump() of a long double that takes the snprintf path replaces the
+        // whole decimal point, not only its first byte (#5993)
+        CHECK(long_double_json(1.5L).dump() == "1.5");
+        CHECK(long_double_json(2.0L).dump() == "2.0");
+        CHECK(long_double_json(-0.25L).dump() == "-0.25");
+        CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L));
+        CHECK(long_double_json::parse(ld.dump()) == expected_long_double);
     }
     if (!tested)
     {
@@ -525,7 +533,7 @@ struct locale_switching_token
         {
             switched = std::setlocale(LC_NUMERIC, switch_to.c_str()) != nullptr;
         }
-        return &text[0];
+        return &text[0]; // NOLINT(readability-container-data-pointer): std::string::data() is const before C++17
     }
 
     std::string text;
@@ -647,7 +655,7 @@ TEST_CASE("convert_float_locale_aware")
 
             // "C" -> comma locale: strtod stops at the '.', so the conversion is repeated
             {
-                locale_switching_token token("1.5", comma_locale.c_str());
+                locale_switching_token token("1.5", comma_locale);
                 double value = 0;
                 nlohmann::detail::convert_float_locale_aware(token, 1, value);
                 if (token.switched)
@@ -660,7 +668,7 @@ TEST_CASE("convert_float_locale_aware")
             // the same for the other floating-point types
             REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr);
             {
-                locale_switching_token token("2.25", comma_locale.c_str());
+                locale_switching_token token("2.25", comma_locale);
                 float value = 0;
                 nlohmann::detail::convert_float_locale_aware(token, 1, value);
                 if (token.switched)
@@ -670,7 +678,7 @@ TEST_CASE("convert_float_locale_aware")
             }
             REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr);
             {
-                locale_switching_token token("2.25", comma_locale.c_str());
+                locale_switching_token token("2.25", comma_locale);
                 long double value = 0;
                 nlohmann::detail::convert_float_locale_aware(token, 1, value);
                 if (token.switched)

tests/src/unit-msgpack-cpp17.cpp

@@ -122,7 +122,8 @@ TEST_CASE("JSON text with std::byte")
     SECTION("invalid text")
     {
         CHECK_FALSE(json::accept(to_bytes("[1,\n2,")));
-        CHECK_THROWS_WITH_AS(json::parse(to_bytes("[1,\n2,")),
+        json _;
+        CHECK_THROWS_WITH_AS(_ = json::parse(to_bytes("[1,\n2,")),
                              "[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&);
     }
 }

tests/src/unit-ordered_json.cpp

@@ -541,6 +541,7 @@ TEST_CASE("object_key_index")
         };
 
         std::vector<std::size_t> hashes;
+        hashes.reserve(initial);
         for (std::size_t i = 0; i < initial; ++i)
         {
             hashes.push_back(hash_of(i));

tests/src/unit-regression1.cpp

@@ -1533,7 +1533,7 @@ TEST_CASE("regression tests 1")
         {
             my_json::object_t top = orig.get_ref<const my_json::object_t&>();
             top.at("x") = my_json::parse(R"({"a":{"b":1},"n":null,"arr":[10,20],"objs":[{"m":5},{"m":6}]})");
-            return my_json(std::move(top));
+            return my_json(std::move(top)); // NOLINT(modernize-return-braced-init-list): {top} would be an array
         };
 
         SECTION("existing members")

tests/src/unit-stream-bulk-scan.cpp

@@ -120,6 +120,7 @@ class oversized_get_area_streambuf : public std::streambuf
   private:
     std::array<char, 8> m_data{{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}};
 };
+
 #endif
 
 // how a test reads from the stream
@@ -463,15 +464,18 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk")
         CHECK(token.empty());
     }
 
+#if SIZE_MAX > UINT32_MAX
+    // a get area larger than an int needs a 64-bit std::size_t (checked by the
+    // preprocessor, as a constant condition triggers MSVC's C4127)
     SECTION("skipping more characters than gbump() takes at once")
     {
         // gbump() takes an int, but a get area can be larger
-        const std::size_t claimed = sizeof(std::size_t) < 8 ? 0 : std::size_t{3} << 30u;
+        const std::size_t claimed = std::size_t{3} << 30u;
         oversized_get_area_streambuf buf(claimed);
-        if (claimed == 0 || static_cast<std::size_t>(buf.in_avail()) != claimed)
+        if (static_cast<std::size_t>(buf.in_avail()) != claimed)
         {
-            // a 32-bit size_t, or a standard library whose streambuf stores
+            // a standard library whose streambuf stores the size of the get
-            // the size of the get area in an int (MSVC)
+            // area in an int (MSVC)
             MESSAGE("a get area larger than an int is not supported");
         }
         else
@@ -491,6 +495,7 @@ TEST_CASE("parsing from a std::istream scans the get area in bulk")
             buf.restore_get_area();
         }
     }
+#endif
 #endif
 
     SECTION("SAX parsing from a stream")