From 3720e4b3aa6c7f6cadb075df69fd765a8a020727 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 22:48:42 +0100 Subject: [PATCH] Replace a multibyte decimal point in dump() of a long double (#5993) The snprintf path of dump_float(), used when number_float_t is neither IEEE single nor double (a long double on x86-64 and aarch64 Linux), read localeconv()->decimal_point and ->thousands_sep as single chars. Under a locale with a multibyte decimal point (glibc's ps_AF: U+066B), it removed the first byte as a "thousands separator" and left the second one in the output, so 1.5L dumped as "1\xab5.0". Find the whole decimal point with std::search and replace it with '.', as the parser already does since #5660. Drop the thousands-separator removal: %g never groups digits. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/output/serializer.hpp | 39 +++++++++---------- single_include/nlohmann/json.hpp | 39 +++++++++---------- tests/src/unit-locale-cpp.cpp | 8 ++++ 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 4656dbdf1..597bb7f7d 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -8,14 +8,14 @@ #pragma once -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf -#include // memcpy, memset +#include // memcpy, memset, strlen #include // next #include // numeric_limits #include // string, char_traits @@ -2095,31 +2095,30 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see lexer::get_decimal_point()); it is a + // string, which can be longer than one byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::strlen(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (decimal_point_len != 0 && !(decimal_point_len == 1 && decimal_point[0] == '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + // move the fraction and exponent behind the '.' (a no-op for a one-byte decimal point) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 693320ed5..056596771 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -33473,14 +33473,14 @@ NLOHMANN_JSON_NAMESPACE_END -#include // remove, fill, find, none_of, min +#include // copy, none_of, min, search #include // array #include // localeconv, lconv #include // isfinite #include // size_t, ptrdiff_t #include // uint8_t #include // snprintf -#include // memcpy, memset +#include // memcpy, memset, strlen #include // next #include // numeric_limits #include // string, char_traits @@ -37342,31 +37342,30 @@ class serializer // check if the buffer was large enough JSON_ASSERT(static_cast(len) < number_buffer.size()); - // look up the locale's thousands separator and decimal point now, - // matching what snprintf_float() just used (see lexer::get_decimal_point()) + // look up the locale's decimal point now, matching what + // snprintf_float() just used (see lexer::get_decimal_point()); it is a + // string, which can be longer than one byte (U+066B in ps_AF, #5993). + // A thousands separator needs no handling: %g never groups digits. const auto* loc = std::localeconv(); JSON_ASSERT(loc != nullptr); - const char thousands_sep = (loc->thousands_sep == nullptr) ? '\0' : *loc->thousands_sep; - const char decimal_point = (loc->decimal_point == nullptr) ? '\0' : *loc->decimal_point; - - // erase thousands separators - if (thousands_sep != '\0') - { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - JSON_ASSERT((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } + const char* const decimal_point = (loc->decimal_point == nullptr) ? "" : loc->decimal_point; + const std::size_t decimal_point_len = std::strlen(decimal_point); // convert decimal point to '.' - if (decimal_point != '\0' && decimal_point != '.') + if (decimal_point_len != 0 && !(decimal_point_len == 1 && decimal_point[0] == '.')) { - // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) + const auto first = number_buffer.begin(); + const auto last = first + len; + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::search returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::search(first, last, decimal_point, decimal_point + decimal_point_len); + if (dec_pos != last) { *dec_pos = '.'; + // move the fraction and exponent behind the '.' (a no-op for a one-byte decimal point) + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::copy returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::copy(dec_pos + static_cast(decimal_point_len), last, dec_pos + 1); + *end = '\0'; + len = end - first; } } diff --git a/tests/src/unit-locale-cpp.cpp b/tests/src/unit-locale-cpp.cpp index b665b6531..d3f86d892 100644 --- a/tests/src/unit-locale-cpp.cpp +++ b/tests/src/unit-locale-cpp.cpp @@ -397,6 +397,14 @@ TEST_CASE("locale with a multi-byte decimal point") // a value the locale-independent paths convert is not affected CHECK(json::parse("12.5") == 12.5); + + // dump() of a long double that takes the snprintf path replaces the + // whole decimal point, not only its first byte (#5993) + CHECK(long_double_json(1.5L).dump() == "1.5"); + CHECK(long_double_json(2.0L).dump() == "2.0"); + CHECK(long_double_json(-0.25L).dump() == "-0.25"); + CHECK(long_double_json::parse(long_double_json(1.5e-7L).dump()) == long_double_json(1.5e-7L)); + CHECK(long_double_json::parse(ld.dump()) == expected_long_double); } if (!tested) {