wt/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-22 11:36:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Write the elements of a BJData ND-array in blocks

Browse Source 
An ND-array annotation object is checked and then written from its _ArrayData_
in three passes. Every element went through basic_json::get<>() twice or three
times (the range check of the dry run, the conversion for the write) and
through write_number() with a std::array and a sink call of its own, with a
switch on the number kind for each.

The range check and the conversion read the value directly (the elements are
known to be integers or floats by then), and the real pass switches on the
number kind once and encodes its elements in blocks into space reserved from
the sink.

to_bjdata into a std::vector, instructions per call (Linux, GCC 13, thousands): an ND-array of 120000 doubles 14588 -> 9975, an ND-array of 120000 uint8 16583 -> 8790, an ND-array of 120000 doubles with use_size and use_type 14630 -> 10017.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-22 09:29:14 +02:00
1 parent ceb3c92709
commit ff2d59d3f1
2 files changed
+186 -6
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/output/binary_writer.hpp
+93 -3
View File
@@ -3747,8 +3747,8 @@ class binary_writer
static bool bjdata_ndarray_value_in_range(const BasicJsonType& el)
{
return el.is_number_unsigned()
? integer_fits_max<std::uint64_t>(el.m_data.m_value.number_unsigned) && value_in_range_of<TargetType>(el.template get<std::uint64_t>())
: integer_fits<std::int64_t>(el.m_data.m_value.number_integer) && value_in_range_of<TargetType>(el.template get<std::int64_t>());
? integer_fits_max<std::uint64_t>(el.m_data.m_value.number_unsigned) && value_in_range_of<TargetType>(static_cast<std::uint64_t>(el.m_data.m_value.number_unsigned))
: integer_fits<std::int64_t>(el.m_data.m_value.number_integer) && value_in_range_of<TargetType>(static_cast<std::int64_t>(el.m_data.m_value.number_integer));
}
/*!
@@ -3837,7 +3837,7 @@ class binary_writer
*/
bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run)
{
const auto dval = el.template get<double>();
const auto dval = static_cast<double>(el.m_data.m_value.number_float);
if (dry_run)
{
#ifdef __GNUC__
@@ -3861,6 +3861,90 @@ class binary_writer
return true;
}
/*!
@brief write the elements of an ND-array as the number kind @a dtype names
(see @ref write_bjdata_ndarray), after they have been found to be
in range by a dry run of @ref write_bjdata_ndarray_elements
The elements are encoded in blocks into space reserved from the sink.
*/
void write_bjdata_ndarray_values(const BasicJsonType& array_data, const CharType dtype)
{
switch (dtype)
{
case 'U':
case 'C':
case 'B':
write_bjdata_ndarray_block<std::uint8_t>(array_data);
break;
case 'i':
write_bjdata_ndarray_block<std::int8_t>(array_data);
break;
case 'u':
write_bjdata_ndarray_block<std::uint16_t>(array_data);
break;
case 'I':
write_bjdata_ndarray_block<std::int16_t>(array_data);
break;
case 'm':
write_bjdata_ndarray_block<std::uint32_t>(array_data);
break;
case 'l':
write_bjdata_ndarray_block<std::int32_t>(array_data);
break;
case 'M':
write_bjdata_ndarray_block<std::uint64_t>(array_data);
break;
case 'L':
write_bjdata_ndarray_block<std::int64_t>(array_data);
break;
case 'd':
write_bjdata_ndarray_block<float>(array_data);
break;
case 'D':
default:
// 'D' (double) already spans the full range of number_float_t
write_bjdata_ndarray_block<double>(array_data);
break;
}
}
/// the value of an ND-array element that is an integer, as the integer type that @a T is converted from
template<typename T>
static T bjdata_ndarray_integer(const BasicJsonType& el, std::true_type /*is_integral*/) noexcept
{
using storage_type = typename std::conditional<std::is_unsigned<T>::value, std::uint64_t, std::int64_t>::type;
return static_cast<T>(el.is_number_unsigned()
? static_cast<storage_type>(el.m_data.m_value.number_unsigned)
: static_cast<storage_type>(el.m_data.m_value.number_integer));
}
/// the value of an ND-array element that is a float
template<typename T>
static T bjdata_ndarray_integer(const BasicJsonType& el, std::false_type /*is_integral*/) noexcept
{
return static_cast<T>(static_cast<double>(el.m_data.m_value.number_float));
}
/// write the elements of an ND-array as @a T, little-endian, in blocks
template<typename T>
void write_bjdata_ndarray_block(const BasicJsonType& array_data)
{
constexpr std::size_t per_block = max_reserve_size / sizeof(T);
auto it = array_data.m_data.m_value.array->begin();
const auto end = array_data.m_data.m_value.array->end();
while (it != end)
{
CharType* const start = oa.reserve(per_block * sizeof(T));
CharType* out = start;
for (std::size_t n = 0; n < per_block && it != end; ++n, ++it)
{
out = store_number(out, bjdata_ndarray_integer<T>(*it, std::is_integral<T>()), true);
}
oa.commit(static_cast<std::size_t>(out - start));
}
}
/*!
@brief validate or write every element of a BJData ND-array's `_ArrayData_`
@param[in] array_data the `_ArrayData_` array
@@ -3870,6 +3954,12 @@ class binary_writer
*/
bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run)
{
if (!dry_run)
{
write_bjdata_ndarray_values(array_data, dtype);
return true;
}
for (const auto& el : array_data)
{
bool ok = true;
single_include/nlohmann/json.hpp
Generated
+93 -3
View File
@@ -28578,8 +28578,8 @@ class binary_writer
static bool bjdata_ndarray_value_in_range(const BasicJsonType& el)
{
return el.is_number_unsigned()
? integer_fits_max<std::uint64_t>(el.m_data.m_value.number_unsigned) && value_in_range_of<TargetType>(el.template get<std::uint64_t>())
: integer_fits<std::int64_t>(el.m_data.m_value.number_integer) && value_in_range_of<TargetType>(el.template get<std::int64_t>());
? integer_fits_max<std::uint64_t>(el.m_data.m_value.number_unsigned) && value_in_range_of<TargetType>(static_cast<std::uint64_t>(el.m_data.m_value.number_unsigned))
: integer_fits<std::int64_t>(el.m_data.m_value.number_integer) && value_in_range_of<TargetType>(static_cast<std::int64_t>(el.m_data.m_value.number_integer));
}
/*!
@@ -28668,7 +28668,7 @@ class binary_writer
*/
bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run)
{
const auto dval = el.template get<double>();
const auto dval = static_cast<double>(el.m_data.m_value.number_float);
if (dry_run)
{
#ifdef __GNUC__
@@ -28692,6 +28692,90 @@ class binary_writer
return true;
}
/*!
@brief write the elements of an ND-array as the number kind @a dtype names
(see @ref write_bjdata_ndarray), after they have been found to be
in range by a dry run of @ref write_bjdata_ndarray_elements
The elements are encoded in blocks into space reserved from the sink.
*/
void write_bjdata_ndarray_values(const BasicJsonType& array_data, const CharType dtype)
{
switch (dtype)
{
case 'U':
case 'C':
case 'B':
write_bjdata_ndarray_block<std::uint8_t>(array_data);
break;
case 'i':
write_bjdata_ndarray_block<std::int8_t>(array_data);
break;
case 'u':
write_bjdata_ndarray_block<std::uint16_t>(array_data);
break;
case 'I':
write_bjdata_ndarray_block<std::int16_t>(array_data);
break;
case 'm':
write_bjdata_ndarray_block<std::uint32_t>(array_data);
break;
case 'l':
write_bjdata_ndarray_block<std::int32_t>(array_data);
break;
case 'M':
write_bjdata_ndarray_block<std::uint64_t>(array_data);
break;
case 'L':
write_bjdata_ndarray_block<std::int64_t>(array_data);
break;
case 'd':
write_bjdata_ndarray_block<float>(array_data);
break;
case 'D':
default:
// 'D' (double) already spans the full range of number_float_t
write_bjdata_ndarray_block<double>(array_data);
break;
}
}
/// the value of an ND-array element that is an integer, as the integer type that @a T is converted from
template<typename T>
static T bjdata_ndarray_integer(const BasicJsonType& el, std::true_type /*is_integral*/) noexcept
{
using storage_type = typename std::conditional<std::is_unsigned<T>::value, std::uint64_t, std::int64_t>::type;
return static_cast<T>(el.is_number_unsigned()
? static_cast<storage_type>(el.m_data.m_value.number_unsigned)
: static_cast<storage_type>(el.m_data.m_value.number_integer));
}
/// the value of an ND-array element that is a float
template<typename T>
static T bjdata_ndarray_integer(const BasicJsonType& el, std::false_type /*is_integral*/) noexcept
{
return static_cast<T>(static_cast<double>(el.m_data.m_value.number_float));
}
/// write the elements of an ND-array as @a T, little-endian, in blocks
template<typename T>
void write_bjdata_ndarray_block(const BasicJsonType& array_data)
{
constexpr std::size_t per_block = max_reserve_size / sizeof(T);
auto it = array_data.m_data.m_value.array->begin();
const auto end = array_data.m_data.m_value.array->end();
while (it != end)
{
CharType* const start = oa.reserve(per_block * sizeof(T));
CharType* out = start;
for (std::size_t n = 0; n < per_block && it != end; ++n, ++it)
{
out = store_number(out, bjdata_ndarray_integer<T>(*it, std::is_integral<T>()), true);
}
oa.commit(static_cast<std::size_t>(out - start));
}
}
/*!
@brief validate or write every element of a BJData ND-array's `_ArrayData_`
@param[in] array_data the `_ArrayData_` array
@@ -28701,6 +28785,12 @@ class binary_writer
*/
bool write_bjdata_ndarray_elements(const BasicJsonType& array_data, const CharType dtype, const bool dry_run)
{
if (!dry_run)
{
write_bjdata_ndarray_values(array_data, dtype);
return true;
}
for (const auto& el : array_data)
{
bool ok = true;