wt/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-15 10:29:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Restore unflatten() speed after the linear-time rewrite

Browse Source 
The prefix tree introduced for linear unflatten() kept a std::map per prefix,
copied each token into it and looked every token up a second time while
creating the values. Replace the per-prefix maps by one open-addressing table
keyed by (parent, token) that refers to the parsed tokens, skip the lookups for
the prefix shared with the previous key (flattened objects are mostly sorted),
and derive the prefix numbers of the second pass from parent links instead of
lookups.

Time and memory stay linear in the pointer depth, and the result is still
independent of the iteration order of the flattened object.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-15 10:52:34 +02:00
1 parent 693f7a1df1
commit 6e051c3bfb
2 files changed
+278 -100
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/json_pointer.hpp
+139 -50
View File
@@ -16,7 +16,6 @@
#include <iosfwd> // ostream
#endif // JSON_NO_IO
#include <limits> // max
#include <map> // map
#include <string> // string
#include <utility> // move
#include <vector> // vector
@@ -499,21 +498,73 @@ class json_pointer
the result independent of the order in which the flattened object is
iterated, which is unspecified for some object types.
The prefixes form a tree and are numbered, so each of them is stored only
once (as a node) rather than as a copy of all of its reference tokens.
The prefixes form a tree and are numbered (number 0 is the empty prefix),
so each of them is stored only once (as a node) rather than as a copy of
all of its reference tokens. The tree does not own the reference tokens:
it refers to the tokens of the parsed keys, which must outlive it.
*/
struct prefix_tree
{
// children[id] maps a reference token to the number of the prefix
// extended by that token; number 0 is the empty prefix
std::vector<std::map<string_t, std::size_t>> children;
// open-addressing hash table from (parent, last reference token) to
// the number of the node; empty slots have token == nullptr
struct slot
{
std::size_t hash;
std::size_t parent;
const string_t* token;
std::size_t id;
slot() noexcept
: hash(0), parent(0), token(nullptr), id(0)
{}
slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept
: hash(h), parent(p), token(t), id(i)
{}
};
std::vector<slot> slots;
std::size_t used = 0;
static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept
{
std::size_t h = parent * static_cast<std::size_t>(0x9E3779B97F4A7C15ULL);
for (std::size_t i = 0; i < token.size(); ++i)
{
h = (h ^ static_cast<unsigned char>(token[i])) * static_cast<std::size_t>(0x100000001B3ULL);
}
return h ^ (h >> 29u);
}
void grow()
{
std::vector<slot> old(slots.size() * 2);
old.swap(slots);
const std::size_t mask = slots.size() - 1;
for (const auto& s : old)
{
if (s.token != nullptr)
{
std::size_t i = s.hash & mask;
while (slots[i].token != nullptr)
{
i = (i + 1) & mask;
}
slots[i] = s;
}
}
}
// parent[id] is the number of the prefix without its last token
std::vector<std::size_t> parent;
// is_array[id] is true iff some flattened key has the reference token
// 0 directly below the prefix with number id
std::vector<bool> is_array;
// start with the empty prefix only
prefix_tree()
: children(1)
: slots(16)
, parent(1, 0)
, is_array(1, false)
{}
@@ -521,65 +572,67 @@ class json_pointer
// reference_token, adding it if it is new
std::size_t add_child(std::size_t id, const string_t& reference_token)
{
const std::size_t hash = hash_of(id, reference_token);
const std::size_t mask = slots.size() - 1;
std::size_t i = hash & mask;
while (slots[i].token != nullptr)
{
if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token)
{
return slots[i].id;
}
i = (i + 1) & mask;
}
// new prefix
const std::size_t child = parent.size();
slots[i] = slot(hash, id, &reference_token, child);
if (reference_token == "0")
{
is_array[id] = true;
}
// shared prefixes are found without copying the token
const auto found = children[id].find(reference_token);
if (found != children[id].end())
{
return found->second;
}
// read the number before the emplace_back below, which may
// reallocate children and invalidate the iterator
const std::size_t child = children.size();
children[id].emplace(reference_token, child);
children.emplace_back();
parent.push_back(id);
is_array.push_back(false);
return child;
}
// return the number of the prefix with number id extended by
// reference_token, which must have been added before
std::size_t find_child(std::size_t id, const string_t& reference_token) const
{
const auto it = children[id].find(reference_token);
JSON_ASSERT(it != children[id].end());
return it->second;
// keep the load factor below 1/2
if (++used * 2 > slots.size())
{
grow();
}
return child;
}
};
/*!
@brief create and return a reference to the pointed to value
Complexity: Linear in the number of reference tokens (times the logarithm
of the number of siblings for the prefix lookup).
@param[in] j the value to create the pointed to value in
@param[in] tokens the reference tokens of the pointer
@param[in] prefixes prefixes[i] is the number in @a tree of the prefix
consisting of the first i reference tokens
Complexity: Linear in the number of reference tokens.
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if array index is not a number
@throw type_error.313 if value cannot be unflattened
*/
template<typename BasicJsonType>
BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const
static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector<string_t>& tokens,
const std::vector<std::size_t>& prefixes, const prefix_tree& tree)
{
auto* result = &j;
// the number of the prefix consumed so far; used to look up whether
// the value to be created below is an array or an object
std::size_t id = 0;
// in case no reference tokens exist, return a reference to the JSON value
// j which will be overwritten by a primitive value
for (const auto& reference_token : reference_tokens)
for (std::size_t i = 0; i < tokens.size(); ++i)
{
const auto& reference_token = tokens[i];
switch (result->type())
{
case detail::value_t::null:
{
if (tree.is_array[id])
if (tree.is_array[prefixes[i]])
{
// some reference token below this position is 0, so the
// value is an array
@@ -623,8 +676,6 @@ class json_pointer
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j));
}
id = tree.find_child(id, reference_token);
}
return *result;
@@ -1257,25 +1308,53 @@ class json_pointer
BasicJsonType result;
// parse every key once; the reference tokens are kept for the second
// pass (the vectors are never resized, so the tree may refer to them)
std::vector<std::vector<string_t>> keys;
keys.reserve(value.m_data.m_value.object->size());
for (const auto& element : *value.m_data.m_value.object)
{
json_pointer ptr(element.first);
keys.push_back(std::move(ptr.reference_tokens));
}
// collect the pointer prefixes that have a reference token 0 among
// their children; the values below them are arrays, all others are
// objects (see prefix_tree)
// (each key is parsed once and the pointer kept for the second pass)
prefix_tree tree;
std::vector<json_pointer> pointers;
pointers.reserve(value.m_data.m_value.object->size());
for (const auto& element : *value.m_data.m_value.object)
// the number of the prefix with i + 1 tokens of the previous key; keys
// are often sorted, so most of the prefix is the same as for the
// previous key and need not be looked up again
std::vector<std::size_t> path;
const std::vector<string_t>* previous = nullptr;
// the number of the complete prefix (i.e., the key) of each key
std::vector<std::size_t> leaves;
leaves.reserve(keys.size());
for (const auto& tokens : keys)
{
pointers.emplace_back(element.first);
std::size_t id = 0;
for (const auto& reference_token : pointers.back().reference_tokens)
std::size_t common = 0;
if (previous != nullptr)
{
id = tree.add_child(id, reference_token);
const std::size_t limit = (std::min)(tokens.size(), previous->size());
while (common < limit && tokens[common] == (*previous)[common])
{
++common;
}
}
path.resize(common);
std::size_t id = common == 0 ? 0 : path[common - 1];
for (std::size_t i = common; i < tokens.size(); ++i)
{
id = tree.add_child(id, tokens[i]);
path.push_back(id);
}
leaves.push_back(id);
previous = &tokens;
}
// iterate the JSON object values
std::size_t index = 0;
std::vector<std::size_t> prefixes;
std::size_t key_index = 0;
for (const auto& element : *value.m_data.m_value.object)
{
if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))
@@ -1283,11 +1362,21 @@ class json_pointer
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second));
}
const auto& tokens = keys[key_index];
// number the prefixes of this key from its last one to the first
prefixes.resize(tokens.size());
for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i)
{
id = tree.parent[id];
prefixes[i - 1] = id;
}
++key_index;
// Assign the value to the reference pointed to by JSON pointer. Note
// that if the JSON pointer is "" (i.e., points to the whole value),
// function get_and_create returns a reference to the result itself.
// An assignment will then create a primitive value.
pointers[index++].get_and_create(result, tree) = element.second;
get_and_create(result, tokens, prefixes, tree) = element.second;
}
return result;
single_include/nlohmann/json.hpp
Generated
+139 -50
View File
@@ -20642,7 +20642,6 @@ NLOHMANN_JSON_NAMESPACE_END
#include <iosfwd> // ostream
#endif // JSON_NO_IO
#include <limits> // max
#include <map> // map
#include <string> // string
#include <utility> // move
#include <vector> // vector
@@ -21131,21 +21130,73 @@ class json_pointer
the result independent of the order in which the flattened object is
iterated, which is unspecified for some object types.
The prefixes form a tree and are numbered, so each of them is stored only
once (as a node) rather than as a copy of all of its reference tokens.
The prefixes form a tree and are numbered (number 0 is the empty prefix),
so each of them is stored only once (as a node) rather than as a copy of
all of its reference tokens. The tree does not own the reference tokens:
it refers to the tokens of the parsed keys, which must outlive it.
*/
struct prefix_tree
{
// children[id] maps a reference token to the number of the prefix
// extended by that token; number 0 is the empty prefix
std::vector<std::map<string_t, std::size_t>> children;
// open-addressing hash table from (parent, last reference token) to
// the number of the node; empty slots have token == nullptr
struct slot
{
std::size_t hash;
std::size_t parent;
const string_t* token;
std::size_t id;
slot() noexcept
: hash(0), parent(0), token(nullptr), id(0)
{}
slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept
: hash(h), parent(p), token(t), id(i)
{}
};
std::vector<slot> slots;
std::size_t used = 0;
static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept
{
std::size_t h = parent * static_cast<std::size_t>(0x9E3779B97F4A7C15ULL);
for (std::size_t i = 0; i < token.size(); ++i)
{
h = (h ^ static_cast<unsigned char>(token[i])) * static_cast<std::size_t>(0x100000001B3ULL);
}
return h ^ (h >> 29u);
}
void grow()
{
std::vector<slot> old(slots.size() * 2);
old.swap(slots);
const std::size_t mask = slots.size() - 1;
for (const auto& s : old)
{
if (s.token != nullptr)
{
std::size_t i = s.hash & mask;
while (slots[i].token != nullptr)
{
i = (i + 1) & mask;
}
slots[i] = s;
}
}
}
// parent[id] is the number of the prefix without its last token
std::vector<std::size_t> parent;
// is_array[id] is true iff some flattened key has the reference token
// 0 directly below the prefix with number id
std::vector<bool> is_array;
// start with the empty prefix only
prefix_tree()
: children(1)
: slots(16)
, parent(1, 0)
, is_array(1, false)
{}
@@ -21153,65 +21204,67 @@ class json_pointer
// reference_token, adding it if it is new
std::size_t add_child(std::size_t id, const string_t& reference_token)
{
const std::size_t hash = hash_of(id, reference_token);
const std::size_t mask = slots.size() - 1;
std::size_t i = hash & mask;
while (slots[i].token != nullptr)
{
if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token)
{
return slots[i].id;
}
i = (i + 1) & mask;
}
// new prefix
const std::size_t child = parent.size();
slots[i] = slot(hash, id, &reference_token, child);
if (reference_token == "0")
{
is_array[id] = true;
}
// shared prefixes are found without copying the token
const auto found = children[id].find(reference_token);
if (found != children[id].end())
{
return found->second;
}
// read the number before the emplace_back below, which may
// reallocate children and invalidate the iterator
const std::size_t child = children.size();
children[id].emplace(reference_token, child);
children.emplace_back();
parent.push_back(id);
is_array.push_back(false);
return child;
}
// return the number of the prefix with number id extended by
// reference_token, which must have been added before
std::size_t find_child(std::size_t id, const string_t& reference_token) const
{
const auto it = children[id].find(reference_token);
JSON_ASSERT(it != children[id].end());
return it->second;
// keep the load factor below 1/2
if (++used * 2 > slots.size())
{
grow();
}
return child;
}
};
/*!
@brief create and return a reference to the pointed to value
Complexity: Linear in the number of reference tokens (times the logarithm
of the number of siblings for the prefix lookup).
@param[in] j the value to create the pointed to value in
@param[in] tokens the reference tokens of the pointer
@param[in] prefixes prefixes[i] is the number in @a tree of the prefix
consisting of the first i reference tokens
Complexity: Linear in the number of reference tokens.
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if array index is not a number
@throw type_error.313 if value cannot be unflattened
*/
template<typename BasicJsonType>
BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const
static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector<string_t>& tokens,
const std::vector<std::size_t>& prefixes, const prefix_tree& tree)
{
auto* result = &j;
// the number of the prefix consumed so far; used to look up whether
// the value to be created below is an array or an object
std::size_t id = 0;
// in case no reference tokens exist, return a reference to the JSON value
// j which will be overwritten by a primitive value
for (const auto& reference_token : reference_tokens)
for (std::size_t i = 0; i < tokens.size(); ++i)
{
const auto& reference_token = tokens[i];
switch (result->type())
{
case detail::value_t::null:
{
if (tree.is_array[id])
if (tree.is_array[prefixes[i]])
{
// some reference token below this position is 0, so the
// value is an array
@@ -21255,8 +21308,6 @@ class json_pointer
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j));
}
id = tree.find_child(id, reference_token);
}
return *result;
@@ -21889,25 +21940,53 @@ class json_pointer
BasicJsonType result;
// parse every key once; the reference tokens are kept for the second
// pass (the vectors are never resized, so the tree may refer to them)
std::vector<std::vector<string_t>> keys;
keys.reserve(value.m_data.m_value.object->size());
for (const auto& element : *value.m_data.m_value.object)
{
json_pointer ptr(element.first);
keys.push_back(std::move(ptr.reference_tokens));
}
// collect the pointer prefixes that have a reference token 0 among
// their children; the values below them are arrays, all others are
// objects (see prefix_tree)
// (each key is parsed once and the pointer kept for the second pass)
prefix_tree tree;
std::vector<json_pointer> pointers;
pointers.reserve(value.m_data.m_value.object->size());
for (const auto& element : *value.m_data.m_value.object)
// the number of the prefix with i + 1 tokens of the previous key; keys
// are often sorted, so most of the prefix is the same as for the
// previous key and need not be looked up again
std::vector<std::size_t> path;
const std::vector<string_t>* previous = nullptr;
// the number of the complete prefix (i.e., the key) of each key
std::vector<std::size_t> leaves;
leaves.reserve(keys.size());
for (const auto& tokens : keys)
{
pointers.emplace_back(element.first);
std::size_t id = 0;
for (const auto& reference_token : pointers.back().reference_tokens)
std::size_t common = 0;
if (previous != nullptr)
{
id = tree.add_child(id, reference_token);
const std::size_t limit = (std::min)(tokens.size(), previous->size());
while (common < limit && tokens[common] == (*previous)[common])
{
++common;
}
}
path.resize(common);
std::size_t id = common == 0 ? 0 : path[common - 1];
for (std::size_t i = common; i < tokens.size(); ++i)
{
id = tree.add_child(id, tokens[i]);
path.push_back(id);
}
leaves.push_back(id);
previous = &tokens;
}
// iterate the JSON object values
std::size_t index = 0;
std::vector<std::size_t> prefixes;
std::size_t key_index = 0;
for (const auto& element : *value.m_data.m_value.object)
{
if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))
@@ -21915,11 +21994,21 @@ class json_pointer
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second));
}
const auto& tokens = keys[key_index];
// number the prefixes of this key from its last one to the first
prefixes.resize(tokens.size());
for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i)
{
id = tree.parent[id];
prefixes[i - 1] = id;
}
++key_index;
// Assign the value to the reference pointed to by JSON pointer. Note
// that if the JSON pointer is "" (i.e., points to the whole value),
// function get_and_create returns a reference to the result itself.
// An assignment will then create a primitive value.
pointers[index++].get_and_create(result, tree) = element.second;
get_and_create(result, tokens, prefixes, tree) = element.second;
}
return result;