Unflatten in time and memory linear in the pointer depth (#5793)
* Unflatten in time and memory linear in the pointer depth #5443 made unflatten() decide between arrays and objects independently of the iteration order by collecting the pointer prefixes that have a reference token 0 below them in a std::set<std::vector<string_t>>. Every such prefix was stored as a copy of all its reference tokens, and get_and_create() compared whole prefix vectors at every step, so unflattening a pointer of depth d took time and memory quadratic in d: a 10,000-level array pointer took 18 s and 1.3 GB, a 100,000-level one did not finish. The prefixes are now numbered nodes of a tree, so each is stored once and get_and_create() follows the tree token by token. The result is unchanged, including its independence of the iteration order. Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Initialize prefix_tree members to satisfy -Weffc++ Signed-off-by: Niels Lohmann <mail@nlohmann.me> * Move prefix_tree setup and child insertion into member functions The constructor now creates the root node, add_child() inserts a reference token below a prefix and returns the child's number, and find_child() looks one up for get_and_create(). Signed-off-by: Niels Lohmann <mail@nlohmann.me> --------- Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
1 parent
44e8597701
commit
2913e96433
3 files changed
+173
-50
No files matched your search
@@ -16,8 +16,8 @@
|
|||||||
#include <iosfwd> // ostream
|
#include <iosfwd> // ostream
|
||||||
#endif // JSON_NO_IO
|
#endif // JSON_NO_IO
|
||||||
#include <limits> // max
|
#include <limits> // max
|
||||||
|
#include <map> // map
|
||||||
#include <numeric> // accumulate
|
#include <numeric> // accumulate
|
||||||
#include <set> // set
|
|
||||||
#include <string> // string
|
#include <string> // string
|
||||||
#include <utility> // move
|
#include <utility> // move
|
||||||
#include <vector> // vector
|
#include <vector> // vector
|
||||||
@@ -359,33 +359,82 @@ class json_pointer
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
/*!
|
/*!
|
||||||
@brief the reference token sequences that denote arrays
|
@brief the pointer prefixes of a flattened object, and which of them denote arrays
|
||||||
|
|
||||||
@ref unflatten collects the pointer prefixes that have a reference token 0
|
@ref unflatten collects the pointer prefixes that have a reference token 0
|
||||||
among their children; @ref get_and_create creates arrays exactly below
|
among their children; @ref get_and_create creates arrays exactly below
|
||||||
those prefixes and objects everywhere else. Deciding this up front keeps
|
those prefixes and objects everywhere else. Deciding this up front keeps
|
||||||
the result independent of the order in which the flattened object is
|
the result independent of the order in which the flattened object is
|
||||||
iterated, which is unspecified for some object types.
|
iterated, which is unspecified for some object types.
|
||||||
|
|
||||||
|
The prefixes form a tree and are numbered, so each of them is stored only
|
||||||
|
once (as a node) rather than as a copy of all of its reference tokens.
|
||||||
*/
|
*/
|
||||||
using array_parents_t = std::set<std::vector<string_t>>;
|
struct prefix_tree
|
||||||
|
{
|
||||||
|
// children[id] maps a reference token to the number of the prefix
|
||||||
|
// extended by that token; number 0 is the empty prefix
|
||||||
|
std::vector<std::map<string_t, std::size_t>> children;
|
||||||
|
// is_array[id] is true iff some flattened key has the reference token
|
||||||
|
// 0 directly below the prefix with number id
|
||||||
|
std::vector<bool> is_array;
|
||||||
|
|
||||||
|
// start with the empty prefix only
|
||||||
|
prefix_tree()
|
||||||
|
: children(1)
|
||||||
|
, is_array(1, false)
|
||||||
|
{}
|
||||||
|
|
||||||
|
// return the number of the prefix with number id extended by
|
||||||
|
// reference_token, adding it if it is new
|
||||||
|
std::size_t add_child(std::size_t id, string_t&& reference_token)
|
||||||
|
{
|
||||||
|
if (reference_token == "0")
|
||||||
|
{
|
||||||
|
is_array[id] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// read the number before the emplace_back below, which may
|
||||||
|
// reallocate children and invalidate the iterator
|
||||||
|
const std::size_t next = children.size();
|
||||||
|
const auto inserted = children[id].emplace(std::move(reference_token), next);
|
||||||
|
const std::size_t child = inserted.first->second;
|
||||||
|
if (inserted.second)
|
||||||
|
{
|
||||||
|
children.emplace_back();
|
||||||
|
is_array.push_back(false);
|
||||||
|
}
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
// return the number of the prefix with number id extended by
|
||||||
|
// reference_token, which must have been added before
|
||||||
|
std::size_t find_child(std::size_t id, const string_t& reference_token) const
|
||||||
|
{
|
||||||
|
const auto it = children[id].find(reference_token);
|
||||||
|
JSON_ASSERT(it != children[id].end());
|
||||||
|
return it->second;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief create and return a reference to the pointed to value
|
@brief create and return a reference to the pointed to value
|
||||||
|
|
||||||
Complexity: Linear in the number of reference tokens.
|
Complexity: Linear in the number of reference tokens (times the logarithm
|
||||||
|
of the number of siblings for the prefix lookup).
|
||||||
|
|
||||||
@throw parse_error.106 if an array index begins with '0'
|
@throw parse_error.106 if an array index begins with '0'
|
||||||
@throw parse_error.109 if array index is not a number
|
@throw parse_error.109 if array index is not a number
|
||||||
@throw type_error.313 if value cannot be unflattened
|
@throw type_error.313 if value cannot be unflattened
|
||||||
*/
|
*/
|
||||||
template<typename BasicJsonType>
|
template<typename BasicJsonType>
|
||||||
BasicJsonType& get_and_create(BasicJsonType& j, const array_parents_t& array_parents) const
|
BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const
|
||||||
{
|
{
|
||||||
auto* result = &j;
|
auto* result = &j;
|
||||||
|
|
||||||
// the reference tokens that have been consumed so far; used to look up
|
// the number of the prefix consumed so far; used to look up whether
|
||||||
// whether the value to be created below is an array or an object
|
// the value to be created below is an array or an object
|
||||||
std::vector<string_t> prefix;
|
std::size_t id = 0;
|
||||||
|
|
||||||
// in case no reference tokens exist, return a reference to the JSON value
|
// in case no reference tokens exist, return a reference to the JSON value
|
||||||
// j which will be overwritten by a primitive value
|
// j which will be overwritten by a primitive value
|
||||||
@@ -395,7 +444,7 @@ class json_pointer
|
|||||||
{
|
{
|
||||||
case detail::value_t::null:
|
case detail::value_t::null:
|
||||||
{
|
{
|
||||||
if (array_parents.find(prefix) != array_parents.end())
|
if (tree.is_array[id])
|
||||||
{
|
{
|
||||||
// some reference token below this position is 0, so the
|
// some reference token below this position is 0, so the
|
||||||
// value is an array
|
// value is an array
|
||||||
@@ -440,7 +489,7 @@ class json_pointer
|
|||||||
JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j));
|
JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j));
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix.push_back(reference_token);
|
id = tree.find_child(id, reference_token);
|
||||||
}
|
}
|
||||||
|
|
||||||
return *result;
|
return *result;
|
||||||
@@ -1030,19 +1079,15 @@ class json_pointer
|
|||||||
|
|
||||||
// collect the pointer prefixes that have a reference token 0 among
|
// collect the pointer prefixes that have a reference token 0 among
|
||||||
// their children; the values below them are arrays, all others are
|
// their children; the values below them are arrays, all others are
|
||||||
// objects (see array_parents_t)
|
// objects (see prefix_tree)
|
||||||
array_parents_t array_parents;
|
prefix_tree tree;
|
||||||
for (const auto& element : *value.m_data.m_value.object)
|
for (const auto& element : *value.m_data.m_value.object)
|
||||||
{
|
{
|
||||||
json_pointer ptr(element.first);
|
json_pointer ptr(element.first);
|
||||||
std::vector<string_t> prefix;
|
std::size_t id = 0;
|
||||||
for (auto& reference_token : ptr.reference_tokens)
|
for (auto& reference_token : ptr.reference_tokens)
|
||||||
{
|
{
|
||||||
if (reference_token == "0")
|
id = tree.add_child(id, std::move(reference_token));
|
||||||
{
|
|
||||||
array_parents.insert(prefix);
|
|
||||||
}
|
|
||||||
prefix.push_back(std::move(reference_token));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1058,7 +1103,7 @@ class json_pointer
|
|||||||
// that if the JSON pointer is "" (i.e., points to the whole value),
|
// that if the JSON pointer is "" (i.e., points to the whole value),
|
||||||
// function get_and_create returns a reference to the result itself.
|
// function get_and_create returns a reference to the result itself.
|
||||||
// An assignment will then create a primitive value.
|
// An assignment will then create a primitive value.
|
||||||
json_pointer(element.first).get_and_create(result, array_parents) = element.second;
|
json_pointer(element.first).get_and_create(result, tree) = element.second;
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
|
|||||||
@@ -19928,8 +19928,8 @@ NLOHMANN_JSON_NAMESPACE_END
|
|||||||
#include <iosfwd> // ostream
|
#include <iosfwd> // ostream
|
||||||
#endif // JSON_NO_IO
|
#endif // JSON_NO_IO
|
||||||
#include <limits> // max
|
#include <limits> // max
|
||||||
|
#include <map> // map
|
||||||
#include <numeric> // accumulate
|
#include <numeric> // accumulate
|
||||||
#include <set> // set
|
|
||||||
#include <string> // string
|
#include <string> // string
|
||||||
#include <utility> // move
|
#include <utility> // move
|
||||||
#include <vector> // vector
|
#include <vector> // vector
|
||||||
@@ -20277,33 +20277,82 @@ class json_pointer
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
/*!
|
/*!
|
||||||
@brief the reference token sequences that denote arrays
|
@brief the pointer prefixes of a flattened object, and which of them denote arrays
|
||||||
|
|
||||||
@ref unflatten collects the pointer prefixes that have a reference token 0
|
@ref unflatten collects the pointer prefixes that have a reference token 0
|
||||||
among their children; @ref get_and_create creates arrays exactly below
|
among their children; @ref get_and_create creates arrays exactly below
|
||||||
those prefixes and objects everywhere else. Deciding this up front keeps
|
those prefixes and objects everywhere else. Deciding this up front keeps
|
||||||
the result independent of the order in which the flattened object is
|
the result independent of the order in which the flattened object is
|
||||||
iterated, which is unspecified for some object types.
|
iterated, which is unspecified for some object types.
|
||||||
|
|
||||||
|
The prefixes form a tree and are numbered, so each of them is stored only
|
||||||
|
once (as a node) rather than as a copy of all of its reference tokens.
|
||||||
*/
|
*/
|
||||||
using array_parents_t = std::set<std::vector<string_t>>;
|
struct prefix_tree
|
||||||
|
{
|
||||||
|
// children[id] maps a reference token to the number of the prefix
|
||||||
|
// extended by that token; number 0 is the empty prefix
|
||||||
|
std::vector<std::map<string_t, std::size_t>> children;
|
||||||
|
// is_array[id] is true iff some flattened key has the reference token
|
||||||
|
// 0 directly below the prefix with number id
|
||||||
|
std::vector<bool> is_array;
|
||||||
|
|
||||||
|
// start with the empty prefix only
|
||||||
|
prefix_tree()
|
||||||
|
: children(1)
|
||||||
|
, is_array(1, false)
|
||||||
|
{}
|
||||||
|
|
||||||
|
// return the number of the prefix with number id extended by
|
||||||
|
// reference_token, adding it if it is new
|
||||||
|
std::size_t add_child(std::size_t id, string_t&& reference_token)
|
||||||
|
{
|
||||||
|
if (reference_token == "0")
|
||||||
|
{
|
||||||
|
is_array[id] = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// read the number before the emplace_back below, which may
|
||||||
|
// reallocate children and invalidate the iterator
|
||||||
|
const std::size_t next = children.size();
|
||||||
|
const auto inserted = children[id].emplace(std::move(reference_token), next);
|
||||||
|
const std::size_t child = inserted.first->second;
|
||||||
|
if (inserted.second)
|
||||||
|
{
|
||||||
|
children.emplace_back();
|
||||||
|
is_array.push_back(false);
|
||||||
|
}
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
|
||||||
|
// return the number of the prefix with number id extended by
|
||||||
|
// reference_token, which must have been added before
|
||||||
|
std::size_t find_child(std::size_t id, const string_t& reference_token) const
|
||||||
|
{
|
||||||
|
const auto it = children[id].find(reference_token);
|
||||||
|
JSON_ASSERT(it != children[id].end());
|
||||||
|
return it->second;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief create and return a reference to the pointed to value
|
@brief create and return a reference to the pointed to value
|
||||||
|
|
||||||
Complexity: Linear in the number of reference tokens.
|
Complexity: Linear in the number of reference tokens (times the logarithm
|
||||||
|
of the number of siblings for the prefix lookup).
|
||||||
|
|
||||||
@throw parse_error.106 if an array index begins with '0'
|
@throw parse_error.106 if an array index begins with '0'
|
||||||
@throw parse_error.109 if array index is not a number
|
@throw parse_error.109 if array index is not a number
|
||||||
@throw type_error.313 if value cannot be unflattened
|
@throw type_error.313 if value cannot be unflattened
|
||||||
*/
|
*/
|
||||||
template<typename BasicJsonType>
|
template<typename BasicJsonType>
|
||||||
BasicJsonType& get_and_create(BasicJsonType& j, const array_parents_t& array_parents) const
|
BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const
|
||||||
{
|
{
|
||||||
auto* result = &j;
|
auto* result = &j;
|
||||||
|
|
||||||
// the reference tokens that have been consumed so far; used to look up
|
// the number of the prefix consumed so far; used to look up whether
|
||||||
// whether the value to be created below is an array or an object
|
// the value to be created below is an array or an object
|
||||||
std::vector<string_t> prefix;
|
std::size_t id = 0;
|
||||||
|
|
||||||
// in case no reference tokens exist, return a reference to the JSON value
|
// in case no reference tokens exist, return a reference to the JSON value
|
||||||
// j which will be overwritten by a primitive value
|
// j which will be overwritten by a primitive value
|
||||||
@@ -20313,7 +20362,7 @@ class json_pointer
|
|||||||
{
|
{
|
||||||
case detail::value_t::null:
|
case detail::value_t::null:
|
||||||
{
|
{
|
||||||
if (array_parents.find(prefix) != array_parents.end())
|
if (tree.is_array[id])
|
||||||
{
|
{
|
||||||
// some reference token below this position is 0, so the
|
// some reference token below this position is 0, so the
|
||||||
// value is an array
|
// value is an array
|
||||||
@@ -20358,7 +20407,7 @@ class json_pointer
|
|||||||
JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j));
|
JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j));
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix.push_back(reference_token);
|
id = tree.find_child(id, reference_token);
|
||||||
}
|
}
|
||||||
|
|
||||||
return *result;
|
return *result;
|
||||||
@@ -20948,19 +20997,15 @@ class json_pointer
|
|||||||
|
|
||||||
// collect the pointer prefixes that have a reference token 0 among
|
// collect the pointer prefixes that have a reference token 0 among
|
||||||
// their children; the values below them are arrays, all others are
|
// their children; the values below them are arrays, all others are
|
||||||
// objects (see array_parents_t)
|
// objects (see prefix_tree)
|
||||||
array_parents_t array_parents;
|
prefix_tree tree;
|
||||||
for (const auto& element : *value.m_data.m_value.object)
|
for (const auto& element : *value.m_data.m_value.object)
|
||||||
{
|
{
|
||||||
json_pointer ptr(element.first);
|
json_pointer ptr(element.first);
|
||||||
std::vector<string_t> prefix;
|
std::size_t id = 0;
|
||||||
for (auto& reference_token : ptr.reference_tokens)
|
for (auto& reference_token : ptr.reference_tokens)
|
||||||
{
|
{
|
||||||
if (reference_token == "0")
|
id = tree.add_child(id, std::move(reference_token));
|
||||||
{
|
|
||||||
array_parents.insert(prefix);
|
|
||||||
}
|
|
||||||
prefix.push_back(std::move(reference_token));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -20976,7 +21021,7 @@ class json_pointer
|
|||||||
// that if the JSON pointer is "" (i.e., points to the whole value),
|
// that if the JSON pointer is "" (i.e., points to the whole value),
|
||||||
// function get_and_create returns a reference to the result itself.
|
// function get_and_create returns a reference to the result itself.
|
||||||
// An assignment will then create a primitive value.
|
// An assignment will then create a primitive value.
|
||||||
json_pointer(element.first).get_and_create(result, array_parents) = element.second;
|
json_pointer(element.first).get_and_create(result, tree) = element.second;
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
|
|||||||
@@ -969,21 +969,54 @@ TEST_CASE("flatten of structured values")
|
|||||||
CHECK(flat.begin().key() == path);
|
CHECK(flat.begin().key() == path);
|
||||||
CHECK(flat.begin().value() == 0);
|
CHECK(flat.begin().value() == 0);
|
||||||
|
|
||||||
// unflatten() is not iterative: it takes time and memory
|
// unflatten() is linear in the depth, so the value roundtrips
|
||||||
// quadratic in the depth, so it is only roundtripped for a
|
CHECK(flat.unflatten() == value);
|
||||||
// moderate depth
|
|
||||||
std::string small_text;
|
|
||||||
for (std::size_t i = 0; i < 500; ++i)
|
|
||||||
{
|
|
||||||
small_text += objects ? "{\"a\":" : "[";
|
|
||||||
}
|
|
||||||
small_text += "0";
|
|
||||||
small_text += std::string(500, objects ? '}' : ']');
|
|
||||||
const auto small_value = json::parse(small_text);
|
|
||||||
CHECK(small_value.flatten().unflatten() == small_value);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SECTION("unflatten of a deeply nested pointer")
|
||||||
|
{
|
||||||
|
const std::size_t depth = 100000;
|
||||||
|
for (const bool objects :
|
||||||
|
{
|
||||||
|
false, true
|
||||||
|
})
|
||||||
|
{
|
||||||
|
CAPTURE(objects)
|
||||||
|
std::string path;
|
||||||
|
for (std::size_t i = 0; i < depth; ++i)
|
||||||
|
{
|
||||||
|
path += objects ? "/a" : "/0";
|
||||||
|
}
|
||||||
|
|
||||||
|
json flat = json::object();
|
||||||
|
flat[path] = 1;
|
||||||
|
const json value = flat.unflatten();
|
||||||
|
|
||||||
|
// walk down iteratively
|
||||||
|
std::size_t levels = 0;
|
||||||
|
const json* current = &value;
|
||||||
|
while (objects ? current->is_object() : current->is_array())
|
||||||
|
{
|
||||||
|
REQUIRE(current->size() == 1);
|
||||||
|
current = objects ? ¤t->at("a") : ¤t->at(0);
|
||||||
|
++levels;
|
||||||
|
}
|
||||||
|
CHECK(levels == depth);
|
||||||
|
CHECK(*current == 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SECTION("unflatten does not depend on the iteration order")
|
||||||
|
{
|
||||||
|
// the "0" key comes after its sibling in iteration order
|
||||||
|
const nlohmann::ordered_json flat_array = nlohmann::ordered_json::parse(R"({"/a/1": 2, "/a/0": 1})");
|
||||||
|
CHECK(flat_array.unflatten() == nlohmann::ordered_json::parse(R"({"a": [1, 2]})"));
|
||||||
|
|
||||||
|
const nlohmann::ordered_json flat_object = nlohmann::ordered_json::parse(R"({"/b/1": 2})");
|
||||||
|
CHECK(flat_object.unflatten() == nlohmann::ordered_json::parse(R"({"b": {"1": 2}})"));
|
||||||
|
}
|
||||||
|
|
||||||
SECTION("objects and arrays interleaved")
|
SECTION("objects and arrays interleaved")
|
||||||
{
|
{
|
||||||
const json value =
|
const json value =
|
||||||
|
|||||||
Reference in new issue
Block a user