diff --git a/tests/benchmarks/json_view/README.md b/tests/benchmarks/json_view/README.md index c4a05d35a..02f33d35f 100644 --- a/tests/benchmarks/json_view/README.md +++ b/tests/benchmarks/json_view/README.md @@ -21,6 +21,10 @@ python3 tests/benchmarks/json_view/compare.py --data - The other libraries come from the system: pkg-config, or Homebrew (`brew install yyjson simdjson boost`). With `--download`, pinned releases are downloaded instead and checked against their SHA-256. Without Boost headers (or with `--no-boost`), the Boost.JSON columns are skipped, and the results say so. +- `--aligned` additionally builds and runs everything with all functions aligned to 64 bytes (`-mllvm + -align-all-functions=6` for Clang, `-falign-functions=64 -falign-loops=64` for GCC), and reports both result + tables as "default build" and "aligned build". Code layout alone can move a result by a few percent; a difference + that appears in only one of the two tables is a layout artifact. Off by default, which doubles the run time. - `--corpus file...` adds files to the corpus benchmark, e.g. those of [simdjson-data](https://github.com/simdjson/simdjson-data) or the [yyjson benchmark](https://github.com/ibireme/yyjson_benchmark). @@ -39,7 +43,7 @@ where `json_view` stands on another architecture; they are not meant for publica ## What is measured -`bench_view.cpp` runs four workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a +`bench_view.cpp` runs these workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a JSON-RPC request (`rpc`): | workload | what it does | @@ -48,6 +52,7 @@ JSON-RPC request (`rpc`): | traverse | parse, then visit every value, convert every number, touch every string and key | | select | parse, then read a few fields per record (e.g. id, user name, and retweet count of each tweet) | | dump | serialize a parsed document (compact) | +| parse+dump | build a fresh document, then serialize it (compact), in one timed call | `bench_corpus.cpp` runs parse, traverse, and dump on any list of files, so that no library is tuned to a handful of documents. Its dump also writes the numbers as they are in the input: `json_view` with `number_format::source`, and @@ -66,11 +71,32 @@ call follows an untimed call of the same engine: otherwise the engine after `jso cleaning up the tens of thousands of nodes `json::parse` just freed (with glibc, this made `json_view` look 1.7 times slower on citm_catalog traverse). +`json_view` is an index into the source text, not a tree of owning nodes. Its like-for-like peers are yyjson and +simdjson DOM (immutable, indexed, no per-node allocation). Boost.JSON and `json::parse` build mutable, owning DOMs and +are reference points for what that costs, not equals; simdjson On-Demand is lazy and forward-only, so it is the other +way round: it has no random access to pay for. All engines validate UTF-8 and syntax by default. + +Number conversion differs: `json_view` converts integers while parsing but converts floating-point numbers lazily, +when they are read or dumped; yyjson (default flags) and simdjson DOM convert every number while parsing. The cost of +float conversion therefore appears in the traverse and dump rows for `json_view`, and in the parse row for the others. +For float-heavy documents (canada), the fair reading is a sum of parse and use: the `traverse` workload already +includes the parse, and `parse+dump` measures a fresh document and its dump in one timed call; the `dump` row alone +leaves the conversion in `json_view` and out of the others. Eager conversion at parse time was prototyped as an opt-in +and measured: canada traverse (with its parse) was 4-7 % faster, but parse+dump 11-30 % slower and parse 36-90 % slower. It was +dropped, so lazy conversion is the only mode. + +`json_view` can also be edited: `json_editable_document` has the same read API, and the `(editable)` rows in +`bench_view` measure it on the same workloads. They are the fair counterpart of Boost.JSON and `json::parse`, which are +mutable too; the plain `json_view` rows are the counterpart of yyjson and simdjson DOM, which are not. `bench_edit` adds +the read-modify-write workloads. Read the tables with that in mind: an immutable-only library is not compared with an +editable `json_view` unless it says so. + The engines do not all offer the same features, which the numbers should be read with: | engine | document | random access | editable | notes | |---|---|---|---|---| | `json_view` | immutable index into the text | yes | no | a fresh document per parse; "reused" parses into the same document | +| `json_view` (editable) | index into the text plus edits | yes | yes, in place | `json_editable_document`; keeps member order and number spellings | | yyjson | immutable (`yyjson_read`) | yes | via a mutable copy | | | simdjson DOM | immutable, parser reused | yes | no | "fresh" uses a new parser per parse | | simdjson On-Demand | none: forward-only, lazy | no | no | only traverse and select | diff --git a/tests/benchmarks/json_view/bench_view.cpp b/tests/benchmarks/json_view/bench_view.cpp index 32d038649..71e19d68b 100644 --- a/tests/benchmarks/json_view/bench_view.cpp +++ b/tests/benchmarks/json_view/bench_view.cpp @@ -8,8 +8,9 @@ // Same-feature-set benchmark: read-only JSON documents with random access. // -// json_view nlohmann/json_view.hpp (fresh document per parse / reused) -// yyjson yyjson_read(): immutable document, random access +// json_view nlohmann/json_view.hpp (fresh document per parse / reused; +// "(editable)": json_editable_document, same read API) +// yyjson yyjson_read(): immutable document, random access // simdjson DOM dom::parser (reused, as recommended; "fresh": a new parser // per parse): immutable, random access // references (different feature sets): @@ -19,7 +20,11 @@ // // Workloads: parse (build + free), traverse (visit everything, convert every // number, touch every string and key), select (a few fields per document), -// dump (compact serialization of the parsed document). +// dump (compact serialization of the parsed document), and parse+dump (a fresh +// document, then its dump, in one timed call). json_view converts floating-point +// numbers lazily (when read or dumped), yyjson and simdjson DOM while parsing: +// for float-heavy documents, traverse (which includes the parse) and parse+dump +// are the fair reading of the parse and dump rows. // All engines run interleaved in every round, each timed call after an untimed // one of the same engine; the best round is reported. #include @@ -43,6 +48,8 @@ using nlohmann::json; using nlohmann::json_document; +using nlohmann::json_editable_document; +using nlohmann::json_editable_view; using nlohmann::json_view; static volatile double g_sink; @@ -55,7 +62,8 @@ struct stats // ---------------- traversal ---------------- -static void walk(json_view v, stats& st) +template +static void walk_view(V v, stats& st) { ++st.nodes; switch (v.type()) @@ -64,32 +72,42 @@ static void walk(json_view v, stats& st) for (auto it = v.begin(); it != v.end(); ++it) { st.str += it.key().size(); - walk(*it, st); + walk_view(*it, st); } break; case json::value_t::array: - for (const json_view e : v) + for (const V e : v) { - walk(e, st); + walk_view(e, st); } break; case json::value_t::string: st.str += v.get_string().size(); break; case json::value_t::number_integer: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_unsigned: - st.num += static_cast(v.get()); + st.num += static_cast(v.template get()); break; case json::value_t::number_float: - st.num += v.get(); + st.num += v.template get(); break; default: break; } } +static void walk(json_view v, stats& st) +{ + walk_view(v, st); +} + +static void walk(json_editable_view v, stats& st) +{ + walk_view(v, st); +} + static void walk(const json& j, stats& st) { ++st.nodes; @@ -300,20 +318,21 @@ static void walk(const boost::json::value& v, stats& st) // status (one tweet): id, user.screen_name, retweet_count // rpc: method, params.minuend, id -static double pick(const std::string& name, json_view r) +template +static double pick_view(const std::string& name, V r) { double acc = 0; if (name == "twitter" || name == "status") { - auto one = [&](json_view s) + auto one = [&](V s) { - acc += static_cast(s["id"].get()); + acc += static_cast(s["id"].template get()); acc += static_cast(s["user"]["screen_name"].get_string().size()); - acc += static_cast(s["retweet_count"].get()); + acc += static_cast(s["retweet_count"].template get()); }; if (name == "twitter") { - for (const json_view s : r["statuses"]) + for (const V s : r["statuses"]) { one(s); } @@ -325,20 +344,20 @@ static double pick(const std::string& name, json_view r) } else if (name == "citm_catalog") { - for (const json_view p : r["performances"]) + for (const V p : r["performances"]) { - acc += static_cast(p["id"].get() + p["eventId"].get() + p["seatCategories"].size()); + acc += static_cast(p["id"].template get() + p["eventId"].template get() + p["seatCategories"].size()); } acc += static_cast(r["events"].size()); } else if (name == "canada") { - const json_view g = r["features"][0]["geometry"]; + const V g = r["features"][0]["geometry"]; acc += static_cast(r["type"].get_string().size() + g["type"].get_string().size() + g["coordinates"].size()); } else if (name == "jeopardy") { - for (const json_view q : r) + for (const V q : r) { acc += q["round"].get_string() == "Final Jeopardy!" ? 1 : 0; acc += static_cast(q["category"].get_string().size()); @@ -347,11 +366,21 @@ static double pick(const std::string& name, json_view r) else if (name == "rpc") { acc += static_cast(r["method"].get_string().size()); - acc += static_cast(r["params"]["minuend"].get() + r["id"].get()); + acc += static_cast(r["params"]["minuend"].template get() + r["id"].template get()); } return acc; } +static double pick(const std::string& name, json_view r) +{ + return pick_view(name, r); +} + +static double pick(const std::string& name, json_editable_view r) +{ + return pick_view(name, r); +} + static double pick(const std::string& name, const json& r) { double acc = 0; @@ -634,15 +663,35 @@ int main(int argc, char** argv) walk(json::parse(dc.text), a); auto d = json_document::parse(dc.text); walk(d.root(), b); + auto ed = json_editable_document::parse(dc.text); + stats e; + walk(ed.root(), e); yyjson_doc* y = yyjson_read(dc.text.data(), dc.text.size(), 0); walk(yyjson_doc_get_root(y), c); simdjson::dom::parser p; walk(p.parse(dc.text).value(), dd); - const bool ok = a.nodes == b.nodes && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str + const bool ok = a.nodes == b.nodes && a.nodes == e.nodes && a.str == e.str && std::fabs(a.num - e.num) <= 1e-9 * std::fabs(a.num) && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str && std::fabs(a.num - b.num) <= 1e-9 * std::fabs(a.num) && std::fabs(a.num - c.num) <= 1e-9 * std::fabs(a.num); - const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); - std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pc && pa == pd) ? "OK" : "MISMATCH"); + const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pe = pick(dc.name, ed.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value()); + std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pe && pa == pc && pa == pd) ? "OK" : "MISMATCH"); yyjson_doc_free(y); + + // the dumps of the engines may format floats differently, so only: every dump is non-empty, and the + // dump of json_view (also editable) and of json::parse parse back to the same number of nodes + const std::string dj = json::parse(dc.text).dump(); + const std::string dv = d.root().dump(); + const std::string de = ed.root().dump(); + std::size_t yn = 0; + yyjson_doc* y2 = yyjson_read(dc.text.data(), dc.text.size(), 0); + char* yo = yyjson_write(y2, 0, &yn); + const bool dumps_ok = !dj.empty() && !dv.empty() && !de.empty() && yo != nullptr && yn > 0 && !simdjson::to_string(p.parse(dc.text).value()).empty(); + std::free(yo); + yyjson_doc_free(y2); + stats sj_nodes, sv_nodes, se_nodes; + walk(json::parse(dj), sj_nodes); + walk(json::parse(dv), sv_nodes); + walk(json::parse(de), se_nodes); + std::printf("check %-13s dump %s\n", dc.name.c_str(), (dumps_ok && sj_nodes.nodes == a.nodes && sv_nodes.nodes == a.nodes && se_nodes.nodes == a.nodes) ? "OK" : "MISMATCH"); } std::FILE* csv = std::fopen("bench_view.csv", "w"); @@ -669,6 +718,7 @@ int main(int argc, char** argv) workloads.push_back({"parse", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = static_cast(d.node_count()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = static_cast(reused.node_count()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = static_cast(d.root().size()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = static_cast(yyjson_doc_get_val_count(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { auto e = sj.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, {"simdjson DOM (fresh)", [&] { simdjson::dom::parser p; auto e = p.parse(ps).value_unsafe(); g_sink = e.is_object(); }}, @@ -680,6 +730,7 @@ int main(int argc, char** argv) workloads.push_back({"traverse", { {"json_view", [&] { auto d = json_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"json_view (reused)", [&] { reused.read(s); stats st; walk(reused.root(), st); g_sink = st.num; }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); stats st; walk(yyjson_doc_get_root(d), st); g_sink = st.num; yyjson_doc_free(d); }}, {"simdjson DOM", [&] { stats st; walk(sj.parse(ps).value_unsafe(), st); g_sink = st.num; }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); stats st; walk_od(d.get_value().value_unsafe(), st); g_sink = st.num; }}, @@ -691,15 +742,31 @@ int main(int argc, char** argv) workloads.push_back({"select", { {"json_view", [&] { auto d = json_document::parse(s); g_sink = pick(name, d.root()); }}, {"json_view (reused)", [&] { reused.read(s); g_sink = pick(name, reused.root()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = pick(name, d.root()); }}, {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = pick(name, yyjson_doc_get_root(d)); yyjson_doc_free(d); }}, {"simdjson DOM", [&] { g_sink = pick(name, sj.parse(ps).value_unsafe()); }}, {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); g_sink = pick_od(name, d); }}, {"json::parse", [&] { json j = json::parse(s); g_sink = pick(name, j); }}, }}); + // pipeline: a fresh document, then a dump of it, in one timed call + // (simdjson DOM reuses its parser, as in the parse row); the traverse + // workload above is already parse+traverse + workloads.push_back({"parse+dump", { + {"json_view", [&] { auto d = json_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast(o.size()); }}, + {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); std::size_t n = 0; char* o = yyjson_write(d, 0, &n); g_sink = static_cast(n); std::free(o); yyjson_doc_free(d); }}, + {"simdjson DOM", [&] { std::string o = simdjson::to_string(sj.parse(ps).value_unsafe()); g_sink = static_cast(o.size()); }}, +#if JSON_VIEW_BENCH_BOOST + {"Boost.JSON", [&] { boost::json::monotonic_resource mr; auto v = boost::json::parse(s, &mr); std::string o = boost::json::serialize(v); g_sink = static_cast(o.size()); }}, +#endif + {"json::parse", [&] { json j = json::parse(s); std::string o = j.dump(); g_sink = static_cast(o.size()); }}, + }}); { // serialization of an already parsed document static json_document vd; vd.read(s); + static json_editable_document ed = json_editable_document::parse(s); + ed = json_editable_document::parse(s); static yyjson_doc* yd = nullptr; if (yd) { @@ -713,6 +780,7 @@ int main(int argc, char** argv) jd = json::parse(s); workloads.push_back({"dump", { {"json_view", [&] { std::string o = vd.root().dump(); g_sink = static_cast(o.size()); }}, + {"json_view (editable)", [&] { std::string o = ed.root().dump(); g_sink = static_cast(o.size()); }}, {"yyjson", [&] { std::size_t n = 0; char* o = yyjson_write(yd, 0, &n); g_sink = static_cast(n); std::free(o); }}, {"simdjson DOM", [&] { std::string o = simdjson::to_string(se); g_sink = static_cast(o.size()); }}, {"json::parse", [&] { std::string o = jd.dump(); g_sink = static_cast(o.size()); }}, @@ -740,7 +808,7 @@ int main(int argc, char** argv) } } const double ref = best[0]; - std::printf("%-13s %-9s", dc.name.c_str(), wl.first.c_str()); + std::printf("%-13s %-14s", dc.name.c_str(), wl.first.c_str()); for (std::size_t k = 0; k < wl.second.size(); ++k) { const double us = best[k] / 1e3; diff --git a/tests/benchmarks/json_view/compare.py b/tests/benchmarks/json_view/compare.py index 39d5048f0..e8fdbcc09 100755 --- a/tests/benchmarks/json_view/compare.py +++ b/tests/benchmarks/json_view/compare.py @@ -239,6 +239,8 @@ def main(): ap.add_argument('--download', action='store_true', help='use pinned downloads instead of system libraries') ap.add_argument('--no-boost', action='store_true', help='skip Boost.JSON') ap.add_argument('--native', action='store_true', help='compile for this CPU (-march=native / -mcpu=native)') + ap.add_argument('--aligned', action='store_true', + help='also build and run with all functions (and loops) aligned, to make code layout artifacts visible') ap.add_argument('--rounds', type=int, default=30, help='rounds of bench_view (default: 30)') ap.add_argument('--corpus', nargs='*', default=[], help='more files for bench_corpus') ap.add_argument('--build-dir', default=os.path.join(HERE, 'build'), help='where to build (default: build/ next to this script)') @@ -282,30 +284,44 @@ def main(): + ['-I' + d for d in lib.include]) objects.append(obj) - binaries = {} - for bench in ['bench_view', 'bench_corpus', 'bench_edit']: - exe = os.path.join(args.build_dir, bench) - run(cxx + flags + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) - binaries[bench] = exe + cxx_version = output(cxx + ['--version']) + is_clang = 'clang' in cxx_version.lower() + # alignment flags: functions and (gcc) loops on 64-byte boundaries + align_flags = ['-mllvm', '-align-all-functions=6'] if is_clang else ['-falign-functions=64', '-falign-loops=64'] + + # the builds: the default one and, with --aligned, one with aligned code + variants = [('default build', args.build_dir, [])] + if args.aligned: + variants.append(('aligned build', os.path.join(args.build_dir, 'aligned'), align_flags)) - # run: bench_view on its documents, bench_corpus on those and the given files corpus = [os.path.join(args.data, f) for f in DEFAULT_CORPUS] + args.corpus - outputs = {} - outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=args.build_dir, - capture_output=True, text=True).stdout - outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir, - capture_output=True, text=True).stdout - for name, text in outputs.items(): - print(text) + # label -> {benchmark: output} + all_outputs = {} + for label, build_dir, extra in variants: + os.makedirs(build_dir, exist_ok=True) + binaries = {} + for bench in ['bench_view', 'bench_corpus', 'bench_edit']: + exe = os.path.join(build_dir, bench) + run(cxx + flags + extra + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe]) + binaries[bench] = exe + + # run: bench_view on its documents, bench_corpus on those and the given files + outputs = {} + outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=build_dir, + capture_output=True, text=True).stdout + outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=build_dir, + capture_output=True, text=True).stdout + for name, text in outputs.items(): + print(text) + all_outputs[label] = outputs # results with their metadata now = datetime.datetime.now() host = re.sub(r'[^A-Za-z0-9-]+', '-', platform.node().split('.')[0]) or 'host' stem = os.path.join(HERE, 'results', f'{now:%Y-%m-%d}-{host}') os.makedirs(os.path.dirname(stem), exist_ok=True) - cxx_version = output(cxx + ['--version']) compiler_version = cxx_version.splitlines()[0] if cxx_version else ' '.join(cxx) meta = [ ('date', f'{now:%Y-%m-%d %H:%M}'), @@ -314,6 +330,7 @@ def main(): ('OS', f'{platform.system()} {platform.release()} ({platform.machine()})'), ('compiler', compiler_version), ('flags', ' '.join(flags)), + ('aligned results', ('included (flags: ' + ' '.join(align_flags) + ')') if args.aligned else 'not included'), ('yyjson', libs['yyjson'].version), ('simdjson', libs['simdjson'].version), ('Boost.JSON', libs['boost'].version if with_boost else 'skipped (not found)'), @@ -326,15 +343,20 @@ def main(): f.write('| | |\n|---|---|\n') for key, value in meta: f.write(f'| {key} | {value} |\n') - for name, text in outputs.items(): - f.write(f'\n## {name}\n\n```\n{text.rstrip()}\n```\n') + for label, outputs in all_outputs.items(): + # (without --aligned there is only the default build, and no extra heading level) + prefix = f'{label}: ' if args.aligned else '' + for name, text in outputs.items(): + f.write(f'\n## {prefix}{name}\n\n```\n{text.rstrip()}\n```\n') with open(stem + '.csv', 'w', encoding='utf-8') as out: out.write(''.join(f'# {key}: {value}\n' for key, value in meta)) - for name in ['bench_view', 'bench_corpus', 'bench_edit']: - path = os.path.join(args.build_dir, name + '.csv') - if os.path.isfile(path): - with open(path, encoding='utf-8') as f: - out.write(f'# {name}\n' + f.read()) + for label, build_dir, _ in variants: + for name in ['bench_view', 'bench_corpus', 'bench_edit']: + path = os.path.join(build_dir, name + '.csv') + if os.path.isfile(path): + with open(path, encoding='utf-8') as f: + # (the default build keeps its plain section names) + out.write(f'# {name}' + (f' ({label})' if args.aligned else '') + '\n' + f.read()) print(f'results: {stem}.md, {stem}.csv')