Let generate_natvis.py cover custom ABI tag combinations (#5880)

Since #5877, the checked-in nlohmann_json.natvis only covers namespaces with up to two ABI tags. The generator gets options to produce a file for other configurations: - --abi-tags TAGS adds the namespaces of one configuration (repeatable) - --all generates every combination of ABI tags - --list-tags prints the tags with the macros that enable them - --output FILE writes somewhere else than the repository root `make natvis NATVIS_ARGS="..."` passes the options through. The default output is unchanged. The options are documented in the generator's README, the debugging page, and FILES.md, and checked in CI. Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-22 11:36:14 +00:00 · 2026-10-21 22:57:54 +02:00
1 parent 9bf531436f
commit 533cbfee51
@@ -154,6 +154,28 @@ jobs:
          # PR content in a job without credentials.
          python3 tools/generate_natvis/generate_natvis.py $MAIN_DIR

+      - name: Check the nlohmann_json.natvis generator options
+        run: |
+          cd $MAIN_DIR
+          # The options for configurations outside the checked-in file write
+          # to a temporary file, so they do not affect the diff below.
+          GENERATOR=tools/generate_natvis/generate_natvis.py
+          python3 $GENERATOR --list-tags
+          TAG_COUNT=$(python3 $GENERATOR --list-tags | wc -l)
+
+          python3 $GENERATOR --abi-tags diag,_dp,_vns --abi-tags _diag --output $RUNNER_TEMP/abi_tags.natvis
+          grep -q '<!-- Namespace nlohmann::json_abi_diag_dp_vns -->' $RUNNER_TEMP/abi_tags.natvis
+          grep -q '<!-- Namespace nlohmann::json_abi_diag_dp_vns_v[0-9_]* -->' $RUNNER_TEMP/abi_tags.natvis
+          test $(grep -c '<!-- Namespace ' $RUNNER_TEMP/abi_tags.natvis) -eq $(( $(grep -c '<!-- Namespace ' nlohmann_json.natvis) + 2 ))
+
+          python3 $GENERATOR --all --output $RUNNER_TEMP/all.natvis
+          test $(grep -c '<!-- Namespace ' $RUNNER_TEMP/all.natvis) -eq $(( (1 << TAG_COUNT) * 2 + 1 ))
+
+          if python3 $GENERATOR --abi-tags _unknown --output $RUNNER_TEMP/unknown.natvis; then
+            echo "generate_natvis.py accepted an unknown ABI tag"
+            exit 1
+          fi
+
      - name: Build patch and check for differences
        id: diff
        run: |
@@ -6,6 +6,7 @@ This file describes the source for supporting files; that is, files that are not
 - [GitHub](#github)
 - [REUSE](#reuse)
 - [Package Managers](#package-managers)
+- [Debugging](#debugging)

 ## Continuous Integration

@@ -289,3 +290,29 @@ The module definition for [Bazel](https://bazel.build)'s [Bzlmod](https://bazel.
 Further documentation:

 - [Bazel modules](https://bazel.build/external/module)
+
+## Debugging
+
+### `nlohmann_json.natvis`
+
+The [Natvis](https://learn.microsoft.com/en-us/visualstudio/debugger/create-custom-views-of-native-objects) file that
+gives `json` values a friendly view in the Visual Studio debugger. The file is generated by
+`tools/generate_natvis/generate_natvis.py` from the template `tools/generate_natvis/nlohmann_json.natvis.j2` and the ABI
+tags and version in `include/nlohmann/detail/abi_macros.hpp`; change the script or the template rather than editing
+the file by hand. The file can be updated by calling
+
+```shell
+make natvis
+```
+
+The "Check amalgamation" workflow fails if the file is out of date.
+
+The file covers the namespace variants with up to two ABI tags. A Natvis file for other configurations can be
+generated with the `--abi-tags` or `--all` options, for example
+`make natvis NATVIS_ARGS="--abi-tags _diag,_dp,_vns --output my_nlohmann_json.natvis"`; see
+[`tools/generate_natvis`](tools/generate_natvis/README.md). With MSVC, CMake adds the file to the `nlohmann_json`
+target and installs it.
+
+Further documentation:
+
+- [Create custom views of C++ objects in the debugger using the Natvis framework](https://learn.microsoft.com/en-us/visualstudio/debugger/create-custom-views-of-native-objects)
@@ -46,7 +46,7 @@ all:
 	@echo "clean - remove built files"
 	@echo "(fuzzers) - see tests/fuzzing.md, or run 'make -C tests fuzzers' to build the AFL fuzzers"
 	@echo "macro_builder_check - check that macro_scope.hpp matches tools/macro_builder's output"
-	@echo "natvis - regenerate nlohmann_json.natvis from the current ABI tags and version"
+	@echo "natvis - regenerate nlohmann_json.natvis from the current ABI tags and version (pass generator options via NATVIS_ARGS, see tools/generate_natvis/README.md)"
 	@echo "pretty - beautify code with Artistic Style"
 	@echo "pvs_studio - run the PVS-Studio static analyzer and open the report"
 	@echo "release - create the release files (include.zip, json.tar.xz, single headers, signatures, hashes)"
@@ -120,13 +120,15 @@ $(AMALGAMATED_VIEW_FILE): $(SRCS)
 	tools/amalgamate/amalgamate.py -c tools/amalgamate/config_json_view.json -s . --verbose=yes

 NATVIS_VENV=tools/generate_natvis/venv
+# extra options for generate_natvis.py, e.g. NATVIS_ARGS="--abi-tags _diag,_dp,_vns --output my.natvis"
+NATVIS_ARGS=

 # regenerate nlohmann_json.natvis from the ABI tags and version in include/nlohmann/detail/abi_macros.hpp
 # (the dependencies are installed into a virtual environment first)
 natvis:
 	@test -d $(NATVIS_VENV) || python3 -mvenv $(NATVIS_VENV)
 	@$(NATVIS_VENV)/bin/python3 -c 'import jinja2' 2>/dev/null || $(NATVIS_VENV)/bin/pip3 install --quiet -r tools/generate_natvis/requirements.txt
-	$(NATVIS_VENV)/bin/python3 tools/generate_natvis/generate_natvis.py .
+	$(NATVIS_VENV)/bin/python3 tools/generate_natvis/generate_natvis.py $(NATVIS_ARGS) .

 # regenerate the two tools/macro_builder blocks of $(MACRO_SCOPE_HPP) (see its README.md) and diff against the
 # checked-in header; phony, because it never writes $(MACRO_SCOPE_HPP) itself
@@ -11,7 +11,27 @@ file that gives `json`/`ordered_json` values a friendly, key/value debugger view
 fields, when debugging with the MSVC debug engine (`cppvsdbg`) in Visual Studio or VS Code.

 The checked-in visualizers cover namespace variants with zero, one, or two ABI tags. Configurations with three or more
-ABI tags are not covered and may show raw internal fields instead.
+ABI tags are not covered and may show raw internal fields instead. To generate a Natvis file for such a configuration:
+
+1. Check out the release tag of the library version you use.
+2. List the ABI tags and the macros that enable them:
+
+    ```shell
+    make natvis NATVIS_ARGS="--list-tags"
+    ```
+
+3. Generate a Natvis file with the tags of your configuration, for example for `JSON_DIAGNOSTICS`,
+    `JSON_DIAGNOSTIC_POSITIONS`, and `JSON_VIEW_NO_SIMD`:
+
+    ```shell
+    make natvis NATVIS_ARGS="--abi-tags _diag,_dp,_vns --output my_nlohmann_json.natvis"
+    ```
+
+4. Use the generated file instead of `nlohmann_json.natvis`.
+
+`--abi-tags` can be repeated for several configurations, and `--all` generates every combination (about 60 MB for 12
+ABI tags, doubling with each further tag). See
+[`tools/generate_natvis`](https://github.com/nlohmann/json/tree/develop/tools/generate_natvis) for all options.

 Debug engines that wrap LLDB instead of the MSVC debug engine (for example, `codelldb` in VS Code) only have
 partial/experimental Natvis support, and commonly fall back to showing raw internal fields even with the
@@ -4,7 +4,7 @@ Generate the Natvis debugger visualization file for namespace combinations with

 The checked-in `nlohmann_json.natvis` covers the default namespace (no ABI tags), every single ABI tag, and every pair
 of ABI tags, with both versioned and unversioned namespace variants. Configurations that enable three or more ABI tags
-are not covered by this reduced file.
+are not covered by this reduced file; use `--abi-tags` or `--all` (see below) to generate a file for them.

 The ABI tag list and the library version are parsed from
 `include/nlohmann/detail/abi_macros.hpp`, so this script must be re-run (via
@@ -23,10 +23,43 @@ make natvis
 `requirements.txt` (Jinja2) and runs the script with it. Alternatively, install the requirements yourself and run:

 ```shell
-./generate_natvis.py [--version X.Y.Z] [repository_root/]
+./generate_natvis.py [--version X.Y.Z] [--abi-tags TAGS]... [--all] [--list-tags] [--output FILE] [repository_root/]
 ```

-`--version` and the output/repository-root directory both default to values
+`--version` and the repository root both default to values
 derived from this script's own location, so they only need to be given
 explicitly when generating a Natvis file for a different checkout or a
 version other than the one in `abi_macros.hpp`.
+
+| Option            | Description                                                                                                                                                                                                                                         |
+|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `--list-tags`     | Print every ABI tag (in namespace order) with the macro that enables it and the `NLOHMANN_JSON_ABI_TAG_*` identifier, then exit.                                                                                                                    |
+| `--abi-tags TAGS` | Add the namespaces of one configuration, given as a comma-separated list of ABI tags (`_diag,_dp,_vns`; the leading underscore is optional, and the order does not matter). Can be repeated for several configurations. Unknown tags are an error. |
+| `--all`           | Generate the namespaces for every combination of ABI tags. The file size doubles with each tag (about 60 MB for 12 tags). Cannot be combined with `--abi-tags`.                                                                                    |
+| `--output FILE`   | Write the Natvis file to `FILE` instead of `nlohmann_json.natvis` in the repository root.                                                                                                                                                           |
+
+Each configuration gets both the versioned (`json_abi_diag_dp_vns_v3_12_0`) and the unversioned (`json_abi_diag_dp_vns`,
+used with `NLOHMANN_JSON_NAMESPACE_NO_VERSION`) namespace. The default namespaces (up to two ABI tags) are always
+included, so a generated file can replace the checked-in one.
+
+### Generating a Natvis file for your configuration
+
+1. Check out the release tag of the library version you use (the ABI tags and the version are read from the checkout).
+2. Find the ABI tags of the macros you enable:
+
+   ```shell
+   make natvis NATVIS_ARGS="--list-tags"
+   ```
+
+3. Generate a Natvis file with them, for example for `JSON_DIAGNOSTICS`, `JSON_DIAGNOSTIC_POSITIONS`, and
+   `JSON_VIEW_NO_SIMD`:
+
+   ```shell
+   make natvis NATVIS_ARGS="--abi-tags _diag,_dp,_vns --output my_nlohmann_json.natvis"
+   ```
+
+4. Use `my_nlohmann_json.natvis` instead of the shipped `nlohmann_json.natvis`, for example by adding it to your Visual
+   Studio project.
+
+Passing `--output` keeps the checked-in `nlohmann_json.natvis` unchanged; without it, `make check-amalgamation` would
+report the file as out of date.
@@ -20,9 +20,11 @@ def semver(v):


 def abi_info(repo_root):
-    """Derive the ABI tag list (in NLOHMANN_JSON_ABI_TAGS_CONCAT order) and the
+    """Derive the ABI tags (in NLOHMANN_JSON_ABI_TAGS_CONCAT order) and the
    library version from <repo_root>/include/nlohmann/detail/abi_macros.hpp,
-    so this script cannot drift from the header it visualizes."""
+    so this script cannot drift from the header it visualizes. Each tag is
+    returned as a (suffix, configuration macro, tag macro) tuple, for example
+    ('_diag', 'JSON_DIAGNOSTICS', 'NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS')."""
    abi_macros_hpp = os.path.join(repo_root, 'include', 'nlohmann', 'detail', 'abi_macros.hpp')
    with open(abi_macros_hpp) as f:
        content = f.read()
@@ -47,7 +49,9 @@ def abi_info(repo_root):
        match = re.search(r'#define\s+' + re.escape(ident) + r'\s+(_\w+)\s*\n', content)
        if not match:
            raise ValueError(f'could not find a non-empty #define for {ident} in {abi_macros_hpp}')
-        abi_tags.append(match.group(1))
+        # the option that enables the tag is the #if/#ifdef right above it
+        option = re.search(r'#\s*(?:if|ifdef)\s+(\w+)\s*\n\s*#define\s+' + re.escape(ident) + r'\s+_', content)
+        abi_tags.append((match.group(1), option.group(1) if option else '?', ident))

    version = {}
    for part in ('MAJOR', 'MINOR', 'PATCH'):
@@ -59,30 +63,79 @@ def abi_info(repo_root):
    return abi_tags, '{MAJOR}.{MINOR}.{PATCH}'.format(**version)


+def parse_tag_list(value, abi_tags):
+    """Turn one --abi-tags value ("_diag,dp") into the tuple of suffixes it
+    names, in namespace order; unknown tags and empty lists are errors."""
+    known = [suffix for suffix, _, _ in abi_tags]
+    requested = set()
+    for tag in value.split(','):
+        tag = tag.strip()
+        if not tag:
+            continue
+        if not tag.startswith('_'):
+            tag = '_' + tag
+        if tag not in known:
+            raise ValueError(f'unknown ABI tag {tag!r}; valid tags: {", ".join(known)} (see --list-tags)')
+        requested.add(tag)
+    if not requested:
+        raise ValueError(f'empty ABI tag list {value!r}; the namespace without tags is always generated')
+    return tuple(tag for tag in known if tag in requested)
+
+
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        description='Generate nlohmann_json.natvis. By default, it covers the namespaces with up to two ABI tags; '
+                    'use --abi-tags or --all for other configurations.')
    parser.add_argument('--version', type=semver,
                         help='Library version number (default: parsed from '
-                              'include/nlohmann/detail/abi_macros.hpp below "output")')
-    parser.add_argument('output', nargs='?', default=REPO_ROOT,
-                         help='Repository root: where include/nlohmann/detail/abi_macros.hpp is '
-                              'read from and where nlohmann_json.natvis is written '
+                              'include/nlohmann/detail/abi_macros.hpp below "repo_root")')
+    selection = parser.add_mutually_exclusive_group()
+    selection.add_argument('--abi-tags', action='append', default=[], metavar='TAGS',
+                           help='Comma-separated list of ABI tags (e.g. "_diag,_dp,_vns", leading underscores are '
+                                'optional) of one configuration to add to the default namespaces; can be repeated')
+    selection.add_argument('--all', action='store_true',
+                           help='Generate the namespaces for every combination of ABI tags; the file size doubles '
+                                'with each ABI tag (about 60 MB for 12 tags)')
+    parser.add_argument('--list-tags', action='store_true',
+                        help='List the ABI tags with the macros that enable them, and exit')
+    parser.add_argument('--output', metavar='FILE',
+                        help='Where to write the Natvis file (default: nlohmann_json.natvis below "repo_root")')
+    parser.add_argument('repo_root', nargs='?', default=REPO_ROOT,
+                         help='Repository root: where include/nlohmann/detail/abi_macros.hpp is read from '
                              '(default: the repository root this script lives in)')
    args = parser.parse_args()

-    derived_tags, derived_version = abi_info(args.output)
+    abi_tags, derived_version = abi_info(args.repo_root)
+
+    if args.list_tags:
+        width = max(len(suffix) for suffix, _, _ in abi_tags)
+        option_width = max(len(option) for _, option, _ in abi_tags)
+        for suffix, option, ident in abi_tags:
+            print(f'{suffix:<{width}}  {option:<{option_width}}  {ident}')
+        sys.exit(0)
+
+    try:
+        extra_combinations = [parse_tag_list(value, abi_tags) for value in args.abi_tags]
+    except ValueError as e:
+        parser.error(str(e))

    namespaces = ['nlohmann']
    abi_prefix = 'json_abi'
-    abi_tags = derived_tags
+    suffixes = [suffix for suffix, _, _ in abi_tags]
    version = '_v' + (args.version or derived_version).replace('.', '_')
-    inline_namespaces = []

-    # generate namespace names for up to two ABI tags
-    for n in range(0, min(len(abi_tags), 2) + 1):
-        for tags in itertools.combinations(abi_tags, n):
-            ns = abi_prefix + ''.join(tags)
-            inline_namespaces += [ns, ns + version]
+    # by default, generate the namespace names for up to two ABI tags, plus
+    # the combinations requested with --abi-tags
+    max_tags = len(suffixes) if args.all else min(len(suffixes), 2)
+    combinations = [tags for n in range(0, max_tags + 1) for tags in itertools.combinations(suffixes, n)]
+    for tags in extra_combinations:
+        if tags not in combinations:
+            combinations.append(tags)
+
+    inline_namespaces = []
+    for tags in combinations:
+        ns = abi_prefix + ''.join(tags)
+        inline_namespaces += [ns, ns + version]

    namespaces += [f'{namespaces[0]}::{ns}' for ns in inline_namespaces]

@@ -91,5 +144,10 @@ if __name__ == '__main__':
    template = env.get_template('nlohmann_json.natvis.j2')
    natvis = template.render(namespaces=namespaces)

-    with open(os.path.join(args.output, 'nlohmann_json.natvis'), 'w') as f:
+    output = args.output or os.path.join(args.repo_root, 'nlohmann_json.natvis')
+    with open(output, 'w') as f:
        f.write(natvis)
+
+    if args.all:
+        print(f'wrote {len(namespaces)} namespaces ({os.path.getsize(output) / 1e6:.1f} MB) to {output}',
+              file=sys.stderr)