Fix dump yaml with comments nested cases

CHANGELOG.rst

@@ -35,6 +35,11 @@ Fixed
 - ``TypedDict`` keys inherited from a base defined in a different module not
   resolving the types only imported in the ``TYPE_CHECKING`` block of that
   module (`#936 <https://github.com/mauvilsa/jsonargparse/pull/936>`__).
+- ``dump`` with ``with_comments=True``, i.e. ``--print_config=comments``, not
+  adding any comment for the ``init_args`` of subclasses and for the fields of
+  dataclass-like types that are not added as a group, including when nested in
+  lists and dicts (`#939
+  <https://github.com/mauvilsa/jsonargparse/pull/939>`__).
 
 Changed
 ^^^^^^^

DOCUMENTATION.rst

@@ -1352,9 +1352,12 @@ Parsers that have an ``action="config"`` argument also include a
 with a large set of options to create an initial config file including all
 default values. If the `ruamel.yaml <https://pypi.org/project/ruamel.yaml>`__
 package is installed, the config can be printed having the help descriptions
-content as YAML comments by using ``--print_config=comments``. Another option is
-``--print_config=skip_unset`` which skips entries whose value is the configured
-unset value (see :ref:`unset-values`).
+content as YAML comments by using ``--print_config=comments``. The comments
+include the descriptions of the groups and arguments of the parser, and for
+values that correspond to a class, e.g. the ``init_args`` of a subclass or the
+fields of a dataclass, the descriptions from the respective class. Another
+option is ``--print_config=skip_unset`` which skips entries whose value is the
+configured unset value (see :ref:`unset-values`).
 
 From within Python it is also possible to serialize a config object by using
 either the :meth:`dump <.ArgumentParser.dump>` or :meth:`save

jsonargparse/_formatters.py

@@ -24,6 +24,8 @@
 from ._common import (
     defaults_cache,
     get_optionals_as_positionals_actions,
+    get_unaliased_type,
+    is_subclasses_disabled,
     parent_parser,
     supports_optionals_as_positionals,
 )
@@ -33,7 +35,13 @@
 from ._optionals import import_ruamel
 from ._subcommands import ActionSubCommands, find_action
 from ._type_checking import ArgumentParser, ruamelCommentedMap
-from ._typehints import ActionTypeHint, type_to_str
+from ._typehints import (
+    ActionTypeHint,
+    get_optional_arg,
+    get_subclass_or_closed_types,
+    is_subclass_spec,
+    type_to_str,
+)
 
 __all__ = ["DefaultHelpFormatter"]
 
@@ -50,6 +58,77 @@
     """  # type: ignore[assignment]
 
 
+def get_subparsers(parser: "ArgumentParser", prefix: str = "") -> dict[str | None, "ArgumentParser"]:
+    """Returns the given parser and all its subcommand parsers, keyed by subcommand key."""
+    parsers: dict[str | None, ArgumentParser] = {}
+    if parser._subcommands_action is not None:
+        for key, subparser in parser._subcommands_action._name_parser_map.items():
+            full_key = (prefix + "." if prefix else "") + key
+            parsers[full_key] = subparser
+            parsers.update(get_subparsers(subparser, prefix=full_key))
+    parsers[None] = parser
+    return parsers
+
+
+def get_group_titles(parser: "ArgumentParser") -> dict:
+    """Returns a map of parser keys to the title of the group they belong to."""
+    group_titles: dict = {}
+    for parser_key, subparser in get_subparsers(parser).items():
+        group_titles[parser_key] = subparser.description
+        prefix = "" if parser_key is None else parser_key + "."
+        for group in subparser._action_groups:
+            actions = filter_non_parsing_actions(group._group_actions)
+            skip_types = (_ActionConfigLoad, ActionConfigFile, ActionSubCommands)
+            actions = [a for a in actions if not isinstance(a, skip_types)]
+            keys = {re.sub(r"\.?[^.]+$", "", a.dest) for a in actions if "." in a.dest}
+            for key in keys:
+                group_titles[prefix + key] = group.title
+    return group_titles
+
+
+def get_class_group_title(class_parser: "ArgumentParser") -> str | None:
+    """Returns the title of the group that a class parser adds for the class itself.
+
+    The group for the class is the first one added, the rest correspond to nested keys.
+    """
+    groups = list((class_parser.groups or {}).values())
+    return groups[0].title if groups else None
+
+
+def get_class_parser(class_type, action: Action | None) -> "ArgumentParser | None":
+    """Returns a parser for the arguments of a class, or None if not possible."""
+    sub_add_kwargs = dict(getattr(action, "sub_add_kwargs", None) or {})
+    sub_add_kwargs.pop("linked_targets", None)
+    try:
+        return ActionTypeHint.get_class_parser(class_type, sub_add_kwargs=sub_add_kwargs)
+    except Exception:
+        return None
+
+
+def get_closed_type_parser(typehint, action: Action | None) -> "ArgumentParser | None":
+    """Returns a parser for the arguments of a closed type, e.g. a dataclass, given its type hint."""
+    types = get_subclass_or_closed_types(typehint, also_lists=True, callable_return=True)
+    if types and len(types) == 1 and is_subclasses_disabled(types[0]):
+        return get_class_parser(types[0], action)
+    return None
+
+
+def get_mapping_value_typehint(typehint):
+    """Returns the type of the values of a mapping type hint, or None if not a mapping."""
+    if not ActionTypeHint.is_mapping_typehint(typehint):
+        return None
+    args = getattr(get_optional_arg(get_unaliased_type(typehint)), "__args__", ())
+    return args[1] if len(args) == 2 else None
+
+
+def remove_leading_blank_line(cfg: ruamelCommentedMap) -> None:
+    """Removes the blank line that comments add before the first key, needed for items of a list."""
+    key = next(iter(cfg.keys()), None)
+    comments = cfg.ca.items.get(key, [None, None])[1] if key is not None else None
+    if comments and comments[0].value == "\n":
+        del comments[0]
+
+
 class YAMLCommentFormatter:
     """Formatter class for adding YAML comments to configuration files."""
 
@@ -64,56 +143,127 @@
         yaml = ruyaml.YAML()
         cfg = yaml.load(cfg)
 
-        def get_parsers(parser: ArgumentParser, prefix="") -> dict[str | None, ArgumentParser]:
-            parsers = {}
-            if parser._subparsers is not None:
-                for key, subparser in parser._subparsers._group_actions[0].choices.items():  # type: ignore[union-attr]
-                    full_key = (prefix + "." if prefix else "") + key
-                    parsers[full_key] = subparser
-                    parsers.update(get_parsers(subparser, prefix=full_key))
-            parsers[None] = parser
-            return parsers
-
         parser = parent_parser.get()
         assert isinstance(parser, ArgumentParser)
-        parsers = get_parsers(parser)
-
-        group_titles = {}
-        for parser_n_key, parser_n in parsers.items():
-            group_titles[parser_n_key] = parser_n.description
-            prefix = "" if parser_n_key is None else parser_n_key + "."
-            for group in parser_n._action_groups:
-                actions = filter_non_parsing_actions(group._group_actions)
-                actions = [
-                    a for a in actions if not isinstance(a, (_ActionConfigLoad, ActionConfigFile, ActionSubCommands))
-                ]
-                keys = {re.sub(r"\.?[^.]+$", "", a.dest) for a in actions if "." in a.dest}
-                for key in keys:
-                    group_titles[prefix + key] = group.title
-
-        def set_comments(cfg, prefix="", depth=0):
-            for key in cfg.keys():
-                full_key = (prefix + "." if prefix else "") + key
-                action = find_action(parser, full_key)
-                text = None
-                if full_key in group_titles and isinstance(cfg[key], dict):
-                    text = group_titles[full_key]
-                elif action is not None and action.help != SUPPRESS:
-                    text = self.help_formatter._expand_help(action)
-                if isinstance(cfg[key], dict):
-                    if text:
-                        self.set_yaml_group_comment(text, cfg, key, depth)
-                    set_comments(cfg[key], full_key, depth + 1)
-                elif text:
-                    self.set_yaml_argument_comment(text, cfg, key, depth)
-
-        if parser.description is not None:
-            self.set_yaml_start_comment(parser.description, cfg)
-        set_comments(cfg)
+        if isinstance(cfg, dict):
+            if parser.description is not None:
+                self.set_yaml_start_comment(parser.description, cfg)
+            self.set_comments(cfg, parser, get_group_titles(parser))
         out = StringIO()
         yaml.dump(cfg, out)
         return out.getvalue()
 
+    def set_comments(
+        self,
+        cfg: ruamelCommentedMap,
+        parser: "ArgumentParser",
+        group_titles: dict,
+        prefix: str = "",
+        depth: int = 0,
+    ) -> None:
+        """Sets the comments for all the keys of a config object.
+
+        Args:
+            cfg: The ruamel.yaml object.
+            parser: The parser from which help content is obtained.
+            group_titles: Map of parser keys to the title of the group they belong to.
+            prefix: The parser key that corresponds to the given config object.
+            depth: The nested level of the keys of the config object.
+        """
+        for key in cfg.keys():
+            full_key = (prefix + "." if prefix else "") + key
+            action = find_action(parser, full_key)
+            value = cfg[key]
+            text = None
+            if full_key in group_titles and isinstance(value, dict):
+                text = group_titles[full_key]
+            elif action is not None and action.help != SUPPRESS:
+                text = self.help_formatter._expand_help(action)
+            if isinstance(value, dict):
+                if text:
+                    self.set_yaml_group_comment(text, cfg, key, depth)
+                self.set_dict_comments(value, action, depth + 1, parser, group_titles, full_key)
+            else:
+                if text:
+                    self.set_yaml_argument_comment(text, cfg, key, depth)
+                if isinstance(value, list):
+                    self.set_list_comments(value, action, depth + 1)
+
+    def set_dict_comments(
+        self,
+        cfg: ruamelCommentedMap,
+        action: Action | None,
+        depth: int,
+        parser: "ArgumentParser | None" = None,
+        group_titles: dict | None = None,
+        prefix: str = "",
+        typehint=None,
+    ) -> None:
+        """Sets the comments for the keys of a config object nested in another one.
+
+        Args:
+            cfg: The ruamel.yaml object.
+            action: The action that corresponds to the given config object, if any.
+            depth: The nested level of the keys of the config object.
+            parser: The parser in which the keys are searched when not a class type.
+            group_titles: Map of parser keys to the title of the group they belong to.
+            prefix: The parser key that corresponds to the given config object.
+            typehint: The type of the config object, defaults to the type of the action.
+        """
+        if is_subclass_spec(cfg):
+            self.set_subclass_comments(cfg, action, depth)
+            return
+        if typehint is None and isinstance(action, ActionTypeHint):
+            typehint = action._typehint
+        class_parser = get_closed_type_parser(typehint, action)
+        value_typehint = get_mapping_value_typehint(typehint)
+        if class_parser is not None:
+            self.set_comments(cfg, class_parser, get_group_titles(class_parser), depth=depth)
+        elif value_typehint is not None:
+            for value in cfg.values():
+                if isinstance(value, dict):
+                    self.set_dict_comments(value, action, depth + 1, typehint=value_typehint)
+                elif isinstance(value, list):
+                    self.set_list_comments(value, action, depth + 1, typehint=value_typehint)
+        elif parser is not None:
+            assert group_titles is not None
+            self.set_comments(cfg, parser, group_titles, prefix, depth)
+
+    def set_subclass_comments(self, cfg: ruamelCommentedMap, action: Action | None, depth: int) -> None:
+        """Sets the comments for the init args of a subclass spec.
+
+        Args:
+            cfg: The ruamel.yaml object that has the class_path key.
+            action: The action that corresponds to the given config object, if any.
+            depth: The nested level of the keys of the config object.
+        """
+        init_args = cfg.get("init_args")
+        if not isinstance(init_args, dict):
+            return
+        class_parser = get_class_parser(cfg["class_path"], action)
+        if class_parser is None:
+            return
+        title = get_class_group_title(class_parser)
+        if title:
+            self.set_yaml_group_comment(title, cfg, "init_args", depth)
+        self.set_comments(init_args, class_parser, get_group_titles(class_parser), depth=depth + 1)
+
+    def set_list_comments(self, cfg: list, action: Action | None, depth: int, typehint=None) -> None:
+        """Sets the comments for the class types that are items of a list.
+
+        Args:
+            cfg: The ruamel.yaml object.
+            action: The action that corresponds to the given list.
+            depth: The nested level of the keys of the items of the list.
+            typehint: The type of the list, defaults to the type of the action.
+        """
+        for item in cfg:
+            if isinstance(item, dict):
+                self.set_dict_comments(item, action, depth, typehint=typehint)
+                remove_leading_blank_line(item)
+            elif isinstance(item, list):
+                self.set_list_comments(item, action, depth + 1, typehint=typehint)
+
     def set_yaml_start_comment(
         self,
         text: str,

jsonargparse/_typehints.py

@@ -1542,11 +1542,13 @@ def is_instance_factory_protocol(class_type, logger=None):
 
 
 def is_subclass_spec(val):
-    is_class = isinstance(val, (dict, Namespace)) and "class_path" in val
-    if is_class:
-        keys = getattr(val, "__dict__", val).keys()
-        is_class = len(set(keys) - _subclass_spec_keys) == 0
-    return is_class
+    if isinstance(val, Namespace):
+        keys = val.__dict__.keys()  # only the top level keys of the namespace
+    elif isinstance(val, dict):
+        keys = val.keys()
+    else:
+        return False
+    return "class_path" in keys and len(set(keys) - _subclass_spec_keys) == 0
 
 
 def subclass_spec_as_namespace(val, prev_val=None):

jsonargparse_tests/conftest.py

@@ -13,7 +13,7 @@
 
 import pytest
 
-from jsonargparse import ArgumentParser, set_parsing_settings
+from jsonargparse import SUPPRESS, ActionParser, ArgumentParser, set_parsing_settings
 from jsonargparse._loaders_dumpers import json_compact_dump, json_load, yaml_dump, yaml_load
 from jsonargparse._optionals import (
     docstring_parser_support,
@@ -22,6 +22,7 @@
     jsonschema_support,
     omegaconf_support,
     pyyaml_available,
+    ruamel_support,
     toml_load_available,
     url_support,
 )
@@ -71,6 +72,11 @@
     reason="docstring-parser package is required",
 )
 
+skip_if_yaml_comments_unavailable = pytest.mark.skipif(
+    not (ruamel_support and docstring_parser_support),
+    reason="ruamel.yaml and docstring-parser packages are required",
+)
+
 skip_if_requests_unavailable = pytest.mark.skipif(
     not url_support,
     reason="requests package is required",
@@ -160,6 +166,18 @@ def example_parser() -> ArgumentParser:
     return parser
 
 
+@pytest.fixture
+def print_parser(parser, subparser) -> ArgumentParser:
+    parser.description = "cli tool"
+    parser.add_argument("--cfg", action="config")
+    parser.add_argument("--v0", help=SUPPRESS, default="0")
+    parser.add_argument("--v1", help="Option v1.", default=1)
+    parser.add_argument("--g1.v2", help="Option v2.", default="2")
+    subparser.add_argument("--v3")
+    parser.add_argument("--g2", action=ActionParser(parser=subparser))
+    return parser
+
+
 @pytest.fixture
 def parsing_settings_patch():
     with patch.dict("jsonargparse._common.parsing_settings"):

jsonargparse_tests/test_cli.py

@@ -14,9 +14,14 @@
 
 from jsonargparse import CLI, auto_cli, auto_parser, capture_parser, lazy_instance
 from jsonargparse._namespace import Namespace
-from jsonargparse._optionals import docstring_parser_support, ruamel_support
+from jsonargparse._optionals import docstring_parser_support
 from jsonargparse.typing import final
-from jsonargparse_tests.conftest import json_or_yaml_dump, json_or_yaml_load, skip_if_docstring_parser_unavailable
+from jsonargparse_tests.conftest import (
+    json_or_yaml_dump,
+    json_or_yaml_load,
+    skip_if_docstring_parser_unavailable,
+    skip_if_yaml_comments_unavailable,
+)
 
 
 def get_cli_stdout(*args, **kwargs) -> str:
@@ -399,8 +404,7 @@ def test_function_and_class_print_config_before_subcommands():
     assert {"Cmd2": {"i1": "d", "method2": {"m2": 0}}} == json_or_yaml_load(out)
 
 
-@skip_if_docstring_parser_unavailable
-@pytest.mark.skipif(not ruamel_support, reason="ruamel.yaml package is required")
+@skip_if_yaml_comments_unavailable
 def test_function_and_class_print_config_comments():
     out = get_cli_stdout([cmd1, Cmd2, cmd3], args=["--print_config=comments", "Cmd2", "method2"])
     assert "# Description of Cmd2" in out