From 9344a197247d021f97aca5ba808d0f7f6a2b7dec Mon Sep 17 00:00:00 2001 From: Tobias Dijkhuis Date: Tue, 11 Aug 2026 11:34:51 +0200 Subject: [PATCH 1/3] Add optional dependency for 'toml', and specify that it needs to be installed for python versions below 3.11. --- README.md | 12 +++++++----- setup.py | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 21e993f..0802d2c 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ Python's command line parsing modules such as argparse have very limited support for config files and environment variables, so this module extends argparse to add these features. -**API docs:** https://bw2.github.io/ConfigArgParse/ - +**API docs:** https://bw2.github.io/ConfigArgParse/ + **PyPI:** http://pypi.python.org/pypi/ConfigArgParse ## Install @@ -158,14 +158,14 @@ Only command line args that have a long version (eg. one that starts with '--') can be set in a config file. For example, "--color" can be set by putting "color=green" in a config file. The config file syntax depends on the constructor arg: `config_file_parser_class` which can be set to one of the provided -classes: +classes: - `DefaultConfigFileParser` - `YAMLConfigFileParser` - `ConfigparserConfigFileParser` - `IniConfigParser` - `TomlConfigParser` - `CompositeConfigParser` - + or to your own subclass of the `ConfigFileParser` abstract class. #### *DefaultConfigFileParser* - the full range of valid syntax is: @@ -332,6 +332,8 @@ parser = configargparse.ArgParser( [TOML](https://github.com/toml-lang/toml/blob/main/toml.md) parser. This config parser can be used to integrate with `pyproject.toml` files. +Requires installation of [toml](https://pypi.org/project/toml/) for Python versions below 3.11. + Example: ```toml @@ -387,7 +389,7 @@ parser = configargparse.ArgParser( ... ``` -Note that it's required to put the TOML parser first because the INI syntax basically would accept anything whereas TOML. +Note that it's required to put the TOML parser first because the INI syntax basically would accept anything whereas TOML is a bit more strict. ## ArgParser Singletons diff --git a/setup.py b/setup.py index 4f95c21..5a347e2 100644 --- a/setup.py +++ b/setup.py @@ -46,8 +46,7 @@ def launch_http_server(directory): logging.debug("All network port. ") except Exception as e: logging.error( - "ERROR: while starting an HTTP server to serve " - "the coverage report: %s" % e + "ERROR: while starting an HTTP server to serve the coverage report: %s" % e ) @@ -82,7 +81,7 @@ def launch_http_server(directory): tests_require = [ "black", "mock", - "toml", + "toml; python_version < '3.11'", "PyYAML", "pytest", "pytest-cov", @@ -126,6 +125,7 @@ def launch_http_server(directory): install_requires=install_requires, tests_require=tests_require, extras_require={ + "toml": ["toml; python_version < '3.11'"], "yaml": ["PyYAML"], "test": tests_require, }, From 8da5faf77a107b893b413bd74e0eba4c004b8c0f Mon Sep 17 00:00:00 2001 From: Tobias Dijkhuis Date: Tue, 11 Aug 2026 11:57:23 +0200 Subject: [PATCH 2/3] Make 'toml' work with binary reads, and raise different exceptions if 'toml' and 'tomllib' are both not available, but they are being used in a CompositeConfigParser. Same for YAML. Correct printing of parser name --- configargparse.py | 49 ++++++++++++++++++++---------------- tests/test_configargparse.py | 34 +++++++------------------ 2 files changed, 36 insertions(+), 47 deletions(-) diff --git a/configargparse.py b/configargparse.py index 01cb589..8cb9886 100644 --- a/configargparse.py +++ b/configargparse.py @@ -134,6 +134,10 @@ class ConfigFileParserException(Exception): """Raised when config file parsing failed.""" +class ConfigFileParserMissingDependency(Exception): + """Raised when an optional dependency is missing.""" + + class DefaultConfigFileParser(ConfigFileParser): """ Based on a simplified subset of INI and YAML formats. Here is the @@ -316,7 +320,7 @@ def _load_yaml(self): try: import yaml except ImportError: - raise ConfigFileParserException( + raise ConfigFileParserMissingDependency( "Could not import yaml. " "It can be installed by running 'pip install PyYAML'" ) @@ -373,7 +377,7 @@ def serialize(self, items, default_flow_style=False): Provides `configargparse.ConfigFileParser` classes to parse ``TOML`` and ``INI`` files with **mandatory** support for sections. Useful to integrate configuration into project files like ``pyproject.toml`` or ``setup.cfg``. -`TomlConfigParser` usage: +`TomlConfigParser` usage: >>> TomlParser = TomlConfigParser(['tool.my_super_tool']) # Simple TOML parser. >>> parser = ArgumentParser(..., default_config_files=['./pyproject.toml'], config_file_parser_class=TomlParser) @@ -522,7 +526,15 @@ def parse(self, stream): """Parses the keys and values from a TOML config file.""" # Use tomllib (Python 3.11+) if available, otherwise fall back to toml package try: - import tomllib + import tomllib as toml + except ImportError: + try: + import toml + except ImportError: + raise ConfigFileParserMissingDependency( + "Could not import toml or tomllib. " + "toml can be installed by running 'pip install toml'" + ) # tomllib.load() requires binary mode, so use loads() for stream compatibility try: @@ -530,15 +542,7 @@ def parse(self, stream): # If content is bytes, decode it; if string, use as-is if isinstance(content, bytes): content = content.decode("utf-8") - config = tomllib.loads(content) - except Exception as e: - raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) - except ImportError: - # Fall back to toml package (supports text mode) - import toml - - try: - config = toml.load(stream) + config = toml.loads(content) except Exception as e: raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) @@ -764,7 +768,7 @@ class CompositeConfigParser(ConfigFileParser): def __init__(self, config_parser_types): super().__init__() - self.parsers = [p() for p in config_parser_types] + self.parsers: list[ConfigFileParser] = [p() for p in config_parser_types] def __call__(self): return self @@ -774,6 +778,9 @@ def parse(self, stream): for i, p in enumerate(self.parsers): try: return p.parse(stream) # type: ignore[no-any-return] + except ConfigFileParserMissingDependency as e: + msg = f"Cannot use parser {p.__class__.__name__} without optional dependency." + raise ConfigFileParserMissingDependency(msg) from e except Exception as e: errors.append(e) # Try to seek back to beginning for next parser @@ -804,7 +811,7 @@ def guess_format_name(classname): msg = "Uses multiple config parser settings (in order): \n" for i, parser in enumerate(self.parsers): - msg += f"[{i+1}] {guess_format_name(parser.__class__.__name__)}: {parser.get_syntax_description()} \n" + msg += f"[{i + 1}] {guess_format_name(parser.__class__.__name__)}: {parser.get_syntax_description()} \n" return msg def serialize(self, items): @@ -1447,9 +1454,9 @@ def convert_item_to_command_line_arg(self, action, key, value): if action is not None and isinstance( action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE ): - assert isinstance( - value, str - ), "config parser should convert anything that is not a list to string." + assert isinstance(value, str), ( + "config parser should convert anything that is not a list to string." + ) if value.lower() in ("true", "yes", "on", "1"): if not is_boolean_optional_action(action): args.append(command_line_key) @@ -1725,7 +1732,7 @@ def format_help(self): added_config_file_help = True msg += ( - "Args that start with '%s' can also be set in " "a config file" + "Args that start with '%s' can also be set in a config file" ) % cc config_arg_string = " or ".join( a.option_strings[0] for a in config_path_actions if a.option_strings @@ -1832,15 +1839,13 @@ def add_argument(self, *args, **kwargs): if action.is_positional_arg and env_var: raise ValueError("env_var can't be set for a positional arg.") if action.is_config_file_arg and not isinstance(action, argparse._StoreAction): - raise ValueError("arg with is_config_file_arg=True must have " "action='store'") + raise ValueError("arg with is_config_file_arg=True must have action='store'") if action.is_write_out_config_file_arg: error_prefix = "arg with is_write_out_config_file_arg=True " if not isinstance(action, argparse._StoreAction): raise ValueError(error_prefix + "must have action='store'") if is_config_file_arg: - raise ValueError( - error_prefix + "can't also have " "is_config_file_arg=True" - ) + raise ValueError(error_prefix + "can't also have is_config_file_arg=True") return action diff --git a/tests/test_configargparse.py b/tests/test_configargparse.py index b6ec6be..7f835bb 100644 --- a/tests/test_configargparse.py +++ b/tests/test_configargparse.py @@ -581,7 +581,7 @@ def testAddArgsErrors(self): ) self.assertRaisesRegex( ValueError, - "arg with " "is_write_out_config_file_arg=True must have action='store'", + "arg with is_write_out_config_file_arg=True must have action='store'", self.add_arg, "-y", "--Y", @@ -1029,7 +1029,7 @@ def testGlobalInstances(self, name=None): self.assertEqual(p.prog, "prog") self.assertRaisesRegex( ValueError, - "kwargs besides 'name' can only be " "passed in the first time", + "kwargs besides 'name' can only be passed in the first time", configargparse.getArgumentParser, name, prog="prog", @@ -1106,7 +1106,8 @@ def testConstructor_ConfigFileArgs(self): r"%s:\n" r" -h, --help\s+ show this help message and exit\n" rf" -c{short_c}, --config CONFIG_FILE\s+ my config file\n" - r" --genome GENOME\s+ Path to genome file\n\n" % OPTIONAL_ARGS_STRING + r" --genome GENOME\s+ Path to genome file\n\n" + % OPTIONAL_ARGS_STRING + 5 * r"(.+\s*)", ) @@ -1188,7 +1189,8 @@ def test_FormatHelp(self): r"Config file syntax allows: key=value, flag=true, stuff=\[a,b,c\] " r"\(for details, see syntax at https://goo.gl/R74nmi\). " r"In general, command-line values override config file values " - r"which override defaults. ".replace(" ", r"\s*") % OPTIONAL_ARGS_STRING, + r"which override defaults. ".replace(" ", r"\s*") + % OPTIONAL_ARGS_STRING, ) def test_FormatHelpProg(self): @@ -1969,7 +1971,7 @@ def testYAMLConfigFileParser_Basic(self): import yaml except: logging.warning( - "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" + "WARNING: PyYAML not installed. Couldn't test YAMLConfigFileParser" ) return @@ -1989,7 +1991,7 @@ def testYAMLConfigFileParser_All(self): import yaml except: logging.warning( - "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" + "WARNING: PyYAML not installed. Couldn't test YAMLConfigFileParser" ) return @@ -2019,7 +2021,7 @@ def testYAMLConfigFileParser_w_ArgumentParser_parsed_values(self): import yaml except: raise AssertionError( - "WARNING: PyYAML not installed. " "Couldn't test YAMLConfigFileParser" + "WARNING: PyYAML not installed. Couldn't test YAMLConfigFileParser" ) return @@ -2090,10 +2092,6 @@ def test_advanced(self): parser = configargparse.TomlConfigParser(["tool.section"]) self.assertEqual(parser.parse(f), {"key1": "toml1", "key2": ["1", "2", "3"]}) - @unittest.skipIf( - sys.version_info < (3, 11), - "Binary mode only supported with tomllib (Python 3.11+)", - ) def test_binary_read_works(self): # Binary mode now works with tomllib (Python 3.11+) f = self.write_toml_file( @@ -2105,20 +2103,6 @@ def test_binary_read_works(self): # Should successfully parse binary stream self.assertEqual(parser.parse(f), {"key1": "toml1"}) - @unittest.skipIf( - sys.version_info >= (3, 11), - "On Python 3.11+, tomllib handles binary; this tests the toml package fallback", - ) - def test_binary_read_fails_without_tomllib(self): - # Without tomllib (Python < 3.11), binary streams should fail - f = self.write_toml_file( - b"""[section]\nkey1 = "toml1"\n""", - obj=BytesIO, - ) - parser = configargparse.TomlConfigParser(["section"]) - with self.assertRaises(configargparse.ConfigFileParserException): - parser.parse(f) - def test_serialize_with_section(self): parser = configargparse.TomlConfigParser(["section"]) try: From f067eaca5e2588deb8960365e33cac0ececb1d25 Mon Sep 17 00:00:00 2001 From: Tobias Dijkhuis Date: Tue, 11 Aug 2026 14:27:56 +0200 Subject: [PATCH 3/3] Test failing, and add warning for dubious order of parsers in composite parser. --- configargparse.py | 42 +++++++++++++++++++++++++----------- tests/test_configargparse.py | 35 ++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/configargparse.py b/configargparse.py index 8cb9886..80da6f2 100644 --- a/configargparse.py +++ b/configargparse.py @@ -17,6 +17,7 @@ import types from collections import OrderedDict import textwrap +import warnings from io import StringIO ACTION_TYPES_THAT_DONT_NEED_A_VALUE = [ @@ -530,21 +531,21 @@ def parse(self, stream): except ImportError: try: import toml - except ImportError: + except ImportError as e: raise ConfigFileParserMissingDependency( "Could not import toml or tomllib. " "toml can be installed by running 'pip install toml'" - ) + ) from e - # tomllib.load() requires binary mode, so use loads() for stream compatibility - try: - content = stream.read() - # If content is bytes, decode it; if string, use as-is - if isinstance(content, bytes): - content = content.decode("utf-8") - config = toml.loads(content) - except Exception as e: - raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) + # tomllib.load() requires binary mode, so use loads() for stream compatibility + try: + content = stream.read() + # If content is bytes, decode it; if string, use as-is + if isinstance(content, bytes): + content = content.decode("utf-8") + config = toml.loads(content) + except Exception as e: + raise ConfigFileParserException("Couldn't parse TOML file: %s" % e) # convert to dict and filter based on section names result = OrderedDict() @@ -581,7 +582,7 @@ def serialize(self, items): try: import toml except ImportError: - raise ConfigFileParserException( + raise ConfigFileParserMissingDependency( "The 'toml' package is required for TOML serialization. " "Install it with: pip install toml" ) @@ -770,6 +771,19 @@ def __init__(self, config_parser_types): super().__init__() self.parsers: list[ConfigFileParser] = [p() for p in config_parser_types] + seen_ini = False + for parser in self.parsers: + if not seen_ini and isinstance(parser, IniConfigParser): + seen_ini = True + continue + if seen_ini and isinstance(parser, TomlConfigParser): + warnings.warn( + "IniConfigParser was found before TomlConfigParser in parsers for " + "CompositeConfigParser. This might lead to a TOML file being " + "parsed as an INI file. Reorder the parsers.", + category=SyntaxWarning, + ) + def __call__(self): return self @@ -777,10 +791,12 @@ def parse(self, stream): errors = [] for i, p in enumerate(self.parsers): try: + print(f"USING PARSER {p.__class__.__name__}") return p.parse(stream) # type: ignore[no-any-return] except ConfigFileParserMissingDependency as e: msg = f"Cannot use parser {p.__class__.__name__} without optional dependency." - raise ConfigFileParserMissingDependency(msg) from e + print(msg) + raise except Exception as e: errors.append(e) # Try to seek back to beginning for next parser diff --git a/tests/test_configargparse.py b/tests/test_configargparse.py index 7f835bb..0bd717b 100644 --- a/tests/test_configargparse.py +++ b/tests/test_configargparse.py @@ -10,7 +10,9 @@ import types import unittest from unittest import mock +import warnings import textwrap +import pytest from io import BytesIO, StringIO @@ -2092,6 +2094,18 @@ def test_advanced(self): parser = configargparse.TomlConfigParser(["tool.section"]) self.assertEqual(parser.parse(f), {"key1": "toml1", "key2": ["1", "2", "3"]}) + def test_read_without_toml_packages(self): + f = self.write_toml_file(""" + [tool.section] + key1 = "toml1" + key2 = [1, 2, 3] + """) + parser = configargparse.TomlConfigParser(["tool.section"]) + + with mock.patch.dict(sys.modules, {"toml": None, "tomllib": None}): + with self.assertRaises(configargparse.ConfigFileParserMissingDependency): + parser.parse(f) + def test_binary_read_works(self): # Binary mode now works with tomllib (Python 3.11+) f = self.write_toml_file( @@ -2143,7 +2157,7 @@ def test_serialize_without_toml_package(self): import unittest.mock as mock with mock.patch.dict(sys.modules, {"toml": None}): - with self.assertRaises(configargparse.ConfigFileParserException): + with self.assertRaises(configargparse.ConfigFileParserMissingDependency): parser.serialize(items) @@ -2187,9 +2201,9 @@ def setUp(self): default_config_files=["config.yaml", "config.toml", "config.ini"], config_file_parser_class=configargparse.CompositeConfigParser( [ - configargparse.IniConfigParser(["section"], False), configargparse.TomlConfigParser(["section"]), configargparse.YAMLConfigFileParser, + configargparse.IniConfigParser(["section"], False), ] ), ) @@ -2282,6 +2296,23 @@ def test_toml_extra(self): with self.assertRaises(SystemExit): self.parser.parse_args([]) + def test_composite_fails_if_missing_dependency(self): + self.write_yaml_file() + self.write_ini_file() + + with mock.patch.dict(sys.modules, {"toml": None, "tomllib": None}): + with self.assertRaises(configargparse.ConfigFileParserMissingDependency): + self.parser.parse_args([]) + + def test_composite_warns_if_wrong_order(self): + with self.assertWarns(SyntaxWarning): + composite = configargparse.CompositeConfigParser( + [ + configargparse.IniConfigParser(["section"], False), + configargparse.TomlConfigParser(["section"]), + ] + )() + def test_composite_serialize_delegates_to_first_parser(self): composite = configargparse.CompositeConfigParser( [