commit 96e0a97c3b51f4237973611ff46116664224d5f9
parent f1ffff8b903610a3c8d050a2567e1cc509c1a5c7
Author: nolash <dev@holbrook.no>
Date: Mon, 23 Aug 2021 08:27:48 +0200
Add docstrings for config, test for config
Diffstat:
8 files changed, 253 insertions(+), 26 deletions(-)
diff --git a/chainlib/cli/arg.py b/chainlib/cli/arg.py
@@ -17,6 +17,13 @@ logg = logging.getLogger(__name__)
def stdin_arg():
+ """Retreive input arguments from stdin if they exist.
+
+ Method does not block, and expects arguments to be ready on stdin before being called.
+
+ :rtype: str
+ :returns: Input arguments string
+ """
h = select.select([sys.stdin], [], [], 0)
if len(h[0]) > 0:
v = h[0][0].read()
@@ -25,6 +32,23 @@ def stdin_arg():
class ArgumentParser(argparse.ArgumentParser):
+ """Extends the standard library argument parser to construct arguments based on configuration flags.
+
+ The extended class is set up to facilitate piping of single positional arguments via stdin. For this reason, positional arguments should be added using the locally defined add_positional method instead of add_argument.
+
+ Calls chainlib.cli.args.ArgumentParser.process_flags with arg_flags and env arguments, see the method's documentation for further details.
+
+ :param arg_flags: Argument flag bit vector to generate configuration values for.
+ :type arg_flags: chainlib.cli.Flag
+ :param env: Environment variables
+ :type env: dict
+ :param usage: Usage string, passed to parent
+ :type usage: str
+ :param description: Description string, passed to parent
+ :type description: str
+ :param epilog: Epilog string, passed to parent
+ :type epilog: str
+ """
def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs):
super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog)
@@ -33,10 +57,34 @@ class ArgumentParser(argparse.ArgumentParser):
def add_positional(self, name, type=str, help=None, required=True):
+ """Add a positional argument.
+
+ Stdin piping will only be possible in the event a single positional argument is defined.
+
+ If the "required" is set, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line.
+
+ :param name: Attribute name of argument
+ :type name: str
+ :param type: Argument type
+ :type type: str
+ :param help: Help string
+ :type help: str
+ :param required: If true, argument will be set to required
+ :type required: bool
+ """
self.pos_args.append((name, type, help, required,))
def parse_args(self, argv=sys.argv[1:]):
+ """Overrides the argparse.ArgumentParser.parse_args method.
+
+ Implements reading arguments from stdin if a single positional argument is defined (and not set to required).
+
+ If the "required" was set for the single positional argument, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line.
+
+ :param argv: Argument vector to process
+ :type argv: list
+ """
if len(self.pos_args) == 1:
arg = self.pos_args[0]
self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2])
@@ -62,6 +110,20 @@ class ArgumentParser(argparse.ArgumentParser):
def process_flags(self, arg_flags, env):
+ """Configures the arguments of the parser using the provided flags.
+
+ Environment variables are used for default values for:
+
+ CONFINI_DIR: -c, --config
+ CONFINI_ENV_PREFIX: --env-prefix
+
+ This method is called by the constructor, and is not intended to be called directly.
+
+ :param arg_flags: Argument flag bit vector to generate configuration values for.
+ :type arg_flags: chainlib.cli.Flag
+ :param env: Environment variables
+ :type env: dict
+ """
if arg_flags & Flag.VERBOSE:
self.add_argument('-v', action='store_true', help='Be verbose')
self.add_argument('-vv', action='store_true', help='Be more verbose')
diff --git a/chainlib/cli/config.py b/chainlib/cli/config.py
@@ -11,22 +11,87 @@ from .base import (
default_config_dir as default_parent_config_dir,
)
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
+logg = logging.getLogger(__name__)
def logcallback(config):
+ """Callback to dump config contents to log after completed config load
+
+ :param config: Config object
+ :type config: confini.Config
+ """
logg.debug('config loaded:\n{}'.format(config))
class Config(confini.Config):
+ """Extends confini.Config.
+ Processes argument parser attributes to configuration variables.
+
+ Provides sane configuration overrides and fallbacks.
+
+ """
default_base_config_dir = default_parent_config_dir
default_fee_limit = 0
@classmethod
- def from_args(cls, args, arg_flags, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback):
-
+ def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback):
+ """Parses arguments in argparse.ArgumentParser instance, then match and override configuration values that match them.
+
+ The method processes all known argument flags from chainlib.cli.Flag passed in the "args" argument.
+
+ All entries in extra_args may be used to associate arguments not defined in the argument flags with configuration variables, in the following manner:
+
+ - The value of argparser.ArgumentParser instance attribute with the dictionary key string is looked up.
+ - If the value is None (defined but empty), any existing value for the configuration directive will be kept.
+ - If the value of the extra_args dictionary entry is None, then the value will be stored in the configuration under the upper-case value of the key string, prefixed with "_" ("foo_bar" becomes "_FOO_BAR")
+ - If the value of the extra_args dictionary entries is a string, then the value will be stored in the configuration under that literal string.
+
+ Missing attributes defined by both the "args" and "extra_args" arguments will both raise an AttributeError.
+
+ The python package "confini" is used to process and render the configuration.
+
+ The confini config schema is determined in the following manner:
+
+ - If nothing is set, only the config folder in chainlib.data.config will be used as schema.
+ - If base_config_dir is a string or list, the config directives from the path(s) will be added to the schema.
+
+ The global override config directories are determined in the following manner:
+
+ - If no default_config_dir is defined, the environment variable CONFINI_DIR will be used.
+ - If default_config_dir is a string or list, values from the config directives from the path(s) will override those defined in the schema(s).
+
+ The user override config directories work the same way as the global ones, but the namespace - if defined - are dependent on them. They are only applied if the CONFIG arg flag is set. User override config directories are determined in the following manner:
+
+ - If --config argument is not defined and the pyxdg module is present, the first available xdg basedir is used.
+ - If --config argument is defined, the directory defined by its value will be used.
+
+ The namespace, if defined, will be stored under the CONFIG_USER_NAMESPACE configuration key.
+
+ :param args: Argument parser object
+ :type args: argparse.ArgumentParser
+ :param arg_flags: Argument flags defining which arguments to process into configuration.
+ :type arg_flags: confini.cli.args.ArgumentParser
+ :param env: Environment variables selection
+ :type env: dict
+ :param extra_args: Extra arguments to process and override.
+ :type extra_args: dict
+ :param base_config_dir: Path(s) to one or more directories extending the base chainlib config schema.
+ :type base_config_dir: list or str
+ :param default_config_dir: Path(s) to one or more directories overriding the defaults defined in the schema config directories.
+ :type default_config_dir: list or str
+ :param user_config_dir: User xdg config basedir, with namespace
+ :type user_config_dir: str
+ :param default_fee_limit: Default value for fee limit argument
+ :type default_fee_limit: int
+ :param logger: Logger instance to use during argument processing (will use package namespace logger if None)
+ :type logger: logging.Logger
+ :param load_callback: Callback receiving config instance as argument after config processing and load completes.
+ :type load_callback: function
+ :raises AttributeError: Attribute defined in flag not found in parsed arguments
+ :rtype: confini.Config
+ :return: Processed configuation
+ """
if logger == None:
logger = logging.getLogger()
@@ -58,7 +123,7 @@ class Config(confini.Config):
# confini dir env var will be used for override configs only in this case
if default_config_dir == None:
- default_config_dir = os.environ.get('CONFINI_DIR')
+ default_config_dir = env.get('CONFINI_DIR')
if default_config_dir != None:
if isinstance(default_config_dir, str):
default_config_dir = [default_config_dir]
@@ -67,32 +132,27 @@ class Config(confini.Config):
# process config command line arguments
if arg_flags & Flag.CONFIG:
-
effective_user_config_dir = getattr(args, 'config', None)
if effective_user_config_dir == None:
effective_user_config_dir = user_config_dir
-
if effective_user_config_dir != None:
- if config_dir == None:
- if getattr(args, 'namespace', None) != None:
- arg_config_dir = os.path.join(effective_user_config_dir, args.namespace)
- config_dir = [cls.default_base_config_dir, effective_user_config_dir]
- logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir))
- else:
- if getattr(args, 'namespace', None) != None:
- arg_config_dir = os.path.join(effective_user_config_dir, args.namespace)
- override_config_dirs.append(effective_user_config_dir)
- logg.debug('using config arg as config override {}'.format(effective_user_config_dir))
-
-
- if config_dir == None:
- if default_config_dir == None:
- default_config_dir = default_parent_config_dir
- config_dir = default_config_dir
- override_config_dirs = []
+ if getattr(args, 'namespace', None) != None:
+ effective_user_config_dir = os.path.join(effective_user_config_dir, args.namespace)
+ #if config_dir == None:
+ # config_dir = [cls.default_base_config_dir, effective_user_config_dir]
+ # logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir))
+ #else:
+ override_config_dirs.append(effective_user_config_dir)
+ logg.debug('using config arg as config override {}'.format(effective_user_config_dir))
+
+ #if config_dir == None:
+ # if default_config_dir == None:
+ # default_config_dir = default_parent_config_dir
+ # config_dir = default_config_dir
+ # override_config_dirs = []
env_prefix = getattr(args, 'env_prefix', None)
- config = confini.Config(config_dir, env_prefix=args.env_prefix, override_dirs=override_config_dirs)
+ config = confini.Config(config_dir, env_prefix=env_prefix, override_dirs=override_config_dirs)
config.process()
args_override = {}
@@ -140,6 +200,9 @@ class Config(confini.Config):
config.add(getattr(args, 'raw'), '_RAW')
+ if arg_flags & Flag.CONFIG:
+ config.add(getattr(args, 'namespace'), 'CONFIG_USER_NAMESPACE')
+
for k in extra_args.keys():
v = extra_args[k]
if v == None:
diff --git a/run_tests.sh b/run_tests.sh
@@ -2,7 +2,7 @@
set -e
set -x
-export PYTHONPATH=${PYTHONPATH:.}
+export PYTHONPATH=$PYTHONPATH:.
for f in `ls tests/*.py`; do
python $f
done
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -0,0 +1,94 @@
+# standard imports
+import unittest
+import os
+
+# local imports
+import chainlib.cli
+from chainlib.cli.base import argflag_std_base
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+data_dir = os.path.join(script_dir, 'testdata')
+config_dir = os.path.join(data_dir, 'config')
+
+
+class TestCli(unittest.TestCase):
+
+ def test_args_process_single(self):
+ ap = chainlib.cli.arg.ArgumentParser()
+ argv = [
+ '-vv',
+ '-n',
+ 'foo',
+ ]
+ args = ap.parse_args(argv)
+ config = chainlib.cli.config.Config.from_args(args)
+ self.assertEqual(config.get('CONFIG_USER_NAMESPACE'), 'foo')
+
+
+ def test_args_process_schema_override(self):
+ ap = chainlib.cli.arg.ArgumentParser()
+ args = ap.parse_args([])
+ config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir)
+ self.assertEqual(config.get('FOO_BAR'), 'baz')
+
+
+ def test_args_process_arg_override(self):
+ ap = chainlib.cli.arg.ArgumentParser()
+ argv = [
+ '-c',
+ config_dir,
+ '-n',
+ 'foo',
+ ]
+ args = ap.parse_args(argv)
+ config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir)
+ self.assertEqual(config.get('FOO_BAR'), 'bazbazbaz')
+
+
+ def test_args_process_internal_override(self):
+ ap = chainlib.cli.arg.ArgumentParser()
+ args = ap.parse_args()
+ default_config_dir = os.path.join(config_dir, 'default')
+ config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir)
+ self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo')
+
+ user_config_dir = os.path.join(default_config_dir, 'user')
+ config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=user_config_dir)
+ self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo')
+
+ config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir)
+ self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo')
+
+ ap = chainlib.cli.arg.ArgumentParser()
+ argv = [
+ '-n',
+ 'user',
+ ]
+ args = ap.parse_args(argv)
+ config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir)
+ self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo')
+
+
+ def test_args_process_extra(self):
+ ap = chainlib.cli.arg.ArgumentParser()
+ ap.add_argument('--foo', type=str)
+ argv = [
+ '--foo',
+ 'bar',
+ ]
+ args = ap.parse_args(argv)
+ extra_args = {
+ 'foo': None,
+ }
+ config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args)
+ self.assertEqual(config.get('_FOO'), 'bar')
+
+ extra_args = {
+ 'foo': 'FOOFOO',
+ }
+ config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args)
+ self.assertEqual(config.get('FOOFOO'), 'bar')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/testdata/config/config.ini b/tests/testdata/config/config.ini
@@ -0,0 +1,2 @@
+[foo]
+bar = baz
diff --git a/tests/testdata/config/default/chain.ini b/tests/testdata/config/default/chain.ini
@@ -0,0 +1,2 @@
+[chain]
+spec = baz:bar:13:foo
diff --git a/tests/testdata/config/default/user/chain.ini b/tests/testdata/config/default/user/chain.ini
@@ -0,0 +1,2 @@
+[chain]
+spec = foo:foo:666:foo
diff --git a/tests/testdata/config/foo/config.ini b/tests/testdata/config/foo/config.ini
@@ -0,0 +1,2 @@
+[foo]
+bar = bazbazbaz
