chainlib

Generic blockchain access library and tooling
Log | Files | Refs | README | LICENSE

commit 96e0a97c3b51f4237973611ff46116664224d5f9
parent f1ffff8b903610a3c8d050a2567e1cc509c1a5c7
Author: nolash <dev@holbrook.no>
Date:   Mon, 23 Aug 2021 08:27:48 +0200

Add docstrings for config, test for config

Diffstat:
Mchainlib/cli/arg.py | 62++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mchainlib/cli/config.py | 113+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------
Mrun_tests.sh | 2+-
Atests/test_cli.py | 94+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atests/testdata/config/config.ini | 2++
Atests/testdata/config/default/chain.ini | 2++
Atests/testdata/config/default/user/chain.ini | 2++
Atests/testdata/config/foo/config.ini | 2++
8 files changed, 253 insertions(+), 26 deletions(-)

diff --git a/chainlib/cli/arg.py b/chainlib/cli/arg.py @@ -17,6 +17,13 @@ logg = logging.getLogger(__name__) def stdin_arg(): + """Retreive input arguments from stdin if they exist. + + Method does not block, and expects arguments to be ready on stdin before being called. + + :rtype: str + :returns: Input arguments string + """ h = select.select([sys.stdin], [], [], 0) if len(h[0]) > 0: v = h[0][0].read() @@ -25,6 +32,23 @@ def stdin_arg(): class ArgumentParser(argparse.ArgumentParser): + """Extends the standard library argument parser to construct arguments based on configuration flags. + + The extended class is set up to facilitate piping of single positional arguments via stdin. For this reason, positional arguments should be added using the locally defined add_positional method instead of add_argument. + + Calls chainlib.cli.args.ArgumentParser.process_flags with arg_flags and env arguments, see the method's documentation for further details. + + :param arg_flags: Argument flag bit vector to generate configuration values for. + :type arg_flags: chainlib.cli.Flag + :param env: Environment variables + :type env: dict + :param usage: Usage string, passed to parent + :type usage: str + :param description: Description string, passed to parent + :type description: str + :param epilog: Epilog string, passed to parent + :type epilog: str + """ def __init__(self, arg_flags=0x0f, env=os.environ, usage=None, description=None, epilog=None, *args, **kwargs): super(ArgumentParser, self).__init__(usage=usage, description=description, epilog=epilog) @@ -33,10 +57,34 @@ class ArgumentParser(argparse.ArgumentParser): def add_positional(self, name, type=str, help=None, required=True): + """Add a positional argument. + + Stdin piping will only be possible in the event a single positional argument is defined. + + If the "required" is set, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line. + + :param name: Attribute name of argument + :type name: str + :param type: Argument type + :type type: str + :param help: Help string + :type help: str + :param required: If true, argument will be set to required + :type required: bool + """ self.pos_args.append((name, type, help, required,)) def parse_args(self, argv=sys.argv[1:]): + """Overrides the argparse.ArgumentParser.parse_args method. + + Implements reading arguments from stdin if a single positional argument is defined (and not set to required). + + If the "required" was set for the single positional argument, the resulting parsed arguments must have provided a value either from stdin or excplicitly on the command line. + + :param argv: Argument vector to process + :type argv: list + """ if len(self.pos_args) == 1: arg = self.pos_args[0] self.add_argument(arg[0], nargs='?', type=arg[1], default=stdin_arg(), help=arg[2]) @@ -62,6 +110,20 @@ class ArgumentParser(argparse.ArgumentParser): def process_flags(self, arg_flags, env): + """Configures the arguments of the parser using the provided flags. + + Environment variables are used for default values for: + + CONFINI_DIR: -c, --config + CONFINI_ENV_PREFIX: --env-prefix + + This method is called by the constructor, and is not intended to be called directly. + + :param arg_flags: Argument flag bit vector to generate configuration values for. + :type arg_flags: chainlib.cli.Flag + :param env: Environment variables + :type env: dict + """ if arg_flags & Flag.VERBOSE: self.add_argument('-v', action='store_true', help='Be verbose') self.add_argument('-vv', action='store_true', help='Be more verbose') diff --git a/chainlib/cli/config.py b/chainlib/cli/config.py @@ -11,22 +11,87 @@ from .base import ( default_config_dir as default_parent_config_dir, ) -#logg = logging.getLogger(__name__) -logg = logging.getLogger() +logg = logging.getLogger(__name__) def logcallback(config): + """Callback to dump config contents to log after completed config load + + :param config: Config object + :type config: confini.Config + """ logg.debug('config loaded:\n{}'.format(config)) class Config(confini.Config): + """Extends confini.Config. + Processes argument parser attributes to configuration variables. + + Provides sane configuration overrides and fallbacks. + + """ default_base_config_dir = default_parent_config_dir default_fee_limit = 0 @classmethod - def from_args(cls, args, arg_flags, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback): - + def from_args(cls, args, arg_flags=0x0f, env=os.environ, extra_args={}, base_config_dir=None, default_config_dir=None, user_config_dir=None, default_fee_limit=None, logger=None, load_callback=logcallback): + """Parses arguments in argparse.ArgumentParser instance, then match and override configuration values that match them. + + The method processes all known argument flags from chainlib.cli.Flag passed in the "args" argument. + + All entries in extra_args may be used to associate arguments not defined in the argument flags with configuration variables, in the following manner: + + - The value of argparser.ArgumentParser instance attribute with the dictionary key string is looked up. + - If the value is None (defined but empty), any existing value for the configuration directive will be kept. + - If the value of the extra_args dictionary entry is None, then the value will be stored in the configuration under the upper-case value of the key string, prefixed with "_" ("foo_bar" becomes "_FOO_BAR") + - If the value of the extra_args dictionary entries is a string, then the value will be stored in the configuration under that literal string. + + Missing attributes defined by both the "args" and "extra_args" arguments will both raise an AttributeError. + + The python package "confini" is used to process and render the configuration. + + The confini config schema is determined in the following manner: + + - If nothing is set, only the config folder in chainlib.data.config will be used as schema. + - If base_config_dir is a string or list, the config directives from the path(s) will be added to the schema. + + The global override config directories are determined in the following manner: + + - If no default_config_dir is defined, the environment variable CONFINI_DIR will be used. + - If default_config_dir is a string or list, values from the config directives from the path(s) will override those defined in the schema(s). + + The user override config directories work the same way as the global ones, but the namespace - if defined - are dependent on them. They are only applied if the CONFIG arg flag is set. User override config directories are determined in the following manner: + + - If --config argument is not defined and the pyxdg module is present, the first available xdg basedir is used. + - If --config argument is defined, the directory defined by its value will be used. + + The namespace, if defined, will be stored under the CONFIG_USER_NAMESPACE configuration key. + + :param args: Argument parser object + :type args: argparse.ArgumentParser + :param arg_flags: Argument flags defining which arguments to process into configuration. + :type arg_flags: confini.cli.args.ArgumentParser + :param env: Environment variables selection + :type env: dict + :param extra_args: Extra arguments to process and override. + :type extra_args: dict + :param base_config_dir: Path(s) to one or more directories extending the base chainlib config schema. + :type base_config_dir: list or str + :param default_config_dir: Path(s) to one or more directories overriding the defaults defined in the schema config directories. + :type default_config_dir: list or str + :param user_config_dir: User xdg config basedir, with namespace + :type user_config_dir: str + :param default_fee_limit: Default value for fee limit argument + :type default_fee_limit: int + :param logger: Logger instance to use during argument processing (will use package namespace logger if None) + :type logger: logging.Logger + :param load_callback: Callback receiving config instance as argument after config processing and load completes. + :type load_callback: function + :raises AttributeError: Attribute defined in flag not found in parsed arguments + :rtype: confini.Config + :return: Processed configuation + """ if logger == None: logger = logging.getLogger() @@ -58,7 +123,7 @@ class Config(confini.Config): # confini dir env var will be used for override configs only in this case if default_config_dir == None: - default_config_dir = os.environ.get('CONFINI_DIR') + default_config_dir = env.get('CONFINI_DIR') if default_config_dir != None: if isinstance(default_config_dir, str): default_config_dir = [default_config_dir] @@ -67,32 +132,27 @@ class Config(confini.Config): # process config command line arguments if arg_flags & Flag.CONFIG: - effective_user_config_dir = getattr(args, 'config', None) if effective_user_config_dir == None: effective_user_config_dir = user_config_dir - if effective_user_config_dir != None: - if config_dir == None: - if getattr(args, 'namespace', None) != None: - arg_config_dir = os.path.join(effective_user_config_dir, args.namespace) - config_dir = [cls.default_base_config_dir, effective_user_config_dir] - logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir)) - else: - if getattr(args, 'namespace', None) != None: - arg_config_dir = os.path.join(effective_user_config_dir, args.namespace) - override_config_dirs.append(effective_user_config_dir) - logg.debug('using config arg as config override {}'.format(effective_user_config_dir)) - - - if config_dir == None: - if default_config_dir == None: - default_config_dir = default_parent_config_dir - config_dir = default_config_dir - override_config_dirs = [] + if getattr(args, 'namespace', None) != None: + effective_user_config_dir = os.path.join(effective_user_config_dir, args.namespace) + #if config_dir == None: + # config_dir = [cls.default_base_config_dir, effective_user_config_dir] + # logg.debug('using config arg as base config addition {}'.format(effective_user_config_dir)) + #else: + override_config_dirs.append(effective_user_config_dir) + logg.debug('using config arg as config override {}'.format(effective_user_config_dir)) + + #if config_dir == None: + # if default_config_dir == None: + # default_config_dir = default_parent_config_dir + # config_dir = default_config_dir + # override_config_dirs = [] env_prefix = getattr(args, 'env_prefix', None) - config = confini.Config(config_dir, env_prefix=args.env_prefix, override_dirs=override_config_dirs) + config = confini.Config(config_dir, env_prefix=env_prefix, override_dirs=override_config_dirs) config.process() args_override = {} @@ -140,6 +200,9 @@ class Config(confini.Config): config.add(getattr(args, 'raw'), '_RAW') + if arg_flags & Flag.CONFIG: + config.add(getattr(args, 'namespace'), 'CONFIG_USER_NAMESPACE') + for k in extra_args.keys(): v = extra_args[k] if v == None: diff --git a/run_tests.sh b/run_tests.sh @@ -2,7 +2,7 @@ set -e set -x -export PYTHONPATH=${PYTHONPATH:.} +export PYTHONPATH=$PYTHONPATH:. for f in `ls tests/*.py`; do python $f done diff --git a/tests/test_cli.py b/tests/test_cli.py @@ -0,0 +1,94 @@ +# standard imports +import unittest +import os + +# local imports +import chainlib.cli +from chainlib.cli.base import argflag_std_base + +script_dir = os.path.dirname(os.path.realpath(__file__)) +data_dir = os.path.join(script_dir, 'testdata') +config_dir = os.path.join(data_dir, 'config') + + +class TestCli(unittest.TestCase): + + def test_args_process_single(self): + ap = chainlib.cli.arg.ArgumentParser() + argv = [ + '-vv', + '-n', + 'foo', + ] + args = ap.parse_args(argv) + config = chainlib.cli.config.Config.from_args(args) + self.assertEqual(config.get('CONFIG_USER_NAMESPACE'), 'foo') + + + def test_args_process_schema_override(self): + ap = chainlib.cli.arg.ArgumentParser() + args = ap.parse_args([]) + config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir) + self.assertEqual(config.get('FOO_BAR'), 'baz') + + + def test_args_process_arg_override(self): + ap = chainlib.cli.arg.ArgumentParser() + argv = [ + '-c', + config_dir, + '-n', + 'foo', + ] + args = ap.parse_args(argv) + config = chainlib.cli.config.Config.from_args(args, base_config_dir=config_dir) + self.assertEqual(config.get('FOO_BAR'), 'bazbazbaz') + + + def test_args_process_internal_override(self): + ap = chainlib.cli.arg.ArgumentParser() + args = ap.parse_args() + default_config_dir = os.path.join(config_dir, 'default') + config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir) + self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo') + + user_config_dir = os.path.join(default_config_dir, 'user') + config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=user_config_dir) + self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo') + + config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir) + self.assertEqual(config.get('CHAIN_SPEC'), 'baz:bar:13:foo') + + ap = chainlib.cli.arg.ArgumentParser() + argv = [ + '-n', + 'user', + ] + args = ap.parse_args(argv) + config = chainlib.cli.config.Config.from_args(args, default_config_dir=default_config_dir, user_config_dir=default_config_dir) + self.assertEqual(config.get('CHAIN_SPEC'), 'foo:foo:666:foo') + + + def test_args_process_extra(self): + ap = chainlib.cli.arg.ArgumentParser() + ap.add_argument('--foo', type=str) + argv = [ + '--foo', + 'bar', + ] + args = ap.parse_args(argv) + extra_args = { + 'foo': None, + } + config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args) + self.assertEqual(config.get('_FOO'), 'bar') + + extra_args = { + 'foo': 'FOOFOO', + } + config = chainlib.cli.config.Config.from_args(args, extra_args=extra_args) + self.assertEqual(config.get('FOOFOO'), 'bar') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/testdata/config/config.ini b/tests/testdata/config/config.ini @@ -0,0 +1,2 @@ +[foo] +bar = baz diff --git a/tests/testdata/config/default/chain.ini b/tests/testdata/config/default/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = baz:bar:13:foo diff --git a/tests/testdata/config/default/user/chain.ini b/tests/testdata/config/default/user/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = foo:foo:666:foo diff --git a/tests/testdata/config/foo/config.ini b/tests/testdata/config/foo/config.ini @@ -0,0 +1,2 @@ +[foo] +bar = bazbazbaz