From 746081e933367ecd08ba9e90b23ae44e4cce8140 Mon Sep 17 00:00:00 2001 From: Vanessasaurus <814322+vsoch@users.noreply.github.com> Date: Fri, 12 Mar 2021 22:31:26 -0700 Subject: adding spack -c to set one off config arguments (#22251) This pull request will add the ability for a user to add a configuration argument on the fly, on the command line, e.g.,: ```bash $ spack -c config:install_tree:root:/path/to/config.yaml -c packages:all:compiler:[gcc] list --help ``` The above command doesn't do anything (I'm just getting help for list) but you can imagine having another root of packages, and updating it on the fly for a command (something I'd like to do in the near future!) I've moved the logic for config_add that used to be in spack/cmd/config.py into spack/config.py proper, and now both the main.py (where spack commands live) and spack/cmd/config.py use these functions. I only needed spack config add, so I didn't move the others. We can move the others if there are also needed in multiple places. --- lib/spack/spack/cmd/config.py | 64 +------------------------------- lib/spack/spack/config.py | 75 ++++++++++++++++++++++++++++++++++++++ lib/spack/spack/main.py | 8 +++- lib/spack/spack/test/cmd/config.py | 26 ++++++++----- lib/spack/spack/test/config.py | 31 ++++++++++++++++ 5 files changed, 132 insertions(+), 72 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index cc7923d450..68ce8362fd 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -200,71 +200,11 @@ def config_add(args): scope, section = _get_scope_and_section(args) - # Updates from file if args.file: - # Get file as config dict - data = spack.config.read_config_file(args.file) - if any(k in data for k in spack.schema.env.keys): - data = ev.config_dict(data) - - # update all sections from config dict - # We have to iterate on keys to keep overrides from the file - for section in data.keys(): - if section in spack.config.section_schemas.keys(): - # Special handling for compiler scope difference - # Has to be handled after we choose a section - if scope is None: - scope = spack.config.default_modify_scope(section) - - value = data[section] - existing = spack.config.get(section, scope=scope) - new = spack.config.merge_yaml(existing, value) - - spack.config.set(section, new, scope) + spack.config.add_from_file(args.file, scope=scope) if args.path: - components = spack.config.process_config_path(args.path) - - has_existing_value = True - path = '' - override = False - for idx, name in enumerate(components[:-1]): - # First handle double colons in constructing path - colon = '::' if override else ':' if path else '' - path += colon + name - if getattr(name, 'override', False): - override = True - else: - override = False - - # Test whether there is an existing value at this level - existing = spack.config.get(path, scope=scope) - - if existing is None: - has_existing_value = False - # We've nested further than existing config, so we need the - # type information for validation to know how to handle bare - # values appended to lists. - existing = spack.config.get_valid_type(path) - - # construct value from this point down - value = syaml.load_config(components[-1]) - for component in reversed(components[idx + 1:-1]): - value = {component: value} - break - - if has_existing_value: - path, _, value = args.path.rpartition(':') - value = syaml.load_config(value) - existing = spack.config.get(path, scope=scope) - - # append values to lists - if isinstance(existing, list) and not isinstance(value, list): - value = [value] - - # merge value into existing - new = spack.config.merge_yaml(existing, value) - spack.config.set(path, new, scope) + spack.config.add(args.path, scope=scope) def config_remove(args): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 5f707e883f..f5fc488f52 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -806,6 +806,81 @@ def _config(): config = llnl.util.lang.Singleton(_config) +def add_from_file(filename, scope=None): + """Add updates to a config from a filename + """ + import spack.environment as ev + + # Get file as config dict + data = read_config_file(filename) + if any(k in data for k in spack.schema.env.keys): + data = ev.config_dict(data) + + # update all sections from config dict + # We have to iterate on keys to keep overrides from the file + for section in data.keys(): + if section in section_schemas.keys(): + # Special handling for compiler scope difference + # Has to be handled after we choose a section + if scope is None: + scope = default_modify_scope(section) + + value = data[section] + existing = get(section, scope=scope) + new = merge_yaml(existing, value) + + # We cannot call config.set directly (set is a type) + config.set(section, new, scope) + + +def add(fullpath, scope=None): + """Add the given configuration to the specified config scope. + Add accepts a path. If you want to add from a filename, use add_from_file""" + + components = process_config_path(fullpath) + + has_existing_value = True + path = '' + override = False + for idx, name in enumerate(components[:-1]): + # First handle double colons in constructing path + colon = '::' if override else ':' if path else '' + path += colon + name + if getattr(name, 'override', False): + override = True + else: + override = False + + # Test whether there is an existing value at this level + existing = get(path, scope=scope) + + if existing is None: + has_existing_value = False + # We've nested further than existing config, so we need the + # type information for validation to know how to handle bare + # values appended to lists. + existing = get_valid_type(path) + + # construct value from this point down + value = syaml.load_config(components[-1]) + for component in reversed(components[idx + 1:-1]): + value = {component: value} + break + + if has_existing_value: + path, _, value = fullpath.rpartition(':') + value = syaml.load_config(value) + existing = get(path, scope=scope) + + # append values to lists + if isinstance(existing, list) and not isinstance(value, list): + value = [value] + + # merge value into existing + new = merge_yaml(existing, value) + config.set(path, new, scope) + + def get(path, default=None, scope=None): """Module-level wrapper for ``Configuration.get()``.""" return config.get(path, default, scope) diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 6b323f8eb3..1aaf447324 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -40,7 +40,6 @@ import spack.util.path import spack.util.executable as exe from spack.error import SpackError - #: names of profile statistics stat_names = pstats.Stats.sort_arg_dict_default @@ -358,6 +357,9 @@ def make_argument_parser(**kwargs): '--color', action='store', default='auto', choices=('always', 'never', 'auto'), help="when to colorize output (default: auto)") + parser.add_argument( + '-c', '--config', default=None, action="append", dest="config_vars", + help="add one or more custom, one off config settings.") parser.add_argument( '-C', '--config-scope', dest='config_scopes', action='append', metavar='DIR', help="add a custom configuration scope") @@ -463,6 +465,10 @@ def setup_main_options(args): tty.warn("You asked for --insecure. Will NOT check SSL certificates.") spack.config.set('config:verify_ssl', False, scope='command_line') + # Use the spack config command to handle parsing the config strings + for config_var in (args.config_vars or []): + spack.config.add(path=config_var, scope="command_line") + # when to use color (takes always, auto, or never) color.set_color_when(args.color) diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py index 2739df1e1e..32996310a6 100644 --- a/lib/spack/spack/test/cmd/config.py +++ b/lib/spack/spack/test/cmd/config.py @@ -87,6 +87,7 @@ repos: def test_config_edit(): """Ensure `spack config edit` edits the right paths.""" + dms = spack.config.default_modify_scope('compilers') dms_path = spack.config.config.scopes[dms].path user_path = spack.config.config.scopes['user'].path @@ -204,20 +205,27 @@ def test_config_add_override_leaf(mutable_empty_config): def test_config_add_update_dict(mutable_empty_config): - config('add', 'packages:all:compiler:[gcc]') - config('add', 'packages:all:version:1.0.0') + config('add', 'packages:all:version:[1.0.0]') output = config('get', 'packages') - expected = """packages: - all: - compiler: [gcc] - version: - - 1.0.0 -""" - + expected = 'packages:\n all:\n version: [1.0.0]\n' assert output == expected +def test_config_with_c_argument(mutable_empty_config): + + # I don't know how to add a spack argument to a Spack Command, so we test this way + config_file = 'config:install_root:root:/path/to/config.yaml' + parser = spack.main.make_argument_parser() + args = parser.parse_args(['-c', config_file]) + assert config_file in args.config_vars + + # Add the path to the config + config("add", args.config_vars[0], scope='command_line') + output = config("get", 'config') + assert "config:\n install_root:\n - root: /path/to/config.yaml" in output + + def test_config_add_ordered_dict(mutable_empty_config): config('add', 'mirrors:first:/path/to/first') config('add', 'mirrors:second:/path/to/second') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 06cfd018f1..a62cfa1cc3 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -258,6 +258,37 @@ def test_write_to_same_priority_file(mock_low_high_config, compiler_specs): repos_low = {'repos': ["/some/path"]} repos_high = {'repos': ["/some/other/path"]} +# Test setting config values via path in filename + + +def test_add_config_path(): + + # Try setting a new install tree root + path = "config:install_tree:root:/path/to/config.yaml" + spack.config.add(path, scope="command_line") + set_value = spack.config.get('config')['install_tree']['root'] + assert set_value == '/path/to/config.yaml' + + # Now a package:all setting + path = "packages:all:compiler:[gcc]" + spack.config.add(path, scope="command_line") + compilers = spack.config.get('packages')['all']['compiler'] + assert "gcc" in compilers + + +def test_add_config_filename(mock_low_high_config, tmpdir): + + config_yaml = tmpdir.join('config-filename.yaml') + config_yaml.ensure() + with config_yaml.open('w') as f: + syaml.dump_config(config_low, f) + + spack.config.add_from_file(str(config_yaml), scope="low") + assert "build_stage" in spack.config.get('config') + build_stages = spack.config.get('config')['build_stage'] + for stage in config_low['config']['build_stage']: + assert stage in build_stages + # repos def test_write_list_in_memory(mock_low_high_config): -- cgit v1.2.3-60-g2f50