From df84fe0b58522597c9c16e59ceeabb9c3163c1ea Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 16 Jun 2016 18:10:38 -0700 Subject: Added init_config parameter so we don't initialize the config when user runs spack compiler add. Should ONLY add what user specifies. Find compilers takes in paths parameter. --- lib/spack/spack/cmd/compiler.py | 6 +++-- lib/spack/spack/compilers/__init__.py | 44 +++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 030aa77c30..94d83512e1 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -77,10 +77,12 @@ def compiler_add(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers(scope=args.scope)] + if c.spec not in spack.compilers.all_compilers(scope=args.scope, + init_config=False)] if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope, + init_config=False) n = len(compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 4b546c2cbf..6de2d87b32 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -79,12 +79,10 @@ def _to_dict(compiler): if compiler.alias: d['alias'] = compiler.alias - return { - 'compiler': d - } + return {'compiler': d} -def get_compiler_config(scope=None): +def get_compiler_config(scope=None, init_config=True): """Return the compiler configuration for the specified architecture. """ def init_compiler_config(): @@ -100,7 +98,8 @@ def get_compiler_config(scope=None): # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if not config: + empty_config = [] + if not config and init_config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -112,18 +111,20 @@ def get_compiler_config(scope=None): if not site_config: init_compiler_config() config = spack.config.get_config('compilers', scope=scope) - - return config + return config + else: + return empty_config -def add_compilers_to_config(compilers, scope=None): +def add_compilers_to_config(compilers, scope=None, init_config=True): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope) + compiler_config = get_compiler_config(scope, init_config) + print compiler_config for compiler in compilers: compiler_config.append(_to_dict(compiler)) @@ -148,25 +149,23 @@ def remove_compiler_from_config(compiler_spec, scope=None): spack.config.update_config('compilers', compiler_config, scope) _cache_config_file = {} - -def all_compilers_config(scope=None): +def all_compilers_config(scope=None, init_config=True): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. global _cache_config_file #Create a cache of the config file so we don't load all the time. - if not _cache_config_file: - _cache_config_file = get_compiler_config(scope) + _cache_config_file = get_compiler_config(scope, init_config) return _cache_config_file - else: return _cache_config_file -def all_compilers(scope=None): + +def all_compilers(scope=None, init_config=True): # Return compiler specs from the merged config. return [spack.spec.CompilerSpec(s['compiler']['spec']) - for s in all_compilers_config(scope)] + for s in all_compilers_config(scope, init_config)] def default_compiler(): @@ -181,7 +180,13 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(): +def find_compilers(*paths): + """ Call find compilers help and return the list it finds using + the operating system method (PATHS, MODULES) """ + return _find_compilers(*paths) + + +def _find_compilers(*paths): """Return a list of compilers found in the suppied paths. This invokes the find_compilers() method for each operating system associated with the host platform, and appends @@ -190,9 +195,8 @@ def find_compilers(): # Find compilers for each operating system class oss = all_os_classes() compiler_lists = [] - for os in oss: - compiler_lists.extend(os.find_compilers()) - + for o in oss: + compiler_lists.extend(o.find_compilers(*paths)) return compiler_lists def supported_compilers(): -- cgit v1.2.3-60-g2f50 From fe5eec5230dd86e2ed247a178a54230f631c1c60 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 17 Jun 2016 09:16:03 -0700 Subject: 'find and 'add' are options for adding compilers via command line --- lib/spack/spack/cmd/compiler.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 94d83512e1..3a496492ff 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -69,7 +69,7 @@ def setup_parser(subparser): help="Configuration scope to read from.") -def compiler_add(args): +def compiler_find(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" paths = args.add_paths @@ -79,7 +79,6 @@ def compiler_add(args): compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope, init_config=False)] - if compilers: spack.compilers.add_compilers_to_config(compilers, scope=args.scope, init_config=False) @@ -139,9 +138,10 @@ def compiler_list(args): def compiler(parser, args): - action = { 'add' : compiler_add, - 'remove' : compiler_remove, - 'rm' : compiler_remove, - 'info' : compiler_info, - 'list' : compiler_list } + action = {'add' : compiler_find, + 'find' : compiler_find, + 'remove' : compiler_remove, + 'rm' : compiler_remove, + 'info' : compiler_info, + 'list' : compiler_list } action[args.compiler_command](args) -- cgit v1.2.3-60-g2f50 From c703bfb54d0452418fea95839226fb5bab7e5893 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 00:48:08 -0700 Subject: Got rid of init_config flag since we don't need it. Instead we update the cache config if there are any changes --- lib/spack/spack/compilers/__init__.py | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6de2d87b32..0543ce8af1 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -52,6 +52,7 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _other_instance_vars = ['modules', 'operating_system'] +_cache_config_file = [] # TODO: customize order in config file if platform.system() == 'Darwin': @@ -82,7 +83,7 @@ def _to_dict(compiler): return {'compiler': d} -def get_compiler_config(scope=None, init_config=True): +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ def init_compiler_config(): @@ -97,9 +98,7 @@ def get_compiler_config(scope=None, init_config=True): # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. -# if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - empty_config = [] - if not config and init_config: + if not config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -112,11 +111,13 @@ def get_compiler_config(scope=None, init_config=True): init_compiler_config() config = spack.config.get_config('compilers', scope=scope) return config + elif config: + return config else: - return empty_config + return [] # Return empty list which we will later append to. -def add_compilers_to_config(compilers, scope=None, init_config=True): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: @@ -124,10 +125,10 @@ def add_compilers_to_config(compilers, scope=None, init_config=True): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope, init_config) - print compiler_config for compiler in compilers: compiler_config.append(_to_dict(compiler)) - + global _cache_config_file + _cache_config_file = compiler_config spack.config.update_config('compilers', compiler_config, scope) @@ -140,32 +141,35 @@ def remove_compiler_from_config(compiler_spec, scope=None): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope) - matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] - if len(matches) == 1: - del compiler_config[matches[0][0]] - else: + config_length = len(compiler_config) + + filtered_compiler_config = [comp for comp in compiler_config + if spack.spec.CompilerSpec(comp['compiler']['spec']) != compiler_spec] + # Need a better way for this + global _cache_config_file + _cache_config_file = filtered_compiler_config # Update the cache for changes + if len(filtered_compiler_config) == config_length: # No items removed CompilerSpecInsufficientlySpecificError(compiler_spec) + spack.config.update_config('compilers', filtered_compiler_config, scope) - spack.config.update_config('compilers', compiler_config, scope) -_cache_config_file = {} -def all_compilers_config(scope=None, init_config=True): +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - _cache_config_file = get_compiler_config(scope, init_config) + _cache_config_file = get_compiler_config(scope) return _cache_config_file else: return _cache_config_file -def all_compilers(scope=None, init_config=True): +def all_compilers(scope=None): # Return compiler specs from the merged config. return [spack.spec.CompilerSpec(s['compiler']['spec']) - for s in all_compilers_config(scope, init_config)] + for s in all_compilers_config(scope)] def default_compiler(): -- cgit v1.2.3-60-g2f50 From ccb62b46204531fa4ceeadd3b1722b0bb0e30f4f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:49:39 -0700 Subject: Getting rid of no longer used init_config arg --- lib/spack/spack/cmd/compiler.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3a496492ff..c95045ef85 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -77,11 +77,9 @@ def compiler_find(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers(scope=args.scope, - init_config=False)] + if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: - spack.compilers.add_compilers_to_config(compilers, scope=args.scope, - init_config=False) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) s = 's' if n > 1 else '' filename = spack.config.get_config_filename(args.scope, 'compilers') @@ -94,7 +92,6 @@ def compiler_find(args): def compiler_remove(args): cspec = CompilerSpec(args.compiler_spec) compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) - if not compilers: tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: -- cgit v1.2.3-60-g2f50 From 36275f8e6eec5b7238e17743a3ec9a2aff371941 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:50:55 -0700 Subject: More clean up of init_config, also no need for a different find_compilers function. --- lib/spack/spack/compilers/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 0543ce8af1..ae72b743b2 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -124,7 +124,7 @@ def add_compilers_to_config(compilers, scope=None): - compilers: a list of Compiler objects. - scope: configuration scope to modify. """ - compiler_config = get_compiler_config(scope, init_config) + compiler_config = get_compiler_config(scope) for compiler in compilers: compiler_config.append(_to_dict(compiler)) global _cache_config_file @@ -185,12 +185,6 @@ def default_compiler(): def find_compilers(*paths): - """ Call find compilers help and return the list it finds using - the operating system method (PATHS, MODULES) """ - return _find_compilers(*paths) - - -def _find_compilers(*paths): """Return a list of compilers found in the suppied paths. This invokes the find_compilers() method for each operating system associated with the host platform, and appends @@ -203,6 +197,7 @@ def _find_compilers(*paths): compiler_lists.extend(o.find_compilers(*paths)) return compiler_lists + def supported_compilers(): """Return a set of names of compilers supported by Spack. -- cgit v1.2.3-60-g2f50 From 15e6b88a8b5f1e7ae1d06b4b4edc1126b4d68420 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 10:51:38 -0700 Subject: Adding new tests for compiler command for spack's test suite --- lib/spack/spack/test/cmd/test_compiler_cmd.py | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 lib/spack/spack/test/cmd/test_compiler_cmd.py diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py new file mode 100644 index 0000000000..81bc7aacf2 --- /dev/null +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -0,0 +1,34 @@ +import spack.spec +import spack.cmd.compiler +import spack.compilers +from spack.test.mock_packages_test import * + + +class MockArgs(object): + def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): + self.add_paths = add_paths + self.scope = scope + self.compiler_spec = compiler_spec + self.all = all + + +class CompilerCmdTest(MockPackagesTest): + """ Test compiler commands for add and remove """ + + def test_compiler_remove(self): + args = MockArgs(all=True, compiler_spec='gcc@4.5.0') + spack.cmd.compiler.compiler_remove(args) + compilers = spack.compilers.all_compilers() + self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers) + + def test_compiler_add(self): + # Probably not a good a assumption but might try finding local + # compilers + # installed in /usr + compilers = spack.compilers.all_compilers() + s = set(compilers) + args = MockArgs(add_paths=["/usr"]) + spack.cmd.compiler.compiler_find(args) + new_compilers = spack.compilers.all_compilers() + new_compiler = [x for x in new_compilers if x not in s] + self.assertTrue(new_compiler) -- cgit v1.2.3-60-g2f50 From 64c83638fff9716477ecbc3d87989b6b8437b05f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 18 Jun 2016 11:11:46 -0700 Subject: Adding test to list of test_names --- lib/spack/spack/test/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 97f142e746..fb91f24721 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -40,7 +40,7 @@ test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'pack 'cc', 'link_tree', 'spec_yaml', 'optional_deps', 'make_executable', 'configure_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', - 'cmd.uninstall', 'cmd.test_install'] + 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd'] def list_tests(): -- cgit v1.2.3-60-g2f50 From 84e21703bde307c920e83e6cd4452feaacbaf5f2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 20 Jun 2016 09:28:24 -0700 Subject: Test no longer assumes compilers exist in /usr. - makes a fake gcc instead, and tests that. --- lib/spack/spack/test/cmd/test_compiler_cmd.py | 67 +++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/test/cmd/test_compiler_cmd.py b/lib/spack/spack/test/cmd/test_compiler_cmd.py index 81bc7aacf2..d89814154b 100644 --- a/lib/spack/spack/test/cmd/test_compiler_cmd.py +++ b/lib/spack/spack/test/cmd/test_compiler_cmd.py @@ -1,8 +1,16 @@ +import os +import shutil +from tempfile import mkdtemp + +from llnl.util.filesystem import set_executable, mkdirp + import spack.spec import spack.cmd.compiler import spack.compilers +from spack.version import Version from spack.test.mock_packages_test import * +test_version = '4.5-spacktest' class MockArgs(object): def __init__(self, add_paths=[], scope=None, compiler_spec=None, all=None): @@ -12,23 +20,62 @@ class MockArgs(object): self.all = all +def make_mock_compiler(): + """Make a directory containing a fake, but detectable compiler.""" + mock_compiler_dir = mkdtemp() + bin_dir = os.path.join(mock_compiler_dir, 'bin') + mkdirp(bin_dir) + + gcc_path = os.path.join(bin_dir, 'gcc') + gxx_path = os.path.join(bin_dir, 'g++') + gfortran_path = os.path.join(bin_dir, 'gfortran') + + with open(gcc_path, 'w') as f: + f.write("""\ +#!/bin/sh + +for arg in "$@"; do + if [ "$arg" = -dumpversion ]; then + echo '%s' + fi +done +""" % test_version) + + # Create some mock compilers in the temporary directory + set_executable(gcc_path) + shutil.copy(gcc_path, gxx_path) + shutil.copy(gcc_path, gfortran_path) + + return mock_compiler_dir + + class CompilerCmdTest(MockPackagesTest): """ Test compiler commands for add and remove """ + def test_compiler_remove(self): args = MockArgs(all=True, compiler_spec='gcc@4.5.0') spack.cmd.compiler.compiler_remove(args) compilers = spack.compilers.all_compilers() self.assertTrue(spack.spec.CompilerSpec("gcc@4.5.0") not in compilers) + def test_compiler_add(self): - # Probably not a good a assumption but might try finding local - # compilers - # installed in /usr - compilers = spack.compilers.all_compilers() - s = set(compilers) - args = MockArgs(add_paths=["/usr"]) - spack.cmd.compiler.compiler_find(args) - new_compilers = spack.compilers.all_compilers() - new_compiler = [x for x in new_compilers if x not in s] - self.assertTrue(new_compiler) + # compilers available by default. + old_compilers = set(spack.compilers.all_compilers()) + + # add our new compiler and find again. + compiler_dir = make_mock_compiler() + + try: + args = MockArgs(add_paths=[compiler_dir]) + spack.cmd.compiler.compiler_find(args) + + # ensure new compiler is in there + new_compilers = set(spack.compilers.all_compilers()) + new_compiler = new_compilers - old_compilers + self.assertTrue(new_compiler) + self.assertTrue(new_compiler.pop().version == Version(test_version)) + + finally: + shutil.rmtree(compiler_dir, ignore_errors=True) -- cgit v1.2.3-60-g2f50