From c174fe6199eaa86fccf3fafa9bbfbff6f3385089 Mon Sep 17 00:00:00 2001 From: Saravan Pantham Date: Mon, 6 Apr 2015 14:03:09 -0700 Subject: Fixed BLAS and Lapack installations --- lib/spack/llnl/util/filesystem.py | 1 - lib/spack/llnl/util/filesystem.py.save | 0 lib/spack/llnl/util/filesystem.py.save.1 | 229 +++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 lib/spack/llnl/util/filesystem.py.save create mode 100644 lib/spack/llnl/util/filesystem.py.save.1 (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 3b34e04740..8eff642938 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -40,7 +40,6 @@ from tempfile import NamedTemporaryFile import llnl.util.tty as tty from spack.util.compression import ALLOWED_ARCHIVE_TYPES - def filter_file(regex, repl, *filenames, **kwargs): """Like sed, but uses python regular expressions. diff --git a/lib/spack/llnl/util/filesystem.py.save b/lib/spack/llnl/util/filesystem.py.save new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/llnl/util/filesystem.py.save.1 b/lib/spack/llnl/util/filesystem.py.save.1 new file mode 100644 index 0000000000..82cfe965b2 --- /dev/null +++ b/lib/spack/llnl/util/filesystem.py.save.1 @@ -0,0 +1,229 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +__all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', + 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] + +import os +import sys +import re +import shutil +import stat +import errno +import getpass +from contextlib import contextmanager, closing +from tempfile import NamedTemporaryFile + +import llnl.util.tty as tty +from spack.util.compression import ALLOWED_ARCHIVE_TYPES + +def filter_file(regex, repl, *filenames, **kwargs): + """Like sed, but uses python regular expressions. + + Filters every line of file through regex and replaces the file + with a filtered version. Preserves mode of filtered files. + + As with re.sub, ``repl`` can be either a string or a callable. + If it is a callable, it is passed the match object and should + return a suitable replacement string. If it is a string, it + can contain ``\1``, ``\2``, etc. to represent back-substitution + as sed would allow. + + Keyword Options: + string[=False] If True, treat regex as a plain string. + backup[=True] Make a backup files suffixed with ~ + ignore_absent[=False] Ignore any files that don't exist. + """ + string = kwargs.get('string', False) + backup = kwargs.get('backup', True) + ignore_absent = kwargs.get('ignore_absent', False) + + # Allow strings to use \1, \2, etc. for replacement, like sed + if not callable(repl): + unescaped = repl.replace(r'\\', '\\') + def replace_groups_with_groupid(m): + def groupid_to_group(x): + return m.group(int(x.group(1))) + return re.sub(r'\\([1-9])', groupid_to_group, unescaped) + repl = replace_groups_with_groupid + + if string: + regex = re.escape(regex) + + for filename in filenames: + backup = filename + "~" + + if ignore_absent and not os.path.exists(filename): + continue + + shutil.copy(filename, backup) + try: + with closing(open(backup)) as infile: + with closing(open(filename, 'w')) as outfile: + for line in infile: + foo = re.sub(regex, repl, line) + outfile.write(foo) + except: + # clean up the original file on failure. + shutil.move(backup, filename) + raise + + finally: + if not backup: + shutil.rmtree(backup, ignore_errors=True) + + +def change_sed_delimiter(old_delim, new_delim, *filenames): + """Find all sed search/replace commands and change the delimiter. + e.g., if the file contains seds that look like 's///', you can + call change_sed_delimeter('/', '@', file) to change the + delimiter to '@'. + + NOTE that this routine will fail if the delimiter is ' or ". + Handling those is left for future work. + """ + assert(len(old_delim) == 1) + assert(len(new_delim) == 1) + + # TODO: handle these cases one day? + assert(old_delim != '"') + assert(old_delim != "'") + assert(new_delim != '"') + assert(new_delim != "'") + + whole_lines = "^s@([^@]*)@(.*)@[gIp]$" + whole_lines = whole_lines.replace('@', old_delim) + + single_quoted = r"'s@((?:\\'|[^@'])*)@((?:\\'|[^'])*)@[gIp]?'" + single_quoted = single_quoted.replace('@', old_delim) + + double_quoted = r'"s@((?:\\"|[^@"])*)@((?:\\"|[^"])*)@[gIp]?"' + double_quoted = double_quoted.replace('@', old_delim) + + repl = r's@\1@\2@g' + repl = repl.replace('@', new_delim) + + for f in filenames: + filter_file(whole_lines, repl, f) + filter_file(single_quoted, "'%s'" % repl, f) + filter_file(double_quoted, '"%s"' % repl, f) + + +def set_install_permissions(path): + """Set appropriate permissions on the installed file.""" + if os.path.isdir(path): + os.chmod(path, 0755) + else: + os.chmod(path, 0644) + + +def install(src, dest): + """Manually install a file to a particular location.""" + tty.info("Installing %s to %s" % (src, dest)) + shutil.copy(src, dest) + set_install_permissions(dest) + + src_mode = os.stat(src).st_mode + dest_mode = os.stat(dest).st_mode + if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + os.chmod(dest, dest_mode) + + +def is_exe(path): + """True if path is an executable file.""" + return os.path.isfile(path) and os.access(path, os.X_OK) + + +def expand_user(path): + """Find instances of '%u' in a path and replace with the current user's + username.""" + username = getpass.getuser() + if not username and '%u' in path: + tty.die("Couldn't get username to complete path '%s'" % path) + + return path.replace('%u', username) + + +def mkdirp(*paths): + """Creates a directory, as well as parent directories if needed.""" + for path in paths: + if not os.path.exists(path): + os.makedirs(path) + elif not os.path.isdir(path): + raise OSError(errno.EEXIST, "File alredy exists", path) + + +def force_remove(*paths): + """Remove files without printing errors. Like rm -f, does NOT + remove directories.""" + for path in paths: + try: + os.remove(path) + except OSError, e: + pass + +@contextmanager +def working_dir(dirname, **kwargs): + if kwargs.get('create', False): + mkdirp(dirname) + + orig_dir = os.getcwd() + os.chdir(dirname) + yield + os.chdir(orig_dir) + + +def touch(path): + """Creates an empty file at the specified path.""" + with closing(open(path, 'a')) as file: + os.utime(path, None) + + +def touchp(path): + """Like touch, but creates any parent directories needed for the file.""" + mkdirp(os.path.dirname(path)) + touch(path) + + +def join_path(prefix, *args): + path = str(prefix) + for elt in args: + path = os.path.join(path, str(elt)) + return path + + +def ancestor(dir, n=1): + """Get the nth ancestor of a directory.""" + parent = os.path.abspath(dir) + for i in range(n): + parent = os.path.dirname(parent) + return parent + + +def can_access(file_name): + """True if we have read/write access to the file.""" + return os.access(file_name, os.R_OK|os.W_OK) -- cgit v1.2.3-70-g09d2 From 454d06c25c5aade4315a3f263adf2d0727966148 Mon Sep 17 00:00:00 2001 From: Saravan Pantham Date: Mon, 6 Apr 2015 14:13:58 -0700 Subject: Removed autosave files from previous commit. --- lib/spack/llnl/util/filesystem.py.save | 0 lib/spack/llnl/util/filesystem.py.save.1 | 229 ------------------------------- 2 files changed, 229 deletions(-) delete mode 100644 lib/spack/llnl/util/filesystem.py.save delete mode 100644 lib/spack/llnl/util/filesystem.py.save.1 (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py.save b/lib/spack/llnl/util/filesystem.py.save deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/lib/spack/llnl/util/filesystem.py.save.1 b/lib/spack/llnl/util/filesystem.py.save.1 deleted file mode 100644 index 82cfe965b2..0000000000 --- a/lib/spack/llnl/util/filesystem.py.save.1 +++ /dev/null @@ -1,229 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -__all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', - 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] - -import os -import sys -import re -import shutil -import stat -import errno -import getpass -from contextlib import contextmanager, closing -from tempfile import NamedTemporaryFile - -import llnl.util.tty as tty -from spack.util.compression import ALLOWED_ARCHIVE_TYPES - -def filter_file(regex, repl, *filenames, **kwargs): - """Like sed, but uses python regular expressions. - - Filters every line of file through regex and replaces the file - with a filtered version. Preserves mode of filtered files. - - As with re.sub, ``repl`` can be either a string or a callable. - If it is a callable, it is passed the match object and should - return a suitable replacement string. If it is a string, it - can contain ``\1``, ``\2``, etc. to represent back-substitution - as sed would allow. - - Keyword Options: - string[=False] If True, treat regex as a plain string. - backup[=True] Make a backup files suffixed with ~ - ignore_absent[=False] Ignore any files that don't exist. - """ - string = kwargs.get('string', False) - backup = kwargs.get('backup', True) - ignore_absent = kwargs.get('ignore_absent', False) - - # Allow strings to use \1, \2, etc. for replacement, like sed - if not callable(repl): - unescaped = repl.replace(r'\\', '\\') - def replace_groups_with_groupid(m): - def groupid_to_group(x): - return m.group(int(x.group(1))) - return re.sub(r'\\([1-9])', groupid_to_group, unescaped) - repl = replace_groups_with_groupid - - if string: - regex = re.escape(regex) - - for filename in filenames: - backup = filename + "~" - - if ignore_absent and not os.path.exists(filename): - continue - - shutil.copy(filename, backup) - try: - with closing(open(backup)) as infile: - with closing(open(filename, 'w')) as outfile: - for line in infile: - foo = re.sub(regex, repl, line) - outfile.write(foo) - except: - # clean up the original file on failure. - shutil.move(backup, filename) - raise - - finally: - if not backup: - shutil.rmtree(backup, ignore_errors=True) - - -def change_sed_delimiter(old_delim, new_delim, *filenames): - """Find all sed search/replace commands and change the delimiter. - e.g., if the file contains seds that look like 's///', you can - call change_sed_delimeter('/', '@', file) to change the - delimiter to '@'. - - NOTE that this routine will fail if the delimiter is ' or ". - Handling those is left for future work. - """ - assert(len(old_delim) == 1) - assert(len(new_delim) == 1) - - # TODO: handle these cases one day? - assert(old_delim != '"') - assert(old_delim != "'") - assert(new_delim != '"') - assert(new_delim != "'") - - whole_lines = "^s@([^@]*)@(.*)@[gIp]$" - whole_lines = whole_lines.replace('@', old_delim) - - single_quoted = r"'s@((?:\\'|[^@'])*)@((?:\\'|[^'])*)@[gIp]?'" - single_quoted = single_quoted.replace('@', old_delim) - - double_quoted = r'"s@((?:\\"|[^@"])*)@((?:\\"|[^"])*)@[gIp]?"' - double_quoted = double_quoted.replace('@', old_delim) - - repl = r's@\1@\2@g' - repl = repl.replace('@', new_delim) - - for f in filenames: - filter_file(whole_lines, repl, f) - filter_file(single_quoted, "'%s'" % repl, f) - filter_file(double_quoted, '"%s"' % repl, f) - - -def set_install_permissions(path): - """Set appropriate permissions on the installed file.""" - if os.path.isdir(path): - os.chmod(path, 0755) - else: - os.chmod(path, 0644) - - -def install(src, dest): - """Manually install a file to a particular location.""" - tty.info("Installing %s to %s" % (src, dest)) - shutil.copy(src, dest) - set_install_permissions(dest) - - src_mode = os.stat(src).st_mode - dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH - os.chmod(dest, dest_mode) - - -def is_exe(path): - """True if path is an executable file.""" - return os.path.isfile(path) and os.access(path, os.X_OK) - - -def expand_user(path): - """Find instances of '%u' in a path and replace with the current user's - username.""" - username = getpass.getuser() - if not username and '%u' in path: - tty.die("Couldn't get username to complete path '%s'" % path) - - return path.replace('%u', username) - - -def mkdirp(*paths): - """Creates a directory, as well as parent directories if needed.""" - for path in paths: - if not os.path.exists(path): - os.makedirs(path) - elif not os.path.isdir(path): - raise OSError(errno.EEXIST, "File alredy exists", path) - - -def force_remove(*paths): - """Remove files without printing errors. Like rm -f, does NOT - remove directories.""" - for path in paths: - try: - os.remove(path) - except OSError, e: - pass - -@contextmanager -def working_dir(dirname, **kwargs): - if kwargs.get('create', False): - mkdirp(dirname) - - orig_dir = os.getcwd() - os.chdir(dirname) - yield - os.chdir(orig_dir) - - -def touch(path): - """Creates an empty file at the specified path.""" - with closing(open(path, 'a')) as file: - os.utime(path, None) - - -def touchp(path): - """Like touch, but creates any parent directories needed for the file.""" - mkdirp(os.path.dirname(path)) - touch(path) - - -def join_path(prefix, *args): - path = str(prefix) - for elt in args: - path = os.path.join(path, str(elt)) - return path - - -def ancestor(dir, n=1): - """Get the nth ancestor of a directory.""" - parent = os.path.abspath(dir) - for i in range(n): - parent = os.path.dirname(parent) - return parent - - -def can_access(file_name): - """True if we have read/write access to the file.""" - return os.access(file_name, os.R_OK|os.W_OK) -- cgit v1.2.3-70-g09d2 From 5d7a6c0c4671626fd402245fe57cec526e6365f8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 6 Jun 2015 14:01:06 -0700 Subject: Add FileFilter convenience class to llnl.util.filesystem. --- lib/spack/llnl/util/filesystem.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 8eff642938..029a7536df 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] import os import sys @@ -96,6 +96,15 @@ def filter_file(regex, repl, *filenames, **kwargs): shutil.rmtree(backup, ignore_errors=True) +class FileFilter(object): + """Convenience class for calling filter_file a lot.""" + def __init__(self, *filenames): + self.filenames = filenames + + def filter(self, regex, repl, **kwargs): + return filter_file(regex, repl, *self.filenames, **kwargs) + + def change_sed_delimiter(old_delim, new_delim, *filenames): """Find all sed search/replace commands and change the delimiter. e.g., if the file contains seds that look like 's///', you can -- cgit v1.2.3-70-g09d2 From 53feb12ea078cd9661eca2311fdbc18b352ddc37 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 6 Jun 2015 15:50:01 -0700 Subject: Cleanup and consolidate error handling --- bin/spack | 9 +-------- lib/spack/spack/build_environment.py | 4 ++++ lib/spack/spack/error.py | 15 +++++++++++++++ lib/spack/spack/package.py | 11 +---------- 4 files changed, 21 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index 354754594e..5c042edd2d 100755 --- a/bin/spack +++ b/bin/spack @@ -126,14 +126,7 @@ def main(): try: return_val = command(parser, args) except SpackError, e: - if spack.debug: - # In debug mode, raise with a full stack trace. - raise - elif e.long_message: - tty.die(e.message, e.long_message) - else: - tty.die(e.message) - + e.die() except KeyboardInterrupt: sys.stderr.write('\n') tty.die("Keyboard interrupt.") diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index f9e795ac42..81fbedc689 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -280,6 +280,10 @@ def fork(pkg, function): # Use os._exit here to avoid raising a SystemExit exception, # which interferes with unit tests. os._exit(0) + + except spack.error.SpackError, e: + e.die() + except: # Child doesn't raise or return to main spack code. # Just runs default exception handler and exits. diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index e8fa756682..bfa7951a47 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -22,6 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import sys +import llnl.util.tty as tty +import spack class SpackError(Exception): """This is the superclass for all Spack errors. @@ -38,6 +42,17 @@ class SpackError(Exception): return self._long_message + def die(self): + if spack.debug: + sys.excepthook(*sys.exc_info()) + os._exit(1) + else: + tty.error(self.message) + if self.long_message: + print self.long_message + os._exit(1) + + def __str__(self): msg = self.message if self.long_message: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5dd410d0e4..e3d766f582 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -816,17 +816,8 @@ class Package(object): except ProcessError, e: # Annotate with location of build log. e.build_log = log_path - - # One of the processes returned an error code. - # Suppress detailed stack trace here unless in debug mode - if spack.debug: - raise e - else: - tty.error(e) - - # Still need to clean up b/c there was an error. cleanup() - os._exit(1) + raise e except: # other exceptions just clean up and raise. -- cgit v1.2.3-70-g09d2 From eb9859ce750def63db8bc436338f94e875276a2e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 6 Jun 2015 16:15:34 -0700 Subject: Executables now have a useful __str__ function. --- lib/spack/spack/util/executable.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 6eede0f78e..1dcda0d87f 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -124,6 +124,11 @@ class Executable(object): return "" % self.exe + def __str__(self): + return ' '.join(self.exe) + + + def which(name, **kwargs): """Finds an executable in the path like command-line which.""" path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) -- cgit v1.2.3-70-g09d2 From 0fc3b58890ddb18c4a2384c6f015c8f9417a1c01 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Jun 2015 15:21:31 -0700 Subject: SPACK-38: Allow specs to be indexed by virtual dependencies. - The following now work differently: spec['mpi'] spec['blas'] This can return a spec for openmpi, mpich, mvapich, etc., EVEN if the spec is already concretized. This means that in a package that `depends_on('mpi')`, you can do `spec['mpi']` to see what it was concretized to. This should simplify MPI and BLAS packages. 'mpi' in spec 'blas' in spec Previously, if the spec had been concretized, these would be `False` because there was not a dependency in the DAG with either of these names. These will now be `True` even if the spec has been concretized. So, e.g., this will print "YES" s = Spec('callpath ^mpich') if 'mpi' in spec: print "YES" - Similarly, this will be True: Spec('mpich').satisfies('mpi') - Because of the way virtual dependencies are currently implemented, the above required some fiddling around with `package.py` so that it would never call `Spec.__contains__` (and result in endless recursion). - This should be fixed by allowing virutal dependnecies to have their own package class. - This would allow a quicker check for vdeps, without a call to `all_packages`. - For the time being, `package.py` shouldn't call `__contains__` --- lib/spack/spack/package.py | 35 ++++++++++++------- lib/spack/spack/spec.py | 33 ++++++++++++++++-- lib/spack/spack/test/concretize.py | 9 +++-- lib/spack/spack/test/spec_semantics.py | 61 ++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e3d766f582..5abf2a6bb3 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -471,13 +471,18 @@ class Package(object): """Spec of the extendee of this package, or None if it is not an extension.""" if not self.extendees: return None + + # TODO: allow more than one extendee. name = next(iter(self.extendees)) - if not name in self.spec: - spec, kwargs = self.extendees[name] - return spec - # Need to do this to get the concrete version of the spec - return self.spec[name] + # If the extendee is in the spec's deps already, return that. + for dep in self.spec.traverse(): + if name == dep.name: + return dep + + # Otherwise return the spec from the extends() directive + spec, kwargs = self.extendees[name] + return spec @property @@ -542,7 +547,7 @@ class Package(object): def provides(self, vpkg_name): """True if this package provides a virtual package with the specified name.""" - return vpkg_name in self.provided + return any(s.name == vpkg_name for s in self.provided) def virtual_dependencies(self, visited=None): @@ -561,8 +566,11 @@ class Package(object): on this one.""" dependents = [] for spec in spack.db.installed_package_specs(): - if self.name != spec.name and self.spec in spec: - dependents.append(spec) + if self.name == spec.name: + continue + for dep in spec.traverse(): + if spec == dep: + dependents.append(spec) return dependents @@ -985,10 +993,13 @@ class Package(object): activated = spack.install_layout.extension_map(self.extendee_spec) for name, aspec in activated.items(): - if aspec != self.spec and self.spec in aspec: - raise ActivationError( - "Cannot deactivate %s beacuse %s is activated and depends on it." - % (self.spec.short_spec, aspec.short_spec)) + if aspec == self.spec: + continue + for dep in aspec.traverse(): + if self.spec == dep: + raise ActivationError( + "Cannot deactivate %s beacuse %s is activated and depends on it." + % (self.spec.short_spec, aspec.short_spec)) self.extendee_spec.package.deactivate(self, **self.extendee_args) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index aa13f0422c..5876fc6cf8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -498,7 +498,13 @@ class Spec(object): Possible idea: just use conventin and make virtual deps all caps, e.g., MPI vs mpi. """ - return not spack.db.exists(self.name) + return Spec.is_virtual(self.name) + + + @staticmethod + def is_virtual(name): + """Test if a name is virtual without requiring a Spec.""" + return not spack.db.exists(name) @property @@ -1224,7 +1230,17 @@ class Spec(object): """ other = self._autospec(other) - # First thing we care about is whether the name matches + # A concrete provider can satisfy a virtual dependency. + if not self.virtual and other.virtual: + pkg = spack.db.get(self.name) + if pkg.provides(other.name): + for provided, when_spec in pkg.provided.items(): + if self.satisfies(when_spec, deps=False, strict=strict): + if provided.satisfies(other): + return True + return False + + # Otherwise, first thing we care about is whether the name matches if self.name != other.name: return False @@ -1364,11 +1380,21 @@ class Spec(object): def __getitem__(self, name): - """TODO: reconcile __getitem__, _add_dependency, __contains__""" + """Get a dependency from the spec by its name.""" for spec in self.traverse(): if spec.name == name: return spec + if Spec.is_virtual(name): + # TODO: this is a kind of kludgy way to find providers + # TODO: should we just keep virtual deps in the DAG instead of + # TODO: removing them on concretize? + for spec in self.traverse(): + if spec.virtual: + continue + if spec.package.provides(name): + return spec + raise KeyError("No spec with name %s in %s" % (name, self)) @@ -1380,6 +1406,7 @@ class Spec(object): for s in self.traverse(): if s.satisfies(spec, strict=True): return True + return False diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index cc839a2340..b3a77d076a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -152,7 +152,10 @@ class ConcretizeTest(MockPackagesTest): spec.concretize() self.assertTrue('zmpi' in spec.dependencies) - self.assertFalse('mpi' in spec) + self.assertTrue(all(not 'mpi' in d.dependencies for d in spec.traverse())) + self.assertTrue('zmpi' in spec) + self.assertTrue('mpi' in spec) + self.assertTrue('fake' in spec.dependencies['zmpi']) @@ -168,7 +171,9 @@ class ConcretizeTest(MockPackagesTest): self.assertTrue('zmpi' in spec.dependencies['callpath'].dependencies) self.assertTrue('fake' in spec.dependencies['callpath'].dependencies['zmpi'].dependencies) - self.assertFalse('mpi' in spec) + self.assertTrue(all(not 'mpi' in d.dependencies for d in spec.traverse())) + self.assertTrue('zmpi' in spec) + self.assertTrue('mpi' in spec) def test_my_dep_depends_on_provider_of_my_virtual_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 20df2603f5..6666dbbb52 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -189,6 +189,67 @@ class SpecSematicsTest(MockPackagesTest): self.check_unsatisfiable('mpich+foo', 'mpich~foo') + def test_satisfies_virtual(self): + self.assertTrue(Spec('mpich').satisfies(Spec('mpi'))) + self.assertTrue(Spec('mpich2').satisfies(Spec('mpi'))) + self.assertTrue(Spec('zmpi').satisfies(Spec('mpi'))) + + + # ================================================================================ + # Indexing specs + # ================================================================================ + def test_self_index(self): + s = Spec('callpath') + self.assertTrue(s['callpath'] == s) + + + def test_dep_index(self): + s = Spec('callpath') + s.normalize() + + self.assertTrue(s['callpath'] == s) + self.assertTrue(type(s['dyninst']) == Spec) + self.assertTrue(type(s['libdwarf']) == Spec) + self.assertTrue(type(s['libelf']) == Spec) + self.assertTrue(type(s['mpi']) == Spec) + + self.assertTrue(s['dyninst'].name == 'dyninst') + self.assertTrue(s['libdwarf'].name == 'libdwarf') + self.assertTrue(s['libelf'].name == 'libelf') + self.assertTrue(s['mpi'].name == 'mpi') + + + def test_spec_contains_deps(self): + s = Spec('callpath') + s.normalize() + self.assertTrue('dyninst' in s) + self.assertTrue('libdwarf' in s) + self.assertTrue('libelf' in s) + self.assertTrue('mpi' in s) + + + def test_virtual_index(self): + s = Spec('callpath') + s.concretize() + + s_mpich = Spec('callpath ^mpich') + s_mpich.concretize() + + s_mpich2 = Spec('callpath ^mpich2') + s_mpich2.concretize() + + s_zmpi = Spec('callpath ^zmpi') + s_zmpi.concretize() + + + self.assertTrue(s['mpi'].name != 'mpi') + self.assertTrue(s_mpich['mpi'].name == 'mpich') + self.assertTrue(s_mpich2['mpi'].name == 'mpich2') + self.assertTrue(s_zmpi['zmpi'].name == 'zmpi') + + for spec in [s, s_mpich, s_mpich2, s_zmpi]: + self.assertTrue('mpi' in spec) + # ================================================================================ # Constraints -- cgit v1.2.3-70-g09d2 From 3f3ceb24c49b8d466048491743e9628bab44f46a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Jun 2015 15:39:40 -0700 Subject: Add some comments b/c I didn't understand my own test. --- lib/spack/spack/test/directory_layout.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index beac038410..b3ad8efec4 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -167,12 +167,15 @@ class DirectoryLayoutTest(unittest.TestCase): def test_find(self): """Test that finding specs within an install layout works.""" packages = list(spack.db.all_packages())[:max_packages] + + # Create install prefixes for all packages in the list installed_specs = {} for pkg in packages: spec = pkg.spec.concretized() installed_specs[spec.name] = spec self.layout.create_install_directory(spec) + # Make sure all the installed specs appear in DirectoryLayout.all_specs() found_specs = dict((s.name, s) for s in self.layout.all_specs()) for name, spec in found_specs.items(): self.assertTrue(name in found_specs) -- cgit v1.2.3-70-g09d2 From 0a92349f90703bb21e58d8e46d7027d654d31eff Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 7 Jun 2015 15:40:01 -0700 Subject: Try a little harder in concretize_version() -- concretize unsafe versions too. - This can result in the user being prompted to download an unsafe version. - Avoids overly strict errors when something *could* be satisfiable but we don't know about hte version. --- lib/spack/spack/concretize.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 15e886ad3c..2e1d5d7f03 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -75,7 +75,23 @@ class DefaultConcretizer(object): if valid_versions: spec.versions = ver([valid_versions[-1]]) else: - raise NoValidVersionError(spec) + # We don't know of any SAFE versions that match the given + # spec. Grab the spec's versions and grab the highest + # *non-open* part of the range of versions it specifies. + # Someone else can raise an error if this happens, + # e.g. when we go to fetch it and don't know how. But it + # *might* work. + if not spec.versions or spec.versions == VersionList([':']): + raise NoValidVersionError(spec) + else: + last = spec.versions[-1] + if isinstance(last, VersionRange): + if last.end: + spec.versions = ver([last.end]) + else: + spec.versions = ver([last.start]) + else: + spec.versions = ver([last]) def concretize_architecture(self, spec): @@ -174,8 +190,8 @@ class UnavailableCompilerVersionError(spack.error.SpackError): class NoValidVersionError(spack.error.SpackError): - """Raised when there is no available version for a package that - satisfies a spec.""" + """Raised when there is no way to have a concrete version for a + particular spec.""" def __init__(self, spec): super(NoValidVersionError, self).__init__( - "No available version of %s matches '%s'" % (spec.name, spec.versions)) + "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) -- cgit v1.2.3-70-g09d2