diff options
Diffstat (limited to 'lib')
28 files changed, 296 insertions, 158 deletions
diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 0966277a91..aacba996b3 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -94,11 +94,11 @@ case "$command" in command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf) + f77|gfortran|ifort|pgf77|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/env/nag/nagfor b/lib/spack/env/nag/nagfor new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/nag/nagfor @@ -0,0 +1 @@ +../cc
\ No newline at end of file diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index e4ec7da35d..bdbd623b39 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,7 +42,7 @@ def get_origin_url(): git = which('git', required=True) origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - return_output=True) + output=str) return origin_url.strip() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7cea39cb55..edcea0718c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -132,7 +132,7 @@ class ConfigureGuesser(object): # Peek inside the tarball. tar = which('tar') output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) + "--exclude=*/*/*", "-tf", stage.archive_file, output=str) lines = output.split("\n") # Set the configure line to the one that matched. diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index a0c78bf755..d6b23d6a08 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -37,7 +37,7 @@ def setup_parser(subparser): help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( '-a', '--all', action='store_true', - help="Deactivate all extensions of an extendable pacakge, or " + help="Deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 39c225e9b2..307ee8982d 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path import spack import spack.cmd -description="Print out locations of various diectories used by Spack" +description="Print out locations of various directories used by Spack" def setup_parser(subparser): global directories diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 885483a840..8e9438c1a3 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -53,11 +53,13 @@ def setup_parser(subparser): create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") create_parser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + 'specs', nargs=argparse.REMAINDER, + help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") create_parser.add_argument( - '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', + help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -74,7 +76,8 @@ def setup_parser(subparser): help="Configuration scope to modify.") # Remove - remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) + remove_parser = sp.add_parser('remove', aliases=['rm'], + help=mirror_remove.__doc__) remove_parser.add_argument('name') remove_parser.add_argument( '--scope', choices=scopes, default=spack.cmd.default_modify_scope, @@ -141,15 +144,17 @@ def mirror_list(args): def _read_specs_from_file(filename): + specs = [] with open(filename, "r") as stream: for i, string in enumerate(stream): try: s = Spec(string) s.package - args.specs.append(s) + specs.append(s) except SpackError, e: tty.die("Parse error in %s, line %d:" % (args.file, i+1), ">>> " + string, str(e)) + return specs def mirror_create(args): @@ -169,6 +174,7 @@ def mirror_create(args): specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: new_specs = set() for spec in specs: diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 448f762841..cf478d3763 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -79,7 +79,7 @@ def list_packages(rev): git = get_git() relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, - return_output=True) + output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 74017f59fb..656873a2f0 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -37,20 +37,20 @@ from spack.build_environment import InstallError from spack.fetch_strategy import FetchError import spack.cmd -description = "Treat package installations as unit tests and output formatted test results" +description = "Run package installation as a unit test, output formatted results." def setup_parser(subparser): subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") - + subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") - + subparser.add_argument( '-o', '--output', action='store', help="test output goes in this file") - + subparser.add_argument( 'package', nargs=argparse.REMAINDER, help="spec of package to install") @@ -59,10 +59,10 @@ class JunitResultFormat(object): def __init__(self): self.root = ET.Element('testsuite') self.tests = [] - + def add_test(self, buildId, testResult, buildInfo=None): self.tests.append((buildId, testResult, buildInfo)) - + def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) for buildId, testResult, buildInfo in self.tests: @@ -84,25 +84,25 @@ class TestResult(object): PASSED = 0 FAILED = 1 SKIPPED = 2 - + class BuildId(object): def __init__(self, spec): self.name = spec.name self.version = spec.version self.hashId = spec.dag_hash() - + def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) def __hash__(self): return hash((self.name, self.version, self.hashId)) - + def __eq__(self, other): if not isinstance(other, BuildId): return False - - return ((self.name, self.version, self.hashId) == + + return ((self.name, self.version, self.hashId) == (other.name, other.version, other.hashId)) @@ -114,12 +114,12 @@ def fetch_log(path): def failed_dependencies(spec): - return set(childSpec for childSpec in spec.dependencies.itervalues() if not + return set(childSpec for childSpec in spec.dependencies.itervalues() if not spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): - # Post-order traversal is not strictly required but it makes sense to output + # Post-order traversal is not strictly required but it makes sense to output # tests for dependencies first. for spec in topSpec.traverse(order='post'): if spec not in newInstalls: @@ -143,12 +143,12 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): re.search('error:', line, re.IGNORECASE)) errOutput = errMessages if errMessages else lines[-10:] errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, + [spec.to_yaml(), "Errors:"], errOutput, ["Build Log:", package.build_log_path])) else: result = TestResult.PASSED errOutput = None - + bId = BuildId(spec) output.add_test(bId, result, errOutput) @@ -163,18 +163,18 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - + specs = spack.cmd.parse_specs(args.package, concretize=True) if len(specs) > 1: tty.die("Only 1 top-level package can be specified") topSpec = iter(specs).next() - + newInstalls = set() for spec in topSpec.traverse(): package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) - + if not args.output: bId = BuildId(topSpec) outputDir = join_path(os.getcwd(), "test-output") @@ -183,7 +183,7 @@ def test_install(parser, args): outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) else: outputFpath = args.output - + for spec in topSpec.traverse(order='post'): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which @@ -202,7 +202,7 @@ def test_install(parser, args): pass except FetchError: pass - + jrf = JunitResultFormat() handled = {} create_test_output(topSpec, newInstalls, jrf) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a665f6062d..12c02e0ea2 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -51,7 +51,7 @@ _version_cache = {} def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=os.devnull) + output = compiler(version_arg, output=str, error=str) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index facc9c338b..6159ef576c 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -51,7 +51,7 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] if platform.system() == 'Darwin': _default_order = ['clang', 'gcc', 'intel'] else: - _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] + _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] def _auto_compiler_spec(function): diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 340051019c..e406d86a24 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -22,7 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import re +import spack.compiler as cpr from spack.compiler import * +from spack.util.executable import * class Clang(Compiler): # Subclasses use possible names of C compiler @@ -47,11 +50,34 @@ class Clang(Compiler): @classmethod def default_version(self, comp): """The '--version' option works for clang compilers. - Output looks like this:: + On most platforms, output looks like this:: clang version 3.1 (trunk 149096) Target: x86_64-unknown-linux-gnu Thread model: posix + + On Mac OS X, it looks like this: + + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix + """ - return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM) ([^ )]+)') + if comp not in cpr._version_cache: + compiler = Executable(comp) + output = compiler('--version', output=str, error=str) + + ver = 'unknown' + match = re.search(r'^Apple LLVM version ([^ )]+)', output) + if match: + # Apple's LLVM compiler has its own versions, so suffix them. + ver = match.group(1) + '-apple' + else: + # Normal clang compiler versions are left as-is + match = re.search(r'^clang version ([^ )]+)', output) + if match: + ver = match.group(1) + + cpr._version_cache[comp] = ver + + return cpr._version_cache[comp] diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py new file mode 100644 index 0000000000..527a05a090 --- /dev/null +++ b/lib/spack/spack/compilers/nag.py @@ -0,0 +1,33 @@ +from spack.compiler import * + +class Nag(Compiler): + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['nagfor'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['nagfor'] + + # Named wrapper links within spack.build_env_path + link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml + 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'nag/nagfor', + 'fc' : 'nag/nagfor' } + + @classmethod + def default_version(self, comp): + """The '-V' option works for nag compilers. + Output looks like this:: + + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux + Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K. + """ + return get_compiler_version( + comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 0fa18db34b..9cbe7de44a 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -489,7 +489,7 @@ class Database(object): 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts - and remvoes them if they are no longer needed. + and removes them if they are no longer needed. """ # Take a lock around the entire removal. diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0657146bf6..337dd1e198 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -154,7 +154,7 @@ class URLFetchStrategy(FetchStrategy): # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -375,7 +375,7 @@ class GitFetchStrategy(VCSFetchStrategy): @property def git_version(self): - vstring = self.git('--version', return_output=True).lstrip('git version ') + vstring = self.git('--version', output=str).lstrip('git version ') return Version(vstring) @@ -518,7 +518,7 @@ class SvnFetchStrategy(VCSFetchStrategy): def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" - status = self.svn('status', '--no-ignore', return_output=True) + status = self.svn('status', '--no-ignore', output=str) self.svn('status', '--no-ignore') for line in status.split('\n'): if not re.match('^[I?]', line): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef2..341cc4cb88 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -78,6 +78,7 @@ def get_matching_versions(specs, **kwargs): continue num_versions = kwargs.get('num_versions', 0) + matching_spec = [] for i, v in enumerate(reversed(sorted(pkg.versions))): # Generate no more than num_versions versions for each spec. if num_versions and i >= num_versions: @@ -88,7 +89,11 @@ def get_matching_versions(specs, **kwargs): s = Spec(pkg.name) s.versions = VersionList([v]) s.variants = spec.variants.copy() - matching.append(s) + matching_spec.append(s) + + if not matching_spec: + tty.warn("No known version matches spec: %s" % spec) + matching.extend(matching_spec) return matching @@ -147,7 +152,11 @@ def create(path, specs, **kwargs): # Get the absolute path of the root before we start jumping around. mirror_root = os.path.abspath(path) if not os.path.isdir(mirror_root): - mkdirp(mirror_root) + try: + mkdirp(mirror_root) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. present = [] @@ -164,7 +173,11 @@ def create(path, specs, **kwargs): # create a subdirectory for the current package@version archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) - mkdirp(subdir) + try: + mkdirp(subdir) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % subdir, str(e)) if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fba2912b75..8cb947c276 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -631,7 +631,7 @@ class Package(object): spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self): + def do_fetch(self, mirror_only=False): """Creates a stage directory and downloads the taball for this package. Working directory will be set to the stage directory. """ @@ -656,7 +656,7 @@ class Package(object): raise FetchError( "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) - self.stage.fetch() + self.stage.fetch(mirror_only) ########## # Fetch resources @@ -677,7 +677,8 @@ class Package(object): if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: @@ -691,8 +692,7 @@ class Package(object): else: tty.msg("Already staged %s in %s." % (name, stage.path)) - - self.do_fetch() + self.do_fetch(mirror_only) _expand_archive(self.stage) ########## @@ -835,10 +835,6 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def _build_logger(self, log_path): - """Create a context manager to log build output.""" - - def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 31596cee7a..b5df1168b6 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -233,6 +233,11 @@ class RepoPath(object): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 31635a854a..79c9030e20 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -97,7 +97,6 @@ class Stage(object): self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') - self.tmp_root = find_tmp_root() self.path = None @@ -240,11 +239,13 @@ class Stage(object): tty.die("Setup failed: no such directory: " + self.path) - def fetch(self): + def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.default_fetcher] + fetchers = [] + if not mirror_only: + fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a @@ -252,7 +253,13 @@ class Stage(object): self.skip_checksum_for_mirror = True if self.mirror_path: mirrors = spack.config.get_config('mirrors') - urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] + + # Join URLs of mirror roots with mirror paths. Because + # urljoin() will strip everything past the final '/' in + # the root, so we add a '/' if it is not present. + mirror_roots = [root if root.endswith('/') else root + '/' + for root in mirrors.values()] + urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that @@ -261,10 +268,11 @@ class Stage(object): if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest - # Have to skip the checkesum for things archived from + # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) + # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 4188b8d550..905af28a06 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -65,17 +65,17 @@ class CompilerTest(unittest.TestCase): def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cc(*args, return_output=True).strip(), expected) + self.assertEqual(self.cc(*args, output=str).strip(), expected) def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.ld(*args, return_output=True).strip(), expected) + self.assertEqual(self.ld(*args, output=str).strip(), expected) def check_cpp(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cpp(*args, return_output=True).strip(), expected) + self.assertEqual(self.cpp(*args, output=str).strip(), expected) def test_vcheck_mode(self): diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 3813079065..d84433176a 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -56,10 +56,7 @@ class GitFetchTest(MockPackagesTest): def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index ee8327aec8..bbcb64e4c1 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -53,10 +53,7 @@ class HgFetchTest(MockPackagesTest): def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 628329a423..c09bd24c8e 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -59,9 +59,7 @@ class InstallTest(MockPackagesTest): def tearDown(self): super(InstallTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() + self.repo.destroy() # Turn checksumming back on spack.do_checksum = True diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 09efec8580..d568a28d44 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -56,47 +56,47 @@ class MakeExecutableTest(unittest.TestCase): def test_make_normal(self): make = MakeExecutable('make', 8) - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') def test_make_explicit(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') def test_make_one_job(self): make = MakeExecutable('make', 1) - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') def test_make_parallel_false(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=False, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=False, output=str).strip(), '') + self.assertEqual(make('install', parallel=False, output=str).strip(), 'install') def test_make_parallel_disabled(self): make = MakeExecutable('make', 8) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] @@ -106,20 +106,20 @@ class MakeExecutableTest(unittest.TestCase): # These should work os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 04e9e3db2e..046ec56604 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -44,8 +44,16 @@ class MirrorTest(MockPackagesTest): self.repos = {} + def tearDown(self): + """Destroy all the stages created by the repos in setup.""" + super(MirrorTest, self).tearDown() + for repo in self.repos.values(): + repo.destroy() + self.repos.clear() + + def set_up_package(self, name, MockRepoClass, url_attr): - """Use this to set up a mock package to be mirrored. + """Set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. 2. Point the package's version args at that repo. @@ -65,21 +73,14 @@ class MirrorTest(MockPackagesTest): pkg.versions[v][url_attr] = repo.url - def tearDown(self): - """Destroy all the stages created by the repos in setup.""" - super(MirrorTest, self).tearDown() - - for name, repo in self.repos.items(): - if repo.stage: - pass #repo.stage.destroy() - - self.repos.clear() - - def check_mirror(self): stage = Stage('spack-mirror-test') mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) + try: os.chdir(stage.path) spack.mirror.create( @@ -88,7 +89,7 @@ class MirrorTest(MockPackagesTest): # Stage directory exists self.assertTrue(os.path.isdir(mirror_root)) - # subdirs for each package + # check that there are subdirs for each package for name in self.repos: subdir = join_path(mirror_root, name) self.assertTrue(os.path.isdir(subdir)) @@ -96,40 +97,37 @@ class MirrorTest(MockPackagesTest): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Decompress archive in the mirror - archive = files[0] - archive_path = join_path(subdir, archive) - decomp = decompressor_for(archive_path) - - with working_dir(subdir): - decomp(archive_path) + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package - # Find the untarred archive directory. - files = os.listdir(subdir) - self.assertEqual(len(files), 2) - self.assertTrue(archive in files) - files.remove(archive) - - expanded_archive = join_path(subdir, files[0]) - self.assertTrue(os.path.isdir(expanded_archive)) + saved_checksum_setting = spack.do_checksum + try: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) # Compare the original repo with the expanded archive - repo = self.repos[name] - if not 'svn' in name: - original_path = repo.path - else: - co = 'checked_out' - svn('checkout', repo.url, co) - original_path = join_path(subdir, co) + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) - dcmp = dircmp(original_path, expanded_archive) + dcmp = dircmp(original_path, pkg.stage.source_path) # make sure there are no new files in the expanded tarball self.assertFalse(dcmp.right_only) + + # and that all original files are present. self.assertTrue(all(l in exclude for l in dcmp.left_only)) + finally: + spack.do_checksum = saved_checksum_setting + pkg.do_clean() finally: - pass #stage.destroy() + stage.destroy() def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index c454b1f106..ed94023b0e 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -55,6 +55,12 @@ class MockRepo(object): mkdirp(self.path) + def destroy(self): + """Destroy resources associated with this mock repo.""" + if self.stage: + self.stage.destroy() + + class MockArchive(MockRepo): """Creates a very simple archive directory with a configure script and a makefile that installs to a prefix. Tars it up into an archive.""" @@ -141,7 +147,7 @@ class MockGitRepo(MockVCSRepo): self.url = self.path def rev_hash(self, rev): - return git('rev-parse', rev, return_output=True).strip() + return git('rev-parse', rev, output=str).strip() class MockSvnRepo(MockVCSRepo): @@ -193,4 +199,4 @@ class MockHgRepo(MockVCSRepo): def get_rev(self): """Get current mercurial revision.""" - return hg('id', '-i', return_output=True).strip() + return hg('id', '-i', output=str).strip() diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 2ee4748fdb..454a7f1d1f 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -55,17 +55,14 @@ class SvnFetchTest(MockPackagesTest): def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - + self.repo.destroy() self.pkg.do_clean() def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" def get_rev(): - output = svn('info', return_output=True) + output = svn('info', output=str) self.assertTrue("Revision" in output) for line in output.split('\n'): match = re.match(r'Revision: (\d+)', line) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index ba765eb662..fc27b789d0 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -55,24 +55,80 @@ class Executable(object): def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - ignore_errors = kwargs.get("ignore_errors", ()) + """Run this executable in a subprocess. + + Arguments + args + command line arguments to the executable to run. + + Optional arguments + + fail_on_error + + Raise an exception if the subprocess returns an + error. Default is True. When not set, the return code is + avaiale as `exe.returncode`. + + ignore_errors + + An optional list/tuple of error codes that can be + *ignored*. i.e., if these codes are returned, this will + not raise an exception when `fail_on_error` is `True`. + + output, error + + These arguments allow you to specify new stdout and stderr + values. They default to `None`, which means the + subprocess will inherit the parent's file descriptors. + + You can set these to: + - python streams, e.g. open Python file objects, or os.devnull; + - filenames, which will be automatically opened for writing; or + - `str`, as in the Python string type. If you set these to `str`, + output and error will be written to pipes and returned as + a string. If both `output` and `error` are set to `str`, + then one string is returned containing output concatenated + with error. + + input + + Same as output, error, but `str` is not an allowed value. + + Deprecated arguments + + return_output[=False] + + Setting this to True is the same as setting output=str. + This argument may be removed in future Spack versions. + + """ + fail_on_error = kwargs.pop("fail_on_error", True) + ignore_errors = kwargs.pop("ignore_errors", ()) + + # TODO: This is deprecated. Remove in a future version. + return_output = kwargs.pop("return_output", False) # Default values of None says to keep parent's file descriptors. - output = kwargs.get("output", None) - error = kwargs.get("error", None) - input = kwargs.get("input", None) + if return_output: + output = str + else: + output = kwargs.pop("output", None) + + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) + if input is str: + raise ValueError("Cannot use `str` as input stream.") def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True + elif arg is str: + return subprocess.PIPE, False else: return arg, False - output, ostream = streamify(output, 'w') - error, estream = streamify(error, 'w') - input, istream = streamify(input, 'r') + ostream, close_ostream = streamify(output, 'w') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -92,19 +148,20 @@ class Executable(object): tty.debug(cmd_line) try: - if return_output: - output = subprocess.PIPE - proc = subprocess.Popen( - cmd, stdin=input, stderr=error, stdout=output) + cmd, stdin=istream, stderr=estream, stdout=ostream) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + + if output is str or error is str: + result = '' + if output is str: result += out + if error is str: result += err + return result except OSError, e: raise ProcessError( @@ -119,9 +176,9 @@ class Executable(object): % (proc.returncode, cmd_line)) finally: - if ostream: output.close() - if estream: error.close() - if istream: input.close() + if close_ostream: output.close() + if close_estream: error.close() + if close_istream: input.close() def __eq__(self, other): |